From 51be765c709ac2d5cb38452925eab77f568ab5ae Mon Sep 17 00:00:00 2001 From: Einar Rasmussen Date: Wed, 11 Nov 2026 18:16:48 +0100 Subject: [PATCH] kona-engine: wait for EL sync instead of resetting on a missing consolidation block Kona-node resets whenever derived attributes name a block the EL lacks below the unsafe head, so after an EL datadir wipe it re-derives from genesis in a loop and finalization stays at genesis. While the EL answers `SYNCING`, the task now waits (new `Wait` severity): it is parked with its own 2s retry deadline, tasks of other kinds (imports, builds, finalizations) keep running, and nothing resets, so the finalization queue survives. Each retry looks the block up again; every 30s it also reads the EL's sync progress and resends the unchanged forkchoice, so an EL that reached the unsafe head is consolidated, and a `VALID` answer records the EL as synced. An EL idle at the derived block's parent, reporting no sync, gets the block built from the attributes instead. Progress, or its absence, is logged every 5 minutes; the engine resets once progress and latest block stop moving for 24 h (`--l2.el-sync-stall-timeout`), as neither op-geth nor op-reth reports progress while downloading headers. Failed lookups and probes are Temporary. The delegated path is #23900. Part of #23931. BREAKING CHANGE: `EngineTaskErrorSeverity::Wait`; `ConsolidateTaskError::{MissingUnsafeL2BlockDuringElSync, MissingUnsafeL2BlockProbeFailed}`; `Engine::drain` takes a `DrainScope`; `EngineConfig` gains `el_sync_stall_timeout` (default `DEFAULT_EL_SYNC_STALL_TIMEOUT`); `ConsolidateTask::new` is non-const and takes the stall timeout. Co-Authored-By: Claude Opus 5.5 --- .../kona-node/configuration.mdx | 1 + .../bin/node/src/commands/node/command.rs | 8 +- .../bin/node/src/flags/engine/providers.rs | 37 +- rust/kona/crates/node/engine/src/lib.rs | 8 +- .../crates/node/engine/src/task_queue/core.rs | 361 ++++++++++- .../crates/node/engine/src/task_queue/mod.rs | 2 +- .../tasks/consolidate/el_sync_progress.rs | 306 +++++++++ .../src/task_queue/tasks/consolidate/error.rs | 17 +- .../src/task_queue/tasks/consolidate/mod.rs | 3 + .../src/task_queue/tasks/consolidate/task.rs | 599 ++++++++++++++++-- .../node/engine/src/task_queue/tasks/mod.rs | 9 +- .../task_queue/tasks/synchronize/helper.rs | 66 +- .../src/task_queue/tasks/synchronize/mod.rs | 5 +- .../node/engine/src/task_queue/tasks/task.rs | 5 + .../node/engine/src/test_utils/blocks.rs | 19 +- .../crates/node/engine/src/test_utils/mod.rs | 4 +- .../crates/node/engine/src/test_utils/rpc.rs | 20 + .../node/service/src/actors/engine/actor.rs | 98 ++- .../node/service/src/actors/engine/config.rs | 4 +- .../service/src/actors/engine/queue_tests.rs | 397 +++++++++++- .../node/service/src/service/node/rollup.rs | 3 +- 21 files changed, 1829 insertions(+), 143 deletions(-) create mode 100644 rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/el_sync_progress.rs diff --git a/docs/public-docs/node-operators/kona-node/configuration.mdx b/docs/public-docs/node-operators/kona-node/configuration.mdx index 30db30bd372..bad550822a4 100644 --- a/docs/public-docs/node-operators/kona-node/configuration.mdx +++ b/docs/public-docs/node-operators/kona-node/configuration.mdx @@ -27,6 +27,7 @@ For more details on each flag, see the inline help (`kona-node node --help`) or | `--l1-beacon ` | `KONA_NODE_L1_BEACON` | URL of the L1 beacon API | Yes | - | | `--l2-engine-rpc ` | `KONA_NODE_L2_ENGINE_RPC` | URL of the engine API endpoint of an L2 execution client | Yes | - | | `--l2-engine-jwt-secret ` | `KONA_NODE_L2_ENGINE_AUTH` | Path to file containing the hex-encoded JWT secret for the execution client | No | - | +| `--l2-el-sync-stall-timeout ` | `KONA_NODE_L2_EL_SYNC_STALL_TIMEOUT` | How long to wait for a missing derived block while the execution client answers `SYNCING` and neither its reported sync progress nor its latest block moves, before resetting. Execution clients report no progress while they download headers, which can take hours | No | `86400` | | `--l2-config-file ` | `KONA_NODE_ROLLUP_CONFIG` | Path to a custom L2 rollup configuration file | No | - | | `--l1-runtime-config-reload-interval ` | `KONA_NODE_L1_RUNTIME_CONFIG_RELOAD_INTERVAL` | Poll interval for reloading runtime config | No | `600` | diff --git a/rust/kona/bin/node/src/commands/node/command.rs b/rust/kona/bin/node/src/commands/node/command.rs index b96712488c6..0faa36cd4f4 100644 --- a/rust/kona/bin/node/src/commands/node/command.rs +++ b/rust/kona/bin/node/src/commands/node/command.rs @@ -336,8 +336,12 @@ impl NodeCommand { let p2p_config = self.p2p_flags.clone().config(&cfg, args, Some(&l1_provider)).await?; let rpc_config = self.rpc_flags.clone().into(); - let engine_config = - EngineConfig { config: Arc::new(cfg.clone()), mode: self.node_mode, provider: engine }; + let engine_config = EngineConfig { + config: Arc::new(cfg.clone()), + el_sync_stall_timeout: self.l2_client_args.el_sync_stall_timeout(), + mode: self.node_mode, + provider: engine, + }; let dependency_set = self.load_dependency_set(&cfg)?; let safe_db = self.open_safe_db()?; diff --git a/rust/kona/bin/node/src/flags/engine/providers.rs b/rust/kona/bin/node/src/flags/engine/providers.rs index 20b434fb96a..f8217cb3011 100644 --- a/rust/kona/bin/node/src/flags/engine/providers.rs +++ b/rust/kona/bin/node/src/flags/engine/providers.rs @@ -1,9 +1,11 @@ use alloy_rpc_types_engine::JwtSecret; -use std::path::PathBuf; +use kona_engine::DEFAULT_EL_SYNC_STALL_TIMEOUT; +use std::{path::PathBuf, time::Duration}; use url::Url; -const DEFAULT_L2_ENGINE_TIMEOUT: u64 = 30_000; const DEFAULT_L1_TRUST_RPC: bool = false; +const DEFAULT_L2_EL_SYNC_STALL_TIMEOUT: u64 = DEFAULT_EL_SYNC_STALL_TIMEOUT.as_secs(); +const DEFAULT_L2_ENGINE_TIMEOUT: u64 = 30_000; /// L1 client arguments. #[derive(Clone, Debug, clap::Args)] @@ -73,6 +75,22 @@ pub struct L2ClientArgs { default_value_t = DEFAULT_L2_ENGINE_TIMEOUT )] pub l2_engine_timeout: u64, + /// Seconds to wait for a missing derived block while the execution layer answers `SYNCING` + /// and neither its reported sync progress nor its latest block moves, before resetting. + #[arg( + long, + visible_alias = "l2.el-sync-stall-timeout", + env = "KONA_NODE_L2_EL_SYNC_STALL_TIMEOUT", + default_value_t = DEFAULT_L2_EL_SYNC_STALL_TIMEOUT + )] + pub l2_el_sync_stall_timeout: u64, +} + +impl L2ClientArgs { + /// How long a consolidation waits for an execution layer whose sync reports no progress. + pub const fn el_sync_stall_timeout(&self) -> Duration { + Duration::from_secs(self.l2_el_sync_stall_timeout) + } } impl Default for L2ClientArgs { @@ -82,6 +100,7 @@ impl Default for L2ClientArgs { l2_engine_jwt_secret: None, l2_engine_jwt_encoded: None, l2_engine_timeout: DEFAULT_L2_ENGINE_TIMEOUT, + l2_el_sync_stall_timeout: DEFAULT_L2_EL_SYNC_STALL_TIMEOUT, } } } @@ -136,6 +155,20 @@ mod tests { assert_eq!(parsed.l1.l1_trust_rpc, l1); } + #[rstest] + #[case::default(&[], DEFAULT_EL_SYNC_STALL_TIMEOUT)] + #[case::dotted_alias(&["--l2.el-sync-stall-timeout=600"], Duration::from_secs(600))] + #[case::long_flag(&["--l2-el-sync-stall-timeout", "7200"], Duration::from_secs(7200))] + fn test_parse_el_sync_stall_timeout(#[case] flags: &[&str], #[case] expected: Duration) { + // Keep these CLI/default tests independent of the operator's environment. + let command = + ClientArgs::command().mut_arg("l2_el_sync_stall_timeout", |arg| arg.env(None::<&str>)); + let args = [CLIENT_ARGS, flags].concat(); + let matches = command.try_get_matches_from(args).unwrap(); + let parsed = ClientArgs::from_arg_matches(&matches).unwrap(); + assert_eq!(parsed.l2.el_sync_stall_timeout(), expected); + } + #[test] fn test_default_l1_trust_rpc() { assert!(!L1ClientArgs::default().l1_trust_rpc); diff --git a/rust/kona/crates/node/engine/src/lib.rs b/rust/kona/crates/node/engine/src/lib.rs index eb26238be30..990a21e2624 100644 --- a/rust/kona/crates/node/engine/src/lib.rs +++ b/rust/kona/crates/node/engine/src/lib.rs @@ -40,10 +40,10 @@ extern crate tracing; mod task_queue; pub use task_queue::{ BuildTask, BuildTaskError, CanonicalizeTask, CanonicalizeTaskError, ConsolidateInput, - ConsolidateTask, ConsolidateTaskError, Engine, EngineBuildError, EngineResetError, EngineTask, - EngineTaskError, EngineTaskErrorSeverity, EngineTaskErrors, ExecutableEngineTask, - FinalizeBlockId, FinalizeTask, FinalizeTaskError, InsertTask, InsertTaskError, SealTask, - SealTaskError, SynchronizeTaskError, + ConsolidateTask, ConsolidateTaskError, DEFAULT_EL_SYNC_STALL_TIMEOUT, DrainScope, Engine, + EngineBuildError, EngineResetError, EngineTask, EngineTaskError, EngineTaskErrorSeverity, + EngineTaskErrors, ExecutableEngineTask, FinalizeBlockId, FinalizeTask, FinalizeTaskError, + InsertTask, InsertTaskError, SealTask, SealTaskError, SynchronizeTaskError, }; mod attributes; diff --git a/rust/kona/crates/node/engine/src/task_queue/core.rs b/rust/kona/crates/node/engine/src/task_queue/core.rs index 65ef7bdc252..14b8364113e 100644 --- a/rust/kona/crates/node/engine/src/task_queue/core.rs +++ b/rust/kona/crates/node/engine/src/task_queue/core.rs @@ -49,8 +49,11 @@ impl Ord for QueuedTask { /// /// Tasks within the queue are also considered fallible. If they fail with a temporary error, /// they are not popped from the queue, the error is returned, and they are retried on the -/// next call to [`Engine::drain`]. A task that fails with a flush error has already replaced its -/// block, so it is popped before the error is returned. +/// next call to [`Engine::drain`]. A task that fails with a wait error is parked instead: until its +/// retry, a drain of [`DrainScope::AheadOfWaitingTask`] runs only tasks of other kinds, while tasks +/// of its own kind keep their place behind it. A task +/// that fails with a flush error has already replaced its block, so it is popped before the error +/// is returned. #[derive(Debug)] pub struct Engine { /// The state of the engine. @@ -62,7 +65,9 @@ pub struct Engine { next_sequence: u64, /// Preserve the task being retried ahead of newly enqueued work: it may have already /// performed part of an Engine API operation. - active: Option, + active: Option, + /// The task that last failed with a wait error, parked until the next [`Engine::drain`]. + waiting: Option, } impl Engine { @@ -74,6 +79,7 @@ impl Engine { tasks: BinaryHeap::default(), active: None, next_sequence: 0, + waiting: None, } } @@ -95,9 +101,55 @@ impl Engine { self.tasks.push(QueuedTask { sequence, task }); } - /// Number of queued and active tasks, including work awaiting a retry. + /// Number of queued, active and waiting tasks, including work awaiting a retry. pub fn len(&self) -> usize { - self.tasks.len() + usize::from(self.active.is_some()) + self.tasks.len() + usize::from(self.active.is_some()) + usize::from(self.waiting.is_some()) + } + + /// Whether a task is parked until its retry. + pub const fn has_waiting_task(&self) -> bool { + self.waiting.is_some() + } + + /// Whether a drain of [`DrainScope::AheadOfWaitingTask`] has a task to run: a task being + /// retried, or a queued task of another kind than the waiting task, or any queued task without + /// one. + pub fn has_work_ahead_of_waiting_task(&self) -> bool { + self.active.is_some() || + self.waiting.as_ref().map_or(!self.tasks.is_empty(), |waiting| { + self.tasks.iter().any(|queued| !is_same_kind(queued, waiting)) + }) + } + + /// Pops the next queued task that may run. Before the waiting task's retry, tasks of its kind + /// stay queued behind it, in their order, while tasks of other kinds run: a finalization + /// queued behind a consolidation that waits for the execution layer would otherwise wait as + /// long as its sync does. + fn pop_runnable_task(&mut self) -> Option { + let Some(waiting) = &self.waiting else { return self.tasks.pop() }; + let mut held_behind_waiting = Vec::new(); + let mut runnable = None; + while let Some(next) = self.tasks.pop() { + if is_same_kind(&next, waiting) { + held_behind_waiting.push(next); + } else { + runnable = Some(next); + break; + } + } + self.tasks.extend(held_behind_waiting); + runnable + } + + /// Parks `task` until its retry. A task parked before it goes back into the queue with its + /// sequence. + fn park(&mut self, task: QueuedTask) { + self.tasks.extend(self.waiting.replace(task)); + } + + /// Puts the waiting task back into the queue with its sequence, to be retried in its turn. + fn unpark(&mut self) { + self.tasks.extend(self.waiting.take()); } /// Whether there is no pending work. @@ -140,30 +192,42 @@ impl Engine { pub fn clear(&mut self) { self.tasks.clear(); self.active = None; + self.waiting = None; } - /// Attempts to drain the queue by executing all [`EngineTask`]s in-order. If any task returns - /// an error along the way, the error is returned and the task is not popped from the queue (in - /// case it must be retried), unless it is a flush or skip error. - pub async fn drain(&mut self) -> Result<(), EngineTaskErrors> { + /// Attempts to drain the queue by executing the [`EngineTask`]s that `scope` allows, in order. + /// If any task returns an error along the way, the error is returned and the task is not popped + /// from the queue (in case it must be retried), unless it is a flush or skip error. A waiting + /// task is retried only once every queued task of another kind has run, so a failure among + /// those leaves it parked; if it still waits, it is parked again. + pub async fn drain(&mut self, scope: DrainScope) -> Result<(), EngineTaskErrors> { // Drain tasks in order of priority, halting on errors for a retry to be attempted. loop { if self.active.is_none() { - self.active = self.tasks.pop().map(|queued| queued.task); + self.active = self.next_task(scope); } - let Some(task) = &self.active else { break }; - let result = task.execute(&mut self.state).await; - - // A flush reports that the task replaced its block with a deposits-only block, so the - // task is done; retrying it would replace that block again. A skip reports a task that - // no longer applies, so retrying it cannot succeed either. - if let Err(err) = &result && - !matches!( - err.severity(), - EngineTaskErrorSeverity::Flush | EngineTaskErrorSeverity::Skip - ) - { - return result; + let Some(active) = &self.active else { break }; + let result = active.task.execute(&mut self.state).await; + + if let Err(err) = &result { + match err.severity() { + // A waiting task leaves the engine state unchanged. Parked with its sequence, + // it keeps its place among same-kind tasks but no longer blocks + // higher-priority work enqueued before its retry. + EngineTaskErrorSeverity::Wait => { + if let Some(waiting) = self.active.take() { + self.park(waiting); + } + return result; + } + // A flush reports that the task replaced its block with a deposits-only block, + // so the task is done; retrying it would replace that block again. A skip + // reports a task that no longer applies, so retrying it cannot succeed either. + EngineTaskErrorSeverity::Flush | EngineTaskErrorSeverity::Skip => {} + EngineTaskErrorSeverity::Critical | + EngineTaskErrorSeverity::Reset | + EngineTaskErrorSeverity::Temporary => return result, + } } self.state_sender.send_replace(self.state); @@ -173,6 +237,34 @@ impl Engine { Ok(()) } + + /// The next task to run: a queued task of another kind than the waiting task, and once none is + /// left, for [`DrainScope::AllTasks`], the waiting task in its turn. + fn next_task(&mut self, scope: DrainScope) -> Option { + if let Some(task) = self.pop_runnable_task() { + return Some(task); + } + if scope == DrainScope::AllTasks && self.waiting.is_some() { + self.unpark(); + return self.tasks.pop(); + } + None + } +} + +/// Whether two queued tasks are of the same [`EngineTask`] kind, which the queue orders alike. +fn is_same_kind(a: &QueuedTask, b: &QueuedTask) -> bool { + a.task.cmp(&b.task) == Ordering::Equal +} + +/// Which tasks an [`Engine::drain`] runs. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DrainScope { + /// Only tasks of other kinds than the waiting task, which stays parked. Without a waiting + /// task, every task. + AheadOfWaitingTask, + /// Every task, retrying the waiting task in its turn. + AllTasks, } /// An error occurred while attempting to reset the [`Engine`]. @@ -242,10 +334,13 @@ mod recovery_tests { Arc::new(NoopBlockSink), )))); // The old inline retry loop never returned from this call. - let error = tokio::time::timeout(std::time::Duration::from_secs(1), engine.drain()) - .await - .expect("temporary errors must yield to the actor") - .unwrap_err(); + let error = tokio::time::timeout( + std::time::Duration::from_secs(1), + engine.drain(DrainScope::AllTasks), + ) + .await + .expect("temporary errors must yield to the actor") + .unwrap_err(); assert_eq!(error.severity(), EngineTaskErrorSeverity::Temporary); assert_eq!(engine.len(), 1); // Accumulate additional unsafe imports during the outage. Equal-priority heap @@ -274,13 +369,55 @@ mod recovery_tests { ForkchoiceUpdated::new(PayloadStatus::from_status(PayloadStatusEnum::Valid)), ); } - engine.drain().await.unwrap(); + engine.drain(DrainScope::AllTasks).await.unwrap(); assert!(engine.is_empty()); assert_eq!(engine.state().sync_state.unsafe_head().block_info.number, 3); l1.assert_finished(); l2.assert_finished(); } + // A drain that may retry the waiting task runs the queued tasks of other kinds first. When one + // of them fails, the waiting task is not retried yet and stays parked for its own retry. + #[tokio::test] + async fn failure_ahead_of_the_waiting_task_leaves_it_parked() { + let payload = OpExecutionPayloadEnvelope::V1(ExecutionPayloadV1::from_block_slow( + &alloy_consensus::Block::::default(), + )); + let block: op_alloy_consensus::OpBlock = payload.clone().try_into_block().unwrap(); + let mut config = RollupConfig::default(); + config.genesis.l2.hash = block.header.hash_slow(); + let config = Arc::new(config); + let (client, l1, l2) = test_engine_client(config.clone()); + let client = Arc::new(client); + l2.expect_error("engine_newPayloadV2"); + let (state_tx, _state_rx) = tokio::sync::watch::channel(EngineState::default()); + let mut engine = Engine::new(EngineState::default(), state_tx); + engine.enqueue(EngineTask::Consolidate(Box::new(ConsolidateTask::new( + client.clone(), + config.clone(), + ConsolidateInput::Attributes(Box::new(TestAttributesBuilder::new().build())), + Arc::new(NoopBlockSink), + crate::DEFAULT_EL_SYNC_STALL_TIMEOUT, + )))); + let waiting = engine.pop_runnable_task().expect("the queued consolidation"); + engine.park(waiting); + engine.enqueue(EngineTask::Insert(Box::new(InsertTask::new( + client, + config, + payload, + false, + Arc::new(NoopBlockSink), + )))); + + let error = engine.drain(DrainScope::AllTasks).await.unwrap_err(); + + assert_eq!(error.severity(), EngineTaskErrorSeverity::Temporary, "got {error:?}"); + assert!(engine.has_waiting_task()); + assert_eq!(engine.len(), 2); + l1.assert_finished(); + l2.assert_finished(); + } + // A flush means the task already replaced its block with a deposits-only block. Keeping it // would rebuild that replacement on every later drain. #[tokio::test] @@ -346,9 +483,10 @@ mod recovery_tests { config, ConsolidateInput::Attributes(Box::new(attributes)), Arc::new(NoopBlockSink), + crate::DEFAULT_EL_SYNC_STALL_TIMEOUT, )))); - let error = engine.drain().await.unwrap_err(); + let error = engine.drain(DrainScope::AllTasks).await.unwrap_err(); assert_eq!(error.severity(), EngineTaskErrorSeverity::Flush, "got {error:?}"); l1.assert_finished(); l2.assert_finished(); @@ -404,7 +542,7 @@ mod recovery_tests { FinalizeBlockId::ByNumber(5), )))); - let error = engine.drain().await.unwrap_err(); + let error = engine.drain(DrainScope::AllTasks).await.unwrap_err(); assert_eq!(error.severity(), EngineTaskErrorSeverity::Skip, "got {error:?}"); assert!(engine.is_empty()); @@ -413,3 +551,166 @@ mod recovery_tests { l2.assert_finished(); } } + +#[cfg(test)] +mod scheduling_tests { + use super::*; + use crate::{ + ConsolidateInput, ConsolidateTask, FinalizeBlockId, FinalizeTask, InsertTask, + NoopBlockSink, + test_utils::{TestAttributesBuilder, test_engine_client}, + }; + use TaskKind::*; + use alloy_rpc_types_engine::ExecutionPayloadV1; + use op_alloy_rpc_types_engine::OpExecutionPayloadEnvelope; + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + enum TaskKind { + Consolidate, + Finalize, + Insert, + } + + /// A queued task, as its kind and sequence. + type Scheduled = (TaskKind, u64); + + /// An engine whose queue holds tasks that are scheduled but never executed. + struct Scheduler { + client: Arc, + config: Arc, + engine: Engine, + } + + impl Scheduler { + fn new() -> Self { + let config = Arc::new(RollupConfig::default()); + let (client, _, _) = test_engine_client(config.clone()); + let (state_tx, _) = tokio::sync::watch::channel(EngineState::default()); + Self { + client: Arc::new(client), + config, + engine: Engine::new(EngineState::default(), state_tx), + } + } + + fn enqueue(&mut self, kind: TaskKind) { + let client = self.client.clone(); + let config = self.config.clone(); + self.engine.enqueue(match kind { + Consolidate => EngineTask::Consolidate(Box::new(ConsolidateTask::new( + client, + config, + ConsolidateInput::Attributes(Box::new(TestAttributesBuilder::new().build())), + Arc::new(NoopBlockSink), + crate::DEFAULT_EL_SYNC_STALL_TIMEOUT, + ))), + Finalize => EngineTask::Finalize(Box::new(FinalizeTask::new( + client, + config, + FinalizeBlockId::ByNumber(1), + ))), + Insert => EngineTask::Insert(Box::new(InsertTask::new( + client, + config, + OpExecutionPayloadEnvelope::V1(ExecutionPayloadV1::from_block_slow( + &alloy_consensus::Block::::default(), + )), + false, + Arc::new(NoopBlockSink), + ))), + }); + } + + /// Pops the next queued task, as a drain would, and parks it as a task that waits. + fn pop_and_park(&mut self) -> Scheduled { + let task = self.engine.pop_runnable_task().expect("a queued task"); + let scheduled = scheduled(&task); + self.engine.park(task); + scheduled + } + + /// The order in which a drain would run the queued tasks that may run now. + fn runnable_order(&mut self) -> Vec { + std::iter::from_fn(|| self.engine.pop_runnable_task()).map(|t| scheduled(&t)).collect() + } + + fn waiting(&self) -> Option { + self.engine.waiting.as_ref().map(scheduled) + } + } + + fn scheduled(queued: &QueuedTask) -> Scheduled { + let kind = match queued.task { + EngineTask::Consolidate(_) => Consolidate, + EngineTask::Finalize(_) => Finalize, + EngineTask::Insert(_) => Insert, + _ => unreachable!("only consolidate, finalize and insert tasks are scheduled"), + }; + (kind, queued.sequence) + } + + /// A consolidation parked while it waits (sequence 0) lets tasks of other kinds run before its + /// retry, imports and finalizations alike, then runs in its turn ahead of later + /// consolidations. + #[rstest::rstest] + #[case::insert_and_finalize_run_ahead(&[Finalize, Insert], &[(Insert, 2), (Finalize, 1)], &[(Consolidate, 0)])] + #[case::finalize_runs_ahead_of_later_consolidation(&[Consolidate, Finalize], &[(Finalize, 2)], &[(Consolidate, 0), (Consolidate, 1)])] + #[case::later_consolidation_stays_behind(&[Consolidate], &[], &[(Consolidate, 0), (Consolidate, 1)])] + #[case::inserts_run_ahead_in_order(&[Insert, Insert], &[(Insert, 1), (Insert, 2)], &[(Consolidate, 0)])] + fn waiting_task_keeps_its_place( + #[case] queued_after_parking: &[TaskKind], + #[case] expected_ahead_of_retry: &[Scheduled], + #[case] expected_at_retry: &[Scheduled], + ) { + let mut scheduler = Scheduler::new(); + scheduler.enqueue(Consolidate); + scheduler.pop_and_park(); + for kind in queued_after_parking { + scheduler.enqueue(*kind); + } + + let ahead_of_retry = scheduler.runnable_order(); + scheduler.engine.unpark(); + let at_retry = scheduler.runnable_order(); + + assert_eq!(ahead_of_retry, expected_ahead_of_retry); + assert_eq!(at_retry, expected_at_retry); + } + + /// A waiting task that waits again at its retry is parked again with its sequence, still ahead + /// of a later consolidation that does not run before it. + #[test] + fn task_waiting_again_is_parked_again_with_its_sequence() { + let mut scheduler = Scheduler::new(); + scheduler.enqueue(Consolidate); + scheduler.pop_and_park(); + scheduler.enqueue(Consolidate); + + scheduler.engine.unpark(); + let retried = scheduler.pop_and_park(); + + assert_eq!(retried, (Consolidate, 0)); + assert_eq!(scheduler.waiting(), Some((Consolidate, 0))); + assert_eq!(scheduler.runnable_order(), []); + scheduler.engine.unpark(); + assert_eq!(scheduler.runnable_order(), [(Consolidate, 0), (Consolidate, 1)]); + } + + /// A task that outranks the waiting one and waits too is parked in its place, and the earlier + /// one goes back into the queue with its sequence instead of being dropped. + #[test] + fn parking_a_task_requeues_the_one_waiting_before_it() { + let mut scheduler = Scheduler::new(); + scheduler.enqueue(Consolidate); + scheduler.pop_and_park(); + scheduler.enqueue(Insert); + + let parked_second = scheduler.pop_and_park(); + + assert_eq!(parked_second, (Insert, 1)); + assert_eq!(scheduler.waiting(), Some((Insert, 1))); + assert_eq!(scheduler.engine.len(), 2); + scheduler.engine.unpark(); + assert_eq!(scheduler.runnable_order(), [(Insert, 1), (Consolidate, 0)]); + } +} diff --git a/rust/kona/crates/node/engine/src/task_queue/mod.rs b/rust/kona/crates/node/engine/src/task_queue/mod.rs index 57fd6ee9907..3dc2799860a 100644 --- a/rust/kona/crates/node/engine/src/task_queue/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/mod.rs @@ -1,7 +1,7 @@ //! The [`Engine`] task queue and the [`EngineTask`]s it can execute. mod core; -pub use core::{Engine, EngineResetError}; +pub use core::{DrainScope, Engine, EngineResetError}; mod tasks; pub use tasks::*; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/el_sync_progress.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/el_sync_progress.rs new file mode 100644 index 00000000000..2a592f9bcd1 --- /dev/null +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/el_sync_progress.rs @@ -0,0 +1,306 @@ +//! Whether the execution layer makes progress while a consolidation waits for it to sync. + +use serde_json::{Map, Value}; +use std::{mem, time::Duration}; +use tokio::time::Instant; + +/// The default bound on how long a consolidation waiting for the execution layer to sync tolerates +/// no reported progress before the engine resets. +/// +/// Neither execution layer reports progress while it downloads headers: op-geth answers +/// `eth_syncing` with `false` until its beacon skeleton links to the local chain, and op-reth's +/// headers stage moves its checkpoint only once the whole range is downloaded. That download can +/// take hours on a large chain, and a reset during it brings back the reset loop the wait exists to +/// avoid, while a longer bound only delays the reset of an execution layer that can never fetch the +/// missing block. +pub const DEFAULT_EL_SYNC_STALL_TIMEOUT: Duration = Duration::from_secs(24 * 60 * 60); + +/// How often a waiting consolidation checks on the execution layer: it reads the sync progress and +/// resends the unchanged forkchoice to learn whether the execution layer now has the unsafe head. +/// The retries in between only look the derived block up, which is cheap for an execution layer +/// that is busy syncing. +pub(super) const CHECK_INTERVAL: Duration = Duration::from_secs(30); + +/// How often a waiting consolidation reports the execution layer's progress, or its lack of it. +pub(super) const PROGRESS_REPORT_INTERVAL: Duration = Duration::from_secs(5 * 60); + +/// What the execution layer reports about its sync on one check of a waiting consolidation. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum ElSyncReading { + /// `eth_syncing` reports no sync in progress, and this is the latest block number. + NotSyncing { + /// The number of the execution layer's latest block. + latest_number: u64, + }, + /// `eth_syncing` reports a sync in progress, with these progress fields. + Syncing(Map), +} + +impl ElSyncReading { + /// Whether the execution layer reports no sync while its latest block is `parent_number`: it + /// has the parent of the next block but fetches nothing toward the unsafe head. + pub(super) const fn is_idle_at(&self, parent_number: u64) -> bool { + matches!(self, Self::NotSyncing { latest_number } if *latest_number == parent_number) + } + + /// The progress in an `eth_syncing` status object: every field but the block the sync started + /// from, which never changes, and the block it syncs toward, which can move while the sync + /// itself makes no progress. + pub(super) fn from_sync_status(mut status: Map) -> Self { + status.remove("startingBlock"); + status.remove("highestBlock"); + Self::Syncing(status) + } +} + +/// Whether a consolidation keeps waiting for the execution layer to sync. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum ElSyncWaitOutcome { + /// The execution layer reported progress. + Progressed, + /// The execution layer has reported no progress for the stall timeout. + Stalled, + /// The wait starts. + Started, + /// The execution layer reported no progress, or could not be read, within the stall timeout. + Unchanged, +} + +/// What one check of a waiting consolidation found. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct ElSyncWaitStep { + /// Whether the wait goes on, starts, or has stalled. + pub(super) outcome: ElSyncWaitOutcome, + /// Whether to report the execution layer's progress, or its lack of it. + pub(super) reports_progress: bool, + /// How long the execution layer may still report no progress before the engine resets. + pub(super) time_until_reset: Duration, +} + +/// Tracks the execution layer's sync across the retries of a waiting consolidation. +#[derive(Debug)] +pub(super) struct ElSyncProgress { + /// When a check's forkchoice probe was last answered. + checked_at: Option, + /// When the wait started or the execution layer last reported progress. + last_progress_at: Option, + /// The first reading that could be read, or the last one that differed from it. + last_reading: Option, + /// When the execution layer's progress was last reported. + progress_reported_at: Option, + /// How long the execution layer may report no progress. + stall_timeout: Duration, +} + +impl ElSyncProgress { + /// Tracks a wait bounded by `stall_timeout`. + pub(super) const fn new(stall_timeout: Duration) -> Self { + Self { + checked_at: None, + last_progress_at: None, + last_reading: None, + progress_reported_at: None, + stall_timeout, + } + } + + /// Whether a retry at `now` checks on the execution layer: when the wait starts, every + /// [`CHECK_INTERVAL`] after the last answered check, and on the retry after a check whose + /// forkchoice probe failed. + pub(super) fn is_check_due(&self, now: Instant) -> bool { + self.checked_at.is_none_or(|checked_at| now.duration_since(checked_at) >= CHECK_INTERVAL) + } + + /// Records the execution layer's sync `reading` at `now`, `None` when it could not be read, + /// and returns what this check found. + /// + /// Any change of a reading counts as progress, including a lower latest block: an execution + /// layer whose datadir is wiped while it is awaited starts over from genesis. An unreadable + /// reading is no evidence either way, so it neither counts as progress nor replaces the last + /// reading, and the first reading that can be read is the baseline later ones are compared + /// with. Progress, or its lack, is reported when the wait starts, when a sync starts or + /// finishes, and otherwise at most every [`PROGRESS_REPORT_INTERVAL`]. + pub(super) fn record( + &mut self, + reading: Option, + now: Instant, + ) -> ElSyncWaitStep { + let Some(last_progress_at) = self.last_progress_at else { + self.last_progress_at = Some(now); + self.last_reading = reading; + self.progress_reported_at = Some(now); + return ElSyncWaitStep { + outcome: ElSyncWaitOutcome::Started, + reports_progress: true, + time_until_reset: self.stall_timeout, + }; + }; + + let outcome = match (&self.last_reading, reading) { + (None, Some(first_readable)) => { + self.last_reading = Some(first_readable); + self.unchanged_or_stalled(last_progress_at, now) + } + (Some(last), Some(reading)) if *last != reading => { + let sync_kind_changed = mem::discriminant(last) != mem::discriminant(&reading); + self.last_reading = Some(reading); + self.last_progress_at = Some(now); + if sync_kind_changed { + self.progress_reported_at = None; + } + ElSyncWaitOutcome::Progressed + } + _ => self.unchanged_or_stalled(last_progress_at, now), + }; + + let reports_progress = outcome != ElSyncWaitOutcome::Stalled && + self.progress_reported_at.is_none_or(|reported_at| { + now.duration_since(reported_at) >= PROGRESS_REPORT_INTERVAL + }); + if reports_progress { + self.progress_reported_at = Some(now); + } + let since_progress = now.duration_since(self.last_progress_at.unwrap_or(now)); + ElSyncWaitStep { + outcome, + reports_progress, + time_until_reset: self.stall_timeout.saturating_sub(since_progress), + } + } + + /// Whether a wait has started, which an answered check's reading does. + #[cfg(test)] + pub(super) const fn has_started(&self) -> bool { + self.last_progress_at.is_some() + } + + /// Records that the execution layer answered a check's forkchoice probe at `now`. A probe that + /// failed is not recorded, so the next retry checks again. + pub(super) const fn check_answered(&mut self, now: Instant) { + self.checked_at = Some(now); + } + + /// Whether the execution layer has reported no progress for the stall timeout at `now`. + fn unchanged_or_stalled(&self, last_progress_at: Instant, now: Instant) -> ElSyncWaitOutcome { + if now.duration_since(last_progress_at) >= self.stall_timeout { + ElSyncWaitOutcome::Stalled + } else { + ElSyncWaitOutcome::Unchanged + } + } +} + +#[cfg(test)] +mod tests { + use super::{ElSyncWaitOutcome::*, *}; + + const STALL_SECS: u64 = 3600; + const CHECK_SECS: u64 = CHECK_INTERVAL.as_secs(); + const REPORT_SECS: u64 = PROGRESS_REPORT_INTERVAL.as_secs(); + + /// A reading of an execution layer that is not syncing a range, at `latest_number`. + const fn at(latest_number: u64) -> Option { + Some(ElSyncReading::NotSyncing { latest_number }) + } + + /// A reading of an execution layer syncing toward `highest_block`, whose `stage` checkpoint is + /// at `stage_block`. + fn staged(stage_block: u64, highest_block: u64) -> Option { + let status = serde_json::json!({ + "currentBlock": "0x0", + "highestBlock": format!("{highest_block:#x}"), + "stages": [{ "name": "Headers", "block": format!("{stage_block:#x}") }], + "startingBlock": "0x0", + }); + let serde_json::Value::Object(status) = status else { unreachable!() }; + Some(ElSyncReading::from_sync_status(status)) + } + + /// Records each check, `(seconds since the previous one, reading)`, and returns the last one. + fn last_step(checks: &[(u64, Option)]) -> ElSyncWaitStep { + let mut progress = ElSyncProgress::new(Duration::from_secs(STALL_SECS)); + let mut now = Instant::now(); + let mut last = None; + for (elapsed_secs, reading) in checks { + now += Duration::from_secs(*elapsed_secs); + last = Some(progress.record(reading.clone(), now)); + } + last.expect("at least one check") + } + + #[rstest::rstest] + #[case::first_reading_starts_the_wait(&[(0, at(100))], Started)] + #[case::first_unreadable_reading_starts_the_wait(&[(0, None)], Started)] + #[case::unchanged_just_below_bound_waits(&[(0, at(100)), (STALL_SECS - 1, at(100))], Unchanged)] + #[case::unchanged_for_bound_stalls(&[(0, at(100)), (STALL_SECS, at(100))], Stalled)] + #[case::advancing_latest_block_progresses(&[(0, at(100)), (STALL_SECS - 1, at(101))], Progressed)] + #[case::stalls_after_progress(&[(0, at(100)), (STALL_SECS - 1, at(101)), (STALL_SECS, at(101))], Stalled)] + #[case::wiped_el_restarting_counts_as_progress(&[(0, at(100)), (STALL_SECS - 1, at(0))], Progressed)] + #[case::sync_starting_counts_as_progress(&[(0, at(100)), (STALL_SECS - 1, staged(10, 500))], Progressed)] + #[case::unchanged_sync_for_bound_stalls(&[(0, staged(10, 500)), (STALL_SECS, staged(10, 500))], Stalled)] + #[case::advancing_stage_progresses(&[(0, staged(10, 500)), (STALL_SECS - 1, staged(20, 500))], Progressed)] + #[case::moving_sync_target_alone_stalls(&[(0, staged(10, 500)), (STALL_SECS, staged(10, 600))], Stalled)] + #[case::unreadable_reading_is_not_progress(&[(0, at(100)), (STALL_SECS - 1, None), (1, at(100))], Stalled)] + #[case::unreadable_readings_stall(&[(0, None), (STALL_SECS, None)], Stalled)] + #[case::first_readable_reading_is_not_progress(&[(0, None), (1, at(100))], Unchanged)] + #[case::first_readable_reading_keeps_the_bound(&[(0, None), (1, at(100)), (STALL_SECS - 1, at(100))], Stalled)] + fn sync_progress_bounds_the_wait( + #[case] checks: &[(u64, Option)], + #[case] expected: ElSyncWaitOutcome, + ) { + assert_eq!(last_step(checks).outcome, expected); + } + + /// Whether a retry `elapsed_secs` after the wait starts checks on the execution layer, when + /// the first check's forkchoice probe was answered or failed. + #[rstest::rstest] + #[case::wait_start_checks(None, 0, true)] + #[case::retry_before_the_interval_does_not_check(Some(true), CHECK_SECS - 1, false)] + #[case::retry_after_the_interval_checks(Some(true), CHECK_SECS, true)] + #[case::failed_probe_checks_on_the_next_retry(Some(false), 2, true)] + fn checks_are_due_on_their_cadence( + #[case] first_check_answered: Option, + #[case] elapsed_secs: u64, + #[case] expected: bool, + ) { + let mut progress = ElSyncProgress::new(Duration::from_secs(STALL_SECS)); + let started_at = Instant::now(); + if let Some(answered) = first_check_answered { + progress.record(at(100), started_at); + if answered { + progress.check_answered(started_at); + } + } + assert_eq!(progress.is_check_due(started_at + Duration::from_secs(elapsed_secs)), expected); + } + + #[rstest::rstest] + #[case::wait_start_reports(&[(0, at(100))], true)] + #[case::progress_before_the_interval_is_quiet(&[(0, at(100)), (REPORT_SECS - 1, at(101))], false)] + #[case::progress_after_the_interval_reports(&[(0, at(100)), (REPORT_SECS, at(101))], true)] + #[case::no_progress_before_the_interval_is_quiet(&[(0, at(100)), (REPORT_SECS - 1, at(100))], false)] + #[case::no_progress_after_the_interval_reports(&[(0, at(100)), (REPORT_SECS, at(100))], true)] + #[case::unreadable_after_the_interval_reports(&[(0, at(100)), (REPORT_SECS, None)], true)] + #[case::sync_starting_reports(&[(0, at(100)), (1, staged(10, 500))], true)] + #[case::sync_finishing_reports(&[(0, staged(10, 500)), (1, at(500))], true)] + #[case::first_readable_reading_is_quiet(&[(0, None), (1, at(100))], false)] + #[case::stall_is_not_reported_as_progress(&[(0, at(100)), (STALL_SECS, at(100))], false)] + fn progress_is_reported_on_its_cadence( + #[case] checks: &[(u64, Option)], + #[case] expected: bool, + ) { + assert_eq!(last_step(checks).reports_progress, expected); + } + + #[rstest::rstest] + #[case::wait_start_has_the_whole_bound(&[(0, at(100))], STALL_SECS)] + #[case::no_progress_uses_up_the_bound(&[(0, at(100)), (100, at(100))], STALL_SECS - 100)] + #[case::progress_restores_the_bound(&[(0, at(100)), (100, at(101))], STALL_SECS)] + #[case::first_readable_reading_does_not_restore_the_bound(&[(0, None), (100, at(100))], STALL_SECS - 100)] + fn time_until_reset_counts_from_the_last_progress( + #[case] checks: &[(u64, Option)], + #[case] expected_secs: u64, + ) { + assert_eq!(last_step(checks).time_until_reset, Duration::from_secs(expected_secs)); + } +} diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs index 2ce1b187a76..8a7ef63879f 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/error.rs @@ -19,9 +19,17 @@ pub enum ConsolidateTaskError { /// The unsafe L2 block is missing. #[error("Unsafe L2 block is missing {0}")] MissingUnsafeL2Block(u64), + /// The unsafe L2 block is missing because the execution layer is still syncing toward the + /// unsafe head. + #[error("Unsafe L2 block {0} is missing while the execution layer syncs")] + MissingUnsafeL2BlockDuringElSync(u64), /// Failed to fetch the unsafe L2 block. #[error("Failed to fetch the unsafe L2 block")] FailedToFetchUnsafeL2Block, + /// The unchanged forkchoice update asking whether the execution layer still syncs toward the + /// unsafe head, for a missing unsafe L2 block, failed. + #[error("Forkchoice update for the missing unsafe L2 block failed: {0}")] + MissingUnsafeL2BlockProbeFailed(#[source] SynchronizeTaskError), /// Starting the build of a block from the payload attributes failed. #[error(transparent)] BuildTaskFailed(#[from] BuildTaskError), @@ -47,12 +55,19 @@ impl EngineTaskError for ConsolidateTaskError { match self { Self::BreaksFinality(_) => EngineTaskErrorSeverity::Skip, Self::MissingUnsafeL2Block(_) => EngineTaskErrorSeverity::Reset, + // Only an execution layer that answers but still syncs makes the task wait on a fixed + // interval. A failed RPC is retried on the growing failure backoff, so an outage is + // not polled at the wait's cadence; the tasks that outrank this one would hit the same + // failing execution layer anyway. Self::FailedToFetchDelegatedSafeBlock(_) | Self::FailedToFetchUnsafeL2Block => { EngineTaskErrorSeverity::Temporary } + Self::MissingUnsafeL2BlockDuringElSync(_) => EngineTaskErrorSeverity::Wait, Self::BuildTaskFailed(inner) => inner.severity(), Self::SealTaskFailed(inner) => inner.severity(), - Self::ForkchoiceUpdateFailed(inner) => inner.severity(), + Self::ForkchoiceUpdateFailed(inner) | Self::MissingUnsafeL2BlockProbeFailed(inner) => { + inner.severity() + } } } } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/mod.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/mod.rs index 21b125bd135..17f14044070 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/mod.rs @@ -1,5 +1,8 @@ //! Consolidation Task +mod el_sync_progress; +pub use el_sync_progress::DEFAULT_EL_SYNC_STALL_TIMEOUT; + mod error; pub use error::ConsolidateTaskError; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs index 00f31457729..012959918bd 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/consolidate/task.rs @@ -1,16 +1,25 @@ //! A task to consolidate the engine state. +use super::el_sync_progress::{ElSyncProgress, ElSyncReading, ElSyncWaitOutcome, ElSyncWaitStep}; use crate::{ ConsolidateTaskError, EngineClient, EngineState, ExecutableEngineTask, ImportedBlockSink, state::EngineSyncStateUpdate, - task_queue::{build_and_import, synchronize, synchronize_unconditionally}, + task_queue::{ + ForkchoiceReply, build_and_import, record_forkchoice_reply, send_forkchoice, synchronize, + synchronize_unconditionally, + }, }; +use alloy_provider::Provider; use alloy_rpc_types_eth::Block; use async_trait::async_trait; use kona_genesis::RollupConfig; use kona_protocol::{L2BlockInfo, OpAttributesWithParent}; use op_alloy_rpc_types::Transaction; -use std::sync::Arc; +use std::{ + sync::{Arc, Mutex, MutexGuard, PoisonError}, + time::Duration, +}; +use tokio::time::Instant; /// Input for consolidation - either derived attributes or safe L2 block #[derive(Debug, Clone)] @@ -58,6 +67,15 @@ impl ConsolidateInput { } } +/// What a consolidation waiting for the execution layer does with a derived block it lacked. +#[derive(Debug)] +enum DerivedBlock { + /// Build the block from the derived attributes on its parent, which the execution layer has. + BuildFromAttributes, + /// Consolidate the block the execution layer committed since the lookup. + Committed(Box>), +} + /// The [`ConsolidateTask`] attempts to consolidate the engine state /// using the specified payload attributes or block info. #[derive(Debug, Clone)] @@ -70,17 +88,28 @@ pub struct ConsolidateTask { pub input: ConsolidateInput, /// Where to hand the decoded block once the engine has canonicalized it. pub block_sink: Arc, + /// The execution layer's progress across the retries of this task while it waits for the + /// execution layer to sync. + el_sync_progress: Arc>, } impl ConsolidateTask { - /// Creates a new [`ConsolidateTask`] with the specified input - pub const fn new( + /// Creates a new [`ConsolidateTask`] with the specified input. A wait for the execution layer + /// to sync resets the engine once its sync reports no progress for `el_sync_stall_timeout`. + pub fn new( client: Arc, cfg: Arc, input: ConsolidateInput, block_sink: Arc, + el_sync_stall_timeout: Duration, ) -> Self { - Self { client, cfg, input, block_sink } + let el_sync_progress = Arc::new(Mutex::new(ElSyncProgress::new(el_sync_stall_timeout))); + Self { block_sink, cfg, client, el_sync_progress, input } + } + + /// The execution layer's progress across the retries of this task. + fn el_sync_progress(&self) -> MutexGuard<'_, ElSyncProgress> { + self.el_sync_progress.lock().unwrap_or_else(PoisonError::into_inner) } /// This is used when the [`ConsolidateTask`] fails to consolidate the engine state @@ -268,6 +297,196 @@ impl ConsolidateTask { } } + /// Waits for the execution layer to sync a derived block that it did not have, at or below the + /// unsafe head, and returns the block if the execution layer has it after all. + /// + /// Retries between checks only rely on the block lookup that precedes this call. A check reads + /// the execution layer's sync progress and resends the engine's unchanged forkchoice: + /// + /// - `SYNCING`: the execution layer is still syncing toward the unsafe head and will fill the + /// block in, so the task waits. Resetting instead would restart from the execution layer's + /// own labels, which are genesis after a datadir wipe, and re-derive the chain in a loop. + /// - `VALID`: the execution layer has the unsafe head, so it has the block unless it lost it. + /// It may have committed the block since the lookup, so the block is looked up once more. + /// + /// An execution layer that answers `SYNCING` while it reports no sync and sits at the derived + /// block's parent is not fetching anything, as without peers, so the block is built from the + /// derived attributes instead. A `SYNCING` answer after the progress has stalled for the bound + /// resets the engine. Only a `VALID` answer changes the engine state, so a task that then waits + /// has changed nothing. + async fn derived_block_once_el_syncs( + &self, + state: &mut EngineState, + block_number: u64, + ) -> Result { + let unsafe_number = state.sync_state.unsafe_head().block_info.number; + if !self.el_sync_progress().is_check_due(Instant::now()) { + debug!( + target: "engine", + block_number, + unsafe_number, + "Still waiting for the execution layer to sync the derived block" + ); + return Err(ConsolidateTaskError::MissingUnsafeL2BlockDuringElSync(block_number)); + } + + let reading = self.el_sync_reading().await; + // Every check probes, a stalled one included: the execution layer may have reached the + // unsafe head since the last check, and then the block is consolidated instead of reset. + if self.el_has_unsafe_head(state, block_number).await? { + record_forkchoice_reply(state, ForkchoiceReply::Valid); + return self + .derived_block_committed_since_lookup(block_number) + .await + .map(|block| DerivedBlock::Committed(Box::new(block))); + } + // The reading counts, and its report goes out, only once the probe is answered. + let now = Instant::now(); + self.el_sync_progress().check_answered(now); + let step = self.el_sync_progress().record(reading.clone(), now); + if reading + .as_ref() + .is_some_and(|reading| reading.is_idle_at(block_number.saturating_sub(1))) + { + info!( + target: "engine", + block_number, + ?reading, + unsafe_number, + "Execution layer is idle at the derived block's parent; building the derived block" + ); + return Ok(DerivedBlock::BuildFromAttributes); + } + if step.outcome == ElSyncWaitOutcome::Stalled { + warn!( + target: "engine", + block_number, + ?reading, + unsafe_number, + "Execution layer stopped making progress toward the unsafe head; resetting" + ); + return Err(ConsolidateTaskError::MissingUnsafeL2Block(block_number)); + } + Self::log_wait_for_el_sync(step, reading.as_ref(), block_number, unsafe_number); + Err(ConsolidateTaskError::MissingUnsafeL2BlockDuringElSync(block_number)) + } + + /// Resends the engine's unchanged forkchoice and returns whether the execution layer applied + /// it, rather than syncing toward its head. + async fn el_has_unsafe_head( + &self, + state: &EngineState, + block_number: u64, + ) -> Result { + debug!(target: "engine", block_number, "Asking the execution layer whether it has the unsafe head"); + let reply = send_forkchoice(self.client.as_ref(), &state.sync_state).await.map_err(|err| { + warn!(target: "engine", ?err, block_number, "Forkchoice update for a missing derived block failed"); + ConsolidateTaskError::MissingUnsafeL2BlockProbeFailed(err) + })?; + debug!(target: "engine", block_number, ?reply, "Execution layer answered the forkchoice update"); + Ok(reply == ForkchoiceReply::Valid) + } + + /// Logs a retry that keeps waiting: at `info!` when the wait starts and when progress is + /// reported, at `warn!` when no progress is reported on that cadence, and at `debug!` + /// otherwise. + fn log_wait_for_el_sync( + step: ElSyncWaitStep, + reading: Option<&ElSyncReading>, + block_number: u64, + unsafe_number: u64, + ) { + let time_until_reset = step.time_until_reset; + match step.outcome { + ElSyncWaitOutcome::Started => info!( + target: "engine", + block_number, + ?reading, + ?time_until_reset, + unsafe_number, + "Execution layer is syncing toward the unsafe head; waiting for the derived block" + ), + ElSyncWaitOutcome::Progressed if step.reports_progress => info!( + target: "engine", + block_number, + ?reading, + ?time_until_reset, + unsafe_number, + "Execution layer sync is progressing; still waiting for the derived block" + ), + _ if step.reports_progress => warn!( + target: "engine", + block_number, + ?reading, + ?time_until_reset, + unsafe_number, + "Execution layer reports no sync progress; the engine resets if none comes in time" + ), + _ => debug!( + target: "engine", + block_number, + ?reading, + ?time_until_reset, + "Still waiting for the execution layer to sync the derived block" + ), + } + } + + /// What the execution layer reports about its sync: the progress of the range it syncs, or + /// its latest block number when it syncs none. `None` if that cannot be read. + async fn el_sync_reading(&self) -> Option { + trace!(target: "engine", "Fetching the execution layer's sync status"); + match self.client.client().request_noparams::("eth_syncing").await { + Ok(serde_json::Value::Object(status)) => Some(ElSyncReading::from_sync_status(status)), + Ok(serde_json::Value::Bool(false)) => self + .el_latest_block_number() + .await + .map(|latest_number| ElSyncReading::NotSyncing { latest_number }), + Ok(status) => { + warn!(target: "engine", %status, "Execution layer returned an unexpected sync status"); + None + } + Err(err) => { + warn!(target: "engine", ?err, "Failed to fetch the execution layer's sync status"); + None + } + } + } + + /// The number of the execution layer's latest block, or `None` if it cannot be read. + async fn el_latest_block_number(&self) -> Option { + trace!(target: "engine", "Fetching the execution layer's latest block number"); + self.client + .get_block_number() + .await + .map_err(|err| { + warn!(target: "engine", ?err, "Failed to fetch the execution layer's latest block number"); + }) + .ok() + } + + /// Looks up the derived block again after the execution layer answered `VALID` for the unsafe + /// head; the engine resets if the block is still missing. + async fn derived_block_committed_since_lookup( + &self, + block_number: u64, + ) -> Result, ConsolidateTaskError> { + match self.client.l2_block_by_label(block_number.into()).await { + Ok(Some(block)) => { + info!(target: "engine", block_number, "Execution layer committed the derived block since the lookup"); + Ok(block) + } + Ok(None) => { + warn!(target: "engine", block_number, "Execution layer has the unsafe head but not the derived block"); + Err(ConsolidateTaskError::MissingUnsafeL2Block(block_number)) + } + Err(err) => { + warn!(target: "engine", ?err, block_number, "Failed to look up the derived block again"); + Err(ConsolidateTaskError::FailedToFetchUnsafeL2Block) + } + } + } + /// Attempts consolidation on the engine state. #[tracing::instrument( name = "consolidate", @@ -285,8 +504,12 @@ impl ConsolidateTask { if let ConsolidateInput::BlockInfo(safe_l2) = &self.input { return self.follow_safe_head_during_el_sync(state, safe_l2).await; } - warn!(target: "engine", "Received `None` block for {}", block_num); - return Err(ConsolidateTaskError::MissingUnsafeL2Block(block_num)); + match self.derived_block_once_el_syncs(state, block_num).await? { + DerivedBlock::Committed(block) => *block, + DerivedBlock::BuildFromAttributes => { + return self.reconcile_unsafe_to_safe(state).await; + } + } } Err(_) => { return Err(ConsolidateTaskError::FailedToFetchUnsafeL2Block); @@ -435,17 +658,19 @@ impl ExecutableEngineTask for ConsolidateTask { mod tests { use super::*; use crate::{ - EngineSyncState, EngineTaskError, FinalityViolation, ImportedBlockSink, NoopBlockSink, - SealTaskError, + EngineSyncState, EngineTaskError, + EngineTaskErrorSeverity::{self, Reset, Temporary, Wait}, + FinalityViolation, ImportedBlockSink, NoopBlockSink, SealTaskError, test_utils::{ - TestAttributesBuilder, TestEngineStateBuilder, l1_info_deposit, test_block_info, - test_engine_client, + RpcMock, TestAttributesBuilder, TestEngineStateBuilder, l1_info_deposit, + test_block_info, test_engine_client, test_rpc_block_from_attributes, }, }; - use alloy_consensus::transaction::Recovered; use alloy_eips::{BlockNumHash, Encodable2718}; - use alloy_primitives::{Address, B256}; - use alloy_rpc_types_engine::{ForkchoiceUpdated, PayloadStatus, PayloadStatusEnum}; + use alloy_primitives::B256; + use alloy_rpc_types_engine::{ + ForkchoiceUpdated, INVALID_FORK_CHOICE_STATE_ERROR, PayloadStatus, PayloadStatusEnum, + }; use alloy_rpc_types_eth::{Block as RpcBlock, BlockTransactions, Header as RpcHeader}; use kona_genesis::ChainGenesis; use kona_protocol::BlockInfo; @@ -480,8 +705,13 @@ mod tests { ..Default::default() }; let sink = Arc::new(RecordingSink::default()); - let task = - ConsolidateTask::new(client, cfg, ConsolidateInput::BlockInfo(safe_l2), sink.clone()); + let task = ConsolidateTask::new( + client, + cfg, + ConsolidateInput::BlockInfo(safe_l2), + sink.clone(), + STALL, + ); task.consolidate(&mut EngineState::default()).await.unwrap(); @@ -519,6 +749,7 @@ mod tests { cfg, ConsolidateInput::Attributes(Box::new(attributes.clone())), Arc::new(crate::NoopBlockSink), + STALL, ); task.build_and_import_attributes(&mut state, attributes) .await @@ -700,6 +931,7 @@ mod tests { cfg, ConsolidateInput::BlockInfo(delegated_safe_head), sink.clone(), + STALL, ); task.execute(&mut state).await.unwrap(); @@ -710,36 +942,269 @@ mod tests { l2.assert_finished(); } - /// Unlike a delegated safe head, derived attributes whose block is missing below the unsafe - /// head still reset the engine. - #[tokio::test] - async fn derived_attributes_missing_from_el_reset_the_engine() { - let cfg = Arc::new(RollupConfig::default()); + const STALL_SECS: u64 = 3600; + const STALL: Duration = Duration::from_secs(STALL_SECS); + + /// A derived-attributes task for block 10 and an engine state whose unsafe head is above it. + fn task_for_missing_derived_block(client: EngineClient) -> (ConsolidateTask, EngineState) { let attributes = derived_attributes(true); - let mut state = engine_state_with_safe_head(attributes.parent); - let state_before = state; - let (client, l1, l2) = test_engine_client(cfg.clone()); - l2.expect_params( - "eth_getBlockByNumber", - serde_json::json!(["0xa", true]), - None::>, - ); + let state = engine_state_with_safe_head(attributes.parent); let task = ConsolidateTask::new( Arc::new(client), - cfg, + Arc::new(RollupConfig::default()), ConsolidateInput::Attributes(Box::new(attributes)), Arc::new(NoopBlockSink), + STALL, ); + (task, state) + } + + /// The execution layer lacks block 10, reports no sync in progress, and answers the unchanged + /// forkchoice with `reply`. + fn el_lacks_derived_block(l2: &RpcMock, state: &EngineState, reply: PayloadStatusEnum) { + el_lacks_derived_block_and_reports(l2, IdleAt(100)); + l2.expect_forkchoice_updated(state.sync_state.create_forkchoice_state(), reply); + } + + /// The execution layer lacks block 10 and reports `report` about its sync. + fn el_lacks_derived_block_and_reports(l2: &RpcMock, report: ElReports) { + l2.expect_block_by_number(10, None); + report.expect_on(l2); + } + + fn el_syncing(l2: &RpcMock, state: &EngineState) { + el_lacks_derived_block(l2, state, PayloadStatusEnum::Syncing); + } + + fn el_valid_and_block_still_missing(l2: &RpcMock, state: &EngineState) { + el_lacks_derived_block(l2, state, PayloadStatusEnum::Valid); + l2.expect_block_by_number(10, None); + } + + fn el_valid_and_recheck_fails(l2: &RpcMock, state: &EngineState) { + el_lacks_derived_block(l2, state, PayloadStatusEnum::Valid); + l2.expect_error("eth_getBlockByNumber"); + } + + fn el_invalid(l2: &RpcMock, state: &EngineState) { + el_lacks_derived_block( + l2, + state, + PayloadStatusEnum::Invalid { validation_error: "bad block".into() }, + ); + } + + fn forkchoice_update_fails(l2: &RpcMock, _: &EngineState) { + el_lacks_derived_block_and_reports(l2, IdleAt(100)); + l2.expect_error("engine_forkchoiceUpdatedV3"); + } + + fn forkchoice_state_rejected(l2: &RpcMock, _: &EngineState) { + el_lacks_derived_block_and_reports(l2, IdleAt(100)); + l2.expect_error_code("engine_forkchoiceUpdatedV3", INVALID_FORK_CHOICE_STATE_ERROR as i64); + } + + /// Derived attributes whose block is missing below the unsafe head resend the unchanged + /// forkchoice. An execution layer still syncing toward the unsafe head makes the task wait, and + /// a failed RPC is retried on the failure backoff; one that has the unsafe head but still not + /// the block, or rejects the forkchoice, resets the engine. The labels stay unchanged. + #[rstest::rstest] + #[case::el_syncing_waits(el_syncing, Wait)] + #[case::el_valid_and_block_still_missing_resets(el_valid_and_block_still_missing, Reset)] + #[case::el_valid_and_recheck_failing_is_retried(el_valid_and_recheck_fails, Temporary)] + #[case::el_invalid_resets(el_invalid, Reset)] + #[case::forkchoice_update_failing_is_retried(forkchoice_update_fails, Temporary)] + #[case::forkchoice_state_rejected_resets(forkchoice_state_rejected, Reset)] + #[tokio::test] + async fn derived_block_missing_from_el_asks_whether_it_syncs( + #[case] el_answers: fn(&RpcMock, &EngineState), + #[case] expected_severity: EngineTaskErrorSeverity, + ) { + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + let (task, mut state) = task_for_missing_derived_block(client); + let state_before = state; + el_answers(&l2, &state); let err = task.execute(&mut state).await.unwrap_err(); - assert!(matches!(err, ConsolidateTaskError::MissingUnsafeL2Block(10)), "got {err:?}"); - assert_eq!(err.severity(), crate::EngineTaskErrorSeverity::Reset); + assert_eq!(err.severity(), expected_severity, "got {err:?}"); assert_eq!(state, state_before); l1.assert_finished(); l2.assert_finished(); } + /// A probe answered `VALID` tells the engine the execution layer applied its forkchoice and + /// finished syncing, even when the recheck of the block then fails; only the labels stay. + #[tokio::test] + async fn valid_probe_records_the_el_sync_even_when_the_recheck_fails() { + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + let (task, mut state) = task_for_missing_derived_block(client); + state.el_sync_finished = false; + let sync_state_before = state.sync_state; + el_valid_and_recheck_fails(&l2, &state); + + let err = task.execute(&mut state).await.unwrap_err(); + + assert_eq!(err.severity(), Temporary, "got {err:?}"); + assert!(state.el_sync_finished); + assert_eq!(state.sync_state, sync_state_before); + l1.assert_finished(); + l2.assert_finished(); + } + + /// An execution layer that commits the derived block between the lookup and the forkchoice + /// update answers `VALID`; the block is then consolidated instead of resetting the engine. + #[tokio::test] + async fn derived_block_committed_during_the_probe_is_consolidated() { + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + let (task, mut state) = task_for_missing_derived_block(client); + let ConsolidateInput::Attributes(attributes) = &task.input else { unreachable!() }; + let block = test_rpc_block_from_attributes(attributes); + let expected_sync_state = + with_safe_head(&state, block_info_of(&RollupConfig::default(), &block)); + el_lacks_derived_block(&l2, &state, PayloadStatusEnum::Valid); + l2.expect_block_by_number(10, Some(block)); + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + + task.execute(&mut state).await.unwrap(); + + assert_eq!(state.sync_state, expected_sync_state); + l1.assert_finished(); + l2.assert_finished(); + } + + /// What the execution layer reports about its sync on one retry. + #[derive(Debug, Clone, Copy)] + enum ElReports { + /// No sync in progress, at this latest block. + IdleAt(u64), + /// A sync in progress, at this current block. + SyncingAt(u64), + } + use ElReports::{IdleAt, SyncingAt}; + + impl ElReports { + fn expect_on(self, l2: &RpcMock) { + match self { + IdleAt(latest_number) => l2.expect_el_not_syncing_at(latest_number), + SyncingAt(current_number) => l2.expect_el_syncing_at(current_number), + } + } + } + + /// One retry of a task waiting for block 10, `elapsed_secs` after the previous one. The + /// execution layer lacks the block; on a check it also reports `check` about its sync and + /// answers the resent forkchoice with `SYNCING`. + #[derive(Debug, Clone, Copy)] + struct Retry { + check: Option, + elapsed_secs: u64, + } + + /// A retry that checks on the execution layer. + const fn checking(elapsed_secs: u64, report: ElReports) -> Retry { + Retry { check: Some(report), elapsed_secs } + } + + /// A retry between checks, which only looks the block up. + const fn looking_up(elapsed_secs: u64) -> Retry { + Retry { check: None, elapsed_secs } + } + + impl Retry { + fn expect_on(self, l2: &RpcMock, state: &EngineState) { + l2.expect_block_by_number(10, None); + if let Some(report) = self.check { + report.expect_on(l2); + l2.expect_forkchoice_updated( + state.sync_state.create_forkchoice_state(), + PayloadStatusEnum::Syncing, + ); + } + } + } + + const CHECK_SECS: u64 = super::super::el_sync_progress::CHECK_INTERVAL.as_secs(); + + /// Across the retries of one task, the execution layer reports its sync on each check. Each + /// retry yields one severity: the task waits while the report changes, and resets once it has + /// stayed the same for the bound and the final check's forkchoice is still answered + /// `SYNCING`. Retries between checks make no RPC but the block lookup. + #[rstest::rstest] + #[case::progressing_el_keeps_waiting(&[checking(0, IdleAt(100)), checking(STALL_SECS - 1, IdleAt(101)), checking(STALL_SECS - 1, IdleAt(102))], &[Wait, Wait, Wait])] + #[case::stuck_el_resets_after_the_bound(&[checking(0, IdleAt(100)), checking(STALL_SECS - CHECK_SECS, IdleAt(100)), checking(CHECK_SECS, IdleAt(100))], &[Wait, Wait, Reset])] + #[case::stuck_sync_resets_after_the_bound(&[checking(0, SyncingAt(100)), checking(STALL_SECS, SyncingAt(100))], &[Wait, Reset])] + #[case::retries_between_checks_only_look_up(&[checking(0, IdleAt(100)), looking_up(2), looking_up(2)], &[Wait, Wait, Wait])] + #[case::check_interval_elapsing_checks(&[checking(0, IdleAt(100)), looking_up(2), checking(CHECK_SECS - 2, IdleAt(100))], &[Wait, Wait, Wait])] + #[tokio::test(start_paused = true)] + async fn wait_for_el_sync_is_bounded_by_its_progress( + #[case] retries: &[Retry], + #[case] expected_severities: &[EngineTaskErrorSeverity], + ) { + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + let (task, mut state) = task_for_missing_derived_block(client); + let mut severities = Vec::new(); + for retry in retries { + tokio::time::advance(Duration::from_secs(retry.elapsed_secs)).await; + retry.expect_on(&l2, &state); + severities.push(task.execute(&mut state).await.unwrap_err().severity()); + } + + assert_eq!(severities, expected_severities); + l1.assert_finished(); + l2.assert_finished(); + } + + /// The check that finds the progress stalled still resends the forkchoice. An execution layer + /// that reached the unsafe head since the last check answers `VALID`, and the derived block is + /// consolidated instead of resetting the engine. + #[tokio::test(start_paused = true)] + async fn stalled_el_that_reached_the_unsafe_head_is_consolidated() { + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + let (task, mut state) = task_for_missing_derived_block(client); + checking(0, IdleAt(100)).expect_on(&l2, &state); + let first = task.execute(&mut state).await.unwrap_err().severity(); + + tokio::time::advance(STALL).await; + let ConsolidateInput::Attributes(attributes) = &task.input else { unreachable!() }; + let block = test_rpc_block_from_attributes(attributes); + let expected_sync_state = + with_safe_head(&state, block_info_of(&RollupConfig::default(), &block)); + el_lacks_derived_block(&l2, &state, PayloadStatusEnum::Valid); + l2.expect_block_by_number(10, Some(block)); + l2.expect_forkchoice_updated( + expected_sync_state.create_forkchoice_state(), + PayloadStatusEnum::Valid, + ); + task.execute(&mut state).await.unwrap(); + + assert_eq!(first, Wait); + assert_eq!(state.sync_state, expected_sync_state); + l1.assert_finished(); + l2.assert_finished(); + } + + /// A check whose forkchoice update fails is not counted as answered, so the next retry checks + /// again instead of waiting for the check interval. + #[tokio::test(start_paused = true)] + async fn failed_probe_is_retried_on_the_next_retry() { + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + let (task, mut state) = task_for_missing_derived_block(client); + forkchoice_update_fails(&l2, &state); + let first = task.execute(&mut state).await.unwrap_err().severity(); + + tokio::time::advance(Duration::from_secs(2)).await; + checking(0, IdleAt(100)).expect_on(&l2, &state); + let second = task.execute(&mut state).await.unwrap_err().severity(); + + assert_eq!([first, second], [Temporary, Wait]); + l1.assert_finished(); + l2.assert_finished(); + } + fn derived_attributes(is_last_in_span: bool) -> OpAttributesWithParent { let mut attributes = TestAttributesBuilder::new() .with_parent(test_block_info(9)) @@ -749,26 +1214,6 @@ mod tests { attributes } - /// The unsafe block that `attributes` derive, so consolidation finds it consistent. - fn unsafe_block_matching(attributes: &OpAttributesWithParent) -> RpcBlock { - let payload_attributes = &attributes.attributes.payload_attributes; - let header = alloy_consensus::Header { - beneficiary: payload_attributes.suggested_fee_recipient, - gas_limit: attributes.attributes.gas_limit.unwrap(), - mix_hash: payload_attributes.prev_randao, - number: attributes.block_number(), - parent_beacon_block_root: payload_attributes.parent_beacon_block_root, - parent_hash: attributes.parent.block_info.hash, - timestamp: payload_attributes.timestamp, - ..Default::default() - }; - let deposit = Transaction::from_transaction( - Recovered::new_unchecked(l1_info_deposit(), Address::ZERO), - Default::default(), - ); - RpcBlock::new(RpcHeader::new(header), BlockTransactions::Full(vec![deposit])) - } - /// Derived blocks inside a span batch only move the safe label; the span's last block sends /// the forkchoice update that delivers it. #[rstest::rstest] @@ -781,7 +1226,7 @@ mod tests { ) { let cfg = Arc::new(RollupConfig::default()); let attributes = derived_attributes(is_last_in_span); - let block = unsafe_block_matching(&attributes); + let block = test_rpc_block_from_attributes(&attributes); let derived_safe_head = block_info_of(&cfg, &block); let mut state = engine_state_with_safe_head(attributes.parent); let expected_sync_state = with_safe_head(&state, derived_safe_head); @@ -798,6 +1243,7 @@ mod tests { cfg, ConsolidateInput::Attributes(Box::new(attributes)), Arc::new(NoopBlockSink), + STALL, ); task.execute(&mut state).await.unwrap(); @@ -876,6 +1322,7 @@ mod tests { cfg, ConsolidateInput::BlockInfo(delegated_safe_head), Arc::new(NoopBlockSink), + STALL, ); let result = task.execute(&mut state).await; @@ -915,6 +1362,7 @@ mod tests { cfg, ConsolidateInput::BlockInfo(delegated_safe_head), sink.clone(), + STALL, ); task.execute(&mut state).await.unwrap(); @@ -951,6 +1399,7 @@ mod tests { cfg, ConsolidateInput::BlockInfo(delegated_safe_head), Arc::new(NoopBlockSink), + STALL, ); task.execute(&mut state).await.unwrap(); @@ -989,6 +1438,7 @@ mod tests { cfg, ConsolidateInput::BlockInfo(delegated_safe_head), Arc::new(NoopBlockSink), + STALL, ); task.execute(&mut state).await.unwrap(); @@ -1026,6 +1476,7 @@ mod tests { cfg, ConsolidateInput::BlockInfo(delegated_safe_head), Arc::new(NoopBlockSink), + STALL, ); let err = task.execute(&mut state).await.unwrap_err(); @@ -1036,6 +1487,27 @@ mod tests { l2.assert_finished(); } + /// An execution layer that answers `SYNCING` while it reports no sync and sits at the derived + /// block's parent fetches nothing, as without peers, so the block is built from the derived + /// attributes on that parent instead of waiting: the build's forkchoice update follows. + #[tokio::test] + async fn el_idle_at_the_parent_builds_the_derived_block() { + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + let (task, mut state) = task_for_missing_derived_block(client); + el_lacks_derived_block_and_reports(&l2, IdleAt(9)); + l2.expect_forkchoice_updated( + state.sync_state.create_forkchoice_state(), + PayloadStatusEnum::Syncing, + ); + l2.expect_error("engine_forkchoiceUpdatedV2"); + + let err = task.execute(&mut state).await.unwrap_err(); + + assert!(matches!(err, ConsolidateTaskError::BuildTaskFailed(_)), "got {err:?}"); + l1.assert_finished(); + l2.assert_finished(); + } + /// Labels with the unsafe head rewound below the safe head are moved back into order by any /// delegated safe head above finality instead of being kept: at or above the unsafe head the /// engine reorgs onto it, and below it the matching block consolidates. Keeping them would let @@ -1078,6 +1550,7 @@ mod tests { cfg, ConsolidateInput::BlockInfo(delegated_safe_head), Arc::new(NoopBlockSink), + STALL, ); task.execute(&mut state).await.unwrap(); @@ -1120,6 +1593,7 @@ mod tests { cfg, ConsolidateInput::BlockInfo(delegated_safe_head), Arc::new(NoopBlockSink), + STALL, ); task.execute(&mut state).await.unwrap(); @@ -1128,4 +1602,23 @@ mod tests { l1.assert_finished(); l2.assert_finished(); } + + /// A check whose probe fails records nothing: the wait has not started, so the next check's + /// reading starts it and its report still goes out. + #[tokio::test(start_paused = true)] + async fn failed_probe_does_not_start_the_wait() { + let (client, l1, l2) = test_engine_client(Arc::new(RollupConfig::default())); + let (task, mut state) = task_for_missing_derived_block(client); + forkchoice_update_fails(&l2, &state); + task.execute(&mut state).await.unwrap_err(); + let has_started_after_failure = task.el_sync_progress().has_started(); + + checking(0, IdleAt(100)).expect_on(&l2, &state); + task.execute(&mut state).await.unwrap_err(); + + assert!(!has_started_after_failure); + assert!(task.el_sync_progress().has_started()); + l1.assert_finished(); + l2.assert_finished(); + } } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs index 9eddaa76d00..593b3a29d9f 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/mod.rs @@ -7,7 +7,10 @@ pub use task::{ mod synchronize; pub use synchronize::SynchronizeTaskError; -pub(super) use synchronize::{synchronize, synchronize_unconditionally}; +pub(super) use synchronize::{ + ForkchoiceReply, record_forkchoice_reply, send_forkchoice, synchronize, + synchronize_unconditionally, +}; mod insert; pub(super) use insert::insert_payload; @@ -25,7 +28,9 @@ mod canonicalize; pub use canonicalize::{CanonicalizeTask, CanonicalizeTaskError}; mod consolidate; -pub use consolidate::{ConsolidateInput, ConsolidateTask, ConsolidateTaskError}; +pub use consolidate::{ + ConsolidateInput, ConsolidateTask, ConsolidateTaskError, DEFAULT_EL_SYNC_STALL_TIMEOUT, +}; mod finalize; pub use finalize::{FinalizeBlockId, FinalizeTask, FinalizeTaskError}; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs index fc28ccf0026..dc2233aa44a 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/helper.rs @@ -6,6 +6,15 @@ use crate::{ use alloy_rpc_types_engine::{INVALID_FORK_CHOICE_STATE_ERROR, PayloadStatusEnum}; use op_alloy_provider::ext::engine::OpEngineApi; +/// How the execution layer answered a forkchoice update it accepted. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(in crate::task_queue) enum ForkchoiceReply { + /// The execution layer does not have the head block yet and is syncing toward it. + Syncing, + /// The execution layer applied the forkchoice. + Valid, +} + /// Applies `state_update` to the engine's sync state and synchronizes the execution layer's /// forkchoice with it, using an `engine_forkchoiceUpdated` call without payload attributes. An /// update that leaves the labels unchanged is skipped. @@ -38,7 +47,7 @@ pub(in crate::task_queue) async fn synchronize( return Ok(()); } - send_forkchoice(client, state, new_sync_state).await + send_and_apply_forkchoice(client, state, new_sync_state).await } /// Like [`synchronize`], but sends the forkchoice update even when it leaves the labels unchanged. @@ -58,12 +67,12 @@ pub(in crate::task_queue) async fn synchronize_unconditionally( state_update: EngineSyncStateUpdate, ) -> Result<(), SynchronizeTaskError> { let new_sync_state = state.sync_state.apply_update(state_update); - send_forkchoice(client, state, new_sync_state).await + send_and_apply_forkchoice(client, state, new_sync_state).await } /// Sends the forkchoice of `new_sync_state` and, once the execution layer accepts it, makes it the /// engine's sync state. -async fn send_forkchoice( +async fn send_and_apply_forkchoice( client: &EngineClient, state: &mut EngineState, new_sync_state: EngineSyncState, @@ -81,12 +90,40 @@ async fn send_forkchoice( // Send the forkchoice update through the input. let forkchoice = new_sync_state.create_forkchoice_state(); tracing::Span::current().record("forkchoice", tracing::field::debug(forkchoice)); + let reply = send_forkchoice(client, &new_sync_state).await?; + record_forkchoice_reply(state, reply); + + // Apply the new sync state to the engine state. + state.sync_state = new_sync_state; - // Handle the forkchoice update result. + Ok(()) +} + +/// Records how the execution layer answered the engine's forkchoice: whether it applied that +/// forkchoice, which a `SYNCING` answer clears, and whether its sync has finished, which only a +/// `VALID` answer marks and nothing clears. +pub(in crate::task_queue) fn record_forkchoice_reply( + state: &mut EngineState, + reply: ForkchoiceReply, +) { + state.el_applied_last_forkchoice = reply == ForkchoiceReply::Valid; + if reply == ForkchoiceReply::Valid && !state.el_sync_finished { + info!(target: "engine", "Finished execution layer sync."); + state.el_sync_finished = true; + } +} + +/// Sends `sync_state`'s forkchoice to the execution layer, with no payload attributes, and returns +/// whether it applied the forkchoice or is still syncing toward its head. Neither the engine state +/// nor its record of the execution layer's sync changes. +pub(in crate::task_queue) async fn send_forkchoice( + client: &EngineClient, + sync_state: &EngineSyncState, +) -> Result { // NOTE: it doesn't matter which version we use here, because we're not sending any payload // attributes. The forkchoice updated call is version agnostic if no payload attributes are // provided. - let response = client.fork_choice_updated_v3(forkchoice, None).await; + let response = client.fork_choice_updated_v3(sync_state.create_forkchoice_state(), None).await; let valid_response = response.map_err(|e| { // Fatal forkchoice update error. @@ -99,24 +136,13 @@ async fn send_forkchoice( })?; match &valid_response.payload_status.status { - PayloadStatusEnum::Valid => { - state.el_applied_last_forkchoice = true; - if !state.el_sync_finished { - info!(target: "engine", "Finished execution layer sync."); - state.el_sync_finished = true; - } - } + PayloadStatusEnum::Valid => Ok(ForkchoiceReply::Valid), // If we're not building a new payload, we're driving EL sync. PayloadStatusEnum::Syncing => { - state.el_applied_last_forkchoice = false; - debug!(target: "engine", "Attempting to update forkchoice state while EL syncing") + debug!(target: "engine", "Attempting to update forkchoice state while EL syncing"); + Ok(ForkchoiceReply::Syncing) } // Other codes are not expected. - status => return Err(SynchronizeTaskError::UnexpectedPayloadStatus(status.clone())), + status => Err(SynchronizeTaskError::UnexpectedPayloadStatus(status.clone())), } - - // Apply the new sync state to the engine state. - state.sync_state = new_sync_state; - - Ok(()) } diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs index 9de83da3a5c..f620d4c6d79 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/synchronize/mod.rs @@ -1,7 +1,10 @@ //! The forkchoice engine update shared by the engine tasks, and its error type. mod helper; -pub(in crate::task_queue) use helper::{synchronize, synchronize_unconditionally}; +pub(in crate::task_queue) use helper::{ + ForkchoiceReply, record_forkchoice_reply, send_forkchoice, synchronize, + synchronize_unconditionally, +}; mod error; pub use error::SynchronizeTaskError; diff --git a/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs b/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs index 7339a4c8102..b3a7f6a315d 100644 --- a/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs +++ b/rust/kona/crates/node/engine/src/task_queue/tasks/task.rs @@ -36,6 +36,11 @@ pub enum EngineTaskErrorSeverity { /// the engine keeps running. #[display("skip")] Skip, + /// The task changed nothing and cannot make progress yet, typically until the execution layer + /// syncs further. It is retried after a backoff, and higher-priority tasks queued meanwhile + /// run first. + #[display("wait")] + Wait, } /// The interface for an engine task error. diff --git a/rust/kona/crates/node/engine/src/test_utils/blocks.rs b/rust/kona/crates/node/engine/src/test_utils/blocks.rs index 83516a460e1..dfbc1936375 100644 --- a/rust/kona/crates/node/engine/src/test_utils/blocks.rs +++ b/rust/kona/crates/node/engine/src/test_utils/blocks.rs @@ -4,7 +4,7 @@ use super::{l1_info_deposit, l1_info_deposit_for}; use alloy_consensus::{Header, transaction::Recovered}; use alloy_primitives::{Address, B256, Bytes}; use alloy_rpc_types_eth::{BlockTransactions, Header as RpcHeader}; -use kona_protocol::L1BlockInfoBedrock; +use kona_protocol::{L1BlockInfoBedrock, OpAttributesWithParent}; use op_alloy_consensus::OpBlock; /// A full L2 block as the execution layer serves it over RPC. @@ -61,3 +61,20 @@ pub fn rpc_l2_block(header: Header, info: &L1BlockInfoBedrock) -> TestRpcBlock { }]); block } + +/// The block that `attributes` derive, as the execution layer serves it over RPC, so that +/// consolidating `attributes` against it finds a match. `attributes` must carry only the L1-info +/// deposit and a gas limit. +pub fn test_rpc_block_from_attributes(attributes: &OpAttributesWithParent) -> TestRpcBlock { + let payload_attributes = &attributes.attributes.payload_attributes; + let mut block = OpBlock::default(); + block.header.beneficiary = payload_attributes.suggested_fee_recipient; + block.header.gas_limit = attributes.attributes.gas_limit.expect("attributes set a gas limit"); + block.header.mix_hash = payload_attributes.prev_randao; + block.header.number = attributes.block_number(); + block.header.parent_beacon_block_root = payload_attributes.parent_beacon_block_root; + block.header.parent_hash = attributes.parent.block_info.hash; + block.header.timestamp = payload_attributes.timestamp; + block.body.transactions.push(l1_info_deposit()); + test_rpc_block(&block) +} diff --git a/rust/kona/crates/node/engine/src/test_utils/mod.rs b/rust/kona/crates/node/engine/src/test_utils/mod.rs index dd8c55020bd..ac61416649a 100644 --- a/rust/kona/crates/node/engine/src/test_utils/mod.rs +++ b/rust/kona/crates/node/engine/src/test_utils/mod.rs @@ -2,7 +2,9 @@ mod attributes; pub use attributes::TestAttributesBuilder; mod blocks; -pub use blocks::{TestRpcBlock, rpc_l2_block, test_l2_block, test_rpc_block}; +pub use blocks::{ + TestRpcBlock, rpc_l2_block, test_l2_block, test_rpc_block, test_rpc_block_from_attributes, +}; mod deposits; pub use deposits::{l1_info_deposit, l1_info_deposit_for}; diff --git a/rust/kona/crates/node/engine/src/test_utils/rpc.rs b/rust/kona/crates/node/engine/src/test_utils/rpc.rs index cb5aee9403d..cd585f6e681 100644 --- a/rust/kona/crates/node/engine/src/test_utils/rpc.rs +++ b/rust/kona/crates/node/engine/src/test_utils/rpc.rs @@ -93,6 +93,26 @@ impl RpcMock { ); } + /// Expects an `eth_syncing` call answered with no sync in progress, then an `eth_blockNumber` + /// call answered with `latest_number`. + pub fn expect_el_not_syncing_at(&self, latest_number: u64) { + self.expect("eth_syncing", false); + self.expect("eth_blockNumber", alloy_primitives::U64::from(latest_number)); + } + + /// Expects an `eth_syncing` call answered with a sync in progress whose current block is + /// `current_number`. + pub fn expect_el_syncing_at(&self, current_number: u64) { + self.expect( + "eth_syncing", + serde_json::json!({ + "currentBlock": format!("{current_number:#x}"), + "highestBlock": "0xffff", + "startingBlock": "0x0", + }), + ); + } + /// Expects an `engine_forkchoiceUpdatedV3` call for `forkchoice` without payload attributes and /// answers it with `status`. pub fn expect_forkchoice_updated( diff --git a/rust/kona/crates/node/service/src/actors/engine/actor.rs b/rust/kona/crates/node/service/src/actors/engine/actor.rs index b45bfa62238..32251cc5c99 100644 --- a/rust/kona/crates/node/service/src/actors/engine/actor.rs +++ b/rust/kona/crates/node/service/src/actors/engine/actor.rs @@ -4,9 +4,9 @@ use crate::{ use async_trait::async_trait; use kona_derive::{ResetSignal, Signal}; use kona_engine::{ - BuildTask, CanonicalizeTask, ConsolidateTask, Engine, EngineActorRequest, EngineClient, - EngineTask, EngineTaskError, EngineTaskErrorSeverity, EngineTaskErrors, FinalizeTask, - ImportedBlockSink, InsertTask, SealTask, + BuildTask, CanonicalizeTask, ConsolidateTask, DEFAULT_EL_SYNC_STALL_TIMEOUT, DrainScope, + Engine, EngineActorRequest, EngineClient, EngineTask, EngineTaskError, EngineTaskErrorSeverity, + EngineTaskErrors, FinalizeTask, ImportedBlockSink, InsertTask, SealTask, }; use kona_genesis::RollupConfig; use kona_protocol::L2BlockInfo; @@ -20,6 +20,12 @@ use tokio::{ #[path = "queue_tests.rs"] mod queue_tests; +/// How long a task waiting for the execution layer to sync waits between retries. +/// +/// The wait is not a failure, so it does not grow the failure backoff, and each retry costs a few +/// RPCs on an execution layer that is busy syncing. +const WAIT_RETRY_INTERVAL: Duration = Duration::from_secs(2); + /// Responsible for managing the operations sent to the execution layer's Engine API. To accomplish /// this, it uses the [`Engine`] task queue to order Engine API interactions based off of /// the [`Ord`] implementation of [`EngineTask`]. @@ -32,6 +38,8 @@ where derivation_client: DerivationClient, /// Whether the EL sync is complete. This should only ever go from false to true. el_sync_complete: bool, + /// How long a consolidation waits for an execution layer whose sync reports no progress. + el_sync_stall_timeout: Duration, /// The last local-safe head sent to the derivation actor. last_engine_safe_head_sent: L2BlockInfo, /// The error of the last skipped engine task, so an identical repeat logs no new warning @@ -43,17 +51,18 @@ where /// mode. unsafe_head_tx: Option>, - /// The [`RollupConfig`] used to build tasks. - rollup: Arc, + /// Where to hand every imported block, so the derivation providers can read it locally + /// instead of fetching it back from the execution layer. + block_sink: Arc, /// An [`EngineClient`] used for creating engine tasks. client: Arc, /// The [`Engine`] task queue. engine: Engine, /// The inbound request channel. inbound_request_rx: mpsc::Receiver, - /// Where to hand every imported block, so the derivation providers can read it locally - /// instead of fetching it back from the execution layer. - block_sink: Arc, + /// The [`RollupConfig`] used to build tasks. + rollup: Arc, + /// A reset is retained until successful; RPC failure must not lose the request. reset_pending: bool, /// Callers waiting for the retained reset to finish. @@ -62,6 +71,9 @@ where retry_at: Instant, /// Exponentially increasing retry delay, capped to keep recovery responsive. retry_delay: Duration, + /// Earliest retry of the task parked while it waits for the execution layer. Kept apart from + /// `retry_at`, so a failure of a task that runs ahead of it cannot bring its retry forward. + waiting_retry_at: Instant, } impl EngineActor @@ -79,23 +91,32 @@ where block_sink: Arc, ) -> Self { Self { + block_sink, client, derivation_client, el_sync_complete: false, + el_sync_stall_timeout: DEFAULT_EL_SYNC_STALL_TIMEOUT, engine, + inbound_request_rx, last_engine_safe_head_sent: L2BlockInfo::default(), last_skipped_task_error: None, - rollup: config, - unsafe_head_tx, - inbound_request_rx, - block_sink, reset_pending: false, reset_waiters: Vec::new(), retry_at: Instant::now(), retry_delay: Duration::from_millis(100), + rollup: config, + unsafe_head_tx, + waiting_retry_at: Instant::now(), } } + /// Bounds how long a consolidation waits for an execution layer whose sync reports no + /// progress. + pub const fn with_el_sync_stall_timeout(mut self, el_sync_stall_timeout: Duration) -> Self { + self.el_sync_stall_timeout = el_sync_stall_timeout; + self + } + /// Logs a skipped task at `warn!` when its error differs from the last skip, and at `debug!` /// while the identical error repeats, so a target resubmitted on every poll warns once. fn log_skipped_task(&mut self, err: &EngineTaskErrors) { @@ -113,6 +134,30 @@ where self.retry_delay = (self.retry_delay * 2).min(Duration::from_secs(5)); } + /// Schedules the retry of a task waiting for the execution layer, leaving the failure backoff + /// as it is. + fn defer_waiting_retry(&mut self) { + self.waiting_retry_at = Instant::now() + WAIT_RETRY_INTERVAL; + } + + /// Which tasks a drain may run now: the waiting task only once its retry is due. + fn drain_scope(&self, now: Instant) -> DrainScope { + if now >= self.waiting_retry_at { + DrainScope::AllTasks + } else { + DrainScope::AheadOfWaitingTask + } + } + + /// When the actor next has work to retry without a new request, if any: the failure backoff + /// while a reset or a runnable task is pending, and otherwise the waiting task's retry. + fn next_retry_at(&self) -> Option { + if self.reset_pending || self.engine.has_work_ahead_of_waiting_task() { + return Some(self.retry_at); + } + self.engine.has_waiting_task().then(|| self.retry_at.max(self.waiting_retry_at)) + } + fn begin_reset(&mut self) { if !self.reset_pending { self.engine.clear(); @@ -188,10 +233,10 @@ where } /// Drains the inner [`Engine`] task queue and attempts to update the safe head. - async fn drain(&mut self) -> Result<(), EngineError> { - match self.engine.drain().await { + async fn drain(&mut self, scope: DrainScope) -> Result<(), EngineError> { + match self.engine.drain(scope).await { Ok(_) => { - trace!(target: "engine", "[ENGINE] tasks drained"); + trace!(target: "engine", ?scope, "[ENGINE] tasks drained"); self.retry_delay = Duration::from_millis(100); } Err(err) => { @@ -228,6 +273,14 @@ where return Ok(()); } EngineTaskErrorSeverity::Skip => self.log_skipped_task(&err), + // A waiting task changed nothing, but tasks that ran before it in this drain + // did, so their safe head and sync completion still reach derivation below. + // They all succeeded, so the failure backoff starts over too. + EngineTaskErrorSeverity::Wait => { + trace!(target: "engine", %err, "Engine task waits before its retry"); + self.retry_delay = Duration::from_millis(100); + self.defer_waiting_retry(); + } } } } @@ -267,7 +320,7 @@ where async fn send_derivation_actor_safe_head_if_updated(&mut self) -> Result<(), EngineError> { let local_safe_head = self.engine.state().sync_state.local_safe_head(); if local_safe_head == self.last_engine_safe_head_sent { - info!(target: "engine", safe_head = ?local_safe_head, "Local-safe head unchanged"); + debug!(target: "engine", safe_head = ?local_safe_head, "Local-safe head unchanged"); return Ok(()); } @@ -291,11 +344,14 @@ where type Error = EngineError; async fn step(&mut self) -> Result<(), Self::Error> { - if Instant::now() >= self.retry_at { + let now = Instant::now(); + if now >= self.retry_at { if self.reset_pending { self.attempt_reset().await?; } else { - self.drain().await?; + // Work that outranks a task waiting for the execution layer to sync, such as an + // unsafe import, runs as it arrives; the waiting task runs once its retry is due. + self.drain(self.drain_scope(now)).await?; } } @@ -305,13 +361,14 @@ where // Wait for the next processing request. // Retry without needing a new inbound message, while still accepting requests during // backoff. Bound queued work so a prolonged outage cannot grow memory without limit. + let next_retry_at = self.next_retry_at(); let request = tokio::select! { request = self.inbound_request_rx.recv(), if self.engine.len() < 1024 && self.reset_waiters.len() < 1024 => { request.ok_or(EngineError::ChannelClosed)? } - _ = time::sleep_until(self.retry_at), - if self.reset_pending || !self.engine.is_empty() => return Ok(()), + _ = time::sleep_until(next_retry_at.unwrap_or(self.retry_at)), + if next_retry_at.is_some() => return Ok(()), }; match request { @@ -331,6 +388,7 @@ where self.rollup.clone(), safe_signal, Arc::clone(&self.block_sink), + self.el_sync_stall_timeout, ))); self.engine.enqueue(task); } diff --git a/rust/kona/crates/node/service/src/actors/engine/config.rs b/rust/kona/crates/node/service/src/actors/engine/config.rs index 25865cc4e6d..a2abd77fff1 100644 --- a/rust/kona/crates/node/service/src/actors/engine/config.rs +++ b/rust/kona/crates/node/service/src/actors/engine/config.rs @@ -4,13 +4,15 @@ use kona_engine::EngineClient; use kona_genesis::RollupConfig; use kona_providers_alloy::AlloyChainProvider; use op_alloy_network::Optimism; -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; /// Configuration for the Engine Actor. #[derive(Debug, Clone)] pub struct EngineConfig { /// The [`RollupConfig`]. pub config: Arc, + /// How long a consolidation waits for an execution layer whose sync reports no progress. + pub el_sync_stall_timeout: Duration, /// The mode of operation for the node. /// When the node is in sequencer mode, the engine actor will receive requests to build blocks diff --git a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs index c7399efc6e6..872f862d83b 100644 --- a/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs +++ b/rust/kona/crates/node/service/src/actors/engine/queue_tests.rs @@ -1,5 +1,6 @@ use super::*; -use crate::QueuedEngineDerivationClient; +use crate::{DerivationActorRequest, QueuedEngineDerivationClient}; +use alloy_eips::Encodable2718; use alloy_primitives::B256; use alloy_rpc_types_engine::{ ExecutionPayloadV1, ForkchoiceState, PayloadStatus, PayloadStatusEnum, @@ -8,8 +9,9 @@ use kona_engine::{ ConsolidateInput, EngineState, EngineSyncStateUpdate, FinalizeBlockId, NoopBlockSink, ResetRequest, test_utils::{ - RpcMock, TestEngineStateBuilder, TestRpcBlock, expect_sync_start, test_block_info, - test_engine_client, test_l2_block, test_rpc_block, + RpcMock, TestAttributesBuilder, TestEngineStateBuilder, TestRpcBlock, expect_sync_start, + l1_info_deposit, test_block_info, test_engine_client, test_l2_block, test_rpc_block, + test_rpc_block_from_attributes, }, }; use op_alloy_consensus::{OpBlock, OpTxEnvelope}; @@ -199,6 +201,395 @@ async fn delegated_poll_during_el_sync_moves_safe_labels_and_skips_finalization( l2.assert_finished(); } +/// After an execution layer datadir wipe, or while the execution layer commits blocks in bursts, +/// derived attributes can name a block below the unsafe head that the execution layer does not +/// have yet. While it answers `SYNCING`, the consolidation waits without a reset, which would +/// clear the derivation actor's finalization queue and re-derive from the execution layer's labels. +/// Unsafe imports received meanwhile run as they arrive rather than at the waiting task's retry, +/// the wait leaves the failure backoff untouched, and the retry consolidates the block once the +/// execution layer has it. +#[tokio::test(start_paused = true)] +async fn derived_block_missing_during_el_sync_waits_without_reset() { + let config = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(config.clone()); + let unsafe_head = test_block_info(20); + let finalized_head = test_block_info(5); + let mut attributes = TestAttributesBuilder::new() + .with_parent(test_block_info(9)) + .with_transactions(vec![l1_info_deposit().encoded_2718().into()]) + .build(); + attributes.is_last_in_span = true; + let safe_head = attributes.parent; + let derived_block = test_rpc_block_from_attributes(&attributes); + let derived_safe_head = L2BlockInfo::from_block_and_genesis( + &derived_block.clone().into_consensus().map_transactions(|t| t.inner.inner.into_inner()), + &config.genesis, + ) + .unwrap(); + let gossiped_payload = + ExecutionPayloadV1::from_block_slow(&test_l2_block(21, unsafe_head.hash(), 0)); + let gossiped_head = L2BlockInfo::from_block_and_genesis( + &OpExecutionPayloadEnvelope::V1(gossiped_payload.clone()) + .try_into_block::() + .unwrap(), + &config.genesis, + ) + .unwrap(); + let state = TestEngineStateBuilder::new() + .with_unsafe_head(unsafe_head) + .with_safe_head(safe_head) + .with_finalized_head(finalized_head) + .build(); + let (state_tx, state_rx) = watch::channel(state); + let (requests, request_rx) = mpsc::channel(8); + let (derivation_tx, mut derivation_rx) = mpsc::channel(8); + let mut actor = EngineActor::new( + Arc::new(client), + config, + QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_tx }, + Engine::new(state, state_tx), + None, + request_rx, + Arc::new(NoopBlockSink), + ); + + // The execution layer has neither the derived block nor the unsafe head, and answers the + // unchanged forkchoice with `SYNCING`. An unsafe block arrives while the consolidation waits. + l2.expect_block_by_number(10, None); + l2.expect_el_not_syncing_at(0); + l2.expect_forkchoice_updated( + forkchoice(unsafe_head, safe_head, finalized_head), + PayloadStatusEnum::Syncing, + ); + requests.send(EngineActorRequest::ProcessSafeL2Signal(attributes.into())).await.unwrap(); + actor.step().await.unwrap(); + requests + .send(EngineActorRequest::ProcessUnsafeL2Block(Box::new(OpExecutionPayloadEnvelope::V1( + gossiped_payload.clone(), + )))) + .await + .unwrap(); + actor.step().await.unwrap(); + assert_eq!(actor.engine.len(), 2, "the consolidation waits behind the unsafe import"); + assert!(!actor.reset_pending); + assert_eq!(actor.retry_delay, Duration::from_millis(100), "waiting is not a failure"); + + // The unsafe import runs at the next step, well before the waiting consolidation's retry + // elapses; the step then idles toward that retry. + l2.expect_params( + "engine_newPayloadV2", + serde_json::json!([gossiped_payload]), + PayloadStatus::from_status(PayloadStatusEnum::Valid), + ); + l2.expect_forkchoice_updated( + forkchoice(gossiped_head, safe_head, finalized_head), + PayloadStatusEnum::Valid, + ); + assert!(time::timeout(Duration::from_millis(50), actor.step()).await.is_err()); + assert_eq!(state_rx.borrow().sync_state.unsafe_head(), gossiped_head); + assert_eq!(actor.engine.len(), 1, "only the waiting consolidation may remain"); + + // The next step only idles until the waiting consolidation's retry is due. + assert!(actor.engine.has_waiting_task()); + time::timeout(WAIT_RETRY_INTERVAL, actor.step()).await.unwrap().unwrap(); + assert!(actor.engine.has_waiting_task(), "the consolidation is not retried before it is due"); + + // The step after it retries the consolidation, which finds the derived block; with nothing left + // to do, the actor then waits for requests instead of retrying. + l2.expect_block_by_number(10, Some(derived_block)); + l2.expect_forkchoice_updated( + forkchoice(gossiped_head, derived_safe_head, finalized_head), + PayloadStatusEnum::Valid, + ); + assert!(time::timeout(Duration::from_secs(60), actor.step()).await.is_err()); + + assert!(actor.engine.is_empty()); + assert!(!actor.reset_pending); + assert_eq!(state_rx.borrow().sync_state.safe_head(), derived_safe_head); + assert_eq!(state_rx.borrow().sync_state.unsafe_head(), gossiped_head); + let mut derivation_requests = Vec::new(); + while let Ok(request) = derivation_rx.try_recv() { + derivation_requests.push(request); + } + assert!( + !derivation_requests.iter().any(|request| matches!( + request, + DerivationActorRequest::ProcessEngineSignalRequest(_) + )), + "no reset may reach derivation: {derivation_requests:?}" + ); + assert!(matches!( + derivation_requests.last(), + Some(DerivationActorRequest::ProcessEngineSafeHeadUpdateRequest(head)) if **head == derived_safe_head + )); + l1.assert_finished(); + l2.assert_finished(); +} + +/// A failure of an unsafe import that runs ahead of a consolidation waiting for the execution +/// layer retries the import on the failure backoff, but leaves the consolidation parked until its +/// own retry is due. +#[tokio::test(start_paused = true)] +async fn failing_import_ahead_of_a_waiting_consolidation_keeps_its_retry() { + let config = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(config.clone()); + let unsafe_head = test_block_info(20); + let finalized_head = test_block_info(5); + let mut attributes = TestAttributesBuilder::new() + .with_parent(test_block_info(9)) + .with_transactions(vec![l1_info_deposit().encoded_2718().into()]) + .build(); + attributes.is_last_in_span = true; + let safe_head = attributes.parent; + let gossiped_payload = + ExecutionPayloadV1::from_block_slow(&test_l2_block(21, unsafe_head.hash(), 0)); + let gossiped_head = L2BlockInfo::from_block_and_genesis( + &OpExecutionPayloadEnvelope::V1(gossiped_payload.clone()) + .try_into_block::() + .unwrap(), + &config.genesis, + ) + .unwrap(); + let state = TestEngineStateBuilder::new() + .with_unsafe_head(unsafe_head) + .with_safe_head(safe_head) + .with_finalized_head(finalized_head) + .build(); + let (state_tx, state_rx) = watch::channel(state); + let (requests, request_rx) = mpsc::channel(8); + let (derivation_tx, _derivation_rx) = mpsc::channel(8); + let mut actor = EngineActor::new( + Arc::new(client), + config, + QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_tx }, + Engine::new(state, state_tx), + None, + request_rx, + Arc::new(NoopBlockSink), + ); + + // The consolidation finds no derived block and parks while the execution layer syncs. + l2.expect_block_by_number(10, None); + l2.expect_el_not_syncing_at(0); + l2.expect_forkchoice_updated( + forkchoice(unsafe_head, safe_head, finalized_head), + PayloadStatusEnum::Syncing, + ); + requests.send(EngineActorRequest::ProcessSafeL2Signal(attributes.into())).await.unwrap(); + actor.step().await.unwrap(); + l2.expect_error("engine_newPayloadV2"); + requests + .send(EngineActorRequest::ProcessUnsafeL2Block(Box::new(OpExecutionPayloadEnvelope::V1( + gossiped_payload.clone(), + )))) + .await + .unwrap(); + actor.step().await.unwrap(); + let parked_at = Instant::now(); + assert!(actor.engine.has_waiting_task()); + + // The unsafe import runs ahead of it and fails once, then succeeds on its backoff, both well + // before the consolidation's retry; the step then idles toward that retry. + actor.step().await.unwrap(); + l2.expect_params( + "engine_newPayloadV2", + serde_json::json!([gossiped_payload]), + PayloadStatus::from_status(PayloadStatusEnum::Valid), + ); + l2.expect_forkchoice_updated( + forkchoice(gossiped_head, safe_head, finalized_head), + PayloadStatusEnum::Valid, + ); + assert!(time::timeout(WAIT_RETRY_INTERVAL / 2, actor.step()).await.is_err()); + + assert_eq!(state_rx.borrow().sync_state.unsafe_head(), gossiped_head); + assert!(actor.engine.has_waiting_task(), "the consolidation stays parked"); + assert!( + Instant::now() < parked_at + WAIT_RETRY_INTERVAL, + "the import ran before the consolidation's retry was due" + ); + assert_eq!(actor.engine.len(), 1, "only the waiting consolidation remains"); + l1.assert_finished(); + l2.assert_finished(); +} + +/// A drain in which an unsafe import completes the execution layer's sync and a consolidation then +/// waits still reports the import to derivation: the waiting task changed nothing, but the import +/// moved the engine's labels and finished its sync. +#[tokio::test(start_paused = true)] +async fn sync_completed_before_a_waiting_consolidation_reaches_derivation() { + let config = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(config.clone()); + let client = Arc::new(client); + let unsafe_head = test_block_info(20); + let finalized_head = test_block_info(5); + let mut attributes = TestAttributesBuilder::new() + .with_parent(test_block_info(9)) + .with_transactions(vec![l1_info_deposit().encoded_2718().into()]) + .build(); + attributes.is_last_in_span = true; + let safe_head = attributes.parent; + let gossiped_payload = + ExecutionPayloadV1::from_block_slow(&test_l2_block(21, unsafe_head.hash(), 0)); + let gossiped_head = L2BlockInfo::from_block_and_genesis( + &OpExecutionPayloadEnvelope::V1(gossiped_payload.clone()) + .try_into_block::() + .unwrap(), + &config.genesis, + ) + .unwrap(); + let state = TestEngineStateBuilder::new() + .with_unsafe_head(unsafe_head) + .with_safe_head(safe_head) + .with_finalized_head(finalized_head) + .with_el_sync_finished(false) + .build(); + let (state_tx, _state_rx) = watch::channel(state); + let mut engine = Engine::new(state, state_tx); + engine.enqueue(EngineTask::Consolidate(Box::new(ConsolidateTask::new( + client.clone(), + config.clone(), + attributes.into(), + Arc::new(NoopBlockSink), + DEFAULT_EL_SYNC_STALL_TIMEOUT, + )))); + engine.enqueue(EngineTask::Insert(Box::new(InsertTask::new( + client.clone(), + config.clone(), + OpExecutionPayloadEnvelope::V1(gossiped_payload.clone()), + false, + Arc::new(NoopBlockSink), + )))); + let (_requests, request_rx) = mpsc::channel(8); + let (derivation_tx, mut derivation_rx) = mpsc::channel(8); + let mut actor = EngineActor::new( + client, + config, + QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_tx }, + engine, + None, + request_rx, + Arc::new(NoopBlockSink), + ); + + // The import's forkchoice update is answered `VALID`, which finishes the sync; the + // consolidation then finds no derived block and waits. + l2.expect_params( + "engine_newPayloadV2", + serde_json::json!([gossiped_payload]), + PayloadStatus::from_status(PayloadStatusEnum::Valid), + ); + l2.expect_forkchoice_updated( + forkchoice(gossiped_head, safe_head, finalized_head), + PayloadStatusEnum::Valid, + ); + l2.expect_block_by_number(10, None); + l2.expect_el_not_syncing_at(21); + l2.expect_forkchoice_updated( + forkchoice(gossiped_head, safe_head, finalized_head), + PayloadStatusEnum::Syncing, + ); + actor.step().await.unwrap(); + + assert!(actor.el_sync_complete); + assert_eq!(actor.engine.len(), 1, "only the waiting consolidation remains"); + let mut derivation_requests = Vec::new(); + while let Ok(request) = derivation_rx.try_recv() { + derivation_requests.push(request); + } + assert!( + derivation_requests.iter().any(|request| matches!( + request, + DerivationActorRequest::ProcessEngineSyncCompletionRequest(_) + )), + "the sync completion must reach derivation: {derivation_requests:?}" + ); + l1.assert_finished(); + l2.assert_finished(); +} + +/// A drain in which an unsafe import succeeds and a consolidation then waits restarts the failure +/// backoff: every task ahead of the waiting one succeeded, so a later unrelated failure is retried +/// on the shortest delay rather than on one grown by earlier failures. +#[tokio::test(start_paused = true)] +async fn waiting_after_successful_tasks_restarts_the_failure_backoff() { + let config = Arc::new(RollupConfig::default()); + let (client, l1, l2) = test_engine_client(config.clone()); + let client = Arc::new(client); + let unsafe_head = test_block_info(20); + let finalized_head = test_block_info(5); + let mut attributes = TestAttributesBuilder::new() + .with_parent(test_block_info(9)) + .with_transactions(vec![l1_info_deposit().encoded_2718().into()]) + .build(); + attributes.is_last_in_span = true; + let safe_head = attributes.parent; + let gossiped_payload = + ExecutionPayloadV1::from_block_slow(&test_l2_block(21, unsafe_head.hash(), 0)); + let gossiped_head = L2BlockInfo::from_block_and_genesis( + &OpExecutionPayloadEnvelope::V1(gossiped_payload.clone()) + .try_into_block::() + .unwrap(), + &config.genesis, + ) + .unwrap(); + let state = TestEngineStateBuilder::new() + .with_unsafe_head(unsafe_head) + .with_safe_head(safe_head) + .with_finalized_head(finalized_head) + .build(); + let (state_tx, _state_rx) = watch::channel(state); + let mut engine = Engine::new(state, state_tx); + engine.enqueue(EngineTask::Consolidate(Box::new(ConsolidateTask::new( + client.clone(), + config.clone(), + attributes.into(), + Arc::new(NoopBlockSink), + DEFAULT_EL_SYNC_STALL_TIMEOUT, + )))); + engine.enqueue(EngineTask::Insert(Box::new(InsertTask::new( + client.clone(), + config.clone(), + OpExecutionPayloadEnvelope::V1(gossiped_payload.clone()), + false, + Arc::new(NoopBlockSink), + )))); + let (_requests, request_rx) = mpsc::channel(8); + let (derivation_tx, _derivation_rx) = mpsc::channel(8); + let mut actor = EngineActor::new( + client, + config, + QueuedEngineDerivationClient { derivation_actor_request_tx: derivation_tx }, + engine, + None, + request_rx, + Arc::new(NoopBlockSink), + ); + actor.retry_delay = Duration::from_secs(5); + + l2.expect_params( + "engine_newPayloadV2", + serde_json::json!([gossiped_payload]), + PayloadStatus::from_status(PayloadStatusEnum::Valid), + ); + l2.expect_forkchoice_updated( + forkchoice(gossiped_head, safe_head, finalized_head), + PayloadStatusEnum::Valid, + ); + l2.expect_block_by_number(10, None); + l2.expect_el_not_syncing_at(21); + l2.expect_forkchoice_updated( + forkchoice(gossiped_head, safe_head, finalized_head), + PayloadStatusEnum::Syncing, + ); + actor.step().await.unwrap(); + + assert_eq!(actor.engine.len(), 1, "only the waiting consolidation remains"); + assert_eq!(actor.retry_delay, Duration::from_millis(100)); + l1.assert_finished(); + l2.assert_finished(); +} + fn forkchoice(head: L2BlockInfo, safe: L2BlockInfo, finalized: L2BlockInfo) -> ForkchoiceState { ForkchoiceState { finalized_block_hash: finalized.hash(), diff --git a/rust/kona/crates/node/service/src/service/node/rollup.rs b/rust/kona/crates/node/service/src/service/node/rollup.rs index 10e926c33ed..702b6339dd6 100644 --- a/rust/kona/crates/node/service/src/service/node/rollup.rs +++ b/rust/kona/crates/node/service/src/service/node/rollup.rs @@ -259,7 +259,8 @@ impl RollupNode { unsafe_head_tx_opt, engine_request_rx, Arc::new(BufferImportedBlocks::new(self.l2_block_buffer.clone())), - ); + ) + .with_el_sync_stall_timeout(self.engine_config.el_sync_stall_timeout); (actor, engine_client.query_client(), engine_state_rx) }