Repository navigation
kona-node: back off repeated pipeline reset failures of any kind #23901
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -57,42 +57,41 @@ where | |
|
|
||
| /// A pipeline reset that has not yet succeeded. | ||
| #[derive(Debug)] | ||
| enum PendingReset { | ||
| /// The reset head was reorged out of the L2 chain, so the engine was asked for a fresh one. | ||
| AwaitingFreshHead, | ||
| /// The reset signal failed temporarily and is retried with backoff. | ||
| Retry(ScheduledRetry), | ||
| } | ||
|
|
||
| /// A reset signal retried after a temporary failure. | ||
| #[derive(Debug)] | ||
| struct ScheduledRetry { | ||
| struct PendingReset { | ||
| /// Consecutive failed attempts since the last successful reset, whichever kind each was. | ||
| attempts: u32, | ||
| retry_at: Instant, | ||
| signal: ResetSignal, | ||
| next_step: NextResetStep, | ||
| } | ||
|
|
||
| impl ScheduledRetry { | ||
| /// op-node retries a temporarily failed reset through its exponential step backoff, capped | ||
| /// at ten seconds. | ||
| const MAX_RETRY_DELAY: Duration = Duration::from_secs(10); | ||
| const MIN_RETRY_DELAY: Duration = Duration::from_millis(100); | ||
|
|
||
| fn first(signal: ResetSignal) -> Self { | ||
| Self::with_attempts(signal, 1) | ||
| } | ||
| /// What brings a pending reset closer to succeeding. | ||
| #[derive(Clone, Copy, Debug)] | ||
| enum NextResetStep { | ||
| /// The engine was asked for a fresh reset head and its reset signal is awaited. | ||
| AwaitFreshHead, | ||
| /// The reset head was reorged out of the L2 chain, so a fresh one is requested at `at`. | ||
| RequestFreshHead { at: Instant }, | ||
| /// The reset signal failed temporarily and is signaled again at `at`. | ||
| RetrySignal { at: Instant, signal: ResetSignal }, | ||
| } | ||
|
|
||
| fn retried(&self) -> Self { | ||
| Self::with_attempts(self.signal, self.attempts + 1) | ||
| } | ||
| impl PendingReset { | ||
| /// op-node retries a failed reset, temporary or not, through its exponential step backoff, | ||
| /// capped at ten seconds. | ||
| const MAX_DELAY: Duration = Duration::from_secs(10); | ||
| const MIN_DELAY: Duration = Duration::from_millis(100); | ||
|
|
||
| fn with_attempts(signal: ResetSignal, attempts: u32) -> Self { | ||
| Self { attempts, retry_at: Instant::now() + Self::retry_delay(attempts), signal } | ||
| fn delay(attempts: u32) -> Duration { | ||
| let doublings = attempts.saturating_sub(1).min(16); | ||
| Self::MIN_DELAY.saturating_mul(1 << doublings).min(Self::MAX_DELAY) | ||
| } | ||
|
|
||
| fn retry_delay(attempts: u32) -> Duration { | ||
| let doublings = attempts.saturating_sub(1).min(16); | ||
| Self::MIN_RETRY_DELAY.saturating_mul(1 << doublings).min(Self::MAX_RETRY_DELAY) | ||
| const fn due_at(&self) -> Option<Instant> { | ||
| match self.next_step { | ||
| NextResetStep::AwaitFreshHead => None, | ||
| NextResetStep::RequestFreshHead { at } | NextResetStep::RetrySignal { at, .. } => { | ||
| Some(at) | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
|
|
@@ -129,6 +128,10 @@ where | |
| /// derivation state intact, so it is logged and the actor continues. A reset is handled by | ||
| /// [`Self::reset_pipeline_and_resume`] instead. | ||
| pub(super) async fn forward_non_reset_signal(&mut self, signal: Signal) { | ||
| debug_assert!( | ||
| !matches!(signal, Signal::Reset(_)), | ||
| "a reset must go through reset_pipeline_and_resume" | ||
| ); | ||
| match self.pipeline.signal(signal).await { | ||
| Ok(()) => info!(target: "derivation", ?signal, "[SIGNAL] Executed Successfully"), | ||
| Err(e) => { | ||
|
|
@@ -144,8 +147,8 @@ where | |
| /// pipeline reset has succeeded; [`Self::attempt_derivation`] holds while a reset is pending, | ||
| /// whichever actor requested it. Like op-node, a temporary pipeline failure, such as an L2 | ||
| /// RPC error while the pipeline walks back to its L1 origin, schedules a retry with backoff, | ||
| /// and a reset head that was reorged out asks the engine for a fresh one. Only a critical | ||
| /// failure stops the actor. | ||
| /// and a reset head that was reorged out asks the engine for a fresh one after that same | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nit on the op-node comparison. op-node's Both limit how fast the loop can run, so the behaviour is fine. The visible difference is that kona leaves the engine forkchoice un-reset for up to 10 s, where op-node resets it immediately. Could the wording here, and in the PR body ("routes Generated by Claude Code |
||
| /// backoff. Only a critical failure stops the actor. | ||
| pub(super) async fn reset_pipeline_and_resume( | ||
| &mut self, | ||
| reset: ResetSignal, | ||
|
|
@@ -161,7 +164,8 @@ where | |
| return Ok(()); | ||
| } | ||
| Err(PipelineErrorKind::Reset(error)) => { | ||
| return self.request_fresh_reset_head(reset, error).await; | ||
| self.schedule_fresh_reset_head_request(reset, error); | ||
| return Ok(()); | ||
| } | ||
| Err(error @ PipelineErrorKind::Critical(_)) => { | ||
| error!(target: "derivation", ?error, ?reset, "Failed to reset derivation pipeline"); | ||
|
|
@@ -180,54 +184,69 @@ where | |
| self.attempt_derivation().await | ||
| } | ||
|
|
||
| /// Schedules `reset` to be signaled again after a backoff that grows with each failed | ||
| /// attempt of that same signal. | ||
| /// Schedules `reset` to be signaled again after the backoff for one more failed attempt. | ||
| fn schedule_reset_retry(&mut self, reset: ResetSignal, error: PipelineError) { | ||
| let retry = match self.pending_reset.take() { | ||
| Some(PendingReset::Retry(retry)) if retry.signal == reset => retry.retried(), | ||
| _ => ScheduledRetry::first(reset), | ||
| }; | ||
| let attempts = self.failed_reset_attempts() + 1; | ||
| let delay = PendingReset::delay(attempts); | ||
| warn!( | ||
| target: "derivation", | ||
| ?error, | ||
| ?reset, | ||
| attempt = retry.attempts, | ||
| retry_delay = ?ScheduledRetry::retry_delay(retry.attempts), | ||
| attempt = attempts, | ||
| retry_delay = ?delay, | ||
| "Pipeline reset failed temporarily; scheduling a retry" | ||
| ); | ||
| metrics::gauge!(Metrics::DERIVATION_PENDING_RESET_ATTEMPTS).set(f64::from(retry.attempts)); | ||
| self.pending_reset = Some(PendingReset::Retry(retry)); | ||
| let next_step = NextResetStep::RetrySignal { at: Instant::now() + delay, signal: reset }; | ||
| self.set_pending_reset(attempts, next_step); | ||
| } | ||
|
|
||
| /// Asks the engine for a fresh reset head. The pipeline could not walk back from the head in | ||
| /// `reset`, so signaling it again cannot succeed; like op-node, that failure starts a new | ||
| /// reset rather than a retry. | ||
| async fn request_fresh_reset_head( | ||
| &mut self, | ||
| reset: ResetSignal, | ||
| error: ResetError, | ||
| ) -> Result<(), DerivationError> { | ||
| /// Schedules a request for a fresh reset head after the backoff for one more failed | ||
| /// attempt. The pipeline could not walk back from the head in `reset`, so signaling it | ||
| /// again cannot succeed; like op-node, that failure starts a new reset rather than a retry. | ||
| fn schedule_fresh_reset_head_request(&mut self, reset: ResetSignal, error: ResetError) { | ||
| let attempts = self.failed_reset_attempts() + 1; | ||
| let delay = PendingReset::delay(attempts); | ||
| warn!( | ||
| target: "derivation", | ||
| ?error, | ||
| ?reset, | ||
| "Pipeline reset head is no longer canonical; requesting a fresh engine reset" | ||
| attempt = attempts, | ||
| request_delay = ?delay, | ||
| "Pipeline reset head is no longer canonical; scheduling a fresh engine reset" | ||
| ); | ||
| self.pending_reset = Some(PendingReset::AwaitingFreshHead); | ||
| metrics::gauge!(Metrics::DERIVATION_PENDING_RESET_ATTEMPTS).set(0.0); | ||
| self.engine_client.reset_engine_forkchoice().await.map_err(|e| { | ||
| error!(target: "derivation", ?e, "Failed to send reset request"); | ||
| DerivationError::Sender(Box::new(e)) | ||
| }) | ||
| let next_step = NextResetStep::RequestFreshHead { at: Instant::now() + delay }; | ||
| self.set_pending_reset(attempts, next_step); | ||
| } | ||
|
|
||
| fn failed_reset_attempts(&self) -> u32 { | ||
| self.pending_reset.as_ref().map_or(0, |pending| pending.attempts) | ||
| } | ||
|
|
||
| async fn retry_pending_reset(&mut self) -> Result<(), DerivationError> { | ||
| let Some(PendingReset::Retry(retry)) = &self.pending_reset else { | ||
| fn set_pending_reset(&mut self, attempts: u32, next_step: NextResetStep) { | ||
| metrics::gauge!(Metrics::DERIVATION_PENDING_RESET_ATTEMPTS).set(f64::from(attempts)); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This gauge fix is the change dashboards depend on, but no test reads Generated by Claude Code |
||
| self.pending_reset = Some(PendingReset { attempts, next_step }); | ||
| } | ||
|
|
||
| async fn run_due_reset_step(&mut self) -> Result<(), DerivationError> { | ||
| let Some(pending) = &self.pending_reset else { | ||
| return Ok(()); | ||
| }; | ||
| let reset = retry.signal; | ||
| info!(target: "derivation", ?reset, attempt = retry.attempts + 1, "Retrying pipeline reset"); | ||
| self.reset_pipeline_and_resume(reset).await | ||
| let failed_attempts = pending.attempts; | ||
| match pending.next_step { | ||
| NextResetStep::AwaitFreshHead => Ok(()), | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nit: this arm can't be reached, and neither can the Generated by Claude Code |
||
| NextResetStep::RetrySignal { signal: reset, .. } => { | ||
| info!(target: "derivation", ?reset, attempt = failed_attempts + 1, "Retrying pipeline reset"); | ||
| self.reset_pipeline_and_resume(reset).await | ||
| } | ||
| NextResetStep::RequestFreshHead { .. } => { | ||
| info!(target: "derivation", failed_attempts, "Requesting a fresh engine reset"); | ||
| self.set_pending_reset(failed_attempts, NextResetStep::AwaitFreshHead); | ||
| self.engine_client.reset_engine_forkchoice().await.map_err(|e| { | ||
| error!(target: "derivation", ?e, "Failed to send reset request"); | ||
| DerivationError::Sender(Box::new(e)) | ||
| }) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /// Refreshes an established origin without republishing one invalidated by a reset. | ||
|
|
@@ -471,7 +490,7 @@ where | |
| if let Some(pending_reset) = &self.pending_reset { | ||
| // SafeDB is already truncated to the reset head, so the pipeline must reset before | ||
| // it is stepped. Wait for that reset the same way a drained data source is awaited. | ||
| info!(target: "derivation", ?pending_reset, "Holding derivation until the pipeline reset succeeds."); | ||
| info!(target: "derivation", failed_attempts = pending_reset.attempts, "Holding derivation until the pipeline reset succeeds."); | ||
| self.derivation_state_machine.update(&DerivationStateUpdate::MoreDataNeeded)?; | ||
| return Ok(()); | ||
| } | ||
|
|
@@ -524,18 +543,15 @@ where | |
| type Error = DerivationError; | ||
|
|
||
| async fn step(&mut self) -> Result<(), Self::Error> { | ||
| let retry_at = match &self.pending_reset { | ||
| Some(PendingReset::Retry(retry)) => Some(retry.retry_at), | ||
| Some(PendingReset::AwaitingFreshHead) | None => None, | ||
| }; | ||
| let due_at = self.pending_reset.as_ref().and_then(PendingReset::due_at); | ||
| let pending_reset_due = async move { | ||
| match retry_at { | ||
| Some(retry_at) => tokio::time::sleep_until(retry_at).await, | ||
| match due_at { | ||
| Some(due_at) => tokio::time::sleep_until(due_at).await, | ||
| None => std::future::pending().await, | ||
| } | ||
| }; | ||
| tokio::select! { | ||
| // Inbound requests win, so a busy queue can hold a due retry past `retry_at`. | ||
| // Inbound requests win, so a busy queue can hold a due reset step past `due_at`. | ||
| // Requests are infrequent, so that delay is accepted. | ||
| biased; | ||
| request = self.inbound_request_rx.recv() => { | ||
|
|
@@ -548,7 +564,7 @@ where | |
| })?; | ||
| self.handle_derivation_actor_request(request).await | ||
| } | ||
| () = pending_reset_due => self.retry_pending_reset().await, | ||
| () = pending_reset_due => self.run_due_reset_step().await, | ||
| } | ||
| } | ||
| } | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -4,6 +4,8 @@ | |
| //! error into `ResetEvent`, which starts a new engine reset. Only `ErrCritical` stops the node. | ||
| use super::*; | ||
| use std::time::Duration; | ||
| use tokio::time::Instant; | ||
|
|
||
| fn engine_reset(head: u64) -> DerivationActorRequest { | ||
| DerivationActorRequest::ProcessEngineSignalRequest(Box::new(Signal::Reset( | ||
|
|
@@ -19,6 +21,18 @@ fn el_sync_completed(head: u64) -> DerivationActorRequest { | |
| DerivationActorRequest::ProcessEngineSyncCompletionRequest(Box::new(l2_block(head))) | ||
| } | ||
|
|
||
| /// The error the pipeline's walk-back reports when the reset head's parent was reorged out. | ||
| fn reorged_parent_error() -> PipelineErrorKind { | ||
| ResetError::BlockNotFound(l2_block(RESUME_HEAD).block_info.parent_hash.into()).reset() | ||
| } | ||
|
|
||
| /// Runs the scheduled reset step and returns how long the actor waited before it. | ||
| async fn step_when_due(node: &mut RecordingActor) -> Duration { | ||
| let started = Instant::now(); | ||
| node.actor.step().await.expect("the scheduled reset step must succeed"); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Suppose the fresh-head request went back to being sent immediately. Two orders and limits aren't covered:
Generated by Claude Code |
||
| started.elapsed() | ||
| } | ||
|
|
||
| /// A reset whose pipeline signal fails once with a temporary provider error neither stops the | ||
| /// actor nor lets derivation resume, whichever actor requested the reset. A request that would | ||
| /// otherwise start deriving holds until the scheduled retry adopts the reset head. | ||
|
|
@@ -72,7 +86,7 @@ async fn temporary_reset_failure_holds_derivation_until_the_retry_like_op_node( | |
| /// A reset head the pipeline can no longer walk back from, because the L2 chain reorged under | ||
| /// it, asks the engine for a fresh head rather than stopping the actor. Derivation holds until | ||
| /// the fresh reset succeeds. | ||
| #[tokio::test] | ||
| #[tokio::test(start_paused = true)] | ||
| async fn reorged_reset_head_requests_a_fresh_engine_reset_like_op_node() { | ||
| let mut engine_client = MockDerivationEngineClient::new(); | ||
| engine_client.expect_reset_engine_forkchoice().times(1).returning(|| Ok(())); | ||
|
|
@@ -82,9 +96,7 @@ async fn reorged_reset_head_requests_a_fresh_engine_reset_like_op_node() { | |
| node.actor.pipeline.attributes.clear(); | ||
| node.complete_el_sync_at(DERIVED_HEAD).await; | ||
| node.actor.pipeline.attributes.push_back(derived_attributes()); | ||
| let reorged_parent = l2_block(RESUME_HEAD).block_info.parent_hash; | ||
| node.actor.pipeline.signal_error = | ||
| Some(ResetError::BlockNotFound(reorged_parent.into()).reset()); | ||
| node.actor.pipeline.signal_error = Some(reorged_parent_error()); | ||
|
|
||
| node.handle(engine_reset(RESUME_HEAD)).await; | ||
| node.handle(l1_head_update(DERIVED_FROM_L1)).await; | ||
|
|
@@ -96,7 +108,69 @@ async fn reorged_reset_head_requests_a_fresh_engine_reset_like_op_node() { | |
| assert_eq!(node.actor.pipeline.attributes.len(), 1, "the pipeline must not be stepped"); | ||
| assert!(node.actor.attributes_awaiting_confirmation.is_none()); | ||
|
|
||
| // No request is queued, so the next step is the scheduled fresh-head request. | ||
| step_when_due(&mut node).await; | ||
| node.handle(engine_reset(RESUME_HEAD - 1)).await; | ||
|
|
||
| assert_eq!( | ||
| node.actor.derivation_state_machine.last_confirmed_safe_head(), | ||
| l2_block(RESUME_HEAD - 1) | ||
| ); | ||
| assert!(node.actor.attributes_awaiting_confirmation.is_some()); | ||
| } | ||
|
|
||
| /// Consecutive reset failures share one backoff schedule, so an EL that keeps answering "not | ||
| /// found" for a canonical block, as a lagging replica does, cannot drive a hot loop of engine | ||
| /// resets and L2 walk-backs. op-node routes `ResetEvent` through the same step backoff as a | ||
| /// temporary error. | ||
| #[tokio::test(start_paused = true)] | ||
| async fn repeated_reorged_reset_heads_back_off_like_op_node() { | ||
| let mut engine_client = MockDerivationEngineClient::new(); | ||
| engine_client.expect_reset_engine_forkchoice().times(2).returning(|| Ok(())); | ||
| engine_client.expect_send_safe_l2_signal().times(1).returning(|_| Ok(())); | ||
| engine_client.expect_send_finalized_l2_block().times(0); | ||
| let mut node = RecordingActor::new(engine_client); | ||
| node.actor.pipeline.attributes.clear(); | ||
| node.complete_el_sync_at(DERIVED_HEAD).await; | ||
| node.actor.pipeline.attributes.push_back(derived_attributes()); | ||
|
|
||
| node.actor.pipeline.signal_error = Some(reorged_parent_error()); | ||
| node.handle(engine_reset(RESUME_HEAD)).await; | ||
| assert_eq!(step_when_due(&mut node).await, Duration::from_millis(100)); | ||
|
|
||
| node.actor.pipeline.signal_error = Some(reorged_parent_error()); | ||
| node.handle(engine_reset(RESUME_HEAD - 1)).await; | ||
| assert_eq!(step_when_due(&mut node).await, Duration::from_millis(200)); | ||
|
|
||
| node.handle(engine_reset(RESUME_HEAD - 2)).await; | ||
|
|
||
| assert_eq!( | ||
| node.actor.derivation_state_machine.last_confirmed_safe_head(), | ||
| l2_block(RESUME_HEAD - 2) | ||
| ); | ||
| assert!(node.actor.attributes_awaiting_confirmation.is_some()); | ||
| } | ||
|
|
||
| /// The failure count carries across a fresh-head request, so a temporary failure of the fresh | ||
| /// reset waits for the next backoff rather than starting the schedule over. | ||
| #[tokio::test(start_paused = true)] | ||
| async fn a_temporary_failure_after_a_fresh_head_continues_the_backoff() { | ||
| let mut engine_client = MockDerivationEngineClient::new(); | ||
| engine_client.expect_reset_engine_forkchoice().times(1).returning(|| Ok(())); | ||
| engine_client.expect_send_safe_l2_signal().times(1).returning(|_| Ok(())); | ||
| engine_client.expect_send_finalized_l2_block().times(0); | ||
| let mut node = RecordingActor::new(engine_client); | ||
| node.actor.pipeline.attributes.clear(); | ||
| node.complete_el_sync_at(DERIVED_HEAD).await; | ||
| node.actor.pipeline.attributes.push_back(derived_attributes()); | ||
|
|
||
| node.actor.pipeline.signal_error = Some(reorged_parent_error()); | ||
| node.handle(engine_reset(RESUME_HEAD)).await; | ||
| assert_eq!(step_when_due(&mut node).await, Duration::from_millis(100)); | ||
|
|
||
| node.actor.pipeline.signal_error = Some(l2_provider_error().temp()); | ||
| node.handle(engine_reset(RESUME_HEAD - 1)).await; | ||
| assert_eq!(step_when_due(&mut node).await, Duration::from_millis(200)); | ||
|
|
||
| assert_eq!( | ||
| node.actor.derivation_state_machine.last_confirmed_safe_head(), | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This schedule isn't op-node's, although this comment and the PR body say it is. op-node's step backoff is
retry.Exponential()(op-service/retry/strategies.go:49):Min(0) + 2^attempt × 1s, capped at 10 s, plus 0–250 ms of jitter.AttemptStepraisesstepAttemptsto 1 before the first failure, so op-node waits about 2 s, 4 s, 8 s, then 10 s. This code waits 100 ms, 200 ms, 400 ms and so on, without jitter:The 100 ms floor came from #23815, but this PR now applies it to fresh-head requests too. Could we either match op-node (2 s base plus jitter) or reword this comment to claim only the 10 s cap?
Generated by Claude Code