net: fix four ways IBD can stall

changelog.d/ibd-stall-recovery.md

@@ -0,0 +1,6 @@
+Fixed
+
+- **Header-cache cleanup keeps the height and parent of stored path headers still waiting for the IBD download queue.** The queue can refill from them.
+- **A new lookahead header batch does not jump ahead of stored headers waiting for queue space.** Those slots stay for the next contiguous headers.
+- **Only an empty reconnect round with no live IBD peer spends the reconnect budget.** A replacement connection that already started is allowed to finish. A spent budget still stops IBD without entering tip mode.
+- **A follow-session peer that goes over the message or byte rate budget is disconnected on that frame.** Its unanswered block asks are released however the session ends, including `disconnectnode` and eviction, so another peer can be asked right away. A second rate-limit disconnect from the same address within an hour refuses that address for a day. Noban, manual, and local peers leave no strike.

crates/rbitcoin-net/src/ibd/events/mod.rs

@@ -213,6 +213,9 @@ fn try_enqueue_ordered_header(
     if !st.is_on_path(&hash, ht) || st.ordered.len() >= MAX_ORDERED_HEADERS {
         return false;
     }
+    if super::path::next_path_height(st, hub).is_some_and(|next| ht > next) {
+        return false;
+    }
     // The queue is a window over headers already written. Stop at the soft cap.
     if st.header_walk.has_checkpoints() && st.ordered.len() >= ORDERED_HEADERS_SOFT_CAP {
         return false;

crates/rbitcoin-net/src/ibd/exit.rs

@@ -6,7 +6,7 @@
 use super::state::IbdWorkState;
 
 /// Max empty redial rounds while fully dark before giving up mid catch-up.
-pub const MAX_DARK_EMPTY_REDIALS: u32 = 4;
+const MAX_DARK_EMPTY_REDIALS: u32 = 4;
 
 /// Highest connected best-chain height we already occupy (not competing `hash_height`).
 pub(crate) fn path_high_water(st: &IbdWorkState, tip_h: u32) -> u32 {
@@ -173,17 +173,32 @@ pub enum AllPeersDead {
     GiveUpMidCatchup,
 }
 
+/// Empty redial rounds in a row with no live peer. A round that adds a peer,
+/// or an empty top-up while a peer is still up, starts the count over.
+pub(crate) fn dark_empty_after(dark: u32, added: usize, any_alive: bool) -> u32 {
+    if added > 0 || any_alive {
+        0
+    } else {
+        dark.saturating_add(1)
+    }
+}
+
+/// A live peer can always be topped up. With none, dialing stops after
+/// [`MAX_DARK_EMPTY_REDIALS`] dark empty rounds.
+pub(crate) fn may_spawn_redial(alive_n: usize, dark: u32) -> bool {
+    alive_n > 0 || dark < MAX_DARK_EMPTY_REDIALS
+}
+
 /// Decide what to do when `slots` has no live peers.
 pub fn all_peers_dead_action(
     st: &IbdWorkState,
     tip_h: u32,
     redial_in_flight: bool,
-    dark_redial_empty: u32,
 ) -> AllPeersDead {
     if ibd_caught_up(st, tip_h) {
         return AllPeersDead::CatchupComplete;
     }
-    if !redial_in_flight || dark_redial_empty >= MAX_DARK_EMPTY_REDIALS {
+    if !redial_in_flight {
         return AllPeersDead::GiveUpMidCatchup;
     }
     AllPeersDead::WaitRedial
@@ -193,6 +208,39 @@ pub fn all_peers_dead_action(
 mod tests {
     use super::*;
 
+    #[test]
+    fn redial_budget_spends_only_dark_empty_rounds_and_waits_out_a_started_dial() {
+        assert_eq!(
+            dark_empty_after(MAX_DARK_EMPTY_REDIALS - 1, 0, true),
+            0,
+            "an empty top-up while a peer is up does not spend the dark budget"
+        );
+        assert_eq!(dark_empty_after(MAX_DARK_EMPTY_REDIALS, 1, false), 0);
+
+        let mut dark = 0;
+        for _ in 0..MAX_DARK_EMPTY_REDIALS {
+            assert!(may_spawn_redial(0, dark));
+            dark = dark_empty_after(dark, 0, false);
+        }
+        assert!(
+            !may_spawn_redial(0, dark),
+            "dark empty rounds stop the next dial"
+        );
+        assert!(may_spawn_redial(1, dark), "a live peer still tops up");
+
+        let mut st = IbdWorkState::new(Vec::new(), None, Some(21_792));
+        st.max_peer_height = 326_676;
+        assert_eq!(
+            all_peers_dead_action(&st, 21_792, true),
+            AllPeersDead::WaitRedial,
+            "a dial already started is waited out"
+        );
+        assert_eq!(
+            all_peers_dead_action(&st, 21_792, false),
+            AllPeersDead::GiveUpMidCatchup
+        );
+    }
+
     /// all_peers_dead + catchup_complete edge matrix (mid-chain / caught-up / tip-0).
     #[allow(clippy::cognitive_complexity)] // one fixture, many exit arms
     #[test]
@@ -202,11 +250,11 @@ mod tests {
         mid.max_peer_height = 958_820;
         mid.max_ready_height = 161_000;
         assert_eq!(
-            all_peers_dead_action(&mid, 161_249, false, 0),
+            all_peers_dead_action(&mid, 161_249, false),
             AllPeersDead::GiveUpMidCatchup
         );
         assert_eq!(
-            all_peers_dead_action(&mid, 161_249, true, 0),
+            all_peers_dead_action(&mid, 161_249, true),
             AllPeersDead::WaitRedial
         );
 
@@ -216,7 +264,7 @@ mod tests {
         done.max_ready_height = 100;
         done.headers_done = true;
         assert_eq!(
-            all_peers_dead_action(&done, 100, false, 0),
+            all_peers_dead_action(&done, 100, false),
             AllPeersDead::CatchupComplete
         );
 
@@ -239,13 +287,7 @@ mod tests {
         assert!(!ibd_caught_up(&zero, 0));
         assert!(!ibd_caught_up(&zero, 0));
         assert_eq!(
-            all_peers_dead_action(&zero, 0, false, 0),
-            AllPeersDead::GiveUpMidCatchup
-        );
-
-        // Dark redial budget exhausted while redial still marked in-flight.
-        assert_eq!(
-            all_peers_dead_action(&mid, 161_249, true, MAX_DARK_EMPTY_REDIALS),
+            all_peers_dead_action(&zero, 0, false),
             AllPeersDead::GiveUpMidCatchup
         );
 
@@ -487,7 +529,7 @@ mod tests {
         zero.max_ready_height = 0;
         assert!(!ibd_caught_up(&zero, 0));
         assert_eq!(
-            all_peers_dead_action(&zero, 0, false, 0),
+            all_peers_dead_action(&zero, 0, false),
             AllPeersDead::GiveUpMidCatchup
         );
 
@@ -497,7 +539,7 @@ mod tests {
         done.max_ready_height = 100;
         done.headers_done = true;
         assert_eq!(
-            all_peers_dead_action(&done, 100, false, 0),
+            all_peers_dead_action(&done, 100, false),
             AllPeersDead::CatchupComplete
         );
 

crates/rbitcoin-net/src/ibd/header_walk.rs

@@ -5752,6 +5752,66 @@ mod tests {
         }
     }
 
+    #[test]
+    fn new_lookahead_batch_does_not_skip_stored_queue_successors() {
+        let (_dir, mut hub) = crate::chain::tiny_regtest_hub_labeled("header-walk-no-queue-gap");
+        hub.ensure_genesis().unwrap();
+        let genesis = hub.tip_hash().unwrap();
+        floor_unreachable(&mut hub);
+        let (peer, _rx) = slot(0);
+        let mut st = IbdWorkState::new(vec![peer], Some(genesis), Some(0));
+        let tail = mine(genesis, 1);
+        queue_at_cap_ending_on(&mut st, &hub, tail);
+        assert!(send_getheaders(&mut st, &hub).unwrap());
+        let next = mine(tail.block_hash(), 2);
+        let third = mine(next.block_hash(), 3);
+        apply(&mut st, &hub, 0, vec![next, third]);
+        for _ in 0..2 {
+            let dropped = st.ordered.pop_front().unwrap();
+            st.ordered_set.remove(&dropped);
+        }
+        let fourth = mine(third.block_hash(), 4);
+        apply(&mut st, &hub, 0, vec![fourth]);
+        super::super::path::append_stored_headers(&mut st, &hub);
+        let queued: Vec<_> = st.ordered.iter().rev().take(2).copied().collect();
+        assert_eq!(
+            queued,
+            vec![third.block_hash(), next.block_hash()],
+            "new lookahead must leave the free queue slots for already stored successors"
+        );
+        assert_eq!(st.header_walk.tip_hash(), Some(fourth.block_hash()));
+    }
+
+    #[test]
+    fn drained_queue_refills_stored_headers_after_known_header_pruning() {
+        let (_dir, mut hub) = crate::chain::tiny_regtest_hub_labeled("header-walk-pruned-refill");
+        hub.ensure_genesis().unwrap();
+        let genesis = hub.tip_hash().unwrap();
+        floor_unreachable(&mut hub);
+        let (peer, _rx) = slot(0);
+        let mut st = IbdWorkState::new(vec![peer], Some(genesis), Some(0));
+        let tail = mine(genesis, 1);
+        queue_at_cap_ending_on(&mut st, &hub, tail);
+        st.known_headers.extend(st.ordered.iter().copied());
+        assert!(send_getheaders(&mut st, &hub).unwrap());
+        let next = mine(tail.block_hash(), 2);
+        let last = mine(next.block_hash(), 3);
+        apply(&mut st, &hub, 0, vec![next, last]);
+        assert_eq!(st.header_walk.tip_hash(), Some(last.block_hash()));
+        while st.ordered.len() > 1 {
+            let dropped = st.ordered.pop_front().unwrap();
+            st.ordered_set.remove(&dropped);
+        }
+        assert_eq!(st.ordered.back(), Some(&tail.block_hash()));
+        st.hygiene();
+        assert_eq!(
+            super::super::path::append_stored_headers(&mut st, &hub),
+            2,
+            "both stored successors must remain available to the download queue after pruning"
+        );
+        assert_eq!(path_top(&st, &hub), Some((last.block_hash(), 3)));
+    }
+
     #[test]
     fn a_heavier_fork_below_the_floor_is_written_when_the_queue_has_room() {
         let (_dir, mut hub) = crate::chain::tiny_regtest_hub_labeled("header-walk-stray-write");

crates/rbitcoin-net/src/ibd/mod.rs

@@ -57,8 +57,9 @@ use events::{apply_confirm_events, apply_peer_event, disconnect_all_peers, updat
 #[cfg(test)]
 pub(crate) use events::{apply_confirm_reject, apply_headers_event};
 use exit::{
-    all_peers_dead_action, best_chain_remainder, empty_path_header_fan, header_lag_behind_peers,
-    ibd_caught_up, path_drained, should_unlatch_headers_done, AllPeersDead,
+    all_peers_dead_action, best_chain_remainder, dark_empty_after, empty_path_header_fan,
+    header_lag_behind_peers, ibd_caught_up, may_spawn_redial, path_drained,
+    should_unlatch_headers_done, AllPeersDead,
 };
 #[cfg(test)]
 pub(crate) use header_walk::{
@@ -785,18 +786,18 @@ pub async fn ibd_cancellable(
                         st.slots.push(s);
                         n += 1;
                     }
+                    dark_redial_empty =
+                        dark_empty_after(dark_redial_empty, n, st.slots.iter().any(|s| s.alive));
                     if n > 0 {
                         st.slots.sort_by_key(|s| s.id);
                         info!(
                             "ibd: redial added {n} peer(s); live={}",
                             st.slots.iter().filter(|s| s.alive).count()
                         );
-                        dark_redial_empty = 0;
                         if st.ordered.is_empty() && !st.headers_done {
                             enqueue_header_cmd(st, HeaderCmd::Ask);
                         }
                     } else {
-                        dark_redial_empty = dark_redial_empty.saturating_add(1);
                         warn!("ibd: redial returned 0 peers (empty_rounds={dark_redial_empty})");
                     }
                 }
@@ -818,6 +819,7 @@ pub async fn ibd_cancellable(
             };
             if redial_handle.is_none()
                 && alive_n < target
+                && may_spawn_redial(alive_n, dark_redial_empty)
                 && !peer_sess.book().is_empty()
                 && last_redial.elapsed() >= redial_interval
             {
@@ -980,7 +982,7 @@ pub async fn ibd_cancellable(
             // All peers dead — never treat mid-chain peer death as catch-up complete.
             if st.slots.iter().all(|s| !s.alive) {
                 let tip_h = hub.tip_height().unwrap_or(0);
-                match all_peers_dead_action(st, tip_h, redial_handle.is_some(), dark_redial_empty) {
+                match all_peers_dead_action(st, tip_h, redial_handle.is_some()) {
                     AllPeersDead::CatchupComplete => {
                         info!(
                         "ibd: catch-up complete (no live peers) tip={tip_h} max_peer_height={} max_ready={} — exiting IBD",

crates/rbitcoin-net/src/ibd/state.rs

@@ -437,8 +437,14 @@ impl IbdWorkState {
                     || (*ht > queue_tail && self.header_walk.indexes_through(*ht)))
         });
         if self.known_headers.len() > live.len().saturating_add(4096) {
-            self.known_headers
-                .retain(|h| live.contains(h) || inflight.contains_key(h));
+            self.known_headers.retain(|h| {
+                live.contains(h)
+                    || inflight.contains_key(h)
+                    || self
+                        .hash_height
+                        .get(h)
+                        .is_some_and(|ht| self.height_to_hash.get(ht) == Some(h))
+            });
             self.header_fks.retain(|h, _| {
                 live.contains(h) || inflight.contains_key(h) || self.known_headers.contains(h)
             });