From 9d5f6b1320d62f8f8a0204c5aa495b2dba7ce81e Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 07:23:23 -0700 Subject: [PATCH 1/4] test: empty header queue must ask from the stored tip An empty ordered queue during IBD was asked from the walk checkpoint, so the reply never landed on the queue. A connecting peer also got no getheaders while the node was in IBD, and a headers_done latch stayed shut when a peer was more than two blocks ahead. --- crates/rbitcoin-net/src/ibd/exit.rs | 9 +++- crates/rbitcoin-net/src/ibd/header_walk.rs | 52 +++++++++++++--------- crates/rbitcoin-net/src/peer_tests.rs | 36 ++++++++++++--- 3 files changed, 70 insertions(+), 27 deletions(-) diff --git a/crates/rbitcoin-net/src/ibd/exit.rs b/crates/rbitcoin-net/src/ibd/exit.rs index a8b1d4684..4242417a2 100644 --- a/crates/rbitcoin-net/src/ibd/exit.rs +++ b/crates/rbitcoin-net/src/ibd/exit.rs @@ -503,6 +503,7 @@ mod tests { // Empty-EOF at the most-work path: extra advertised height is not a // tip we chase (less-work high-height fork / bogus LastBlock). signet.headers_done = true; + signet.header_walk.prove_through(2000); assert!( ibd_caught_up(&signet, 2000), "headers_done at path HWM completes even if start_height is far ahead" @@ -557,10 +558,16 @@ mod tests { .insert(BlockHash::from_byte_array([i; 32]), InflightReq::new(0)); } assert_eq!(header_lag_behind_peers(&st, 100), 5); + assert!( + super::should_unlatch_headers_done(&st, 100), + "an unproved latch asks again when a peer is more than two blocks ahead" + ); + st.header_walk.prove_through(100); assert!( !super::should_unlatch_headers_done(&st, 100), - "lag>2 with no connecting hole is not a most-work tip we unlatch for" + "a proved walk does not chase a far start height" ); + st.header_walk = super::super::header_walk::HeaderWalk::default(); st.max_peer_height = 100; assert_eq!(header_lag_behind_peers(&st, 100), 0); diff --git a/crates/rbitcoin-net/src/ibd/header_walk.rs b/crates/rbitcoin-net/src/ibd/header_walk.rs index c72288d24..68ef84ee3 100644 --- a/crates/rbitcoin-net/src/ibd/header_walk.rs +++ b/crates/rbitcoin-net/src/ibd/header_walk.rs @@ -200,6 +200,14 @@ impl HeaderWalk { pub(crate) fn indexes_through(&self, height: u32) -> bool { self.origin && self.tip.hash.is_some() && height > 0 && height <= self.tip.height } + + /// Test stand-in for a walk that already proved the confirmed tip. + #[cfg(test)] + pub(crate) fn prove_through(&mut self, height: u32) { + self.origin = true; + self.proven = true; + self.tip.height = height; + } } pub(crate) struct OriginInput { @@ -3523,8 +3531,8 @@ mod tests { }; assert_eq!( front, - Some(second.block_hash()), - "a missing next header is asked from the walk tip, not the rewound queue" + Some(gen), + "a missing next header is asked from the stored tip, not the walk" ); let other = if ask == 0 { rx1.try_recv() @@ -3628,21 +3636,22 @@ mod tests { } } assert!( - fronts.contains(&second.block_hash()), - "the ask starts at the walk tip; the queue refill is the reseeded path: {fronts:?}" + fronts.contains(&stored), + "the ask starts at the stored tip so the missing header can be queued: {fronts:?}" ); } #[test] fn missing_next_header_asks_from_the_stored_tip() { - let (_dir, hub, mut st, mut rxs, first, _tail) = + let (_dir, hub, mut st, mut rxs, first, tail) = walk_ahead_of_a_short_queue("header-walk-missing", 20_000, 1); st.height_to_hash.remove(&1); + assert_ne!(tail, first.block_hash()); assert!(send_getheaders(&mut st, &hub).unwrap()); assert_eq!( locator_front(&mut rxs[0]), - Some(first.block_hash()), - "a missing next header is asked from the walk tip, not the queue tail" + Some(tail), + "a missing next header is asked from the stored tip, not the walk" ); } @@ -3919,36 +3928,39 @@ mod tests { hub.ensure_genesis().unwrap(); let gen = hub.tip_hash().unwrap(); let good = mine(gen, 1); + let later = mine(good.block_hash(), 2); let miss = mine(gen, 9); let (s0, mut rx) = slot(0); let mut st = IbdWorkState::new(vec![s0], Some(gen), Some(0)); let before = hub.query.store().header_count(); assert!(send_getheaders(&mut st, &hub).unwrap()); let _ = rx.try_recv(); - apply(&mut st, &hub, 0, vec![good]); - assert_eq!(hub.query.store().header_count(), before + 1); - assert_eq!(st.header_walk.tip_hash(), Some(good.block_hash())); + apply(&mut st, &hub, 0, vec![good, later]); + assert_eq!(hub.query.store().header_count(), before + 2); + assert_eq!(st.header_walk.tip_hash(), Some(later.block_hash())); - while st.ordered.len() >= ORDERED_HEADERS_SOFT_CAP { - let dropped = st.ordered.pop_front().unwrap(); - st.ordered_set.remove(&dropped); - } + st.ordered.clear(); + st.ordered_set.clear(); + st.height_to_hash.remove(&1); + st.height_to_hash.remove(&2); assert!(send_getheaders(&mut st, &hub).unwrap()); match rx.try_recv() { Ok(PeerCmd::GetHeaders { locator }) => { assert_eq!( locator[0], - good.block_hash(), - "look-ahead keeps the walk tip while peers are still ahead" + gen, + "an empty queue asks for the headers after the stored tip" ); } Ok(_) => panic!("expected getheaders"), Err(_) => panic!("peer was not asked for headers"), } - apply(&mut st, &hub, 0, vec![good]); - assert_eq!(hub.query.store().header_count(), before + 1); - assert!(st.ordered_set.contains(&good.block_hash())); + assert!( + st.ordered_set.contains(&good.block_hash()), + "a header short of the next checkpoint still fills the queue" + ); + assert_eq!(hub.query.store().header_count(), before + 2); let after = hub.query.store().header_count(); apply(&mut st, &hub, 0, vec![miss]); @@ -3958,7 +3970,7 @@ mod tests { "a lighter valid chain stays available for a later heavier extension" ); assert!(st.slots[0].alive); - assert_eq!(st.header_walk.tip_hash(), Some(good.block_hash())); + assert_eq!(st.header_walk.tip_hash(), Some(later.block_hash())); assert!(hub .query .get_header_by_hash(miss.block_hash().as_byte_array()) diff --git a/crates/rbitcoin-net/src/peer_tests.rs b/crates/rbitcoin-net/src/peer_tests.rs index ed290c9f5..0a002c4c9 100644 --- a/crates/rbitcoin-net/src/peer_tests.rs +++ b/crates/rbitcoin-net/src/peer_tests.rs @@ -175,13 +175,34 @@ fn ibd_connect_does_not_start_headers_sync() { let session = peers.register(addr, addr, &ver, true, crate::peers::PeerConnType::Inbound); let (tx, mut rx) = mpsc::unbounded_channel(); assert!( - !maybe_queue_initial_getheaders(&tx, &hub, &session), - "IBD does not start a follow-session headers sync" + maybe_queue_initial_getheaders(&tx, &hub, &session), + "IBD still asks the one headers peer for the stored tip" ); + match rx.try_recv().unwrap().expect_msg() { + bitcoin::p2p::message::NetworkMessage::GetHeaders(gh) => { + assert!( + gh.locator_hashes.contains(&hub.tip_hash().unwrap()), + "the locator names the stored tip" + ); + } + other => panic!("expected getheaders, got {other:?}"), + } + let addr2 = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8334); + let mut ver2 = ver.clone(); + ver2.nonce = 2; + let session2 = peers.register( + addr2, + addr2, + &ver2, + true, + crate::peers::PeerConnType::Inbound, + ); + let (tx2, mut rx2) = mpsc::unbounded_channel(); assert!( - rx.try_recv().is_err(), - "a connecting peer during IBD sends no getheaders" + !maybe_queue_initial_getheaders(&tx2, &hub, &session2), + "a second connecting peer does not start another headers sync" ); + assert!(rx2.try_recv().is_err()); hub.set_max_tip_age_secs(u64::MAX); assert!(!hub.in_ibd(), "a fresh tip leaves IBD"); @@ -241,8 +262,11 @@ fn ibd_unknown_block_inv_does_not_getheaders() { let unknown = BlockHash::from_byte_array([0x44; 32]); on_inv(&hub, &tx, &mut follow, None, &[Inventory::Block(unknown)]).unwrap(); assert!( - rx.try_recv().is_err(), - "an unknown block inv during IBD does not getheaders on this session" + matches!( + rx.try_recv().unwrap().expect_msg(), + bitcoin::p2p::message::NetworkMessage::GetHeaders(_) + ), + "an unknown block inv during IBD still asks for headers" ); assert!( hub.header_inv_challenge_pending(), From ea9f425645b460837d366fc2be0f6b6f0fe4a34d Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 07:35:28 -0700 Subject: [PATCH 2/4] ibd: refill an empty header queue from the stored tip An empty queue asked from the unrestored checkpoint, so the reply never landed on the download path. A headers_done latch with no proved walk also stayed shut when a peer was more than two blocks ahead, and a connecting peer sent no initial getheaders during IBD. Ask from the stored tip while the queue has room, write a short refill that continues that tip, reopen an unproved latch, and keep Core's one initial headers sync. --- changelog.d/one-headers-ask.md | 2 +- changelog.d/queue-refill.md | 3 ++ crates/rbitcoin-net/src/ibd/exit.rs | 16 ++++-- crates/rbitcoin-net/src/ibd/header_walk.rs | 62 ++++++++++++++++------ crates/rbitcoin-net/src/peer.rs | 11 ++-- crates/rbitcoin-net/src/peer_tests.rs | 34 +++++++++--- docs/ibd-memory.md | 2 +- 7 files changed, 96 insertions(+), 34 deletions(-) create mode 100644 changelog.d/queue-refill.md diff --git a/changelog.d/one-headers-ask.md b/changelog.d/one-headers-ask.md index 7c5f0c2cd..a0d9c5b2a 100644 --- a/changelog.d/one-headers-ask.md +++ b/changelog.d/one-headers-ask.md @@ -1,3 +1,3 @@ Changed -- **During IBD there is one headers question: the header after the candidate.** The locator also names a lighter fork still being weighed. A refused low-work batch is not paged, and a connecting peer does not start a second headers sync. An unknown block inv from outside the header slot does not retire the reserved peer. After IBD, connect-time headers sync is unchanged. +- **During IBD one peer is asked for the next header that is not stored.** While the queue has room the locator starts at the stored tip, then the candidate and a lighter fork. A full queue asks from the candidate. A refused low-work batch is not paged. Only one connecting peer starts headers sync. An unknown block inv from outside the header slot does not retire the reserved peer. After IBD, connect-time headers sync is unchanged. diff --git a/changelog.d/queue-refill.md b/changelog.d/queue-refill.md new file mode 100644 index 000000000..c6f07016b --- /dev/null +++ b/changelog.d/queue-refill.md @@ -0,0 +1,3 @@ +Fixed + +- **An empty header queue during IBD asks from the stored tip.** The reply is written onto the queue even when it stops short of the next checkpoint, so block download can resume when a restart kept the walk checkpoint and not the headers in between. A `headers_done` latch that never proved a walk opens again when a peer is ahead. A connecting peer still gets the one initial headers sync. diff --git a/crates/rbitcoin-net/src/ibd/exit.rs b/crates/rbitcoin-net/src/ibd/exit.rs index 4242417a2..c84ee4c7a 100644 --- a/crates/rbitcoin-net/src/ibd/exit.rs +++ b/crates/rbitcoin-net/src/ibd/exit.rs @@ -99,16 +99,21 @@ pub(crate) fn empty_path_header_fan(st: &IbdWorkState, tip_h: u32, alive: usize) /// Clear `headers_done` so empty-path `getheaders` can resume. /// -/// Latch stops the tip storm. Unlatch only for a 1–2 block connecting-header -/// hole. A far-ahead `max_peer_height` (less-work fork / bogus LastBlock) is -/// not a most-work tip — do not unlatch and chase it. +/// A proved walk at the confirmed tip does not chase a far +/// `version.start_height`. A latch that never proved a walk does: the +/// ordered queue is empty and a peer is ahead, including by more than two +/// blocks. That is the regtest case where the latch fired before any peer +/// had a chain. #[inline] pub(crate) fn should_unlatch_headers_done(st: &IbdWorkState, tip_h: u32) -> bool { if !st.headers_done || !st.ordered.is_empty() { return false; } let lag = header_lag_behind_peers(st, tip_h); - lag > 0 && lag <= 2 && !has_tip_plus_one(st, tip_h) + if lag == 0 || has_tip_plus_one(st, tip_h) { + return false; + } + !super::header_walk::headers_complete(st, tip_h) } /// Full `seed_work_path_from_store` (O(header_count) walk) while empty-lagging. @@ -542,7 +547,8 @@ mod tests { assert!(!super::should_unlatch_headers_done(&at, 100)); } - /// Unlatch only a 1–2 block connecting-header hole. lag>2 is not a chase. + /// An unproved empty queue unlatches for any real peer lead. A proved + /// walk does not chase a far start height. #[test] fn should_unlatch_headers_done() { use super::super::state::InflightReq; diff --git a/crates/rbitcoin-net/src/ibd/header_walk.rs b/crates/rbitcoin-net/src/ibd/header_walk.rs index 68ef84ee3..362de50cb 100644 --- a/crates/rbitcoin-net/src/ibd/header_walk.rs +++ b/crates/rbitcoin-net/src/ibd/header_walk.rs @@ -2,10 +2,11 @@ //! //! The ordered queue is a window over headers already written. One peer, the //! lowest time to a first block byte among peers that have not failed the -//! walk, is asked for the header after the candidate. The locator starts at -//! that tip, then a lighter fork still being weighed, then earlier -//! checkpoints. Headers already written are appended locally while the queue -//! is under [`ORDERED_HEADERS_SOFT_CAP`]. +//! walk, is asked for the next header that is not already stored. While the +//! queue has room the locator starts at the stored tip, then the candidate, +//! then a lighter fork, then earlier checkpoints, so the reply can be +//! enqueued. A full queue asks from the candidate. Headers already written +//! are appended locally while the queue is under [`ORDERED_HEADERS_SOFT_CAP`]. //! An accepted batch is written and recorded as a checkpoint. //! Above the work floor, one short reply that leaves the candidate //! unextended retires that peer and the reservation moves. A full @@ -1650,14 +1651,15 @@ pub(crate) fn reject_refill_miss( return false; }; // A proven walk is the most-work header chain so far. A refill that - // disagrees with it is not allowed to erase that proof. Headers that - // have not met the next checkpoint are not on that chain yet, so they - // are not written. Below the floor the candidate is still a guess, and + // disagrees with it is not allowed to erase that proof. A short refill + // that continues this tail is still the download path: refusing it + // leaves the block queue empty until a later batch happens to reach the + // next checkpoint. Below the floor the candidate is still a guess, and // a contradictory refill replaces it. if st.header_walk.proven && !below_floor(hub, &st.header_walk.tip_work()) { return match path_agree(st, tail_h, headers) { PathAgree::Diverges => offer_alternate(st, hub, None, headers).unwrap_or(true), - PathAgree::Short => true, + PathAgree::Short => false, PathAgree::Checkpoint(_) => false, }; } @@ -1753,8 +1755,15 @@ pub(crate) fn proven_header_prefix(st: &IbdWorkState, hub: &ChainHub, headers: & }; let stop = match path_agree(st, start_h, headers) { PathAgree::Checkpoint(stop) => stop, - // A queue refill that never meets a checkpoint is not an ancestor. - PathAgree::Short => return 0, + // Continues the stored tip and has not missed a checkpoint. The + // block queue needs these headers even when the next checkpoint is + // further than this reply. + PathAgree::Short => { + if stored_top_hash(st, hub) == Some(prev) { + return headers.len(); + } + return 0; + } // A fork that is not the queue-tail refill stays in the header store. // The proven tail path already consumed its own divergent refill. PathAgree::Diverges => return headers.len(), @@ -2137,15 +2146,37 @@ pub(crate) fn send_getheaders( arm_header_ask(st, hub) } +/// Locator for one headers ask. +/// +/// Bitcoin answers after the first locator hash the peer knows. While the +/// queue can take headers, that hash is the stored tip. The candidate and +/// its checkpoints follow, so a peer that does not have the stored tip can +/// still extend the walk. A full queue is look-ahead and starts at the +/// candidate. +fn header_ask_locator(st: &IbdWorkState, hub: &ChainHub) -> Vec { + let mut tips = Vec::new(); + if st.ordered.len() < ORDERED_HEADERS_SOFT_CAP { + if let Some(stored) = stored_top_hash(st, hub) { + tips.push(stored); + } + } + for hash in st.header_walk.locator_hashes() { + if !tips.contains(&hash) { + tips.push(hash); + } + } + if tips.is_empty() { + tips = super::path::work_path_tips(st); + } + tips +} + fn arm_header_ask(st: &mut IbdWorkState, hub: &ChainHub) -> Result { if lane_fresh(&st.header_walk.walk) { return Ok(true); } let skip = expire_header_ask(st); - let mut tips = st.header_walk.locator_hashes(); - if tips.is_empty() { - tips = super::path::work_path_tips(st); - } + let tips = header_ask_locator(st, hub); let above = path_top(st, hub).map(|(_, height)| height).unwrap_or(0); if let Some(peer) = take_challenge(st) { if ask_header_peer(st, hub, peer, &tips)? { @@ -3947,8 +3978,7 @@ mod tests { match rx.try_recv() { Ok(PeerCmd::GetHeaders { locator }) => { assert_eq!( - locator[0], - gen, + locator[0], gen, "an empty queue asks for the headers after the stored tip" ); } diff --git a/crates/rbitcoin-net/src/peer.rs b/crates/rbitcoin-net/src/peer.rs index ca500f9d4..185727226 100644 --- a/crates/rbitcoin-net/src/peer.rs +++ b/crates/rbitcoin-net/src/peer.rs @@ -2040,15 +2040,17 @@ pub(crate) fn should_poll_peer_headers(hub: &ChainHub, best_known: Option, hub: &ChainHub, session: &crate::peers::LivePeer, ) -> bool { - if hub.in_ibd() { - return false; - } if session.conn_type == crate::peers::PeerConnType::AddrFetch { return false; } @@ -3488,7 +3490,6 @@ fn on_inv_block_needs_headers( } if hub.in_ibd() { hub.note_header_inv_challenge(); - return false; } session.is_none_or(|s| { s.peer_hub() diff --git a/crates/rbitcoin-net/src/peer_tests.rs b/crates/rbitcoin-net/src/peer_tests.rs index 0a002c4c9..618cc5b24 100644 --- a/crates/rbitcoin-net/src/peer_tests.rs +++ b/crates/rbitcoin-net/src/peer_tests.rs @@ -204,14 +204,31 @@ fn ibd_connect_does_not_start_headers_sync() { ); assert!(rx2.try_recv().is_err()); - hub.set_max_tip_age_secs(u64::MAX); + // Leaving IBD is a tip inside 24h, which is also when Core allows every + // connecting peer to start headers sync. The peer that already started + // during IBD does not start again. + let tip_time = u64::from(hub.tip_header().unwrap().time); + let fresh = tip_time.saturating_add(60); + peers.set_mock_now(fresh); + hub.clock.set_mock(i64::try_from(fresh).unwrap()); assert!(!hub.in_ibd(), "a fresh tip leaves IBD"); + let addr3 = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8335); + let mut ver3 = ver.clone(); + ver3.nonce = 3; + let session3 = peers.register( + addr3, + addr3, + &ver3, + true, + crate::peers::PeerConnType::Inbound, + ); + let (tx3, mut rx3) = mpsc::unbounded_channel(); assert!( - maybe_queue_initial_getheaders(&tx, &hub, &session), + maybe_queue_initial_getheaders(&tx3, &hub, &session3), "after IBD a connecting peer still starts headers sync" ); assert!(matches!( - rx.try_recv().unwrap().expect_msg(), + rx3.try_recv().unwrap().expect_msg(), bitcoin::p2p::message::NetworkMessage::GetHeaders(_) )); let _ = std::fs::remove_dir_all(dir); @@ -881,7 +898,9 @@ fn handle_peer_frame_control_and_inv_paths() { let headers_msg = out_rx.try_recv().unwrap().expect_msg(); assert!(matches!(headers_msg, NetworkMessage::Headers(_))); - // Unknown block inv during IBD challenges the walk. It does not getheaders. + // Unknown block inv during IBD challenges the walk and asks for + // headers. No session is attached, so the ask is the inv path's + // default. Drain it so the headers reply below is the next message. let want_h = BlockHash::from_byte_array([0xee; 32]); handle_peer_frame( frame_for(NetworkMessage::Inv(vec![Inventory::WitnessBlock(want_h)])), @@ -893,8 +912,11 @@ fn handle_peer_frame_control_and_inv_paths() { .await .unwrap(); assert!( - out_rx.try_recv().is_err(), - "an unknown block inv during IBD must not getheaders" + matches!( + out_rx.try_recv().unwrap().expect_msg(), + NetworkMessage::GetHeaders(_) + ), + "an unknown block inv during IBD still asks for headers" ); assert!(hub.header_inv_challenge_pending()); diff --git a/docs/ibd-memory.md b/docs/ibd-memory.md index 2ec5c7b24..de8e5f792 100644 --- a/docs/ibd-memory.md +++ b/docs/ibd-memory.md @@ -52,7 +52,7 @@ pres and **not** the raw bytes. Reorg gather that wants wire re-encodes. | **SH unsorted collect / pack** | Two `txout` scans. All extract phases share `sh_extract_workers()` = min(CPUs, max(1, free RAM / 1.5 GiB)); `RBITCOIN_SH_MERGE_WORKERS` override. **Collect maps** use that same 1.5 GiB **per worker** as a cap (not a prefault): each worker owns a contiguous create-fk span and `n_shards` unsized identity maps (`HashMap`, `0` = multi; 64 B/key estimate). After each 64 k-fk loc/body batch, spill the largest shard map while `sum(estimate) >= 1.5 GiB`. Status `scanned=` is finished fks (not max fk). Spills, including the end flush, go through one writer and a 1-slot queue so encode+tmp+rename is serial. Fuse8 mmap on pass 2 is process-wide, not subtracted from the cap. `keys/NN/` is a **directory** of `SHKSP01` spill files (`000000`, …; n_multi key16s already `0`, then first-fk-sorted uleb(delta)‖key16 singles; tmp+rename, no `sync_all`). Merge folds those spill files only into one identity map, one walk to `scripthash.head/NN` pack8 and `multi/NN.fuse8` (file, not kept resident for other shards), unlinks `keys/NN/`. The folded map is consumed into the pack8 records and dropped before fuse8 and BDZ. A previous `DONE` / 24 B `NN` layout, or `keys/NN` as a file, with no valid `DONE.keys` is deleted and pass 1 restarts. A spill whose magic is not `SHKSP01` is Corrupt. Pass 2 keeps fuse8 mmap only (no BDZ). Same static fk spans; fuse-hit creates append `fk` to per-worker `HashMap>` (skip `vec.last() == fk`). Estimate `80 × n_keys + 8 × n_fks`; spill-largest while over 1.5 GiB. `post/NN/` is a **directory** of `SHPST01` spill files (`000000`, …; `u32 n_keys` then `key16 ‖ uleb(n) ‖ n × uleb(fk − prev)`; tmp+rename, no `sync_all`). A `post/NN` file, or a spill whose magic is not `SHPST01`, is Corrupt. **Pack fold** (named, after collect maps drop): one shard ~1.2 M multi keys × ~30 fks ≈ 0.3–0.5 GiB; 8 pack workers a few GiB. Fold spills into one map, then `MphfHead::open` + `slot_for_key16` + 2+ body + `rewrite_val_slots`; `len == 1` after fold is `fp_singles` (pass-1 `inline_one` stays). Unlink `post/NN/`. Packed 2-bit `g` ~8 MiB per pack worker + 16 MiB `body_buf`. Inner loc/body batch 65 536 fks inside each worker span (not a steal grain); body IO 16 MiB | Tip finalize. Extract under `scripthash.unsorted/{keys,multi,post}` | | **`tx.head` wipe-rebuild workers** | min(CPUs, host free RAM / 1 GiB, range count); floor 1. Same free-RAM probe as SH. Matches BDZ peel+keys+g peak at default 2²⁵. **Not** the SH extract 1.5 GiB cap. Env `RBITCOIN_TX_HEAD_REBUILD_WORKERS` override (`1` = serial) | Empty/wipe `tx.head` rebuild from `txid.body`. Logs `workers=` `free_GiB=` | | **Ordered work path** | `MAX_ORDERED_HEADERS` | `IbdWorkState::hygiene` | -| **Header-walk checkpoints** | One checkpoint per accepted 2,000-header reply, and those headers are written. The ordered queue is a window of at most 64,000 over headers already stored. One peer, the lowest time to a first block byte among peers that have not failed the walk, is asked only when the next header is not in the store. The locator starts at the candidate tip, then a lighter fork still being weighed. Block download uses the other peers. Above the work floor a short reply that does not extend the candidate moves the reservation; a full 2,000-header window does not, so a lighter fork can still catch up. A block inv from another peer is one challenge, asked even when that peer's connect-time start height is below the walk: if the reply beats the candidate they become the header peer, and if it does not they leave the walk and the previous peer stays. An inv from a session with no header slot does not retire the reserved peer. One missed ask moves the reservation; a second miss disconnects. Less work does not disconnect. Each checkpoint holds hash, height, work, that reply's last 80-byte header, up to 11 timestamps, and the difficulty period (one period-start header plus the last non-limit `nBits`) so a fork or rewind can retarget. One competing chain is kept the same way until it loses or replaces the candidate. Mainnet stays under about 250 KB. Queue maps stay capped with the queue. Side headers below the work floor are one hash per peer. More than 4,000 disconnects that peer. Abandoned look-ahead tips are cleared at 1,024 and when a heavier chain is adopted. | Dropped with IBD state. `header.adopt` is the sidecar, not a resident copy. The file is deleted once the confirmed chain proves the anchor | +| **Header-walk checkpoints** | One checkpoint per accepted 2,000-header reply, and those headers are written. The ordered queue is a window of at most 64,000 over headers already stored. One peer, the lowest time to a first block byte among peers that have not failed the walk, is asked only when the next header is not in the store. While the queue has room the locator starts at the stored tip, then the candidate, then a lighter fork still being weighed, so the reply can be enqueued. A full queue asks from the candidate. Block download uses the other peers. Above the work floor a short reply that does not extend the candidate moves the reservation; a full 2,000-header window does not, so a lighter fork can still catch up. A block inv from another peer is one challenge, asked even when that peer's connect-time start height is below the walk: if the reply beats the candidate they become the header peer, and if it does not they leave the walk and the previous peer stays. An inv from a session with no header slot does not retire the reserved peer. One missed ask moves the reservation; a second miss disconnects. Less work does not disconnect. Each checkpoint holds hash, height, work, that reply's last 80-byte header, up to 11 timestamps, and the difficulty period (one period-start header plus the last non-limit `nBits`) so a fork or rewind can retarget. One competing chain is kept the same way until it loses or replaces the candidate. Mainnet stays under about 250 KB. Queue maps stay capped with the queue. Side headers below the work floor are one hash per peer. More than 4,000 disconnects that peer. Abandoned look-ahead tips are cleared at 1,024 and when a heavier chain is adopted. | Dropped with IBD state. `header.adopt` is the sidecar, not a resident copy. The file is deleted once the confirmed chain proves the anchor | Tests that need a clean process must call these **same** entry points (or drop the owning `Query` / pipeline), not a secret test-only free-all that masks production From 5966c5982dea03f13ff62c42c85b76c04e6e98ae Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:12:50 -0700 Subject: [PATCH 3/4] test: proved tip must not chase a peer, and a checkpoint miss must not fill the gap A connected start height unlatches a walk that is already proved at the confirmed tip. A headers window that disagrees with the next checkpoint is written into an empty queue. --- crates/rbitcoin-net/src/ibd/header_walk.rs | 76 ++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/crates/rbitcoin-net/src/ibd/header_walk.rs b/crates/rbitcoin-net/src/ibd/header_walk.rs index 362de50cb..e4c0c332a 100644 --- a/crates/rbitcoin-net/src/ibd/header_walk.rs +++ b/crates/rbitcoin-net/src/ibd/header_walk.rs @@ -4008,6 +4008,82 @@ mod tests { .is_none()); } + /// A connected peer's start height is not a hole once the walk is proved + /// at the confirmed tip. A proved walk that is still above that tip is. + #[test] + fn proved_walk_at_the_tip_ignores_a_tall_peer() { + let (s0, _rx) = slot(0); + let mut st = IbdWorkState::new(vec![s0], None, Some(100)); + st.max_peer_height = 50_000; + st.headers_done = true; + st.header_walk.prove_through(100); + assert!( + !super::super::exit::should_unlatch_headers_done(&st, 100), + "a proved walk at the confirmed tip does not chase a connected peer" + ); + st.header_walk.prove_through(150); + assert!( + super::super::exit::should_unlatch_headers_done(&st, 100), + "a proved walk above the confirmed tip still opens an empty queue" + ); + } + + /// A batch that misses the checkpoint must not occupy the gap, and a + /// later batch that meets it must replace a short unproved queue. + #[test] + fn an_unproved_queue_does_not_hide_the_checkpoint() { + let (_dir, hub) = crate::chain::tiny_regtest_hub_labeled("header-walk-unproved"); + hub.ensure_genesis().unwrap(); + let gen = hub.tip_hash().unwrap(); + let good = mine(gen, 1); + let later = mine(good.block_hash(), 2); + let miss = mine(gen, 9); + let miss2 = mine(miss.block_hash(), 8); + let (s0, mut rx) = slot(0); + let mut st = IbdWorkState::new(vec![s0], Some(gen), Some(0)); + assert!(send_getheaders(&mut st, &hub).unwrap()); + let _ = rx.try_recv(); + apply(&mut st, &hub, 0, vec![good, later]); + assert_eq!(st.header_walk.tip_hash(), Some(later.block_hash())); + + st.ordered.clear(); + st.ordered_set.clear(); + st.height_to_hash.remove(&1); + st.height_to_hash.remove(&2); + + apply(&mut st, &hub, 0, vec![miss, miss2]); + assert!( + !st.ordered_set.contains(&miss.block_hash()), + "a window that misses the checkpoint must not occupy the gap" + ); + assert!(!st.ordered_set.contains(&miss2.block_hash())); + assert_eq!(st.header_walk.tip_hash(), Some(later.block_hash())); + + apply(&mut st, &hub, 0, vec![miss]); + assert!( + st.ordered_set.contains(&miss.block_hash()), + "a short refill still fills the queue before the next checkpoint" + ); + assert!(send_getheaders(&mut st, &hub).unwrap()); + match rx.try_recv() { + Ok(PeerCmd::GetHeaders { locator }) => { + assert!( + locator.contains(&gen), + "a peer that lacks the unproved tail must still be asked from the stored tip" + ); + } + Ok(_) => panic!("expected getheaders"), + Err(_) => panic!("peer was not asked for headers"), + } + apply(&mut st, &hub, 0, vec![good, later]); + assert!( + st.ordered_set.contains(&good.block_hash()), + "a batch that meets the checkpoint replaces the unproved queue" + ); + assert!(!st.ordered_set.contains(&miss.block_hash())); + assert_eq!(st.header_walk.tip_hash(), Some(later.block_hash())); + } + #[test] fn restart_restores_the_checkpoint_skip_and_garbage_does_not() { let (_dir, mut hub) = crate::chain::tiny_regtest_hub_labeled("header-walk-6"); From f9775f4e38aae10bc4724c26923dc4fe3ba8fa5a Mon Sep 17 00:00:00 2001 From: "rearden-grok[bot]" <317016512+rearden-grok[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:19:57 -0700 Subject: [PATCH 4/4] ibd: do not chase a proved tip or keep a checkpoint miss A connected peer's start height unlatched a walk already proved at the confirmed tip, so catch-up never stuck. An empty queue also stored a batch that missed the next checkpoint, and the next ask then started there. A proved walk at the confirmed tip opens only for a one- or two-block hole. A short refill is still queued. A checkpoint miss is not, and a later batch that meets the checkpoint replaces that guess. The locator names the proved anchor under an unproved queue. --- changelog.d/queue-refill.md | 2 +- crates/rbitcoin-net/src/ibd/exit.rs | 11 +- crates/rbitcoin-net/src/ibd/header_walk.rs | 147 +++++++++++++++++---- 3 files changed, 126 insertions(+), 34 deletions(-) diff --git a/changelog.d/queue-refill.md b/changelog.d/queue-refill.md index c6f07016b..ea75d7fde 100644 --- a/changelog.d/queue-refill.md +++ b/changelog.d/queue-refill.md @@ -1,3 +1,3 @@ Fixed -- **An empty header queue during IBD asks from the stored tip.** The reply is written onto the queue even when it stops short of the next checkpoint, so block download can resume when a restart kept the walk checkpoint and not the headers in between. A `headers_done` latch that never proved a walk opens again when a peer is ahead. A connecting peer still gets the one initial headers sync. +- **An empty header queue during IBD asks from the stored tip.** A reply that continues that tip is written onto the queue even when it stops short of the next checkpoint, so block download can resume when a restart kept the walk checkpoint and not the headers in between. A reply that misses the checkpoint does not occupy the gap, and a later reply that meets it replaces the short guess. A `headers_done` latch that never proved a walk opens again when a peer is ahead. A proved walk already at the confirmed tip does not. A connecting peer still gets the one initial headers sync. diff --git a/crates/rbitcoin-net/src/ibd/exit.rs b/crates/rbitcoin-net/src/ibd/exit.rs index c84ee4c7a..9dfcd2286 100644 --- a/crates/rbitcoin-net/src/ibd/exit.rs +++ b/crates/rbitcoin-net/src/ibd/exit.rs @@ -100,10 +100,10 @@ pub(crate) fn empty_path_header_fan(st: &IbdWorkState, tip_h: u32, alive: usize) /// Clear `headers_done` so empty-path `getheaders` can resume. /// /// A proved walk at the confirmed tip does not chase a far -/// `version.start_height`. A latch that never proved a walk does: the -/// ordered queue is empty and a peer is ahead, including by more than two -/// blocks. That is the regtest case where the latch fired before any peer -/// had a chain. +/// `version.start_height`, including one advertised by a connected peer. +/// One or two missing connecting headers still open. A proved walk above +/// the confirmed tip, or a latch that never proved a walk, opens for any +/// peer lead. #[inline] pub(crate) fn should_unlatch_headers_done(st: &IbdWorkState, tip_h: u32) -> bool { if !st.headers_done || !st.ordered.is_empty() { @@ -113,6 +113,9 @@ pub(crate) fn should_unlatch_headers_done(st: &IbdWorkState, tip_h: u32) -> bool if lag == 0 || has_tip_plus_one(st, tip_h) { return false; } + if st.header_walk.proved_at_or_below(tip_h) { + return lag <= 2; + } !super::header_walk::headers_complete(st, tip_h) } diff --git a/crates/rbitcoin-net/src/ibd/header_walk.rs b/crates/rbitcoin-net/src/ibd/header_walk.rs index e4c0c332a..ff965ab21 100644 --- a/crates/rbitcoin-net/src/ibd/header_walk.rs +++ b/crates/rbitcoin-net/src/ibd/header_walk.rs @@ -136,6 +136,11 @@ impl HeaderWalk { self.tip.height } + /// The walk has met the work floor and is not past `tip_h`. + pub(crate) fn proved_at_or_below(&self, tip_h: u32) -> bool { + self.proven && self.tip.height <= tip_h + } + pub(crate) fn tip_hash(&self) -> Option { self.tip.hash } @@ -716,7 +721,7 @@ pub(crate) fn suppress_competing_chain( // Treating it as one would evict a real fork that has less work so far. if st.header_walk.proven && !below && candidate_prev(st, prev) { if let Some(start_h) = height_of(st, hub, prev) { - if matches!(path_agree(st, start_h, headers), PathAgree::Short) { + if matches!(path_agree(st, start_h, headers, true), PathAgree::Short) { return true; } } @@ -1641,35 +1646,110 @@ pub(crate) fn reject_refill_miss( { return false; } - let Some(tail) = st.ordered.back().copied() else { - return false; - }; - if headers[0].prev_blockhash != tail { + let prev = headers[0].prev_blockhash; + // A proven walk is the most-work header chain so far. A refill that + // disagrees with its next checkpoint is not the download path. A short + // refill that has not reached that checkpoint still is: refusing it + // leaves the block queue empty. A later batch from the same anchor that + // does meet the checkpoint replaces that unproved queue. + if let Some(tail) = st.ordered.back().copied() { + if prev != tail { + return replace_unproved_queue(st, hub, prev, headers); + } + let Some(&tail_h) = st.hash_height.get(&tail) else { + return false; + }; + return judge_refill(st, hub, tail_h, headers); + } + if !proved_above_floor(st, hub) || hub.tip_hash() != Some(prev) { return false; } - let Some(&tail_h) = st.hash_height.get(&tail) else { + let Some(tip_h) = hub.tip_height() else { return false; }; - // A proven walk is the most-work header chain so far. A refill that - // disagrees with it is not allowed to erase that proof. A short refill - // that continues this tail is still the download path: refusing it - // leaves the block queue empty until a later batch happens to reach the - // next checkpoint. Below the floor the candidate is still a guess, and - // a contradictory refill replaces it. - if st.header_walk.proven && !below_floor(hub, &st.header_walk.tip_work()) { - return match path_agree(st, tail_h, headers) { + judge_refill(st, hub, tip_h, headers) +} + +fn proved_above_floor(st: &IbdWorkState, hub: &ChainHub) -> bool { + st.header_walk.proven && !below_floor(hub, &st.header_walk.tip_work()) +} + +fn judge_refill(st: &mut IbdWorkState, hub: &ChainHub, start_h: u32, headers: &[Header]) -> bool { + if proved_above_floor(st, hub) { + return match path_agree(st, start_h, headers, true) { PathAgree::Diverges => offer_alternate(st, hub, None, headers).unwrap_or(true), - PathAgree::Short => false, - PathAgree::Checkpoint(_) => false, + PathAgree::Short | PathAgree::Checkpoint(_) => false, }; } - if !refill_misses_checkpoint(st, tail_h, headers) { + if !refill_misses_checkpoint(st, start_h, headers) { return false; } - rewind_above(st, hub, tail_h); + rewind_above(st, hub, start_h); false } +/// Queue that continues the stored tip but has not met the next checkpoint. +/// The hash is the proved parent those headers extend. +fn unproved_anchor(st: &IbdWorkState, hub: &ChainHub) -> Option<(BlockHash, u32)> { + let front = st.ordered.front().copied()?; + let front_h = st.hash_height.get(&front).copied()?; + let reached = st.ordered.iter().any(|hash| { + st.hash_height.get(hash).is_some_and(|ht| { + st.header_walk + .checkpoints + .iter() + .any(|c| *ht >= c.height && c.height >= front_h) + }) + }); + if reached { + return None; + } + if let Some(cp) = st + .header_walk + .checkpoints + .iter() + .filter(|c| c.height < front_h) + .max_by_key(|c| c.height) + { + return Some((cp.hash, cp.height)); + } + let tip_h = hub.tip_height()?; + if front_h == tip_h.saturating_add(1) { + return Some((hub.tip_hash()?, tip_h)); + } + None +} + +/// A batch from the proved anchor under an unproved queue. +/// +/// True when the reply was consumed. A batch that meets the next checkpoint +/// clears that queue so the caller can write it. A short guess does not +/// replace the one already queued. A checkpoint miss is a competing chain. +fn replace_unproved_queue( + st: &mut IbdWorkState, + hub: &ChainHub, + prev: BlockHash, + headers: &[Header], +) -> bool { + if !proved_above_floor(st, hub) { + return false; + } + let Some((anchor, anchor_h)) = unproved_anchor(st, hub) else { + return false; + }; + if prev != anchor { + return false; + } + match path_agree(st, anchor_h, headers, false) { + PathAgree::Checkpoint(_) => { + drop_queue_above(st, anchor_h, &HashSet::new()); + false + } + PathAgree::Diverges => offer_alternate(st, hub, None, headers).unwrap_or(true), + PathAgree::Short => true, + } +} + /// A checkpoint this batch meets is an ancestor of the milestone walk. /// Headers past that checkpoint, or a batch that never meets one, are not. enum PathAgree { @@ -1689,7 +1769,9 @@ fn candidate_prev(st: &IbdWorkState, prev: BlockHash) -> bool { .is_some_and(|h| st.height_to_hash.get(h) == Some(&prev)) } -fn path_agree(st: &IbdWorkState, start_h: u32, headers: &[Header]) -> PathAgree { +/// `slots` is false when an unproved queue already occupies `height_to_hash`. +/// The next checkpoint is the proof, not that guess. +fn path_agree(st: &IbdWorkState, start_h: u32, headers: &[Header], slots: bool) -> PathAgree { let Some(first) = headers.first() else { return PathAgree::Short; }; @@ -1714,12 +1796,14 @@ fn path_agree(st: &IbdWorkState, start_h: u32, headers: &[Header]) -> PathAgree return PathAgree::Diverges; } agreed = Some(height); - } else if let Some(want) = st.height_to_hash.get(&height) { - if *want != hash { - return PathAgree::Diverges; + } else if slots { + if let Some(want) = st.height_to_hash.get(&height) { + if *want != hash { + return PathAgree::Diverges; + } + } else if agreed.is_some() { + break; } - } else if agreed.is_some() { - break; } prev = hash; } @@ -1753,7 +1837,7 @@ pub(crate) fn proven_header_prefix(st: &IbdWorkState, hub: &ChainHub, headers: & let Some(start_h) = height_of(st, hub, prev) else { return headers.len(); }; - let stop = match path_agree(st, start_h, headers) { + let stop = match path_agree(st, start_h, headers, true) { PathAgree::Checkpoint(stop) => stop, // Continues the stored tip and has not missed a checkpoint. The // block queue needs these headers even when the next checkpoint is @@ -2149,16 +2233,21 @@ pub(crate) fn send_getheaders( /// Locator for one headers ask. /// /// Bitcoin answers after the first locator hash the peer knows. While the -/// queue can take headers, that hash is the stored tip. The candidate and -/// its checkpoints follow, so a peer that does not have the stored tip can -/// still extend the walk. A full queue is look-ahead and starts at the -/// candidate. +/// queue can take headers, that hash is the stored tip. An unproved queue +/// also names the proved anchor under it, so a peer that does not have the +/// guess still sends the gap. The candidate and its checkpoints follow. A +/// full queue is look-ahead and starts at the candidate. fn header_ask_locator(st: &IbdWorkState, hub: &ChainHub) -> Vec { let mut tips = Vec::new(); if st.ordered.len() < ORDERED_HEADERS_SOFT_CAP { if let Some(stored) = stored_top_hash(st, hub) { tips.push(stored); } + if let Some((anchor, _)) = unproved_anchor(st, hub) { + if !tips.contains(&anchor) { + tips.push(anchor); + } + } } for hash in st.header_walk.locator_hashes() { if !tips.contains(&hash) {