From be2d297305c8a93f3e51bb88394b7d5869fb7e92 Mon Sep 17 00:00:00 2001 From: Nitesh Balusu Date: Sun, 18 Oct 2026 04:51:23 -0400 Subject: [PATCH 1/6] ibd: pin a same-batch parent on an archived retry A batch with an invalid block is retried from its archived bodies. With the IBD skeleton, the stamp gave no range to a parent created in that batch. Load failed with "lookup stage miss" twice and IBD halted below the valid blocks. Fill ranges for the batch's own creates on that retry. Co-Authored-By: Claude Opus 5.5 --- .../archived-retry-same-batch-parent.md | 7 +++ .../src/ibd/events/confirm_reject_tests.rs | 52 +++++++++++++++++++ .../rbitcoin-query/src/confirm_run/lookup.rs | 11 ++++ docs/invariants.md | 2 +- 4 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 changelog.d/archived-retry-same-batch-parent.md diff --git a/changelog.d/archived-retry-same-batch-parent.md b/changelog.d/archived-retry-same-batch-parent.md new file mode 100644 index 0000000000..b914d1554d --- /dev/null +++ b/changelog.d/archived-retry-same-batch-parent.md @@ -0,0 +1,7 @@ +Fixed + +- **IBD connects archived blocks that spend a create from the same batch.** + A retry from archived bodies found no range for a parent created in that + batch, logged `lookup stage miss`, and halted IBD below valid blocks. The + retry follows an invalid block in a batch, or a restart after a crash + between the Class A write and the tip advance. diff --git a/crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs b/crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs index 7a5ef0b38f..2b5450abec 100644 --- a/crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs +++ b/crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs @@ -2678,6 +2678,58 @@ fn batched_spend_of_a_later_block_is_rejected_alone() { rig.finish(); } +/// A wave whose first block spends a create from its own block and whose +/// second block is invalid at write. The cascade retries the first block from +/// the archive, and that retry must still pin the same-block parent. +#[test] +fn archived_retry_pins_a_same_block_parent() { + use super::super::assign::tests::lock_default_assign_stop; + use super::super::state::InflightReq; + use bitcoin::{Amount, ScriptBuf}; + use rbitcoin_consensus::{grind_regtest_pow, mine_regtest_paying}; + + let _env = lock_default_assign_stop(); + let mut rig = WireRig::new("archived-same-block", 2); + let (t, cbs) = (rig.t, rig.cbs.clone()); + let spk = ScriptBuf::from_bytes(vec![0x51]); + let parent = WireRig::spend(cbs[0]); + let child = WireRig::spend(parent.compute_txid()); + let b1 = mine_regtest_paying( + rig.tip, + rig.tip_time + 600, + t + 1, + spk.clone(), + vec![parent, child], + ); + let mut b2 = mine_regtest_paying(b1.block_hash(), rig.tip_time + 1200, t + 2, spk, vec![]); + b2.txdata[0].output[0].value += Amount::from_sat(1); + b2.header.merkle_root = b2.compute_merkle_root().unwrap(); + grind_regtest_pow(&mut b2.header); + let (h1, h2) = (b1.block_hash(), b2.block_hash()); + rig.plant(&[&b1, &b2]); + for (peer, body) in [(1, &b1), (2, &b2)] { + let hash = body.block_hash(); + rig.st.slots[peer - 1].in_flight.insert(hash); + rig.st.inflight.insert(hash, InflightReq::new(peer)); + rig.deliver(peer, body); + } + rig.start_engine(); + + let rejects = rig.pump( + |_, hash| if hash == h1 { b1.clone() } else { b2.clone() }, + |st, _, _| st.halt.is_some() || st.reorg.invalid.contains(h2.to_byte_array()), + ); + assert_eq!( + rig.st.halt, None, + "rejects={rejects:?} last={:?}", + rig.last_reject_err + ); + assert!(!rig.st.engine_fault_seen.contains(&h1), "{rejects:?}"); + assert!(rig.st.reorg.invalid.contains(h2.to_byte_array())); + assert_eq!(rig.hub.tip_height(), Some(t + 1), "{rejects:?}"); + rig.finish(); +} + /// tip+1 is the last block on the path, and load cannot read its header /// row when it stamps the wave: a local store fault, not a block verdict. /// The wave goes back to lookup, so once the row reads again tip+1 diff --git a/crates/rbitcoin-query/src/confirm_run/lookup.rs b/crates/rbitcoin-query/src/confirm_run/lookup.rs index 60641af3dd..c7142b106e 100644 --- a/crates/rbitcoin-query/src/confirm_run/lookup.rs +++ b/crates/rbitcoin-query/src/confirm_run/lookup.rs @@ -366,6 +366,17 @@ pub(super) fn stamp_parent_pin_archived( query.confirm_stats(), ) .map_err(ConfirmError::from)?; + } else { + // The IBD skeleton carries external ranges. This batch's own creates + // are already archived (plan=None), so only the store has theirs. + let mut own: U64Map = metas + .iter() + .flat_map(|m| m.tx_fks.iter().filter_map(|f| f.get())) + .filter_map(|id| stamp.idents.remove_entry(&id)) + .collect(); + crate::fill_missing_parent_ranges(query.store(), ifo, &mut own, query.confirm_stats()) + .map_err(ConfirmError::from)?; + stamp.idents.extend(own); } for ident in stamp.idents.values() { if ident.txid == [0u8; 32] { diff --git a/docs/invariants.md b/docs/invariants.md index 54fd054948..458fb996a0 100644 --- a/docs/invariants.md +++ b/docs/invariants.md @@ -81,7 +81,7 @@ the Wire CreatePin (`Arc` + tx index). Load still does not head/idx. | Stage | Invariant | Soft path allowed? | |-------|-----------|--------------------| -| Lookup parent stamp | Every external spent parent has create_fk + body_range (or offline in_flight CreatePin) + reverse txid. Archived parents also have computed `spent.body` range on the stamp (same-wave in-flight outs skip loc, miss OK; later-wave InFlight takes loc from the same `CreatePin` Arc write set at Class A, or skeleton TipOnly loc). IBD (`skeleton = Some`) never loc-by-fk: pin loc unset leaves spent unset for write TLS. plan=None leftover still `fill_missing_parent_ranges` | Missing → hard Err at stamp / pin contract. **No** `finish_archive_plan` `fill_missing` after stamp loc. **No** IBD load `create.loc` | +| Lookup parent stamp | Every external spent parent has create_fk + body_range (or offline in_flight CreatePin) + reverse txid. Archived parents also have computed `spent.body` range on the stamp (same-wave in-flight outs skip loc, miss OK; later-wave InFlight takes loc from the same `CreatePin` Arc write set at Class A, or skeleton TipOnly loc). IBD (`skeleton = Some`) never loc-by-fk: pin loc unset leaves spent unset for write TLS. The one exception is an all-archived retry (plan=None), whose own creates only the store can range. plan=None leftover still `fill_missing_parent_ranges` | Missing → hard Err at stamp / pin contract. **No** `finish_archive_plan` `fill_missing` after stamp loc. **No** IBD load `create.loc` | | Parent create_fk | **same-batch** planned fks (offline at pin) → **in-flight** (lookup snapshots `drain_and_fence_hi` **before** the wave's TipOnly read and passes it on the last load batch; load drops tagged map rows with pack height **below** that snapshot after that batch's in-flight read; equality keeps; not Class C tip, not `class_a_hi`, not write freeze; one load-thread HashMap, insert after stamp) → **skeleton** (`BatchParentIds` on the `LoadBatch`: lookup TipOnly fk + body_range + spent_range + per-chunk need-vouts) → **Corrupt** on IBD miss. plan=None / S0 (`skeleton = None`) is in-flight → leftover TipOnly. One helper: [`stamp_external_parents`](../crates/rbitcoin-query/src/stamp.rs). No leftover pending map, no process pin FIFO, no BQ-side hits map, no parent-store create_fk on stamp, no published live_union. Same-wave creates are omitted from TipOnly need. Header-cache GC polls store tip each load pack. In-flight and skeleton maps stay one fk per txid (last write). That limit is only those RAM maps ([`errata.md`](./errata.md)), not a consensus rule for this write. | Miss of in-flight and skeleton → `Corrupt("parent create_fk unresolved")`. A batched miss isolates; a lone block's miss is an **engine fault** (requeue once, then halt IBD; never blacklist), unless the block extends the tip, `tx.head` holds every connected create (`Query::head_covers_fence`; open starts the drain at the head's last create, after its rebuild or tail backfill), and a fresh TipOnly read of its parents on the load thread (reject path only) still misses: that is `ConsensusInvalid`, unless `txid.body` holds the txid at a connected fk in an unsealed `tx.head` segment (the open tail or the in-flight seal), whose slot pages may hold a power-loss hole (`Corrupt("invariant: tx.head misses a connected create")`, engine fault) ([089](./external_findings/089-missing-parent-verdict.md)). A lone block whose parent is not the tip within 10 s is a `Cascade` retry. Identity without loc range → `Corrupt("invariant: loc range missing after identity")`, not a miss | | io_uring harvest | TLS session fail-closed ([`io-modality.md`](./io-modality.md)) | **No** silent success. `Corrupt("invariant: io_uring …")` (not `bdz g page bad slot`). Ring-unavailable still pread-fallback | | Load body outs | By `txout` range only from lookup stamp; incomplete outs → hard Err. Pin **copies** lookup `spent_range` (no loc IO). Later-wave InFlight loc comes from the CreatePin Arc (Class A `set_loc`) | **No** loc cold outs on load; **no** `seqsigwit` on pin; **no** IBD loc-by-fk | From d676fe134c091ca3e326573ee6ef5587f38bece1 Mon Sep 17 00:00:00 2001 From: Nitesh Balusu Date: Sun, 18 Oct 2026 06:55:00 -0400 Subject: [PATCH 2/6] test: assert the archived-retry wave is a cascade Review nit: the test should fail if a batching change stops putting both blocks in one wave. Co-Authored-By: Claude Opus 5.5 --- crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs b/crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs index 2b5450abec..315328278f 100644 --- a/crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs +++ b/crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs @@ -2724,6 +2724,10 @@ fn archived_retry_pins_a_same_block_parent() { "rejects={rejects:?} last={:?}", rig.last_reject_err ); + assert!( + rejects.contains(&(h1, ConfirmRejectClass::Cascade, 2)), + "the wave must be retried from the archive: {rejects:?}" + ); assert!(!rig.st.engine_fault_seen.contains(&h1), "{rejects:?}"); assert!(rig.st.reorg.invalid.contains(h2.to_byte_array())); assert_eq!(rig.hub.tip_height(), Some(t + 1), "{rejects:?}"); From 544a16f5f79dfd75d39628bbfc34ac920db10da3 Mon Sep 17 00:00:00 2001 From: Nitesh Balusu Date: Sun, 18 Oct 2026 08:48:11 -0400 Subject: [PATCH 3/6] ibd: pin a parent from an earlier chunk of an archived wave Review: a wave longer than one load batch notes chunk 1's archived creates in flight with no outs. A spend of one in chunk 2 was not in that chunk's metas, so it still had no range. Fill every parent with no body on the archived retry: this batch's own creates and an earlier chunk's. Skeleton parents keep their body and in-flight pins are still skipped. Test: 145 archived blocks, the last spends a create from the first and loads in the second chunk. The stamp.rs module note and the invariants row name the exception. Co-Authored-By: Claude Opus 5.5 --- .../archived-retry-same-batch-parent.md | 11 +-- .../src/ibd/events/confirm_reject_tests.rs | 85 ++++++++++++++++++- .../rbitcoin-query/src/confirm_run/lookup.rs | 17 ++-- crates/rbitcoin-query/src/stamp.rs | 6 +- docs/invariants.md | 2 +- 5 files changed, 103 insertions(+), 18 deletions(-) diff --git a/changelog.d/archived-retry-same-batch-parent.md b/changelog.d/archived-retry-same-batch-parent.md index b914d1554d..4acfc8a641 100644 --- a/changelog.d/archived-retry-same-batch-parent.md +++ b/changelog.d/archived-retry-same-batch-parent.md @@ -1,7 +1,8 @@ Fixed -- **IBD connects archived blocks that spend a create from the same batch.** - A retry from archived bodies found no range for a parent created in that - batch, logged `lookup stage miss`, and halted IBD below valid blocks. The - retry follows an invalid block in a batch, or a restart after a crash - between the Class A write and the tip advance. +- **IBD connects archived blocks that spend an archived create.** A retry + from archived bodies found no range for a parent created in the same batch, + or in an earlier chunk of the same wave. It logged `lookup stage miss` and + halted IBD below valid blocks. The retry follows an invalid block in a + batch, or a restart after a crash between the Class A write and the tip + advance. diff --git a/crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs b/crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs index 315328278f..4035c741cf 100644 --- a/crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs +++ b/crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs @@ -2726,7 +2726,7 @@ fn archived_retry_pins_a_same_block_parent() { ); assert!( rejects.contains(&(h1, ConfirmRejectClass::Cascade, 2)), - "the wave must be retried from the archive: {rejects:?}" + "both blocks in one wave: {rejects:?}" ); assert!(!rig.st.engine_fault_seen.contains(&h1), "{rejects:?}"); assert!(rig.st.reorg.invalid.contains(h2.to_byte_array())); @@ -2734,6 +2734,89 @@ fn archived_retry_pins_a_same_block_parent() { rig.finish(); } +/// Archived blocks above the tip, as a crash between the Class A write and +/// the tip advance leaves them. The wave is longer than one load batch, so +/// the last block loads in the next chunk and spends a create from the +/// first, noted in flight with no outs. +#[test] +fn archived_wave_pins_a_parent_from_the_previous_chunk() { + use super::super::assign::tests::lock_default_assign_stop; + use super::super::state::InflightReq; + use bitcoin::ScriptBuf; + use rbitcoin_consensus::mine_regtest_paying; + + let _env = lock_default_assign_stop(); + let mut rig = WireRig::new("archived-cross-chunk", 1); + let (t, cbs) = (rig.t, rig.cbs.clone()); + let op_true = ScriptBuf::from_bytes(vec![0x51]); + let n = super::super::confirm::CONFIRM_RUN_MAX_BLOCKS as u32 + 1; + let parent = WireRig::spend(cbs[0]); + let mut child = Some(WireRig::spend(parent.compute_txid())); + let mut extra = vec![parent]; + let mut blocks: Vec = Vec::new(); + for i in 1..=n { + let prev = blocks.last().map_or(rig.tip, |b| b.block_hash()); + if i == n { + extra = child.take().into_iter().collect(); + } + let b = mine_regtest_paying( + prev, + rig.tip_time + 600 * i, + t + i, + op_true.clone(), + std::mem::take(&mut extra), + ); + blocks.push(b); + } + // One Class A write for the run, as the write stage does, with no tip advance. + let bodies: Vec<_> = blocks.iter().cloned().map(std::sync::Arc::new).collect(); + let fks: Vec<_> = bodies + .iter() + .map(|b| rig.hub.ensure_header_fk(&b.header).unwrap()) + .collect(); + let txids: Vec> = bodies + .iter() + .map(|b| { + b.txdata + .iter() + .map(|t| t.compute_txid().to_byte_array()) + .collect() + }) + .collect(); + let items: Vec<_> = bodies + .iter() + .zip(&fks) + .zip(&txids) + .map(|((b, fk), ids)| (*fk, b, ids.as_slice())) + .collect(); + rig.hub.query.archive_class_a_from_wire(&items).unwrap(); + assert_eq!(rig.hub.tip_height(), Some(t)); + let refs: Vec<&bitcoin::Block> = blocks.iter().collect(); + rig.plant(&refs); + for body in &blocks { + let hash = body.block_hash(); + rig.st.slots[0].in_flight.insert(hash); + rig.st.inflight.insert(hash, InflightReq::new(1)); + rig.deliver(1, body); + } + rig.start_engine(); + + let last = blocks.last().unwrap().block_hash(); + let by_hash: std::collections::HashMap<_, _> = + blocks.iter().map(|b| (b.block_hash(), b.clone())).collect(); + let rejects = rig.pump( + |_, hash| by_hash[&hash].clone(), + |st, hub, _| st.halt.is_some() || hub.tip_hash() == Some(last), + ); + assert_eq!( + rig.st.halt, None, + "rejects={rejects:?} last={:?}", + rig.last_reject_err + ); + assert_eq!(rig.hub.tip_height(), Some(t + n), "{rejects:?}"); + rig.finish(); +} + /// tip+1 is the last block on the path, and load cannot read its header /// row when it stamps the wave: a local store fault, not a block verdict. /// The wave goes back to lookup, so once the row reads again tip+1 diff --git a/crates/rbitcoin-query/src/confirm_run/lookup.rs b/crates/rbitcoin-query/src/confirm_run/lookup.rs index c7142b106e..1a543bbce8 100644 --- a/crates/rbitcoin-query/src/confirm_run/lookup.rs +++ b/crates/rbitcoin-query/src/confirm_run/lookup.rs @@ -367,16 +367,15 @@ pub(super) fn stamp_parent_pin_archived( ) .map_err(ConfirmError::from)?; } else { - // The IBD skeleton carries external ranges. This batch's own creates - // are already archived (plan=None), so only the store has theirs. - let mut own: U64Map = metas - .iter() - .flat_map(|m| m.tx_fks.iter().filter_map(|f| f.get())) - .filter_map(|id| stamp.idents.remove_entry(&id)) - .collect(); - crate::fill_missing_parent_ranges(query.store(), ifo, &mut own, query.confirm_stats()) + // The IBD skeleton carries external ranges. A parent with no body is + // an archived create: this batch's own, or an earlier chunk's of the + // same wave (noted in flight with no outs). Only the store has its + // range. In-flight pins are skipped inside the fill. + let mut archived: U64Map = + stamp.idents.extract_if(|_, e| e.body.is_none()).collect(); + crate::fill_missing_parent_ranges(query.store(), ifo, &mut archived, query.confirm_stats()) .map_err(ConfirmError::from)?; - stamp.idents.extend(own); + stamp.idents.extend(archived); } for ident in stamp.idents.values() { if ident.txid == [0u8; 32] { diff --git a/crates/rbitcoin-query/src/stamp.rs b/crates/rbitcoin-query/src/stamp.rs index bb7153a212..e5ecdc5e9e 100644 --- a/crates/rbitcoin-query/src/stamp.rs +++ b/crates/rbitcoin-query/src/stamp.rs @@ -6,8 +6,10 @@ //! rehydrate. In-flight holds CreatePins until load drops map rows below a //! lookup-wave drain+fence snapshot taken before TipOnly. Same-wave creates are //! omitted from that skeleton. Class A `CreatePin::set_loc` after append; -//! later-wave stamp reads pin loc. IBD (`skeleton = Some`) never loc-by-fk: -//! spent is pin loc, skeleton TipOnly loc, or unset for write fill. +//! later-wave stamp reads pin loc. IBD (`skeleton = Some`) never loc-by-fk +//! here: spent is pin loc, skeleton TipOnly loc, or unset for write fill. The +//! all-archived retry (`stamp_parent_pin_archived`) loc-fills its archived +//! parents that have no body. External IBD parents still must not. use crate::{CreatePin, InFlight, QueryError, U64Map}; use crate::{IdMap, TxidHasher}; diff --git a/docs/invariants.md b/docs/invariants.md index 458fb996a0..3918553479 100644 --- a/docs/invariants.md +++ b/docs/invariants.md @@ -81,7 +81,7 @@ the Wire CreatePin (`Arc` + tx index). Load still does not head/idx. | Stage | Invariant | Soft path allowed? | |-------|-----------|--------------------| -| Lookup parent stamp | Every external spent parent has create_fk + body_range (or offline in_flight CreatePin) + reverse txid. Archived parents also have computed `spent.body` range on the stamp (same-wave in-flight outs skip loc, miss OK; later-wave InFlight takes loc from the same `CreatePin` Arc write set at Class A, or skeleton TipOnly loc). IBD (`skeleton = Some`) never loc-by-fk: pin loc unset leaves spent unset for write TLS. The one exception is an all-archived retry (plan=None), whose own creates only the store can range. plan=None leftover still `fill_missing_parent_ranges` | Missing → hard Err at stamp / pin contract. **No** `finish_archive_plan` `fill_missing` after stamp loc. **No** IBD load `create.loc` | +| Lookup parent stamp | Every external spent parent has create_fk + body_range (or offline in_flight CreatePin) + reverse txid. Archived parents also have computed `spent.body` range on the stamp (same-wave in-flight outs skip loc, miss OK; later-wave InFlight takes loc from the same `CreatePin` Arc write set at Class A, or skeleton TipOnly loc). IBD (`skeleton = Some`) never loc-by-fk: pin loc unset leaves spent unset for write TLS. The one exception is an all-archived retry (plan=None): a parent with no body is an archived create (this batch's own, or an earlier chunk's of the same wave), and only the store can range it. plan=None leftover still `fill_missing_parent_ranges` | Missing → hard Err at stamp / pin contract. **No** `finish_archive_plan` `fill_missing` after stamp loc. **No** IBD load `create.loc` | | Parent create_fk | **same-batch** planned fks (offline at pin) → **in-flight** (lookup snapshots `drain_and_fence_hi` **before** the wave's TipOnly read and passes it on the last load batch; load drops tagged map rows with pack height **below** that snapshot after that batch's in-flight read; equality keeps; not Class C tip, not `class_a_hi`, not write freeze; one load-thread HashMap, insert after stamp) → **skeleton** (`BatchParentIds` on the `LoadBatch`: lookup TipOnly fk + body_range + spent_range + per-chunk need-vouts) → **Corrupt** on IBD miss. plan=None / S0 (`skeleton = None`) is in-flight → leftover TipOnly. One helper: [`stamp_external_parents`](../crates/rbitcoin-query/src/stamp.rs). No leftover pending map, no process pin FIFO, no BQ-side hits map, no parent-store create_fk on stamp, no published live_union. Same-wave creates are omitted from TipOnly need. Header-cache GC polls store tip each load pack. In-flight and skeleton maps stay one fk per txid (last write). That limit is only those RAM maps ([`errata.md`](./errata.md)), not a consensus rule for this write. | Miss of in-flight and skeleton → `Corrupt("parent create_fk unresolved")`. A batched miss isolates; a lone block's miss is an **engine fault** (requeue once, then halt IBD; never blacklist), unless the block extends the tip, `tx.head` holds every connected create (`Query::head_covers_fence`; open starts the drain at the head's last create, after its rebuild or tail backfill), and a fresh TipOnly read of its parents on the load thread (reject path only) still misses: that is `ConsensusInvalid`, unless `txid.body` holds the txid at a connected fk in an unsealed `tx.head` segment (the open tail or the in-flight seal), whose slot pages may hold a power-loss hole (`Corrupt("invariant: tx.head misses a connected create")`, engine fault) ([089](./external_findings/089-missing-parent-verdict.md)). A lone block whose parent is not the tip within 10 s is a `Cascade` retry. Identity without loc range → `Corrupt("invariant: loc range missing after identity")`, not a miss | | io_uring harvest | TLS session fail-closed ([`io-modality.md`](./io-modality.md)) | **No** silent success. `Corrupt("invariant: io_uring …")` (not `bdz g page bad slot`). Ring-unavailable still pread-fallback | | Load body outs | By `txout` range only from lookup stamp; incomplete outs → hard Err. Pin **copies** lookup `spent_range` (no loc IO). Later-wave InFlight loc comes from the CreatePin Arc (Class A `set_loc`) | **No** loc cold outs on load; **no** `seqsigwit` on pin; **no** IBD loc-by-fk | From 7f09e5f42bb3c5a9d779510b283a2fd3d3d7020d Mon Sep 17 00:00:00 2001 From: Nitesh Balusu Date: Sun, 18 Oct 2026 05:48:42 -0400 Subject: [PATCH 4/6] ibd: fetch archived bodies above the tip after a crash A crash between the Class A write and the tip advance leaves bodies above the tip. On restart the resume seed marks them archived, densify skips archived hashes, and the tip-hole cover only ran once the queue was a quarter full. With a short chain left, IBD sat at the old tip. Cover archived tip holes with one peer when there is no race. Co-Authored-By: Claude Opus 5.5 --- changelog.d/ibd-archived-tip-hole.md | 6 ++++++ crates/rbitcoin-net/src/ibd/assign.rs | 9 +++++++++ crates/rbitcoin-test/tests/end_of_ibd.rs | 17 +++++++++++++++++ 3 files changed, 32 insertions(+) create mode 100644 changelog.d/ibd-archived-tip-hole.md diff --git a/changelog.d/ibd-archived-tip-hole.md b/changelog.d/ibd-archived-tip-hole.md new file mode 100644 index 0000000000..31f5872af8 --- /dev/null +++ b/changelog.d/ibd-archived-tip-hole.md @@ -0,0 +1,6 @@ +Fixed + +- **IBD fetches archived bodies above the tip again after a crash.** A crash + between the Class A write and the tip advance leaves bodies above the tip. + The restart marked them archived, densify skipped them, and only a + quarter-full queue raced the tip hole, so IBD could sit at the old tip. diff --git a/crates/rbitcoin-net/src/ibd/assign.rs b/crates/rbitcoin-net/src/ibd/assign.rs index 7b99b951d5..d19e2e2ecc 100644 --- a/crates/rbitcoin-net/src/ibd/assign.rs +++ b/crates/rbitcoin-net/src/ibd/assign.rs @@ -294,6 +294,15 @@ pub(crate) fn assign_work_ordered( ); if race_tip { issued += cover_tip_batch_holes(st, hub, cfg, &alive, &tip_holes); + } else { + // Densify skips a Class A seed (`need_hash_at`). Without a race, + // only this cover gets an archived tip hole back into the queue. + let archived: Vec = tip_holes + .iter() + .copied() + .filter(|h| st.body.is_known_archived(h)) + .collect(); + issued += cover_tip_holes(st, hub, cfg, &alive, &archived, 1); } if tip_holes.is_empty() { issued += cover_first_pre_hole(st, hub, cfg, &alive); diff --git a/crates/rbitcoin-test/tests/end_of_ibd.rs b/crates/rbitcoin-test/tests/end_of_ibd.rs index 7e6549d8ae..1693f856ef 100644 --- a/crates/rbitcoin-test/tests/end_of_ibd.rs +++ b/crates/rbitcoin-test/tests/end_of_ibd.rs @@ -524,10 +524,27 @@ async fn partial_with_miner_down(miner_dir: &Path, synced: &Synced) { let miner = start_miner(miner_dir).await; finish_partial(partial_dir.path().as_path(), miner.local_addr, synced).await; + archive_above_tip(&held_dir.store_path(), &miner, held_tip, synced.height); finish_partial(held_dir.path().as_path(), miner.local_addr, synced).await; miner.shutdown().await; } +/// A crash between the Class A write and the tip advance leaves bodies above +/// the tip with no connect. The next start must fetch them again. +fn archive_above_tip(store: &Path, miner: &P2PNode, tip: u32, top: u32) { + use rbitcoin_query::testutil::FixtureChain; + let query = Query::open_or_create_tiny(store).unwrap(); + let params = ChainParams::regtest(); + for h in tip + 1..=(tip + 4).min(top) { + let block = miner.query.reconstruct_block_at_height(Height(h)).unwrap(); + let (rec, txs) = + rbitcoin_query::prepare_block_for_archive(&query, ¶ms, &block).unwrap(); + query.commit_class_a_only(&rec, &txs).unwrap(); + } + query.flush().unwrap(); + assert_eq!(query.tip_height(), Some(Height(tip))); +} + async fn follow_journey() { let (blocks, script_a, script_b) = fixture_chain(); let miner_dir = TestDatadir::new().unwrap(); From eed8b3f5e9912dc0edbbb364df588f7ef4d80213 Mon Sep 17 00:00:00 2001 From: Nitesh Balusu Date: Sun, 18 Oct 2026 07:08:33 -0400 Subject: [PATCH 5/6] docs: Class A can lead the tip after a crash Review: crash-recovery.md said Class A never leads tip. A crash between the Class A write and the tip advance leaves bodies above it. Co-Authored-By: Claude Opus 5.5 --- docs/crash-recovery.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/crash-recovery.md b/docs/crash-recovery.md index ab778ae232..43ae522c2d 100644 --- a/docs/crash-recovery.md +++ b/docs/crash-recovery.md @@ -53,7 +53,7 @@ Open revalidation runs in `Query::open_or_create` **before** P2P can extend tip. ## Class A (archive) - Append-oriented; re-archive is **idempotent** when `header_txs` already present. -- **Never leads tip:** Class A is published only on the confirm-write path with tip advance in the same era (no dual-track archive-ahead). +- **Never leads tip** on a clean run: Class A is published only on the confirm-write path with tip advance in the same era (no dual-track archive-ahead). A crash between the Class A write and the tip advance can leave bodies above the tip. IBD gets them again through the tip-hole cover. - Kill mid-archive without a complete body association ⇒ not treated as archived; re-getdata. ## Spends (v5: annotation on create outputs) From 2bc94cf666bac837f238fd8745aa8ba67c242727 Mon Sep 17 00:00:00 2001 From: Nitesh Balusu Date: Sun, 18 Oct 2026 08:57:42 -0400 Subject: [PATCH 6/6] test: crash with an archived same-batch spend above the tip The end-of-IBD journey now plants two Class A blocks above a catch-up-only tip. The second spends a create of the first and one of its own. The restart must fetch both and connect them. Docs: Class A never leads tip on a clean run; point the crash exception at crash-recovery.md. Co-Authored-By: Claude Opus 5.5 --- crates/rbitcoin-test/tests/end_of_ibd.rs | 100 +++++++++++++++++++---- docs/architecture.md | 8 +- docs/concurrency.md | 2 +- 3 files changed, 92 insertions(+), 18 deletions(-) diff --git a/crates/rbitcoin-test/tests/end_of_ibd.rs b/crates/rbitcoin-test/tests/end_of_ibd.rs index 1693f856ef..a98e41c28c 100644 --- a/crates/rbitcoin-test/tests/end_of_ibd.rs +++ b/crates/rbitcoin-test/tests/end_of_ibd.rs @@ -1,6 +1,7 @@ //! End of IBD: tip follow, scripthash history, and restart on a live miner. use bitcoin::absolute::LockTime; +use bitcoin::hashes::Hash; use bitcoin::script::ScriptBuf; use bitcoin::transaction::Version as TxVersion; use bitcoin::{ @@ -500,7 +501,7 @@ async fn finish_partial(dir: &Path, miner: SocketAddr, synced: &Synced) { stop_run_p2p(rpc, node).await; } -async fn partial_with_miner_down(miner_dir: &Path, synced: &Synced) { +async fn partial_with_miner_down(miner_dir: &Path, synced: &mut Synced) { let miner = start_miner(miner_dir).await; let miner_addr = miner.local_addr; assert_eq!(miner.tip_height().unwrap(), synced.height); @@ -524,25 +525,96 @@ async fn partial_with_miner_down(miner_dir: &Path, synced: &Synced) { let miner = start_miner(miner_dir).await; finish_partial(partial_dir.path().as_path(), miner.local_addr, synced).await; - archive_above_tip(&held_dir.store_path(), &miner, held_tip, synced.height); finish_partial(held_dir.path().as_path(), miner.local_addr, synced).await; + crash_with_class_a_above_tip(&miner, synced).await; miner.shutdown().await; } /// A crash between the Class A write and the tip advance leaves bodies above -/// the tip with no connect. The next start must fetch them again. -fn archive_above_tip(store: &Path, miner: &P2PNode, tip: u32, top: u32) { - use rbitcoin_query::testutil::FixtureChain; - let query = Query::open_or_create_tiny(store).unwrap(); - let params = ChainParams::regtest(); - for h in tip + 1..=(tip + 4).min(top) { - let block = miner.query.reconstruct_block_at_height(Height(h)).unwrap(); - let (rec, txs) = - rbitcoin_query::prepare_block_for_archive(&query, ¶ms, &block).unwrap(); - query.commit_class_a_only(&rec, &txs).unwrap(); +/// the tip with no connect. The second block spends a create of the first and +/// one of its own, so the next start must fetch both and pin those parents +/// from the archive. The store holds catch-up indexes, as a crash in IBD does, +/// so a halted IBD has no tip-follow to fall back on. +async fn crash_with_class_a_above_tip(miner: &P2PNode, synced: &mut Synced) { + let dir = TestDatadir::new().unwrap(); + let query = Query::open_or_create_tiny(dir.store_path()).unwrap(); + let node = P2PNode::start(any_port(), query, ChainParams::regtest(), Milestone::NONE) + .await + .expect("crash syncer"); + let peers = [NetAddr::Ip(miner.local_addr)]; + node.sync(&peers, IbdConfig::for_test()) + .await + .expect("crash syncer ibd"); + node.shutdown().await; + + let op_true = ScriptBuf::from_bytes(vec![0x51]); + let matured = miner + .query + .reconstruct_block_at_height(Height(2)) + .unwrap() + .txdata[0] + .compute_txid(); + let parent = spend_to(matured, Amount::from_sat(49_0000_0000), vec![0x51]); + let child = spend_to( + parent.compute_txid(), + Amount::from_sat(48_0000_0000), + vec![0x51], + ); + let grandchild = spend_to( + child.compute_txid(), + Amount::from_sat(47_0000_0000), + vec![0x51], + ); + let mut blocks = Vec::new(); + for txs in [vec![parent], vec![child, grandchild]] { + miner + .hub + .generate_to_script(1, op_true.clone(), txs) + .unwrap(); + let block = miner + .query + .reconstruct_block_at_height(Height(miner.tip_height().unwrap())); + blocks.push(Arc::new(block.unwrap())); + } + + let query = Query::open_or_create_tiny(dir.store_path()).unwrap(); + let tip = query.tip_height(); + assert_eq!(tip, Some(Height(synced.height))); + let mut prev_fk = query + .get_header_by_hash(blocks[0].header.prev_blockhash.as_byte_array()) + .unwrap() + .unwrap() + .0; + let mut fks = Vec::new(); + let mut txids = Vec::new(); + for block in &blocks { + let hash = block.block_hash().to_byte_array(); + let rec = rbitcoin_query::header_to_record(prev_fk, &block.header, hash); + prev_fk = query.ensure_header(&rec).unwrap(); + fks.push(prev_fk); + let ids: Vec<[u8; 32]> = block + .txdata + .iter() + .map(|t| t.compute_txid().to_byte_array()) + .collect(); + synced + .script_a + .extend(block.txdata.iter().map(|t| t.compute_txid().to_string())); + txids.push(ids); } + let items: Vec<_> = blocks + .iter() + .zip(&fks) + .zip(&txids) + .map(|((b, fk), ids)| (*fk, b, ids.as_slice())) + .collect(); + query.archive_class_a_from_wire(&items).unwrap(); query.flush().unwrap(); - assert_eq!(query.tip_height(), Some(Height(tip))); + assert_eq!(query.tip_height(), tip); + synced.height = miner.tip_height().unwrap(); + synced.hash = miner.hub.tip_hash().unwrap().to_string(); + drop(query); + finish_partial(dir.path().as_path(), miner.local_addr, synced).await; } async fn follow_journey() { @@ -560,7 +632,7 @@ async fn follow_journey() { assert_eq!(synced.height as usize, blocks.len() - 1); ibd_then_restart(miner, miner_addr, &mut synced).await; - partial_with_miner_down(miner_dir.path().as_path(), &synced).await; + partial_with_miner_down(miner_dir.path().as_path(), &mut synced).await; } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] diff --git a/docs/architecture.md b/docs/architecture.md index 416aa94391..831c90ae60 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -36,9 +36,11 @@ appender. Roles: [`concurrency.md`](./concurrency.md). Stage IO: **Invariant — Class A never leads tip:** there is no dual-track “archive Class A far ahead of confirmed tip.” Wire plan + Class A append + Class C tip advance -are one confirm-write era. Do not reintroduce plan-time “archive lead” heuristics -(e.g. body `posix_fadvise(DONTNEED)` when body count ≫ tip) — under this path -just-written body pages stay tip-hot. **No** ContigPark / archive-job fallback +are one confirm-write era on a clean run. A crash inside that era is the +exception ([crash-recovery.md](./crash-recovery.md#class-a-archive)). Do not +reintroduce plan-time “archive lead” heuristics (e.g. body +`posix_fadvise(DONTNEED)` when body count ≫ tip) — under this path just-written +body pages stay tip-hot. **No** ContigPark / archive-job fallback for unknown-height bodies (mark missing → re-getdata). - **Storage center** is a **transaction-relational archive** on **map-free diff --git a/docs/concurrency.md b/docs/concurrency.md index 03e6ef2229..dbd13605ca 100644 --- a/docs/concurrency.md +++ b/docs/concurrency.md @@ -10,7 +10,7 @@ Short map of who may write which tables. **Format is unstable until 1.0.** | Confirm **lookup** | 1 OS thread | pack/hold from BQ stamped Σ inputs; emit: decode BQ raw + TipOnly `head_fk` + **`take_raw` onto loadq** (`Arc` + `TxPrecompute`). Does **not** plan_batch / structure | | Confirm **load** | 1 OS thread | `confirm_wire_lookup_stamp` (structure with **loadq `pres`**, plan_batch binds carried BQ keys, skeleton bind) + pin `txout` + assemble | | Confirm **scripts** | 1 OS thread (`ibd-confirm`) publishes waves + `rbtc-scripts` steal (lock-free claim). The interpreter runs here, not on a peer IO task (`verify_tx_scripts_detached`) | **none** — pure CPU | -| Confirm **write** | 1 OS thread (`ibd-confirm-write`) + 1 process-wide `ibd-confirm-head` | **sole Class A appender** (`txout`+`seqsigwit`+`spent`; IBD encodes ins from `Arc` + SpendEdges) + structural + Class C + spend annotate on **`spent.body`** + tip GC; **`block_queue_dequeue_height`**. `tx.head` write-behind insert runs on **`ibd-confirm-head`** (not a per-batch spawn). `--datadir-hdd` starts that insert after spend annotate; otherwise it overlaps structural and Class C. Class A **never leads tip** (same commit era; no archive-ahead DONTNEED). When `index_live` is set, this thread appends that index after `confirmed[h]` is set (one put per confirm batch) and does not read Class A for it. Startup `prepare_live_indexes` seals a restart gap of at most one write drain (14 × 144 heights) before confirm; a larger gap is left for `rbtc-idx-wb` | +| Confirm **write** | 1 OS thread (`ibd-confirm-write`) + 1 process-wide `ibd-confirm-head` | **sole Class A appender** (`txout`+`seqsigwit`+`spent`; IBD encodes ins from `Arc` + SpendEdges) + structural + Class C + spend annotate on **`spent.body`** + tip GC; **`block_queue_dequeue_height`**. `tx.head` write-behind insert runs on **`ibd-confirm-head`** (not a per-batch spawn). `--datadir-hdd` starts that insert after spend annotate; otherwise it overlaps structural and Class C. Class A **never leads tip** (same commit era on a clean run; no archive-ahead DONTNEED). A crash inside the era can leave it above the tip ([crash-recovery.md](./crash-recovery.md#class-a-archive)). When `index_live` is set, this thread appends that index after `confirmed[h]` is set (one put per confirm batch) and does not read Class A for it. Startup `prepare_live_indexes` seals a restart gap of at most one write drain (14 × 144 heights) before confirm; a larger gap is left for `rbtc-idx-wb` | | IBD main loop | 1 tokio task | none (orchestration only). Event drain + confirm-offer every turn; getdata assign ≤50 ms (immediate if inflight empty); main-loop `getheaders`/`locator_hashes` ≤500 ms (empty path immediate); peer stall/relative-slow and work-path hygiene ≤1 s. Full-batch header continuation stays on the Headers event. Tip hash is the snapshot confirm published with `confirmed[]` (`published_tip`); each admitted header's parent stays on the work state. Those two reads are not store IO. The write holds `tip_commit` from the tip-height check through `flush_class_c_tip`. IBD rewind tries that lock and retries on the next pass. | **IoSession TLS:** one completion session per OS thread (`with_thread_local`). Harvest / poison / drain / do-not-flatten: [`io-modality.md`](./io-modality.md). `RBITCOIN_IO=pread` disables the session. SH k-way merge submits 256 KiB ahead preads on that TLS session and waits only when promote needs a page that has not completed.