From be2d297305c8a93f3e51bb88394b7d5869fb7e92 Mon Sep 17 00:00:00 2001 From: Nitesh Balusu Date: Sun, 18 Oct 2026 04:51:23 -0400 Subject: [PATCH 1/3] ibd: pin a same-batch parent on an archived retry A batch with an invalid block is retried from its archived bodies. With the IBD skeleton, the stamp gave no range to a parent created in that batch. Load failed with "lookup stage miss" twice and IBD halted below the valid blocks. Fill ranges for the batch's own creates on that retry. Co-Authored-By: Claude Opus 5.5 --- .../archived-retry-same-batch-parent.md | 7 +++ .../src/ibd/events/confirm_reject_tests.rs | 52 +++++++++++++++++++ .../rbitcoin-query/src/confirm_run/lookup.rs | 11 ++++ docs/invariants.md | 2 +- 4 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 changelog.d/archived-retry-same-batch-parent.md diff --git a/changelog.d/archived-retry-same-batch-parent.md b/changelog.d/archived-retry-same-batch-parent.md new file mode 100644 index 0000000000..b914d1554d --- /dev/null +++ b/changelog.d/archived-retry-same-batch-parent.md @@ -0,0 +1,7 @@ +Fixed + +- **IBD connects archived blocks that spend a create from the same batch.** + A retry from archived bodies found no range for a parent created in that + batch, logged `lookup stage miss`, and halted IBD below valid blocks. The + retry follows an invalid block in a batch, or a restart after a crash + between the Class A write and the tip advance. diff --git a/crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs b/crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs index 7a5ef0b38f..2b5450abec 100644 --- a/crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs +++ b/crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs @@ -2678,6 +2678,58 @@ fn batched_spend_of_a_later_block_is_rejected_alone() { rig.finish(); } +/// A wave whose first block spends a create from its own block and whose +/// second block is invalid at write. The cascade retries the first block from +/// the archive, and that retry must still pin the same-block parent. +#[test] +fn archived_retry_pins_a_same_block_parent() { + use super::super::assign::tests::lock_default_assign_stop; + use super::super::state::InflightReq; + use bitcoin::{Amount, ScriptBuf}; + use rbitcoin_consensus::{grind_regtest_pow, mine_regtest_paying}; + + let _env = lock_default_assign_stop(); + let mut rig = WireRig::new("archived-same-block", 2); + let (t, cbs) = (rig.t, rig.cbs.clone()); + let spk = ScriptBuf::from_bytes(vec![0x51]); + let parent = WireRig::spend(cbs[0]); + let child = WireRig::spend(parent.compute_txid()); + let b1 = mine_regtest_paying( + rig.tip, + rig.tip_time + 600, + t + 1, + spk.clone(), + vec![parent, child], + ); + let mut b2 = mine_regtest_paying(b1.block_hash(), rig.tip_time + 1200, t + 2, spk, vec![]); + b2.txdata[0].output[0].value += Amount::from_sat(1); + b2.header.merkle_root = b2.compute_merkle_root().unwrap(); + grind_regtest_pow(&mut b2.header); + let (h1, h2) = (b1.block_hash(), b2.block_hash()); + rig.plant(&[&b1, &b2]); + for (peer, body) in [(1, &b1), (2, &b2)] { + let hash = body.block_hash(); + rig.st.slots[peer - 1].in_flight.insert(hash); + rig.st.inflight.insert(hash, InflightReq::new(peer)); + rig.deliver(peer, body); + } + rig.start_engine(); + + let rejects = rig.pump( + |_, hash| if hash == h1 { b1.clone() } else { b2.clone() }, + |st, _, _| st.halt.is_some() || st.reorg.invalid.contains(h2.to_byte_array()), + ); + assert_eq!( + rig.st.halt, None, + "rejects={rejects:?} last={:?}", + rig.last_reject_err + ); + assert!(!rig.st.engine_fault_seen.contains(&h1), "{rejects:?}"); + assert!(rig.st.reorg.invalid.contains(h2.to_byte_array())); + assert_eq!(rig.hub.tip_height(), Some(t + 1), "{rejects:?}"); + rig.finish(); +} + /// tip+1 is the last block on the path, and load cannot read its header /// row when it stamps the wave: a local store fault, not a block verdict. /// The wave goes back to lookup, so once the row reads again tip+1 diff --git a/crates/rbitcoin-query/src/confirm_run/lookup.rs b/crates/rbitcoin-query/src/confirm_run/lookup.rs index 60641af3dd..c7142b106e 100644 --- a/crates/rbitcoin-query/src/confirm_run/lookup.rs +++ b/crates/rbitcoin-query/src/confirm_run/lookup.rs @@ -366,6 +366,17 @@ pub(super) fn stamp_parent_pin_archived( query.confirm_stats(), ) .map_err(ConfirmError::from)?; + } else { + // The IBD skeleton carries external ranges. This batch's own creates + // are already archived (plan=None), so only the store has theirs. + let mut own: U64Map = metas + .iter() + .flat_map(|m| m.tx_fks.iter().filter_map(|f| f.get())) + .filter_map(|id| stamp.idents.remove_entry(&id)) + .collect(); + crate::fill_missing_parent_ranges(query.store(), ifo, &mut own, query.confirm_stats()) + .map_err(ConfirmError::from)?; + stamp.idents.extend(own); } for ident in stamp.idents.values() { if ident.txid == [0u8; 32] { diff --git a/docs/invariants.md b/docs/invariants.md index 54fd054948..458fb996a0 100644 --- a/docs/invariants.md +++ b/docs/invariants.md @@ -81,7 +81,7 @@ the Wire CreatePin (`Arc` + tx index). Load still does not head/idx. | Stage | Invariant | Soft path allowed? | |-------|-----------|--------------------| -| Lookup parent stamp | Every external spent parent has create_fk + body_range (or offline in_flight CreatePin) + reverse txid. Archived parents also have computed `spent.body` range on the stamp (same-wave in-flight outs skip loc, miss OK; later-wave InFlight takes loc from the same `CreatePin` Arc write set at Class A, or skeleton TipOnly loc). IBD (`skeleton = Some`) never loc-by-fk: pin loc unset leaves spent unset for write TLS. plan=None leftover still `fill_missing_parent_ranges` | Missing → hard Err at stamp / pin contract. **No** `finish_archive_plan` `fill_missing` after stamp loc. **No** IBD load `create.loc` | +| Lookup parent stamp | Every external spent parent has create_fk + body_range (or offline in_flight CreatePin) + reverse txid. Archived parents also have computed `spent.body` range on the stamp (same-wave in-flight outs skip loc, miss OK; later-wave InFlight takes loc from the same `CreatePin` Arc write set at Class A, or skeleton TipOnly loc). IBD (`skeleton = Some`) never loc-by-fk: pin loc unset leaves spent unset for write TLS. The one exception is an all-archived retry (plan=None), whose own creates only the store can range. plan=None leftover still `fill_missing_parent_ranges` | Missing → hard Err at stamp / pin contract. **No** `finish_archive_plan` `fill_missing` after stamp loc. **No** IBD load `create.loc` | | Parent create_fk | **same-batch** planned fks (offline at pin) → **in-flight** (lookup snapshots `drain_and_fence_hi` **before** the wave's TipOnly read and passes it on the last load batch; load drops tagged map rows with pack height **below** that snapshot after that batch's in-flight read; equality keeps; not Class C tip, not `class_a_hi`, not write freeze; one load-thread HashMap, insert after stamp) → **skeleton** (`BatchParentIds` on the `LoadBatch`: lookup TipOnly fk + body_range + spent_range + per-chunk need-vouts) → **Corrupt** on IBD miss. plan=None / S0 (`skeleton = None`) is in-flight → leftover TipOnly. One helper: [`stamp_external_parents`](../crates/rbitcoin-query/src/stamp.rs). No leftover pending map, no process pin FIFO, no BQ-side hits map, no parent-store create_fk on stamp, no published live_union. Same-wave creates are omitted from TipOnly need. Header-cache GC polls store tip each load pack. In-flight and skeleton maps stay one fk per txid (last write). That limit is only those RAM maps ([`errata.md`](./errata.md)), not a consensus rule for this write. | Miss of in-flight and skeleton → `Corrupt("parent create_fk unresolved")`. A batched miss isolates; a lone block's miss is an **engine fault** (requeue once, then halt IBD; never blacklist), unless the block extends the tip, `tx.head` holds every connected create (`Query::head_covers_fence`; open starts the drain at the head's last create, after its rebuild or tail backfill), and a fresh TipOnly read of its parents on the load thread (reject path only) still misses: that is `ConsensusInvalid`, unless `txid.body` holds the txid at a connected fk in an unsealed `tx.head` segment (the open tail or the in-flight seal), whose slot pages may hold a power-loss hole (`Corrupt("invariant: tx.head misses a connected create")`, engine fault) ([089](./external_findings/089-missing-parent-verdict.md)). A lone block whose parent is not the tip within 10 s is a `Cascade` retry. Identity without loc range → `Corrupt("invariant: loc range missing after identity")`, not a miss | | io_uring harvest | TLS session fail-closed ([`io-modality.md`](./io-modality.md)) | **No** silent success. `Corrupt("invariant: io_uring …")` (not `bdz g page bad slot`). Ring-unavailable still pread-fallback | | Load body outs | By `txout` range only from lookup stamp; incomplete outs → hard Err. Pin **copies** lookup `spent_range` (no loc IO). Later-wave InFlight loc comes from the CreatePin Arc (Class A `set_loc`) | **No** loc cold outs on load; **no** `seqsigwit` on pin; **no** IBD loc-by-fk | From d676fe134c091ca3e326573ee6ef5587f38bece1 Mon Sep 17 00:00:00 2001 From: Nitesh Balusu Date: Sun, 18 Oct 2026 06:55:00 -0400 Subject: [PATCH 2/3] test: assert the archived-retry wave is a cascade Review nit: the test should fail if a batching change stops putting both blocks in one wave. Co-Authored-By: Claude Opus 5.5 --- crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs b/crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs index 2b5450abec..315328278f 100644 --- a/crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs +++ b/crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs @@ -2724,6 +2724,10 @@ fn archived_retry_pins_a_same_block_parent() { "rejects={rejects:?} last={:?}", rig.last_reject_err ); + assert!( + rejects.contains(&(h1, ConfirmRejectClass::Cascade, 2)), + "the wave must be retried from the archive: {rejects:?}" + ); assert!(!rig.st.engine_fault_seen.contains(&h1), "{rejects:?}"); assert!(rig.st.reorg.invalid.contains(h2.to_byte_array())); assert_eq!(rig.hub.tip_height(), Some(t + 1), "{rejects:?}"); From 544a16f5f79dfd75d39628bbfc34ac920db10da3 Mon Sep 17 00:00:00 2001 From: Nitesh Balusu Date: Sun, 18 Oct 2026 08:48:11 -0400 Subject: [PATCH 3/3] ibd: pin a parent from an earlier chunk of an archived wave Review: a wave longer than one load batch notes chunk 1's archived creates in flight with no outs. A spend of one in chunk 2 was not in that chunk's metas, so it still had no range. Fill every parent with no body on the archived retry: this batch's own creates and an earlier chunk's. Skeleton parents keep their body and in-flight pins are still skipped. Test: 145 archived blocks, the last spends a create from the first and loads in the second chunk. The stamp.rs module note and the invariants row name the exception. Co-Authored-By: Claude Opus 5.5 --- .../archived-retry-same-batch-parent.md | 11 +-- .../src/ibd/events/confirm_reject_tests.rs | 85 ++++++++++++++++++- .../rbitcoin-query/src/confirm_run/lookup.rs | 17 ++-- crates/rbitcoin-query/src/stamp.rs | 6 +- docs/invariants.md | 2 +- 5 files changed, 103 insertions(+), 18 deletions(-) diff --git a/changelog.d/archived-retry-same-batch-parent.md b/changelog.d/archived-retry-same-batch-parent.md index b914d1554d..4acfc8a641 100644 --- a/changelog.d/archived-retry-same-batch-parent.md +++ b/changelog.d/archived-retry-same-batch-parent.md @@ -1,7 +1,8 @@ Fixed -- **IBD connects archived blocks that spend a create from the same batch.** - A retry from archived bodies found no range for a parent created in that - batch, logged `lookup stage miss`, and halted IBD below valid blocks. The - retry follows an invalid block in a batch, or a restart after a crash - between the Class A write and the tip advance. +- **IBD connects archived blocks that spend an archived create.** A retry + from archived bodies found no range for a parent created in the same batch, + or in an earlier chunk of the same wave. It logged `lookup stage miss` and + halted IBD below valid blocks. The retry follows an invalid block in a + batch, or a restart after a crash between the Class A write and the tip + advance. diff --git a/crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs b/crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs index 315328278f..4035c741cf 100644 --- a/crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs +++ b/crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs @@ -2726,7 +2726,7 @@ fn archived_retry_pins_a_same_block_parent() { ); assert!( rejects.contains(&(h1, ConfirmRejectClass::Cascade, 2)), - "the wave must be retried from the archive: {rejects:?}" + "both blocks in one wave: {rejects:?}" ); assert!(!rig.st.engine_fault_seen.contains(&h1), "{rejects:?}"); assert!(rig.st.reorg.invalid.contains(h2.to_byte_array())); @@ -2734,6 +2734,89 @@ fn archived_retry_pins_a_same_block_parent() { rig.finish(); } +/// Archived blocks above the tip, as a crash between the Class A write and +/// the tip advance leaves them. The wave is longer than one load batch, so +/// the last block loads in the next chunk and spends a create from the +/// first, noted in flight with no outs. +#[test] +fn archived_wave_pins_a_parent_from_the_previous_chunk() { + use super::super::assign::tests::lock_default_assign_stop; + use super::super::state::InflightReq; + use bitcoin::ScriptBuf; + use rbitcoin_consensus::mine_regtest_paying; + + let _env = lock_default_assign_stop(); + let mut rig = WireRig::new("archived-cross-chunk", 1); + let (t, cbs) = (rig.t, rig.cbs.clone()); + let op_true = ScriptBuf::from_bytes(vec![0x51]); + let n = super::super::confirm::CONFIRM_RUN_MAX_BLOCKS as u32 + 1; + let parent = WireRig::spend(cbs[0]); + let mut child = Some(WireRig::spend(parent.compute_txid())); + let mut extra = vec![parent]; + let mut blocks: Vec = Vec::new(); + for i in 1..=n { + let prev = blocks.last().map_or(rig.tip, |b| b.block_hash()); + if i == n { + extra = child.take().into_iter().collect(); + } + let b = mine_regtest_paying( + prev, + rig.tip_time + 600 * i, + t + i, + op_true.clone(), + std::mem::take(&mut extra), + ); + blocks.push(b); + } + // One Class A write for the run, as the write stage does, with no tip advance. + let bodies: Vec<_> = blocks.iter().cloned().map(std::sync::Arc::new).collect(); + let fks: Vec<_> = bodies + .iter() + .map(|b| rig.hub.ensure_header_fk(&b.header).unwrap()) + .collect(); + let txids: Vec> = bodies + .iter() + .map(|b| { + b.txdata + .iter() + .map(|t| t.compute_txid().to_byte_array()) + .collect() + }) + .collect(); + let items: Vec<_> = bodies + .iter() + .zip(&fks) + .zip(&txids) + .map(|((b, fk), ids)| (*fk, b, ids.as_slice())) + .collect(); + rig.hub.query.archive_class_a_from_wire(&items).unwrap(); + assert_eq!(rig.hub.tip_height(), Some(t)); + let refs: Vec<&bitcoin::Block> = blocks.iter().collect(); + rig.plant(&refs); + for body in &blocks { + let hash = body.block_hash(); + rig.st.slots[0].in_flight.insert(hash); + rig.st.inflight.insert(hash, InflightReq::new(1)); + rig.deliver(1, body); + } + rig.start_engine(); + + let last = blocks.last().unwrap().block_hash(); + let by_hash: std::collections::HashMap<_, _> = + blocks.iter().map(|b| (b.block_hash(), b.clone())).collect(); + let rejects = rig.pump( + |_, hash| by_hash[&hash].clone(), + |st, hub, _| st.halt.is_some() || hub.tip_hash() == Some(last), + ); + assert_eq!( + rig.st.halt, None, + "rejects={rejects:?} last={:?}", + rig.last_reject_err + ); + assert_eq!(rig.hub.tip_height(), Some(t + n), "{rejects:?}"); + rig.finish(); +} + /// tip+1 is the last block on the path, and load cannot read its header /// row when it stamps the wave: a local store fault, not a block verdict. /// The wave goes back to lookup, so once the row reads again tip+1 diff --git a/crates/rbitcoin-query/src/confirm_run/lookup.rs b/crates/rbitcoin-query/src/confirm_run/lookup.rs index c7142b106e..1a543bbce8 100644 --- a/crates/rbitcoin-query/src/confirm_run/lookup.rs +++ b/crates/rbitcoin-query/src/confirm_run/lookup.rs @@ -367,16 +367,15 @@ pub(super) fn stamp_parent_pin_archived( ) .map_err(ConfirmError::from)?; } else { - // The IBD skeleton carries external ranges. This batch's own creates - // are already archived (plan=None), so only the store has theirs. - let mut own: U64Map = metas - .iter() - .flat_map(|m| m.tx_fks.iter().filter_map(|f| f.get())) - .filter_map(|id| stamp.idents.remove_entry(&id)) - .collect(); - crate::fill_missing_parent_ranges(query.store(), ifo, &mut own, query.confirm_stats()) + // The IBD skeleton carries external ranges. A parent with no body is + // an archived create: this batch's own, or an earlier chunk's of the + // same wave (noted in flight with no outs). Only the store has its + // range. In-flight pins are skipped inside the fill. + let mut archived: U64Map = + stamp.idents.extract_if(|_, e| e.body.is_none()).collect(); + crate::fill_missing_parent_ranges(query.store(), ifo, &mut archived, query.confirm_stats()) .map_err(ConfirmError::from)?; - stamp.idents.extend(own); + stamp.idents.extend(archived); } for ident in stamp.idents.values() { if ident.txid == [0u8; 32] { diff --git a/crates/rbitcoin-query/src/stamp.rs b/crates/rbitcoin-query/src/stamp.rs index bb7153a212..e5ecdc5e9e 100644 --- a/crates/rbitcoin-query/src/stamp.rs +++ b/crates/rbitcoin-query/src/stamp.rs @@ -6,8 +6,10 @@ //! rehydrate. In-flight holds CreatePins until load drops map rows below a //! lookup-wave drain+fence snapshot taken before TipOnly. Same-wave creates are //! omitted from that skeleton. Class A `CreatePin::set_loc` after append; -//! later-wave stamp reads pin loc. IBD (`skeleton = Some`) never loc-by-fk: -//! spent is pin loc, skeleton TipOnly loc, or unset for write fill. +//! later-wave stamp reads pin loc. IBD (`skeleton = Some`) never loc-by-fk +//! here: spent is pin loc, skeleton TipOnly loc, or unset for write fill. The +//! all-archived retry (`stamp_parent_pin_archived`) loc-fills its archived +//! parents that have no body. External IBD parents still must not. use crate::{CreatePin, InFlight, QueryError, U64Map}; use crate::{IdMap, TxidHasher}; diff --git a/docs/invariants.md b/docs/invariants.md index 458fb996a0..3918553479 100644 --- a/docs/invariants.md +++ b/docs/invariants.md @@ -81,7 +81,7 @@ the Wire CreatePin (`Arc` + tx index). Load still does not head/idx. | Stage | Invariant | Soft path allowed? | |-------|-----------|--------------------| -| Lookup parent stamp | Every external spent parent has create_fk + body_range (or offline in_flight CreatePin) + reverse txid. Archived parents also have computed `spent.body` range on the stamp (same-wave in-flight outs skip loc, miss OK; later-wave InFlight takes loc from the same `CreatePin` Arc write set at Class A, or skeleton TipOnly loc). IBD (`skeleton = Some`) never loc-by-fk: pin loc unset leaves spent unset for write TLS. The one exception is an all-archived retry (plan=None), whose own creates only the store can range. plan=None leftover still `fill_missing_parent_ranges` | Missing → hard Err at stamp / pin contract. **No** `finish_archive_plan` `fill_missing` after stamp loc. **No** IBD load `create.loc` | +| Lookup parent stamp | Every external spent parent has create_fk + body_range (or offline in_flight CreatePin) + reverse txid. Archived parents also have computed `spent.body` range on the stamp (same-wave in-flight outs skip loc, miss OK; later-wave InFlight takes loc from the same `CreatePin` Arc write set at Class A, or skeleton TipOnly loc). IBD (`skeleton = Some`) never loc-by-fk: pin loc unset leaves spent unset for write TLS. The one exception is an all-archived retry (plan=None): a parent with no body is an archived create (this batch's own, or an earlier chunk's of the same wave), and only the store can range it. plan=None leftover still `fill_missing_parent_ranges` | Missing → hard Err at stamp / pin contract. **No** `finish_archive_plan` `fill_missing` after stamp loc. **No** IBD load `create.loc` | | Parent create_fk | **same-batch** planned fks (offline at pin) → **in-flight** (lookup snapshots `drain_and_fence_hi` **before** the wave's TipOnly read and passes it on the last load batch; load drops tagged map rows with pack height **below** that snapshot after that batch's in-flight read; equality keeps; not Class C tip, not `class_a_hi`, not write freeze; one load-thread HashMap, insert after stamp) → **skeleton** (`BatchParentIds` on the `LoadBatch`: lookup TipOnly fk + body_range + spent_range + per-chunk need-vouts) → **Corrupt** on IBD miss. plan=None / S0 (`skeleton = None`) is in-flight → leftover TipOnly. One helper: [`stamp_external_parents`](../crates/rbitcoin-query/src/stamp.rs). No leftover pending map, no process pin FIFO, no BQ-side hits map, no parent-store create_fk on stamp, no published live_union. Same-wave creates are omitted from TipOnly need. Header-cache GC polls store tip each load pack. In-flight and skeleton maps stay one fk per txid (last write). That limit is only those RAM maps ([`errata.md`](./errata.md)), not a consensus rule for this write. | Miss of in-flight and skeleton → `Corrupt("parent create_fk unresolved")`. A batched miss isolates; a lone block's miss is an **engine fault** (requeue once, then halt IBD; never blacklist), unless the block extends the tip, `tx.head` holds every connected create (`Query::head_covers_fence`; open starts the drain at the head's last create, after its rebuild or tail backfill), and a fresh TipOnly read of its parents on the load thread (reject path only) still misses: that is `ConsensusInvalid`, unless `txid.body` holds the txid at a connected fk in an unsealed `tx.head` segment (the open tail or the in-flight seal), whose slot pages may hold a power-loss hole (`Corrupt("invariant: tx.head misses a connected create")`, engine fault) ([089](./external_findings/089-missing-parent-verdict.md)). A lone block whose parent is not the tip within 10 s is a `Cascade` retry. Identity without loc range → `Corrupt("invariant: loc range missing after identity")`, not a miss | | io_uring harvest | TLS session fail-closed ([`io-modality.md`](./io-modality.md)) | **No** silent success. `Corrupt("invariant: io_uring …")` (not `bdz g page bad slot`). Ring-unavailable still pread-fallback | | Load body outs | By `txout` range only from lookup stamp; incomplete outs → hard Err. Pin **copies** lookup `spent_range` (no loc IO). Later-wave InFlight loc comes from the CreatePin Arc (Class A `set_loc`) | **No** loc cold outs on load; **no** `seqsigwit` on pin; **no** IBD loc-by-fk |