ibd: connect archived blocks that spend a same-batch create

changelog.d/archived-retry-same-batch-parent.md

@@ -0,0 +1,8 @@
+Fixed
+
+- **IBD connects archived blocks that spend an archived create.** A retry
+  from archived bodies found no range for a parent created in the same batch,
+  or in an earlier chunk of the same wave. It logged `lookup stage miss` and
+  halted IBD below valid blocks. The retry follows an invalid block in a
+  batch, or a restart after a crash between the Class A write and the tip
+  advance.

crates/rbitcoin-net/src/ibd/events/confirm_reject_tests.rs

@@ -2678,6 +2678,145 @@ fn batched_spend_of_a_later_block_is_rejected_alone() {
     rig.finish();
 }
 
+/// A wave whose first block spends a create from its own block and whose
+/// second block is invalid at write. The cascade retries the first block from
+/// the archive, and that retry must still pin the same-block parent.
+#[test]
+fn archived_retry_pins_a_same_block_parent() {
+    use super::super::assign::tests::lock_default_assign_stop;
+    use super::super::state::InflightReq;
+    use bitcoin::{Amount, ScriptBuf};
+    use rbitcoin_consensus::{grind_regtest_pow, mine_regtest_paying};
+
+    let _env = lock_default_assign_stop();
+    let mut rig = WireRig::new("archived-same-block", 2);
+    let (t, cbs) = (rig.t, rig.cbs.clone());
+    let spk = ScriptBuf::from_bytes(vec![0x51]);
+    let parent = WireRig::spend(cbs[0]);
+    let child = WireRig::spend(parent.compute_txid());
+    let b1 = mine_regtest_paying(
+        rig.tip,
+        rig.tip_time + 600,
+        t + 1,
+        spk.clone(),
+        vec![parent, child],
+    );
+    let mut b2 = mine_regtest_paying(b1.block_hash(), rig.tip_time + 1200, t + 2, spk, vec![]);
+    b2.txdata[0].output[0].value += Amount::from_sat(1);
+    b2.header.merkle_root = b2.compute_merkle_root().unwrap();
+    grind_regtest_pow(&mut b2.header);
+    let (h1, h2) = (b1.block_hash(), b2.block_hash());
+    rig.plant(&[&b1, &b2]);
+    for (peer, body) in [(1, &b1), (2, &b2)] {
+        let hash = body.block_hash();
+        rig.st.slots[peer - 1].in_flight.insert(hash);
+        rig.st.inflight.insert(hash, InflightReq::new(peer));
+        rig.deliver(peer, body);
+    }
+    rig.start_engine();
+
+    let rejects = rig.pump(
+        |_, hash| if hash == h1 { b1.clone() } else { b2.clone() },
+        |st, _, _| st.halt.is_some() || st.reorg.invalid.contains(h2.to_byte_array()),
+    );
+    assert_eq!(
+        rig.st.halt, None,
+        "rejects={rejects:?} last={:?}",
+        rig.last_reject_err
+    );
+    assert!(
+        rejects.contains(&(h1, ConfirmRejectClass::Cascade, 2)),
+        "both blocks in one wave: {rejects:?}"
+    );
+    assert!(!rig.st.engine_fault_seen.contains(&h1), "{rejects:?}");
+    assert!(rig.st.reorg.invalid.contains(h2.to_byte_array()));
+    assert_eq!(rig.hub.tip_height(), Some(t + 1), "{rejects:?}");
+    rig.finish();
+}
+
+/// Archived blocks above the tip, as a crash between the Class A write and
+/// the tip advance leaves them. The wave is longer than one load batch, so
+/// the last block loads in the next chunk and spends a create from the
+/// first, noted in flight with no outs.
+#[test]
+fn archived_wave_pins_a_parent_from_the_previous_chunk() {
+    use super::super::assign::tests::lock_default_assign_stop;
+    use super::super::state::InflightReq;
+    use bitcoin::ScriptBuf;
+    use rbitcoin_consensus::mine_regtest_paying;
+
+    let _env = lock_default_assign_stop();
+    let mut rig = WireRig::new("archived-cross-chunk", 1);
+    let (t, cbs) = (rig.t, rig.cbs.clone());
+    let op_true = ScriptBuf::from_bytes(vec![0x51]);
+    let n = super::super::confirm::CONFIRM_RUN_MAX_BLOCKS as u32 + 1;
+    let parent = WireRig::spend(cbs[0]);
+    let mut child = Some(WireRig::spend(parent.compute_txid()));
+    let mut extra = vec![parent];
+    let mut blocks: Vec<bitcoin::Block> = Vec::new();
+    for i in 1..=n {
+        let prev = blocks.last().map_or(rig.tip, |b| b.block_hash());
+        if i == n {
+            extra = child.take().into_iter().collect();
+        }
+        let b = mine_regtest_paying(
+            prev,
+            rig.tip_time + 600 * i,
+            t + i,
+            op_true.clone(),
+            std::mem::take(&mut extra),
+        );
+        blocks.push(b);
+    }
+    // One Class A write for the run, as the write stage does, with no tip advance.
+    let bodies: Vec<_> = blocks.iter().cloned().map(std::sync::Arc::new).collect();
+    let fks: Vec<_> = bodies
+        .iter()
+        .map(|b| rig.hub.ensure_header_fk(&b.header).unwrap())
+        .collect();
+    let txids: Vec<Vec<[u8; 32]>> = bodies
+        .iter()
+        .map(|b| {
+            b.txdata
+                .iter()
+                .map(|t| t.compute_txid().to_byte_array())
+                .collect()
+        })
+        .collect();
+    let items: Vec<_> = bodies
+        .iter()
+        .zip(&fks)
+        .zip(&txids)
+        .map(|((b, fk), ids)| (*fk, b, ids.as_slice()))
+        .collect();
+    rig.hub.query.archive_class_a_from_wire(&items).unwrap();
+    assert_eq!(rig.hub.tip_height(), Some(t));
+    let refs: Vec<&bitcoin::Block> = blocks.iter().collect();
+    rig.plant(&refs);
+    for body in &blocks {
+        let hash = body.block_hash();
+        rig.st.slots[0].in_flight.insert(hash);
+        rig.st.inflight.insert(hash, InflightReq::new(1));
+        rig.deliver(1, body);
+    }
+    rig.start_engine();
+
+    let last = blocks.last().unwrap().block_hash();
+    let by_hash: std::collections::HashMap<_, _> =
+        blocks.iter().map(|b| (b.block_hash(), b.clone())).collect();
+    let rejects = rig.pump(
+        |_, hash| by_hash[&hash].clone(),
+        |st, hub, _| st.halt.is_some() || hub.tip_hash() == Some(last),
+    );
+    assert_eq!(
+        rig.st.halt, None,
+        "rejects={rejects:?} last={:?}",
+        rig.last_reject_err
+    );
+    assert_eq!(rig.hub.tip_height(), Some(t + n), "{rejects:?}");
+    rig.finish();
+}
+
 /// tip+1 is the last block on the path, and load cannot read its header
 /// row when it stamps the wave: a local store fault, not a block verdict.
 /// The wave goes back to lookup, so once the row reads again tip+1

crates/rbitcoin-query/src/confirm_run/lookup.rs

@@ -366,6 +366,16 @@ pub(super) fn stamp_parent_pin_archived(
             query.confirm_stats(),
         )
         .map_err(ConfirmError::from)?;
+    } else {
+        // The IBD skeleton carries external ranges. A parent with no body is
+        // an archived create: this batch's own, or an earlier chunk's of the
+        // same wave (noted in flight with no outs). Only the store has its
+        // range. In-flight pins are skipped inside the fill.
+        let mut archived: U64Map<crate::ParentIdent> =
+            stamp.idents.extract_if(|_, e| e.body.is_none()).collect();
+        crate::fill_missing_parent_ranges(query.store(), ifo, &mut archived, query.confirm_stats())
+            .map_err(ConfirmError::from)?;
+        stamp.idents.extend(archived);
     }
     for ident in stamp.idents.values() {
         if ident.txid == [0u8; 32] {

crates/rbitcoin-query/src/stamp.rs

@@ -6,8 +6,10 @@
 //! rehydrate. In-flight holds CreatePins until load drops map rows below a
 //! lookup-wave drain+fence snapshot taken before TipOnly. Same-wave creates are
 //! omitted from that skeleton. Class A `CreatePin::set_loc` after append;
-//! later-wave stamp reads pin loc. IBD (`skeleton = Some`) never loc-by-fk:
-//! spent is pin loc, skeleton TipOnly loc, or unset for write fill.
+//! later-wave stamp reads pin loc. IBD (`skeleton = Some`) never loc-by-fk
+//! here: spent is pin loc, skeleton TipOnly loc, or unset for write fill. The
+//! all-archived retry (`stamp_parent_pin_archived`) loc-fills its archived
+//! parents that have no body. External IBD parents still must not.
 
 use crate::{CreatePin, InFlight, QueryError, U64Map};
 use crate::{IdMap, TxidHasher};

docs/invariants.md

@@ -81,7 +81,7 @@ the Wire CreatePin (`Arc<Block>` + tx index). Load still does not head/idx.
 
 | Stage | Invariant | Soft path allowed? |
 |-------|-----------|--------------------|
-| Lookup parent stamp | Every external spent parent has create_fk + body_range (or offline in_flight CreatePin) + reverse txid. Archived parents also have computed `spent.body` range on the stamp (same-wave in-flight outs skip loc, miss OK; later-wave InFlight takes loc from the same `CreatePin` Arc write set at Class A, or skeleton TipOnly loc). IBD (`skeleton = Some`) never loc-by-fk: pin loc unset leaves spent unset for write TLS. plan=None leftover still `fill_missing_parent_ranges` | Missing → hard Err at stamp / pin contract. **No** `finish_archive_plan` `fill_missing` after stamp loc. **No** IBD load `create.loc` |
+| Lookup parent stamp | Every external spent parent has create_fk + body_range (or offline in_flight CreatePin) + reverse txid. Archived parents also have computed `spent.body` range on the stamp (same-wave in-flight outs skip loc, miss OK; later-wave InFlight takes loc from the same `CreatePin` Arc write set at Class A, or skeleton TipOnly loc). IBD (`skeleton = Some`) never loc-by-fk: pin loc unset leaves spent unset for write TLS. The one exception is an all-archived retry (plan=None): a parent with no body is an archived create (this batch's own, or an earlier chunk's of the same wave), and only the store can range it. plan=None leftover still `fill_missing_parent_ranges` | Missing → hard Err at stamp / pin contract. **No** `finish_archive_plan` `fill_missing` after stamp loc. **No** IBD load `create.loc` |
 | Parent create_fk | **same-batch** planned fks (offline at pin) → **in-flight** (lookup snapshots `drain_and_fence_hi` **before** the wave's TipOnly read and passes it on the last load batch; load drops tagged map rows with pack height **below** that snapshot after that batch's in-flight read; equality keeps; not Class C tip, not `class_a_hi`, not write freeze; one load-thread HashMap, insert after stamp) → **skeleton** (`BatchParentIds` on the `LoadBatch`: lookup TipOnly fk + body_range + spent_range + per-chunk need-vouts) → **Corrupt** on IBD miss. plan=None / S0 (`skeleton = None`) is in-flight → leftover TipOnly. One helper: [`stamp_external_parents`](../crates/rbitcoin-query/src/stamp.rs). No leftover pending map, no process pin FIFO, no BQ-side hits map, no parent-store create_fk on stamp, no published live_union. Same-wave creates are omitted from TipOnly need. Header-cache GC polls store tip each load pack. In-flight and skeleton maps stay one fk per txid (last write). That limit is only those RAM maps ([`errata.md`](./errata.md)), not a consensus rule for this write. | Miss of in-flight and skeleton → `Corrupt("parent create_fk unresolved")`. A batched miss isolates; a lone block's miss is an **engine fault** (requeue once, then halt IBD; never blacklist), unless the block extends the tip, `tx.head` holds every connected create (`Query::head_covers_fence`; open starts the drain at the head's last create, after its rebuild or tail backfill), and a fresh TipOnly read of its parents on the load thread (reject path only) still misses: that is `ConsensusInvalid`, unless `txid.body` holds the txid at a connected fk in an unsealed `tx.head` segment (the open tail or the in-flight seal), whose slot pages may hold a power-loss hole (`Corrupt("invariant: tx.head misses a connected create")`, engine fault) ([089](./external_findings/089-missing-parent-verdict.md)). A lone block whose parent is not the tip within 10 s is a `Cascade` retry. Identity without loc range → `Corrupt("invariant: loc range missing after identity")`, not a miss |
 | io_uring harvest | TLS session fail-closed ([`io-modality.md`](./io-modality.md)) | **No** silent success. `Corrupt("invariant: io_uring …")` (not `bdz g page bad slot`). Ring-unavailable still pread-fallback |
 | Load body outs | By `txout` range only from lookup stamp; incomplete outs → hard Err. Pin **copies** lookup `spent_range` (no loc IO). Later-wave InFlight loc comes from the CreatePin Arc (Class A `set_loc`) | **No** loc cold outs on load; **no** `seqsigwit` on pin; **no** IBD loc-by-fk |