|
| 1 | +// Follow the chain with reorg handling. Bootstrap the UTXO set from the snapshot, |
| 2 | +// catch up to the live testnet4 tip, then stay live — and when the chain reorgs, |
| 3 | +// disconnect the orphaned blocks and reconnect the canonical chain *in place*. |
| 4 | +// |
| 5 | +// To disconnect a block you must restore the coins it spent — which aren't in the |
| 6 | +// block, only referenced. So as we connect each block we keep its **undo data** |
| 7 | +// (the spent coins' full records + the keys it created). On a reorg we walk down |
| 8 | +// to the fork point, disconnect newest-first using that undo data, then connect |
| 9 | +// the canonical chain forward. Reorg-aware down to the snapshot height. |
| 10 | + |
| 11 | +import { performance } from 'node:perf_hooks'; |
| 12 | +import { readFile } from 'node:fs/promises'; |
| 13 | +import { Codec } from '@bitcoin-desktop/schema/codec/codec.js'; |
| 14 | +import { BlockEngine } from '@bitcoin-desktop/schema/codec/blocks.js'; |
| 15 | +import { setVerifyBackend } from '@bitcoin-desktop/schema/codec/secp256k1.js'; |
| 16 | +import { setSha256Backend, reverseHex, hexToBytes } from '@bitcoin-desktop/schema/codec/hash.js'; |
| 17 | +import { wasmBackend } from './wasm-secp.js'; |
| 18 | +import { nativeSha256 } from './sha256-native.js'; |
| 19 | +import { FileBlockStore } from './store/block-store.js'; |
| 20 | +import { ShardedUtxo } from './sharded-utxo.js'; |
| 21 | + |
| 22 | +const load = async (p) => JSON.parse(await readFile(new URL(import.meta.resolve('@bitcoin-desktop/schema/' + p)), 'utf8')); |
| 23 | +const codec = new Codec(await load('schema/core.jsonld'), await load('schema/proof.jsonld')); |
| 24 | +const be = BlockEngine.fromSchemas(codec, await load('schema/chain.jsonld'), await load('schema/validate.jsonld'), await load('schema/script.jsonld'), 'btc:testnet4'); |
| 25 | +setVerifyBackend(wasmBackend); |
| 26 | +setSha256Backend(nativeSha256); |
| 27 | + |
| 28 | +const API = 'https://mempool.space/testnet4/api'; |
| 29 | +const UNDO_DEPTH = 1000; // reorgs deeper than this fall back to re-bootstrap |
| 30 | +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); |
| 31 | +async function get(url, kind = 'text') { |
| 32 | + for (let a = 0; a < 6; a++) { |
| 33 | + const r = await fetch(url); |
| 34 | + if (r.status === 429) { await sleep(2000 * (a + 1)); continue; } |
| 35 | + if (!r.ok) throw new Error(`${r.status} ${url}`); |
| 36 | + return kind === 'buf' ? Buffer.from(await r.arrayBuffer()) : (await r.text()); |
| 37 | + } |
| 38 | + throw new Error('rate-limited: ' + url); |
| 39 | +} |
| 40 | + |
| 41 | +// bootstrap |
| 42 | +const bs = new FileBlockStore(new URL('../data/', import.meta.url), codec); |
| 43 | +const bootstrapHeight = (() => { let t = 0; while (bs.has(t + 1)) t++; return t; })(); |
| 44 | +const bootstrapHash = codec.blockHash(bs.get(bootstrapHeight).header); |
| 45 | +let tip = bootstrapHeight, tipHash = bootstrapHash; |
| 46 | +console.log(`bootstrapping from snapshot at height ${tip}…`); |
| 47 | +const snap = new ShardedUtxo(64); |
| 48 | +await snap.load(new URL(`../data/chainstate/utxo-${tip}.ndjson`, import.meta.url)); |
| 49 | +const parse = (v) => { const a = v.indexOf('\t'), b = v.indexOf('\t', a + 1), c = v.lastIndexOf('\t'); return { output: { value: Number(v.slice(0, a)), scriptPubKey: v.slice(a + 1, b) }, height: Number(v.slice(b + 1, c)), coinbase: v.slice(c + 1) === '1' }; }; |
| 50 | +const view = { get: (k) => { const v = snap.get(k); return v === undefined ? undefined : parse(v); }, has: (k) => snap.has(k), get size() { return snap.size; } }; |
| 51 | +console.log(`coin view ready — ${view.size} coins at height ${tip} (${tipHash.slice(0, 16)}…)`); |
| 52 | + |
| 53 | +// undo log: height -> { hash, created:[key], spent:[[key, compactRecord]] } |
| 54 | +const undo = new Map(); |
| 55 | +const NULL_TXID = '00'.repeat(32); |
| 56 | + |
| 57 | +function connect(block, height) { |
| 58 | + const created = [], spent = []; |
| 59 | + block.transactions.forEach((tx, i) => { |
| 60 | + if (i > 0) for (const inp of tx.inputs) { |
| 61 | + const key = `${inp.prevout.txid}:${inp.prevout.vout}`; |
| 62 | + const rec = snap.get(key); |
| 63 | + if (rec !== undefined) { spent.push([key, rec]); snap.delete(key); } |
| 64 | + } |
| 65 | + const txid = codec.txid(tx); |
| 66 | + tx.outputs.forEach((o, vout) => { |
| 67 | + if (!o.scriptPubKey.startsWith('6a')) { |
| 68 | + const key = `${txid}:${vout}`; |
| 69 | + snap.set(key, `${o.value}\t${o.scriptPubKey}\t${height}\t${i === 0 ? 1 : 0}`); |
| 70 | + created.push(key); |
| 71 | + } |
| 72 | + }); |
| 73 | + }); |
| 74 | + undo.set(height, { hash: codec.blockHash(block.header), created, spent }); |
| 75 | + undo.delete(height - UNDO_DEPTH); // prune |
| 76 | + return { created: created.length, spent: spent.length }; |
| 77 | +} |
| 78 | + |
| 79 | +function disconnect(height) { |
| 80 | + const u = undo.get(height); |
| 81 | + // restore spent FIRST, then remove created — so a coin created *and* spent |
| 82 | + // within this same block (key in both lists) nets to correctly absent. |
| 83 | + for (const [key, rec] of u.spent) snap.set(key, rec); // restore what it spent |
| 84 | + for (const key of u.created) snap.delete(key); // remove what it created |
| 85 | + undo.delete(height); |
| 86 | +} |
| 87 | + |
| 88 | +async function findFork() { // highest height where we agree with canonical |
| 89 | + for (let h = tip; h >= bootstrapHeight; h--) { |
| 90 | + const mine = h === bootstrapHeight ? bootstrapHash : undo.get(h)?.hash; |
| 91 | + if (mine === undefined) return null; // undo pruned past the fork — too deep |
| 92 | + const canon = (await get(`${API}/block-height/${h}`)).trim(); |
| 93 | + if (canon === mine) return { height: h, hash: canon }; |
| 94 | + } |
| 95 | + return null; |
| 96 | +} |
| 97 | + |
| 98 | +const tipRev = (h) => reverseHex(hexToBytes(h)); |
| 99 | +let target = Number(await get(`${API}/blocks/tip/height`)); |
| 100 | +console.log(`live tip ${target} — catching up ${target - tip} blocks\n`); |
| 101 | +const t0 = performance.now(); let done = 0; |
| 102 | + |
| 103 | +while (true) { |
| 104 | + const h = tip + 1; |
| 105 | + if (h > target) { |
| 106 | + if (done > 0) { console.log(`\n✅ at live tip ${tip}. following — polling every 30s…`); done = 0; } |
| 107 | + await sleep(30000); |
| 108 | + const nt = Number(await get(`${API}/blocks/tip/height`)); |
| 109 | + if (nt > target) { target = nt; console.log(`live tip advanced to ${target}`); } |
| 110 | + continue; |
| 111 | + } |
| 112 | + let block, hash; |
| 113 | + try { |
| 114 | + hash = (await get(`${API}/block-height/${h}`)).trim(); |
| 115 | + block = codec.decode('Block', (await get(`${API}/block/${hash}/raw`, 'buf')).toString('hex')); |
| 116 | + } catch (e) { console.log(`h ${h}: fetch error (${e.message}) — retry in 5s`); await sleep(5000); continue; } |
| 117 | + |
| 118 | + const prev = block.header.prevBlockHash; |
| 119 | + if (prev !== tipHash && prev !== tipRev(tipHash)) { |
| 120 | + const fork = await findFork(); |
| 121 | + if (!fork) { console.log(`\n⛔ reorg deeper than ${UNDO_DEPTH} blocks of undo — re-bootstrap needed. stopping.`); break; } |
| 122 | + let depth = 0; |
| 123 | + while (tip > fork.height) { disconnect(tip); tip--; depth++; } |
| 124 | + tipHash = fork.hash; |
| 125 | + console.log(`\n⟲ reorg: forked at ${fork.height} (disconnected ${depth} orphaned block${depth === 1 ? '' : 's'}), reconnecting canonical chain…`); |
| 126 | + continue; // refetch fork+1 and connect forward |
| 127 | + } |
| 128 | + |
| 129 | + const failed = [...be.validateBlockStructure(block).results, ...be.validateBlockContext(block, { height: h, utxo: view }).results].filter((r) => r.ok === false); |
| 130 | + if (failed.length) { console.log(`\n❌ block ${h} INVALID: ${failed.map((f) => f.rule).join(', ')} — stopping`); break; } |
| 131 | + const { created, spent } = connect(block, h); |
| 132 | + tip = h; tipHash = hash; done++; |
| 133 | + const rate = done / ((performance.now() - t0) / 1000); |
| 134 | + console.log(`h ${h} ${hash.slice(0, 12)}… txs ${String(block.transactions.length).padStart(4)} +${created} -${spent} utxo ${view.size} ${rate.toFixed(1)} blk/s`); |
| 135 | + await sleep(400); |
| 136 | +} |
0 commit comments