Skip to content

Commit 7437b05

Browse files
Merge pull request #18 from bitcoin-kernel/issue-17-follow-reorg
Follow the chain: live tip-following with reorg handling
2 parents eb33119 + 10a803b commit 7437b05

1 file changed

Lines changed: 136 additions & 0 deletions

File tree

‎src/_follow.mjs‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
// Follow the chain with reorg handling. Bootstrap the UTXO set from the snapshot,
2+
// catch up to the live testnet4 tip, then stay live — and when the chain reorgs,
3+
// disconnect the orphaned blocks and reconnect the canonical chain *in place*.
4+
//
5+
// To disconnect a block you must restore the coins it spent — which aren't in the
6+
// block, only referenced. So as we connect each block we keep its **undo data**
7+
// (the spent coins' full records + the keys it created). On a reorg we walk down
8+
// to the fork point, disconnect newest-first using that undo data, then connect
9+
// the canonical chain forward. Reorg-aware down to the snapshot height.
10+
11+
import { performance } from 'node:perf_hooks';
12+
import { readFile } from 'node:fs/promises';
13+
import { Codec } from '@bitcoin-desktop/schema/codec/codec.js';
14+
import { BlockEngine } from '@bitcoin-desktop/schema/codec/blocks.js';
15+
import { setVerifyBackend } from '@bitcoin-desktop/schema/codec/secp256k1.js';
16+
import { setSha256Backend, reverseHex, hexToBytes } from '@bitcoin-desktop/schema/codec/hash.js';
17+
import { wasmBackend } from './wasm-secp.js';
18+
import { nativeSha256 } from './sha256-native.js';
19+
import { FileBlockStore } from './store/block-store.js';
20+
import { ShardedUtxo } from './sharded-utxo.js';
21+
22+
const load = async (p) => JSON.parse(await readFile(new URL(import.meta.resolve('@bitcoin-desktop/schema/' + p)), 'utf8'));
23+
const codec = new Codec(await load('schema/core.jsonld'), await load('schema/proof.jsonld'));
24+
const be = BlockEngine.fromSchemas(codec, await load('schema/chain.jsonld'), await load('schema/validate.jsonld'), await load('schema/script.jsonld'), 'btc:testnet4');
25+
setVerifyBackend(wasmBackend);
26+
setSha256Backend(nativeSha256);
27+
28+
const API = 'https://mempool.space/testnet4/api';
29+
const UNDO_DEPTH = 1000; // reorgs deeper than this fall back to re-bootstrap
30+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
31+
async function get(url, kind = 'text') {
32+
for (let a = 0; a < 6; a++) {
33+
const r = await fetch(url);
34+
if (r.status === 429) { await sleep(2000 * (a + 1)); continue; }
35+
if (!r.ok) throw new Error(`${r.status} ${url}`);
36+
return kind === 'buf' ? Buffer.from(await r.arrayBuffer()) : (await r.text());
37+
}
38+
throw new Error('rate-limited: ' + url);
39+
}
40+
41+
// bootstrap
42+
const bs = new FileBlockStore(new URL('../data/', import.meta.url), codec);
43+
const bootstrapHeight = (() => { let t = 0; while (bs.has(t + 1)) t++; return t; })();
44+
const bootstrapHash = codec.blockHash(bs.get(bootstrapHeight).header);
45+
let tip = bootstrapHeight, tipHash = bootstrapHash;
46+
console.log(`bootstrapping from snapshot at height ${tip}…`);
47+
const snap = new ShardedUtxo(64);
48+
await snap.load(new URL(`../data/chainstate/utxo-${tip}.ndjson`, import.meta.url));
49+
const parse = (v) => { const a = v.indexOf('\t'), b = v.indexOf('\t', a + 1), c = v.lastIndexOf('\t'); return { output: { value: Number(v.slice(0, a)), scriptPubKey: v.slice(a + 1, b) }, height: Number(v.slice(b + 1, c)), coinbase: v.slice(c + 1) === '1' }; };
50+
const view = { get: (k) => { const v = snap.get(k); return v === undefined ? undefined : parse(v); }, has: (k) => snap.has(k), get size() { return snap.size; } };
51+
console.log(`coin view ready — ${view.size} coins at height ${tip} (${tipHash.slice(0, 16)}…)`);
52+
53+
// undo log: height -> { hash, created:[key], spent:[[key, compactRecord]] }
54+
const undo = new Map();
55+
const NULL_TXID = '00'.repeat(32);
56+
57+
function connect(block, height) {
58+
const created = [], spent = [];
59+
block.transactions.forEach((tx, i) => {
60+
if (i > 0) for (const inp of tx.inputs) {
61+
const key = `${inp.prevout.txid}:${inp.prevout.vout}`;
62+
const rec = snap.get(key);
63+
if (rec !== undefined) { spent.push([key, rec]); snap.delete(key); }
64+
}
65+
const txid = codec.txid(tx);
66+
tx.outputs.forEach((o, vout) => {
67+
if (!o.scriptPubKey.startsWith('6a')) {
68+
const key = `${txid}:${vout}`;
69+
snap.set(key, `${o.value}\t${o.scriptPubKey}\t${height}\t${i === 0 ? 1 : 0}`);
70+
created.push(key);
71+
}
72+
});
73+
});
74+
undo.set(height, { hash: codec.blockHash(block.header), created, spent });
75+
undo.delete(height - UNDO_DEPTH); // prune
76+
return { created: created.length, spent: spent.length };
77+
}
78+
79+
function disconnect(height) {
80+
const u = undo.get(height);
81+
// restore spent FIRST, then remove created — so a coin created *and* spent
82+
// within this same block (key in both lists) nets to correctly absent.
83+
for (const [key, rec] of u.spent) snap.set(key, rec); // restore what it spent
84+
for (const key of u.created) snap.delete(key); // remove what it created
85+
undo.delete(height);
86+
}
87+
88+
async function findFork() { // highest height where we agree with canonical
89+
for (let h = tip; h >= bootstrapHeight; h--) {
90+
const mine = h === bootstrapHeight ? bootstrapHash : undo.get(h)?.hash;
91+
if (mine === undefined) return null; // undo pruned past the fork — too deep
92+
const canon = (await get(`${API}/block-height/${h}`)).trim();
93+
if (canon === mine) return { height: h, hash: canon };
94+
}
95+
return null;
96+
}
97+
98+
const tipRev = (h) => reverseHex(hexToBytes(h));
99+
let target = Number(await get(`${API}/blocks/tip/height`));
100+
console.log(`live tip ${target} — catching up ${target - tip} blocks\n`);
101+
const t0 = performance.now(); let done = 0;
102+
103+
while (true) {
104+
const h = tip + 1;
105+
if (h > target) {
106+
if (done > 0) { console.log(`\n✅ at live tip ${tip}. following — polling every 30s…`); done = 0; }
107+
await sleep(30000);
108+
const nt = Number(await get(`${API}/blocks/tip/height`));
109+
if (nt > target) { target = nt; console.log(`live tip advanced to ${target}`); }
110+
continue;
111+
}
112+
let block, hash;
113+
try {
114+
hash = (await get(`${API}/block-height/${h}`)).trim();
115+
block = codec.decode('Block', (await get(`${API}/block/${hash}/raw`, 'buf')).toString('hex'));
116+
} catch (e) { console.log(`h ${h}: fetch error (${e.message}) — retry in 5s`); await sleep(5000); continue; }
117+
118+
const prev = block.header.prevBlockHash;
119+
if (prev !== tipHash && prev !== tipRev(tipHash)) {
120+
const fork = await findFork();
121+
if (!fork) { console.log(`\n⛔ reorg deeper than ${UNDO_DEPTH} blocks of undo — re-bootstrap needed. stopping.`); break; }
122+
let depth = 0;
123+
while (tip > fork.height) { disconnect(tip); tip--; depth++; }
124+
tipHash = fork.hash;
125+
console.log(`\n⟲ reorg: forked at ${fork.height} (disconnected ${depth} orphaned block${depth === 1 ? '' : 's'}), reconnecting canonical chain…`);
126+
continue; // refetch fork+1 and connect forward
127+
}
128+
129+
const failed = [...be.validateBlockStructure(block).results, ...be.validateBlockContext(block, { height: h, utxo: view }).results].filter((r) => r.ok === false);
130+
if (failed.length) { console.log(`\n❌ block ${h} INVALID: ${failed.map((f) => f.rule).join(', ')} — stopping`); break; }
131+
const { created, spent } = connect(block, h);
132+
tip = h; tipHash = hash; done++;
133+
const rate = done / ((performance.now() - t0) / 1000);
134+
console.log(`h ${h} ${hash.slice(0, 12)}… txs ${String(block.transactions.length).padStart(4)} +${created} -${spent} utxo ${view.size} ${rate.toFixed(1)} blk/s`);
135+
await sleep(400);
136+
}

0 commit comments

Comments
 (0)