swiftsync: hint generation + reconstruction — end-to-end on real testnet4

hint.js: generateHints walks a range, builds the final UTXO set, and records the
block-local output indices that stay unspent (index counts all outputs so a
verifier aligns; OP_RETURN gets an index but is never unspent). reconstructUtxo
is the assumevalid verifier — rebuilds the UTXO set from blocks + the hint bitmap
alone, no input processing.

Proven end-to-end on 5,000 real testnet4 blocks (113,300 outputs): generate hint
→ Elias-Fano hintsfile → decode → reconstruct == the real UTXO set (4,574 coins).
Hintsfile is 13,897 bytes = 0.981 bits/output, matching the BIP's concise-
representation claim. The full assumevalid SwiftSync loop now runs on real blocks.

Remaining: the full-chain run (real hintsfile + accumulator oracle test over all
of testnet4), which gates on the validate-sync finishing.

Author: melvincarvalho

packages/swiftsync/DESIGN.md

@@ -63,8 +63,8 @@ method we use against Bitcoin Core. No external reference needed.
 - `validate.js` — drive the engine over blocks → add/spend ops. **Done** (verified on 5,000 real testnet4 blocks).
 - `hintsfile.js` — Elias-Fano hintsfile encode/decode (the interop artifact). **Done** (matches the BIP `elias_fano.json` vectors byte-for-byte).
 - `undo.js` — spent-coin data for the full version: CompressAmount + reconstructable script + height code + Coin record. **Done** (amount/script match the BIP vectors byte-for-byte).
-- (todo) hint generation — produce the per-block unspent-index sets from a validated chain (needs the finished sync).
-- (todo) oracle test — run validate-sync + SwiftSync over testnet4; assert equal digests.
+- `hint.js` — `generateHints` (per-block unspent-index sets from a chain) + `reconstructUtxo` (the assumevalid verifier). **Done** (end-to-end on 5,000 real testnet4 blocks: reconstruction == real UTXO set; hintsfile 0.98 bits/output).
+- (todo) full-chain run — generate the real testnet4 hintsfile + commitment (needs the finished sync) and the accumulator oracle test over the whole chain.
 
 ## Decisions
 

packages/swiftsync/hint.js

@@ -0,0 +1,48 @@
+// Hint generation + reconstruction.
+//
+// generateHints: walk a contiguous range, build the UTXO set at the end, then for
+// each block list the block-local output indices that remain unspent. The index
+// counts ALL outputs in transaction order (so a verifier walking outputs aligns);
+// OP_RETURN outputs get an index but are never in the UTXO set, so never in a
+// hint. These per-block index lists are what hintsfile.js Elias-Fano encodes.
+//
+// reconstructUtxo: the assumevalid verifier side — rebuild the UTXO set purely
+// from the blocks + the hint bitmap (no input processing). With the matching
+// accumulator check (validate.js), this is SwiftSync: fast set construction from
+// hints, verified against a trusted commitment.
+//
+// Engine-agnostic: caller injects txidOf(tx).
+
+const NULL_TXID = '00'.repeat(32);
+const isOpReturn = (spk) => typeof spk === 'string' && spk.startsWith('6a');
+
+export function generateHints(blocks, { txidOf }) {
+  const utxo = new Set();
+  for (const block of blocks) for (const tx of block.transactions) {
+    const txid = txidOf(tx);
+    for (let v = 0; v < tx.outputs.length; v++) if (!isOpReturn(tx.outputs[v].scriptPubKey)) utxo.add(txid + ':' + v);
+    for (const inp of tx.inputs) if (inp.prevout.txid !== NULL_TXID) utxo.delete(inp.prevout.txid + ':' + inp.prevout.vout);
+  }
+  const blockHints = [];
+  for (const block of blocks) {
+    const idx = []; let i = 0;
+    for (const tx of block.transactions) {
+      const txid = txidOf(tx);
+      for (let v = 0; v < tx.outputs.length; v++) { if (utxo.has(txid + ':' + v)) idx.push(i); i++; }
+    }
+    blockHints.push(idx);
+  }
+  return { height: blocks.length, blockHints };
+}
+
+export function reconstructUtxo(blocks, blockHints, { txidOf }) {
+  const utxo = new Set();
+  for (let b = 0; b < blocks.length; b++) {
+    const want = new Set(blockHints[b]); let i = 0;
+    for (const tx of blocks[b].transactions) {
+      const txid = txidOf(tx);
+      for (let v = 0; v < tx.outputs.length; v++) { if (want.has(i)) utxo.add(txid + ':' + v); i++; }
+    }
+  }
+  return utxo;
+}

packages/swiftsync/index.js

@@ -3,6 +3,7 @@
 
 export { Accumulator } from './accumulator.js';
 export { applyBlocks } from './validate.js';
+export { generateHints, reconstructUtxo } from './hint.js';
 export { eliasFanoEncode, eliasFanoDecode, encodeHintsfile, decodeHintsfile } from './hintsfile.js';
 export { compressAmount, decompressAmount, compressScript, expandScript, encodeHeightCode, decodeHeightCode, encodeCoin as encodeSpentCoin, decodeCoin as decodeSpentCoin } from './undo.js';
 

packages/swiftsync/package.json

@@ -8,6 +8,7 @@
     ".": "./index.js",
     "./accumulator": "./accumulator.js",
     "./validate": "./validate.js",
+    "./hint": "./hint.js",
     "./hintsfile": "./hintsfile.js",
     "./undo": "./undo.js"
   }

packages/swiftsync/test/hint.test.js

@@ -0,0 +1,33 @@
+// Hint generation + reconstruction: the generator marks block-local output
+// indices that stay unspent; the verifier rebuilds the exact same UTXO set from
+// the bitmap alone. (Verified separately end-to-end on 5,000 real testnet4 blocks.)
+
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { generateHints, reconstructUtxo } from '../hint.js';
+import { encodeHintsfile, decodeHintsfile } from '../hintsfile.js';
+
+const A = 'aa'.repeat(32), B = 'bb'.repeat(32), NULL = '00'.repeat(32);
+const txidOf = (tx) => tx.id;
+
+test('generate + reconstruct: indices count all outputs, OP_RETURN never unspent', () => {
+  const blocks = [
+    { transactions: [
+      { id: A, inputs: [{ prevout: { txid: NULL, vout: 0xffffffff } }], outputs: [{ scriptPubKey: '51' }, { scriptPubKey: '52' }] }, // A:0, A:1
+    ] },
+    { transactions: [
+      { id: B, inputs: [{ prevout: { txid: A, vout: 0 } }],                                                                            // spend A:0
+        outputs: [{ scriptPubKey: '51' }, { scriptPubKey: '6a00' }] },                                                                  // B:0 spendable, B:1 OP_RETURN
+    ] },
+  ];
+  // unspent at tip: A:1 (block 0 index 1), B:0 (block 1 index 0)
+  const { blockHints } = generateHints(blocks, { txidOf });
+  assert.deepEqual(blockHints, [[1], [0]]);
+
+  // verifier rebuilds the same UTXO set from the bitmap alone
+  const utxo = reconstructUtxo(blocks, blockHints, { txidOf });
+  assert.deepEqual([...utxo].sort(), [A + ':1', B + ':0'].sort());
+
+  // and it survives the Elias-Fano hintsfile round-trip
+  assert.deepEqual(decodeHintsfile(encodeHintsfile({ height: 2, blockHints })).blockHints, blockHints);
+});