diff --git a/CHANGELOG.md b/CHANGELOG.md index a010265..61d34f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.18.0] - 2026-09-11 + +### Fixed +- A raw `estimatesmartfee` result is clamped to a per-chain sanity ceiling, `FEE_ESTIMATE_SANITY_CEILING`. +- A two-phase reveal is prefunded at a package-aware fee rate. +- UTXO dedup is linearised and tracker freshness is unified into one classifier. +- The bufferutils patch converges with the SDK copy and is pinned by a twin guard. +- The Docker image no longer tries to bake a `.env` file, so the build succeeds on the legacy builder and configuration reaches the container as environment only. + ## [0.17.0] - 2026-09-10 ### Added diff --git a/Dockerfile b/Dockerfile index 18cb224..03e1908 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,10 +13,9 @@ RUN npm ci --omit=dev COPY ./src /XChainEncoder/src COPY ./docs /XChainEncoder/docs -# Glob pattern matches zero or one .env file so the build succeeds when -# the source has no .env (typical in CI / fresh clones). Mirrors the -# pattern used by xchain-indexer/Dockerfile. -COPY ./.en[v] /XChainEncoder/.env +# No .env is baked in: configuration reaches the container as environment +# (xchain-node at `docker run`, docker-compose.yml via env_file). An optional +# `COPY ./.en[v]` glob here builds only under BuildKit. # Run node directly rather than through `npm run api` (which is this exact # command). npm builds a three-process tree, npm -> sh -c -> node, and neither diff --git a/README.md b/README.md index 702c54e..8f14cf2 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ # XChain Platform Encoder

- Version - Tests + Version + Tests Node License

@@ -165,7 +165,7 @@ neither source sets one, so these defaults hold on an unconfigured box: | `npm run build` | Production browser bundle (minified) -> `dist/xchain_encoder.min.js` | | `npm run build:dev` | Development browser bundle (unminified) | | `npm run smoke-test` | Smoke tests (~52 tests, <1s) | -| `npm run test:unit` | Unit tests (815 tests) | +| `npm run test:unit` | Unit tests (870 tests) | | `npm run test:integration` | Integration tests (115 tests) | | `npm run test:boundary` | Boundary condition tests (~101 tests) | | `npm run test:security` | Security tests (63 tests) | @@ -178,7 +178,7 @@ neither source sets one, so these defaults hold on an unconfigured box: | `npm run bench` | Performance benchmarks | | `npm run bench:full` | Extended benchmarks with JSON output | | `npm run bench:soak` | Soak test (sustained load) | -| `npm test` | Unit tests (hermetic, no external services, 815 tests) | +| `npm test` | Unit tests (hermetic, no external services, 870 tests) | | `npm run test:regtest` | Regtest integration tests (requires local bitcoind) | ## Test Suite diff --git a/docker-compose.yml b/docker-compose.yml index 2013c0e..89d9e5c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,9 @@ version: '3' services: xchain_encoder: build: . + # The image bakes no .env (Dockerfile); the README's .env is handed to the + # container here instead. + env_file: .env # HARD CONSTRAINT: exactly one encoder replica per endpoint. The UTXO # outpoint-reservation double-spend guard, the recent-build duplicate # refusal and the rate limiter are all in-process diff --git a/package-lock.json b/package-lock.json index b92aa45..e5d556e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,16 @@ { "name": "xchain-encoder", - "version": "0.17.0", + "version": "0.18.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "xchain-encoder", - "version": "0.17.0", + "version": "0.18.0", "license": "AGPL-3.0-or-later", "dependencies": { "axios": "^1.18.1", + "bip174": "2.1.1", "bip32": "4.0.0", "bip39": "^3.1.0", "bitcoinjs-lib": "6.1.7", diff --git a/package.json b/package.json index f865c32..8825340 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "xchain-encoder", "description": "xchain-encoder encodes XChain Platform ACTION commands into blockchain transactions.", - "version": "0.17.0", + "version": "0.18.0", "license": "AGPL-3.0-or-later", "repository": { "type": "git", @@ -9,6 +9,7 @@ }, "dependencies": { "axios": "^1.18.1", + "bip174": "2.1.1", "bip32": "4.0.0", "bip39": "^3.1.0", "bitcoinjs-lib": "6.1.7", diff --git a/src/BlockchainConnector.js b/src/BlockchainConnector.js index 77edcfc..e9c1760 100644 --- a/src/BlockchainConnector.js +++ b/src/BlockchainConnector.js @@ -37,6 +37,36 @@ function noEstimateRelayMultiplier(){ return (Number.isFinite(raw) && raw > 0) ? raw : 10 } +// Absolute per-chain sanity ceiling (COIN/kB, the same unit estimatesmartfee +// returns) on the RAW non-regtest estimate before anything else sees it. +// getFeePerKilobyte's result is used two ways downstream: as the fee actually +// charged when a caller supplies none, AND as the anchor the caller-facing +// caps (MAX_FEE_RATE_KB, MAX_FEE_RATE_MULTIPLIER) derive their relative +// ceiling from - so a node returning a spiked or malformed estimate on +// testnet/mainnet lifts its own ceiling right along with it, and nothing +// downstream is positioned to catch that; those caps bound a CALLER-supplied +// rate, never the node's own report. Defaults are coin-scale, since fee rates +// run roughly 1000x higher in DOGE/kB than in BTC/kB or LTC/kB for an +// equivalent real-world cost; keyed off NETWORK's coin prefix +// (bitcoin-mainnet, litecoin-testnet, dogecoin-mainnet, ...). An unrecognized +// or missing prefix falls back to the bitcoin-scale default so an +// unconfigured deployment still gets a ceiling rather than none. +// FEE_ESTIMATE_SANITY_CEILING overrides in COIN/kB for every chain; a +// non-positive or unparseable value keeps the coin default (fail-safe: this +// guard can be tightened or loosened by env, never silently disabled). +const DEFAULT_FEE_ESTIMATE_SANITY_CEILING = { + bitcoin: 0.01, // 0.01 BTC/kB + litecoin: 0.01, // 0.01 LTC/kB + dogecoin: 10, // 10 DOGE/kB (~1000x the 0.01 DOGE/kB documented recommended rate) +} + +function feeEstimateSanityCeiling(){ + const override = parseFloat(process.env.FEE_ESTIMATE_SANITY_CEILING) + if (Number.isFinite(override) && override > 0) return override + const coin = String(process.env.NETWORK || '').split('-')[0].toLowerCase() + return DEFAULT_FEE_ESTIMATE_SANITY_CEILING[coin] || DEFAULT_FEE_ESTIMATE_SANITY_CEILING.bitcoin +} + // Sanitize an axios error before it is logged or re-thrown. RPC calls pass // auth:{username,password} to axios, which attaches the request config to the // thrown error, so logging the raw error serializes the node RPC password into @@ -446,7 +476,17 @@ class BlockchainConnector { // sentinel (feerate:-1, not enough data) is treated as an error rather // than silently returned. if (responseData.result && Number(responseData.result.feerate) > 0) { - return responseData.result.feerate; + const feerate = Number(responseData.result.feerate); + const ceiling = feeEstimateSanityCeiling(); + if (feerate > ceiling) { + // Loud diagnostic: this is a money-affecting path and a clamp here + // is silent everywhere else unless it is logged at error level. + console.error('estimatesmartfee returned ' + feerate + '/kB on a non-regtest ' + + 'chain, above the ' + ceiling + '/kB sanity ceiling; clamping to the ceiling. ' + + 'Set FEE_ESTIMATE_SANITY_CEILING to override.'); + return ceiling; + } + return feerate; } // No estimate at all (feerate:-1). A public TESTNET routinely has too few // transactions to estimate from: DOGE testnet answers -1 with an empty diff --git a/src/XChainEncoder.js b/src/XChainEncoder.js index cc7c0f0..98b09ce 100644 --- a/src/XChainEncoder.js +++ b/src/XChainEncoder.js @@ -217,12 +217,19 @@ function maxCpfpUpliftSat(){ // DOGE/kB whose 0.00313 DOGE/kB funding ancestors dragged the package to // 0.00896, under the 0.01 inclusion floor. Returns 0 whenever the package // already clears the target, so this only ever raises a fee. -function packageFeeUpliftSatoshis({ currentFee, txSize, ancestorSize, ancestorFees, targetFeePerBytes, satoshiUnit }){ +// `ancestorFees` is the package's fee in COIN units (what a node's mempool RPC +// reports). `ancestorFeeSatoshis` is the same quantity already in base units and +// takes precedence when supplied: the two-phase prefund below knows the commit's +// fee exactly, as an integer, and routing it through a coin-unit float would +// re-introduce the representation error the epsilon shave exists to absorb. +function packageFeeUpliftSatoshis({ currentFee, txSize, ancestorSize, ancestorFees, ancestorFeeSatoshis, targetFeePerBytes, satoshiUnit }){ if (!(targetFeePerBytes > 0)) return 0 if (!Number.isFinite(txSize) || txSize <= 0) return 0 const packageSize = Number(ancestorSize) if (!Number.isFinite(packageSize) || packageSize <= 0) return 0 - const ancestorFeeSat = Math.round(Number(ancestorFees) * satoshiUnit) + const ancestorFeeSat = (ancestorFeeSatoshis != null) + ? Math.round(Number(ancestorFeeSatoshis)) + : Math.round(Number(ancestorFees) * satoshiUnit) if (!Number.isFinite(ancestorFeeSat) || ancestorFeeSat < 0) return 0 // Rounding up a float product costs a whole base unit whenever the rate // carries binary representation error (0.01/1000 is not exact, so a 3000-byte @@ -249,6 +256,96 @@ function packageFeeUpliftSatoshis({ currentFee, txSize, ancestorSize, ancestorFe // that without false-positiving on routine operation. const DEFAULT_MAX_UTXO_TRACKER_LAG_BLOCKS = 2 +// THE tracker-freshness classifier. Pure: it reads a `sync` object and a lag +// ceiling and returns a verdict; it never throws, logs, or touches a connector. +// +// It exists because the same verdict was written twice against two different +// endpoints and drifted: _buildTransaction read the `sync` sibling that get_utxos +// carries, api.js getServeReadiness() read getSyncStatus(), and each re-derived +// halted / over-lag / behind-node / mempool-ready with its own strict-equality +// chain. Both surfaces now call this, so a create_tx refusal and an unhealthy +// /status can no longer disagree about the same tracker. +// +// Fail-open is deliberate and narrow, NOT an oversight: a field that is absent +// (an older tracker predating the ce16bdd freshness surface, or one predating +// the halt/mempool markers) must not refuse every build in the fleet, so only an +// EXPLICIT negative refuses. `halted` refuses on === true; `synced` refuses on +// === false; `mempool_ready` refuses on === false; a non-numeric lag is unknown +// and bounds nothing. An entirely missing `sync` yields present:false and no code. +// +// Returns: +// present - a usable sync object was supplied +// lag - numeric lag, or null when the tracker did not report one +// overLag - lag is above maxLagBlocks +// behindNode - lag is negative, i.e. the tracker's committed tip sits ABOVE +// the node's, so its outputs live in orphaned blocks +// halted - tracker stopped polling on an unrecoverable reorg +// mempoolReady - mempool index has reconverged (true unless explicitly false) +// syncedClaimed - the tracker's own positive `synced` assertion, truthy-tested. +// Readiness probes need this POSITIVE form (an omitted field is +// not an assertion of health), while the create_tx gate refuses +// only on the explicit negative; that asymmetry is the one real +// difference between the two surfaces and is now named instead +// of being buried in two `===` chains. +// code - null when nothing refuses, else the OperationalError code +// message - operator-facing refusal text, null when code is null +// details - error details payload, null when code is null +// +// Order matters: halted first (most specific physical cause), then staleness, +// then mempool readiness. An orphaned or lagging view also de-asserts +// mempool_ready (the tracker floors that field on the same negative lag), so +// checking readiness first would name mempool reconvergence for what is really a +// node reset. +function classifyTrackerFreshness(sync, maxLagBlocks){ + if (!sync || typeof sync !== 'object'){ + return { + present: false, lag: null, overLag: false, behindNode: false, + halted: false, mempoolReady: true, syncedClaimed: false, + code: null, message: null, details: null + } + } + // NaN is excluded explicitly: it is `typeof number`, so it passed the old check + // and was then reported verbatim as the lag while answering every comparison + // below with a silent false. Unknown is the honest verdict for it. Infinity is + // NOT excluded: it compares, and it compares as unboundedly stale, which is + // exactly the refusal a garbage lag should get. + const lag = (typeof sync.lag === 'number' && !Number.isNaN(sync.lag)) ? sync.lag : null + const overLag = (lag !== null) && (lag > maxLagBlocks) + const behindNode = (lag !== null) && (lag < 0) + const halted = sync.halted === true + const mempoolReady = sync.mempool_ready !== false + const syncedClaimed = !!sync.synced + const heights = { lag, tracker_height: sync.tracker_height, node_height: sync.node_height } + const verdict = { + present: true, lag, overLag, behindNode, halted, mempoolReady, syncedClaimed, + code: null, message: null, details: null + } + if (halted){ + verdict.code = 'UTXO_TRACKER_HALTED' + verdict.message = `utxo-tracker is halted (${sync.halt_reason || 'unrecoverable reorg'}); ` + + 'refusing to select utxos from it' + verdict.details = Object.assign({}, heights, { halt_reason: sync.halt_reason || null }) + return verdict + } + if (sync.synced === false || overLag || behindNode){ + verdict.code = 'UTXO_TRACKER_STALE' + verdict.message = `utxo-tracker view is stale (lag ${lag === null ? 'unknown' : lag} blocks` + + `${overLag ? `, exceeds ${maxLagBlocks}-block threshold` : ''}` + + `${behindNode ? `, tracker is ahead of the node so its view is orphaned` : ''}); ` + + 'refusing to select utxos from it' + verdict.details = heights + return verdict + } + if (!mempoolReady){ + verdict.code = 'UTXO_TRACKER_NOT_READY' + verdict.message = 'utxo-tracker has not reconverged its mempool yet, so an already-spent ' + + 'confirmed output cannot be filtered; refusing to select utxos from it' + verdict.details = heights + return verdict + } + return verdict +} + // How long a selected outpoint stays reserved against concurrent selection. // Long enough for a caller to sign and broadcast, short enough that an // abandoned selection auto-releases without operator intervention. In-memory @@ -1369,51 +1466,12 @@ class XChainEncoder { // `sync` is absent on an older tracker: fail OPEN (old behavior) rather // than block every create_tx, since this ships ahead of every tracker // in the fleet being upgraded. - const sync = fetched && fetched.sync - if (sync && typeof sync === 'object'){ - const lag = (typeof sync.lag === 'number') ? sync.lag : null - const overLag = (lag !== null) && (lag > this.maxUtxoTrackerLagBlocks) - // Negative lag: the tracker's committed tip sits ABOVE the node's, - // so its outputs live in blocks the node reset or reorged away. Only - // the upper bound was checked, so an orphaned view reached selection. - const behindNode = (lag !== null) && (lag < 0) - // Halted: the tracker stopped polling on an unrecoverable reorg and - // froze, possibly mid-rollback. It publishes this independently of - // `synced`, so a frozen height with an acceptable lag passed this - // gate. Strict === true keeps an older tracker, whose - // sync sibling carries no halt marker, on the existing fail-open path. - if (sync.halted === true){ - throw new OperationalError( - 'UTXO_TRACKER_HALTED', - `utxo-tracker is halted (${sync.halt_reason || 'unrecoverable reorg'}); refusing to select utxos from it`, - { lag, tracker_height: sync.tracker_height, node_height: sync.node_height, halt_reason: sync.halt_reason || null } - ) - } - // Position before readiness, most specific cause first. An orphaned or - // lagging view de-asserts mempool_ready too (the tracker floors that - // field on the same negative lag), so checking readiness first would - // name mempool reconvergence for a fault that is really a node reset. - if (sync.synced === false || overLag || behindNode){ - throw new OperationalError( - 'UTXO_TRACKER_STALE', - `utxo-tracker view is stale (lag ${lag === null ? 'unknown' : lag} blocks` + - `${overLag ? `, exceeds ${this.maxUtxoTrackerLagBlocks}-block threshold` : ''}` + - `${behindNode ? `, tracker is ahead of the node so its view is orphaned` : ''}); refusing to select utxos from it`, - { lag, tracker_height: sync.tracker_height, node_height: sync.node_height } - ) - } - // Mempool readiness: block sync flips true before the first mempool - // rebuild finishes, and until it does an empty mempool index cannot - // filter a confirmed output that is already spent in the node's - // mempool, so selection can pick an unspendable input. - // Strict === false again fails open for a pre-mempool_ready tracker. - if (sync.mempool_ready === false){ - throw new OperationalError( - 'UTXO_TRACKER_NOT_READY', - 'utxo-tracker has not reconverged its mempool yet, so an already-spent confirmed output cannot be filtered; refusing to select utxos from it', - { lag, tracker_height: sync.tracker_height, node_height: sync.node_height } - ) - } + // One classifier, shared with api.js getServeReadiness(); the ordering, + // the strict-equality fail-open rules and the operator text all live in + // classifyTrackerFreshness above. + const freshness = classifyTrackerFreshness(fetched && fetched.sync, this.maxUtxoTrackerLagBlocks) + if (freshness.code){ + throw new OperationalError(freshness.code, freshness.message, freshness.details) } utxos = fetched["utxos"] @@ -1478,31 +1536,38 @@ class XChainEncoder { } } - //Remove duplicated utxos (the utxo tracker returns duplicated utxos sometimes, this should be fixed) - //Also if unconfirmed is false, then all mempool txs will be eliminated - let utxoIndex = 0 - while (utxoIndex < utxos.length){ - let nextUtxo = utxos[utxoIndex] - - //if the tx is in the mempool, remove it if unconfirmed is false - if (!unconfirmed && (nextUtxo.confirmations == 0)){ - utxos.splice(utxoIndex, 1) - } else { - - let utxoDupIndex = utxoIndex + 1 - while (utxoDupIndex < utxos.length){ - let nextUtxoDup = utxos[utxoDupIndex] - - if ((nextUtxoDup.txid == nextUtxo.txid) && (nextUtxoDup.vout == nextUtxo.vout)){ - utxos.splice(utxoDupIndex, 1) - } else { - utxoDupIndex = utxoDupIndex + 1 - } - } - - utxoIndex = utxoIndex+1 - } - } + // Remove duplicated utxos (the utxo tracker returns duplicated utxos sometimes) + // and, when unconfirmed is false, every mempool entry along with them. + // + // Single linear pass with an outpoint Set, compacting in place. It replaces a + // nested while whose inner loop re-scanned the tail for every surviving entry + // and called Array.splice on each hit: O(n^2) comparisons plus an O(n) element + // shift per removal, so a duplicate-heavy set was cubic-ish in the worst case. + // The caller-supplied path is capped at MAX_UTXO_COUNT (500), but the + // tracker-fetched path deliberately is NOT (see the per-entry validation note + // above), so a hot address with tens of thousands of outputs spent the build + // inside this loop. + // + // Semantics preserved exactly: first occurrence of an outpoint wins, relative + // order of survivors is unchanged, the mempool filter is applied BEFORE dedup + // (so an unconfirmed duplicate cannot claim the slot of a confirmed one when + // unconfirmed=false), and the array is mutated IN PLACE because downstream code + // and the caller both hold this same reference. + const seenUtxoOutpoints = new Set() + let utxoWriteIndex = 0 + for (let utxoReadIndex = 0; utxoReadIndex < utxos.length; utxoReadIndex++){ + const nextUtxo = utxos[utxoReadIndex] + // if the tx is in the mempool, drop it if unconfirmed is false + if (!unconfirmed && (nextUtxo.confirmations == 0)) continue + // Loose-equality txid/vout matching became string-key matching: both sides + // are concatenated through the same String() coercion, so a numeric 0 and a + // string '0' vout still collide exactly as `==` made them collide. + const outpointKey = nextUtxo.txid + ':' + nextUtxo.vout + if (seenUtxoOutpoints.has(outpointKey)) continue + seenUtxoOutpoints.add(outpointKey) + utxos[utxoWriteIndex++] = nextUtxo + } + utxos.length = utxoWriteIndex //If unconfirmed=false stripped every mempool UTXO and nothing //confirmed remains, surface the same error as a never-funded @@ -1698,6 +1763,15 @@ class XChainEncoder { // the unsigned commit's txid (stable: commit inputs are segwit-only). let envelopeContext = null + // Two-phase reveal prefund bookkeeping, set by whichever commit branch + // sized a whole-reveal fee into a commit output (P2SH first leg, TAPROOT + // commit output). Consumed by the package-prefund pass after the commit's + // own fee is final. See that pass for why it cannot be done in the branch. + // outputIndex - the commit output carrying the reveal's fee money + // revealSize - whole-reveal size estimate, the package's child bytes + // revealFee - the fee that output currently prefunds + let revealPrefund = null + let estimatedTxSize = 0 for (let nextDataBufferIndex in preparedData["dataBufferArray"]){ @@ -1839,6 +1913,13 @@ class XChainEncoder { // The reveal's change is an authored output, so its headroom is the relay floor. let changeHeadroom = needsChangeOutput ? BigInt(this.outputFloor) : 0n spendingP2shEstimatedFee = asSatValue(BigInt(spendingP2shEstimatedFee) + BigInt(revealShortfall) + changeHeadroom) + // This leg is where the reveal's fee money lives, so it is + // the one the package-prefund pass tops up. + revealPrefund = { + outputIndex: psbt.txOutputs.length, + revealSize: this.estimateP2shRevealTx(preparedData["dataBufferArray"], 43), + revealFee: revealFeeNeeded + } } psbt.addOutput({ @@ -2040,6 +2121,12 @@ class XChainEncoder { } const commitValue = revealFee + this.dustAmount + (revealPadNeeded ? this.dustAmount : 0) + revealPrefund = { + outputIndex: psbt.txOutputs.length, + revealSize: revealVsize, + revealFee + } + psbt.addOutput({ script: commitPayment.output, value: commitValue @@ -2367,6 +2454,12 @@ class XChainEncoder { // data, an unreachable node, a connector without the method) leaves the // per-transaction fee exactly as it was. const cpfpUpliftBound = maxCpfpUpliftSat() + // Whatever unconfirmed ancestors this transaction inherits, carried out of + // the block below so the two-phase prefund pass can price the reveal + // against the WHOLE chain (ancestors + commit + reveal), not just against + // the commit. They stay 0 when nothing was measured. + let commitAncestorSize = 0 + let commitAncestorFeeSat = 0 if (unconfirmedInputTxids.length > 0 && feePerBytes > 0 && cpfpUpliftBound > 0 && this.connector && typeof this.connector.getUnconfirmedAncestorPackage === 'function'){ let ancestorPackage = null @@ -2375,6 +2468,14 @@ class XChainEncoder { } catch (err) { console.warn('Package fee sizing skipped: ancestor lookup failed:', err.message) } + if (ancestorPackage && Number.isFinite(Number(ancestorPackage.size)) && Number(ancestorPackage.size) > 0){ + commitAncestorSize = Number(ancestorPackage.size) + commitAncestorFeeSat = Math.round(Number(ancestorPackage.fees) * SATOSHI_UNIT) + if (!Number.isFinite(commitAncestorFeeSat) || commitAncestorFeeSat < 0){ + commitAncestorSize = 0 + commitAncestorFeeSat = 0 + } + } const wanted = ancestorPackage ? packageFeeUpliftSatoshis({ currentFee: estimatedFee, txSize: estimatedTxSize, @@ -2432,6 +2533,79 @@ class XChainEncoder { estimatedFee = this.dustAmount } + // Two-phase package prefund. + // + // A reveal spends nothing but the commit's own outputs, so the two are + // ALWAYS one mempool package and the miner judges them on the package's + // ancestor fee rate. The reveal's fee is money the commit set aside, and + // the reveal has no other input to raise it from, so a commit that ends + // up under the target rate drags the whole package under it and the + // reveal cannot rescue itself: the pair sits unmined with a confirmed + // commit stranded behind it. This is the shape a PRICE batch publish + // takes, and the commit lands under target whenever the caller supplies + // an explicit fee below the node's rate, or its own CPFP uplift was + // clamped (see the warning above), or its size estimate ran short. + // + // So prefund the reveal at the PACKAGE rate rather than at its own: the + // reveal carries the commit's shortfall, plus the shortfall of whatever + // unconfirmed ancestors the commit inherits. The money comes out of + // change, not out of thin air, and the commit's own fee is untouched - + // the pair's total is what a miner weighs, and only the reveal can still + // be raised at this point. + // + // This cannot live in the emission branch that sized the commit output: + // input selection, the caller-fee ceilings and the CPFP uplift all run + // after it, and the commit's fee and size are only final here. + if (revealPrefund && !p2shHash && feePerBytes > 0 && cpfpUpliftBound > 0){ + const wanted = packageFeeUpliftSatoshis({ + currentFee: revealPrefund.revealFee, + txSize: revealPrefund.revealSize, + ancestorSize: estimatedTxSize + commitAncestorSize, + ancestorFeeSatoshis: estimatedFee + commitAncestorFeeSat, + targetFeePerBytes: feePerBytes, + satoshiUnit: SATOSHI_UNIT + }) + + if (wanted > 0){ + // The same absolute bound the single-tx uplift answers to: this + // buys someone else's bytes, so no rate cap measured against one + // transaction's size can bound it. + let allowed = Math.min(wanted, cpfpUpliftBound) + + // Never spend an input that is not there. The prefund is paid out + // of change, so it is bounded by what change actually holds; + // turning a fundable build into INSUFFICIENT_FUNDS would be a + // worse outcome than a package that confirms late. + const headroom = inputSatoshis - outputSatoshis - BigInt(estimatedFee) + if (headroom <= 0n) allowed = 0 + else if (headroom < BigInt(allowed)) allowed = Number(headroom) + if (allowed < 0) allowed = 0 + + if (allowed < wanted){ + const packageSize = commitAncestorSize + estimatedTxSize + revealPrefund.revealSize + const packageFee = commitAncestorFeeSat + estimatedFee + revealPrefund.revealFee + allowed + console.warn(`Reveal package prefund clamped to ${allowed} of ${wanted} base units: this commit ` + + `pays ${estimatedFee} base units over ~${estimatedTxSize} bytes, so the commit/reveal package ` + + `will pay ${Math.round(packageFee / packageSize * 1000)} base units/kB against a target of ` + + `${Math.round(feePerBytes * SATOSHI_UNIT * 1000)}. The reveal cannot raise its own fee later, ` + + `so the pair may stay unmined. Raise MAX_CPFP_UPLIFT_SAT, the commit's fee, or the input balance.`) + } + + if (allowed > 0){ + this._raiseOutputValue(psbt, revealPrefund.outputIndex, allowed) + outputSatoshis = outputSatoshis + BigInt(allowed) + revealPrefund.revealFee = revealPrefund.revealFee + allowed + // The envelope reveal is built in THIS call against these two + // numbers, so both have to move with the output or its change + // output would silently hand the uplift back to the caller. + if (envelopeContext){ + envelopeContext.commitValue = envelopeContext.commitValue + allowed + envelopeContext.revealFee = envelopeContext.revealFee + allowed + } + } + } + } + // Validate the fee BEFORE the BigInt conversion below: a NaN/Infinity // fee would previously surface via the Number.isFinite(changeSatoshis) // check, but BigInt(estimatedFee) throws an opaque error on those. @@ -2536,10 +2710,77 @@ class XChainEncoder { revealEmittedOutputBytes = revealEmittedOutputBytes + TxSizeEstimator.estimateOutputSizeForAddress(customOutputs[i].address, this.network) } } - let revealFeeKept = Math.trunc((this.estimateP2shRevealTx(preparedData["dataBufferArray"], revealEmittedOutputBytes) * feePerBytes) * SATOSHI_UNIT) + const revealSizeForFee = this.estimateP2shRevealTx(preparedData["dataBufferArray"], revealEmittedOutputBytes) + let revealFeeKept = Math.trunc((revealSizeForFee * feePerBytes) * SATOSHI_UNIT) if (revealFeeKept < estimatedFee){ revealFeeKept = estimatedFee } + + // Keep the package prefund as FEE instead of sweeping it home. + // + // The reveal spends only the commit's legs, so commit and reveal are + // one mempool package and the miner weighs them together. The funding + // side sized these legs at the package rate precisely so this + // transaction can pay the commit's shortfall; without this the extra + // lands in the surplus and goes straight back to the caller's change, + // leaving the package exactly as under-target as it was. + // + // The commit's fee is read from the node's mempool rather than + // re-derived: the commit is a broadcast transaction by now, its ACTUAL + // size and fee are what the miner sees, and this also picks up any + // unconfirmed ancestors it inherited. An already-confirmed commit + // reports an empty package and asks for nothing, which is correct - a + // confirmed parent is no longer part of anyone's package. Any failure + // (no method, unreachable node, RPC error) leaves the fee exactly as + // it was; the reveal is then priced per-transaction, as before. + const revealUpliftBound = maxCpfpUpliftSat() + if (revealUpliftBound > 0 && feePerBytes > 0 && + this.connector && typeof this.connector.getUnconfirmedAncestorPackage === 'function'){ + let commitPackage = null + try { + commitPackage = await this.connector.getUnconfirmedAncestorPackage([p2shHash]) + } catch (err) { + console.warn('Reveal package fee sizing skipped: commit lookup failed:', err.message) + } + const wanted = commitPackage ? packageFeeUpliftSatoshis({ + currentFee: revealFeeKept, + txSize: revealSizeForFee, + ancestorSize: commitPackage.size, + ancestorFees: commitPackage.fees, + targetFeePerBytes: feePerBytes, + satoshiUnit: SATOSHI_UNIT + }) : 0 + if (wanted > 0){ + // The legs are the only money this transaction has, and every + // satoshi above the fee is already spoken for by the outputs + // it must emit. Spend the surplus and not one unit more. + // + // A reveal with no customOutputs leaves exactly one value + // output, the swept change, and that output is what keeps it + // off the SDK's FULL_BURN_FEE refusal. Reserve its floor here: + // eating it would strand the confirmed commit outright, which + // is a far worse outcome than a package that confirms late. + const emitsValueOutput = !!(customOutputs && Array.isArray(customOutputs) && customOutputs.length > 0) + const sweepReserve = emitsValueOutput ? 0n : BigInt(this.outputFloor) + const spendable = phaseLegInputSatoshis - outputSatoshis - BigInt(revealFeeKept) - sweepReserve + let allowed = Math.min(wanted, revealUpliftBound) + if (spendable <= 0n) allowed = 0 + else if (spendable < BigInt(allowed)) allowed = Number(spendable) + if (allowed < wanted){ + const packageSize = commitPackage.size + revealSizeForFee + const packageFee = Math.round(commitPackage.fees * SATOSHI_UNIT) + revealFeeKept + allowed + console.warn(`Reveal package fee uplift clamped to ${allowed} of ${wanted} base units: the commit ` + + `package totals ${commitPackage.size} bytes, so the commit/reveal package will pay ` + + `${Math.round(packageFee / packageSize * 1000)} base units/kB against a target of ` + + `${Math.round(feePerBytes * SATOSHI_UNIT * 1000)} and may stay unmined. The reveal has no ` + + `other input to raise, so rebuild the commit with a package-aware fee.`) + } + if (allowed > 0){ + revealFeeKept = revealFeeKept + allowed + } + } + } + let revealSurplus = phaseLegInputSatoshis - outputSatoshis - BigInt(revealFeeKept) let sweepAddress = change || resolveCallerAddress(pubkey, this.network) // The sweep is an authored output: below the relay floor the surplus stays as fee. @@ -2790,6 +3031,34 @@ class XChainEncoder { return size } + // Raise an already-emitted output's value in place. + // + // bitcoinjs exposes no setter for an output's value (updateOutput only + // touches PSBT-level fields), and re-emitting the output is not an option: + // the commit output's index is load-bearing (the envelope reveal spends it by + // vout, the decoder reads the chunk legs in order), so it has to stay put. + // The PSBT is unsigned at this point and every serialization path reads the + // same unsigned transaction object, so the write is seen by psbt.txOutputs, + // psbt.toHex() and the reveal builder alike. Asserted rather than assumed: + // a bitcoinjs release that reshapes this would otherwise underfund a reveal + // silently, which is the exact failure this whole pass exists to prevent. + _raiseOutputValue(psbt, outputIndex, delta){ + if (!Number.isInteger(delta) || delta <= 0){ + throw new RangeError('output uplift must be a positive integer') + } + const outs = psbt.data.globalMap.unsignedTx + && psbt.data.globalMap.unsignedTx.tx + && psbt.data.globalMap.unsignedTx.tx.outs + if (!Array.isArray(outs) || !outs[outputIndex]){ + throw new RangeError(`no output at index ${outputIndex} to raise`) + } + const raised = outs[outputIndex].value + delta + outs[outputIndex].value = raised + if (psbt.txOutputs[outputIndex].value !== raised){ + throw new RangeError('output value uplift did not reach the transaction bitcoinjs will serialize') + } + } + estimateSpendingP2shTx(redeemData){ // Per-chunk embedded value sized to cover the spending tx's worst // case at 1 sat/vbyte. Includes tx overhead, the OP_RETURN marker @@ -3077,6 +3346,9 @@ class XChainEncoder { // same rate createTx would charge; a quote the builder then ignores is worse than // no quote, because a wallet shows the user a fee that never applies. XChainEncoder.suggestedFeeCeilingPerByte = suggestedFeeCeilingPerByte +// Exported so api.js's readiness probe classifies a tracker with the exact same +// rules create_tx refuses one with; see classifyTrackerFreshness. +XChainEncoder.classifyTrackerFreshness = classifyTrackerFreshness XChainEncoder.suggestedFeeCeilingFloorPerByte = suggestedFeeCeilingFloorPerByte XChainEncoder.isTestNetworkKey = isTestNetworkKey XChainEncoder.DEFAULT_SUGGESTED_FEE_MAX_PER_VBYTE = DEFAULT_SUGGESTED_FEE_MAX_PER_VBYTE diff --git a/src/api.js b/src/api.js index 4801412..4d2ce99 100644 --- a/src/api.js +++ b/src/api.js @@ -292,15 +292,18 @@ async function getServeReadiness() { try { const status = await encoder.utxoTrackerConnector.getSyncStatus() tracker_reachable = true - const lag = status.lag !== undefined ? status.lag : null - const overLag = (lag !== null) && (lag > encoder.maxUtxoTrackerLagBlocks) - const behindNode = (lag !== null) && (lag < 0) - tracker_halted = status.halted === true - // Strict === false, matching create_tx's gate: a pre-mempool_ready tracker omits - // the field and stays on the existing fail-open path rather than reading unready. - tracker_mempool_ready = status.mempool_ready !== false - tracker_synced = !!status.synced && !overLag && !behindNode && !tracker_halted && tracker_mempool_ready - tracker_lag = lag + // This probe applies the SAME freshness classifier create_tx refuses a + // tracker with, against this endpoint's payload, so the board and the + // encoder cannot drift apart: both read getSyncStatus() through + // classifyTrackerFreshness instead of each re-deriving over-lag, + // behind-node, halted and mempool-ready. The remaining asymmetry is + // intentional: readiness needs the tracker's POSITIVE synced assertion + // (syncedClaimed), while the create_tx gate refuses only on negatives. + const freshness = XChainEncoder.classifyTrackerFreshness(status, encoder.maxUtxoTrackerLagBlocks) + tracker_halted = freshness.halted + tracker_mempool_ready = freshness.mempoolReady + tracker_synced = freshness.syncedClaimed && freshness.code === null + tracker_lag = freshness.lag } catch (_err) { // tracker unreachable; fields stay at defaults } diff --git a/src/applyBufferutilsPatch.js b/src/applyBufferutilsPatch.js index bc32cae..efce4b2 100644 --- a/src/applyBufferutilsPatch.js +++ b/src/applyBufferutilsPatch.js @@ -21,15 +21,26 @@ * addOutput at 21M BTC in sats (2.1e15). Dogecoin has no supply cap, so a * single output can legitimately exceed both ceilings (>~90.07M DOGE for * 2^53); without this patch the encoder cannot build such a PSBT at all. - * Sibling of xchain-decoder/src/applyBufferutilsPatch.js, which - * fixes the same 2^53 wall on the read side of the pipeline. * - * The patch teaches the loaded modules to carry a satoshi value as either - * a Number (unchanged fast path) or a BigInt up to 2^64-1 (the wire - * format's true ceiling). Readers return a Number whenever the value is - * exactly representable and a BigInt only above 2^53-1, so existing - * Number-based callers see identical behavior for every value they could - * already handle. + * This is the WRITE-SIDE contract, shared with xchain-sdk/src/ + * applyBufferutilsPatch.js: the patch teaches the loaded modules to carry a + * satoshi value as either a Number (unchanged fast path) or a BigInt up to + * 2^64-1 (the wire format's true ceiling). Readers return a Number whenever + * the value is exactly representable and a BigInt only above 2^53-1, so + * existing Number-based callers see identical behavior for every value they + * could already handle. The READ-SIDE copies (xchain-decoder and + * xchain-utxo-tracker, src/applyBufferutilsPatch.js) lift the same 2^53 wall + * for block decode but deliberately implement a DIFFERENT contract: + * BufferReader.readUInt64 always returns a BigInt and the module-level + * readUInt64LE/writeUInt64LE helpers keep the stock 2^53-1 ceiling. Only the + * 2^53 probe and the error strings are common to both, so this file is NOT a + * drop-in for either read-side copy and neither of those is one for this. + * + * Re-sync seam (uuid:311476c7). Everything below this banner is pinned + * byte-identical to the SDK copy's body by + * test/unit/applyBufferutilsPatch.test.js. Convergence runs SDK-ward: change + * the SDK copy first, then copy its body here. Only the two headers differ, + * because each names the other and its own read-side relationship. * * All bitcoinjs-lib/bip174 consumers hold the patched modules' exports * objects and class prototypes rather than destructured copies, so @@ -46,21 +57,28 @@ const MAX_SAFE_BIG = BigInt(Number.MAX_SAFE_INTEGER) // Bounds-check a satoshi value that may be a Number or a BigInt, mirroring // stock verifuint's error strings so callers relying on them keep working. +// Guard order is stock verifuint's, not a preference: non-number, negative, +// upper bound, fractional. Testing fractional first made -0.5 report a +// fractional component where stock reports a negative value, and converting to +// BigInt before the range check let +/-Infinity escape as a native BigInt +// RangeError instead of either stock string. Comparing before converting is +// what fixes both, and mixed Number/BigInt relational comparison is legal, so +// one pair of comparisons covers both accepted types (NaN fails both and falls +// to the fractional branch, exactly as stock does). function verifyU64(value) { if (typeof value !== 'number' && typeof value !== 'bigint') { throw new Error('cannot write a non-number as a number') } - if (typeof value === 'number' && Math.floor(value) !== value) { - throw new Error('value has a fractional component') - } - const big = BigInt(value) - if (big < 0n) { + if (value < 0) { throw new Error('specified a negative value for writing an unsigned value') } - if (big > MAX_U64) { + if (value > MAX_U64) { throw new Error('RangeError: value out of range') } - return big + if (typeof value === 'number' && Math.floor(value) !== value) { + throw new Error('value has a fractional component') + } + return BigInt(value) } // Number when exactly representable, BigInt above 2^53-1. Keeps every @@ -170,8 +188,14 @@ if (!psbtTxProto.__xchainBigIntValues) { // the public methods instead: try the stock implementation first (zero // behavior change for all-Number PSBTs) and, only on the BigInt-mixing // TypeError, recompute the amounts BigInt-safely, prime the caches the -// stock helper would have primed, and re-run the stock method (which then -// returns the cached result). +// stock helper would have primed, and answer from those caches ourselves. +// getFee / getFeeRate must NOT be re-run through stock after priming: stock +// getTxCacheValue tests the cache for TRUTHINESS, so a primed 0 (zero-fee +// PSBT, or any fee under 1 sat/vbyte) falls straight back into the Number +// summing and re-throws the very TypeError this wrapper exists to absorb. +// extractTransaction is safe to re-run: its checkFees reads the primed +// __FEE_RATE or calls the (wrapped) getFeeRate, and it returns the primed +// __EXTRACTED_TX. function witnessStackFromScriptWitness(buffer) { // Inverse of psbt.js's internal scriptWitnessToWitnessStack: a varint // element count followed by var-sliced witness elements. @@ -229,6 +253,8 @@ if (!Psbt.prototype.__xchainBigIntAmts) { } catch (err) { if (!(err instanceof TypeError) || !/BigInt/.test(err.message)) throw err primeBigIntSafeAmts(this) + if (method === 'getFee') return this.__CACHE.__FEE + if (method === 'getFeeRate') return this.__CACHE.__FEE_RATE return stock.apply(this, args) } } diff --git a/src/coins/BTC.js b/src/coins/BTC.js index c846405..caf3614 100644 --- a/src/coins/BTC.js +++ b/src/coins/BTC.js @@ -347,9 +347,14 @@ module.exports = { // capability SLASH block. Mainnet-inert until the EQUIV_HEADER flag-day. CONFIG_SLASH: { BOUNTY_BPS: 500, BOUNTY_FLOOR: '50.00000000', BOUNTY_CAP: '1000.00000000' }, - // Full-node possession-proof / verified-validator tier (NODEPROOF.md). The - // regtest-only env/sidecar overrides are applied by index.js for regtest ONLY; - // mainnet/testnet keep these frozen defaults with no env surface. + // Full-node possession-proof / verified-validator tier; see + // xchain-documentation/protocol/actions/nodeproof.md for the challenge/verdict + // action and xchain-documentation/getting-started/running-a-validator.md for the + // tier walkthrough. REWARD_SHARE 0 and an empty GENESIS_VERIFIERS below are the + // deliberate pre-activation state: the tier ships inert until a later flag-day + // arms it, not a bug. The regtest-only env/sidecar overrides are applied by + // index.js for regtest ONLY; mainnet/testnet keep these frozen defaults with no + // env surface. FULLNODE: { CHALLENGE_INTERVAL_BLOCKS: 144, CONFIRM_DEPTH: 100, diff --git a/src/coins/consensus_pin.js b/src/coins/consensus_pin.js index 53a2113..1adccc8 100644 --- a/src/coins/consensus_pin.js +++ b/src/coins/consensus_pin.js @@ -68,13 +68,15 @@ module.exports = { // untouched and were re-verified as unchanged by this edit. // REGENERATED 2026-09-01: GAS_SCHEDULE gains SWEEP_BASE, // SWEEP_PER_ITEM, CALLBACK_BASE and CALLBACK_PER_RECIPIENT, the unified prices - // SWEEP and CALLBACK move onto at the UNIFIED_FEES_SWEEP_CALLBACK flag day (both - // networks UNARMED; regtest genesis-active). GAS_SCHEDULE is hashed whole by - // consensusSubset(), so ADDING a key moves every hash even while the flag that - // reads it is unarmed, and the same one-wave rollout rule as every regeneration - // above applies in full: every service bundling these must ship the new values - // together, and a straggler fail-closes on verifyConsensusPin() at boot rather - // than forking. Mainnet stays null (Phase 6 arms it). + // SWEEP and CALLBACK move onto at the UNIFIED_FEES_SWEEP_CALLBACK flag day + // (mainnet armed at genesis since the genesis arm; testnet armed + // 2026-10-01T00:00:00Z; regtest genesis-active). GAS_SCHEDULE is hashed whole by + // consensusSubset(), so ADDING a key moves every hash regardless of which + // networks have the flag armed, and the same one-wave rollout rule as every + // regeneration above applies in full: every service bundling these must ship + // the new values together, and a straggler fail-closes on verifyConsensusPin() + // at boot rather than forking. CONSENSUS_CONFIG_PIN.mainnet above stays null + // regardless (Phase 6 arms that separate pin). testnet: { BTC: 'd3c66a4fb288b2666a2a4fad85200bbeac162bb36fed8a3eddcfc7b2d4d48070', LTC: 'ae94a951a838e64f9c36e503b978d9b9ad5ea74f7b443465baaabca8f675ea0d', diff --git a/test/unit/BlockchainConnector.test.js b/test/unit/BlockchainConnector.test.js index c906be1..571951a 100644 --- a/test/unit/BlockchainConnector.test.js +++ b/test/unit/BlockchainConnector.test.js @@ -660,6 +660,95 @@ describe('BlockchainConnector.getFeePerKilobyte()', () => { /Error getting smart fee from node/ ) }) + + // estimatesmartfee was taken verbatim on testnet/mainnet with no + // upper bound. A misbehaving/compromised/misconfigured node feeding a + // spiked estimate on this money-affecting path fed straight into what the + // caller was charged, and into the caller-facing caps' own anchor. + describe('sanity ceiling on the raw non-regtest estimate', () => { + let originalNetwork, originalCeiling, warnSpy, errorSpy + + beforeEach(() => { + originalNetwork = process.env.NETWORK + originalCeiling = process.env.FEE_ESTIMATE_SANITY_CEILING + delete process.env.NETWORK + delete process.env.FEE_ESTIMATE_SANITY_CEILING + errorSpy = [] + const originalError = console.error + console.error = (...args) => { errorSpy.push(args.join(' ')); } + warnSpy = originalError + }) + + afterEach(() => { + if (originalNetwork === undefined) delete process.env.NETWORK; else process.env.NETWORK = originalNetwork + if (originalCeiling === undefined) delete process.env.FEE_ESTIMATE_SANITY_CEILING; else process.env.FEE_ESTIMATE_SANITY_CEILING = originalCeiling + console.error = warnSpy + }) + + it('clamps a spiked mainnet estimate to the coin-default ceiling and logs loudly', async () => { + process.env.NETWORK = 'bitcoin-mainnet' + axios.post = async (url, data) => { + if (data.method === 'getblockchaininfo') return { data: { result: { chain: 'main' } } } + if (data.method === 'estimatesmartfee') return { data: { result: { feerate: 5 } } } // 5 BTC/kB: absurd + return { data: { result: {} } } + } + const c = makeConnector() + const feerate = await c.getFeePerKilobyte(1) + assert.strictEqual(feerate, 0.01, 'expected clamp to the bitcoin default ceiling 0.01 BTC/kB') + assert.ok(errorSpy.some(m => /sanity ceiling/.test(m)), 'expected a loud diagnostic') + }) + + it('clamps a spiked testnet estimate using the dogecoin-scale default when NETWORK says dogecoin', async () => { + process.env.NETWORK = 'dogecoin-testnet' + axios.post = async (url, data) => { + if (data.method === 'getblockchaininfo') return { data: { result: { chain: 'test' } } } + if (data.method === 'estimatesmartfee') return { data: { result: { feerate: 5000 } } } // 5000 DOGE/kB: absurd + return { data: { result: {} } } + } + const c = makeConnector() + const feerate = await c.getFeePerKilobyte(6) + assert.strictEqual(feerate, 10, 'expected clamp to the dogecoin default ceiling 10 DOGE/kB') + }) + + it('passes through an estimate under the ceiling unchanged', async () => { + process.env.NETWORK = 'bitcoin-mainnet' + axios.post = async (url, data) => { + if (data.method === 'getblockchaininfo') return { data: { result: { chain: 'main' } } } + if (data.method === 'estimatesmartfee') return { data: { result: { feerate: 0.0005 } } } + return { data: { result: {} } } + } + const c = makeConnector() + const feerate = await c.getFeePerKilobyte(1) + assert.strictEqual(feerate, 0.0005) + assert.strictEqual(errorSpy.length, 0, 'must not log when under the ceiling') + }) + + it('honours FEE_ESTIMATE_SANITY_CEILING as an override', async () => { + process.env.NETWORK = 'bitcoin-mainnet' + process.env.FEE_ESTIMATE_SANITY_CEILING = '0.0001' + axios.post = async (url, data) => { + if (data.method === 'getblockchaininfo') return { data: { result: { chain: 'main' } } } + if (data.method === 'estimatesmartfee') return { data: { result: { feerate: 0.0005 } } } + return { data: { result: {} } } + } + const c = makeConnector() + const feerate = await c.getFeePerKilobyte(1) + assert.strictEqual(feerate, 0.0001, 'expected clamp to the overridden ceiling') + }) + + it('ignores a non-positive or unparseable override and keeps the coin default', async () => { + process.env.NETWORK = 'litecoin-mainnet' + process.env.FEE_ESTIMATE_SANITY_CEILING = '-1' + axios.post = async (url, data) => { + if (data.method === 'getblockchaininfo') return { data: { result: { chain: 'main' } } } + if (data.method === 'estimatesmartfee') return { data: { result: { feerate: 5 } } } + return { data: { result: {} } } + } + const c = makeConnector() + const feerate = await c.getFeePerKilobyte(1) + assert.strictEqual(feerate, 0.01, 'expected fallback to the litecoin default ceiling') + }) + }) }) // Behavioral lock: every RPC call passes auth:{username,password} to axios, and diff --git a/test/unit/XChainEncoder.trackerFreshnessClassifier.test.js b/test/unit/XChainEncoder.trackerFreshnessClassifier.test.js new file mode 100644 index 0000000..66b3bd1 --- /dev/null +++ b/test/unit/XChainEncoder.trackerFreshnessClassifier.test.js @@ -0,0 +1,231 @@ +'use strict'; + +/********************************************************************* + * + * Copyright © 2025–2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC – https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ********************************************************************** + * ONE tracker-freshness classifier. + * + * There is exactly one place the verdict is written. Split across two + * endpoints, _buildTransaction reading the `sync` sibling on get_utxos and + * api.js getServeReadiness() reading getSyncStatus(), each re-derives halted / + * over-lag / behind-node / mempool-ready with its own strict-equality chain. + * Two chains drift, and the drift shows up as a status board painting Online on + * an encoder that refuses every create_tx. + * + * The first block pins the classifier's own behaviour. The second block is the + * one that actually matters: it drives BOTH surfaces with the same payload and + * asserts they reach the same verdict, so a future edit to one of them alone + * goes red. + * + ********************************************************************/ + +// api.js constructs an encoder from env at load time; set the minimum viable +// env BEFORE requiring it, exactly as reg-10 does. +process.env.NETWORK = process.env.NETWORK || 'bitcoin-regtest' +process.env.NODE_URL = process.env.NODE_URL || '127.0.0.1' +process.env.NODE_PORT = process.env.NODE_PORT || '8332' +process.env.NODE_USER = process.env.NODE_USER || 'test' +process.env.NODE_PASSWORD = process.env.NODE_PASSWORD || 'test' + +const assert = require('assert') +const XChainEncoder = require('../../src/XChainEncoder') +const { jsonRpcController, encoder: apiEncoder } = require('../../src/api') +const { + TXID_A, + makeUtxo, + makeEncoder, + getTestAddress +} = require('../integration/helpers/utxoFactory') + +const classify = XChainEncoder.classifyTrackerFreshness + +describe('classifyTrackerFreshness(): the single tracker-freshness verdict', function () { + + it('reports no refusal, and present:false, for a tracker with no freshness surface', function () { + for (const absent of [undefined, null, 'not an object', 42]) { + const v = classify(absent, 2) + assert.strictEqual(v.code, null, 'a pre-freshness tracker must fail open') + assert.strictEqual(v.present, false) + assert.strictEqual(v.lag, null) + } + }) + + it('refuses a halted tracker even at lag 0, and carries the halt reason', function () { + const v = classify({ tracker_height: 100, node_height: 100, lag: 0, synced: true, halted: true, + halt_reason: 'rolled back past the recovery window' }, 2) + assert.strictEqual(v.code, 'UTXO_TRACKER_HALTED') + assert.strictEqual(v.halted, true) + assert.strictEqual(v.details.halt_reason, 'rolled back past the recovery window') + assert.strictEqual(v.details.tracker_height, 100) + }) + + it('names the halt, not the staleness, when a halted tracker is ALSO stale', function () { + // Ordering is load-bearing: a halt froze the tracker mid-rollback, and the + // lag it froze at is a consequence, not the cause the operator needs. + const v = classify({ lag: 99, synced: false, halted: true, mempool_ready: false }, 2) + assert.strictEqual(v.code, 'UTXO_TRACKER_HALTED') + }) + + it('refuses a tracker that de-asserts synced, whatever the lag', function () { + const v = classify({ tracker_height: 100, node_height: 100, lag: 0, synced: false }, 2) + assert.strictEqual(v.code, 'UTXO_TRACKER_STALE') + assert.strictEqual(v.details.lag, 0) + }) + + it('refuses a lag above the supplied ceiling and names the ceiling it used', function () { + const v = classify({ lag: 7, synced: true }, 2) + assert.strictEqual(v.code, 'UTXO_TRACKER_STALE') + assert.strictEqual(v.overLag, true) + assert.ok(/exceeds 2-block threshold/.test(v.message), v.message) + }) + + it('takes the ceiling from its argument, not from a baked-in constant', function () { + // Same payload, two deployments: UTXO_TRACKER_MAX_LAG_BLOCKS moves the gate. + assert.strictEqual(classify({ lag: 7, synced: true }, 2).code, 'UTXO_TRACKER_STALE') + assert.strictEqual(classify({ lag: 7, synced: true }, 10).code, null) + }) + + it('serves at exactly the ceiling (lag == max is not "above")', function () { + const v = classify({ lag: 2, synced: true }, 2) + assert.strictEqual(v.code, null) + assert.strictEqual(v.overLag, false) + }) + + it('refuses an orphaned view (tracker committed above the node) and says so', function () { + const v = classify({ tracker_height: 1000, node_height: 900, lag: -100, synced: true }, 2) + assert.strictEqual(v.code, 'UTXO_TRACKER_STALE') + assert.strictEqual(v.behindNode, true) + assert.ok(/orphaned/.test(v.message), v.message) + }) + + it('refuses a tracker whose mempool has not reconverged', function () { + const v = classify({ lag: 0, synced: true, mempool_ready: false }, 2) + assert.strictEqual(v.code, 'UTXO_TRACKER_NOT_READY') + assert.strictEqual(v.mempoolReady, false) + }) + + it('fails open on every field it is not explicitly told is bad', function () { + // An older tracker publishes lag/synced only. Nothing here is an explicit + // negative, so nothing refuses; this is the property that let the gate ship + // ahead of the fleet. + const v = classify({ tracker_height: 100, node_height: 100, lag: 0, synced: true }, 2) + assert.strictEqual(v.code, null) + assert.strictEqual(v.halted, false) + assert.strictEqual(v.mempoolReady, true) + }) + + it('treats a non-numeric lag as unknown rather than comparing it', function () { + // A lag that is null, absent or a string bounds nothing; comparing it would + // make `'7' > 2` decide whether money moves. + for (const lag of [undefined, null, '7', NaN]) { + const v = classify({ lag, synced: true }, 2) + assert.strictEqual(v.overLag, false, `lag ${String(lag)} must not read as over-lag`) + assert.strictEqual(v.behindNode, false) + assert.strictEqual(v.lag, null) + assert.strictEqual(v.code, null) + } + }) + + it('separates the tracker\'s positive synced claim from the refusal verdict', function () { + // The one real asymmetry between the two surfaces: a readiness probe needs a + // POSITIVE assertion of health (an omitted `synced` is not one), while the + // create_tx gate refuses only on an explicit negative. + const omitted = classify({ lag: 0 }, 2) + assert.strictEqual(omitted.syncedClaimed, false, 'an omitted synced is not an assertion') + assert.strictEqual(omitted.code, null, 'but it is not an explicit negative either') + }) + + it('does not mutate the sync object it is handed', function () { + const sync = { tracker_height: 100, node_height: 100, lag: 0, synced: true } + const before = JSON.stringify(sync) + classify(sync, 2) + assert.strictEqual(JSON.stringify(sync), before) + }) +}) + +// The anti-drift test. Each row is one tracker state; both surfaces are driven +// with it and must agree. Before the shared classifier these two derivations +// lived 1200 lines apart and were edited independently. +const PARITY_CASES = [ + { name: 'healthy', sync: { tracker_height: 100, node_height: 100, lag: 0, synced: true, mempool_ready: true }, + expectCode: null }, + { name: 'lag at the ceiling', sync: { tracker_height: 98, node_height: 100, lag: 2, synced: true, mempool_ready: true }, + expectCode: null }, + { name: 'lag one over the ceiling', sync: { tracker_height: 97, node_height: 100, lag: 3, synced: true, mempool_ready: true }, + expectCode: 'UTXO_TRACKER_STALE' }, + { name: 'tracker de-asserts synced', sync: { tracker_height: 100, node_height: 100, lag: 0, synced: false, mempool_ready: true }, + expectCode: 'UTXO_TRACKER_STALE' }, + { name: 'orphaned view', sync: { tracker_height: 1000, node_height: 900, lag: -100, synced: true, mempool_ready: true }, + expectCode: 'UTXO_TRACKER_STALE' }, + { name: 'halted', sync: { tracker_height: 100, node_height: 100, lag: 0, synced: true, mempool_ready: true, halted: true, + halt_reason: 'unrecoverable reorg' }, + expectCode: 'UTXO_TRACKER_HALTED' }, + { name: 'mempool not reconverged', sync: { tracker_height: 100, node_height: 100, lag: 0, synced: true, mempool_ready: false }, + expectCode: 'UTXO_TRACKER_NOT_READY' } +] + +describe('create_tx and health() reach the same verdict on the same tracker @regression', function () { + + async function createTxVerdict(sync) { + const enc = makeEncoder('bitcoin-regtest') + const address = getTestAddress('bitcoin-regtest') + enc.utxoTrackerConnector.getUtxosFromAddress = async () => ({ + utxos: [makeUtxo('bitcoin-regtest', TXID_A, 0, 100000000)], + sync + }) + try { + const result = await enc.createTransaction( + null, address, null, 'test', null, 10000, false, null, address, + null, null, null, true, 0.00001 + ) + assert.ok(result.psbt) + return null + } catch (err) { + if (!err.operational) throw err + return err.xchainCode + } + } + + async function healthVerdict(sync) { + const orig = apiEncoder.utxoTrackerConnector.getSyncStatus + apiEncoder.utxoTrackerConnector.getSyncStatus = async () => sync + try { + return await jsonRpcController.health() + } finally { + apiEncoder.utxoTrackerConnector.getSyncStatus = orig + } + } + + for (const testCase of PARITY_CASES) { + it(`agrees on: ${testCase.name}`, async function () { + const code = await createTxVerdict(testCase.sync) + assert.strictEqual(code, testCase.expectCode, + `create_tx verdict for "${testCase.name}"`) + + const health = await healthVerdict(testCase.sync) + assert.strictEqual(health.tracker_synced, testCase.expectCode === null, + `health() must read ${testCase.expectCode === null ? 'serveable' : 'un-serveable'} ` + + `for "${testCase.name}", the same way create_tx does`) + }) + } + + it('reports the halt and the mempool state alongside the verdict', async function () { + const halted = await healthVerdict({ lag: 0, synced: true, halted: true, halt_reason: 'unrecoverable reorg' }) + assert.strictEqual(halted.tracker_halted, true) + assert.strictEqual(halted.tracker_synced, false) + + const unready = await healthVerdict({ lag: 0, synced: true, mempool_ready: false }) + assert.strictEqual(unready.tracker_mempool_ready, false) + assert.strictEqual(unready.tracker_synced, false) + }) +}) diff --git a/test/unit/XChainEncoder.utxoDedup.test.js b/test/unit/XChainEncoder.utxoDedup.test.js new file mode 100644 index 0000000..e7d870f --- /dev/null +++ b/test/unit/XChainEncoder.utxoDedup.test.js @@ -0,0 +1,183 @@ +'use strict'; + +/********************************************************************* + * + * Copyright © 2025–2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC – https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ********************************************************************** + * (a): the outpoint dedup / mempool filter in _buildTransaction. + * + * It was a nested while whose inner loop re-scanned the tail for every + * surviving entry and called Array.splice on each hit: quadratic comparisons + * plus an O(n) element shift per removal. The caller path is capped at + * MAX_UTXO_COUNT, but the tracker-fetched path is deliberately uncapped, so a + * hot address spent the whole build inside that loop. + * + * The semantics tests below pin what must not change; the last one pins that it + * is no longer quadratic, and is the test that goes red if the old loop returns. + * + ********************************************************************/ + +const assert = require('assert') +const { + TXID_A, + TXID_B, + TXID_C, + makeUtxo, + makeMempoolUtxo, + makeEncoder, + getTestAddress +} = require('../integration/helpers/utxoFactory') + +const NETWORK = 'bitcoin-regtest' + +// The dedup filters the caller's array IN PLACE, so the array a test hands in is +// the observation surface. A fresh encoder holds no reservations, so the +// head-of-order splice further down never fires and the surviving order is the +// dedup's own output. +function build(encoder, address, utxos, unconfirmed) { + return encoder.createTransaction( + utxos, address, null, 'test', null, 10000, false, null, address, + null, null, null, unconfirmed, 0.00001 + ) +} + +function outpoints(utxos) { + return utxos.map((u) => u.txid + ':' + u.vout) +} + +describe('_buildTransaction outpoint dedup / mempool filter', function () { + + it('keeps the FIRST occurrence of a repeated outpoint and drops the rest', async function () { + const encoder = makeEncoder(NETWORK) + const address = getTestAddress(NETWORK) + const first = makeUtxo(NETWORK, TXID_A, 0, 100000000) + const dup = makeUtxo(NETWORK, TXID_A, 0, 100000000) + const utxos = [first, dup, makeUtxo(NETWORK, TXID_B, 0, 100000000)] + + const result = await build(encoder, address, utxos, true) + assert.ok(result.psbt) + assert.deepStrictEqual(outpoints(utxos), [TXID_A + ':0', TXID_B + ':0']) + assert.strictEqual(utxos[0], first, 'the FIRST occurrence must be the survivor, not the last') + }) + + it('preserves the relative order of the survivors', async function () { + const encoder = makeEncoder(NETWORK) + const address = getTestAddress(NETWORK) + // Interleaved duplicates: an implementation that compacts by swapping with + // the tail instead of shifting would reorder these. + const utxos = [ + makeUtxo(NETWORK, TXID_A, 0, 100000000), + makeUtxo(NETWORK, TXID_B, 0, 100000000), + makeUtxo(NETWORK, TXID_A, 0, 100000000), + makeUtxo(NETWORK, TXID_C, 0, 100000000), + makeUtxo(NETWORK, TXID_B, 0, 100000000), + makeUtxo(NETWORK, TXID_C, 1, 100000000) + ] + + await build(encoder, address, utxos, true) + assert.deepStrictEqual(outpoints(utxos), + [TXID_A + ':0', TXID_B + ':0', TXID_C + ':0', TXID_C + ':1']) + }) + + it('distinguishes two vouts of the same txid', async function () { + const encoder = makeEncoder(NETWORK) + const address = getTestAddress(NETWORK) + const utxos = [ + makeUtxo(NETWORK, TXID_A, 0, 100000000), + makeUtxo(NETWORK, TXID_A, 1, 100000000), + makeUtxo(NETWORK, TXID_A, 2, 100000000) + ] + + await build(encoder, address, utxos, true) + assert.strictEqual(utxos.length, 3, 'same txid, different vout, is not a duplicate') + }) + + it('drops every mempool entry when unconfirmed is false', async function () { + const encoder = makeEncoder(NETWORK) + const address = getTestAddress(NETWORK) + const utxos = [ + makeMempoolUtxo(TXID_A, 0, 100000000), + makeUtxo(NETWORK, TXID_B, 0, 100000000), + makeMempoolUtxo(TXID_C, 0, 100000000) + ] + + await build(encoder, address, utxos, false) + assert.deepStrictEqual(outpoints(utxos), [TXID_B + ':0']) + }) + + it('keeps mempool entries when unconfirmed is true', async function () { + const encoder = makeEncoder(NETWORK) + const address = getTestAddress(NETWORK) + const utxos = [ + makeMempoolUtxo(TXID_A, 0, 100000000), + makeUtxo(NETWORK, TXID_B, 0, 100000000) + ] + + await build(encoder, address, utxos, true) + assert.deepStrictEqual(outpoints(utxos), [TXID_A + ':0', TXID_B + ':0']) + }) + + it('applies the mempool filter BEFORE dedup, so a confirmed twin survives', async function () { + // The ordering is load-bearing. Dedup-first would let the unconfirmed entry + // claim the outpoint's slot and then the mempool filter would delete it, + // silently losing a confirmed, spendable output. + const encoder = makeEncoder(NETWORK) + const address = getTestAddress(NETWORK) + const confirmed = makeUtxo(NETWORK, TXID_A, 0, 100000000) + const utxos = [makeMempoolUtxo(TXID_A, 0, 100000000), confirmed] + + await build(encoder, address, utxos, false) + assert.deepStrictEqual(outpoints(utxos), [TXID_A + ':0']) + assert.strictEqual(utxos[0], confirmed, 'the confirmed twin must be what survives') + assert.strictEqual(utxos[0].confirmations, 6) + }) + + it('collapses a set that is nothing but copies of one outpoint', async function () { + const encoder = makeEncoder(NETWORK) + const address = getTestAddress(NETWORK) + const utxos = [] + for (let i = 0; i < 5000; i++) utxos.push(makeUtxo(NETWORK, TXID_A, 0, 100000000)) + + const result = await build(encoder, address, utxos, true) + assert.ok(result.psbt) + assert.strictEqual(utxos.length, 1, '5000 copies of one outpoint collapse to one') + }) + + // The defect itself, in the shape that actually hurts: an address holding many + // DISTINCT outputs. Nothing is spliced, so the cost is pure comparison, and the + // old inner while re-scanned the whole tail for every surviving entry: n^2/2 + // txid string compares. Measured on this machine with the old loop: 5000 -> 119ms, + // 10000 -> 423ms, 20000 -> 1739ms (a clean 4x per doubling). The linear pass: + // 14ms at 20000, 26ms at 40000. The caller path is capped at MAX_UTXO_COUNT, but + // the tracker-fetched path is deliberately uncapped, so this is reachable. + // + // The bound measures the complexity class, not the machine: it sits ~35x above + // the linear timing and ~3.5x below the quadratic one. + it('filters a large set of DISTINCT outpoints in linear time', async function () { + this.timeout(30000) + const encoder = makeEncoder(NETWORK) + const address = getTestAddress(NETWORK) + const utxos = [] + for (let i = 0; i < 20000; i++) { + utxos.push(makeUtxo(NETWORK, i.toString(16).padStart(64, '0'), 0, 1000)) + } + + const started = Date.now() + const result = await build(encoder, address, utxos, true) + const elapsed = Date.now() - started + + assert.ok(result.psbt) + assert.strictEqual(utxos.length, 20000, 'distinct outpoints are not duplicates') + assert.ok(elapsed < 500, + `filtering 20000 distinct outpoints took ${elapsed}ms; the quadratic scan is back`) + }) +}) diff --git a/test/unit/applyBufferutilsPatch.test.js b/test/unit/applyBufferutilsPatch.test.js index 1f48fc6..091338e 100644 --- a/test/unit/applyBufferutilsPatch.test.js +++ b/test/unit/applyBufferutilsPatch.test.js @@ -5,8 +5,21 @@ // relies on. const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); const bufferutils = require('../../src/applyBufferutilsPatch.js'); +// The two copies share one body and differ only in the licence/doc banner, +// because each header names the other copy and its own read-side relationship. +// Split on the banner terminator and compare what follows. +const BANNER_END = '********************************************************************/'; +function patchBody(file) { + const source = fs.readFileSync(file, 'utf8'); + const end = source.indexOf(BANNER_END); + assert.notStrictEqual(end, -1, 'no banner terminator in ' + file); + return source.slice(end + BANNER_END.length); +} + describe('applyBufferutilsPatch', function () { it('exposes the patched bufferutils surface', function () { assert.strictEqual(typeof bufferutils.readUInt64LE, 'function'); @@ -38,4 +51,142 @@ describe('applyBufferutilsPatch', function () { assert.strictEqual(Number(dec), n, `varuint failed for ${n}`); } }); + + // The write-side contract this copy ships (and that the header documents): + // readers narrow to a Number at or below 2^53-1 and return a BigInt above + // it; the module-level helpers accept the full u64 range. The decoder and + // utxo-tracker copies deliberately differ (always-BigInt reader, stock + // 2^53-1 helper ceiling), so this pins THIS copy rather than a shared one. + it('narrows a representable value to Number and keeps a BigInt only above 2^53-1', function () { + const buf = Buffer.alloc(8); + bufferutils.writeUInt64LE(buf, 9007199254740991n, 0); // 2^53-1 + assert.strictEqual(bufferutils.readUInt64LE(buf, 0), 9007199254740991); + assert.strictEqual(typeof new bufferutils.BufferReader(buf).readUInt64(), 'number'); + bufferutils.writeUInt64LE(buf, 9007199254740992n, 0); // 2^53 + assert.strictEqual(bufferutils.readUInt64LE(buf, 0), 9007199254740992n); + assert.strictEqual(typeof new bufferutils.BufferReader(buf).readUInt64(), 'bigint'); + }); + + it('module-level helpers accept the full u64 range and reject one past it', function () { + const buf = Buffer.alloc(8); + bufferutils.writeUInt64LE(buf, 0xffffffffffffffffn, 0); + assert.strictEqual(bufferutils.readUInt64LE(buf, 0), 0xffffffffffffffffn); + assert.throws(() => bufferutils.writeUInt64LE(buf, 0x10000000000000000n, 0), /value out of range/); + }); + + // The header promises stock verifuint's error strings, and a caller that + // branches on them only keeps working if EVERY rejected value produces the + // one stock verifuint produces. Testing fractional before the range checks + // broke that for -0.5, and converting to BigInt before them let +/-Infinity + // out as a native BigInt RangeError that is neither stock string. + it('rejects every invalid value with the stock verifuint string', function () { + const buf = Buffer.alloc(8); + const cases = [ + ['5', /cannot write a non-number as a number/], + [-0.5, /specified a negative value for writing an unsigned value/], + [-1, /specified a negative value for writing an unsigned value/], + [-1n, /specified a negative value for writing an unsigned value/], + [-Infinity, /specified a negative value for writing an unsigned value/], + [Infinity, /value out of range/], + [1e30, /value out of range/], + [0.5, /value has a fractional component/], + [NaN, /value has a fractional component/] + ]; + for (const [value, expected] of cases) { + assert.throws(() => bufferutils.writeUInt64LE(buf, value, 0), expected, + 'wrong error string for ' + String(value)); + // The native BigInt conversion error is the specific escape hatch + // the guard order closes, so name it rather than only the shape. + assert.throws(() => bufferutils.writeUInt64LE(buf, value, 0), + (err) => !/cannot be converted to a BigInt/.test(err.message), + 'a native BigInt conversion error escaped for ' + String(value)); + } + }); + + // Fee-accounting wrapper: bitcoinjs-lib's stock cache getter tests __FEE / + // __FEE_RATE for truthiness, so a primed 0 (zero fee, or any fee under + // 1 sat/vbyte) must be answered by the wrapper itself rather than by + // re-running stock, which would re-throw the BigInt-mixing TypeError. + describe('BigInt fee accounting (getFee / getFeeRate / extractTransaction)', function () { + const { Psbt } = require('bitcoinjs-lib'); + function finalizedPsbt(inputValue, outputValue) { + const psbt = new Psbt(); + psbt.addInput({ + hash: Buffer.alloc(32, 1), index: 0, + witnessUtxo: { script: Buffer.from('0014' + '11'.repeat(20), 'hex'), value: inputValue } + }); + psbt.addOutput({ script: Buffer.from('0014' + '22'.repeat(20), 'hex'), value: outputValue }); + // An empty witness stack is enough for isFinalized; no signing needed. + psbt.updateInput(0, { finalScriptWitness: Buffer.from([0]) }); + return psbt; + } + + it('returns 0 for a zero-fee PSBT with BigInt values instead of re-throwing', function () { + const psbt = finalizedPsbt(1000n, 1000n); + assert.strictEqual(psbt.getFee(), 0); + assert.strictEqual(psbt.getFeeRate(), 0); + assert.ok(psbt.extractTransaction()); + }); + + it('returns a 0 fee rate for a sub-1-sat/vbyte fee instead of re-throwing', function () { + const psbt = finalizedPsbt(100000n, 99990n); + assert.strictEqual(psbt.getFee(), 10); + assert.strictEqual(psbt.getFeeRate(), 0); + assert.ok(psbt.extractTransaction()); + }); + + it('still computes a non-zero fee and fee rate across the BigInt path', function () { + const psbt = finalizedPsbt(9007199254740993n, 9007199254740000n); // input above 2^53 + assert.strictEqual(psbt.getFee(), 993); + assert.ok(psbt.getFeeRate() > 0); + assert.ok(psbt.extractTransaction()); + }); + + it('the all-Number fast path is unchanged', function () { + const psbt = finalizedPsbt(100000, 90000); + assert.strictEqual(psbt.getFee(), 10000); + assert.ok(psbt.getFeeRate() > 0); + }); + }); + + // Twin guard. This file is a fork of xchain-sdk/src/applyBufferutilsPatch.js + // and has drifted apart three times. Convergence runs SDK-ward, so pin the + // bodies byte-identical and let the fork be a deliberate, visible act + // rather than a silent one. + describe('twin guard against the SDK copy', function () { + const repoRoot = path.resolve(__dirname, '../../..'); + const sdkRoot = path.join(repoRoot, 'xchain-sdk'); + const mine = path.resolve(__dirname, '../../src/applyBufferutilsPatch.js'); + const twin = path.join(sdkRoot, 'src/applyBufferutilsPatch.js'); + + it('is byte-identical to the SDK copy below the banner', function () { + // Skipped ONLY where the sibling repo is absent entirely (a + // standalone encoder checkout); inside the monorepo it must run, + // so a missing twin file next to a present xchain-sdk is a failure. + if (!fs.existsSync(sdkRoot)) return this.skip(); + assert.ok(fs.existsSync(twin), 'xchain-sdk is present but its copy of the patch is missing: ' + twin); + assert.strictEqual(patchBody(mine), patchBody(twin), + 'applyBufferutilsPatch.js has drifted from the SDK copy; converge SDK-ward'); + }); + + it('declares every package it deep-requires', function () { + // The patch reaches into bip174 internals. An undeclared dependency + // resolves today only because bitcoinjs-lib happens to hoist it, + // and vanishes on any transitive bump. + const pkg = JSON.parse(fs.readFileSync(path.resolve(__dirname, '../../package.json'), 'utf8')); + const source = fs.readFileSync(mine, 'utf8'); + const required = new Set(); + for (const m of source.matchAll(/require\('([^']+)'\)/g)) { + if (m[1].startsWith('.')) continue; + const parts = m[1].split('/'); + required.add(m[1].startsWith('@') ? parts.slice(0, 2).join('/') : parts[0]); + } + for (const name of required) { + assert.ok( + Object.prototype.hasOwnProperty.call(pkg.dependencies || {}, name), + name + ' is required by src/applyBufferutilsPatch.js but not declared in package.json dependencies' + ); + } + }); + }); }); diff --git a/test/unit/packageFeeSizing.test.js b/test/unit/packageFeeSizing.test.js index 80bf0b7..4a19f2b 100644 --- a/test/unit/packageFeeSizing.test.js +++ b/test/unit/packageFeeSizing.test.js @@ -219,6 +219,20 @@ describe('packageFeeUpliftSatoshis() @regression @tier1', () => { }), 0, 'a rich ancestor must never LOWER this fee') }) + it('takes the ancestor fee in base units when one is given, ahead of the coin-unit figure', () => { + // The two-phase prefund knows the commit's fee exactly, as an integer. A + // coin-unit round trip would reintroduce the float error the ceiling shaves. + const owed = uplift({ + currentFee: 0, + txSize: 1000, + ancestorSize: 2000, + ancestorFees: 1, // a wildly different coin-unit figure + ancestorFeeSatoshis: 626000, + targetFeePerBytes: 0.01 / 1000 + }) + assert.strictEqual(owed, 2374000, 'the base-unit figure wins') + }) + it('returns 0 for an empty package or a missing target', () => { const base = { currentFee: 0, txSize: 250, ancestorSize: 0, ancestorFees: 0, targetFeePerBytes: 0.00001 } assert.strictEqual(uplift(base), 0) @@ -447,3 +461,327 @@ describe('XChainEncoder package-aware fee sizing @regression @tier1', () => { assert.ok(outputs <= 400000, 'the transaction may never pay out more than it takes in') }) }) + +// A reveal spends nothing but the commit's own outputs, so the pair is +// always one mempool package and a miner weighs them together. The reveal's fee +// is money the commit set aside and the reveal has no second input to raise it +// from, so a commit that lands under the target rate drags the package under it +// and nothing downstream can fix that: the pair sits unmined behind a confirmed +// commit. These pin that the commit prefunds the reveal at the PACKAGE rate, and +// that the P2SH reveal keeps that money as fee instead of sweeping it home. +describe('two-phase reveal package prefund @regression @tier1', () => { + const ecc = require('tiny-secp256k1') + const { ECPairFactory } = require('ecpair') + bitcoin.initEccLib(ecc) + const KEY = ECPairFactory(ecc).fromPrivateKey(Buffer.alloc(32, 7)) + const PUBKEY_HEX = Buffer.from(KEY.publicKey).toString('hex') + + const BTC_REGTEST = require('../../src/CryptoNetworks').getBitcoinJsNetwork('bitcoin-regtest') + const BTC_RATE_KB = 0.0001 // 10 sat/byte + const BTC_TARGET_PER_BYTE = BTC_RATE_KB * SATOSHI_UNIT / 1000 + const COMMIT_INPUT_VALUE = 100000000 + // Far under the target rate for a commit of any size: the congestion-clamped + // commit the ledger entry describes, reproduced as an explicit caller fee. + const UNDER_TARGET_COMMIT_FEE = 700 + const ENVELOPE_PAYLOAD = 'x'.repeat(20000) + + afterEach(() => { delete process.env.MAX_CPFP_UPLIFT_SAT }) + + function envelopeEncoder (ancestorPackage) { + const encoder = new XChainEncoder('bitcoin-regtest', '127.0.0.1', '8333', 'rpc', 'rpc', '', '') + encoder.connector = { + getFeePerKilobyte: async () => BTC_RATE_KB, + // Without a relayfee the suggested-rate ceiling clamps to its default and + // the probe rate below would never be priced at all. + getNetworkInfo: async () => ({ relayfee: 0.00001 }), + getTransactionHex: async () => { throw new Error('unit test: no node') } + } + if (ancestorPackage !== undefined) { + encoder.connector.getUnconfirmedAncestorPackage = async () => + (typeof ancestorPackage === 'function' ? ancestorPackage() : ancestorPackage) + } + return encoder + } + + function callerAddress () { + return bitcoin.payments.p2wpkh({ pubkey: Buffer.from(KEY.publicKey), network: BTC_REGTEST }).address + } + + function segwitUtxo (value, confirmations) { + const p2wpkh = bitcoin.payments.p2wpkh({ pubkey: Buffer.from(KEY.publicKey), network: BTC_REGTEST }) + return { txid: TXID_PARENT_A, vout: 0, value, confirmations, scriptPubKey: p2wpkh.output.toString('hex') } + } + + async function buildEnvelope (encoder, { commitFee = null, inputValue = COMMIT_INPUT_VALUE, confirmations = 6 } = {}) { + encoder.clearReservations() + const addr = callerAddress() + return encoder.createTransaction( + [segwitUtxo(inputValue, confirmations)], addr, null, 'FILE|0|package-prefund', + ENVELOPE_PAYLOAD, commitFee, false, 'TAPROOT', addr, null, null, PUBKEY_HEX) + } + + function unsignedTx (psbt) { + return bitcoin.Transaction.fromBuffer(psbt.data.globalMap.unsignedTx.toBuffer()) + } + + // Sign both halves so the package is measured on the bytes a miner actually + // sees, not on the builder's own estimate of them. + function signedPair (result) { + const signer = { + publicKey: Buffer.from(KEY.publicKey), + sign: (h) => Buffer.from(KEY.sign(h)), + signSchnorr: (h) => Buffer.from(ecc.signSchnorr(h, KEY.privateKey)) + } + const commitPsbt = bitcoin.Psbt.fromHex(result.psbt.toHex()) + commitPsbt.signAllInputs(signer) + commitPsbt.finalizeAllInputs() + const commitTx = commitPsbt.extractTransaction() + + const revealPsbt = result.revealPsbt + revealPsbt.signInput(0, signer) + revealPsbt.finalizeAllInputs() + const revealTx = revealPsbt.extractTransaction() + return { commitTx, revealTx } + } + + it('lifts the prefund so a commit paying under the target rate still clears it as a package', async () => { + const result = await buildEnvelope(envelopeEncoder(), { commitFee: UNDER_TARGET_COMMIT_FEE }) + const { commitTx, revealTx } = signedPair(result) + + const commitFee = COMMIT_INPUT_VALUE - commitTx.outs.reduce((s, o) => s + o.value, 0) + assert.strictEqual(commitFee, UNDER_TARGET_COMMIT_FEE, 'the commit still pays exactly what the caller asked') + assert.ok(commitFee / commitTx.virtualSize() < BTC_TARGET_PER_BYTE, + 'this probe is only meaningful while the commit itself is under target') + + const revealFee = commitTx.outs[result.envelope.commitVout].value - + revealTx.outs.reduce((s, o) => s + o.value, 0) + const packageRate = (commitFee + revealFee) / (commitTx.virtualSize() + revealTx.virtualSize()) + assert.ok(packageRate >= BTC_TARGET_PER_BYTE, + `commit+reveal package pays ${packageRate} sat/byte, under the ${BTC_TARGET_PER_BYTE} target`) + }) + + it('leaves the reveal its change output rather than paying the uplift out of it', async () => { + const encoder = envelopeEncoder() + const result = await buildEnvelope(encoder, { commitFee: UNDER_TARGET_COMMIT_FEE }) + // commitValue = revealFee + dust by construction; both move together or the + // uplift silently returns to the caller as change instead of reaching miners. + assert.strictEqual(result.envelope.commitValue, result.envelope.revealFee + encoder.dustAmount) + assert.strictEqual(Number(result.revealPsbt.txOutputs[0].value), encoder.dustAmount, + 'the reveal still leaves exactly one dust output back to the caller') + const commitTx = unsignedTx(result.psbt) + assert.strictEqual(commitTx.outs[result.envelope.commitVout].value, result.envelope.commitValue, + 'the commit output on the wire carries the raised value') + }) + + it('adds nothing when the commit already pays the target rate on its own bytes', async () => { + // Same build, node-derived fee: the commit pays the target, so the package + // already clears it and the prefund must not grow by a single satoshi. + const withPackaging = await buildEnvelope(envelopeEncoder()) + process.env.MAX_CPFP_UPLIFT_SAT = '0' + const withoutPackaging = await buildEnvelope(envelopeEncoder()) + assert.strictEqual(withPackaging.envelope.revealFee, withoutPackaging.envelope.revealFee, + 'a commit at target must not buy the reveal a bigger prefund') + }) + + it('carries ancestors the commit\'s own capped uplift could not pay for', async () => { + // A commit whose CPFP uplift is capped at the node rate cannot pay for the + // free ancestors it spends: the rate cap measures a fee against the commit's + // OWN bytes, so there is no room in it for anyone else's. The reveal is the + // only half of the package left that can still be raised, and its prefund is + // bought with change rather than with commit fee, so the cap does not bind it. + const capped = (ancestorPackage) => { + const encoder = new XChainEncoder('bitcoin-regtest', '127.0.0.1', '8333', 'rpc', 'rpc', '', '', + BTC_RATE_KB * SATOSHI_UNIT) + encoder.connector = { + getFeePerKilobyte: async () => BTC_RATE_KB, + getNetworkInfo: async () => ({ relayfee: 0.00001 }), + getTransactionHex: async () => { throw new Error('unit test: no node') }, + getUnconfirmedAncestorPackage: async () => ancestorPackage + } + return encoder + } + const originalWarn = console.warn + console.warn = () => {} + let withAncestors, withoutAncestors + try { + withAncestors = await buildEnvelope(capped({ size: 2000, fees: 0 }), { confirmations: 0 }) + withoutAncestors = await buildEnvelope(capped({ size: 0, fees: 0 }), { confirmations: 0 }) + } finally { + console.warn = originalWarn + } + assert.ok(withAncestors.envelope.revealFee > withoutAncestors.envelope.revealFee, + `unpaid ancestors must raise the prefund (${withAncestors.envelope.revealFee} vs ${withoutAncestors.envelope.revealFee})`) + // 2000 free ancestor bytes at the 10 sat/byte target is what is owed, give or + // take the single base unit the rate's float representation costs. + const owed = withAncestors.envelope.revealFee - withoutAncestors.envelope.revealFee + assert.ok(Math.abs(owed - 2000 * BTC_TARGET_PER_BYTE) <= 1, + `owed ${owed} against ${2000 * BTC_TARGET_PER_BYTE} for the unpaid ancestor bytes`) + }) + + it('MAX_CPFP_UPLIFT_SAT=0 turns the prefund off entirely', async () => { + const baseline = await buildEnvelope(envelopeEncoder(), { commitFee: UNDER_TARGET_COMMIT_FEE }) + process.env.MAX_CPFP_UPLIFT_SAT = '0' + const disabled = await buildEnvelope(envelopeEncoder(), { commitFee: UNDER_TARGET_COMMIT_FEE }) + assert.ok(baseline.envelope.revealFee > disabled.envelope.revealFee, + 'the probe must actually be lifting something when sizing is on') + }) + + it('clamps at MAX_CPFP_UPLIFT_SAT and warns that the package stays under target', async () => { + process.env.MAX_CPFP_UPLIFT_SAT = '0' + const disabled = await buildEnvelope(envelopeEncoder(), { commitFee: UNDER_TARGET_COMMIT_FEE }) + + process.env.MAX_CPFP_UPLIFT_SAT = '11' + const warnings = [] + const originalWarn = console.warn + console.warn = (...args) => warnings.push(args.join(' ')) + let clamped + try { + clamped = await buildEnvelope(envelopeEncoder(), { commitFee: UNDER_TARGET_COMMIT_FEE }) + } finally { + console.warn = originalWarn + } + assert.strictEqual(clamped.envelope.revealFee, disabled.envelope.revealFee + 11, + 'the prefund stops at exactly the configured bound') + assert.ok(warnings.some(w => /Reveal package prefund clamped/.test(w) && /MAX_CPFP_UPLIFT_SAT/.test(w)), + 'the operator must be told the package will stay under target: ' + warnings.join(' | ')) + }) + + it('never prefunds more than the commit inputs hold', async () => { + // The input barely covers the commit's own outputs and fee. The build must + // still produce a signable pair rather than fail with INSUFFICIENT_FUNDS. + const encoder = envelopeEncoder({ size: 500000, fees: 0 }) + const result = await buildEnvelope(encoder, { commitFee: UNDER_TARGET_COMMIT_FEE, inputValue: 53000, confirmations: 0 }) + const commitTx = unsignedTx(result.psbt) + const outputs = commitTx.outs.reduce((s, o) => s + o.value, 0) + assert.ok(outputs + UNDER_TARGET_COMMIT_FEE <= 53000, + `commit pays out ${outputs} + ${UNDER_TARGET_COMMIT_FEE} fee against a ${53000} input`) + assert.strictEqual(result.envelope.commitValue, commitTx.outs[result.envelope.commitVout].value) + }) +}) + +// The P2SH lane splits the two phases across two calls, so the commit funds the +// package rate into its first leg and the reveal has to KEEP that money as fee +// instead of sweeping it back to the caller. +describe('P2SH two-phase package prefund @regression @tier1', () => { + const DOGE = 'dogecoin-regtest' + const DOGE_REGTEST_NET = require('../../src/CryptoNetworks').getBitcoinJsNetwork(DOGE) + const RATE_KB = 1000000 // 1000 koinu/byte, the venue rate + const TARGET_PER_BYTE = RATE_KB / 1000 + const pubkeyBuf = Buffer.from('0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798', 'hex') + const CALLER = bitcoin.payments.p2pkh({ pubkey: pubkeyBuf, network: DOGE_REGTEST_NET }).address + // Four inputs this size: selection takes two, which is enough to leave a + // change output at or above the 0.01 DOGE relay floor. That matters - a change + // that folds into the fee instead of being emitted quietly lifts the commit + // back over the target rate and the probe stops probing anything. + const FUNDING_INPUT = 2000000 + const PAYLOAD = 'x'.repeat(400) + // The dust floor, and far under the target rate for a two-input funding tx: + // the congestion-clamped commit the ledger entry describes. + const UNDER_TARGET_COMMIT_FEE = 100000 + + function prevTxHex () { + const tx = new bitcoin.Transaction() + tx.addInput(Buffer.alloc(32, 0x11), 0) + for (let i = 0; i < 4; i++) { + tx.addOutput(bitcoin.payments.p2pkh({ pubkey: pubkeyBuf, network: bitcoin.networks.regtest }).output, FUNDING_INPUT) + } + return tx.toHex() + } + + function makeEncoder (commitPackage) { + const encoder = new XChainEncoder(DOGE, '127.0.0.1', '8333', 'rpc', 'rpc', '', '') + encoder.connector = { + getFeePerKilobyte: async () => 0.01, + getTransactionHex: async () => prevTxHex() + } + if (commitPackage !== undefined) { + encoder.connector.getUnconfirmedAncestorPackage = async () => + (typeof commitPackage === 'function' ? commitPackage() : commitPackage) + } + return encoder + } + + function legacyUtxos () { + const p2pkh = bitcoin.payments.p2pkh({ pubkey: pubkeyBuf, network: bitcoin.networks.regtest }) + return [0, 1, 2, 3].map(vout => ({ + txid: TXID_PARENT_A, vout, value: FUNDING_INPUT, confirmations: 6, scriptPubKey: p2pkh.output.toString('hex') + })) + } + + async function buildFunding (encoder, commitFee) { + encoder.clearReservations() + const res = await encoder.createTransaction( + legacyUtxos(), CALLER, null, PAYLOAD, null, commitFee, false, 'P2SH', CALLER, + null, null, null, true, RATE_KB) + return res.psbt.__CACHE.__TX + } + + async function buildReveal (encoder, fundingTx) { + const res = await encoder.createTransaction( + [], CALLER, null, PAYLOAD, null, null, false, 'P2SH', CALLER, + fundingTx.getId(), fundingTx.toHex(), null, true, RATE_KB) + return res.psbt.__CACHE.__TX + } + + const legTotal = (tx) => tx.outs + .filter(o => o.script.toString('hex').startsWith('a914')) + .reduce((s, o) => s + o.value, 0) + + it('funds the package rate into the leg when the commit pays under target', async () => { + const atTarget = await buildFunding(makeEncoder(), null) + const underTarget = await buildFunding(makeEncoder(), UNDER_TARGET_COMMIT_FEE) + assert.ok(legTotal(underTarget) > legTotal(atTarget), + `an under-target commit must over-fund its leg (${legTotal(underTarget)} vs ${legTotal(atTarget)})`) + }) + + it('the reveal keeps the package money as fee instead of sweeping it back', async () => { + const funding = await buildFunding(makeEncoder(), UNDER_TARGET_COMMIT_FEE) + const commitSize = funding.virtualSize() + // What the commit ACTUALLY pays: a sub-floor change would have folded into + // the fee and quietly lifted it back over target. + const commitFee = funding.ins.length * FUNDING_INPUT - funding.outs.reduce((s, o) => s + o.value, 0) + assert.ok(commitFee / commitSize < TARGET_PER_BYTE, + `this probe is only meaningful while the commit is under target (${commitFee / commitSize})`) + + // The node reports the commit exactly as it was broadcast: its real size and + // the under-target fee the caller chose. + const packaged = await buildReveal( + makeEncoder({ size: commitSize, fees: commitFee / SATOSHI_UNIT }), funding) + // Control: the same reveal built against a node that cannot price the commit. + const unpackaged = await buildReveal(makeEncoder(), funding) + + const legs = legTotal(funding) + const packagedFee = legs - packaged.outs.reduce((s, o) => s + o.value, 0) + const unpackagedFee = legs - unpackaged.outs.reduce((s, o) => s + o.value, 0) + assert.ok(packagedFee > unpackagedFee, + `the reveal must keep the package money (${packagedFee} vs ${unpackagedFee})`) + + // The reveal's own priced size, recovered from the control: at exactly + // 1000 koinu/byte an unpackaged fee IS the size in bytes. + const revealSize = unpackagedFee / TARGET_PER_BYTE + const packageRate = (commitFee + packagedFee) / (commitSize + revealSize) + assert.ok(packageRate >= TARGET_PER_BYTE, + `commit+reveal package pays ${packageRate} koinu/byte, under the ${TARGET_PER_BYTE} target`) + + // And it is still not a full burn: the SDK refuses to sign an outputless reveal. + const swept = packaged.outs.reduce((s, o) => s + o.value, 0) + assert.ok(swept > 0, 'the reveal must still leave the caller an output') + }) + + it('a confirmed commit asks the reveal for nothing', async () => { + const funding = await buildFunding(makeEncoder(), UNDER_TARGET_COMMIT_FEE) + // An empty package is what the connector reports once the commit confirms; a + // confirmed parent is nobody's ancestor for fee purposes any more. + const confirmed = await buildReveal(makeEncoder({ size: 0, fees: 0 }), funding) + const unpackaged = await buildReveal(makeEncoder(), funding) + assert.strictEqual(confirmed.toHex(), unpackaged.toHex(), + 'a confirmed commit must leave the reveal byte-identical') + }) + + it('degrades to the per-transaction fee when the commit lookup throws', async () => { + const funding = await buildFunding(makeEncoder(), UNDER_TARGET_COMMIT_FEE) + const thrown = await buildReveal(makeEncoder(() => { throw new Error('node RPC exploded') }), funding) + const unpackaged = await buildReveal(makeEncoder(), funding) + assert.strictEqual(thrown.toHex(), unpackaged.toHex()) + }) +})