diff --git a/.env.example b/.env.example index 7385068d..4762140a 100644 --- a/.env.example +++ b/.env.example @@ -36,8 +36,8 @@ DECODER_API_PORT=3002 # Optional: API rate limit in requests per minute per IP (default 100). # DECODER_RATE_LIMIT_RPM=100 -# Optional: enable AuxPoW (merged-mining) block-header parsing, required for -# Dogecoin-family chains. Set to 1/true to enable. +# Legacy, ignored: AuxPoW (merged-mining) block-header parsing is selected from +# NETWORK (on for dogecoin-*, off otherwise). Kept for backward compatibility. # AUX_POW= # Optional: coin node RPC timeout in ms (default 30000). diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 6241305c..061bdbb5 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,5 +1,9 @@ +## Base branch + +- [ ] This pull request targets `develop`. `master` only takes release merges, and a pull request into it from a fork cannot be merged. + ## What this changes diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01b2ecab..32fc4ec0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ on: # so the merge commit a release tag points at has its own green run. branches: [master, develop] pull_request: - branches: [master] + branches: [master, develop] jobs: ci: diff --git a/CHANGELOG.md b/CHANGELOG.md index f83cd92c..c9af3e64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.24.0] - 2026-10-17 + +### Added +- Added a ready endpoint and reported readiness on the live endpoint. +- Added the dispenser parent-floor activation that anchors dispenser grace to the parent protocol time. + +### Fixed +- Refused a block whose transactions do not fill it or match its merkle root, and parsed LTC MWEB-form transactions. +- Refused unsafe MWEB CompactSize values and trailing transaction bytes with a typed parse error. +- Settled a lower node tip with two agreeing readings before rolling back, and bounded the current block MTP history. + +### Security +- Bumped ip-address, axios and proxy-addr past their advisories. + ## [0.23.0] - 2026-10-11 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3a415bc8..15935408 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -110,7 +110,7 @@ this service's entire input surface is attacker-controlled. Match the existing log style: a concise subject line, then a short body explaining what changed and why. -- Branch off `master` and keep history linear (rebase, don't merge). +- Fork the repository, branch off `develop`, and open your pull request against `develop`. Keep history linear (rebase, don't merge). `master` is what production runs and only ever receives release merges from `develop`; a pull request into `master` from a fork or any other branch fails the `base-branch guard` check and cannot be merged. - One logical change per commit; don't batch unrelated work. - **No `Co-Authored-By` trailers.** This is a project policy. - **Never `--no-verify`.** If a hook fails, fix the cause; don't bypass it. diff --git a/bin/pins/identity.json b/bin/pins/identity.json index 6308fb20..0a32bcc4 100644 --- a/bin/pins/identity.json +++ b/bin/pins/identity.json @@ -2,8 +2,8 @@ "repo": "xchain-decoder", "what": "sha256 of every file this repo vendors from a canonical it does not own", "coins": { - "src/coins/BTC.js": "182baf5173e3846b94a5edc1ae15913a54fb266addfc45a3cf9a013bd9ec16c6", - "src/coins/DOGE.js": "4659b45fb7a5c959ce2777ab9d5e7c193f3515f03cdd8cd23bb6b9c4d012fc11", + "src/coins/BTC.js": "6868afc2d18911ba112234c980c094fda71cfc3e744a00f493def781c7610f8d", + "src/coins/DOGE.js": "29ac02299b0c779f4e3eda579eeaa9f673c454b73cfe9978a35f03f0e9aed33c", "src/coins/LTC.js": "6155195bfa5fdfeab77dec3802ce67b5cc47a77695e81a4a5545d980e4c421dd", "src/coins/consensus_pin.js": "342bf7af224a19ea0a98ff9cfb2c9a70b3fd5be75de7ecef2db05936f82baa63", "src/coins/index.js": "08b678d934fa647d3c7d14dc5ccdaeb86722ee32a3662de69e4a31cd7b09234c" diff --git a/bin/pins/suite-title-renames.json b/bin/pins/suite-title-renames.json index 3f5e7484..a7d9d99a 100644 --- a/bin/pins/suite-title-renames.json +++ b/bin/pins/suite-title-renames.json @@ -5,6 +5,9 @@ "test/unit/db_queries.test/04_delete_block_by_index.test.js": { "Database#deleteBlockByIndex() executes 4 DELETE queries and returns true on success": "Database#deleteBlockByIndex() executes 5 DELETE queries and returns true on success" }, + "test/unit/db_queries.test/20_error_path_transaction_connection_branches.test.js": { + "Database error-path transactionConnection branches createAddress: swallows INSERT error even with active transactionConnection": "Database error-path transactionConnection branches createAddress: rethrows an INSERT error inside an active transaction so the block rolls back" + }, "test/unit/fee_destination.test.js": { "resolveFeeDestination @unit honors an env override on regtest ONLY": "resolveFeeDestination @unit ignores a bare env override on regtest too, so capture and indexer validation resolve one address" }, diff --git a/package-lock.json b/package-lock.json index b20b5e06..0b9878f5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "xchain-decoder", - "version": "0.23.0", + "version": "0.24.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "xchain-decoder", - "version": "0.23.0", + "version": "0.24.0", "license": "AGPL-3.0-or-later", "dependencies": { "axios": "^1.18.1", @@ -21,7 +21,7 @@ "express-json-rpc-router": "^1.4.0", "express-rate-limit": "^8.5.2", "helmet": "^8.2.0", - "mariadb": "3.5.3", + "mariadb": "3.5.4", "tiny-secp256k1": "2.2.4" }, "devDependencies": { @@ -1737,13 +1737,13 @@ "license": "MIT" }, "node_modules/axios": { - "version": "1.18.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.18.1.tgz", - "integrity": "sha512-3nTvFlvpn9Zu/RkHUqtc7/+al4UpRW5az71ap5zccp6e8RAYEzhMTecX8Dz1wWDYrPpUoB1HAQEGEAEvUr7S9g==", + "version": "1.20.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.20.0.tgz", + "integrity": "sha512-r8aOh8j9cGKpgQAqpzrUHnSIc6a59Y3Xf/cv8sy1DrHCkZHzQGEuoq1tARk6qSyDdtQGSDgpb9kFlruzPvrgwg==", "license": "MIT", "dependencies": { "follow-redirects": "^1.16.0", - "form-data": "^4.0.5", + "form-data": "^4.0.6", "https-proxy-agent": "^5.0.1", "proxy-from-env": "^2.1.0" } @@ -3825,9 +3825,9 @@ "license": "ISC" }, "node_modules/ip-address": { - "version": "10.5.0", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.5.0.tgz", - "integrity": "sha512-R5SnVLJmgYYvf2F2ZgwSBnelz5G4q5AxIC277GDfUaNbrZKNANcBC7RHqYYePlszf4kBolVkJauG0ZjHHFh55g==", + "version": "10.7.4", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.4.tgz", + "integrity": "sha512-DWBMKyeM25D70nrIiTkgQth59/G+naKOTep6huF9j+GmeK3DMD60Zvxg5d5qQnz+38AHgou6ean2LECX+lrKDg==", "license": "MIT", "engines": { "node": ">= 12" @@ -4333,9 +4333,9 @@ } }, "node_modules/mariadb": { - "version": "3.5.3", - "resolved": "https://registry.npmjs.org/mariadb/-/mariadb-3.5.3.tgz", - "integrity": "sha512-i053Kc0MgdUv/hu9mCyq67TYfPXFj3/MV8I7ZW5wvJNixIyXC0VztMPUjIVj/449nQo+BsxFD4Fdk/sA/uqKPQ==", + "version": "3.5.4", + "resolved": "https://registry.npmjs.org/mariadb/-/mariadb-3.5.4.tgz", + "integrity": "sha512-3m0vdfgRkjle5/9hHl4R7rlVvnDA7QfuFSNkdIAQNROWKOS0J3b9IQK9HaZSCGEVKiDUbFXFiINfC+cmR5P/aA==", "license": "LGPL-2.1-or-later", "dependencies": { "@types/geojson": "^7946.0.16", @@ -4957,9 +4957,9 @@ } }, "node_modules/proxy-addr": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", - "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.8.tgz", + "integrity": "sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ==", "license": "MIT", "dependencies": { "forwarded": "0.2.0", @@ -4967,6 +4967,10 @@ }, "engines": { "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/proxy-from-env": { diff --git a/package.json b/package.json index 7a68c9ae..3a634a05 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "xchain-decoder", "description": "xchain-decoder decodes XChain platform transactions from a given blockchain and populates a database with the decoded data.", - "version": "0.23.0", + "version": "0.24.0", "license": "AGPL-3.0-or-later", "repository": { "type": "git", @@ -20,7 +20,7 @@ "express-json-rpc-router": "^1.4.0", "express-rate-limit": "^8.5.2", "helmet": "^8.2.0", - "mariadb": "3.5.3", + "mariadb": "3.5.4", "tiny-secp256k1": "2.2.4" }, "scripts": { @@ -93,7 +93,7 @@ "diff": "^8.0.3", "tmp": "^0.2.7", "fast-uri": "^3.1.5", - "ip-address": "^10.3.1", + "ip-address": "^10.5.1", "tough-cookie": "^6.0.2", "uuid": "^11.1.1", "request": { diff --git a/src/XChainDecoder.js b/src/XChainDecoder.js index bbfd7761..cc01a47a 100644 --- a/src/XChainDecoder.js +++ b/src/XChainDecoder.js @@ -210,6 +210,10 @@ function initializeDecoderMode(decoder) { decoder.rpcErrors = 0 decoder.parseErrors = 0 + decoder.quarantinedTxs = 0 + decoder.lastQuarantinedBlock = null + decoder.lastQuarantinedTxHash = null + decoder.lastQuarantinedAt = null decoder.dispenserPurgeFailures = 0 } diff --git a/src/XChainDecoder/block_store.js b/src/XChainDecoder/block_store.js index 43212adf..b12c5542 100644 --- a/src/XChainDecoder/block_store.js +++ b/src/XChainDecoder/block_store.js @@ -14,17 +14,21 @@ const { format: formatLogLine } = require('node:util') const { isDispenserExpiryRealignActive } = require('../protocol/dispenser_expiry_realign') -const { cancelGraceFloor, purgeGraceFloor } = require('../protocol/dispenser_cancel_grace') +const { + cancelGraceFloor, + isDispenserCancelGraceParentFloorActive, + purgeGraceFloor, +} = require('../protocol/dispenser_cancel_grace') const protocolTime = require('../protocol/protocol_time') const { logger, SYNCED_THRESHOLD, DB_TRANSACTION_BLOCKS_QUANTITY, LOG_BLOCK_INTERVAL, DISPENSER_EXPIRE_SAFE_DEPTH } = require('./constants.js') const { ingestTransaction, clearQuarantineState } = require('./transaction_ingest.js') -async function loadOpenDispenserAddresses(block, nextBlockHeight){ +async function loadOpenDispenserAddresses(block, nextBlockHeight, parentProtocolTime){ // Load the set once per block after any legacy expiry and before the // realigned end-of-block expiry. The grace floor preserves dispensers // that the indexer still considers fillable during cancellation grace. let openDispenserAddresses = await this.db.getAllOpenDispenserAddresses( - cancelGraceFloor(this.consensusNetwork, block.timestamp)) + cancelGraceFloor(this.consensusNetwork, block.timestamp, parentProtocolTime)) if (openDispenserAddresses == null){ logger.error(`Could not load open dispenser addresses for block ${nextBlockHeight}; retrying block`) await this.db.endTransaction() @@ -129,13 +133,11 @@ async function finishBlock(loop, block, nextBlockHeight, nextBlockHash, openDisp this.lastAdvanceAt = Date.now() } -async function fetchPreviousBlockTimes(nextBlockHeight, span){ - // Walk strictly backward so the result is newest first by block height and - // never wider than the median span, even when asked for more. A failed +async function fetchBlockTimes(nextBlockHeight, span){ + // Walk strictly backward so the result is newest first by block height. A failed // lookup returns no partial window, avoiding mixed reorg states. - const boundedSpan = Math.min(span, protocolTime.MEDIAN_TIME_SPAN) let previousBlockTimes = [] - for (let height = nextBlockHeight - 1; height >= 0 && previousBlockTimes.length < boundedSpan; height--){ + for (let height = nextBlockHeight - 1; height >= 0 && previousBlockTimes.length < span; height--){ let previousBlock try { previousBlock = await this.db.getBlockByIndex(height) @@ -149,18 +151,34 @@ async function fetchPreviousBlockTimes(nextBlockHeight, span){ return previousBlockTimes } +async function fetchPreviousBlockTimes(nextBlockHeight, span){ + return fetchBlockTimes.call( + this, nextBlockHeight, Math.min(span, protocolTime.MEDIAN_TIME_SPAN)) +} + async function resolveBlockTimeContext(block, nextBlockHeight){ - // MTP networks resolve against the previous MEDIAN_TIME_SPAN blocks. + const mtpActive = protocolTime.isProtocolTimeMtpActive(this.consensusNetwork) + const parentFloorActive = isDispenserCancelGraceParentFloorActive( + this.consensusNetwork, block.timestamp) + const historySpan = mtpActive + ? protocolTime.MEDIAN_TIME_SPAN + (parentFloorActive ? 1 : 0) + : (parentFloorActive ? 1 : 0) let previousBlockTimes = [] - if (protocolTime.isProtocolTimeMtpActive(this.consensusNetwork)){ - previousBlockTimes = await fetchPreviousBlockTimes.call(this, nextBlockHeight, protocolTime.MEDIAN_TIME_SPAN) + if (historySpan > 0){ + previousBlockTimes = await fetchBlockTimes.call(this, nextBlockHeight, historySpan) if (previousBlockTimes === null){ await this.db.endTransaction() return null } } - return protocolTime.createBlockTimeContext( - this.consensusNetwork, block.timestamp, previousBlockTimes) + const current = protocolTime.createBlockTimeContext( + this.consensusNetwork, block.timestamp, + previousBlockTimes.slice(0, protocolTime.MEDIAN_TIME_SPAN)) + const parentProtocolTime = previousBlockTimes.length === 0 + ? null + : protocolTime.protocolTime( + this.consensusNetwork, previousBlockTimes[0], previousBlockTimes.slice(1)) + return Object.freeze({ ...current, parentProtocolTime }) } async function insertBlockRecord(blockTimeContext, nextBlockHeight, nextBlockHash, previousBlockHash){ @@ -203,7 +221,8 @@ async function storeBlock(loop, block, nextBlockHeight, nextBlockHash, previousB if ((await expireDispensersAtBlockStart.call( this, block, nextBlockHeight, expireAtEnd)) === 'rollback') return 'rollback' - const openAddresses = await loadOpenDispenserAddresses.call(this, block, nextBlockHeight) + const openAddresses = await loadOpenDispenserAddresses.call( + this, block, nextBlockHeight, blockTimeContext.parentProtocolTime) if (openAddresses === 'rollback') return 'rollback' return await finishBlock.call( this, loop, block, nextBlockHeight, nextBlockHash, openAddresses, expireAtEnd) diff --git a/src/XChainDecoder/chain_integrity.js b/src/XChainDecoder/chain_integrity.js index 6cc0aaa4..6436104b 100644 --- a/src/XChainDecoder/chain_integrity.js +++ b/src/XChainDecoder/chain_integrity.js @@ -19,19 +19,24 @@ ********************************************************************/ const { chainGenesisMismatch, chainGenesisUnpinned } = require('../protocol/chain_identity') -const { logger, REORG_HALT_PROBE_INTERVAL_MS, AUXPOW_REASSEMBLE_AFTER } = require('./constants.js') +const { logger, REORG_HALT_PROBE_INTERVAL_MS, AUXPOW_REASSEMBLE_AFTER, TIP_SETTLE_DELAY_MS, TIP_SETTLE_MAX_READS } = require('./constants.js') // Read the durable REORG_HALT marker through whichever probe the db exposes, or // null when it exposes none (or there is no db) and the cached state must stand. +// A probe that exists but answers with nothing throws instead, so checkReorgHalt +// logs the fault and keeps the cached state: an empty answer is never a clear. async function readReorgHaltMarker(db){ if (!db) return null if (typeof db.getReorgHaltMarker === 'function'){ - return (await db.getReorgHaltMarker()) || { halted: false, at: null, reason: null } + const marker = await db.getReorgHaltMarker() + if (!marker || typeof marker !== 'object') throw new Error('getReorgHaltMarker returned no marker') + return marker } if (typeof db.isReorgHalted === 'function'){ // Older/minimal db shapes (and the mocks in the verifyReorg suites) // expose only the boolean probe. const halted = await db.isReorgHalted() + if (halted == null) throw new Error('isReorgHalted returned no answer') return { halted: !!(halted && halted.halted !== undefined ? halted.halted : halted), at: null, reason: null } } return null @@ -236,5 +241,24 @@ module.exports = { // refresh the timestamp, so the refusal is re-proved on every retry. if (!mismatch) this.chainGenesisCheckedAt = Date.now() return mismatch + }, + + // Re-read with readOnce() until two readings TIP_SETTLE_DELAY_MS apart agree, starting + // from `first`, the reading that raised the question. readOnce() returns null when the + // node cannot be read or a gate refuses the reply. Returns the settled reading, or null + // when a re-read fails or the reading is still moving after TIP_SETTLE_MAX_READS, so + // the caller ends its pass with nothing deleted and the next pass reads afresh. + async settleNodeReading(first, readOnce, what){ + let previous = first + for (let read = 0; read < TIP_SETTLE_MAX_READS; read++){ + await this.sleep(TIP_SETTLE_DELAY_MS) + const next = await readOnce() + if (next === null) return null + if (next === previous) return next + previous = next + } + logger.warn('The ' + what + ' is still moving after ' + TIP_SETTLE_MAX_READS + ' readings (last ' + previous + + '); not rolling back on a moving reading, re-checking next pass.') + return null } } diff --git a/src/XChainDecoder/constants.js b/src/XChainDecoder/constants.js index 17f3a0a8..ab7fcfa7 100644 --- a/src/XChainDecoder/constants.js +++ b/src/XChainDecoder/constants.js @@ -28,6 +28,14 @@ const lenientTextDecoder = new TextDecoder('utf-8') const CHECK_BLOCK_DELAY_MS = 1000 //1 second to continously ask for new block when all has been parsed const BLOCKCHAIN_INFO_REFRESH_MS = 30000 //Re-poll the node tip at least this often during catch-up so reported lag stays accurate const MEMPOOL_INTERVAL = 60000 //60 seconds between mempool checks +// A node tip below the stored tip, or a different block at the stored height, is acted +// on only once two readings TIP_SETTLE_DELAY_MS apart agree; a reading that keeps +// changing for TIP_SETTLE_MAX_READS re-reads ends the pass with nothing deleted. One +// stale reply (an RPC failover onto a lagging endpoint) otherwise rolls back valid +// blocks and writes REORG markers for a reorg that never happened. Same values as +// xchain-utxo-tracker, which settles its tip the same way before a rollback. +const TIP_SETTLE_DELAY_MS = 2000 +const TIP_SETTLE_MAX_READS = 5 // How often a health surface may re-probe the durable REORG_HALT marker. The marker // changes at most once in a decoder's life, so a slow TTL is ample; the point of the // cache is that an unauthenticated health endpoint must not turn into one DB query @@ -164,6 +172,8 @@ module.exports = { CHECK_BLOCK_DELAY_MS, BLOCKCHAIN_INFO_REFRESH_MS, MEMPOOL_INTERVAL, + TIP_SETTLE_DELAY_MS, + TIP_SETTLE_MAX_READS, REORG_HALT_PROBE_INTERVAL_MS, REORG_HALT_PARK_TICK_MS, STALL_ALERT_MS, diff --git a/src/XChainDecoder/mempool_refresh.js b/src/XChainDecoder/mempool_refresh.js index 1e7c196a..020e0ed5 100644 --- a/src/XChainDecoder/mempool_refresh.js +++ b/src/XChainDecoder/mempool_refresh.js @@ -19,7 +19,11 @@ ********************************************************************/ const { format: formatLogLine } = require('node:util') -const { cancelGraceFloor } = require('../protocol/dispenser_cancel_grace') +const { + cancelGraceFloor, + isDispenserCancelGraceParentFloorActive, +} = require('../protocol/dispenser_cancel_grace') +const protocolTime = require('../protocol/protocol_time') const { logger, MEMPOOL_BATCH_SIZE } = require('./constants.js') function dedupNodeMempool(rawMempoolUnordered){ @@ -73,8 +77,36 @@ function decodeMempoolTransaction(nextTxHex, nextTxHexIndex){ // an empty set: the mempool view is non-consensus, so a pending row that blanks an // unattributed ACTION for one poll is corrected by the block path on confirm, and the // rest of the cycle (pruning, new arrivals) still runs. +function* loadMempoolParentProtocolTime(blockTime){ + if (!isDispenserCancelGraceParentFloorActive(this.consensusNetwork, blockTime)) return null + if (this.lastProcessedBlockIndex < 0 || + typeof this.mempoolDb.getBlockByIndex !== 'function') return null + + const span = protocolTime.isProtocolTimeMtpActive(this.consensusNetwork) + ? protocolTime.MEDIAN_TIME_SPAN + 1 + : 1 + const previousBlockTimes = [] + for (let height = this.lastProcessedBlockIndex; + height >= 0 && previousBlockTimes.length < span; height--){ + let block + try { + block = yield this.mempoolDb.getBlockByIndex(height) + } catch (err) { + logger.error(formatLogLine('Mempool: failed to resolve parent protocol time:', err)) + return null + } + if (!block) break + previousBlockTimes.push(block.block_time) + } + if (previousBlockTimes.length === 0) return null + return protocolTime.protocolTime( + this.consensusNetwork, previousBlockTimes[0], previousBlockTimes.slice(1)) +} + function* loadMempoolOpenDispenserAddresses(){ - const floor = cancelGraceFloor(this.consensusNetwork, Math.floor(Date.now() / 1000)) + const blockTime = Math.floor(Date.now() / 1000) + const parentProtocolTime = yield* loadMempoolParentProtocolTime.call(this, blockTime) + const floor = cancelGraceFloor(this.consensusNetwork, blockTime, parentProtocolTime) let openDispenserAddresses = null try { openDispenserAddresses = yield this.mempoolDb.getAllOpenDispenserAddresses(floor) @@ -130,13 +162,23 @@ function admitMempoolAction(parseResult, nextTransactionHash){ return this.buildStoredActionRecord(parseResult, nextTransactionHash, true) } -function* storeMempoolTransaction(nextTxHex, nextTxHexIndex, openDispenserAddresses){ +// The row is keyed on the txid the node listed, and the parsed tx must hash to it. +// A mismatch means this parse read different bytes than the node's (an unhandled +// serialization), so its source, outputs and payload cannot be trusted: the tx is +// skipped and counted in parseErrors, and the next cycle tries it again. +function* storeMempoolTransaction(nodeTxid, nextTxHex, nextTxHexIndex, openDispenserAddresses){ let nextTx = decodeMempoolTransaction.call(this, nextTxHex, nextTxHexIndex) if (nextTx == null) { return false } - let nextTransactionHash = nextTx.getId() + const computedTxid = nextTx.getId() + if (computedTxid !== nodeTxid) { + this.parseErrors++ + logger.error(`Mempool: tx ${nodeTxid} parses to txid ${computedTxid}, skipping`) + return false + } + let nextTransactionHash = nodeTxid let parseResult = yield* parseMempoolTransaction.call(this, nextTx, nextTransactionHash, openDispenserAddresses) if (parseResult == null) { @@ -198,7 +240,9 @@ function* storeMempoolBatches(rawMempool, openDispenserAddresses){ continue } - if (yield* storeMempoolTransaction.call(this, nextTxHex, nextTxHexIndex, openDispenserAddresses)) { + // getRawTransactions answers in request order, so the index names the node txid. + const nodeTxid = nextRawMempoolChunk[nextTxHexIndex] + if (yield* storeMempoolTransaction.call(this, nodeTxid, nextTxHex, nextTxHexIndex, openDispenserAddresses)) { validTransactionsCount = validTransactionsCount + 1 } } diff --git a/src/XChainDecoder/reorg_verification.js b/src/XChainDecoder/reorg_verification.js index ad83f63a..42557df9 100644 --- a/src/XChainDecoder/reorg_verification.js +++ b/src/XChainDecoder/reorg_verification.js @@ -174,7 +174,10 @@ function nodeTipStillCatchingUp(info){ return nodeStillCatchingUp(info) || info["verificationprogress"] < MIN_VERIFICATION_PROGRESS_TO_PARSE } -async function refreshReorgTip(nodeTip){ +// Read the node tip through the block loop's gates. Answers nodeTip unchanged when the +// node cannot be read or a gate refuses the reply (foreign tier, foreign genesis, a node +// still catching up), so a refused reading is never below the tip the walk already has. +async function readReorgTip(nodeTip){ try { const info = await this.connector.getBlockchainInfo() // Apply the block loop's chain-identity gate here too. This is @@ -208,6 +211,18 @@ async function refreshReorgTip(nodeTip){ return nodeTip } +// Re-read the node tip after a failed hash read mid-walk. A higher or equal tip deletes +// nothing by height and is taken at once; a lower one becomes the height every stored +// block above it is deleted against, so it is taken only once two readings agree. A +// reading that does not settle keeps the current tip and the walk retries. A refused +// re-read answers the current tip, so only two real readings can settle a lower one. +async function refreshReorgTip(nodeTip){ + const tip = await readReorgTip.call(this, nodeTip) + if (tip === nodeTip || (nodeTip != null && tip >= nodeTip)) return tip + const settled = await this.settleNodeReading(tip, () => readReorgTip.call(this, nodeTip), 'mid-walk node tip') + return settled === null ? nodeTip : settled +} + async function deleteForkedBlock(lastBlockIndex, lastBlock, priorDepth, blocksDeleted, retryCount){ await assertWithinSafeDepth.call(this, lastBlockIndex, priorDepth, blocksDeleted) try { diff --git a/src/XChainDecoder/source_resolution.js b/src/XChainDecoder/source_resolution.js index 4a83c78a..5d2f97db 100644 --- a/src/XChainDecoder/source_resolution.js +++ b/src/XChainDecoder/source_resolution.js @@ -53,7 +53,7 @@ function* fetchPrevoutTransaction(txId, outputIndex){ // deterministic. start() refuses to run a Dogecoin decoder whose BigInt-safe // bufferutils reader is inactive for the same reason: that is the one decode fault // that would differ between instances. - // MUST parse through transactionFromHex (strips the LTC MWEB marker+flag), not + // MUST parse through transactionFromHex (reads the LTC MWEB serialization), not // bitcoin.Transaction.fromHex: a Litecoin funding/prevout tx can carry the MWEB // flag (0x08/0x09) and vanilla strict parsing throws a UInt64 range error on it. // transactionFromHex is the same parser the block path uses; for BTC/DOGE and @@ -179,7 +179,7 @@ module.exports = { async parseRawTransaction(rawTransaction){ // Parse via xchainBlockDecoder.transactionFromHex, not bitcoin.Transaction.fromHex: - // the former strips the LTC MWEB marker+flag (0x08/0x09) that makes vanilla strict + // the former reads the LTC MWEB marker+flag (0x08/0x09) form that makes vanilla strict // parsing throw a deterministic UInt64 range error. See getSourceFromOutput. return await this.parseTransaction(this.xchainBlockDecoder.transactionFromHex(rawTransaction)) }, diff --git a/src/XChainDecoder/sync_loop.js b/src/XChainDecoder/sync_loop.js index 64336f22..ea8b24b5 100644 --- a/src/XChainDecoder/sync_loop.js +++ b/src/XChainDecoder/sync_loop.js @@ -57,18 +57,39 @@ async function leaveReorgHaltPark(loop){ loop.lastBlockchainInfo = null } +// A different hash at the stored height is a reorg only once two readings of it agree: +// one stray answer (a failover onto an endpoint on another branch) must not roll back +// the stored tip. An unreadable re-read (the node's tip dropped below this height) ends +// the check with nothing deleted; the tip-regression branch takes that case next pass. +async function equalHeightSwapHolds(height, nodeHash, storedHash){ + const settled = await this.settleNodeReading(nodeHash, async () => { + try { + return (await this.connector.getBlockHash(height)) || null + } catch (e){ + logger.error(formatLogLine('Error re-reading the node block hash at height ' + height + ' before a reconcile:', e)) + return null + } + }, 'node block hash at height ' + height) + return settled !== null && settled !== storedHash +} + async function reconcileEqualHeightTip(loop){ loop.tipHashCheckedAt = loop.lastBlockchainInfoRefreshAt // Guard ONLY the detection reads: an RPC/DB blip there is transient and // should log-and-skip until the next refresh, as before. let needsReconcile = false + let nodeHash = null + let storedBlock = null try { - const nodeHash = await this.connector.getBlockHash(loop.lastProcessedBlockIndex) - const storedBlock = await this.db.getBlockByIndex(loop.lastProcessedBlockIndex) + nodeHash = await this.connector.getBlockHash(loop.lastProcessedBlockIndex) + storedBlock = await this.db.getBlockByIndex(loop.lastProcessedBlockIndex) needsReconcile = !!(storedBlock && nodeHash && storedBlock.block_hash !== nodeHash) } catch (e){ logger.error(formatLogLine('Error during equal-height tip-hash detection reads, skipping:', e)) } + if (needsReconcile){ + needsReconcile = await equalHeightSwapHolds.call(this, loop.lastProcessedBlockIndex, nodeHash, storedBlock.block_hash) + } if (needsReconcile){ // Run the reconcile OUTSIDE the detection try so a fail-closed verifyReorg // abort is never swallowed as a transient blip, which left a partially diff --git a/src/XChainDecoder/tip_refresh.js b/src/XChainDecoder/tip_refresh.js index eb8bb0b4..6bce8f3e 100644 --- a/src/XChainDecoder/tip_refresh.js +++ b/src/XChainDecoder/tip_refresh.js @@ -207,7 +207,27 @@ async function reconcileTipRegression(loop, lastProcessedBlockIndex, lastBlockch return await reconcileOrphanBlocks.call(this, loop, lastProcessedBlockIndex) } +// Settle a node tip below ours before anything is rolled back against it. One stale +// getblockchaininfo reply (a failover onto a lagging endpoint) otherwise deletes every +// valid block above it and writes a REORG marker for each. Runs before endTransaction, +// so a reading that does not hold leaves the open batch and the cursor as they were. +// Returns true only when two readings agree on a tip still below lastProcessedBlockIndex. +async function settleTipBelow(loop, lastProcessedBlockIndex){ + const settled = await this.settleNodeReading(this.blockchainInfoLastBlock, async () => { + if ((await fetchChainTip.call(this, loop)) === 'continue') return null + // A node that went back into initial block download is the wait branch's case. + if (nodeStillCatchingUp(loop.lastBlockchainInfo)) return null + return this.blockchainInfoLastBlock + }, 'node tip') + if (settled === null) return false + if (settled < lastProcessedBlockIndex) return true + this.log("A node tip below the last processed block (" + lastProcessedBlockIndex + ") did not hold: the node settled at " + + settled + ", so nothing is rolled back.") + return false +} + async function reconcileOrphanBlocks(loop, lastProcessedBlockIndex){ + if (!(await settleTipBelow.call(this, loop, lastProcessedBlockIndex))) return 'continue' // The node's tip has dropped BELOW our last-processed height (deep // reorg, node rollback, or restart onto a shorter/different chain). // The forward hash-compare reorg path (below) is unreachable in this @@ -217,8 +237,8 @@ async function reconcileOrphanBlocks(loop, lastProcessedBlockIndex){ // inherits as permanently divergent history. Reconcile now: // verifyReorg(tip) deletes every stored block above the tip via a // deterministic height compare, then walks the hash-compare back to - // the fork point. blockchainInfoLastBlock was just refreshed above, so - // the tip is current. + // the fork point. blockchainInfoLastBlock holds the settled reading + // from settleTipBelow, so the tip is current and not a one-off reply. if (!loop.tipBelowStoredTipRefused){ this.logWarn("The last processed block height ("+lastProcessedBlockIndex+") is greater than the last block from the node ("+this.blockchainInfoLastBlock+"). Reconciling orphan blocks...") } diff --git a/src/XChainDecoder/transaction_ingest.js b/src/XChainDecoder/transaction_ingest.js index 2475e3ff..24770f5a 100644 --- a/src/XChainDecoder/transaction_ingest.js +++ b/src/XChainDecoder/transaction_ingest.js @@ -45,6 +45,15 @@ function clearQuarantineState(loop){ loop.insertQuarantine.clear() } +// Count a transaction this decoder skipped for good, apart from the routine rejections +// parseErrors also counts. A later whole-block rollback can count one position twice. +function noteQuarantine(decoder, height, hash){ + decoder.quarantinedTxs = (decoder.quarantinedTxs || 0) + 1 + decoder.lastQuarantinedBlock = height + decoder.lastQuarantinedTxHash = hash || null + decoder.lastQuarantinedAt = new Date().toISOString() +} + async function skipQuarantinedTransaction(block, nextBlockHeight, nextTransaction, txIndex){ this.parseErrors++ let quarantinedHash = null @@ -59,10 +68,19 @@ async function skipQuarantinedTransaction(block, nextBlockHeight, nextTransactio // insertEvent already rolled the block transaction back return 'rollback' } + noteQuarantine(this, nextBlockHeight, quarantinedHash) return 'continue' } async function handleParseFailure(loop, e, block, nextBlockHeight, nextTransactionHash, txIndex, retryKey){ + if (e && e.dbLookupFailure){ + // A DB lookup failed inside the block transaction (a timeout, a lock wait, a + // dropped connection). Infrastructure trouble like the RPC case below, never a + // poison transaction, so retry the block indefinitely rather than quarantine. + logger.error(formatLogLine(`DB lookup failed in block ${nextBlockHeight} (tx position ${txIndex}), retrying block:`, e)) + await this.db.endTransaction() + return 'rollback' + } if (e && e.rpcLookupFailure){ // A prevout/fee-output RPC lookup failed even after the // connector's internal retries. That is node/infrastructure @@ -100,6 +118,7 @@ async function handleParseFailure(loop, e, block, nextBlockHeight, nextTransacti // insertEvent already rolled the block transaction back return 'rollback' } + noteQuarantine(this, nextBlockHeight, nextTransactionHash) return 'continue' } @@ -150,6 +169,13 @@ async function insertTransactionRow(loop, parseResult, nextTransactionHash, next // already rolled the block back. Retry indefinitely (never skip // a tx a healthy instance accepts). return 'rollback' + } else if (insertResult === this.db.DUPLICATED_TRANSACTION){ + // A duplicate tx_index or tx hash never happens on a clean instance; writing the outputs + // anyway would attach them to another row, so roll back (a drifted counter re-reads + // MAX(tx_index), a stale pre-reorg row stalls here loudly). The transaction is still open. + logger.error(`insertTransaction hit a duplicate key in block ${nextBlockHeight} (tx position ${txIndex}, tx_index ${loop.lastProcessedTxIndex}, hash ${nextTransactionHash}); possible stale pre-reorg row, rolling the block back`) + await this.db.endTransaction() + return 'rollback' } } @@ -258,7 +284,20 @@ async function persistTransaction(loop, block, nextBlockHeight, openDispenserAdd // A transaction carrying a single create (every non-BATCH transaction, // and every transaction below the gate) collapses to that create // unchanged, so this insert is byte-identical to the one it replaces. - for (let nextRegistration of collapseDispenserRegistrations(dispenserCreateCandidates, this.consensusNetwork, block.timestamp)){ + const addressIdOf = typeof this.db.createAddress === 'function' + ? address => this.db.createAddress(address) + : undefined + let dispenserRegistrations + try { + dispenserRegistrations = await collapseDispenserRegistrations( + dispenserCreateCandidates, this.consensusNetwork, block.timestamp, addressIdOf) + } catch (e) { + // An address lookup failed in the block transaction: roll back and retry the block. + logger.error(formatLogLine(`dispenser address lookup failed at block ${nextBlockHeight}; block rolled back, retrying:`, e)) + await this.db.endTransaction() + return 'rollback' + } + for (const nextRegistration of dispenserRegistrations){ if ((await registerDispenser.call(this, loop, nextRegistration, openDispenserAddresses)) === 'rollback') return 'rollback' } diff --git a/src/XChainDecoder/transaction_parsing.js b/src/XChainDecoder/transaction_parsing.js index 4201f5ba..24ddfc51 100644 --- a/src/XChainDecoder/transaction_parsing.js +++ b/src/XChainDecoder/transaction_parsing.js @@ -254,7 +254,7 @@ function* parseTransactionSteps(transaction, openDispenserAddresses, db, blockHe if (!db) db = this.db // A zero-input transaction has no ins[0] to dereference below (the coinbase/ // standard_input guard also reads ins[0]). An LTC MWEB/HogEx integration tx can - // parse to zero canonical inputs after marker+flag stripping; such a tx carries no + // parse to zero canonical inputs once its MWEB part is set aside; such a tx carries no // XChain data. Skip it cleanly here, mirroring the mempool path's ins.length guard, // so it never throws a TypeError that costs 3 wasted block re-parses + a spurious // PARSE_ERROR quarantine event. diff --git a/src/api.js b/src/api.js index 4fc8dd3c..7944032d 100644 --- a/src/api.js +++ b/src/api.js @@ -46,6 +46,7 @@ const jsonRouter = require('express-json-rpc-router') const { makeRpcBatchGuard, registerLiveRoute, + registerReadyRoute, readReorgHaltForProbe, noteProbeFailure, nodeReachabilityFields, @@ -119,6 +120,9 @@ function buildHealthResult(decoder, state, decoderRunning, decoderError){ : null, rpc_errors: decoder.rpcErrors + decoder.connector.rpcErrors, parse_errors: decoder.parseErrors, + quarantined_txs: decoder.quarantinedTxs || 0, + last_quarantined_block: decoder.lastQuarantinedBlock ?? null, + last_quarantined_tx_hash: decoder.lastQuarantinedTxHash || null, dispenser_purge_failures: decoder.dispenserPurgeFailures || 0, error: decoderError ? decoderError.message : null } @@ -300,11 +304,12 @@ function installAppMiddleware(app, decoder){ installDecoderObservability(app, decoder, { COIN: config.COIN, NETWORK }) } -// Routes in registration order: GET /status, GET /live, the batch guard, the +// Routes in registration order: GET /status, GET /live, GET /ready, the batch guard, the // empty-body default, then the JSON-RPC router mounted at the root. function registerRoutes(app, decoder, isDecoderRunning, getDecoderError){ registerStatusRoute(app, decoder, isDecoderRunning) registerLiveRoute(app, decoder, isDecoderRunning) + registerReadyRoute(app, decoder, isDecoderRunning) // Bound JSON-RPC batch size (see makeRpcBatchGuard). Must run after bodyParser // (req.body parsed) and before the router (dispatch). @@ -386,4 +391,4 @@ if (require.main === module) startApi() // startApi is exported so the crash handlers it installs can be driven for real // rather than asserted against the source text; the require.main guard above // still keeps a plain require from opening a port or a DB connection. -module.exports = { makeRpcBatchGuard, registerLiveRoute, registerStatusRoute, startApi, createMempoolMethods, noteProbeFailure, nodeReachabilityFields, resetProbeLogState, ageProbeLogState, PROBE_LOG_WINDOW_MS } +module.exports = { makeRpcBatchGuard, registerLiveRoute, registerReadyRoute, registerStatusRoute, startApi, createMempoolMethods, noteProbeFailure, nodeReachabilityFields, resetProbeLogState, ageProbeLogState, PROBE_LOG_WINDOW_MS } diff --git a/src/api/probe_routes.js b/src/api/probe_routes.js index 7f2bb002..441e2921 100644 --- a/src/api/probe_routes.js +++ b/src/api/probe_routes.js @@ -169,9 +169,37 @@ async function getLiveProbeState(decoder, isDecoderRunning) { return { decoderRunning, dbOk, stalled, pollSilent, reorgHalt, syncStatus, healthy } } -function sendLiveResponse(res, decoder, state) { +// The READINESS verdict over one /live probe state: whether this decoder is serving +// the chain, as distinct from whether its process is worth keeping alive. Returns the +// ordered list of every condition that holds, empty when ready. Every liveness fault +// is a readiness fault; on top of those, a live REORG_HALT marker (latent or parked), +// a marker probe that has never completed, since "no answer yet" is not "clear", and +// a frozen node tip, under which no height this decoder reports can be trusted as current. +// +// These extra reasons are exactly the ones /live must NOT gate on: the marker survives +// restarts and is released only by an audited operator clear, and a node outage is +// upstream, so autoheal would restart-loop the container on them. They are answered by GET /ready instead, and +// carried on the /live body as ready / not_ready_reasons for anything already polling it. +function decoderNotReadyReasons(state) { + const { decoderRunning, dbOk, stalled, pollSilent, reorgHalt, syncStatus } = state + const reasons = [] + if (!decoderRunning) reasons.push('not_running') + if (!dbOk) reasons.push('db_unreachable') + if (stalled) reasons.push('stalled') + if (pollSilent) reasons.push('poll_silent') + if (reorgHalt && reorgHalt.halted === true) reasons.push('reorg_halt') + if (reorgHalt && reorgHalt.parked === true) reasons.push('reorg_halt_parked') + if (!reorgHalt || !reorgHalt.checked_at) reasons.push('reorg_halt_unchecked') + if (syncStatus && syncStatus.node_height_stale === true) reasons.push('node_height_stale') + return reasons +} + +// The /live JSON body. /live and /ready answer the same body and differ only in which +// verdict sets the status code. +function liveBody(decoder, state) { const { decoderRunning, dbOk, stalled, pollSilent, reorgHalt, syncStatus, healthy } = state - res.status(healthy ? 200 : 503).json({ + const notReady = decoderNotReadyReasons(state) + return { status: healthy ? 'healthy' : 'unhealthy', db: dbOk, running: decoderRunning, @@ -211,8 +239,21 @@ function sendLiveResponse(res, decoder, state) { node_height: syncStatus.node_height, lag: syncStatus.lag, parse_errors: decoder.parseErrors, - rpc_errors: decoder.rpcErrors + decoder.connector.rpcErrors - }) + // Transactions skipped for good, reported but never gating: a restart cannot undo a skip. + quarantined_txs: decoder.quarantinedTxs || 0, + last_quarantined_block: decoder.lastQuarantinedBlock ?? null, + last_quarantined_tx_hash: decoder.lastQuarantinedTxHash || null, + last_quarantined_at: decoder.lastQuarantinedAt || null, + rpc_errors: decoder.rpcErrors + decoder.connector.rpcErrors, + // The readiness verdict (decoderNotReadyReasons above). Reported here, never + // gating this route: GET /ready answers it as a status code. + ready: notReady.length === 0, + not_ready_reasons: notReady + } +} + +function sendLiveResponse(res, decoder, state) { + res.status(state.healthy ? 200 : 503).json(liveBody(decoder, state)) } function registerLiveRoute(app, decoder, isDecoderRunning){ @@ -221,4 +262,16 @@ function registerLiveRoute(app, decoder, isDecoderRunning){ }) } -module.exports = { makeRpcBatchGuard, registerLiveRoute, readReorgHaltForProbe, noteProbeFailure, nodeReachabilityFields, resetProbeLogState, ageProbeLogState, PROBE_LOG_WINDOW_MS } +// GET /ready, the READINESS probe: the /live body, answering 503 while +// not_ready_reasons is non-empty. Meant for monitors, load balancers and anything +// deciding whether to trust this decoder's tip, never for the container +// healthcheck: autoheal restarts on that 503, and a REORG_HALT outlives a restart. +function registerReadyRoute(app, decoder, isDecoderRunning){ + app.get('/ready', async (req, res) => { + const state = await getLiveProbeState(decoder, isDecoderRunning) + const body = liveBody(decoder, state) + res.status(body.ready ? 200 : 503).json(body) + }) +} + +module.exports = { makeRpcBatchGuard, registerLiveRoute, registerReadyRoute, decoderNotReadyReasons, readReorgHaltForProbe, noteProbeFailure, nodeReachabilityFields, resetProbeLogState, ageProbeLogState, PROBE_LOG_WINDOW_MS } diff --git a/src/chain/XChainBlockDecoder.js b/src/chain/XChainBlockDecoder.js index 3af7309c..2583d794 100644 --- a/src/chain/XChainBlockDecoder.js +++ b/src/chain/XChainBlockDecoder.js @@ -19,8 +19,21 @@ const coins = require('../coins'); const LITECOIN_HOGEX_FLAG = 0x08 const LITECOIN_MWEB_SEGWIT_FLAG = 0x09 +class TransactionParseError extends Error { + constructor(message, cause) { + super(message, cause ? { cause } : undefined) + this.name = 'TransactionParseError' + this.code = 'ERR_TRANSACTION_PARSE' + } +} + +function transactionParseError(message, cause) { + if (cause instanceof TransactionParseError) return cause + return new TransactionParseError(message, cause) +} + // Block/tx wire-serialization families this decoder actually implements. The -// per-coin branches below (Litecoin 'mweb' marker/flag stripping, Dogecoin +// per-coin branches below (Litecoin 'mweb' MWEB-flagged tx reading, Dogecoin // 'auxpow' handled upstream in BlockchainConnector.getBlockWithoutAuxPow, // 'default' plain bitcoinjs) are keyed on the coin's declared wireFormat from // the canonical coin registry (src/coins) rather than a hardcoded coin-name @@ -31,6 +44,145 @@ const LITECOIN_MWEB_SEGWIT_FLAG = 0x09 // special block. const HANDLED_WIRE_FORMATS = new Set(['default', 'mweb', 'auxpow']) +// Litecoin's flagged transaction serialization (litecoin src/primitives/transaction.h +// SerializeTransaction): version | 0x00 marker | flags | vin | vout | witness when +// flags & 1 | mweb_tx when flags & 8 | nLockTime. The readers below walk it with +// bounds checks so a short buffer fails by name instead of over-reading. +// Keep in sync with xchain-utxo-tracker/src/chain/XChainBlockDecoder.js: every function from +// mwebShortError through assertBlockBodyIntact is byte-identical in both repos, and +// xchain-utxo-tracker/test/unit/block_decoder_twin_parity.test.js compares verdicts. +function mwebShortError(what) { + return new Error('MWEB tx parse: buffer too short for ' + what) +} + +function skipMwebBytes(buf, offset, count, what) { + if (count > buf.length - offset) throw mwebShortError(what) + return offset + count +} + +// A CompactSize at offset, as { value, end }. +function readMwebCompactSize(buf, offset, what) { + if (offset >= buf.length) throw mwebShortError(what) + const first = buf[offset] + if (first < 0xFD) return { value: first, end: offset + 1 } + const width = first === 0xFD ? 2 : first === 0xFE ? 4 : 8 + const end = skipMwebBytes(buf, offset + 1, width, what) + if (width === 2) return { value: buf.readUInt16LE(offset + 1), end } + if (width === 4) return { value: buf.readUInt32LE(offset + 1), end } + const value = buf.readBigUInt64LE(offset + 1) + if (value > BigInt(Number.MAX_SAFE_INTEGER)) { + throw new TransactionParseError('MWEB tx parse: ' + what + ' exceeds Number.MAX_SAFE_INTEGER') + } + return { value: Number(value), end } +} + +function skipMwebVarBytes(buf, offset, what) { + const length = readMwebCompactSize(buf, offset, what) + return skipMwebBytes(buf, length.end, length.value, what) +} + +// Where vin and vout end, where vout starts, and the input count the witness needs. +function skipMwebInsAndOuts(buf, offset) { + const ins = readMwebCompactSize(buf, offset, 'input count') + offset = ins.end + for (let i = 0; i < ins.value; i++) { + offset = skipMwebBytes(buf, offset, 36, 'input prevout') + offset = skipMwebVarBytes(buf, offset, 'input script') + offset = skipMwebBytes(buf, offset, 4, 'input sequence') + } + const outsStart = offset + const outs = readMwebCompactSize(buf, offset, 'output count') + offset = outs.end + for (let i = 0; i < outs.value; i++) { + offset = skipMwebBytes(buf, offset, 8, 'output value') + offset = skipMwebVarBytes(buf, offset, 'output script') + } + return { end: offset, outsStart, nIns: ins.value } +} + +function skipMwebWitness(buf, offset, nIns) { + for (let i = 0; i < nIns; i++) { + const items = readMwebCompactSize(buf, offset, 'witness item count') + offset = items.end + for (let j = 0; j < items.value; j++) offset = skipMwebVarBytes(buf, offset, 'witness item') + } + return offset +} + +// True when the tx at buf[offset] is version 1 or 2 with the 0x00 marker and flag +// 0x08 (MWEB) or 0x09 (witness and MWEB): the forms this decoder reads itself. +function hasMwebFlag(buf, offset) { + if (buf.length - offset < 6) return false + const version = buf.readUInt32LE(offset) + const flag = buf[offset + 5] + return (version === 1 || version === 2) && buf[offset + 4] === 0x00 && + (flag === LITECOIN_HOGEX_FLAG || flag === LITECOIN_MWEB_SEGWIT_FLAG) +} + +// bitcoinjs reads an empty input count followed by 0x01 as the segwit marker, so a +// tx with no transparent inputs is built field by field rather than re-parsed. +function buildInputlessTransaction(buf, start, outsStart, lockTimeAt) { + const tx = new bitcoinjs.Transaction() + tx.version = buf.readInt32LE(start) + const outs = readMwebCompactSize(buf, outsStart, 'output count') + let offset = outs.end + for (let i = 0; i < outs.value; i++) { + // The same unsigned BigInt the patched bufferutils reader gives every output value. + const value = buf.readBigUInt64LE(offset) + const script = readMwebCompactSize(buf, offset + 8, 'output script') + offset = script.end + script.value + tx.outs.push({ script: Buffer.from(buf.subarray(script.end, offset)), value }) + } + tx.locktime = buf.readUInt32LE(lockTimeAt) + return tx +} + +// Read one MWEB-flagged tx at buf[start]. Returns it without its MWEB part, so +// getId() is Litecoin's ComputeHash (no witness, no MWEB) and the witness, where a +// P2WSH carrier keeps its payload, survives; plus the bytes consumed. mweb_tx is an +// OptionalPtr: tag 0x00 means no MWEB part (the HogEx), any other tag starts an +// MWEB::Tx body this file does not parse. A standalone tx (getrawtransaction) ends +// with its nLockTime, so the body runs to four bytes before the end; inside a block +// nothing records the body's length, so a non-null MWEB part there is refused. +function readMwebFlaggedTransaction(buf, start, standalone) { + const withWitness = (buf[start + 5] & 1) === 1 + const insOuts = skipMwebInsAndOuts(buf, start + 6) + const witnessEnd = withWitness ? skipMwebWitness(buf, insOuts.end, insOuts.nIns) : insOuts.end + let lockTimeAt = skipMwebBytes(buf, witnessEnd, 1, 'MWEB tag') + if (buf[witnessEnd] !== 0x00) { + if (!standalone) throw new Error('MWEB tx parse: a non-null MWEB part inside a block has no recorded length') + if (buf.length - 4 < lockTimeAt) throw mwebShortError('locktime') + lockTimeAt = buf.length - 4 + } + const end = skipMwebBytes(buf, lockTimeAt, 4, 'locktime') + if (insOuts.nIns === 0) { + return { tx: buildInputlessTransaction(buf, start, insOuts.outsStart, lockTimeAt), consumed: end - start } + } + const plain = Buffer.concat([ + buf.subarray(start, start + 4), + Buffer.from(withWitness ? [0x00, 0x01] : []), + buf.subarray(start + 6, witnessEnd), + buf.subarray(lockTimeAt, end), + ]) + return { tx: bitcoinjs.Transaction.fromBuffer(plain), consumed: end - start } +} + +// bitcoinjs Block.fromBuffer checks neither the merkle root nor what follows the +// last transaction, so a parsed block body is held to three things here: at least +// one transaction (the coinbase), txids that hash to the header's merkle root, and +// no bytes after the last transaction except where Litecoin writes the MWEB +// extension block, after a HogEx in last place. +function assertBlockBodyIntact(block, bufferLength, bodyEnd, extensionFollows) { + const txs = block.transactions || [] + if (txs.length < 1) throw new Error('Block holds no transactions') + if (bodyEnd !== bufferLength && !extensionFollows) { + throw new Error('Block has ' + (bufferLength - bodyEnd) + ' bytes after its last transaction') + } + if (!bitcoinjs.Block.calculateMerkleRoot(txs).equals(block.merkleRoot)) { + throw new Error('Block merkle root mismatch: the header root is not the root of its transactions') + } +} + function readMwebTransactions(bufferReader, block) { const readTransaction = () => { const tx = transaction_js_1.Transaction.fromBuffer( @@ -55,36 +207,29 @@ function readMwebTransactions(bufferReader, block) { remainingBytes + ' bytes remain (invalid transaction count)'); } block.transactions = []; + let hogExLast = false for (let i = 0; i < nTransactions; ++i) { - try { - if (i == nTransactions - 1){//If it's the last transaction, then check if it's the HogEx - let nextTxBuffer = bufferReader.buffer.slice(bufferReader.offset) - let txVersion = nextTxBuffer.readUInt32LE(); - let marker = nextTxBuffer.readUInt8(4); - let flag = nextTxBuffer.readUInt8(5); - - if ((txVersion == 0x01 || txVersion == 0x02) && (marker == 0x00) && (flag == LITECOIN_HOGEX_FLAG || flag == LITECOIN_MWEB_SEGWIT_FLAG)){ - let removeOffsetStart = bufferReader.offset + 4 //4 bytes for txVersion - let removeOffsetEnd = removeOffsetStart + 2 //2 bytes for marker + flag - - //Remove the marker+flag (0x08 pure-MWEB, or 0x09 segwit+MWEB), so bitcoinjs-lib parses this tx as a normal transaction - let bufferReaderBeforeFlag = bufferReader.buffer.slice(0, removeOffsetStart) - let bufferReaderAfterFlag = bufferReader.buffer.slice(removeOffsetEnd) - bufferReader.buffer = Buffer.concat([bufferReaderBeforeFlag, bufferReaderAfterFlag]) - - } + // Only the last tx can be the HogEx, which Litecoin writes with the MWEB + // flag; it is read whole, so its nLockTime and txid are its real ones. + if (i == nTransactions - 1 && hasMwebFlag(bufferReader.buffer, bufferReader.offset)) { + const read = readMwebFlaggedTransaction(bufferReader.buffer, bufferReader.offset, false) + bufferReader.offset += read.consumed + block.transactions.push(read.tx) + hogExLast = true + } else { + block.transactions.push(readTransaction()) } - - let tx = readTransaction(); - block.transactions.push(tx); - } catch (err){ - throw err - } } + // The MWEB extension block follows only a HogEx in last place. + return hogExLast } class XChainBlockDecoder { + // Callers tell a wire-parse failure apart by this class, reached through the one + // export, so the module keeps a single export shape. + static TransactionParseError = TransactionParseError + constructor(networkName) { let networkNameSplit = networkName.split("-") this.coin = networkNameSplit[0] @@ -103,21 +248,28 @@ class XChainBlockDecoder { } transactionFromHex(txHex){ - if (this.wireFormat === "mweb") { - const marker = txHex.substr(8, 2) - const flag = txHex.substr(10, 2) - if ( - (txHex.substr(0, 8) === "01000000" || txHex.substr(0, 8) === "02000000") && - marker === "00" && - (flag === "08" || flag === "09") - ) { - // Strip the 2-byte marker+flag (MWEB flag 0x08, or segwit+MWEB 0x09). - // bitcoinjs-lib misreads these flags causing UInt64 range errors. - txHex = txHex.substr(0, 8) + txHex.substr(12) + try { + if (typeof txHex !== 'string' || txHex.length === 0 || (txHex.length % 2) !== 0 || !/^[0-9a-fA-F]+$/.test(txHex)) { + throw new TransactionParseError('Transaction hex must be a non-empty, even-length hexadecimal string') + } + const buffer = Buffer.from(txHex, "hex") + if (this.wireFormat === "mweb" && hasMwebFlag(buffer, 0)) { + // Litecoin's getrawtransaction answers in the MWEB serialization when the + // tx has an MWEB part; bitcoinjs cannot read that flag, so read it here. + const parsed = readMwebFlaggedTransaction(buffer, 0, true) + if (parsed.consumed !== buffer.length) { + throw new TransactionParseError('Transaction has ' + (buffer.length - parsed.consumed) + ' trailing byte(s)') + } + return parsed.tx } + const tx = transaction_js_1.Transaction.fromBuffer(buffer, true) + if (tx.byteLength() !== buffer.length) { + throw new TransactionParseError('Transaction has ' + (buffer.length - tx.byteLength()) + ' trailing byte(s)') + } + return tx + } catch (err) { + throw transactionParseError('Transaction parse failed: ' + (err && err.message ? err.message : err), err) } - // Non-strict mode ignores trailing MWEB extension data after the tx. - return transaction_js_1.Transaction.fromBuffer(Buffer.from(txHex, "hex"), true) } doubleSha256AndReverse(data){ @@ -144,13 +296,29 @@ class XChainBlockDecoder { block.bits = bufferReader.readUInt32(); block.nonce = bufferReader.readUInt32(); if (buffer.length === 80) return block; - readMwebTransactions(bufferReader, block) + let extensionFollows + try { + extensionFollows = readMwebTransactions(bufferReader, block) + } catch (err) { + throw transactionParseError('Block transaction parse failed: ' + (err && err.message ? err.message : err), err) + } + assertBlockBodyIntact(block, buffer.length, bufferReader.offset, extensionFollows) const witnessCommit = block.getWitnessCommit(); // This Block contains a witness commit if (witnessCommit) block.witnessCommit = witnessCommit; - return block; - default: - return bitcoinjs.Block.fromBuffer(buffer) + return block; + default: { + let plainBlock + try { + plainBlock = bitcoinjs.Block.fromBuffer(buffer) + } catch (err) { + throw transactionParseError('Block transaction parse failed: ' + (err && err.message ? err.message : err), err) + } + // An 80-byte buffer is a header-only parse, with no body to check. + if (buffer.length === 80) return plainBlock + assertBlockBodyIntact(plainBlock, buffer.length, plainBlock.byteLength(), false) + return plainBlock + } } } } diff --git a/src/chain/blockchain_connector/auxpow_codec.js b/src/chain/blockchain_connector/auxpow_codec.js index 00a15c6c..fde30470 100644 --- a/src/chain/blockchain_connector/auxpow_codec.js +++ b/src/chain/blockchain_connector/auxpow_codec.js @@ -12,6 +12,11 @@ * ********************************************************************/ +const bitcoinjs = require('bitcoinjs-lib') +// BigInt-safe 64-bit reader, so a Dogecoin output above 2^53-1 sat parses in +// assertWholeBlock exactly as it does in the block decoder. +require('../apply_bufferutils_patch') + // Decode a Bitcoin-style varint from `buf` at `offset`. // Returns { value, bytes } where `bytes` is the number of bytes consumed. // Keep in sync with xchain-utxo-tracker/src/chain/blockchain_connector/auxpow_codec.js readVarint. @@ -136,12 +141,41 @@ function skipAuxPow(buf, start) { return offset } +// Check that a stripped block hex is one whole block: at least one transaction (the +// coinbase), no bytes after the last transaction, and a merkle root over the parsed +// txids equal to the header's. bitcoinjs Block.fromBuffer checks none of these, so a +// cut that lands a few bytes off (one changed branch-count byte is enough) can still +// read as a block, often one whose tx count is 0 with the real transactions ignored +// as trailing bytes. Throwing here keeps that inside the caller's AuxPoW try, which +// tags it auxPowParseFailure so the fetch path escalates to per-tx reassembly. +// Keep in sync with xchain-utxo-tracker/src/chain/blockchain_connector/auxpow_codec.js assertWholeBlock. +function assertWholeBlock(blockHex) { + const buf = Buffer.from(blockHex, 'hex') + if (buf.length * 2 !== blockHex.length) throw new Error('AuxPoW parse: stripped block is not whole hex bytes') + let block + try { + block = bitcoinjs.Block.fromBuffer(buf) + } catch (err) { + throw new Error('AuxPoW parse: stripped block does not parse: ' + err.message) + } + const txs = block.transactions || [] + if (txs.length < 1) throw new Error('AuxPoW parse: stripped block holds no transactions') + const used = block.byteLength() + if (used !== buf.length) { + throw new Error('AuxPoW parse: stripped block has ' + (buf.length - used) + ' bytes after its last transaction') + } + if (!bitcoinjs.Block.calculateMerkleRoot(txs).equals(block.merkleRoot)) { + throw new Error('AuxPoW parse: stripped block merkle root does not match its transactions') + } +} + // Strip the AuxPoW section from a merge-mined block's hex, preserving the 80-byte // (160 hex char) standard header. Two daemon behaviors are handled: an older daemon // whose getblockheader already includes the AuxPoW bytes (length-based strip via the // header/block length delta), and Dogecoin Core 1.14 whose getblockheader always // returns exactly 160 chars, requiring the AuxPoW size to be parsed structurally from -// the block hex (skipAuxPow). Non-AuxPoW blocks pass through unchanged. +// the block hex (skipAuxPow). Either cut must leave one whole block (assertWholeBlock). +// Non-AuxPoW blocks pass through unchanged. // Keep in sync with xchain-utxo-tracker/src/chain/blockchain_connector/auxpow_codec.js stripAuxPowFromBlockHex. // test/unit/auxpow_strip_parity.test.js asserts byte identity of the two function bodies, // so a strip correction cannot land in one repo alone. @@ -149,7 +183,9 @@ function stripAuxPowFromBlockHex(headerHex, blockHex) { const dataToRemove = headerHex.length - 160 // 160 hex chars = 80-byte standard header if (dataToRemove > 0) { // Legacy path: getblockheader included AuxPoW bytes (older daemon). - return blockHex.substring(0, 160) + blockHex.substring(160 + dataToRemove) + const stripped = blockHex.substring(0, 160) + blockHex.substring(160 + dataToRemove) + assertWholeBlock(stripped) + return stripped } if (blockHex.length >= 8) { const versionLE = parseInt(blockHex.substring(0, 8), 16) @@ -160,7 +196,9 @@ function stripAuxPowFromBlockHex(headerHex, blockHex) { // (Dogecoin Core 1.14). Parse the AuxPoW size from the block hex directly. const blockBuf = Buffer.from(blockHex, 'hex') const afterAuxPow = skipAuxPow(blockBuf, 80) - return blockHex.substring(0, 160) + blockHex.substring(afterAuxPow * 2) + const stripped = blockHex.substring(0, 160) + blockHex.substring(afterAuxPow * 2) + assertWholeBlock(stripped) + return stripped } } return blockHex @@ -170,5 +208,6 @@ module.exports = { readVarint, encodeVarintHex, skipAuxPow, + assertWholeBlock, stripAuxPowFromBlockHex, } diff --git a/src/coins/BTC.js b/src/coins/BTC.js index b50f3d4a..8dfbdfbc 100644 --- a/src/coins/BTC.js +++ b/src/coins/BTC.js @@ -162,10 +162,19 @@ module.exports = { // plus airdropSetHash - the sha256 over the canonical `name:hash:amount` lines, // which genesis.js verifies before it credits anything - followed by a // bin/sync-coins.sh re-vendoring wave. + // Both hashes re-pinned 2026-10-13: the name registrations BASE, + // DASH, HBAR, HOOD, HYPE and NEAR left the CSV manifest so the bridge owns + // those chain roots, six rows out of 121,716. They were supply-0, balance-free + // name rows with no subassets and no post-genesis action on either chain, so + // nothing else in the manifest or the dump moved: the dump loses only those six + // names' rows and its ledger and contracts block hashes are unchanged. + // The six stay reserved names (RESERVED_FUTURE_ROOTS in xchain-indexer, operator + // ruling 2026-10-17): with no row left, a new ISSUE of any of them is refused + // wherever TICK_NAMESPACE_ACTIVATION binds (unarmed on mainnet today). genesis: { block: 950000, - ledgerHash: 'f347a0499654b128dd0461441ed6341ac27a24b909d6ff6e3995db4e69dc23e5', - dumpHash: 'e416c382dd6951ce5c4da7e053f6dbb407ca3d966a513209eb59886dbe95e738', + ledgerHash: '13aeeebb9f2e27d507ca96fb7c54a1bfa98b9cc80c3feebd5d1ad022ce7e2b73', + dumpHash: '85431c2c41d6442e38aa78af804148dd6ead207aca036f6e44fbf2bee8c2846e', airdropPaths: [], airdropHashes: [], airdropAmounts: [], diff --git a/src/coins/DOGE.js b/src/coins/DOGE.js index 787327f3..41e51c0f 100644 --- a/src/coins/DOGE.js +++ b/src/coins/DOGE.js @@ -93,10 +93,19 @@ module.exports = { // plus airdropSetHash - the sha256 over the canonical `name:hash:amount` lines, // which genesis.js verifies before it credits anything - followed by a // bin/sync-coins.sh re-vendoring wave. + // Both hashes re-pinned 2026-10-13: the name registrations BASE, + // DASH, HBAR, HOOD, HYPE and NEAR left the CSV manifest so the bridge owns + // those chain roots, six rows out of 42,704. They were supply-0, balance-free + // name rows with no subassets and no post-genesis action on either chain, so + // nothing else in the manifest or the dump moved: the dump loses only those six + // names' rows and its ledger and contracts block hashes are unchanged. + // The six stay reserved names (RESERVED_FUTURE_ROOTS in xchain-indexer, operator + // ruling 2026-10-17): with no row left, a new ISSUE of any of them is refused + // wherever TICK_NAMESPACE_ACTIVATION binds (unarmed on mainnet today). genesis: { block: 6240000, - ledgerHash: '87abac0b03cbd24694f7c5e666425bdae6f9a9c51b826826bd4848bd4ade991b', - dumpHash: '81d3d2cb4714bc40ec12c2d1c8573040858ddf66e4ba89a05d44d86d3bffa0f1', + ledgerHash: '888fc0be89d4ac58dff24ef5b02c8ccfb947cef9c3d4bc3b28db2202e68da5df', + dumpHash: '9aae4cdf2bc14de119e089c267d442aec4671e01f6139aff486460798bb96a92', airdropPaths: [], airdropHashes: [], airdropAmounts: [], diff --git a/src/db/addresses_and_events.js b/src/db/addresses_and_events.js index dc65c053..ff9d1759 100644 --- a/src/db/addresses_and_events.js +++ b/src/db/addresses_and_events.js @@ -15,6 +15,16 @@ const { format: formatLogLine } = require('node:util'); const { logger } = require('./constants.js') const { jsonBigIntSafe } = require('./query_helpers.js') +const { DETERMINISTIC_WRITE_ERRNOS } = require('./constants.js') + +// Inside the block transaction a failed lookup must roll the block back: a swallowed error +// hands back a NULL id that commits with the block. Outside it (the mempool handle) the +// caller keeps its null or false fallback, so this returns and the caller logs and goes on. +function rethrowInTransaction(database, err){ + if (database.transactionConnection == null) return + if (err && typeof err === 'object') err.dbLookupFailure = true + throw err +} module.exports = { async getAddressId(address){ @@ -27,6 +37,7 @@ module.exports = { id = rows[0].id; } catch (err) { logger.error(formatLogLine('Error looking up address record id in index_addresses table:', err)); + rethrowInTransaction(this, err) } finally { if (this.transactionConnection == null){ await db.release() @@ -50,6 +61,7 @@ module.exports = { await db.query(query, [address]); } catch (err) { logger.error(formatLogLine('Error trying to create address record in index_addresses table:', err)); + rethrowInTransaction(this, err) } finally { if (this.transactionConnection == null){ await db.release() @@ -67,6 +79,7 @@ module.exports = { return rows.length > 0 } catch (err) { logger.error(formatLogLine('Error checking pubkey existence:', err)) + rethrowInTransaction(this, err) return false } finally { if (this.transactionConnection == null){ @@ -82,6 +95,9 @@ module.exports = { return true } catch (err) { logger.error(formatLogLine('Error inserting pubkey:', err)) + // A deterministic content error on a pubkey never fails the block (it would + // quarantine a good transaction); only a transient one rolls the block back. + if (!DETERMINISTIC_WRITE_ERRNOS.has(err && err.errno)) rethrowInTransaction(this, err) return false } finally { if (this.transactionConnection == null){ diff --git a/src/db/blocks.js b/src/db/blocks.js index cb0cdf7c..308855a3 100644 --- a/src/db/blocks.js +++ b/src/db/blocks.js @@ -255,9 +255,6 @@ module.exports = { ) VALUES (?, ?, ?, ?); `; - let blockHashId = await this.createTransaction(block.block_hash) - let previousBlockHashId = await this.createTransaction(block.previous_block_hash) - let connection = await this.getConnection() // Snapshot whether WE acquired this lease. Inside a block transaction // getConnection() returns the shared this.transactionConnection, and the catch @@ -267,6 +264,10 @@ module.exports = { const ownLease = (this.transactionConnection == null) try { + // Inside the try, so a lookup that fails in the block transaction rolls it back + // through the catch below instead of escaping the block loop. + let blockHashId = await this.createTransaction(block.block_hash) + let previousBlockHashId = await this.createTransaction(block.previous_block_hash) await connection.query(query, [ block.block_index, blockHashId, diff --git a/src/db/connection_lifecycle.js b/src/db/connection_lifecycle.js index 55557be3..9675c6b0 100644 --- a/src/db/connection_lifecycle.js +++ b/src/db/connection_lifecycle.js @@ -115,32 +115,50 @@ module.exports = { } }, + // Every step after the lock is taken sits inside one guard that releases the lock + // exactly once on failure, so a DB outage (getConnection gives up after its retry + // budget) cannot leave the lock held and wedge the next caller, such as a reorg retry. async beginTransaction(){ await this.acquireTransactionLock() - - if (this.transactionConnection != null){ - await this.endTransaction() - } - - this.transactionConnection = await this.getConnection() try { + if (this.transactionConnection != null){ + await this.discardStaleTransactionConnection() + } + this.transactionConnection = await this.getConnection() await this.transactionConnection.beginTransaction() } catch(err){ - await this.transactionConnection.release() - this.transactionConnection = null + await this.releaseTransactionConnectionQuietly() this.releaseTransactionLock() throw err } }, + // Roll back and drop a connection left over from an earlier transaction without + // touching the lock: the caller has just acquired it and still holds it. + async discardStaleTransactionConnection(){ + try { await this.transactionConnection.rollback() } catch(_){} + await this.releaseTransactionConnectionQuietly() + }, + + // Release the transaction connection, ignoring a release error on a dead connection. + async releaseTransactionConnectionQuietly(){ + if (this.transactionConnection == null) return + try { await this.transactionConnection.release() } catch(_){} + this.transactionConnection = null + }, + + // The lock is released in a finally, so a rollback that throws on a dead + // connection still frees it; the rollback error itself is passed on. async endTransaction(){ - if (this.transactionConnection != null){ - logger.info("rolling back") - await this.transactionConnection.rollback() - await this.transactionConnection.release() - this.transactionConnection = null + try { + if (this.transactionConnection != null){ + logger.info("rolling back") + await this.transactionConnection.rollback() + } + } finally { + await this.releaseTransactionConnectionQuietly() + this.releaseTransactionLock() } - this.releaseTransactionLock() }, async commitTransaction(){ @@ -153,7 +171,10 @@ module.exports = { return true } catch (e){ logger.error("There was an error trying to commit a transaction: " + e.code) - await this.endTransaction() + // A failed commit reports false to the caller even when the rollback after it also fails. + try { await this.endTransaction() } catch(rollbackErr){ + logger.error("Rollback after a failed commit also failed: " + (rollbackErr && rollbackErr.code)) + } } } diff --git a/src/db/migration_live_schema_guard.js b/src/db/migration_live_schema_guard.js index 3d7159e0..12eb6259 100644 --- a/src/db/migration_live_schema_guard.js +++ b/src/db/migration_live_schema_guard.js @@ -21,6 +21,7 @@ * information_schema for every MODIFY in an auto file and names each loss. A type change * it has no widening rule for counts as a loss, and a text MODIFY that names no charset * or collation is compared against the table default the server would fall back to. + * A MODIFY keyword the parse cannot place refuses the file instead of being skipped. * ********************************************************************/ @@ -106,18 +107,39 @@ function stripBlockComments(text){ return out; } -// Every `ALTER TABLE t ... MODIFY [COLUMN] c ` clause in a statement list. +// MariaDB's `ALTER [ONLINE] [IGNORE] TABLE [IF EXISTS] t [WAIT n | NOWAIT]` head. The lock-wait +// option sits between the table and the first alter_spec, so it is consumed here, not read as one. +const ALTER_HEAD = /^ALTER\s+(?:ONLINE\s+)?(?:IGNORE\s+)?TABLE\s+(?:IF\s+EXISTS\s+)?((?:`[^`]+`|[\w$]+)(?:\s*\.\s*(?:`[^`]+`|[\w$]+))?)\s+(?:(?:WAIT\s+\d+(?:\.\d+)?|NOWAIT)\s+)?([\s\S]*)$/i; + +// The table and top-level alter_specs of an ALTER TABLE statement, or null when the head does not parse. +function alterSpecs(stmt){ + const head = ALTER_HEAD.exec(stripBlockComments(stmt).trim()); + if(!head) return null; + return { table: unquote(head[1].split('.').pop().trim()), specs: splitTopLevel(head[2]) }; +} + +// True when text carries a MODIFY keyword outside every quoted literal and identifier. +const namesModify = (text) => /\bMODIFY\b/i.test(blankLiterals(text)); + +// Every `ALTER TABLE t ... MODIFY [COLUMN] c ` clause in a statement list. A MODIFY +// keyword this parse cannot place comes back as { unparsed } so the caller fails closed on it. function modifyClauses(statements){ const found = []; for(const stmt of statements){ - const head = /^ALTER\s+(?:ONLINE\s+)?(?:IGNORE\s+)?TABLE\s+(?:IF\s+EXISTS\s+)?((?:`[^`]+`|\w+)(?:\s*\.\s*(?:`[^`]+`|\w+))?)\s+([\s\S]*)$/i.exec(stripBlockComments(stmt).trim()); - if(!head) continue; - const table = unquote(head[1].split('.').pop().trim()); - for(const clause of splitTopLevel(head[2])){ + const parsed = alterSpecs(stmt); + if(!parsed){ + const text = stripBlockComments(stmt).trim(); + if(/^ALTER\b/i.test(text) && namesModify(text)) found.push({ unparsed: text }); + continue; + } + for(const clause of parsed.specs){ const m = /^\s*MODIFY\s+(?:COLUMN\s+)?(?:IF\s+EXISTS\s+)?(`[^`]+`|\w+)\s+([\s\S]+)$/i.exec(clause); - if(!m) continue; + if(!m){ + if(namesModify(clause)) found.push({ table: parsed.table, unparsed: clause.trim() }); + continue; + } const definition = m[2].replace(/\s+(?:FIRST|AFTER\s+(?:`[^`]+`|\w+))\s*$/i, '').trim(); - found.push({ table, column: unquote(m[1]), definition }); + found.push({ table: parsed.table, column: unquote(m[1]), definition }); } } return found; @@ -258,7 +280,13 @@ function losses(live, definition){ // Throw when an auto file's MODIFY would strip or narrow a live column. An absent live // column (a fresh install, or one the file itself adds) is not drift and passes through. async function assertNoLiveColumnLoss(conn, file, statements){ - for(const { table, column, definition } of modifyClauses(statements)){ + for(const { table, column, definition, unparsed } of modifyClauses(statements)){ + if(unparsed != null){ + throw new Error('runMigrations: ' + file + ' is tagged mode=auto but carries a MODIFY this guard cannot read: "' + + unparsed.slice(0, 160) + (unparsed.length > 160 ? '...' : '') + '", so the live column it lands on goes unchecked. ' + + 'Rewrite it as `ALTER TABLE t [WAIT n | NOWAIT] MODIFY [COLUMN] c `, or re-tag the file ' + + '`-- xchain:migration mode=manual` and apply it deliberately via `node src/db/migrate.js`.'); + } const rows = await conn.query( 'SELECT COLUMN_TYPE, COLUMN_DEFAULT, EXTRA, COLUMN_COMMENT, GENERATION_EXPRESSION, CHARACTER_SET_NAME, ' + 'COLLATION_NAME, TABLE_COLLATION FROM information_schema.columns c JOIN information_schema.tables t ' + @@ -277,4 +305,4 @@ async function assertNoLiveColumnLoss(conn, file, statements){ } } -module.exports = { assertNoLiveColumnLoss, modifyClauses, losses, stripBlockComments }; +module.exports = { assertNoLiveColumnLoss, modifyClauses, alterSpecs, losses, stripBlockComments, blankLiterals }; diff --git a/src/db/migration_preconditions.js b/src/db/migration_preconditions.js index a286f72c..dcbd04a8 100644 --- a/src/db/migration_preconditions.js +++ b/src/db/migration_preconditions.js @@ -27,7 +27,63 @@ const Database = require('../db.js') // both properties intact and covers every invocation route (startup, blanket // `node src/db/migrate.js`, and a targeted `--file` rollout), since all three funnel through // this loop. +const BIGINT_ID_MIGRATION = '2026-06-02-widen-ids-to-bigint.sql'; + +// Every (table, column) pair BIGINT_ID_MIGRATION MODIFYs, in file order. +const BIGINT_ID_COLUMNS = { + index_addresses: ['id'], + index_transactions: ['id'], + pubkeys: ['address_id'], + blocks: ['block_index', 'block_time', 'block_hash_id', 'previous_block_hash_id'], + transactions: ['tx_index', 'tx_hash_id', 'block_index', 'source_id', 'destination_id'], + dispensers: ['tx_index', 'address_id'], + transaction_outputs: ['tx_index', 'vout', 'destination_id'], +}; + Database.MIGRATION_PRECONDITIONS = { + // Widens every id/key column to BIGINT UNSIGNED and re-adds pubkeys_ibfk_1. It is + // mode=manual and a deploy precondition, so on a database built from the current + // src/sql (already BIGINT UNSIGNED, with the unnamed pubkeys FK auto-named + // pubkeys_ibfk_1) it would sit PENDING with no ledger row, and the deploy tool + // refuses every update while a required file has no row. + // + // Baseline only when all 17 columns the file MODIFYs are already BIGINT UNSIGNED, + // both surrogate keys keep AUTO_INCREMENT, and the FK is present. Any narrower, + // signed, absent or unreadable column, or a missing FK (a crash between the file's + // drop and re-add), is deliberately NOT baselined: the file still has work to do. + [BIGINT_ID_MIGRATION]: { + sql: "SELECT c.TABLE_NAME AS tbl, c.COLUMN_NAME AS col, c.DATA_TYPE AS dataType, " + + "c.COLUMN_TYPE AS columnType, c.EXTRA AS extra, " + + "(SELECT COUNT(*) FROM information_schema.key_column_usage k " + + "WHERE k.table_schema = c.table_schema AND k.table_name = 'pubkeys' " + + "AND k.column_name = 'address_id' AND k.constraint_name = 'pubkeys_ibfk_1' " + + "AND k.referenced_table_name = 'index_addresses' AND k.referenced_column_name = 'id') AS fkCount " + + "FROM information_schema.columns c WHERE c.table_schema = ? AND c.table_name IN (" + + Object.keys(BIGINT_ID_COLUMNS).map(t => "'" + t + "'").join(', ') + ")", + skipWhen: (rows) => { + if(!Array.isArray(rows) || !rows.length) return null; + const live = new Map(); + let fk = false; + for(const row of rows){ + if(!row || row.tbl == null || row.col == null) continue; + live.set(String(row.tbl).toLowerCase() + '.' + String(row.col).toLowerCase(), row); + if(row.fkCount != null && Number(row.fkCount) >= 1) fk = true; + } + if(!fk) return null; + for(const [table, columns] of Object.entries(BIGINT_ID_COLUMNS)){ + for(const column of columns){ + const row = live.get(table + '.' + column); + if(!row || row.dataType == null || row.columnType == null) return null; + if(String(row.dataType).toLowerCase() !== 'bigint') return null; + if(!/\bunsigned\b/.test(String(row.columnType).toLowerCase())) return null; + if(column === 'id' && !/\bauto_increment\b/.test(String(row.extra || '').toLowerCase())) return null; + } + } + return 'every id/key column widen-ids MODIFYs is already BIGINT UNSIGNED and pubkeys_ibfk_1 is present, ' + + 'so there is nothing to widen.'; + } + }, + // DATETIME -> BIGINT UNSIGNED converter. It is mode=manual, so it stays PENDING on a // database created from the current dispensers.sql (already BIGINT UNSIGNED) - and the // documented blanket `npm run migrate` applies every pending manual file. Run against a @@ -201,8 +257,6 @@ Database.backdatedFrontierViolation = function(pendingName, appliedNames){ // startup that sees it, so it can never be the missing precondition. Database.DEPLOY_PRECONDITION_TAG = 'deploy-precondition=required'; -const BIGINT_ID_MIGRATION = '2026-06-02-widen-ids-to-bigint.sql'; - // Migrations this tree ASSERTS at startup: the service refuses to run when the // target database has not applied them. // @@ -224,7 +278,9 @@ const BIGINT_ID_MIGRATION = '2026-06-02-widen-ids-to-bigint.sql'; // here must exist, be mode=manual, and carry the tag. // // ADDING A STARTUP ASSERTION: register it here and tag its migration file, or the -// next fleet deploy discovers the requirement as a crash-loop again. A server-setting +// next fleet deploy discovers the requirement as a crash-loop again. Give the file a +// MIGRATION_PRECONDITIONS baseline too, or a database built from the current src/sql +// never records it and the deploy tool refuses every update. A server-setting // assertion with no migration behind it (assertStrictSqlMode) is not registered: the // pool pins what it checks, so it holds by construction and cannot crash-loop a deploy. Database.STARTUP_ASSERTED_MIGRATIONS = [ diff --git a/src/db/migration_statements.js b/src/db/migration_statements.js index 5ca5cd2d..83eb01a8 100644 --- a/src/db/migration_statements.js +++ b/src/db/migration_statements.js @@ -14,7 +14,7 @@ const { opensBackslashEscape, createNamesNonInnoDb, dropTargetDestroys } = require('./query_helpers.js') const { ensureLedgerAppliedAtDatetime } = require('./migration_ledger.js') -const { stripBlockComments } = require('./migration_live_schema_guard.js') +const { stripBlockComments, alterSpecs, blankLiterals } = require('./migration_live_schema_guard.js') // True when a `#` sits outside every quoted span - a line comment // stripSqlLineComments should already have removed. Quote-aware so a `#` @@ -129,14 +129,9 @@ function isDestructiveAlter(stmt, safeAlterDrop){ // CONVERT TO CHARACTER SET rewrites every text column of the table and can truncate or // re-encode row data. if(/\bCONVERT\s+TO\b/i.test(stmt)) return true; - // MODIFY that adds NOT NULL narrows the column domain - except an - // AUTO_INCREMENT attribute repair: an AUTO_INCREMENT column is - // definitionally NOT NULL, so no domain is narrowed (see the - // committed 2026-06-10-mirror-id-autoincrement-repair.sql pattern). - // Check per top-level clause: a statement-wide AUTO_INCREMENT test - // would let one AUTO_INCREMENT clause exempt a sibling NOT NULL clause - // in the same multi-clause ALTER (e.g. `MODIFY id ... AUTO_INCREMENT, - // MODIFY source VARCHAR(255) NOT NULL`). + // MODIFY that adds NOT NULL narrows the column domain, except an AUTO_INCREMENT repair + // (definitionally NOT NULL; 2026-06-10-mirror-id-autoincrement-repair.sql). Checked per + // top-level clause, so one AUTO_INCREMENT clause cannot exempt a sibling NOT NULL clause. let mDepth = 0, mStart = 0; const mClauses = []; for(let i=0;i= 8 +} + module.exports = { // Durable reorg-halt flag. verifyReorg's fail-closed safe-depth // ceiling is a per-invocation counter: on a reorg deeper than @@ -36,6 +45,8 @@ module.exports = { // index. Returns { halted, id, at, reason, cleared_at, cleared_reason }. // Fail-closed: a halt row whose id or payload cannot be read still counts as // live, because "we could not tell" must never reach a caller as "not halted". + // The same holds for a clear row: one without the audited shape clearReorgHalt + // writes reads as a live halt flagged unaudited_clear, not as a clear. async readReorgHaltState(){ const query = `SELECT id, time, code, data FROM events WHERE code IN ('REORG_HALT', 'REORG_HALT_CLEARED') ORDER BY id DESC LIMIT 1;` const none = { halted: false, id: null, at: null, reason: null, cleared_at: null, cleared_reason: null } @@ -53,9 +64,6 @@ module.exports = { } const at = (payload && payload.at) ? payload.at : (row.time != null ? String(row.time) : null) const reason = (payload && payload.reason) ? payload.reason : null - if (row.code === 'REORG_HALT_CLEARED'){ - return { ...none, cleared_at: at, cleared_reason: reason } - } // events.id is a BIGINT column, and the pool below sets insertIdAsNumber // but not bigIntAsNumber, so the driver hands row.id back as a JS BigInt. // An events id never approaches Number.MAX_SAFE_INTEGER, so normalise to a @@ -63,6 +71,11 @@ module.exports = { // audit payload (clearReorgHalt's cleared_halt_id) gets a safe value // instead of a BigInt that JSON.stringify throws on. const id = (row.id != null) ? Number(row.id) : null + if (row.code === 'REORG_HALT_CLEARED'){ + if (isAuditedClear(payload)) return { ...none, cleared_at: at, cleared_reason: reason } + // Live under the clear row's own id, so clear-reorg-halt can pin it. + return { halted: true, id: id, at: at, reason: UNAUDITED_CLEAR_REASON, cleared_at: null, cleared_reason: null, unaudited_clear: true } + } // Any other shape (the expected REORG_HALT, or a row whose code could not // be read) is a live halt. return { halted: true, id: id, at: at, reason: reason, cleared_at: null, cleared_reason: null } @@ -218,7 +231,10 @@ module.exports = { // silently resumed over-deep rollback, and every consumer of it (the entry // guard, the health surfaces, the bootstrap gate) reads the ROW, not the ack. async markReorgHalted(reason){ - if (await this.isReorgHalted()) return true + // An unaudited clear row reads as halted but is no REORG_HALT row, and the + // bootstrap gate counts only those rows, so that halt still gets one written. + const live = await this.readReorgHaltState() + if (live.halted && !live.unaudited_clear) return true const written = await this.insertEvent('REORG_HALT', { reason: reason, at: new Date().toISOString() }) // Anything other than a clean insert is a failure. DUPLICATED_TRANSACTION // is truthy and would otherwise read as success, so the read-back below diff --git a/src/db/transactions.js b/src/db/transactions.js index 09085eb5..07a0a778 100644 --- a/src/db/transactions.js +++ b/src/db/transactions.js @@ -16,6 +16,14 @@ const { getLogger } = require('../observability') const { format: formatLogLine } = require('node:util'); const { DETERMINISTIC_WRITE_ERRNOS, logger } = require('./constants.js') +// Rethrow a lookup failure inside the block transaction so the block rolls back instead of +// committing a NULL id; outside it (the mempool handle) the null fallback stays. +function rethrowInTransaction(database, err){ + if (database.transactionConnection == null) return + if (err && typeof err === 'object') err.dbLookupFailure = true + throw err +} + async function insertTransactionRow(database, connection, query, tx){ let txHashId = await database.createTransaction(tx.hash) let sourceId = await database.createAddress(tx.source) @@ -27,9 +35,9 @@ async function insertTransactionRow(database, connection, query, tx){ // first-ever action from an address leaves source_pubkey permanently NULL // across the decoder->indexer seam. Inside the block's open transaction, so // it commits or rolls back with the block. Sentinel id 1 (empty address) is - // never a real source. insertPubkey is INSERT IGNORE against a PRIMARY KEY - // and swallows its own errors, so a pubkey hiccup can never turn a good - // transaction into a quarantined poison row. + // never a real source. insertPubkey is INSERT IGNORE against a PRIMARY KEY and + // swallows a deterministic content error, so a pubkey can never turn a good + // transaction into a quarantined poison row; a transient one rolls the block back. if (tx.source_pubkey && sourceId != null && sourceId !== 1){ await database.insertPubkey(sourceId, tx.source_pubkey) } @@ -182,6 +190,8 @@ module.exports = { return true } catch (err) { if (err.errno == 1062){ + // Name the colliding key (PRIMARY tx_index or the tx_hash_id unique key) for the caller's stall. + logger.error(formatLogLine('Duplicate key inserting transaction:', err)); return this.DUPLICATED_TRANSACTION } else { logger.error(formatLogLine('Error inserting transaction:', err)); @@ -212,6 +222,7 @@ module.exports = { id = rows[0].id; } catch (err) { logger.error(formatLogLine('Error looking up hash record id in index_transactions table:', err)); + rethrowInTransaction(this, err) } finally { if (this.transactionConnection == null){ await db.release() @@ -238,6 +249,7 @@ module.exports = { await db.query(query, [hash]); } catch (err) { logger.error(formatLogLine('Error trying to create hash record in index_transactions table:', err)); + rethrowInTransaction(this, err) } finally { if (this.transactionConnection == null){ await db.release() diff --git a/src/metrics/decoder_metrics.js b/src/metrics/decoder_metrics.js index fbccf8a3..b30ae4b6 100644 --- a/src/metrics/decoder_metrics.js +++ b/src/metrics/decoder_metrics.js @@ -45,6 +45,7 @@ const DECODER_GAUGES = [ const DECODER_COUNTERS = [ ['parse_errors_total', 'Transactions the decoder failed to parse since process start'], + ['quarantined_tx_total', 'Transactions this decoder permanently skipped (PARSE_ERROR quarantine) since process start'], ['rpc_errors_total', 'Node RPC errors seen since process start'], ['reorgs_total', 'Reorgs this decoder has rolled back since process start'], ['dispenser_purge_failures_total', 'Post-commit expired-dispenser purges that failed since process start'] @@ -90,6 +91,7 @@ function collectDecoderMetrics(decoder, gauges, counters) { const rpcErrors = (decoder.rpcErrors || 0) + ((decoder.connector && decoder.connector.rpcErrors) || 0); counters.rpc_errors_total.setMonotonic({}, rpcErrors); counters.parse_errors_total.setMonotonic({}, decoder.parseErrors || 0); + counters.quarantined_tx_total.setMonotonic({}, decoder.quarantinedTxs || 0); counters.dispenser_purge_failures_total.setMonotonic({}, decoder.dispenserPurgeFailures || 0); // Reorg churn. The durable REORG rows and the indexer's reorgsProcessed cover diff --git a/src/protocol/batch_sub_command_capture.js b/src/protocol/batch_sub_command_capture.js index 350f8873..b1d9c1f0 100644 --- a/src/protocol/batch_sub_command_capture.js +++ b/src/protocol/batch_sub_command_capture.js @@ -157,32 +157,37 @@ function isDispenserAddressIdCollapseActive(consensusNetwork, blockTime){ // Registering ONE of the two is strictly better than the pre-gate behaviour, where a // batch registers NEITHER. // -// Above the address-id collapse gate, use the same collation key as index_addresses so -// spellings that resolve to one address_id cannot attempt two inserts for the same primary -// key. Raw spellings remain distinct below the gate for historical replay compatibility. -function collapseDispenserRegistrations(candidates, consensusNetwork, blockTime){ - if (isDispenserAddressIdCollapseActive(consensusNetwork, blockTime)) - return collapseRegistrationsByKey(candidates, dispenserAddressKey) - const collapsed = new Map() +// Above the address-id collapse gate, use the resolved index_addresses id so every +// equality of the table's collation shares one key. Raw spellings remain distinct below +// the gate for historical replay compatibility. +async function collapseRegistrationsByAddressId(candidates, addressIdOf){ if (!Array.isArray(candidates)) return [] + const rawAddresses = new Set() for (const candidate of candidates){ - if (!candidate || !candidate.address) continue - const existing = collapsed.get(candidate.address) - if (existing === undefined){ - collapsed.set(candidate.address, { - address: candidate.address, - sourceAddress: candidate.sourceAddress, - oracleAddress: candidate.oracleAddress || null, - expiration: candidate.expiration, - }) - continue - } - if (candidate.expiration > existing.expiration) - existing.expiration = candidate.expiration - if (!existing.oracleAddress && candidate.oracleAddress) - existing.oracleAddress = candidate.oracleAddress + if (candidate && candidate.address) rawAddresses.add(candidate.address) } - return [...collapsed.values()] + if (rawAddresses.size < 2) return collapseRegistrationsByKey(candidates, null) + + const keys = new Map() + for (const candidate of candidates){ + if (!candidate || !candidate.address || keys.has(candidate.address)) continue + keys.set(candidate.address, await dispenserAddressKey(candidate.address, addressIdOf)) + } + return collapseRegistrationsByKey(candidates, address => keys.get(address)) +} + +function legacyDispenserAddressKey(address){ + return address.replace(/[A-Z]/g, letter => letter.toLowerCase()).replace(/ +$/, '') +} + +function collapseDispenserRegistrations(candidates, consensusNetwork, blockTime, addressIdOf){ + if (isDispenserAddressIdCollapseActive(consensusNetwork, blockTime)){ + if (typeof addressIdOf === 'function') + return collapseRegistrationsByAddressId(candidates, addressIdOf) + return collapseRegistrationsByKey(candidates, legacyDispenserAddressKey) + } + // Pinned by the frozen-reference test: a merge-rule change here needs its own gate. + return collapseRegistrationsByKey(candidates, null) } module.exports = { diff --git a/src/protocol/batch_sub_command_capture/dispenser_address_key.js b/src/protocol/batch_sub_command_capture/dispenser_address_key.js index 5604d537..7f42f5b9 100644 --- a/src/protocol/batch_sub_command_capture/dispenser_address_key.js +++ b/src/protocol/batch_sub_command_capture/dispenser_address_key.js @@ -14,8 +14,11 @@ 'use strict' -function dispenserAddressKey(address){ - return address.replace(/[A-Z]/g, letter => letter.toLowerCase()).replace(/ +$/, '') +async function dispenserAddressKey(address, addressIdOf){ + const addressId = await addressIdOf(address) + return addressId === null || addressId === undefined + ? 'address:' + address + : 'address-id:' + String(addressId) } module.exports = { dispenserAddressKey } diff --git a/src/protocol/batch_sub_command_capture/sub_commands.js b/src/protocol/batch_sub_command_capture/sub_commands.js index 56c322f2..64b2ea89 100644 --- a/src/protocol/batch_sub_command_capture/sub_commands.js +++ b/src/protocol/batch_sub_command_capture/sub_commands.js @@ -115,7 +115,7 @@ function subCommandActionName(command){ // for(let command of commands){ // let action = String(command).split('|')[0]; // if(normalize) action = this.normalizeSubAction(action); -// if(!error && dispatchedOnly && !DISPATCHED_ACTIONS.has(action)) +// if(!error && dispatchedOnly && action && !DISPATCHED_ACTIONS.has(action)) // error = 'invalid: ACTION (unknown)'; // else if(!error && await this.protocolChanges.isEnabled(action, ...) == false) // error = 'invalid: ACTION (unknown)'; diff --git a/src/protocol/constants.js b/src/protocol/constants.js index 64e6271f..7f1e3c49 100644 --- a/src/protocol/constants.js +++ b/src/protocol/constants.js @@ -104,6 +104,8 @@ module.exports = { DISPENSER_EXPIRY_REALIGN_ACTIVATION: activation.DISPENSER_EXPIRY_REALIGN_ACTIVATION, DISPENSER_CANCEL_GRACE_ACTIVATION: activation.DISPENSER_CANCEL_GRACE_ACTIVATION, DISPENSER_PURGE_GRACE_ACTIVATION: activation.DISPENSER_PURGE_GRACE_ACTIVATION, + DISPENSER_CANCEL_GRACE_PARENT_FLOOR_ACTIVATION: + activation.DISPENSER_CANCEL_GRACE_PARENT_FLOOR_ACTIVATION, DISPENSER_ADDRESS_ID_COLLAPSE_ACTIVATION: activation.DISPENSER_ADDRESS_ID_COLLAPSE_ACTIVATION, DISPENSER_WIDE_EXPIRATION_ACTIVATION: dispenserWideExpiration.DISPENSER_WIDE_EXPIRATION_GATES, EXACT_INTEGER_EXPIRATION_ACTIVATION: activation.EXACT_INTEGER_EXPIRATION_ACTIVATION, diff --git a/src/protocol/constants/activation.js b/src/protocol/constants/activation.js index d41157a0..adefa88e 100644 --- a/src/protocol/constants/activation.js +++ b/src/protocol/constants/activation.js @@ -259,6 +259,12 @@ const DISPENSER_PURGE_GRACE_ACTIVATION = { regtest: 0, }; +const DISPENSER_CANCEL_GRACE_PARENT_FLOOR_ACTIVATION = { + mainnet: 9999999999, + testnet: 9999999999, + regtest: 0, +}; + const DISPENSER_ADDRESS_ID_COLLAPSE_ACTIVATION = { mainnet: 9999999999, testnet: 9999999999, @@ -331,6 +337,7 @@ module.exports = { DISPENSER_EXPIRY_REALIGN_ACTIVATION, DISPENSER_CANCEL_GRACE_ACTIVATION, DISPENSER_PURGE_GRACE_ACTIVATION, + DISPENSER_CANCEL_GRACE_PARENT_FLOOR_ACTIVATION, DISPENSER_ADDRESS_ID_COLLAPSE_ACTIVATION, EXACT_INTEGER_EXPIRATION_ACTIVATION, BATCH_SUBCOMMAND_OUTPUT_CAPTURE_ACTIVATION, diff --git a/src/protocol/dispenser_cancel_grace.js b/src/protocol/dispenser_cancel_grace.js index 98beaac6..57a10a97 100644 --- a/src/protocol/dispenser_cancel_grace.js +++ b/src/protocol/dispenser_cancel_grace.js @@ -67,6 +67,7 @@ const { DISPENSER_CANCEL_GRACE_ACTIVATION, DISPENSER_PURGE_GRACE_ACTIVATION, + DISPENSER_CANCEL_GRACE_PARENT_FLOOR_ACTIVATION, } = require('./constants.js') // Seconds a soft-expired dispenser stays an eligible payment destination at/above the gate. @@ -99,15 +100,27 @@ function isDispenserCancelGraceActive(consensusNetwork, blockTime){ return t >= activation } -// The capture floor for a block: the oldest expiration still eligible for payment capture. -// -// Returns null below the gate, which getAllOpenDispenserAddresses reads as "keep the -// unwidened set", so a from-genesis re-decode of pre-flag-day history reproduces the output -// set the fleet wrote live, byte for byte. Above it the floor is a pure function of the -// block's own header time, so two honest nodes load the identical capture set. -function cancelGraceFloor(consensusNetwork, blockTime){ +// The capture floor is null below the gate, preserving the pre-flag-day set. Above the +// parent-floor gate, the earlier block or parent protocol time carries capture across +// the first block past the close boundary. +function isDispenserCancelGraceParentFloorActive(consensusNetwork, blockTime){ + const activation = DISPENSER_CANCEL_GRACE_PARENT_FLOOR_ACTIVATION[consensusNetwork] + if (typeof activation !== 'number') return false + const t = Number(blockTime) + if (!Number.isFinite(t)) return false + return t >= activation +} + +function cancelGraceFloor(consensusNetwork, blockTime, parentProtocolTime){ if (!isDispenserCancelGraceActive(consensusNetwork, blockTime)) return null - return Number(blockTime) - DISPENSER_CANCEL_GRACE_SECONDS + const block = Number(blockTime) + let anchor = block + if (isDispenserCancelGraceParentFloorActive(consensusNetwork, block)){ + const parent = Number(parentProtocolTime) + if (parentProtocolTime !== null && parentProtocolTime !== undefined && + Number.isFinite(parent)) anchor = Math.min(block, parent) + } + return anchor - DISPENSER_CANCEL_GRACE_SECONDS } function isDispenserPurgeGraceActive(consensusNetwork, blockTime){ @@ -126,8 +139,10 @@ function purgeGraceFloor(consensusNetwork, blockTime){ module.exports = { DISPENSER_CANCEL_GRACE_ACTIVATION, DISPENSER_PURGE_GRACE_ACTIVATION, + DISPENSER_CANCEL_GRACE_PARENT_FLOOR_ACTIVATION, DISPENSER_CANCEL_GRACE_SECONDS, isDispenserCancelGraceActive, + isDispenserCancelGraceParentFloorActive, cancelGraceFloor, isDispenserPurgeGraceActive, purgeGraceFloor, diff --git a/test/chaos/support/helpers.js b/test/chaos/support/helpers.js index f09ce0c2..b47b6c59 100644 --- a/test/chaos/support/helpers.js +++ b/test/chaos/support/helpers.js @@ -16,6 +16,7 @@ * Provides mock factories and fault injection utilities for testing * the decoder's resilience without requiring real infrastructure. */ +const crypto = require('crypto') const sinon = require('sinon') const EventEmitter = require('events') @@ -107,15 +108,6 @@ function createMockConnector(overrides = {}) { */ function createMinimalBlockHex() { // A minimal valid block: 80-byte header + 1 coinbase tx - // Version 1, prev hash all zeros, merkle root, timestamp, bits, nonce - const header = '01000000' + - '0000000000000000000000000000000000000000000000000000000000000000' + - '3ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a' + - '29ab5f49' + - 'ffff001d' + - '1dac2b7c' - // varint 1 tx - const txCount = '01' // Minimal coinbase tx const coinbaseTx = '01000000' + // version @@ -130,6 +122,18 @@ function createMinimalBlockHex() { '43' + // script length (67 bytes) '4104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac' + '00000000' // locktime + // Version 1, prev hash all zeros, merkle root, timestamp, bits, nonce. A block of + // one tx has that tx's hash as its merkle root; the decoder refuses a mismatch. + const merkleRoot = crypto.createHash('sha256') + .update(crypto.createHash('sha256').update(Buffer.from(coinbaseTx, 'hex')).digest()).digest('hex') + const header = '01000000' + + '0000000000000000000000000000000000000000000000000000000000000000' + + merkleRoot + + '29ab5f49' + + 'ffff001d' + + '1dac2b7c' + // varint 1 tx + const txCount = '01' return header + txCount + coinbaseTx } diff --git a/test/fixtures/decoder_parse_vectors.json b/test/fixtures/decoder_parse_vectors.json new file mode 100644 index 00000000..f3cd8a80 --- /dev/null +++ b/test/fixtures/decoder_parse_vectors.json @@ -0,0 +1,12 @@ +{ + "bitcoinGenesisCoinbase": { + "form": "legacy", + "txid": "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b", + "hex": "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000" + }, + "taprootWitnessSpend": { + "form": "segwit-v1", + "txid": "065e8cd3378b9925feee408ae9e5f58e5bba75e8d7682a7fbcf230a3b435f7bb", + "hex": "0200000000010111111111111111111111111111111111111111111111111111111111111111110300000000fdffffff013930000000000000225120222222222222222222222222222222222222222222222222222222222222222201403333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333340d10c00" + } +} diff --git a/test/fixtures/wire_parse_vectors.json b/test/fixtures/wire_parse_vectors.json new file mode 100644 index 00000000..2a726eed --- /dev/null +++ b/test/fixtures/wire_parse_vectors.json @@ -0,0 +1,31 @@ +{ + "$comment": [ + "Real mainnet wire vectors for the AuxPoW stripped-block integrity and Litecoin MWEB serialization tests.", + "doge4500000: Dogecoin block 4500000 rebuilt byte-exact from verbose getblock (header fields and auxpow object) and each raw tx; the header hash equals hash.", + "ltcPegin: Litecoin tx 4e54ce26..., a segwit tx whose output is an MWEB peg-in (witness v9), in its non-MWEB serialization.", + "ltcHogEx3195200: the HogEx (last tx) of Litecoin block 3195200 as serialized in the block: flag 0x08, a null MWEB tag 0x00, then nLockTime." + ], + "doge4500000": { + "hash": "66d27fdeb0f694d06d1276b2d3d72c4dbefe294651ddc1ade220951823243396", + "header": "04016200214b69e27b09a709e69279f7730410dafe6ad24b30ff734ce1e7fd3c1c73ce47af9644e48992d389a485c0c51a630037afb3c7dec10fddafb3b5e8664a72c4d3223d8d63fc0c021a00000000", + "auxpow": "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff62036154241b2f5669614254432f4d696e656420627920727562656e686f6f642f2cfabe6d6da0cfe03f4076ae75a950e7b5a092cce6225a1508ab8c3de2d495e81606c9c0a6100000000000000010c671c6026f9adb604b88006b0000000000190300ffffffff028149c74a000000001976a914e16c28146ed4869c190b3f0bdc18d80d45f9213488ac0000000000000000266a24aa21a9edff42cfee49791bf95a1ec3388636cad51050db6ae37942ed6fa27f83450c0baf000000006a8b87ebf3d5bc54173bc3ac0299e53456cb0384d243037e3d488088e6178cf50929e06eb3fbffa2741cf18b61d642e6beb4ceec0d5782e66eb6e3e23d5844a86e8e59ffa53953a3fc1a5bbafc12daa70633e5faab2abdc7b5faeca0bbe7b280864714d482049e4192baefe7f686275ba9282ac7e2fa58ee267c5dae43004c1cb76413bf16ce402680a2be1f9d7be3767f702287daa83a6f84ef1b3975ba52466efb5c1ac43881abfcc678aa2235494747fa07afc7a1be2d5fe845882634a85d304c439aef2f4c7af6e545ee32542ec67c659f62205c6c250d6d55259a970a1b08b8acb4bbd723254ffbafaba102d05f95c5b69fe291723213e693ee14da635fada04e8062b21250073bd94d4e29dfbb2955a25d97c3a82392ad402d42cbae448fcbce55511fdddc91ef6baa66a85d3aa4194ce58dcbf68e0021e36fb20bc8f22d00000000040000000000000000000000000000000000000000000000000000000000000000e2f61c3f71d1defd3fa999dfa36953755c690689799962b48bebd836974e8cf9a8106811fef1ffc754d0d05e0e96a688ec32d5ac549bc6bd6c20d7b3a99723c894857d3e08918f70395d9206410fbfa942f1a889aa5ab8188ec33c2f6e207dc70800000000000020959f11a38573c927a248683351feb669e2ac995dddd8078d3c2705e037a774b137a5c80a940f1cbde7ddf0c539874f2c4a72564f257b356fdc4848f8a8d58446233d8d6319d0001abc3d8173", + "txs": [ + "01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff060320aa440101ffffffff01c0a0bedbe8000000232103c72fa19c28f1326f0cc5f5af64abe1902626e38007993a1cfecae67e1414fc2cac00000000", + "0100000001026d47733fbfd7c06ec43bb3317d3a43eca442fe33f87bcfd9849edd02c8e621060000006b4830450221008832a1f3299cb5c2aea69692b444be727c2548e4bb87d91d28563d61e06425c50220329fd05003e6d7fd884619cc39b2afee8e9a040380a7a2c310cb64b45f59916c012102ba13b5c51ec5807012434d8b4ea3bc5633dcee8f1e2b92b98c6285544deb8f48ffffffff033aca2900000000001976a914514a54610af79c8bfd7834413b46cdf60d6b489988acf0da2000000000001976a9149980d2b4d6153ed4196803d74cd2dc8570726bcb88ac002d4e4d5d0600001976a914b89dbe8f22ebc12da7add2743e2e4a4cc045078b88ac00000000", + "01000000010277eef6d6e3cb1a841232714e7b1c64bfd99e79f4375c441283bbd229ccc91b010000006b483045022100df048fe3b4f72f5886d0ccdfe3ccc69d7cbf68c7b61f945597cc00243fef81da0220530967bfd110072b73cf305e77bdea1ffda431b52b900c1398cf343c4434723d0121033cb935220607ccdbc31dfeea8f5620b7d88fd4f6350458eef40ff8c9b10d6560feffffff01f334bac4000000001976a91482e4c1bcc62243f4a7e6ca3e81f85281c627b22b88ac00000000" + ], + "txids": [ + "e8c228912d12f22aa5ba5c46cf5c674369fb3703ddceaaa6575bf53379b7ce23", + "c9bd1c3bb4255a77d3e56104468c10129aa61315062cb98798852a5513aa8bc0", + "7ae1930e89d5fefc88e5cc1d5fa7baad39cc444edda42d9a023414a68469f912" + ] + }, + "ltcPegin": { + "txid": "4e54ce26e51ff5115e7ac0d499f9dc81dae7edefa493d565ec3d6006eef3e3f0", + "hex": "0200000000010111df042d08f2a72d5509cfb03da5cfb0f69468cc3f2b509f6e30efe3fb18c48d0000000000feffffff01e2527307000000002259207b367c6028c3c990e52f1fe8fb2cfa6dd3d001596fac08eba68b51008ce51b0102473044022063d3002a1307359181e04e34e97eb3974add62ee3368a07ca6332acc64358760022029eb75a2897cd9036935ab3a5a51a64635c78594d4456a9c2060bba0de28c00f012102a2fedc3dc40793175bc2c4264a770de763579c47addf0108006c5dc686b8b2cb7f932200" + }, + "ltcHogEx3195200": { + "txid": "4c2393b5f649cf651ef0a55aef7688ce87c3cf137faf5af9a6ba311bbde53419", + "hex": "020000000008012beb4a8a05f0ecd5c58e183dfac4bd750de5bbd40f41e4a99ec16f25427ab5060000000000ffffffff01ba3c2a7f9d28000022582089e183a8217880feb0fa0dee7fce949b7d1dab12be24029f701dca1c9873c25b0000000000" + } +} diff --git a/test/fuzz/harness/parse_transaction.fuzz/support.cjs b/test/fuzz/harness/parse_transaction.fuzz/support.cjs index bac4ac7b..aab3301d 100644 --- a/test/fuzz/harness/parse_transaction.fuzz/support.cjs +++ b/test/fuzz/harness/parse_transaction.fuzz/support.cjs @@ -44,8 +44,12 @@ function createDecoder() { } // Errors thrown by bitcoinjs-lib during Transaction.fromHex/fromBuffer are expected -// when we feed it corrupted hex. These are not decoder bugs. +// when we feed it corrupted hex. These are not decoder bugs. transactionFromHex +// refuses malformed, truncated and trailing-byte hex with a TransactionParseError, +// keyed here on its code rather than its message, so a refusal stops counting as +// expected the moment the decoder stops typing it. function isBitcoinjsParseError(err) { + if (err != null && err.code === 'ERR_TRANSACTION_PARSE') return true const msg = err.message || '' return msg.includes('Cannot read slice out of bounds') || msg.includes('Transaction has unexpected data') || diff --git a/test/integration/raw_transaction_guard.test.js b/test/integration/raw_transaction_guard.test.js new file mode 100644 index 00000000..fdae6438 --- /dev/null +++ b/test/integration/raw_transaction_guard.test.js @@ -0,0 +1,122 @@ +/********************************************************************* + * + * Copyright © 2025–2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC – https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ********************************************************************** + * Integration test: getRawTransaction passes only whole hex through + * wholeHexResult and retries past malformed rpcPost answers. + * + * rpcPost is stubbed, so no node is needed. The same scenarios also run + * against a copy of the module whose wholeHexResult guard is removed and must + * FAIL there, so this file cannot pass when the guard behavior is absent. + */ + +'use strict' + +const assert = require('assert') +const fs = require('fs') +const Module = require('module') +const path = require('path') + +const MODULE_PATH = path.resolve(__dirname, '../../src/chain/blockchain_connector/transaction_queries.js') +const real = require(MODULE_PATH) + +/* eslint-disable no-underscore-dangle -- Module internals are needed to compile a mutated copy */ +// Load a mutated copy of the module (relative requires resolve as for the original). +function loadWithoutGuard() { + const source = fs.readFileSync(MODULE_PATH, 'utf8') + const mutated = source.replace( + /function wholeHexResult\(result, txid\) \{[\s\S]*?\n\}\n/, + 'function wholeHexResult(result) { return result }\n' + ) + assert.notStrictEqual(mutated, source, 'mutation must change wholeHexResult') + const mod = new Module(MODULE_PATH, module) + mod.filename = MODULE_PATH + mod.paths = Module._nodeModulePaths(path.dirname(MODULE_PATH)) + mod._compile(mutated, MODULE_PATH) + return mod.exports +} +/* eslint-enable no-underscore-dangle */ + +// Script rpcPost answers in order (the last repeats) and record the run. +async function drive(queries, answers) { + let calls = 0 + const sleeps = [] + const connector = { + rpcErrors: 0, + sleep: async (ms) => { sleeps.push(ms) }, + rpcPost: async () => ({ status: 200, data: { result: answers[Math.min(calls++, answers.length - 1)] } }), + } + try { + const value = await queries.getRawTransaction.call(connector, 'guard-txid') + return { value, calls, sleeps, rpcErrors: connector.rpcErrors } + } catch (error) { + return { error, calls, sleeps, rpcErrors: connector.rpcErrors } + } +} + +const MALFORMED = [ + ['an object', { hex: '00' }], + ['a number', 42], + ['an odd-length hex string', 'abc'], + ['a non-hex string', 'busy'], +] + +// Each scenario asserts the guarded behavior; returns normally only when it holds. +const scenarios = { + async 'passes whole hex through unchanged on the first rpcPost call'(queries) { + const got = await drive(queries, ['0100abCD']) + assert.strictEqual(got.value, '0100abCD') + assert.strictEqual(got.calls, 1) + assert.deepStrictEqual(got.sleeps, []) + }, + async 'retries past every malformed rpcPost answer to the whole hex that follows'(queries) { + for (const [name, bad] of MALFORMED) { + const got = await drive(queries, [bad, bad, '00ff']) + assert.strictEqual(got.value, '00ff', `${name} must not reach the caller`) + assert.strictEqual(got.calls, 3, `${name} must be retried`) + assert.deepStrictEqual(got.sleeps, [500, 500]) + assert.strictEqual(got.rpcErrors, 0) + } + }, + async 'rejects naming the txid when rpcPost never answers whole hex'(queries) { + const got = await drive(queries, [{ hex: '00' }]) + assert.ok(got.error, 'must reject, not resolve a malformed answer') + assert.strictEqual(got.calls, 10) + assert.strictEqual(got.rpcErrors, 1) + assert.ok(/guard-txid/.test(got.error.message), got.error.message) + assert.ok(/malformed/.test(got.error.message), got.error.message) + }, +} + +describe('getRawTransaction wholeHexResult guard (stubbed rpcPost)', function () { + for (const [name, fn] of Object.entries(scenarios)) { + it(name, async function () { + await fn(real) + }) + } + + it('keeps an empty rpcPost result on the tolerant null path', async function () { + const got = await drive(real, [null]) + assert.strictEqual(got.value, null) + assert.strictEqual(got.calls, 1) + }) + + describe('non-vacuity: the scenarios fail when wholeHexResult is removed', function () { + const unguarded = loadWithoutGuard() + for (const [name, fn] of Object.entries(scenarios)) { + if (name.startsWith('passes whole hex')) continue // true with or without the guard + it(`detects the missing guard: ${name}`, async function () { + await assert.rejects(() => fn(unguarded), assert.AssertionError) + }) + } + }) +}) diff --git a/test/unit/auxpow_reassembly.test.js b/test/unit/auxpow_reassembly.test.js index 93dc34bc..c6fab3d6 100644 --- a/test/unit/auxpow_reassembly.test.js +++ b/test/unit/auxpow_reassembly.test.js @@ -16,6 +16,7 @@ // AuxPoW bytes at all. const assert = require('assert') +const crypto = require('crypto') const BlockchainConnector = require('../../src/chain/blockchain_connector') const { encodeVarintHex } = require('../../src/chain/blockchain_connector') const XChainDecoder = require('../../src/XChainDecoder') @@ -31,8 +32,14 @@ const TX_HEX = '01' + '0100000000000000' + '00' + '00000000' const TXID = 'a'.repeat(64) -// 80-byte header (160 hex chars); content is irrelevant to reassembly. -const HEADER_HEX = '04016200' + '11'.repeat(76) +// 80-byte header (160 hex chars). Reassembly copies it as is; its merkle root is +// that of a block holding TX_HEX twice, so the two-tx reassembled block decodes +// (the block decoder refuses a root that does not match the transactions). +const TX_HASH = crypto.createHash('sha256').update(crypto.createHash('sha256') + .update(Buffer.from(TX_HEX, 'hex')).digest()).digest() +const TWO_TX_ROOT = crypto.createHash('sha256').update(crypto.createHash('sha256') + .update(Buffer.concat([TX_HASH, TX_HASH])).digest()).digest('hex') +const HEADER_HEX = '04016200' + '11'.repeat(32) + TWO_TX_ROOT + '11'.repeat(12) function makeConnector(overrides) { const connector = new BlockchainConnector('127.0.0.1', 0, 'user', 'pass') diff --git a/test/unit/auxpow_strip_parity.test.js b/test/unit/auxpow_strip_parity.test.js index b75e57c5..44298f59 100644 --- a/test/unit/auxpow_strip_parity.test.js +++ b/test/unit/auxpow_strip_parity.test.js @@ -31,6 +31,7 @@ // of green-by-skip. const assert = require('assert') +const crypto = require('crypto') const fs = require('fs') const path = require('path') @@ -54,6 +55,7 @@ const SHARED_FUNCTIONS = [ 'readVarint', 'encodeVarintHex', 'skipAuxPow', + 'assertWholeBlock', 'stripAuxPowFromBlockHex', ] @@ -72,12 +74,6 @@ function extractFunction(source, name) { return lines.slice(start, end + 1).join('\n') } -// 80-byte header, version 0x00620104 little-endian: AuxPoW bit (0x100) SET. -const AUXPOW_HEADER = '04016200' + '11'.repeat(76) -// Same header with the AuxPoW bit clear (version 2). -const PLAIN_HEADER = '02000000' + '11'.repeat(76) -const TX_COUNT_AND_TXS = '01' + '0a'.repeat(60) - // Minimal legacy coinbase tx: 1 input, 1 empty-script output. const COINBASE = '01000000' + @@ -85,6 +81,15 @@ const COINBASE = '00'.repeat(32) + 'ffffffff' + '00' + 'ffffffff' + '01' + '0100000000000000' + '00' + '00000000' +// The coinbase is the block's only tx, so the header's merkle root is its txid +// in internal byte order; the strip refuses a block whose root does not match. +const COINBASE_MERKLE_ROOT = crypto.createHash('sha256') + .update(crypto.createHash('sha256').update(Buffer.from(COINBASE, 'hex')).digest()).digest('hex') +// 80-byte header, version 0x00620104 little-endian: AuxPoW bit (0x100) SET. +const AUXPOW_HEADER = '04016200' + '11'.repeat(32) + COINBASE_MERKLE_ROOT + '11'.repeat(12) +// Same header with the AuxPoW bit clear (version 2). +const PLAIN_HEADER = '02000000' + '11'.repeat(32) + COINBASE_MERKLE_ROOT + '11'.repeat(12) +const TX_COUNT_AND_TXS = '01' + COINBASE // AuxPoW tail: parent hash (32 B) + empty coinbase branch (count 0 + index 4 B) // + empty chain branch + parent header (80 B). const AUXPOW_TAIL = diff --git a/test/unit/batch_dispenser_registration.test/03_two_creates_on_the_same_operating_address_the_primary_key_collision.test.js b/test/unit/batch_dispenser_registration.test/03_two_creates_on_the_same_operating_address_the_primary_key_collision.test.js index dd6afd5a..0f891819 100644 --- a/test/unit/batch_dispenser_registration.test/03_two_creates_on_the_same_operating_address_the_primary_key_collision.test.js +++ b/test/unit/batch_dispenser_registration.test/03_two_creates_on_the_same_operating_address_the_primary_key_collision.test.js @@ -14,7 +14,10 @@ const { EXP_LATE, ORACLE_A, ORACLE_B, + SOURCE, + DispenserModel, assert, + buildDecoder, create, runOne, } = require('./support/helpers.js') @@ -44,6 +47,29 @@ describe('BATCH dispenser registration', function () { ].join(';'), ABOVE_GATE) assert.strictEqual(decoder.model.rows.length, 1) assert.strictEqual(decoder.model.rows[0].expiration, EXP_LATE) + + const firstAddress = 'Málaga' + const secondAddress = 'malaga' + const action = 'BATCH|0|' + [ + create({ getAddress: firstAddress, expiration: EXP_EARLY }), + create({ getAddress: secondAddress, expiration: EXP_LATE }), + ].join(';') + const model = new DispenserModel() + const addressIdDecoder = buildDecoder([ + { id: 'tx01', action, source: SOURCE, outputs: [] }, + ], model, ABOVE_GATE) + const resolved = [] + addressIdDecoder.db.createAddress = async address => { + resolved.push(address) + return 73 + } + + await addressIdDecoder.start() + + assert.deepStrictEqual(resolved, [firstAddress, secondAddress]) + assert.strictEqual(model.insertCalls, 1) + assert.strictEqual(model.rows[0].address, firstAddress) + assert.strictEqual(model.rows[0].expiration, EXP_LATE) }) }) }) diff --git a/test/unit/batch_dispenser_registration.test/14_below_gate_collapse_frozen_reference.test.js b/test/unit/batch_dispenser_registration.test/14_below_gate_collapse_frozen_reference.test.js new file mode 100644 index 00000000..8be792a5 --- /dev/null +++ b/test/unit/batch_dispenser_registration.test/14_below_gate_collapse_frozen_reference.test.js @@ -0,0 +1,108 @@ +// Copyright © 2025-2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC - https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +const assert = require('assert') +const { collapseDispenserRegistrations } = require('../../../src/protocol/batch_sub_command_capture.js') + +// Frozen reference for the raw-address collapse every pre-gate block re-decodes with. +// Never edit it: a different merge rule belongs behind a new activation gate, and this +// copy makes a change to the shared helper fail here before it can move replay. +function legacyRawAddressCollapse(candidates){ + const collapsed = new Map() + if (!Array.isArray(candidates)) return [] + for (const candidate of candidates){ + if (!candidate || !candidate.address) continue + const existing = collapsed.get(candidate.address) + if (existing === undefined){ + collapsed.set(candidate.address, { + address: candidate.address, + sourceAddress: candidate.sourceAddress, + oracleAddress: candidate.oracleAddress || null, + expiration: candidate.expiration, + }) + continue + } + if (candidate.expiration > existing.expiration) + existing.expiration = candidate.expiration + if (!existing.oracleAddress && candidate.oracleAddress) + existing.oracleAddress = candidate.oracleAddress + } + return [...collapsed.values()] +} + +// Deterministic linear congruential generator, so a failure names a reproducible case. +function seededRandom(seed){ + let state = seed >>> 0 + return () => { + state = (Math.imul(state, 1664525) + 1013904223) >>> 0 + return state / 4294967296 + } +} + +function pick(random, pool){ + return pool[Math.floor(random() * pool.length)] +} + +function assertMatchesFrozenReference(input){ + const snapshot = JSON.stringify(input) + const expected = legacyRawAddressCollapse(input) + for (const actual of [collapseDispenserRegistrations(input), collapseDispenserRegistrations(input, 'mainnet', 1)]){ + assert.deepStrictEqual(actual, expected) + assert.strictEqual(JSON.stringify(input), snapshot) + if (Array.isArray(input)) + for (const row of actual) assert.ok(!input.includes(row)) + } +} + +describe('collapseDispenserRegistrations below the address-id gate', function () { + + it('matches the frozen raw-address reference on non-list and sparse inputs', function () { + for (const input of [undefined, null, 'mabc', { address: 'mabc' }, [], [null, undefined], + [{ address: '' }, { sourceAddress: 's' }, { address: 'mq', sourceAddress: 's', oracleAddress: null, expiration: 1 }]]) + assertMatchesFrozenReference(input) + }) + + it('matches the frozen raw-address reference on order, ties, oracles and spellings', function () { + assertMatchesFrozenReference([ + { address: 'mb', sourceAddress: 's1', oracleAddress: '', expiration: 5 }, + { address: 'ma', sourceAddress: 's2', oracleAddress: 0, expiration: 7 }, + { address: 'mb', sourceAddress: 's3', oracleAddress: null, expiration: 5 }, + { address: 'ma', sourceAddress: 's4', oracleAddress: undefined, expiration: '9' }, + { address: 'mb', sourceAddress: 's5', oracleAddress: 'oracle-b', expiration: undefined }, + { address: 'ma', sourceAddress: 's6', oracleAddress: 'oracle-a', expiration: null }, + { address: 'mA', sourceAddress: 's7', oracleAddress: 'oracle-c', expiration: 3 }, + { address: 'ma ', sourceAddress: 's8', oracleAddress: null, expiration: 11 }, + ]) + assert.strictEqual(collapseDispenserRegistrations([ + { address: 'mA', expiration: 1 }, { address: 'ma', expiration: 2 }, { address: 'ma ', expiration: 3 }, + ], 'mainnet', 1).length, 3) + }) + + it('matches the frozen raw-address reference across seeded random batches', function () { + const random = seededRandom(20261012) + const addresses = ['ma', 'mA', 'mb', 'ma ', '', null] + const oracles = ['', null, undefined, 0, 'oracle-1', 'oracle-2'] + const expirations = [undefined, null, 0, 1, 2, 2, 10, '3', '10'] + for (let round = 0; round < 400; round++){ + const input = [] + const size = Math.floor(random() * 7) + for (let i = 0; i < size; i++){ + if (random() < 0.08) { input.push(null); continue } + input.push({ + address: pick(random, addresses), + sourceAddress: 'source-' + i, + oracleAddress: pick(random, oracles), + expiration: pick(random, expirations), + }) + } + assertMatchesFrozenReference(input) + } + }) +}) diff --git a/test/unit/blockchain_connector.test/01_get_block_without_aux_pow.test.js b/test/unit/blockchain_connector.test/01_get_block_without_aux_pow.test.js index 7e9c4b42..89839d22 100644 --- a/test/unit/blockchain_connector.test/01_get_block_without_aux_pow.test.js +++ b/test/unit/blockchain_connector.test/01_get_block_without_aux_pow.test.js @@ -11,8 +11,15 @@ const assert = require('assert') const sinon = require('sinon') const axios = require('axios') +const crypto = require('crypto') const BlockchainConnector = require('../../../src/chain/blockchain_connector') +// A coinbase tx and the merkle root of a block holding only it (its txid, in +// internal byte order): the strip refuses a block whose root does not match. +const COINBASE_HEX = '01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0704ffff001d0104ffffffff0100f2052a010000001976a914aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa88ac00000000' +const COINBASE_ROOT = crypto.createHash('sha256') + .update(crypto.createHash('sha256').update(Buffer.from(COINBASE_HEX, 'hex')).digest()).digest('hex') + let connector let axiosStub @@ -34,10 +41,12 @@ describe('BlockchainConnector', () => { it('[REGRESSION P2] R-NET-003: should strip AuxPoW data from block hex', async () => { // Header = 200 hex chars (100 bytes, includes 20 bytes of AuxPoW) // Standard bitcoin header = 160 hex chars (80 bytes) - // So dataToRemove = 200 - 160 = 40 hex chars - const auxPowHeader = 'a'.repeat(200) - const blockBody = 'b'.repeat(100) - const fullBlockHex = auxPowHeader.substring(0, 160) + 'x'.repeat(40) + blockBody + // So dataToRemove = 200 - 160 = 40 hex chars. The body is a real one-tx + // block body under a header carrying its merkle root. + const baseHeader = '00016200' + 'aa'.repeat(32) + COINBASE_ROOT + 'aa'.repeat(12) + const auxPowHeader = baseHeader + 'cc'.repeat(20) + const blockBody = '01' + COINBASE_HEX + const fullBlockHex = baseHeader + 'cc'.repeat(20) + blockBody // getBlockHeader returns the full header including AuxPoW axiosStub.onCall(0).resolves({ data: { result: auxPowHeader } }) // getBlockHeader @@ -101,7 +110,7 @@ describe('BlockchainConnector', () => { const BASE_HEADER_HEX = '00016200' + // version 0x00620100 (LE), AuxPoW flag (0x100) set 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + // prevHash 32 bytes - 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' + // merkleRoot 32 bytes + COINBASE_ROOT + // merkleRoot 32 bytes: the coinbase txid '00f15365' + // timestamp 1700000000 (LE) 'ffff001d' + // bits '39300000' // nonce @@ -153,7 +162,7 @@ describe('BlockchainConnector', () => { const BASE_HEADER_HEX = '00016200' + // version 0x00620100 (LE), AuxPoW flag (0x100) set 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + // prevHash 32 bytes - 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb' + // merkleRoot 32 bytes + COINBASE_ROOT + // merkleRoot 32 bytes: the coinbase txid '00f15365' + // timestamp 1700000000 (LE) 'ffff001d' + // bits '39300000' // nonce diff --git a/test/unit/db.test/08_database_transaction_lock_release_on_failure.test.js b/test/unit/db.test/08_database_transaction_lock_release_on_failure.test.js new file mode 100644 index 00000000..0a33383c --- /dev/null +++ b/test/unit/db.test/08_database_transaction_lock_release_on_failure.test.js @@ -0,0 +1,112 @@ +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +'use strict' + +const assert = require('assert') +const sinon = require('sinon') +const Database = require('../../../src/db.js') + +function makeDb() { + return new Database('127.0.0.1', 3306, 'test_db', 'user', 'pass') +} + +function fakeConn(overrides = {}) { + return Object.assign({ + release: sinon.stub().resolves(), + beginTransaction: sinon.stub().resolves(), + commit: sinon.stub().resolves(), + rollback: sinon.stub().resolves(), + }, overrides) +} + +// Resolve true when the lock can be acquired again before a short deadline, false when it is wedged. +async function lockIsFree(db) { + const acquired = db.acquireTransactionLock().then(() => true) + const timedOut = new Promise(resolve => setTimeout(() => resolve(false), 50)) + return Promise.race([acquired, timedOut]) +} + +// A begin/end failure must hand the transaction lock back, or the next caller waits forever. +describe('Database transaction lock release on failure', () => { + afterEach(() => sinon.restore()) + + it('releases the lock when getConnection gives up during beginTransaction', async () => { + const db = makeDb() + sinon.stub(db, 'getConnection').rejects(new Error('Failed to get database connection after 30 attempts: ECONNREFUSED')) + await assert.rejects(db.beginTransaction(), /Failed to get database connection/) + assert.strictEqual(db.transactionConnection, null) + assert.strictEqual(await lockIsFree(db), true) + }) + + it('lets a retried beginTransaction proceed after a failed one', async () => { + const db = makeDb() + const conn = fakeConn() + const getConnection = sinon.stub(db, 'getConnection') + getConnection.onFirstCall().rejects(new Error('db down')) + getConnection.onSecondCall().resolves(conn) + await assert.rejects(db.beginTransaction(), /db down/) + await db.beginTransaction() + assert.strictEqual(db.transactionConnection, conn) + assert.ok(conn.beginTransaction.calledOnce) + }) + + it('releases the lock and the connection when conn.beginTransaction throws', async () => { + const db = makeDb() + const conn = fakeConn({ beginTransaction: sinon.stub().rejects(new Error('begin failed')) }) + sinon.stub(db, 'getConnection').resolves(conn) + await assert.rejects(db.beginTransaction(), /begin failed/) + assert.ok(conn.release.calledOnce) + assert.strictEqual(db.transactionConnection, null) + assert.strictEqual(await lockIsFree(db), true) + }) + + it('keeps the lock while discarding a stale connection, so a queued caller stays queued', async () => { + const db = makeDb() + const stale = fakeConn() + const fresh = fakeConn() + db.transactionConnection = stale + sinon.stub(db, 'getConnection').callsFake(async () => db.transactionConnection || fresh) + await db.beginTransaction() + assert.ok(stale.rollback.calledOnce) + assert.ok(stale.release.calledOnce) + assert.strictEqual(db.transactionConnection, fresh) + // The lock is still ours: a second acquirer must wait rather than be woken. + assert.strictEqual(await lockIsFree(db), false) + }) +}) + +describe('Database transaction lock release on failure', () => { + afterEach(() => sinon.restore()) + + it('releases the lock and the connection when rollback throws in endTransaction', async () => { + const db = makeDb() + await db.acquireTransactionLock() + const conn = fakeConn({ rollback: sinon.stub().rejects(new Error('connection lost')) }) + db.transactionConnection = conn + await assert.rejects(db.endTransaction(), /connection lost/) + assert.ok(conn.release.calledOnce) + assert.strictEqual(db.transactionConnection, null) + assert.strictEqual(await lockIsFree(db), true) + }) + + it('returns false and frees the lock when commit and the rollback after it both fail', async () => { + const db = makeDb() + await db.acquireTransactionLock() + const conn = fakeConn({ + commit: sinon.stub().rejects(Object.assign(new Error('commit failed'), { code: 'ER_X' })), + rollback: sinon.stub().rejects(new Error('connection lost')), + }) + db.transactionConnection = conn + assert.strictEqual(await db.commitTransaction(), false) + assert.strictEqual(db.transactionConnection, null) + assert.strictEqual(await lockIsFree(db), true) + }) +}) diff --git a/test/unit/db_lookup_failure_rollback.test.js b/test/unit/db_lookup_failure_rollback.test.js new file mode 100644 index 00000000..d1b23e1e --- /dev/null +++ b/test/unit/db_lookup_failure_rollback.test.js @@ -0,0 +1,114 @@ +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +'use strict' + +const assert = require('assert') +const sinon = require('sinon') +const Database = require('../../src/db.js') +const XChainDecoder = require('../../src/XChainDecoder') +const { TX_PARSE_MAX_RETRIES } = require('../../src/XChainDecoder/constants.js') +const { ABOVE_GATE, DELEGATE_A, DELEGATE_B, DispenserModel, SOURCE, buildDecoder, create } = require('./batch_dispenser_registration.test/support/helpers.js') + +function lockWait() { + return Object.assign(new Error('Lock wait timeout exceeded'), { errno: 1205, code: 'ER_LOCK_WAIT_TIMEOUT' }) +} + +// A Database inside a block transaction whose connection fails every statement `failWhen` matches. +function dbInTransaction(failWhen) { + const db = new Database('127.0.0.1', 3306, 'test_db', 'u', 'p') + const statements = [] + db.transactionConnection = { + query: sinon.stub().callsFake(async (sql, params) => { + statements.push({ sql: String(sql).trim(), params }) + if (failWhen(String(sql))) throw lockWait() + if (/^SELECT id FROM/.test(String(sql).trim())) return [{ id: 7 }] + return [] + }), + release: sinon.stub().resolves(), + rollback: sinon.stub().resolves(), + commit: sinon.stub().resolves(), + } + db._transactionLock = true + return { db, statements } +} + +// A failed id lookup inside the block transaction rolls the block back instead of committing a NULL id. +describe('DB lookup failure inside the block transaction', () => { + afterEach(() => sinon.restore()) + + it('insertTransaction rolls back and writes no row when the source address lookup fails', async () => { + const { db, statements } = dbInTransaction(sql => /FROM index_addresses/.test(sql)) + const result = await db.insertTransaction({ index: 1, hash: 'ab'.repeat(32), block_index: 5, source: 'srcaddr', destination: null, amount: 0, fee: 0, data: 'SEND|x' }) + assert.strictEqual(result, false, 'a transient lookup failure retries the block') + assert.ok(!statements.some(s => /^INSERT INTO transactions/.test(s.sql)), 'no transactions row is written with a NULL source_id') + assert.strictEqual(db.transactionConnection, null, 'the block transaction was rolled back') + }) + + it('insertBlock rolls back and writes no row when the block hash lookup fails', async () => { + const { db, statements } = dbInTransaction(sql => /FROM index_transactions/.test(sql)) + const result = await db.insertBlock({ block_index: 5, block_hash: 'cd'.repeat(32), previous_block_hash: 'ef'.repeat(32), block_time: 1 }) + assert.strictEqual(result, false) + assert.ok(!statements.some(s => /INSERT INTO blocks/.test(s.sql)), 'no blocks row is written with a NULL block_hash_id') + assert.strictEqual(db.transactionConnection, null) + }) + + it('insertPubkey keeps swallowing a deterministic content error inside the transaction', async () => { + const db = new Database('127.0.0.1', 3306, 'test_db', 'u', 'p') + db.transactionConnection = { query: sinon.stub().rejects(Object.assign(new Error('Data too long'), { errno: 1406 })), release: sinon.stub().resolves() } + assert.strictEqual(await db.insertPubkey(7, 'aa'), false) + }) +}) + +describe('DB lookup failure inside the block transaction', function () { + this.timeout(0) + + it('retries the block past the parse retry budget instead of quarantining the transaction', async function () { + const decoder = new XChainDecoder('bitcoin-regtest', 'h', '0', 'db', 'u', 'p', 'h', '0', 'u', 'p', false, null) + decoder.startBlockIndex = 0 + decoder.sleep = async () => {} + const events = [] + let parses = 0 + decoder.connector = { getBlockchainInfo: async () => ({ verificationprogress: 1, blocks: 0 }), getBlockHash: async () => 'blockA', getBlock: async () => '' } + decoder.db = { + createDatabase: async () => true, verifyDatabase: async () => true, verifyTables: async () => true, + runMigrations: async () => ({ applied: [], pending: [] }), getLastBlockIndex: async () => -1, getLastTxIndex: async () => 0, + beginTransaction: async () => {}, endTransaction: async () => {}, + commitTransaction: async () => { decoder.stopFlag = true; return true }, + deleteOpenDispensers: async () => true, purgeExpiredDispensers: async () => true, + getAllOpenDispenserAddresses: async () => new Set(), insertBlock: async () => true, + insertEvent: async (code, data) => { events.push({ code, data }); return true }, + } + decoder.xchainBlockDecoder = { blockFromHex: () => ({ prevHash: Buffer.alloc(32), timestamp: 1700000000, transactions: [{ getId: () => 'cafe01', outs: [] }] }) } + decoder.parseTransaction = async () => { + parses++ + if (parses <= TX_PARSE_MAX_RETRIES + 2) throw Object.assign(lockWait(), { dbLookupFailure: true }) + return null + } + await decoder.start() + assert.strictEqual(parses, TX_PARSE_MAX_RETRIES + 3, 'the block kept retrying until the lookup recovered') + assert.deepStrictEqual(events, [], 'a DB lookup failure is never quarantined with PARSE_ERROR') + assert.strictEqual(decoder.quarantinedTxs || 0, 0) + }) + + it('rolls the block back when a dispenser address lookup fails during the collapse', async function () { + const model = new DispenserModel() + // Two operating addresses, so the address-id collapse has to look both up. + const action = 'BATCH|0|' + create({ getAddress: DELEGATE_A }) + ';' + create({ getAddress: DELEGATE_B }) + const decoder = buildDecoder([{ id: 'tx01', action, source: SOURCE, outputs: [] }], model, ABOVE_GATE) + let lookups = 0 + let rollbacks = 0 + decoder.db.createAddress = async () => { lookups++; if (lookups === 1) throw Object.assign(lockWait(), { dbLookupFailure: true }); return 40 + lookups } + decoder.db.endTransaction = async () => { rollbacks++ } + await decoder.start() + assert.strictEqual(rollbacks, 1, 'the failed collapse rolled the block back once') + assert.deepStrictEqual(model.rows.map(r => r.address).sort(), [DELEGATE_A, DELEGATE_B].sort(), 'each dispenser is registered once, on the retry') + }) +}) diff --git a/test/unit/db_queries.test/20_error_path_transaction_connection_branches.test.js b/test/unit/db_queries.test/20_error_path_transaction_connection_branches.test.js index 0373bdc1..bb2ec5b8 100644 --- a/test/unit/db_queries.test/20_error_path_transaction_connection_branches.test.js +++ b/test/unit/db_queries.test/20_error_path_transaction_connection_branches.test.js @@ -49,7 +49,7 @@ function injectPool(db, pool) { describe('Database error-path transactionConnection branches', () => { afterEach(() => sinon.restore()); - it('createAddress: swallows INSERT error even with active transactionConnection', async () => { + it('createAddress: rethrows an INSERT error inside an active transaction so the block rolls back', async () => { const db = makeDb(); // Use a fake transactionConnection so getConnection returns it const txConn = { @@ -63,10 +63,22 @@ describe('Database error-path transactionConnection branches', () => { }; db.transactionConnection = txConn; db._transactionLock = true; - // Should not throw; error in INSERT catch is logged and swallowed - const id = await db.createAddress('newaddr2'); - // id may be 11 from re-fetch or null if re-fetch also fails; just assert no throw - assert.ok(id === 11 || id === null); + // A swallowed error would hand back a NULL id that commits with the block. + await assert.rejects(db.createAddress('newaddr2'), (err) => err.dbLookupFailure === true && /insert addr fail/.test(err.message)); + assert.strictEqual(txConn.query.callCount, 2, 'no re-fetch runs after the failed INSERT'); + }); + + it('createAddress: still swallows an INSERT error with no transaction open (mempool handle)', async () => { + const db = makeDb(); + const conn = { + query: sinon.stub() + .onFirstCall().resolves([]) + .onSecondCall().rejects(new Error('insert addr fail')) + .onThirdCall().resolves([{ id: 11 }]), + release: sinon.stub().resolves(), + }; + injectPool(db, { getConnection: sinon.stub().resolves(conn) }); + assert.strictEqual(await db.createAddress('newaddr2'), 11); }); // Regression guard: on a generic error inside an active transaction, insertEvent diff --git a/test/unit/decoder_quarantine_signal.test.js b/test/unit/decoder_quarantine_signal.test.js new file mode 100644 index 00000000..e3300e75 --- /dev/null +++ b/test/unit/decoder_quarantine_signal.test.js @@ -0,0 +1,91 @@ +/********************************************************************* + * + * Copyright © 2025-2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC - https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + *********************************************************************/ + +'use strict'; + +// A quarantined transaction is skipped for good, so it gets a signal of its own on every +// polled surface, apart from parse_errors, which routine rejections also raise. +const assert = require('assert'); +const http = require('http'); +const express = require('express'); +const XChainDecoder = require('../../src/XChainDecoder'); +const { registerLiveRoute } = require('../../src/api'); +const { registerDecoderMetrics } = require('../../src/metrics/decoder_metrics'); +const { Registry } = require('../../src/observability/metrics'); + +function caughtUpDecoder() { + const decoder = new XChainDecoder( + 'bitcoin-regtest', 'h', '0', 'db', 'u', 'p', 'h', '0', 'u', 'p', false, null + ); + decoder.lastProcessedBlockIndex = 150; + decoder.blockchainInfoLastBlock = 150; + decoder.blockchainInfoLastRefreshAt = Date.now(); + decoder.lastAdvanceAt = Date.now(); + decoder.lastPollAt = Date.now(); + decoder.synced = true; + decoder.db = { ping: async () => true }; + decoder.connector = { rpcErrors: 0 }; + return decoder; +} + +function getLive(decoder) { + const app = express(); + registerLiveRoute(app, decoder, () => true); + return new Promise((resolve, reject) => { + const server = app.listen(0, () => { + http.get({ port: server.address().port, path: '/live' }, (res) => { + let body = ''; + res.on('data', (c) => { body += c; }); + res.on('end', () => { server.close(); resolve({ status: res.statusCode, body: JSON.parse(body) }); }); + }).on('error', (e) => { server.close(); reject(e); }); + }); + }); +} + +describe('decoder quarantine signal', function () { + it('/live reports zero and nulls on a decoder that never skipped a transaction', async function () { + const res = await getLive(caughtUpDecoder()); + assert.strictEqual(res.body.quarantined_txs, 0); + assert.strictEqual(res.body.last_quarantined_block, null); + assert.strictEqual(res.body.last_quarantined_tx_hash, null); + assert.strictEqual(res.body.last_quarantined_at, null); + }); + + it('/live reports a quarantine and stays 200, since a restart cannot undo a skip', async function () { + const decoder = caughtUpDecoder(); + decoder.parseErrors = 40; + decoder.quarantinedTxs = 1; + decoder.lastQuarantinedBlock = 0; + decoder.lastQuarantinedTxHash = 'cafe01'; + decoder.lastQuarantinedAt = '2026-10-12T00:00:00.000Z'; + const res = await getLive(decoder); + assert.strictEqual(res.status, 200); + assert.strictEqual(res.body.quarantined_txs, 1); + assert.strictEqual(res.body.parse_errors, 40, 'the routine-rejection count stays separate'); + assert.strictEqual(res.body.last_quarantined_block, 0, 'height 0 is a height, not a missing value'); + assert.strictEqual(res.body.last_quarantined_tx_hash, 'cafe01'); + assert.strictEqual(res.body.last_quarantined_at, '2026-10-12T00:00:00.000Z'); + }); + + it('exports the skip count as its own Prometheus counter', function () { + const registry = new Registry(); + const decoder = caughtUpDecoder(); + decoder.parseErrors = 40; + decoder.quarantinedTxs = 2; + registerDecoderMetrics(registry, decoder); + const body = registry.render(); + assert.match(body, /^xchain_decoder_quarantined_tx_total 2$/m); + assert.match(body, /^xchain_decoder_parse_errors_total 40$/m); + }); +}); diff --git a/test/unit/decoder_ready_probe.test/01_ready_route_reports_reorg_halt.test.js b/test/unit/decoder_ready_probe.test/01_ready_route_reports_reorg_halt.test.js new file mode 100644 index 00000000..28811975 --- /dev/null +++ b/test/unit/decoder_ready_probe.test/01_ready_route_reports_reorg_halt.test.js @@ -0,0 +1,114 @@ +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +// A decoder carrying a live REORG_HALT marker answered /live with 200 and the +// container read healthy for as long as the marker stood. That 200 is deliberate: +// /live drives autoheal, and a restart does not clear the marker, so gating it there +// would restart-loop the container. GET /ready is the surface that answers 503 for +// it, and /live carries the same verdict as ready / not_ready_reasons without +// changing its own status code. +// +// Drives the REAL registrars out of src/api.js against a REAL XChainDecoder; only +// the database round-trips are stubbed. + +'use strict' + +const assert = require('assert') +const http = require('http') +const express = require('express') +const XChainDecoder = require('../../../src/XChainDecoder') +const { registerLiveRoute, registerReadyRoute } = require('../../../src/api') +const { decoderNotReadyReasons } = require('../../../src/api/probe_routes') + +// A caught-up decoder whose halt marker read answers `marker`. +function decoderWithMarker(marker, dbOk = true) { + const decoder = new XChainDecoder( + 'bitcoin-regtest', 'h', '0', 'db', 'u', 'p', 'h', '0', 'u', 'p', false, null + ) + decoder.lastProcessedBlockIndex = 150 + decoder.blockchainInfoLastBlock = 150 + decoder.blockchainInfoLastRefreshAt = Date.now() + decoder.lastAdvanceAt = Date.now() + decoder.lastPollAt = Date.now() + decoder.synced = true + decoder.db = { ping: async () => dbOk, getReorgHaltMarker: async () => marker } + decoder.connector = { rpcErrors: 0 } + return decoder +} + +function get(decoder, path) { + const app = express() + registerLiveRoute(app, decoder, () => true) + registerReadyRoute(app, decoder, () => true) + return new Promise((resolve, reject) => { + const server = app.listen(0, '127.0.0.1', () => { + http.get({ hostname: '127.0.0.1', port: server.address().port, path }, (res) => { + let body = '' + res.on('data', (c) => { body += c }) + res.on('end', () => { server.close(); resolve({ status: res.statusCode, body: JSON.parse(body) }) }) + }).on('error', (e) => { server.close(); reject(e) }) + }) + }) +} + +const CLEAN = { halted: false, at: null, reason: null } +const HALTED = { halted: true, at: '2026-10-12T21:00:00.000Z', reason: 'reorg deeper than the safe rewind window' } + +describe('GET /ready answers the REORG_HALT that /live must not gate on', function () { + it('answers 200 on /live and /ready for a clean, caught-up decoder', async function () { + const live = await get(decoderWithMarker(CLEAN), '/live') + const ready = await get(decoderWithMarker(CLEAN), '/ready') + assert.strictEqual(live.status, 200) + assert.strictEqual(ready.status, 200) + assert.strictEqual(ready.body.ready, true) + assert.deepStrictEqual(ready.body.not_ready_reasons, []) + }) + + it('keeps /live at 200 on a latent halt and reports it not ready', async function () { + const live = await get(decoderWithMarker(HALTED), '/live') + assert.strictEqual(live.status, 200, 'autoheal reads /live; a restart cannot clear the marker') + assert.strictEqual(live.body.reorg_halted, true) + assert.strictEqual(live.body.ready, false) + assert.deepStrictEqual(live.body.not_ready_reasons, ['reorg_halt']) + }) + + it('answers 503 on /ready while the marker stands', async function () { + const ready = await get(decoderWithMarker(HALTED), '/ready') + assert.strictEqual(ready.status, 503) + assert.deepStrictEqual(ready.body.not_ready_reasons, ['reorg_halt']) + assert.strictEqual(ready.body.reorg_halt_reason, HALTED.reason) + }) + + it('fails closed when the marker has never been read', async function () { + const ready = await get(decoderWithMarker(CLEAN, false), '/ready') + assert.strictEqual(ready.status, 503) + assert.deepStrictEqual(ready.body.not_ready_reasons, ['db_unreachable', 'reorg_halt_unchecked']) + }) +}) + +describe('decoderNotReadyReasons()', function () { + const base = { decoderRunning: true, dbOk: true, stalled: false, pollSilent: false, + reorgHalt: { halted: false, parked: false, checked_at: 1 } } + + it('names a parked halt beside the halt itself', function () { + const state = { ...base, reorgHalt: { halted: true, parked: true, checked_at: 1 } } + assert.deepStrictEqual(decoderNotReadyReasons(state), ['reorg_halt', 'reorg_halt_parked']) + }) + + it('names a frozen node tip, which /live reports but never gates on', function () { + const state = { ...base, syncStatus: { node_height_stale: true } } + assert.deepStrictEqual(decoderNotReadyReasons(state), ['node_height_stale']) + }) + + it('carries every liveness fault as a readiness fault', function () { + const state = { ...base, decoderRunning: false, stalled: true, pollSilent: true } + assert.deepStrictEqual(decoderNotReadyReasons(state), ['not_running', 'stalled', 'poll_silent']) + }) +}) diff --git a/test/unit/decoder_stress_sweep.test.js b/test/unit/decoder_stress_sweep.test.js index 3beedf2c..372b30a2 100644 --- a/test/unit/decoder_stress_sweep.test.js +++ b/test/unit/decoder_stress_sweep.test.js @@ -24,8 +24,10 @@ const { makeRpcBatchGuard } = require('../../src/api') const CANONICAL_TX_HEX = '0200000001aabbccdd11223344eeff5566778899001122334455667788aabbccddeeff0011010000006b4830303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303030303021020202020202020202020202020202020202020202020202020202020202020202ffffffff020000000000000000166a145ed141846fd6cbef65cb28316aff11ba07152fcf00e1f505000000001976a914aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa88ac00000000' // The same tx with the LTC MWEB marker (0x00) + flag (0x08) injected after the -// 4-byte version, exactly as a HogEx/MWEB-carrying LTC tx serializes on the wire. -const MWEB_FLAGGED_TX_HEX = '02000000' + '0008' + CANONICAL_TX_HEX.substr(8) +// 4-byte version and the MWEB tag 0x00 (no MWEB part) before the locktime, +// exactly as a HogEx/MWEB-carrying LTC tx serializes on the wire. +const MWEB_FLAGGED_TX_HEX = '02000000' + '0008' + + CANONICAL_TX_HEX.slice(8, -8) + '00' + CANONICAL_TX_HEX.slice(-8) function newDecoder(network){ return new XChainDecoder(network, 'h', '0', 'db', 'u', 'p', 'h', '0', 'u', 'p', false, null) diff --git a/test/unit/dispenser_address_key.test.js b/test/unit/dispenser_address_key.test.js index df6368a8..984b8270 100644 --- a/test/unit/dispenser_address_key.test.js +++ b/test/unit/dispenser_address_key.test.js @@ -15,24 +15,42 @@ const { dispenserAddressKey } = require('../../src/protocol/batch_sub_command_ca describe('dispenserAddressKey', function () { - it('folds ASCII letters to lower case', function () { - assert.strictEqual(dispenserAddressKey('BCRT1QXYZ'), 'bcrt1qxyz') - assert.strictEqual(dispenserAddressKey('mAbC'), dispenserAddressKey('mabc')) + it('folds ASCII letters to lower case', async function () { + const addressIdOf = async address => address === 'BCRT1QXYZ' || address === 'bcrt1qxyz' ? 41 : 42 + assert.strictEqual( + await dispenserAddressKey('BCRT1QXYZ', addressIdOf), + await dispenserAddressKey('bcrt1qxyz', addressIdOf)) }) - it('drops trailing spaces', function () { - assert.strictEqual(dispenserAddressKey('mAbC '), 'mabc') - assert.strictEqual(dispenserAddressKey(' '), '') + it('drops trailing spaces', async function () { + const resolved = [] + const addressIdOf = async address => { + resolved.push(address) + if (address === 'mAbC ' || address === 'mabc') return 12 + throw new Error('unexpected address: ' + address) + } + assert.strictEqual( + await dispenserAddressKey('mAbC ', addressIdOf), + await dispenserAddressKey('mabc', addressIdOf)) + assert.deepStrictEqual(resolved, ['mAbC ', 'mabc']) }) - it('keeps leading spaces and other characters', function () { - assert.notStrictEqual(dispenserAddressKey(' mabc'), dispenserAddressKey('mabc')) - assert.strictEqual(dispenserAddressKey(' MABC '), ' mabc') - assert.strictEqual(dispenserAddressKey('ÄBC\t\u00a0'), 'Äbc\t\u00a0') + it('keeps leading spaces and other characters', async function () { + const addressIdOf = async address => address === ' MABC ' ? 7 : 8 + assert.notStrictEqual( + await dispenserAddressKey(' MABC ', addressIdOf), + await dispenserAddressKey('mabc', addressIdOf)) }) - it('keeps distinct addresses distinct', function () { - assert.notStrictEqual(dispenserAddressKey('mabc'), dispenserAddressKey('mabd')) - assert.strictEqual(typeof dispenserAddressKey('mAbC'), 'string') + it('keeps distinct addresses distinct', async function () { + assert.notStrictEqual( + await dispenserAddressKey('first', async () => 7), + await dispenserAddressKey('second', async () => 8)) + assert.strictEqual( + await dispenserAddressKey('first', async () => 12), + await dispenserAddressKey('second', async () => '12')) + assert.notStrictEqual( + await dispenserAddressKey('Málaga', async () => null), + await dispenserAddressKey('malaga', async () => undefined)) }) }) diff --git a/test/unit/dispenser_cancel_grace.test.js b/test/unit/dispenser_cancel_grace.test.js index 0c041c5b..e6871e3c 100644 --- a/test/unit/dispenser_cancel_grace.test.js +++ b/test/unit/dispenser_cancel_grace.test.js @@ -138,15 +138,22 @@ function makeDecoder(consensusNetwork){ return decoder } -// Drive the real block loop over two blocks on `consensusNetwork`: -// block 0 at `expireAt` - the decoder's own soft-expire stamps the dispenser here; -// block 1 at `payAt` - the payment block whose capture set the test asserts on. +function configureBlockDecoder(decoder, blockTimes){ + decoder.xchainBlockDecoder = { + blockFromHex: (hex) => ({ + prevHash: Buffer.from(PREV_WIRE), + timestamp: blockTimes[Number(String(hex).split(':')[1])], + transactions: [fakeTx('tx-at-' + String(hex))], + }) + } +} + +// Drive the real block loop over the supplied protocol times on `consensusNetwork`. // Nothing is pre-stamped by hand: the stamp under test is written by the production // deleteOpenDispensers call site. -function runTwoBlocks(consensusNetwork, expireAt, payAt, model){ +function runBlocks(consensusNetwork, blockTimes, model){ const decoder = makeDecoder(consensusNetwork) - const timesByHeight = { 0: expireAt, 1: payAt } const setsSeenByParse = [] decoder.parseTransaction = async (tx, openDispenserAddresses) => { setsSeenByParse.push(openDispenserAddresses) @@ -154,12 +161,13 @@ function runTwoBlocks(consensusNetwork, expireAt, payAt, model){ } decoder.connector = { - getBlockchainInfo: async () => ({ verificationprogress: 1, blocks: 1 }), + getBlockchainInfo: async () => ({ verificationprogress: 1, blocks: blockTimes.length - 1 }), getBlockHash: async (height) => 'height:' + height, getBlock: async (hash) => hash, } let commits = 0 + const storedBlocks = new Map() decoder.db = { createDatabase: async () => true, verifyDatabase: async () => true, @@ -170,11 +178,17 @@ function runTwoBlocks(consensusNetwork, expireAt, payAt, model){ // Block 1 runs the reorg check against block 0's stored hash. Both blocks carry the // same PREV_WIRE, so answering with that value keeps the chain contiguous and the loop // out of its reorg branch, which this suite does not exercise. - getBlockByIndex: async () => ({ block_hash: PREV_HASH }), + getBlockByIndex: async (height) => storedBlocks.get(height) || null, beginTransaction: async () => {}, endTransaction: async () => {}, - commitTransaction: async () => { if (++commits >= 2) decoder.stopFlag = true; return true }, - insertBlock: async () => true, + commitTransaction: async () => { + if (++commits >= blockTimes.length) decoder.stopFlag = true + return true + }, + insertBlock: async (row) => { + storedBlocks.set(row.block_index, { ...row, block_hash: PREV_HASH }) + return true + }, insertEvent: async () => true, insertTransaction: async () => true, insertTransactionOutput: async () => true, @@ -190,17 +204,15 @@ function runTwoBlocks(consensusNetwork, expireAt, payAt, model){ getOpenDispenserOracleAddressesBySource: (s) => model.getOpenDispenserOracleAddressesBySource(s), } - decoder.xchainBlockDecoder = { - blockFromHex: (hex) => ({ - prevHash: Buffer.from(PREV_WIRE), - timestamp: timesByHeight[Number(String(hex).split(':')[1])], - transactions: [fakeTx('tx-at-' + String(hex))], - }) - } + configureBlockDecoder(decoder, blockTimes) return decoder.start().then(() => ({ setsSeenByParse })) } +function runTwoBlocks(consensusNetwork, expireAt, payAt, model){ + return runBlocks(consensusNetwork, [expireAt, payAt], model) +} + // One funded dispenser, cancelled shortly before its expiration, as the decoder holds it: // the decoder mirrors no cancel, so the row carries only its own expiration. function fundedCancelledDispenser(){ @@ -228,8 +240,8 @@ function registerPostExpiryCaptureTest(){ assert.strictEqual(model.captureLoads.length, 2) const payLoad = model.captureLoads[1] - assert.strictEqual(payLoad.floor, payAt - DISPENSER_CANCEL_GRACE_SECONDS, - 'the block loop must pass the grace floor derived from this block header time') + assert.strictEqual(payLoad.floor, EXPIRATION + 1 - DISPENSER_CANCEL_GRACE_SECONDS, + 'the block loop must anchor the grace floor to the parent protocol time') assert.deepStrictEqual(model.purgeCalls[1], { safeHeight: 1 - XChainDecoder.DISPENSER_EXPIRE_SAFE_DEPTH, graceFloor: payAt - DISPENSER_CANCEL_GRACE_SECONDS, @@ -298,19 +310,20 @@ describe('dispenser cancellation grace: decoder capture outlasts the indexer fil }) it('closes capture once the indexer can no longer settle a fill', async () => { - // The grace is a window, not an amnesty: past mark + grace the address leaves the - // capture set, and by then the indexer stopped matching the dispenser long ago. - // The window is measured from the SOFT-EXPIRE MARK block, which is the last block a - // cancel can be accepted in, so the probe steps one second past mark + grace. + // The grace is measured from the last block that can accept a cancel. + // The first block across that boundary is retained; the next block proves + // that advancing the parent clock advances the floor. const markAt = EXPIRATION + 1 - const payAt = markAt + DISPENSER_CANCEL_GRACE_SECONDS + 1 + const payAt = markAt + DISPENSER_CANCEL_GRACE_SECONDS + 2 assert.ok(!indexerStillSettlesFill(payAt), 'the probe block must be one the indexer has already closed') const model = fundedCancelledDispenser() - await runTwoBlocks('regtest', markAt, payAt, model) + await runBlocks('regtest', [markAt, payAt - 1, payAt], model) - assert.ok(!model.captureLoads[1].set.has(ADDR), + assert.ok(model.captureLoads[1].set.has(ADDR), + 'the first block across the close boundary remains conservatively captured') + assert.ok(!model.captureLoads[2].set.has(ADDR), 'past the grace window the dispenser leaves the capture set') }) }) @@ -334,11 +347,8 @@ describe('dispenser cancellation grace: decoder capture outlasts the indexer fil `block time ${payAt}: the indexer still settles fills here, so the decoder ` + 'must still capture payments to the dispenser') } - // Outside the indexer's window capture is merely allowed to continue to the end of - // the grace: over-capture is the direction the advisory contract calls safe, and - // the indexer drops the surplus. - if (payAt > markAt + DISPENSER_CANCEL_GRACE_SECONDS) - assert.ok(!captured, `block time ${payAt}: capture must end with the grace window`) + // A two-block jump deliberately anchors this candidate block to the mark block's + // protocol time. The next sequential block is what advances the floor. } // Guard against a vacuous sweep: an arithmetic slip that made the window empty would // otherwise pass every assertion above. @@ -381,7 +391,7 @@ describe('dispenser cancellation grace: decoder capture outlasts the indexer fil assert.strictEqual(model.rows[0].expiredBlockTime, MARK_AT, 'the production soft-expire must stamp the mark block header time') const payLoad = model.captureLoads[1] - assert.strictEqual(payLoad.floor, payAt - DISPENSER_CANCEL_GRACE_SECONDS) + assert.strictEqual(payLoad.floor, MARK_AT - DISPENSER_CANCEL_GRACE_SECONDS) assert.ok(payLoad.set.has(ADDR), 'a payment the indexer would still settle must be captured by the decoder') }) @@ -389,11 +399,13 @@ describe('dispenser cancellation grace: decoder capture outlasts the indexer fil it('stops capturing once the indexer has closed the boundary-cancelled dispenser', async () => { const payAt = INDEXER_CLOSE + 1 const model = fundedCancelledDispenser() - await runTwoBlocks('regtest', MARK_AT, payAt, model) + await runBlocks('regtest', [MARK_AT, payAt, payAt + 1], model) assert.strictEqual(model.rows[0].expiredBlockTime, MARK_AT) - assert.ok(!model.captureLoads[1].set.has(ADDR), - 'past the indexer close the dispenser leaves the capture set') + assert.ok(model.captureLoads[1].set.has(ADDR), + 'the first block after close must use the parent protocol clock') + assert.ok(!model.captureLoads[2].set.has(ADDR), + 'the following block leaves the capture set after its parent advances') }) }) }) diff --git a/test/unit/dispenser_cancel_grace_parent_floor.test.js b/test/unit/dispenser_cancel_grace_parent_floor.test.js new file mode 100644 index 00000000..94df016a --- /dev/null +++ b/test/unit/dispenser_cancel_grace_parent_floor.test.js @@ -0,0 +1,142 @@ +'use strict' + +// Copyright © 2025-2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC - https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +const assert = require('assert') +const sinon = require('sinon') + +const XChainDecoder = require('../../src/XChainDecoder') +const { storeBlock } = require('../../src/XChainDecoder/block_store') +const protocolTime = require('../../src/protocol/protocol_time') +const { + DISPENSER_CANCEL_GRACE_PARENT_FLOOR_ACTIVATION, + DISPENSER_CANCEL_GRACE_SECONDS, + cancelGraceFloor, +} = require('../../src/protocol/dispenser_cancel_grace') + +function makeBlockStoreContext(blockReads, floors){ + return { + consensusNetwork: 'testnet', + blockchainInfoLastBlock: 100, + log: () => {}, + millisecondsToTimeString: () => '', + db: { + beginTransaction: async () => {}, + getBlockByIndex: async (height) => { + blockReads.push(height) + return { block_time: 100 + height } + }, + insertBlock: async () => true, + deleteOpenDispensers: async () => true, + commitTransaction: async () => true, + purgeExpiredDispensers: async () => true, + getAllOpenDispenserAddresses: async (floor) => { + floors.push(floor) + return new Set() + }, + }, + } +} + +function makeBlockLoop(){ + return { + blocksQuantity: 0, + blocksCount: 0, + transactionsCount: 0, + validTransactionsCount: 0, + outputCount: 0, + startTimeStamp: Date.now(), + txParseRetryCounts: new Map(), + insertQuarantineCounts: new Map(), + insertQuarantine: new Set(), + } +} + +describe('dispenser cancellation grace parent protocol floor', function () { + afterEach(function () { + sinon.restore() + }) + + it('uses the earlier parent protocol time only on an active network', function () { + assert.strictEqual(cancelGraceFloor('regtest', 10000, 9000), 5400) + assert.strictEqual(cancelGraceFloor('regtest', 10000, 12000), 6400) + assert.strictEqual(cancelGraceFloor('mainnet', 10000, 9000), 6400) + assert.strictEqual(cancelGraceFloor('testnet', 10000, 9000), 6400) + }) + + it('falls back to the block clock when no finite parent clock is available', function () { + assert.strictEqual( + cancelGraceFloor('regtest', 10000, null), + 10000 - DISPENSER_CANCEL_GRACE_SECONDS) + assert.strictEqual( + cancelGraceFloor('regtest', 10000, 'not-a-time'), + 10000 - DISPENSER_CANCEL_GRACE_SECONDS) + }) + + it('anchors a mempool refresh to the latest stored parent protocol time', async function () { + sinon.stub(Date, 'now').returns(10000 * 1000) + const decoder = new XChainDecoder( + 'bitcoin-regtest', 'h', '0', 'db', 'u', 'p', 'h', '0', 'u', 'p', false, null + ) + decoder.consensusNetwork = 'regtest' + decoder.lastProcessedBlockIndex = 7 + + const blockReads = [] + const floors = [] + decoder.mempoolDb = { + getBlockByIndex: async (height) => { + blockReads.push(height) + return { block_time: 9000 } + }, + getAllOpenDispenserAddresses: async (floor) => { + floors.push(floor) + return new Set() + }, + deleteAndCompareTxsNotInList: async () => ({ transactionsDeleted: 0 }), + } + decoder.connector = { + getRawMempool: async () => [], + getRawTransactions: async () => [], + } + + await decoder.updateMempool() + + assert.deepStrictEqual(blockReads, [7]) + assert.deepStrictEqual(floors, [9000 - DISPENSER_CANCEL_GRACE_SECONDS]) + }) +}) + +describe('dispenser cancellation grace parent protocol floor', function () { + afterEach(function () { + sinon.restore() + }) + + it('keeps the current block MTP window bounded when resolving its parent clock', async function () { + const activation = DISPENSER_CANCEL_GRACE_PARENT_FLOOR_ACTIVATION + const previousActivation = activation.testnet + activation.testnet = 0 + const createContext = sinon.spy(protocolTime, 'createBlockTimeContext') + const blockReads = [] + const floors = [] + const context = makeBlockStoreContext(blockReads, floors) + + try { + await storeBlock.call( + context, makeBlockLoop(), { timestamp: 500, transactions: [] }, 12, 'hash', 'parent') + } finally { + activation.testnet = previousActivation + } + + assert.deepStrictEqual(blockReads, [11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0]) + assert.strictEqual(createContext.firstCall.args[2].length, protocolTime.MEDIAN_TIME_SPAN) + assert.deepStrictEqual(floors, [105 - DISPENSER_CANCEL_GRACE_SECONDS]) + }) +}) diff --git a/test/unit/dispenser_cancel_grace_parent_floor_activation.test.js b/test/unit/dispenser_cancel_grace_parent_floor_activation.test.js new file mode 100644 index 00000000..702a561b --- /dev/null +++ b/test/unit/dispenser_cancel_grace_parent_floor_activation.test.js @@ -0,0 +1,34 @@ +'use strict'; + +// Copyright © 2025-2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC - https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +const assert = require('assert'); + +const activation = require('../../src/protocol/constants/activation.js'); +const constants = require('../../src/protocol/constants.js'); + +const NAME = 'DISPENSER_CANCEL_GRACE_PARENT_FLOOR_ACTIVATION'; + +describe('DISPENSER_CANCEL_GRACE_PARENT_FLOOR_ACTIVATION conformance', function () { + + it('keeps mainnet and testnet inert while regtest is genesis-active', function () { + assert.deepStrictEqual(activation[NAME], { + mainnet: 9999999999, + testnet: 9999999999, + regtest: 0, + }); + }); + + it('is re-exported by the public constants module', function () { + assert.ok(activation[NAME]); + assert.strictEqual(constants[NAME], activation[NAME]); + }); +}); diff --git a/test/unit/duplicate_transaction_insert_rollback.test.js b/test/unit/duplicate_transaction_insert_rollback.test.js new file mode 100644 index 00000000..39672dc8 --- /dev/null +++ b/test/unit/duplicate_transaction_insert_rollback.test.js @@ -0,0 +1,88 @@ +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +'use strict' + +const assert = require('assert') +const XChainDecoder = require('../../src/XChainDecoder') + +// Passes of one block before a test calls the loop wedged and stops it. +const PASS_LIMIT = 8 + +function fakeTx(id) { + return { getId: () => id, outs: [] } +} + +// Drive the real block loop at height 0 with `insert` deciding each insertTransaction result. +function buildDecoder(insert) { + const decoder = new XChainDecoder('bitcoin-regtest', 'h', '0', 'db', 'u', 'p', 'h', '0', 'u', 'p', false, null) + decoder.startBlockIndex = 0 + decoder.sleep = async () => {} + const calls = { passes: 0, wedged: false, commit: 0, endTransaction: 0, outputs: [], insertEvent: [] } + decoder.connector = { + getBlockchainInfo: async () => ({ verificationprogress: 1, blocks: 0 }), + getBlockHash: async () => 'blockA', + getBlock: async () => '' + } + decoder.db = { + createDatabase: async () => true, verifyDatabase: async () => true, + verifyTables: async () => true, runMigrations: async () => ({ applied: [], pending: [] }), + getLastBlockIndex: async () => -1, getLastTxIndex: async () => 0, + beginTransaction: async () => {}, + endTransaction: async () => { calls.endTransaction++ }, + commitTransaction: async () => { calls.commit++; decoder.stopFlag = true; return true }, + deleteOpenDispensers: async () => true, purgeExpiredDispensers: async () => true, + getAllOpenDispenserAddresses: async () => new Set(), + insertEvent: async (code, data) => { calls.insertEvent.push({ code, data }); return true }, + insertBlock: async () => true, + insertTransaction: async (tx) => insert(tx), + insertTransactionOutput: async (output) => { calls.outputs.push(output.txIndex); return true }, + DUPLICATED_TRANSACTION: 1, POISON_ROW: 2 + } + decoder.xchainBlockDecoder = { + blockFromHex: () => { + calls.passes++ + if (calls.passes > PASS_LIMIT){ calls.wedged = true; decoder.stopFlag = true } + return { prevHash: Buffer.alloc(32), timestamp: 1700000000, transactions: [fakeTx('cafe01')] } + } + } + decoder.parseTransaction = async () => ({ + data: Buffer.from('SEND|x'), compiledDataLength: 6, rawData: null, + source: 'someaddr', destination: null, amount: 0, + dispenseOutputs: [{ vout: 0, destinationAddress: 'destaddr', amount: 1n }], paymentOutputs: [] + }) + return { decoder, calls } +} + +// A duplicate key on the transactions row rolls the block back instead of writing its outputs. +describe('duplicate transaction insert rolls the block back', function () { + this.timeout(0) + + it('writes no output under a tx_index whose row collided and commits on the clean retry', async function () { + let attempts = 0 + const { decoder, calls } = buildDecoder(() => (++attempts === 1 ? decoder.db.DUPLICATED_TRANSACTION : true)) + await decoder.start() + assert.strictEqual(calls.wedged, false, 'the block must not wedge') + assert.strictEqual(attempts, 2, 'the block is retried after the duplicate') + assert.strictEqual(calls.endTransaction, 1, 'the open block transaction is ended on the duplicate') + assert.deepStrictEqual(calls.outputs, [1], 'the output is written once, on the retry, under the retried tx_index') + assert.strictEqual(calls.commit, 1, 'the block commits on the retry') + assert.deepStrictEqual(calls.insertEvent, [], 'a duplicate is never quarantined with PARSE_ERROR') + }) + + it('keeps retrying a persistent duplicate without ever committing or skipping the tx', async function () { + const { decoder, calls } = buildDecoder(() => decoder.db.DUPLICATED_TRANSACTION) + await decoder.start() + assert.strictEqual(calls.wedged, true, 'a persistent duplicate stalls the block') + assert.strictEqual(calls.commit, 0, 'nothing commits past the collision') + assert.deepStrictEqual(calls.outputs, [], 'no output is written for a tx whose row collided') + assert.deepStrictEqual(calls.insertEvent, [], 'the tx is never skipped with PARSE_ERROR') + }) +}) diff --git a/test/unit/litecoin_block.test.js b/test/unit/litecoin_block.test.js index 3d30e6e8..b98fe23f 100644 --- a/test/unit/litecoin_block.test.js +++ b/test/unit/litecoin_block.test.js @@ -54,19 +54,46 @@ function buildMinimalTxBuf({ version = 1, markerFlag = null } = {}) { const locktime = Buffer.alloc(4, 0) if (markerFlag) { - // Segwit / MWEB style: version + 0x00 marker + flag + inputs + outputs + locktime. - // With zero inputs there are no witness stacks to serialize between the two. + // MWEB style: version + 0x00 marker + flag + inputs + outputs + MWEB tag + + // locktime. With zero inputs there are no witness stacks to serialize, and the + // MWEB tag 0x00 says there is no MWEB part (as in a HogEx). const marker = Buffer.from([0x00, markerFlag]) - // For segwit: after 0-input 0-output, we need witnesses (one per input = 0) and locktime - return Buffer.concat([versionBuf, marker, varint(0), varint(0), locktime]) + return Buffer.concat([versionBuf, marker, varint(0), varint(0), Buffer.from([0x00]), locktime]) } // Standard: version + 0 inputs + 0 outputs + locktime return Buffer.concat([versionBuf, varint(0), varint(0), locktime]) } -// Compose a block buffer: header + varint(txCount) + tx1 bytes + ... + txN bytes +const dsha = (buf) => crypto.createHash('sha256').update(crypto.createHash('sha256').update(buf).digest()).digest() + +// The hash a tx's txid is read from: its plain serialization. The MWEB-flagged txs +// built above have no witness and a null MWEB tag, so the plain form drops the +// marker, flag and tag. +function plainTxHash(txBuf) { + if (txBuf[4] === 0x00 && (txBuf[5] === 0x08 || txBuf[5] === 0x09)) { + return dsha(Buffer.concat([txBuf.subarray(0, 4), txBuf.subarray(6, txBuf.length - 5), txBuf.subarray(txBuf.length - 4)])) + } + return bitcoin.Transaction.fromBuffer(txBuf).getHash() +} + +// The merkle root over tx hashes, the last one doubled on an odd level. +function merkleRoot(hashes) { + let level = hashes + while (level.length > 1) { + if (level.length % 2) level = level.concat([level[level.length - 1]]) + const next = [] + for (let i = 0; i < level.length; i += 2) next.push(dsha(Buffer.concat([level[i], level[i + 1]]))) + level = next + } + return level[0] +} + +// Compose a block buffer: header + varint(txCount) + tx1 bytes + ... + txN bytes, +// with the header's merkle root set to the txs' root (the decoder refuses a mismatch). function buildBlockBuf(header, txBuffers) { - return Buffer.concat([header, varint(txBuffers.length), ...txBuffers]) + const rooted = Buffer.from(header) + merkleRoot(txBuffers.map(plainTxHash)).copy(rooted, 36) + return Buffer.concat([rooted, varint(txBuffers.length), ...txBuffers]) } // ─── tests ────────────────────────────────────────────────────────────────── @@ -197,15 +224,7 @@ describe('XChainBlockDecoder litecoin blockFromBuffer', () => { // - the coinbase's first input has a 32-byte witness (satisfies txesHaveWitnessCommit) // - the coinbase output #1 is OP_RETURN 0xaa21a9ed + 32-byte witness root hash // This exercises the `if (witnessCommit) block.witnessCommit = witnessCommit` branch. - const SEGWIT_LITECOIN_BLOCK_HEX = - '04000000' + // version 4 - 'aa'.repeat(32) + // prevHash - 'bb'.repeat(32) + // merkleRoot - '00f15365' + // timestamp - 'ffff001d' + // bits - '78563412' + // nonce - '01' + // 1 transaction - // coinbase segwit tx: + const SEGWIT_COINBASE_HEX = '01000000' + // version '00' + // segwit marker '01' + // segwit flag @@ -218,6 +237,15 @@ describe('XChainBlockDecoder litecoin blockFromBuffer', () => { '01' + // 1 witness stack item '20' + '00'.repeat(32) + // 32-byte reserved value '00000000' // locktime + const SEGWIT_LITECOIN_BLOCK_HEX = + '04000000' + // version 4 + 'aa'.repeat(32) + // prevHash + plainTxHash(Buffer.from(SEGWIT_COINBASE_HEX, 'hex')).toString('hex') + // merkleRoot: the coinbase txid + '00f15365' + // timestamp + 'ffff001d' + // bits + '78563412' + // nonce + '01' + // 1 transaction + SEGWIT_COINBASE_HEX const block = decoder.blockFromBuffer(Buffer.from(SEGWIT_LITECOIN_BLOCK_HEX, 'hex')) diff --git a/test/unit/migration_preconditions.test.js b/test/unit/migration_preconditions.test.js index 24aa4f6b..26580a89 100644 --- a/test/unit/migration_preconditions.test.js +++ b/test/unit/migration_preconditions.test.js @@ -145,6 +145,19 @@ describe('Database.STARTUP_ASSERTED_MIGRATIONS @regression @tier1', function () }); }); +describe('Database.STARTUP_ASSERTED_MIGRATIONS @regression @tier1', function () { + + Database.STARTUP_ASSERTED_MIGRATIONS.forEach(function (entry) { + it(entry.file + ': carries a MIGRATION_PRECONDITIONS baseline predicate', function () { + const pre = Database.MIGRATION_PRECONDITIONS[entry.file]; + assert.ok(pre && typeof pre.sql === 'string' && typeof pre.skipWhen === 'function', + entry.file + ' is a startup-asserted manual migration with no baseline predicate, so on a ' + + 'database built from the current src/sql it parks PENDING with no ledger row and the deploy ' + + 'tool refuses every update on a requirement the schema already satisfies.'); + }); + }); +}); + describe('Database.STARTUP_ASSERTED_MIGRATIONS @regression @tier1', function () { describe('startupAssertedMigrationFile()', function () { diff --git a/test/unit/migration_preconditions.test/01_widen_ids_baseline_predicate.test.js b/test/unit/migration_preconditions.test/01_widen_ids_baseline_predicate.test.js new file mode 100644 index 00000000..4cf1ab43 --- /dev/null +++ b/test/unit/migration_preconditions.test/01_widen_ids_baseline_predicate.test.js @@ -0,0 +1,93 @@ +'use strict'; + +/********************************************************************* + * + * Copyright © 2025–2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC – https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ********************************************************************** + * Baseline predicate for the widen-ids deploy precondition: a database built + * from the current src/sql records the file, and any narrower shape keeps it + * pending for the operator run. + * + ********************************************************************/ + +const assert = require('assert'); + +const Database = require('../../../src/db'); + +// The fresh src/sql shape for every column widen-ids MODIFYs, as information_schema +// reports it, with the FK count the correlated subquery returns (a BigInt from the driver). +const WIDEN_IDS_FRESH = [ + ['index_addresses', 'id'], ['index_transactions', 'id'], ['pubkeys', 'address_id'], + ['blocks', 'block_index'], ['blocks', 'block_time'], ['blocks', 'block_hash_id'], ['blocks', 'previous_block_hash_id'], + ['transactions', 'tx_index'], ['transactions', 'tx_hash_id'], ['transactions', 'block_index'], + ['transactions', 'source_id'], ['transactions', 'destination_id'], + ['dispensers', 'tx_index'], ['dispensers', 'address_id'], + ['transaction_outputs', 'tx_index'], ['transaction_outputs', 'vout'], ['transaction_outputs', 'destination_id'], +]; + +function widenIdsRows(edit) { + const rows = WIDEN_IDS_FRESH.map(([tbl, col]) => ({ + tbl, col, dataType: 'bigint', columnType: 'bigint(20) unsigned', + extra: col === 'id' ? 'auto_increment' : '', fkCount: 1n + })); + return edit ? edit(rows) : rows; +} + +describe('Database.MIGRATION_PRECONDITIONS: widen-ids-to-bigint predicate @regression', function () { + + const pre = Database.MIGRATION_PRECONDITIONS['2026-06-02-widen-ids-to-bigint.sql']; + const narrow = (tbl, col, type) => (rows) => rows.map(r => (r.tbl === tbl && r.col === col) + ? { ...r, dataType: type.split('(')[0], columnType: type } : r); + + it('baselines on the fresh src/sql shape', function () { + assert.strictEqual(WIDEN_IDS_FRESH.length, 17); + assert.match(pre.skipWhen(widenIdsRows()), /already BIGINT UNSIGNED/); + }); + + it('does NOT baseline while pubkeys.address_id or index_addresses.id is still INT', function () { + assert.strictEqual(pre.skipWhen(widenIdsRows(narrow('pubkeys', 'address_id', 'int(10) unsigned'))), null); + assert.strictEqual(pre.skipWhen(widenIdsRows(narrow('index_addresses', 'id', 'int(10) unsigned'))), null); + }); + + it('does NOT baseline on a signed BIGINT', function () { + assert.strictEqual(pre.skipWhen(widenIdsRows(narrow('transaction_outputs', 'vout', 'bigint(20)'))), null); + }); + + it('does NOT baseline when any one column is absent', function () { + for (let i = 0; i < WIDEN_IDS_FRESH.length; i++) { + assert.strictEqual(pre.skipWhen(widenIdsRows(rows => rows.filter((_, j) => j !== i))), null, + WIDEN_IDS_FRESH[i].join('.') + ' absent must not baseline'); + } + }); + + it('does NOT baseline when pubkeys_ibfk_1 is missing', function () { + assert.strictEqual(pre.skipWhen(widenIdsRows(rows => rows.map(r => ({ ...r, fkCount: 0n })))), null); + }); + + it('does NOT baseline when a surrogate key lacks AUTO_INCREMENT', function () { + assert.strictEqual(pre.skipWhen(widenIdsRows(rows => rows.map(r => + (r.tbl === 'index_transactions' ? { ...r, extra: '' } : r)))), null); + }); + + it('does NOT baseline on an unreadable type or an empty answer', function () { + assert.strictEqual(pre.skipWhen(widenIdsRows(rows => rows.map(r => + (r.tbl === 'blocks' && r.col === 'block_time' ? { ...r, columnType: null } : r)))), null); + assert.strictEqual(pre.skipWhen([]), null); + }); + + it('reads information_schema for this database through exactly one placeholder', function () { + assert.match(pre.sql, /information_schema\.columns/i); + assert.match(pre.sql, /table_schema\s*=\s*\?/i, 'must be parameterised on the database name'); + assert.strictEqual((pre.sql.match(/\?/g) || []).length, 1, 'the runner binds only the database name'); + assert.match(pre.sql, /pubkeys_ibfk_1/); + }); +}); diff --git a/test/unit/migration_runner.test/05_run_migrations_migration_preconditions.test.js b/test/unit/migration_runner.test/05_run_migrations_migration_preconditions.test.js index 2ea432b8..621e2a28 100644 --- a/test/unit/migration_runner.test/05_run_migrations_migration_preconditions.test.js +++ b/test/unit/migration_runner.test/05_run_migrations_migration_preconditions.test.js @@ -170,3 +170,55 @@ describe('runMigrations() migration preconditions @regression', function () { .then((r) => assert.strictEqual(r, null, 'unlisted files short-circuit without a query')); }); }); + +// The startup run (no includeManual) on a database built from the current src/sql +// records widen-ids, so the deploy tool finds its ledger row; a still-INT database +// keeps it PENDING for the operator run. +describe('runMigrations() migration preconditions @regression', function () { + + const WIDEN = '2026-06-02-widen-ids-to-bigint.sql'; + const PAIRS = { + index_addresses: ['id'], index_transactions: ['id'], pubkeys: ['address_id'], + blocks: ['block_index', 'block_time', 'block_hash_id', 'previous_block_hash_id'], + transactions: ['tx_index', 'tx_hash_id', 'block_index', 'source_id', 'destination_id'], + dispensers: ['tx_index', 'address_id'], transaction_outputs: ['tx_index', 'vout', 'destination_id'], + }; + + function widenDb(addressIdType) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'decoder-widen-')); + fs.mkdirSync(path.join(root, 'migrations')); + fs.copyFileSync(path.join(__dirname, '..', '..', '..', 'src', 'sql', 'migrations', WIDEN), + path.join(root, 'migrations', WIDEN)); + const rows = []; + for (const [tbl, cols] of Object.entries(PAIRS)) for (const col of cols) { + const type = (tbl === 'pubkeys') ? addressIdType : 'bigint(20) unsigned'; + rows.push({ tbl, col, dataType: type.split('(')[0], columnType: type, + extra: col === 'id' ? 'auto_increment' : '', fkCount: 1n }); + } + const { db, ledgered, executed } = makeDb(root, 'bigint'); + const base = db.getConnection; + db.getConnection = async () => { + const conn = await base(); + const query = conn.query; + return { ...conn, query: async (sql, params) => (/pubkeys_ibfk_1/.test(sql) && /information_schema\.columns/.test(sql)) + ? rows : query(sql, params) }; + }; + return { db, ledgered, executed }; + } + + it('baselines widen-ids at startup on a fresh-shape database', async function () { + const { db, ledgered, executed } = widenDb('bigint(20) unsigned'); + const res = await db.runMigrations(); + assert.deepStrictEqual(res.baselined, [WIDEN]); + assert.deepStrictEqual(ledgered, [WIDEN], 'the ledger row the deploy tool reads is written'); + assert.deepStrictEqual(executed, [], 'no statement of the file ran'); + }); + + it('keeps widen-ids PENDING at startup while a column is still INT', async function () { + const { db, ledgered, executed } = widenDb('int(10) unsigned'); + const res = await db.runMigrations(); + assert.deepStrictEqual(res.pending, [WIDEN]); + assert.deepStrictEqual(ledgered, []); + assert.deepStrictEqual(executed, []); + }); +}); diff --git a/test/unit/migration_runner.test/18_destructive_guard_lock_wait.test.js b/test/unit/migration_runner.test/18_destructive_guard_lock_wait.test.js new file mode 100644 index 00000000..7095409d --- /dev/null +++ b/test/unit/migration_runner.test/18_destructive_guard_lock_wait.test.js @@ -0,0 +1,99 @@ +'use strict'; + +/********************************************************************* + * + * Copyright © 2025–2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC – https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ********************************************************************** + * Schema migration runner: the WAIT n / NOWAIT lock-wait option and unplaced MODIFYs. + * + * MariaDB allows `ALTER TABLE t [WAIT n | NOWAIT] MODIFY ...`. Both the auto-apply scan and + * the live-schema guard must read the MODIFY behind that option, and a MODIFY keyword neither + * can place must refuse the file rather than slip past both checks. + * + ********************************************************************/ + +const assert = require('assert'); + +const Database = require('../../../src/db'); + +const destructiveOf = Database.prototype.destructiveAutoStatement.bind(Database.prototype); +const statementsOf = (raw) => Database.prototype.splitSqlStatements.call(Database.prototype, raw); +const { assertNoLiveColumnLoss, modifyClauses } = require('../../../src/db/migration_live_schema_guard.js'); + +// The guard's verdict on one statement against one live column row (null when it passes). +async function guardVerdict(live, sql) { + const conn = { query: async () => (live ? [live] : []) }; + try { await assertNoLiveColumnLoss(conn, 'f.sql', statementsOf(sql)); return null; } + catch (e) { return e.message; } +} + +describe('Database.destructiveAutoStatement() lock-wait head and unplaced MODIFY @regression', function () { + it('flags a MODIFY of id that drops AUTO_INCREMENT behind WAIT n, NOWAIT or IF EXISTS', function () { + for (const stmt of [ + 'ALTER TABLE t NOWAIT MODIFY id BIGINT UNSIGNED', + 'ALTER TABLE t WAIT 10 MODIFY id BIGINT', + 'ALTER TABLE t WAIT 2.5 MODIFY COLUMN id BIGINT', + 'ALTER TABLE IF EXISTS t MODIFY id BIGINT', + 'ALTER ONLINE TABLE `t` NOWAIT ADD COLUMN c INT, MODIFY id BIGINT', + ]) { + assert.strictEqual(destructiveOf([stmt]), stmt, stmt + ' must not be auto-eligible'); + } + }); + + it('flags a MODIFY keyword that does not open its alter_spec', function () { + for (const stmt of [ + 'ALTER TABLE t FROBNICATE MODIFY id BIGINT NOT NULL AUTO_INCREMENT', + 'ALTER TABLE `db`.`t` WAIT x MODIFY n INT', + 'ALTER TABLE t ADD COLUMN c INT, ALGORITHM=INPLACE MODIFY n INT', + ]) { + assert.strictEqual(destructiveOf([stmt]), stmt, stmt + ' must not be auto-eligible'); + } + }); + + it('keeps lock-wait ALTERs that restate AUTO_INCREMENT or only add auto-eligible', function () { + for (const stmt of [ + 'ALTER TABLE t WAIT 10 MODIFY id BIGINT NOT NULL AUTO_INCREMENT', + 'ALTER TABLE t NOWAIT ADD COLUMN c INT', + 'ALTER TABLE IF EXISTS t NOWAIT MODIFY memo MEDIUMTEXT NULL', + "ALTER TABLE t ADD COLUMN c INT COMMENT 'NOWAIT MODIFY later'", + ]) { + assert.strictEqual(destructiveOf([stmt]), null, stmt + ' must stay auto-eligible'); + } + }); +}); + +describe('migration live-schema MODIFY guard: lock-wait head and unplaced MODIFY @regression', function () { + it('finds the MODIFY behind WAIT n and NOWAIT', function () { + for (const sql of ['ALTER TABLE t NOWAIT MODIFY id BIGINT UNSIGNED;', 'ALTER TABLE t WAIT 30 MODIFY id BIGINT UNSIGNED;']) { + assert.deepStrictEqual(modifyClauses(statementsOf(sql)).map(m => [m.table, m.column, m.definition]), + [['t', 'id', 'BIGINT UNSIGNED']], sql); + } + }); + + it('refuses a lock-wait MODIFY that strips AUTO_INCREMENT or a DEFAULT', async function () { + assert.match(await guardVerdict({ COLUMN_TYPE: 'bigint(20) unsigned', EXTRA: 'auto_increment' }, + 'ALTER TABLE price_snapshots NOWAIT MODIFY id BIGINT UNSIGNED;'), /strips AUTO_INCREMENT/); + assert.match(await guardVerdict({ COLUMN_TYPE: 'tinyint(1)', COLUMN_DEFAULT: '0' }, + 'ALTER TABLE tokens WAIT 10 MODIFY lock_bridge TINYINT(1);'), /strips DEFAULT 0/); + }); + + it('refuses a MODIFY it cannot place instead of skipping it', async function () { + for (const sql of ['ALTER TABLE t FROBNICATE MODIFY n INT;', 'ALTER TABLE `db`.`t` WAIT x MODIFY n INT;']) { + assert.match(await guardVerdict(null, sql), /carries a MODIFY this guard cannot read/, sql); + } + }); + + it('does not count a MODIFY inside a quoted literal', async function () { + assert.strictEqual(await guardVerdict(null, "ALTER TABLE t NOWAIT ADD COLUMN c INT COMMENT 'modify later';"), null); + assert.deepStrictEqual(modifyClauses(statementsOf("ALTER TABLE t ADD COLUMN c INT COMMENT 'x, MODIFY y';")), []); + }); +}); diff --git a/test/unit/parse_loop_quarantine.test.js b/test/unit/parse_loop_quarantine.test.js index 403bb24f..dd2ee2ee 100644 --- a/test/unit/parse_loop_quarantine.test.js +++ b/test/unit/parse_loop_quarantine.test.js @@ -113,6 +113,7 @@ describe('XChainDecoder parse-loop quarantine', function () { assert.strictEqual(calls.insertBlock, 1, 'the block should be inserted on the successful retry') assert.strictEqual(calls.endTransaction, 1, 'the failed attempt should roll back') assert.strictEqual(decoder.parseErrors, 1) + assert.strictEqual(decoder.quarantinedTxs, 0, 'a retried block skips nothing') assert.strictEqual(calls.insertEvent.length, 0, 'a block-level failure is never quarantined') }) }) @@ -138,6 +139,7 @@ describe('XChainDecoder parse-loop quarantine', function () { assert.strictEqual(calls.endTransaction, 1, 'the failed attempt should roll back') assert.strictEqual(calls.commitTransaction, 1, 'the block should commit on the retry') assert.strictEqual(calls.insertEvent.length, 0, 'a transiently failing tx must NOT be quarantined') + assert.strictEqual(decoder.quarantinedTxs, 0, 'a retried tx is not counted as skipped') }) }) @@ -164,6 +166,9 @@ describe('XChainDecoder parse-loop quarantine', function () { assert.strictEqual(calls.insertEvent[0].data.tx_hash, 'cafe01') assert.strictEqual(calls.insertEvent[0].data.block_index, 0) assert.strictEqual(calls.insertEvent[0].data.error, 'poison transaction') + assert.strictEqual(decoder.quarantinedTxs, 1, 'the skip has its own count, apart from parse_errors') + assert.strictEqual(decoder.lastQuarantinedBlock, 0) + assert.strictEqual(decoder.lastQuarantinedTxHash, 'cafe01') }) }) @@ -176,9 +181,11 @@ describe('XChainDecoder parse-loop quarantine', function () { ) decoder.sleep = async () => {} + // Each stub tx hashes to the txid the node listed for it, so both reach the + // parser (a tx whose id differs from the node txid is skipped before it). decoder.connector = { - getRawMempool: async () => ['aa', 'bb'], - getRawTransactions: async () => ['deadbeef01', 'deadbeef02'] + getRawMempool: async () => ['deadbeef01', 'deadbeef02'], + getRawTransactions: async (txids) => txids } decoder.xchainBlockDecoder = { transactionFromHex: (hex) => ({ ins: [{}], outs: [], getId: () => hex }) @@ -201,7 +208,7 @@ describe('XChainDecoder parse-loop quarantine', function () { await decoder.updateMempool() - assert.deepStrictEqual(parsed, ['deadbeef01', 'deadbeef02'], 'the second tx must still be processed after the first throws') + assert.deepStrictEqual(parsed.slice().sort(), ['deadbeef01', 'deadbeef02'], 'the second tx must still be processed after the first throws') assert.strictEqual(decoder.parseErrors, 1) assert.strictEqual(decoder.mempoolBusy, false, 'the busy flag must be released') }) @@ -360,5 +367,8 @@ describe('quarantine retry state is per tx position and per block', function () assert.strictEqual(calls.commit, 1, 'the block commits') assert.deepStrictEqual([...new Set(quarantinedHashes(calls))], ['cafe01'], 'only the poison row is quarantined') assert.deepStrictEqual(calls.inserted, ['beef02'], 'the second tx is inserted on its retry') + assert.ok(decoder.quarantinedTxs >= 1, 'an INSERT quarantine is counted as a skip') + assert.strictEqual(decoder.quarantinedTxs, quarantinedHashes(calls).length, 'one count per recorded PARSE_ERROR') + assert.strictEqual(decoder.lastQuarantinedTxHash, 'cafe01') }) }) diff --git a/test/unit/reorg_halt_clear.test/01_clear_confirms_on_read_back.test.js b/test/unit/reorg_halt_clear.test/01_clear_confirms_on_read_back.test.js new file mode 100644 index 00000000..d1b7b5ab --- /dev/null +++ b/test/unit/reorg_halt_clear.test/01_clear_confirms_on_read_back.test.js @@ -0,0 +1,63 @@ +/********************************************************************* + * + * Copyright © 2025-2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC - https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ********************************************************************** + * The audited REORG_HALT clear reports cleared only on what a post-write + * read sees, never on the write's ack. + */ + +'use strict' + +const assert = require('assert') +const sinon = require('sinon') +const Database = require('../../../src/db.js') + +function dbAnswering(handler) { + const db = new Database('127.0.0.1', 3306, 'xchain_btc_mainnet', 'u', 'p') + const query = sinon.stub().callsFake(async (sql, params) => handler(sql, params)) + db.pool = { getConnection: sinon.stub().resolves({ query, release: sinon.stub().resolves() }) } + return { db, query } +} + +const halt = (id) => ({ id, time: '2026-09-07 06:29:07', code: 'REORG_HALT', data: JSON.stringify({ reason: 'safe-depth', at: '2026-09-07T06:29:07Z' }) }) + +// The INSERT binds the code as a parameter, so a SELECT naming REORG_HALT_CLEARED +// in its text is always a halt-state read. +const stateReads = (query) => query.getCalls() + .map(c => String(c.args[0])) + .filter(s => /SELECT[\s\S]*REORG_HALT_CLEARED/.test(s)).length + +describe('Database: the newest REORG_HALT / REORG_HALT_CLEARED row decides', function () { + afterEach(() => sinon.restore()) + + it('clearReorgHalt reports not-cleared when the write lands but the read-back still shows a halt', async function () { + const { db, query } = dbAnswering((sql) => { + if (/INSERT INTO events/.test(sql)) return { affectedRows: 1 } + return [halt(7)] + }) + const res = await db.clearReorgHalt({ reason: 'long enough reason', expectedHaltId: 7 }) + assert.deepStrictEqual(res, { cleared: false, alreadyClear: false }) + const inserts = query.getCalls().filter(c => /INSERT INTO events/.test(String(c.args[0]))).length + assert.strictEqual(inserts, 1, 'the clear row was written') + assert.strictEqual(stateReads(query), 2, 'the pre-check read and the post-write read-back both ran') + }) + + it('clearReorgHalt refuses a DUPLICATED_TRANSACTION write (errno 1062) without a read-back', async function () { + const { db, query } = dbAnswering((sql) => { + if (/INSERT INTO events/.test(sql)) throw Object.assign(new Error('Duplicate entry'), { errno: 1062 }) + return [halt(7)] + }) + const res = await db.clearReorgHalt({ reason: 'long enough reason', expectedHaltId: 7 }) + assert.deepStrictEqual(res, { cleared: false, alreadyClear: false }) + assert.strictEqual(stateReads(query), 1, 'a write that is not exactly true refuses before any read-back') + }) +}) diff --git a/test/unit/reorg_halt_park.test.js b/test/unit/reorg_halt_park.test.js index ac87ece9..6eaebc39 100644 --- a/test/unit/reorg_halt_park.test.js +++ b/test/unit/reorg_halt_park.test.js @@ -290,10 +290,10 @@ describe('a parked decoder resumes on the clear, with no restart', function () { let probes = 0 let halted = true const { decoder } = buildDecoder({ - // Once the operator has cleared, the node is also past our tip again, so the - // resumed loop leaves the reconcile branch and reaches the parse path. - nodeTips: [NODE_TIP, STORED_TIP + 50], - maxSleeps: 6, + // The low tip is read twice (a lower tip is reconciled only once two readings agree, + // one more sleep); once cleared the node is past our tip, so the loop reaches the parse path. + nodeTips: [NODE_TIP, NODE_TIP, STORED_TIP + 50], + maxSleeps: 7, dbOverrides: { isReorgHalted: async () => { probes++; if (probes >= 3) halted = false; return halted } } diff --git a/test/unit/reorg_halt_surface.test.js b/test/unit/reorg_halt_surface.test.js index 478d2c95..c3690280 100644 --- a/test/unit/reorg_halt_surface.test.js +++ b/test/unit/reorg_halt_surface.test.js @@ -249,9 +249,13 @@ describe('Database.markReorgHalted reports the row, not the ack', function () { function stubMarkerDb({ alreadyHalted = false, insertResult = true, readBack = true, readBackThrows = false } = {}) { const db = new Database('127.0.0.1', 3306, 'test_db', 'u', 'p') const calls = { insert: 0, probes: 0 } + // The entry probe reads the full state; the read-back after the write asks isReorgHalted. + db.readReorgHaltState = async () => { + calls.probes++ + return { halted: alreadyHalted } + } db.isReorgHalted = async () => { calls.probes++ - if (calls.probes === 1) return alreadyHalted if (readBackThrows) throw new Error('connection lost during read-back') return readBack } diff --git a/test/unit/reorg_halt_surface.test/01_empty_probe_answer_keeps_the_halt.test.js b/test/unit/reorg_halt_surface.test/01_empty_probe_answer_keeps_the_halt.test.js new file mode 100644 index 00000000..4be528a1 --- /dev/null +++ b/test/unit/reorg_halt_surface.test/01_empty_probe_answer_keeps_the_halt.test.js @@ -0,0 +1,75 @@ +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +// A halt probe that exists but answers with nothing is a fault, not a clear: the +// cached halt must stand, and checked_at must not move, exactly as for a thrown read. + +const assert = require('assert') +const XChainDecoder = require('../../../src/XChainDecoder') + +function makeDecoder() { + return new XChainDecoder( + 'bitcoin-regtest', 'h', '0', 'db', 'u', 'p', 'h', '0', 'u', 'p', false, null + ) +} + +function haltedThenEmpty(empty) { + let answer = { halted: true, at: null, reason: 'over-deep rollback' } + const decoder = makeDecoder() + decoder.db = { getReorgHaltMarker: async () => answer } + return { decoder, goEmpty: () => { answer = empty } } +} + +describe('XChainDecoder latent REORG_HALT reporting: an empty probe answer', function () { + + for (const empty of [null, undefined]) { + it('keeps a halt read earlier when getReorgHaltMarker resolves ' + String(empty), async function () { + const { decoder, goEmpty } = haltedThenEmpty(empty) + await decoder.checkReorgHalt({ force: true }) + const checkedAt = decoder.getReorgHaltStatus().checked_at + goEmpty() + const status = await decoder.checkReorgHalt({ force: true }) + assert.strictEqual(status.halted, true, 'an empty answer must never read as an operator clear') + assert.strictEqual(status.reason, 'over-deep rollback') + assert.strictEqual(status.checked_at, checkedAt, 'an empty answer is not a successful probe') + }) + } + + it('leaves a fresh decoder unchecked when getReorgHaltMarker resolves null', async function () { + const decoder = makeDecoder() + decoder.db = { getReorgHaltMarker: async () => null } + const status = await decoder.checkReorgHalt({ force: true }) + assert.strictEqual(status.halted, false) + assert.strictEqual(status.checked_at, null, '"never answered" must stay distinguishable from "checked and clear"') + }) + + it('keeps a parked decoder parked when the probe starts answering null', async function () { + const { decoder, goEmpty } = haltedThenEmpty(null) + decoder.logError = () => {} + decoder.log = () => {} + await decoder.checkReorgHalt({ force: true }) + decoder.parkOnReorgHalt('safe-depth window exceeded', 100) + goEmpty() + decoder.reorgHaltCheckedAt = 0 + assert.strictEqual(await decoder.resumeFromReorgHaltPark(), false) + assert.strictEqual(decoder.getReorgHaltStatus().parked, true) + assert.strictEqual(decoder.getReorgHaltStatus().halted, true) + }) + + it('keeps a halt read earlier when the boolean isReorgHalted probe resolves undefined', async function () { + let answer = true + const decoder = makeDecoder() + decoder.db = { isReorgHalted: async () => answer } + await decoder.checkReorgHalt({ force: true }) + answer = undefined + const status = await decoder.checkReorgHalt({ force: true }) + assert.strictEqual(status.halted, true) + }) +}) diff --git a/test/unit/verify_reorg_retry.test/04_settle_the_node_tip_before_a_rollback.test.js b/test/unit/verify_reorg_retry.test/04_settle_the_node_tip_before_a_rollback.test.js new file mode 100644 index 00000000..f7aeff13 --- /dev/null +++ b/test/unit/verify_reorg_retry.test/04_settle_the_node_tip_before_a_rollback.test.js @@ -0,0 +1,205 @@ +/********************************************************************* + * + * Copyright © 2025-2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC - https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ********************************************************************** + * A single stale tip reading is not a rollback. + * + * One getblockchaininfo reply that runs behind the real tip (an RPC failover + * onto a lagging fallback endpoint is the realistic source) must not send the + * decoder into verifyReorg: acting on it would delete valid blocks and write + * REORG markers (one reply 3 low, 3 of each; one reply 100 low, 100), and those + * markers feed the indexer, so the marker stream would follow RPC timing + * instead of chain history. A lower tip (and a swapped hash at the stored + * height) is acted on only once two readings TIP_SETTLE_DELAY_MS apart agree, the + * rule xchain-utxo-tracker already applies; a real, settled reorg rolls back + * exactly what it did before. + */ + +'use strict' + +const assert = require('assert') +const XChainDecoder = require('../../../src/XChainDecoder') +const { refreshChainTip } = require('../../../src/XChainDecoder/tip_refresh.js') +const { waitAtTip } = require('../../../src/XChainDecoder/sync_loop.js') + +function makeDecoder() { + const decoder = new XChainDecoder( + 'bitcoin-regtest', 'h', '0', 'db', 'u', 'p', 'h', '0', 'u', 'p', false, null + ) + decoder.startBlockIndex = 0 + decoder.chainGenesisHash = null + decoder.sleep = async () => {} + return decoder +} + +const info = (blocks) => ({ chain: 'regtest', blocks, verificationprogress: 1, initialblockdownload: false }) + +// A stored chain 0..top whose hashes agree with the node at and below `nodeTip` +// (and, above it, belong to a branch the node no longer has). `tips` is the run +// of getblockchaininfo heights the node answers in turn; the last one repeats. +function decoderOver({ top, nodeTip, tips, forkAt = null }) { + const decoder = makeDecoder() + const state = { top, deleted: [], markers: [], tipReads: 0 } + const stored = (h) => (forkAt != null && h > forkAt ? 'old' + h : 'hash' + h) + decoder.connector = { + getBlockchainInfo: async () => info(tips[Math.min(state.tipReads++, tips.length - 1)]), + getBlockHash: async (h) => { + if (h > nodeTip) throw new Error('Block height out of range') + return 'hash' + h + } + } + decoder.db = { + endTransaction: async () => {}, + getLastBlockIndex: async () => state.top, + getLastTxIndex: async () => 0, + getBlockByIndex: async (h) => (h < 0 || h > state.top ? null : { block_index: h, block_hash: stored(h) }), + // The REORG marker commits with its delete, so the two lists move together. + deleteBlockByIndex: async (h, hash) => { state.deleted.push(h); state.markers.push({ block_index: h, block_hash: hash }); state.top = h - 1; return true }, + isReorgHalted: async () => false, + countReorgDeletesAboveTip: async () => 0, + markReorgHalted: async () => true + } + return { decoder, state } +} + +// One parse-loop pass at the stored tip, as startup.js runs it. +async function onePass(decoder, top) { + const loop = { lastProcessedBlockIndex: top } + const result = await refreshChainTip.call(decoder, loop, top) + return { loop, result } +} + +describe('a node tip below the stored tip is settled before any rollback [REGRESSION P1]', function () { + + it('one stale reply 3 below the real tip deletes nothing and writes no REORG marker', async function () { + const { decoder, state } = decoderOver({ top: 200, nodeTip: 200, tips: [197, 200] }) + const { result } = await onePass(decoder, 200) + assert.strictEqual(result, 'continue') + assert.deepStrictEqual(state.deleted, [], 'valid blocks rolled back on one stale reading') + assert.deepStrictEqual(state.markers, []) + assert.strictEqual(decoder.blockchainInfoLastBlock, 200, 'the settled reading replaces the stale one') + }) + + it('one stale reply 100 below the real tip deletes nothing and writes no REORG marker', async function () { + const { decoder, state } = decoderOver({ top: 200, nodeTip: 200, tips: [100, 200] }) + await onePass(decoder, 200) + assert.deepStrictEqual(state.deleted, []) + assert.deepStrictEqual(state.markers, []) + assert.strictEqual(decoder.blockchainInfoLastBlock, 200) + }) + + it('a tip that keeps moving ends the pass with nothing deleted', async function () { + const { decoder, state } = decoderOver({ top: 200, nodeTip: 200, tips: [190, 191, 192, 193, 194, 195, 196] }) + const { result } = await onePass(decoder, 200) + assert.strictEqual(result, 'continue') + assert.deepStrictEqual(state.deleted, []) + assert.strictEqual(state.tipReads, 6, 'the first reading plus five re-reads, then give up') + }) + + it('an unreadable re-read ends the pass with nothing deleted', async function () { + const { decoder, state } = decoderOver({ top: 200, nodeTip: 194, tips: [194] }) + let reads = 0 + decoder.connector.getBlockchainInfo = async () => { + if (reads++ === 0) return info(194) + throw new Error('ECONNREFUSED') + } + await onePass(decoder, 200) + assert.deepStrictEqual(state.deleted, []) + }) +}) + +describe('a settled lower tip rolls back exactly as before [REGRESSION P1]', function () { + + it('a settled real rollback deletes and marks exactly what it did before the settle', async function () { + // Today's behaviour, measured by calling verifyReorg with the same tip directly. + const direct = decoderOver({ top: 200, nodeTip: 194, tips: [194] }) + await direct.decoder.verifyReorg(194) + + const { decoder, state } = decoderOver({ top: 200, nodeTip: 194, tips: [194] }) + const { loop } = await onePass(decoder, 200) + assert.deepStrictEqual(state.deleted, [200, 199, 198, 197, 196, 195]) + assert.deepStrictEqual(state.deleted, direct.state.deleted) + assert.deepStrictEqual(state.markers, direct.state.markers) + assert.strictEqual(loop.lastProcessedBlockIndex, 194, 'the cursor is re-derived after the rollback') + assert.strictEqual(decoder.reorgCount, 1) + }) + + it('a settled real reorg below a lower tip still walks back to the fork point', async function () { + // Node dropped to 196 on a branch that forked at 193: 200..197 are above the + // tip, 196..194 differ by hash, 193 agrees. + const direct = decoderOver({ top: 200, nodeTip: 196, tips: [196], forkAt: 193 }) + await direct.decoder.verifyReorg(196) + + const { decoder, state } = decoderOver({ top: 200, nodeTip: 196, tips: [196], forkAt: 193 }) + await onePass(decoder, 200) + assert.deepStrictEqual(state.deleted, [200, 199, 198, 197, 196, 195, 194]) + assert.deepStrictEqual(state.markers, direct.state.markers) + }) +}) + +describe('the mid-walk tip refresh is settled before it moves the tip down [REGRESSION P1]', function () { + + it('one stale reading during a walk does not turn valid blocks into above-tip deletes', async function () { + // Stored 0..105 all agree with a node at 105. A transient getBlockHash fault at + // 105 sends the walk to its tip refresh, which answers 102 once. + const { decoder, state } = decoderOver({ top: 105, nodeTip: 105, tips: [102, 105] }) + let faults = 0 + const real = decoder.connector.getBlockHash + decoder.connector.getBlockHash = async (h) => { + if (h === 105 && faults++ === 0) throw new Error('ETIMEDOUT') + return real(h) + } + assert.strictEqual(await decoder.verifyReorg(105), true) + assert.deepStrictEqual(state.deleted, []) + }) + + it('a settled lower tip mid-walk still drains the blocks above it', async function () { + const { decoder, state } = decoderOver({ top: 105, nodeTip: 102, tips: [102] }) + assert.strictEqual(await decoder.verifyReorg(105), true) + assert.deepStrictEqual(state.deleted, [105, 104, 103]) + }) +}) + +describe('an equal-height tip swap is settled before the reconcile [REGRESSION P1]', function () { + + function atTip(hashes) { + const { decoder, state } = decoderOver({ top: 200, nodeTip: 200, tips: [200] }) + let reads = 0 + const real = decoder.connector.getBlockHash + decoder.connector.getBlockHash = async (h) => { + if (h === 200 && reads < hashes.length) return hashes[reads++] + return real(h) + } + decoder.mempoolInterval = 1 + decoder.blockchainInfoLastBlock = 200 + const loop = { lastProcessedBlockIndex: 200, lastBlockchainInfoRefreshAt: 2, tipHashCheckedAt: 1 } + let reconciles = 0 + const verifyReorg = decoder.verifyReorg.bind(decoder) + decoder.verifyReorg = async (tip) => { reconciles++; return verifyReorg(tip) } + return { decoder, state, loop, reconciles: () => reconciles } + } + + it('one stray hash at the stored height does not start a reconcile', async function () { + const { decoder, state, loop, reconciles } = atTip(['stray200']) + await waitAtTip.call(decoder, loop) + assert.strictEqual(reconciles(), 0) + assert.deepStrictEqual(state.deleted, []) + }) + + it('a swap that holds across two readings reconciles as before', async function () { + // The node's block at 200 really is different: every read answers 'new200'. + const { decoder, state, loop, reconciles } = atTip(['new200', 'new200', 'new200', 'new200']) + await waitAtTip.call(decoder, loop) + assert.strictEqual(reconciles(), 1) + assert.deepStrictEqual(state.deleted, [200]) + }) +}) diff --git a/test/unit/wire_parse/auxpow_stripped_block_integrity.test.js b/test/unit/wire_parse/auxpow_stripped_block_integrity.test.js new file mode 100644 index 00000000..fbdb3c23 --- /dev/null +++ b/test/unit/wire_parse/auxpow_stripped_block_integrity.test.js @@ -0,0 +1,137 @@ +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +// A wrong AuxPoW cut must never reach the block decoder as a block. Dogecoin +// mainnet block 4500000 is rebuilt byte-exact from real chain data +// (test/fixtures/wire_parse_vectors.json), then single-byte changes are made in +// its AuxPoW section. Each one moves skipAuxPow's cut, and bitcoinjs +// Block.fromBuffer checks neither the merkle root nor trailing bytes, so the +// stripped hex can read as a block holding zero transactions. The strip must +// refuse every such result inside the AuxPoW try, where the refusal is tagged +// auxPowParseFailure and fetchBlockHex can escalate to per-tx reassembly. +// The plain path (no AuxPoW strip) gets the same three checks in blockFromBuffer. + +const assert = require('assert') +const crypto = require('crypto') +const bitcoinjs = require('bitcoinjs-lib') +const BlockchainConnector = require('../../../src/chain/blockchain_connector') +const { stripAuxPowFromBlockHex } = require('../../../src/chain/blockchain_connector/auxpow_codec.js') +const XChainBlockDecoder = require('../../../src/chain/XChainBlockDecoder') +const VECTORS = require('../../fixtures/wire_parse_vectors.json') + +const DOGE = VECTORS.doge4500000 +const BODY = '03' + DOGE.txs.join('') +const NODE_BLOCK = DOGE.header + DOGE.auxpow + BODY +const PURE_BLOCK = DOGE.header + BODY + +const AUX = Buffer.from(DOGE.auxpow, 'hex') +const COINBASE_LENGTH = bitcoinjs.Transaction.fromBuffer(AUX, true).byteLength() +const CB_BRANCH_COUNT_AT = COINBASE_LENGTH + 32 +const CHAIN_BRANCH_COUNT_AT = CB_BRANCH_COUNT_AT + 1 + AUX[CB_BRANCH_COUNT_AT] * 32 + 4 + +function withAuxByte(offset, value) { + const aux = Buffer.from(AUX) + aux[offset] = value + return DOGE.header + aux.toString('hex') + BODY +} + +// Every entry is a node answer whose AuxPoW cut lands off the real boundary while +// skipAuxPow still traverses it; the strip refuses each one. +const MALFORMED = { + 'chain branch count raised by one': withAuxByte(CHAIN_BRANCH_COUNT_AT, AUX[CHAIN_BRANCH_COUNT_AT] + 1), + 'chain branch count lowered by one': withAuxByte(CHAIN_BRANCH_COUNT_AT, AUX[CHAIN_BRANCH_COUNT_AT] - 1), + 'coinbase branch count raised by one': withAuxByte(CB_BRANCH_COUNT_AT, AUX[CB_BRANCH_COUNT_AT] + 1), + 'one zero byte after the AuxPoW section': DOGE.header + DOGE.auxpow + '00' + BODY, + 'eight zero bytes after the AuxPoW section': DOGE.header + DOGE.auxpow + '00'.repeat(8) + BODY, + 'AuxPoW section with nothing after it': DOGE.header + DOGE.auxpow, + 'parent header dropped': DOGE.header + DOGE.auxpow.slice(0, -160) + BODY, +} + +function connectorServing(blockHex, headerHex) { + const connector = new BlockchainConnector('127.0.0.1', 0, 'user', 'pass') + connector.getBlockHeader = async () => headerHex + connector.getBlock = async () => blockHex + return connector +} + +describe('AuxPoW stripped-block integrity @regression', function () { + describe('a real merge-mined block [REGRESSION P1]', function () { + it('the fixture header hashes to the real block hash', function () { + const hash = crypto.createHash('sha256').update(crypto.createHash('sha256') + .update(Buffer.from(DOGE.header, 'hex')).digest()).digest().reverse().toString('hex') + assert.strictEqual(hash, DOGE.hash) + }) + + it('strips to the pure block when getblockheader returns the 80-byte header', function () { + assert.strictEqual(stripAuxPowFromBlockHex(DOGE.header, NODE_BLOCK), PURE_BLOCK) + }) + + it('strips to the pure block when getblockheader carries the AuxPoW bytes', function () { + assert.strictEqual(stripAuxPowFromBlockHex(DOGE.header + DOGE.auxpow, NODE_BLOCK), PURE_BLOCK) + }) + + it('decodes the stripped block to the real txids', function () { + const block = new XChainBlockDecoder('dogecoin-mainnet').blockFromHex(PURE_BLOCK) + assert.deepStrictEqual(block.transactions.map(t => t.getId()), DOGE.txids) + }) + }) +}) + +describe('AuxPoW stripped-block integrity @regression', function () { + describe('a malformed AuxPoW section is refused, never a block [REGRESSION P1]', function () { + for (const [name, hex] of Object.entries(MALFORMED)) { + it(`${name}: the strip throws instead of returning a block`, function () { + assert.throws(() => stripAuxPowFromBlockHex(DOGE.header, hex), /AuxPoW parse:/) + }) + + it(`${name}: getBlockWithoutAuxPow rejects tagged auxPowParseFailure`, async function () { + await assert.rejects(() => connectorServing(hex, DOGE.header).getBlockWithoutAuxPow(DOGE.hash), + err => err.auxPowParseFailure === true) + }) + } + + it('the legacy long-header path refuses a cut that leaves trailing bytes', function () { + // An older daemon's header that is one byte short of the real AuxPoW size + // makes the length-delta cut land one byte early. + const shortHeader = DOGE.header + DOGE.auxpow.slice(0, -2) + assert.throws(() => stripAuxPowFromBlockHex(shortHeader, NODE_BLOCK), /AuxPoW parse:/) + }) + }) +}) + +describe('AuxPoW stripped-block integrity @regression', function () { + describe('the plain path checks the same three things [REGRESSION P1]', function () { + const decoders = ['dogecoin-mainnet', 'bitcoin-mainnet', 'litecoin-mainnet'] + for (const network of decoders) { + it(`${network}: refuses a tx count of zero with transactions trailing`, function () { + const hex = DOGE.header + '00' + DOGE.txs.join('') + assert.throws(() => new XChainBlockDecoder(network).blockFromHex(hex), /no transactions/) + }) + + it(`${network}: refuses a merkle root that does not match the transactions`, function () { + const hex = DOGE.header + '02' + DOGE.txs.slice(0, 2).join('') + assert.throws(() => new XChainBlockDecoder(network).blockFromHex(hex), /merkle root/) + }) + + it(`${network}: accepts the real body and keeps the 80-byte header-only parse`, function () { + const decoder = new XChainBlockDecoder(network) + assert.deepStrictEqual(decoder.blockFromHex(PURE_BLOCK).transactions.map(t => t.getId()), DOGE.txids) + assert.strictEqual(decoder.blockFromHex(DOGE.header).transactions, undefined) + }) + } + + for (const network of ['dogecoin-mainnet', 'bitcoin-mainnet']) { + it(`${network}: refuses bytes after the last transaction`, function () { + assert.throws(() => new XChainBlockDecoder(network).blockFromHex(PURE_BLOCK + '00'), + /after its last transaction/) + }) + } + }) +}) diff --git a/test/unit/wire_parse/mempool_node_txid.test.js b/test/unit/wire_parse/mempool_node_txid.test.js new file mode 100644 index 00000000..edd18c46 --- /dev/null +++ b/test/unit/wire_parse/mempool_node_txid.test.js @@ -0,0 +1,84 @@ +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +// A mempool row is keyed on the txid the node listed in getrawmempool, and the +// parsed tx must hash to that same txid before it is stored. A Litecoin tx in +// its MWEB serialization (real peg-in bytes, opaque MWEB filler) must therefore +// store under its real txid with its witness intact, and a tx whose parse hashes +// to anything else is skipped and counted, never stored under a made-up hash. + +const assert = require('assert') +const crypto = require('crypto') +const bitcoinjs = require('bitcoinjs-lib') +const XChainDecoder = require('../../../src/XChainDecoder') +const VECTORS = require('../../fixtures/wire_parse_vectors.json') + +const PEGIN = VECTORS.ltcPegin + +function mwebForm(hex) { + const full = Buffer.from(hex, 'hex') + const bare = bitcoinjs.Transaction.fromBuffer(full) + bare.ins.forEach(i => { i.witness = [] }) + const legacy = bare.toBuffer() + const insOuts = legacy.subarray(4, legacy.length - 4) + const body = Buffer.concat([Buffer.from([0x01]), crypto.createHash('sha512').update('mweb body').digest()]) + return Buffer.concat([full.subarray(0, 4), Buffer.from([0x00, 0x09]), insOuts, + full.subarray(6 + insOuts.length, full.length - 4), body, full.subarray(full.length - 4)]).toString('hex') +} + +function buildDecoder(nodeTxid, nodeHex) { + const decoder = new XChainDecoder( + 'litecoin-regtest', 'h', '0', 'db', 'u', 'p', 'h', '0', 'u', 'p', false, null + ) + const seen = { rows: [], parsed: [] } + decoder.mempoolDb = { + getAllOpenDispenserAddresses: async () => new Set(), + deleteAndCompareTxsNotInList: async () => ({ transactionsDeleted: 0 }), + insertMempoolTransaction: async (row) => { seen.rows.push(row); return true }, + } + decoder.connector = { + getRawMempool: async () => [nodeTxid], + getRawTransactions: async () => [nodeHex], + } + decoder.parseTransaction = async (tx) => { + seen.parsed.push(tx) + return { data: Buffer.from('SEND'), compiledDataLength: 4, source: 'src', destination: null, amount: '0' } + } + return { decoder, seen } +} + +describe('decoder mempool rows keyed on the node txid @regression', function () { + describe('an MWEB-form Litecoin tx [REGRESSION P1]', function () { + it('stores under the node txid with the witness handed to the parser', async function () { + const { decoder, seen } = buildDecoder(PEGIN.txid, mwebForm(PEGIN.hex)) + await decoder.updateMempool() + assert.deepStrictEqual(seen.rows.map(r => r.hash), [PEGIN.txid]) + assert.ok(seen.parsed[0].ins[0].witness.length > 0, 'the witness must survive the parse') + }) + + it('skips and counts a tx whose parse does not hash to the node txid', async function () { + const { decoder, seen } = buildDecoder('ab'.repeat(32), PEGIN.hex) + const before = decoder.parseErrors + await decoder.updateMempool() + assert.deepStrictEqual(seen.rows, []) + assert.deepStrictEqual(seen.parsed, []) + assert.strictEqual(decoder.parseErrors, before + 1) + }) + + it('handles a typed malformed-transaction error without aborting the mempool cycle', async function () { + const { decoder, seen } = buildDecoder('cd'.repeat(32), 'not-hex') + const before = decoder.parseErrors + await decoder.updateMempool() + assert.deepStrictEqual(seen.rows, []) + assert.deepStrictEqual(seen.parsed, []) + assert.strictEqual(decoder.parseErrors, before + 1) + }) + }) +}) diff --git a/test/unit/wire_parse/mweb_transaction_serialization.test.js b/test/unit/wire_parse/mweb_transaction_serialization.test.js new file mode 100644 index 00000000..2e5cc86c --- /dev/null +++ b/test/unit/wire_parse/mweb_transaction_serialization.test.js @@ -0,0 +1,135 @@ +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +// Litecoin's getrawtransaction answers in the MWEB serialization (litecoin +// src/primitives/transaction.h SerializeTransaction): version | 0x00 flags | +// vin | vout | witness when flags & 1 | mweb_tx when flags & 8 | nLockTime, +// where mweb_tx is an OptionalPtr (tag 0x00 for none, else a body). Each form +// must parse to Litecoin's own txid (no witness, no MWEB) with its real nLockTime, +// and keep its witness, where a P2WSH carrier keeps its payload; dropping only the +// marker and flag bytes reads the witness or MWEB tag as nLockTime. The vectors are real mainnet bytes +// (test/fixtures/wire_parse_vectors.json); the MWEB body is opaque to both +// services, so a fixed filler stands in for it. + +const assert = require('assert') +const crypto = require('crypto') +const bitcoinjs = require('bitcoinjs-lib') +const XChainBlockDecoder = require('../../../src/chain/XChainBlockDecoder') +const VECTORS = require('../../fixtures/wire_parse_vectors.json') + +const PEGIN = VECTORS.ltcPegin +const HOGEX = VECTORS.ltcHogEx3195200 +const MWEB_BODY = Buffer.concat([Buffer.from([0x01]), crypto.createHash('sha512').update('mweb body').digest()]) +const dsha = (b) => crypto.createHash('sha256').update(crypto.createHash('sha256').update(b).digest()).digest() +const txidOf = (b) => Buffer.from(dsha(b)).reverse().toString('hex') + +// The pieces SerializeTransaction writes, cut from a real segwit tx. +function piecesOf(hex) { + const full = Buffer.from(hex, 'hex') + const tx = bitcoinjs.Transaction.fromBuffer(full) + const bare = tx.clone() + bare.ins.forEach(i => { i.witness = [] }) + const legacy = bare.toBuffer() + const insOuts = legacy.subarray(4, legacy.length - 4) + return { + version: full.subarray(0, 4), + insOuts, + witness: full.subarray(6 + insOuts.length, full.length - 4), + lockTime: full.subarray(full.length - 4), + tx, + } +} + +// The getrawtransaction form of a tx that carries an MWEB part. +function mwebForm(hex, { withWitness }) { + const p = piecesOf(hex) + return Buffer.concat([p.version, Buffer.from([0x00, withWitness ? 0x09 : 0x08]), p.insOuts, + withWitness ? p.witness : Buffer.alloc(0), MWEB_BODY, p.lockTime]).toString('hex') +} + +// The real HogEx with its nLockTime replaced, and the txid Litecoin gives it +// (ComputeHash serializes without witness and without MWEB). +function hogExWithLockTime(lockTime) { + const raw = Buffer.from(HOGEX.hex, 'hex') + const lt = Buffer.alloc(4) + lt.writeUInt32LE(lockTime) + const hex = Buffer.concat([raw.subarray(0, raw.length - 4), lt]) + const plain = Buffer.concat([raw.subarray(0, 4), raw.subarray(6, raw.length - 5), lt]) + return { hex: hex.toString('hex'), txid: txidOf(plain) } +} + +const ltc = () => new XChainBlockDecoder('litecoin-mainnet') + +describe('Litecoin MWEB transaction serialization @regression', function () { + describe('a standalone tx in its MWEB form [REGRESSION P1]', function () { + it('a segwit peg-in with an MWEB part keeps its real txid, witness and locktime', function () { + const p = piecesOf(PEGIN.hex) + const tx = ltc().transactionFromHex(mwebForm(PEGIN.hex, { withWitness: true })) + assert.strictEqual(tx.getId(), PEGIN.txid) + assert.deepStrictEqual(tx.ins.map(i => i.witness), p.tx.ins.map(i => i.witness)) + assert.strictEqual(tx.locktime, p.tx.locktime) + assert.strictEqual(tx.toHex(), PEGIN.hex) + }) + + it('the same tx without its witness (flag 0x08) keeps its real txid', function () { + const tx = ltc().transactionFromHex(mwebForm(PEGIN.hex, { withWitness: false })) + assert.strictEqual(tx.getId(), PEGIN.txid) + assert.strictEqual(tx.ins[0].witness.length, 0) + }) + + it('the real HogEx (null MWEB tag, zero locktime) keeps its txid', function () { + assert.strictEqual(ltc().transactionFromHex(HOGEX.hex).getId(), HOGEX.txid) + }) + + it('a HogEx with a nonzero locktime gets the txid Litecoin computes', function () { + const h = hogExWithLockTime(3195200) + const tx = ltc().transactionFromHex(h.hex) + assert.strictEqual(tx.getId(), h.txid) + assert.strictEqual(tx.locktime, 3195200) + }) + + it('a plain segwit tx parses exactly as before', function () { + assert.strictEqual(ltc().transactionFromHex(PEGIN.hex).getId(), PEGIN.txid) + }) + }) +}) + +// A block's last tx is the HogEx; the MWEB extension block follows it. +function ltcBlock(lastTxHex, extension) { + const coinbase = Buffer.from(VECTORS.doge4500000.txs[0], 'hex') + const last = Buffer.from(lastTxHex, 'hex') + const header = Buffer.alloc(80) + header.writeInt32LE(0x20000000) + // Merkle root over the coinbase and the last tx's Litecoin txid. + const ids = [dsha(coinbase), dsha(Buffer.concat([last.subarray(0, 4), last.subarray(6, last.length - 5), last.subarray(last.length - 4)]))] + dsha(Buffer.concat(ids)).copy(header, 36) + return Buffer.concat([header, Buffer.from([0x02]), coinbase, last, extension]).toString('hex') +} + +describe('Litecoin MWEB transaction serialization @regression', function () { + describe('the HogEx inside a block [REGRESSION P1]', function () { + it('the real HogEx keeps its txid with the extension block after it', function () { + const block = ltc().blockFromHex(ltcBlock(HOGEX.hex, Buffer.from('0102030405', 'hex'))) + assert.strictEqual(block.transactions[1].getId(), HOGEX.txid) + }) + + it('a HogEx with a nonzero locktime gets the txid Litecoin computes', function () { + const h = hogExWithLockTime(7) + const block = ltc().blockFromHex(ltcBlock(h.hex, Buffer.alloc(0))) + assert.strictEqual(block.transactions[1].getId(), h.txid) + }) + + it('refuses a non-null MWEB part inside a block, whose length nothing records', function () { + const raw = Buffer.from(HOGEX.hex, 'hex') + const tagged = Buffer.concat([raw.subarray(0, raw.length - 5), MWEB_BODY, raw.subarray(raw.length - 4)]) + assert.throws(() => ltc().blockFromHex(ltcBlock(tagged.toString('hex'), Buffer.alloc(0))), /MWEB/) + }) + }) +}) diff --git a/test/unit/wire_parse/transaction_parse_conformance.test.js b/test/unit/wire_parse/transaction_parse_conformance.test.js new file mode 100644 index 00000000..0afd5c61 --- /dev/null +++ b/test/unit/wire_parse/transaction_parse_conformance.test.js @@ -0,0 +1,188 @@ +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +'use strict' + +const assert = require('assert') +const bitcoin = require('bitcoinjs-lib') +const ecc = require('tiny-secp256k1') +const XChainDecoder = require('../../../src/XChainDecoder') +const XChainBlockDecoder = require('../../../src/chain/XChainBlockDecoder') +const CryptoNetworks = require('../../../src/chain/crypto_networks') +const NEW_VECTORS = require('../../fixtures/decoder_parse_vectors.json') +const WIRE_VECTORS = require('../../fixtures/wire_parse_vectors.json') + +bitcoin.initEccLib(ecc) + +const P2PKH = Buffer.from('76a914' + '11'.repeat(20) + '88ac', 'hex') +const P2SH = Buffer.from('a914' + '22'.repeat(20) + '87', 'hex') +const P2WPKH = Buffer.from('0014' + '33'.repeat(20), 'hex') +const P2WSH = Buffer.from('0020' + '44'.repeat(32), 'hex') +const P2TR = Buffer.from('5120' + '22'.repeat(32), 'hex') + +describe('transaction wire parsing conformance', function () { + it('matches node txids for legacy, segwit v0, taproot v1, LTC MWEB and DOGE coinbase forms', function () { + const forms = [ + ['bitcoin-mainnet', NEW_VECTORS.bitcoinGenesisCoinbase], + ['bitcoin-mainnet', WIRE_VECTORS.ltcPegin], + ['bitcoin-mainnet', NEW_VECTORS.taprootWitnessSpend], + ['litecoin-mainnet', WIRE_VECTORS.ltcHogEx3195200], + ['dogecoin-mainnet', { hex: WIRE_VECTORS.doge4500000.txs[0], txid: WIRE_VECTORS.doge4500000.txids[0] }], + ] + + for (const [network, vector] of forms) { + assert.strictEqual(new XChainBlockDecoder(network).transactionFromHex(vector.hex).getId(), vector.txid) + } + }) + + it('rejects truncated, non-hex and trailing bytes with a typed error', function () { + const decoder = new XChainBlockDecoder('bitcoin-mainnet') + const valid = NEW_VECTORS.taprootWitnessSpend.hex + const malformed = [ + '', + '0', + 'zz', + valid.slice(0, -2), + valid + '00', + ] + + for (const hex of malformed) { + assert.throws( + () => decoder.transactionFromHex(hex), + err => err instanceof XChainBlockDecoder.TransactionParseError && err.code === 'ERR_TRANSACTION_PARSE' + ) + } + }) + + it('rejects an MWEB CompactSize above 2^53 before using it as a length', function () { + // version 2, MWEB flag 0x08, then input-count CompactSize 2^53. + const unsafeInputCount = '020000000008ff0000000000002000' + assert.throws( + () => new XChainBlockDecoder('litecoin-mainnet').transactionFromHex(unsafeInputCount), + err => err instanceof XChainBlockDecoder.TransactionParseError && + /exceeds Number\.MAX_SAFE_INTEGER/.test(err.message) + ) + }) +}) + +describe('transaction wire parsing conformance', function () { + it('bounds-checks every truncation of a fixed MWEB transaction', function () { + const decoder = new XChainBlockDecoder('litecoin-mainnet') + const valid = WIRE_VECTORS.ltcHogEx3195200.hex + for (let end = 0; end < valid.length; end += 2) { + assert.throws( + () => decoder.transactionFromHex(valid.slice(0, end)), + err => err instanceof XChainBlockDecoder.TransactionParseError + ) + } + assert.throws( + () => decoder.transactionFromHex(valid + '00'), + err => err instanceof XChainBlockDecoder.TransactionParseError && /trailing byte/.test(err.message) + ) + }) + + it('types a truncated transaction inside a block for the block-loop caller', function () { + const block = Buffer.concat([ + Buffer.from(WIRE_VECTORS.doge4500000.header, 'hex'), + Buffer.from([WIRE_VECTORS.doge4500000.txs.length]), + ...WIRE_VECTORS.doge4500000.txs.map(hex => Buffer.from(hex, 'hex')), + ]) + assert.throws( + () => new XChainBlockDecoder('dogecoin-mainnet').blockFromBuffer(block.subarray(0, -1)), + err => err instanceof XChainBlockDecoder.TransactionParseError && err.code === 'ERR_TRANSACTION_PARSE' + ) + }) +}) + +// The addresses each network derives for P2PKH, P2SH, P2WPKH, P2WSH and P2TR, in +// that order; DOGE has no witness outputs, so it lists only the first two. +const EXPECTED_ADDRESSES = { + 'bitcoin-mainnet': [ + '12ZEw5Hcv1hTb6YUQJ69y1V7uhcoDz92PH', + '34oVnh4gNviJGMnNvgquMeLAxvXJuaRVMZ', + 'bc1qxvenxvenxvenxvenxvenxvenxvenxven2ymjt8', + 'bc1qg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zqd8sxw4', + 'bc1pyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3qfpjpqe', + ], + 'bitcoin-testnet': [ + 'mh5CE8Nbj38iND267s4XnvhSmhDW7yWc6Q', + '2MvMhrRzhzPDeU9QvbpTmybKSBGjUmC6TTu', + 'tb1qxvenxvenxvenxvenxvenxvenxvenxvenqzqps5', + 'tb1qg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zq60xf56', + 'tb1pyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3q7fyw6k', + ], + 'bitcoin-regtest': [ + 'mh5CE8Nbj38iND267s4XnvhSmhDW7yWc6Q', + '2MvMhrRzhzPDeU9QvbpTmybKSBGjUmC6TTu', + 'bcrt1qxvenxvenxvenxvenxvenxvenxvenxvenztev8a', + 'bcrt1qg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zqhkv0pq', + 'bcrt1pyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3qnswg0v', + ], + 'litecoin-mainnet': [ + 'LLnCCHbSzfwWquEdaS5TF2Yt7uz5Qb1SZ1', + 'MB1e6aUeL3Zj4s4H2ZqFBHaaHd7kvvzTco', + 'ltc1qxvenxvenxvenxvenxvenxvenxvenxvenwcpknh', + 'ltc1qg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zqwr7k5s', + 'ltc1pyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3q29u36u', + ], + 'litecoin-testnet': [ + 'mh5CE8Nbj38iND267s4XnvhSmhDW7yWc6Q', + '2MvMhrRzhzPDeU9QvbpTmybKSBGjUmC6TTu', + 'tltc1qxvenxvenxvenxvenxvenxvenxvenxvene2zlqa', + 'tltc1qg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zq9v6gt9', + 'tltc1pyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3qp2c09f', + ], + 'litecoin-regtest': [ + 'mh5CE8Nbj38iND267s4XnvhSmhDW7yWc6Q', + '2MvMhrRzhzPDeU9QvbpTmybKSBGjUmC6TTu', + 'rltc1qxvenxvenxvenxvenxvenxvenxvenxvenuxr9sr', + 'rltc1qg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zql4xswd', + 'rltc1pyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3qmnyhqp', + ], + 'dogecoin-mainnet': [ + 'D6hLULEGDRbk86j58t5iWmeinqM6acA16V', + '9uYkXY8aSzbCAj9rLpWKbmxYfVuM33MqfZ', + ], + 'dogecoin-testnet': [ + 'nVkQCLyB9Q4U15JGAhjAmBF22hjPYUwj3w', + '2MvMhrRzhzPDeU9QvbpTmybKSBGjUmC6TTu', + ], + 'dogecoin-regtest': [ + 'mh5CE8Nbj38iND267s4XnvhSmhDW7yWc6Q', + '2MvMhrRzhzPDeU9QvbpTmybKSBGjUmC6TTu', + ], +} + +async function assertNetworkAddresses(network, addresses) { + const decoder = new XChainDecoder( + network, null, null, null, null, null, + '127.0.0.1', 0, null, null, false + ) + const tx = new bitcoin.Transaction() + tx.addInput(Buffer.alloc(32, 1), 0) + const scripts = network.startsWith('dogecoin') + ? [P2PKH, P2SH] + : [P2PKH, P2SH, P2WPKH, P2WSH, P2TR] + for (const script of scripts) tx.addOutput(script, 1) + + const parsed = await decoder.parseTransaction(tx, new Set()) + assert.deepStrictEqual(parsed.paymentOutputs.map(output => output.destinationAddress), addresses) + + if (!network.startsWith('dogecoin')) { + assert.strictEqual(bitcoin.address.fromOutputScript(P2WPKH, CryptoNetworks.getBitcoinJsNetwork(network)), addresses[2]) + assert.strictEqual(bitcoin.address.fromOutputScript(P2TR, CryptoNetworks.getBitcoinJsNetwork(network)), addresses[4]) + } else { + assert.throws(() => bitcoin.address.fromOutputScript(P2WPKH, CryptoNetworks.getBitcoinJsNetwork(network))) + assert.throws(() => bitcoin.address.fromOutputScript(P2TR, CryptoNetworks.getBitcoinJsNetwork(network))) + } +} + +describe('output address derivation conformance', function () { + for (const [network, addresses] of Object.entries(EXPECTED_ADDRESSES)) { + it(network + ' derives the coin prefixes and witness checksum variant', async function () { + await assertNetworkAddresses(network, addresses) + }) + } +}) diff --git a/test/unit/xchain_block_decoder.test.js b/test/unit/xchain_block_decoder.test.js index dc32077d..55ed2c98 100644 --- a/test/unit/xchain_block_decoder.test.js +++ b/test/unit/xchain_block_decoder.test.js @@ -188,24 +188,33 @@ describe('XChainBlockDecoder', () => { it('[REGRESSION P2] R-NET-002: should strip MWEB flag (0x08) from litecoin transactions', () => { const ltcDecoder = new XChainBlockDecoder('litecoin-mainnet') - // Build a v2 tx with marker=0x00, flag=0x08 (MWEB) - // version(4) + marker(1) + flag(1) + varint inputs(1)=0 + varint outputs(1)=0 + locktime(4) - const txHex = '02000000' + '00' + '08' + '00' + '00' + '00000000' - // After stripping marker+flag: 02000000 + 00 + 00 + 00000000 - // This is a tx with 0 inputs, 0 outputs + // An MWEB-only v2 tx: version(4) + marker 0x00 + flag 0x08 + no inputs + + // no outputs + MWEB tag 0x01 and an (opaque) MWEB body + locktime(4). + const txHex = '02000000' + '00' + '08' + '00' + '00' + '01' + 'ab'.repeat(8) + '00000000' const tx = ltcDecoder.transactionFromHex(txHex) assert.ok(tx) assert.strictEqual(tx.ins.length, 0) + assert.strictEqual(tx.outs.length, 0) }) it('[REGRESSION P2] R-NET-002: should strip MWEB+segwit flag (0x09) from litecoin transactions', () => { const ltcDecoder = new XChainBlockDecoder('litecoin-mainnet') - const txHex = '02000000' + '00' + '09' + '00' + '00' + '00000000' + // marker 0x00 + flag 0x09 (witness and MWEB): one input, no outputs, one + // witness item 0xaa, MWEB tag 0x01 and an (opaque) MWEB body, locktime 7. + const txHex = '02000000' + '00' + '09' + + '01' + '07'.repeat(32) + '00000000' + '00' + 'ffffffff' + '00' + + '01' + '01' + 'aa' + '01' + 'ab'.repeat(8) + '07000000' const tx = ltcDecoder.transactionFromHex(txHex) assert.ok(tx) - assert.strictEqual(tx.ins.length, 0) + assert.strictEqual(tx.ins.length, 1) + assert.deepStrictEqual(tx.ins[0].witness, [Buffer.from('aa', 'hex')]) + assert.strictEqual(tx.locktime, 7) }) + }) +}) +describe('XChainBlockDecoder', () => { + describe('#transactionFromHex()', () => { it('[REGRESSION P2] R-NET-002: should not strip flags for bitcoin transactions', () => { const btcDecoder = new XChainBlockDecoder('bitcoin-regtest') // A v1 tx with no special flags, just 0 inputs/outputs