From b6c483e60ba4d3af8a29fdad7509d1df17fb5e8a Mon Sep 17 00:00:00 2001 From: J-Dog Date: Wed, 23 Sep 2026 15:12:26 -0700 Subject: [PATCH 01/50] fix(node): roll-call wiring refusal names the public explorer read with no key The explorer's federation reads are public since the operator ruling of 2026-09-23, so the refusal no longer tells an operator to set DOGE_INDEXER_API_KEY to a federation read key nobody issues. --- src/services/rollcall_wiring/index.js | 12 ++++++------ .../unit/rollcall_wiring.test/02_the_verdict.test.js | 5 +++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/services/rollcall_wiring/index.js b/src/services/rollcall_wiring/index.js index 10ff346f..36a1de63 100644 --- a/src/services/rollcall_wiring/index.js +++ b/src/services/rollcall_wiring/index.js @@ -48,8 +48,8 @@ const DOGE_READ_URL_KEYS = ['DOGE_INDEXER_API_URL', 'DOGE_INDEXER_URL'] // Where an outside validator without its own Dogecoin indexer points: the // public explorer serves the federation reads off its replicated indexer -// databases, gated by the federation read key the validator was issued. -// Regtest is private, so it has no public value to name. +// databases with no key, like the rest of its API. Regtest is private, so it +// has no public value to name. const PUBLIC_DOGE_READ_URL = Object.freeze({ [Network.MAINNET]: 'https://explorer.xchain.io/DOGE/api/', [Network.TESTNET]: 'https://explorer.xchain.io/TDOGE/api/' @@ -63,10 +63,10 @@ function describeWhereToPoint(network) { return 'Set ' + DOGE_READ_URL_KEYS[0] + ' (and DOGE_INDEXER_API_KEY) in the host .env to a reachable dogecoin ' + network + ' indexer, then re-run.' } - return 'Set ' + DOGE_READ_URL_KEYS[0] + '=' + publicUrl + ' in the host .env, with DOGE_INDEXER_API_KEY set to the ' - + 'federation read key this validator was issued (the public explorer serves the federation reads off its ' - + 'replicated indexer databases); a validator running its own dogecoin ' + network - + ' indexer points at that instead. Then re-run.' + return 'Set ' + DOGE_READ_URL_KEYS[0] + '=' + publicUrl + ' in the host .env, with no DOGE_INDEXER_API_KEY ' + + '(the public explorer serves the federation reads off its replicated indexer databases to anyone); ' + + 'a validator running its own dogecoin ' + network + + ' indexer points at that instead, with that indexer\'s key. Then re-run.' } function isTruthyEnv(value) { diff --git a/test/unit/rollcall_wiring.test/02_the_verdict.test.js b/test/unit/rollcall_wiring.test/02_the_verdict.test.js index 6d559520..7fa95b57 100644 --- a/test/unit/rollcall_wiring.test/02_the_verdict.test.js +++ b/test/unit/rollcall_wiring.test/02_the_verdict.test.js @@ -50,10 +50,11 @@ describe('ROLLCALL wiring guard: the refusal', function () { expect(warnStub.called).to.equal(false) }) - it('names the public explorer read for testnet, keyed by the federation read key, and the self-hosted alternative', () => { + it('names the public explorer read for testnet, with no key, and the self-hosted alternative', () => { const message = refusal(INDEXER, 'bitcoin', 'testnet', {}).message expect(message).to.contain('DOGE_INDEXER_API_URL=https://explorer.xchain.io/TDOGE/api/') - expect(message).to.match(/DOGE_INDEXER_API_KEY set to the federation read key this validator was issued/) + expect(message).to.match(/with no DOGE_INDEXER_API_KEY/) + expect(message).to.not.match(/federation read key/) expect(message).to.match(/running its own dogecoin testnet indexer points at that instead/) expect(message).to.not.contain('/DOGE/api/') }) From a6f188a021277023740c35133efb67a6c1692d25 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Wed, 23 Sep 2026 18:55:11 -0700 Subject: [PATCH 02/50] Write fresh sidecar credentials under safe secret names Persist newly generated RPC and database credentials under their preferred secret aliases so fresh installs avoid deprecated-name warnings. Preserve legacy sidecars for compatibility and cover generation, reload, and rotation behavior. --- src/services/config_service/sidecars.js | 14 ++- .../database_credentials.test.js | 10 +- .../preferred_sidecar_names.test.js | 98 +++++++++++++++++++ 3 files changed, 114 insertions(+), 8 deletions(-) create mode 100644 test/unit/config_service.test/preferred_sidecar_names.test.js diff --git a/src/services/config_service/sidecars.js b/src/services/config_service/sidecars.js index b22468f6..fbd5640c 100644 --- a/src/services/config_service/sidecars.js +++ b/src/services/config_service/sidecars.js @@ -80,9 +80,11 @@ function upsertSidecarValues(localFilePath, values) { // renamed to the redaction-safe `*_SECRET` form must not sprout the legacy // twin again on the next rotation: two names for one credential is exactly // the ambiguity foldSecretEnvAliases() refuses to guess through, so the - // rotation would leave the stack unable to start. + // rotation would leave the stack unable to start. When the legacy name is + // not on disk either, there is nothing to preserve, so a brand new key goes + // straight to its redaction-safe name. const alias = preferredSecretEnvName(k) - merged[alias && alias in merged ? alias : k] = values[k] + merged[alias && !(k in merged) ? alias : k] = values[k] } persistSidecarCreds(localFilePath, merged, { overwrite: true }) } @@ -279,7 +281,13 @@ function generateRpcCredentials(defaultConfig, localFilePath) { generated["NODE_USER"] = defaultConfig["NODE_USER"] = crypto.randomBytes(12).toString('hex') } if (!("NODE_PASSWORD" in defaultConfig)) { - generated["NODE_PASSWORD"] = defaultConfig["NODE_PASSWORD"] = crypto.randomBytes(24).toString('hex') + // Absent under NODE_PASSWORD means absent under NODE_SECRET too, since the + // fold above already collapsed either spelling onto this key. Nothing on + // disk to preserve, so the new credential goes straight to its + // redaction-safe name. + const value = crypto.randomBytes(24).toString('hex') + defaultConfig["NODE_PASSWORD"] = value + generated[preferredSecretEnvName("NODE_PASSWORD")] = value } if (Object.keys(generated).length) persistSidecarCreds(localFilePath, generated) } diff --git a/test/unit/config_service.test/database_credentials.test.js b/test/unit/config_service.test/database_credentials.test.js index db51ea2f..2a10e2db 100644 --- a/test/unit/config_service.test/database_credentials.test.js +++ b/test/unit/config_service.test/database_credentials.test.js @@ -29,8 +29,8 @@ function databaseCredentials1() { const config = await cs.getDefaultConfig('xchain-decoder', 'bitcoin', 'mainnet') expect(config['DECODER_DB_PASS']).to.equal('xchain' + SEP + 'password') expect(config['INDEXER_DB_PASS']).to.equal('xchain' + SEP + 'password') - expect(files[coinSidecar] || '').to.not.include('DECODER_DB_PASS=') - expect(files[coinSidecar] || '').to.not.include('INDEXER_DB_PASS=') + expect(files[coinSidecar] || '').to.not.include('DECODER_DB_SECRET=') + expect(files[coinSidecar] || '').to.not.include('INDEXER_DB_SECRET=') }) it('generates per-install DB passwords when a DB container exists (rotation can apply them) and persists them', async function () { @@ -39,8 +39,8 @@ function databaseCredentials1() { expect(config['DECODER_DB_PASS']).to.match(/^[0-9a-f]{48}$/) expect(config['INDEXER_DB_PASS']).to.match(/^[0-9a-f]{48}$/) expect(config['DECODER_DB_PASS']).to.not.equal('xchain' + SEP + 'password') - expect(files[coinSidecar]).to.include('DECODER_DB_PASS=') - expect(files[coinSidecar]).to.include('INDEXER_DB_PASS=') + expect(files[coinSidecar]).to.include('DECODER_DB_SECRET=') + expect(files[coinSidecar]).to.include('INDEXER_DB_SECRET=') }) it('generates per-install DB passwords under EXTERNAL_DB (native-path rotation)', async function () { @@ -107,7 +107,7 @@ function databaseCredentials2() { expect(coinCfg['HUB_DB_PASS']).to.match(/^[0-9a-f]{48}$/) expect(coinCfg['HUB_DB_PASS']).to.not.equal('xchain' + SEP + 'password') expect(hubCfg['HUB_DB_PASS']).to.equal(coinCfg['HUB_DB_PASS']) - expect(files[hubSidecar]).to.include('HUB_DB_PASS=') + expect(files[hubSidecar]).to.include('HUB_DB_SECRET=') }) // #2246: on the non-rotatable path INDEXER_DB_PASS is not yet in diff --git a/test/unit/config_service.test/preferred_sidecar_names.test.js b/test/unit/config_service.test/preferred_sidecar_names.test.js new file mode 100644 index 00000000..520d9394 --- /dev/null +++ b/test/unit/config_service.test/preferred_sidecar_names.test.js @@ -0,0 +1,98 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +const { + sinon, expect, makeMemoryConfigService, CONTAINER_ID, coinSidecar, hubSidecar +} = require('./helpers.test') + +describe('ConfigService', function () { + describe('preferred (_SECRET) sidecar names', function () { + + it('a fresh coin/network sidecar writes RPC and DB credentials under their redaction-safe _SECRET names', async function () { + const { cs, files } = makeMemoryConfigService({}, { dbContainerId: CONTAINER_ID }) + await cs.getDefaultConfig('xchain-decoder', 'bitcoin', 'mainnet') + const coinBody = files[coinSidecar] || '' + expect(coinBody).to.include('NODE_SECRET=') + expect(coinBody).to.include('DECODER_DB_SECRET=') + expect(coinBody).to.include('INDEXER_DB_SECRET=') + expect(coinBody).to.not.include('NODE_PASSWORD=') + expect(coinBody).to.not.include('DECODER_DB_PASS=') + expect(coinBody).to.not.include('INDEXER_DB_PASS=') + const hubBody = files[hubSidecar] || '' + expect(hubBody).to.include('HUB_DB_SECRET=') + expect(hubBody).to.not.include('HUB_DB_PASS=') + }) + + it('a reload of a freshly-generated sidecar resolves the same credentials without regenerating them', async function () { + const { cs } = makeMemoryConfigService({}, { dbContainerId: CONTAINER_ID }) + const first = await cs.getDefaultConfig('xchain-decoder', 'bitcoin', 'mainnet') + const second = await cs.getDefaultConfig('xchain-decoder', 'bitcoin', 'mainnet') + expect(second['NODE_PASSWORD']).to.equal(first['NODE_PASSWORD']) + expect(second['DECODER_DB_PASS']).to.equal(first['DECODER_DB_PASS']) + expect(second['INDEXER_DB_PASS']).to.equal(first['INDEXER_DB_PASS']) + }) + + it('a full install (generate) plus reload emits no deprecated-name warning', async function () { + // getDefaultConfig also warns when the main config file is absent (an + // unrelated, expected notice on a from-scratch install); only the + // deprecated-secret-name warning is what this rename was meant to silence. + const { cs } = makeMemoryConfigService({}, { dbContainerId: CONTAINER_ID }) + const warnSpy = sinon.stub(console, 'warn') + try { + await cs.getDefaultConfig('xchain-decoder', 'bitcoin', 'mainnet') + await cs.getDefaultConfig('xchain-decoder', 'bitcoin', 'mainnet') + } finally { + warnSpy.restore() + } + const deprecatedWarnings = warnSpy.args.filter(args => String(args[0]).includes('deprecated name')) + expect(deprecatedWarnings).to.deep.equal([]) + }) + + it('still warns when an existing sidecar carries a deprecated legacy name', async function () { + const { cs } = makeMemoryConfigService({ + [coinSidecar]: 'NODE_USER=u\nNODE_PASSWORD=legacypass\n' + }) + const warnSpy = sinon.stub(console, 'warn') + try { + await cs.getDefaultConfig('xchain-encoder', 'bitcoin', 'mainnet') + } finally { + warnSpy.restore() + } + expect(warnSpy.called).to.equal(true) + expect(warnSpy.args.some(args => String(args[0]).includes('NODE_PASSWORD'))).to.equal(true) + }) + + it('legacy fallback: an existing old-name sidecar still resolves and is not force-renamed', async function () { + const { cs, files } = makeMemoryConfigService({ + [coinSidecar]: 'NODE_USER=u\nNODE_PASSWORD=legacypass\n' + }) + const config = await cs.getDefaultConfig('xchain-encoder', 'bitcoin', 'mainnet') + expect(config['NODE_PASSWORD']).to.equal('legacypass') + expect(config['NODE_USER']).to.equal('u') + expect(files[coinSidecar]).to.include('NODE_PASSWORD=legacypass') + expect(files[coinSidecar]).to.not.include('NODE_SECRET=') + }) + + it('a rotation on a sidecar already using the new name does not reintroduce the legacy twin', async function () { + const { cs, files } = makeMemoryConfigService({ + [coinSidecar]: 'NODE_USER=u\nNODE_SECRET=n\nDECODER_DB_SECRET=d\nINDEXER_DB_SECRET=i\n' + }, { dbContainerId: CONTAINER_ID }) + const config = await cs.getDefaultConfig('xchain-decoder', 'bitcoin', 'mainnet') + expect(config['NODE_PASSWORD']).to.equal('n') + expect(config['DECODER_DB_PASS']).to.equal('d') + const body = files[coinSidecar] || '' + expect(body).to.not.include('NODE_PASSWORD=') + expect(body).to.not.include('DECODER_DB_PASS=') + expect(body).to.not.include('INDEXER_DB_PASS=') + }) + }) +}) From 068874cc5275ea0d9e7f1f9ae480d425d94a155e Mon Sep 17 00:00:00 2001 From: J-Dog Date: Wed, 23 Sep 2026 18:55:29 -0700 Subject: [PATCH 03/50] Publish the hub peer port in validator mode Add the conditional peer port mapping so validator hubs expose their network listener while standalone hubs remain unchanged. Pin both modes in the service registry tests. --- src/config/service_registry.js | 5 ++++- test/unit/service_registry.test.js | 11 +++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/config/service_registry.js b/src/config/service_registry.js index 879365d6..6bbb3f97 100644 --- a/src/config/service_registry.js +++ b/src/config/service_registry.js @@ -181,7 +181,10 @@ const SERVICE_REGISTRY = { [HUB_MODULE_NAME]: { docker: { singleton: true, - ports: [{ host: 'HUB_PORT', container: 'HUB_PORT', always: true }], + ports: [ + { host: 'HUB_PORT', container: 'HUB_PORT', always: true }, + { host: 'P2P_PORT', container: 'P2P_PORT' } + ], volumes: [{ type: 'hubCapabilityConfig' }, { type: 'hubSignerDir' }] } }, diff --git a/test/unit/service_registry.test.js b/test/unit/service_registry.test.js index e5685776..f9c1d563 100644 --- a/test/unit/service_registry.test.js +++ b/test/unit/service_registry.test.js @@ -227,6 +227,17 @@ describe('SERVICE_REGISTRY', function () { // No HUB_CAPABILITY_CONFIG in env and no signer in the fixture means no volumes. expect(r.volumeArgs).to.deep.equal([]) }) + + it('hub: the P2P_PORT row is non-always, so it publishes in validator mode only', function () { + const row = SERVICE_REGISTRY[HUB_MODULE_NAME].docker.ports.find(p => p.host === 'P2P_PORT') + expect(row).to.deep.equal({ host: 'P2P_PORT', container: 'P2P_PORT' }) + + const standalone = ms.buildModuleDockerArgs(HUB_MODULE_NAME, ENV, 'bitcoin', 'mainnet') + expect(standalone.portArgs).to.deep.equal(['-p', '10000:10000']) + + const validator = ms.buildModuleDockerArgs(HUB_MODULE_NAME, { ...ENV, P2P_PORT: 10002 }, 'bitcoin', 'mainnet') + expect(validator.portArgs).to.deep.equal(['-p', '10000:10000', '-p', '10002:10002']) + }) }) }) From 9d6acdcb3ee4db403f727c59254c428f5b037030 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Wed, 23 Sep 2026 19:02:07 -0700 Subject: [PATCH 04/50] Split preferred sidecar tests into named groups Keep the new credential coverage within the repository's function-size limit while preserving the existing suite and case titles. --- .../preferred_sidecar_names.test.js | 150 +++++++++--------- 1 file changed, 78 insertions(+), 72 deletions(-) diff --git a/test/unit/config_service.test/preferred_sidecar_names.test.js b/test/unit/config_service.test/preferred_sidecar_names.test.js index 520d9394..dcab1121 100644 --- a/test/unit/config_service.test/preferred_sidecar_names.test.js +++ b/test/unit/config_service.test/preferred_sidecar_names.test.js @@ -14,85 +14,91 @@ const { sinon, expect, makeMemoryConfigService, CONTAINER_ID, coinSidecar, hubSidecar } = require('./helpers.test') -describe('ConfigService', function () { - describe('preferred (_SECRET) sidecar names', function () { +function registerPreferredNameTests() { + it('a fresh coin/network sidecar writes RPC and DB credentials under their redaction-safe _SECRET names', async function () { + const { cs, files } = makeMemoryConfigService({}, { dbContainerId: CONTAINER_ID }) + await cs.getDefaultConfig('xchain-decoder', 'bitcoin', 'mainnet') + const coinBody = files[coinSidecar] || '' + expect(coinBody).to.include('NODE_SECRET=') + expect(coinBody).to.include('DECODER_DB_SECRET=') + expect(coinBody).to.include('INDEXER_DB_SECRET=') + expect(coinBody).to.not.include('NODE_PASSWORD=') + expect(coinBody).to.not.include('DECODER_DB_PASS=') + expect(coinBody).to.not.include('INDEXER_DB_PASS=') + const hubBody = files[hubSidecar] || '' + expect(hubBody).to.include('HUB_DB_SECRET=') + expect(hubBody).to.not.include('HUB_DB_PASS=') + }) - it('a fresh coin/network sidecar writes RPC and DB credentials under their redaction-safe _SECRET names', async function () { - const { cs, files } = makeMemoryConfigService({}, { dbContainerId: CONTAINER_ID }) - await cs.getDefaultConfig('xchain-decoder', 'bitcoin', 'mainnet') - const coinBody = files[coinSidecar] || '' - expect(coinBody).to.include('NODE_SECRET=') - expect(coinBody).to.include('DECODER_DB_SECRET=') - expect(coinBody).to.include('INDEXER_DB_SECRET=') - expect(coinBody).to.not.include('NODE_PASSWORD=') - expect(coinBody).to.not.include('DECODER_DB_PASS=') - expect(coinBody).to.not.include('INDEXER_DB_PASS=') - const hubBody = files[hubSidecar] || '' - expect(hubBody).to.include('HUB_DB_SECRET=') - expect(hubBody).to.not.include('HUB_DB_PASS=') - }) + it('a reload of a freshly-generated sidecar resolves the same credentials without regenerating them', async function () { + const { cs } = makeMemoryConfigService({}, { dbContainerId: CONTAINER_ID }) + const first = await cs.getDefaultConfig('xchain-decoder', 'bitcoin', 'mainnet') + const second = await cs.getDefaultConfig('xchain-decoder', 'bitcoin', 'mainnet') + expect(second['NODE_PASSWORD']).to.equal(first['NODE_PASSWORD']) + expect(second['DECODER_DB_PASS']).to.equal(first['DECODER_DB_PASS']) + expect(second['INDEXER_DB_PASS']).to.equal(first['INDEXER_DB_PASS']) + }) - it('a reload of a freshly-generated sidecar resolves the same credentials without regenerating them', async function () { - const { cs } = makeMemoryConfigService({}, { dbContainerId: CONTAINER_ID }) - const first = await cs.getDefaultConfig('xchain-decoder', 'bitcoin', 'mainnet') - const second = await cs.getDefaultConfig('xchain-decoder', 'bitcoin', 'mainnet') - expect(second['NODE_PASSWORD']).to.equal(first['NODE_PASSWORD']) - expect(second['DECODER_DB_PASS']).to.equal(first['DECODER_DB_PASS']) - expect(second['INDEXER_DB_PASS']).to.equal(first['INDEXER_DB_PASS']) - }) + it('a full install (generate) plus reload emits no deprecated-name warning', async function () { + // getDefaultConfig also warns when the main config file is absent (an + // unrelated, expected notice on a from-scratch install); only the + // deprecated-secret-name warning is what this rename was meant to silence. + const { cs } = makeMemoryConfigService({}, { dbContainerId: CONTAINER_ID }) + const warnSpy = sinon.stub(console, 'warn') + try { + await cs.getDefaultConfig('xchain-decoder', 'bitcoin', 'mainnet') + await cs.getDefaultConfig('xchain-decoder', 'bitcoin', 'mainnet') + } finally { + warnSpy.restore() + } + const deprecatedWarnings = warnSpy.args.filter(args => String(args[0]).includes('deprecated name')) + expect(deprecatedWarnings).to.deep.equal([]) + }) +} - it('a full install (generate) plus reload emits no deprecated-name warning', async function () { - // getDefaultConfig also warns when the main config file is absent (an - // unrelated, expected notice on a from-scratch install); only the - // deprecated-secret-name warning is what this rename was meant to silence. - const { cs } = makeMemoryConfigService({}, { dbContainerId: CONTAINER_ID }) - const warnSpy = sinon.stub(console, 'warn') - try { - await cs.getDefaultConfig('xchain-decoder', 'bitcoin', 'mainnet') - await cs.getDefaultConfig('xchain-decoder', 'bitcoin', 'mainnet') - } finally { - warnSpy.restore() - } - const deprecatedWarnings = warnSpy.args.filter(args => String(args[0]).includes('deprecated name')) - expect(deprecatedWarnings).to.deep.equal([]) +function registerLegacyNameTests() { + it('still warns when an existing sidecar carries a deprecated legacy name', async function () { + const { cs } = makeMemoryConfigService({ + [coinSidecar]: 'NODE_USER=u\nNODE_PASSWORD=legacypass\n' }) + const warnSpy = sinon.stub(console, 'warn') + try { + await cs.getDefaultConfig('xchain-encoder', 'bitcoin', 'mainnet') + } finally { + warnSpy.restore() + } + expect(warnSpy.called).to.equal(true) + expect(warnSpy.args.some(args => String(args[0]).includes('NODE_PASSWORD'))).to.equal(true) + }) - it('still warns when an existing sidecar carries a deprecated legacy name', async function () { - const { cs } = makeMemoryConfigService({ - [coinSidecar]: 'NODE_USER=u\nNODE_PASSWORD=legacypass\n' - }) - const warnSpy = sinon.stub(console, 'warn') - try { - await cs.getDefaultConfig('xchain-encoder', 'bitcoin', 'mainnet') - } finally { - warnSpy.restore() - } - expect(warnSpy.called).to.equal(true) - expect(warnSpy.args.some(args => String(args[0]).includes('NODE_PASSWORD'))).to.equal(true) + it('legacy fallback: an existing old-name sidecar still resolves and is not force-renamed', async function () { + const { cs, files } = makeMemoryConfigService({ + [coinSidecar]: 'NODE_USER=u\nNODE_PASSWORD=legacypass\n' }) + const config = await cs.getDefaultConfig('xchain-encoder', 'bitcoin', 'mainnet') + expect(config['NODE_PASSWORD']).to.equal('legacypass') + expect(config['NODE_USER']).to.equal('u') + expect(files[coinSidecar]).to.include('NODE_PASSWORD=legacypass') + expect(files[coinSidecar]).to.not.include('NODE_SECRET=') + }) - it('legacy fallback: an existing old-name sidecar still resolves and is not force-renamed', async function () { - const { cs, files } = makeMemoryConfigService({ - [coinSidecar]: 'NODE_USER=u\nNODE_PASSWORD=legacypass\n' - }) - const config = await cs.getDefaultConfig('xchain-encoder', 'bitcoin', 'mainnet') - expect(config['NODE_PASSWORD']).to.equal('legacypass') - expect(config['NODE_USER']).to.equal('u') - expect(files[coinSidecar]).to.include('NODE_PASSWORD=legacypass') - expect(files[coinSidecar]).to.not.include('NODE_SECRET=') - }) + it('a rotation on a sidecar already using the new name does not reintroduce the legacy twin', async function () { + const { cs, files } = makeMemoryConfigService({ + [coinSidecar]: 'NODE_USER=u\nNODE_SECRET=n\nDECODER_DB_SECRET=d\nINDEXER_DB_SECRET=i\n' + }, { dbContainerId: CONTAINER_ID }) + const config = await cs.getDefaultConfig('xchain-decoder', 'bitcoin', 'mainnet') + expect(config['NODE_PASSWORD']).to.equal('n') + expect(config['DECODER_DB_PASS']).to.equal('d') + const body = files[coinSidecar] || '' + expect(body).to.not.include('NODE_PASSWORD=') + expect(body).to.not.include('DECODER_DB_PASS=') + expect(body).to.not.include('INDEXER_DB_PASS=') + }) +} - it('a rotation on a sidecar already using the new name does not reintroduce the legacy twin', async function () { - const { cs, files } = makeMemoryConfigService({ - [coinSidecar]: 'NODE_USER=u\nNODE_SECRET=n\nDECODER_DB_SECRET=d\nINDEXER_DB_SECRET=i\n' - }, { dbContainerId: CONTAINER_ID }) - const config = await cs.getDefaultConfig('xchain-decoder', 'bitcoin', 'mainnet') - expect(config['NODE_PASSWORD']).to.equal('n') - expect(config['DECODER_DB_PASS']).to.equal('d') - const body = files[coinSidecar] || '' - expect(body).to.not.include('NODE_PASSWORD=') - expect(body).to.not.include('DECODER_DB_PASS=') - expect(body).to.not.include('INDEXER_DB_PASS=') - }) +describe('ConfigService', function () { + describe('preferred (_SECRET) sidecar names', function () { + registerPreferredNameTests() + registerLegacyNameTests() }) }) From de0d1892fe537e3511b240aa28338f547b0502f1 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Wed, 23 Sep 2026 19:02:48 -0700 Subject: [PATCH 05/50] Enforce c8 --all coverage and re-baseline floors to measured values coverage:check previously skipped source files the unit suite never imports, so floors sat far under real coverage. Add --all to both coverage scripts, tighten every floor to within 1.5 points of the full-tree measurement, and pin --all with a sync test. --- bin/coverage-thresholds.json | 10 +++++----- package.json | 4 ++-- test/unit/coverage_thresholds_sync.test.js | 6 ++++++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/bin/coverage-thresholds.json b/bin/coverage-thresholds.json index 48ffaf08..d3b8ade9 100644 --- a/bin/coverage-thresholds.json +++ b/bin/coverage-thresholds.json @@ -1,7 +1,7 @@ { - "comment": "Coverage floors for the CI coverage job (regression floors, ~1-1.5 points below measured, not tier targets; raise as coverage climbs). Mirrored into the coverage:check npm script in package.json and guarded by test/unit/coverage-thresholds-sync.test.js. Re-measured 2026-08-15 against a full sibling checkout: 92.44 lines/statements, 87.61 branches, 90.77 functions over 1477 unit tests. The previous floors came from a 2026-08-12 run and sat nearly 5 points under the lines figure, wide enough for a real regression to pass unseen. This repo declares no .ci-siblings; its one cross-repo guard is the xchain-hub coins byte-identity check, which reads files rather than executing src, so the sibling-less CI run measures the same surface.", - "lines": 91, - "statements": 91, - "branches": 86.5, - "functions": 89.5 + "comment": "Coverage floors for the CI coverage job (regression floors, about 1.3 points below measured, not tier targets; raise as coverage climbs). Mirrored into the coverage:check npm script in package.json and guarded by test/unit/coverage_thresholds_sync.test.js. Re-measured 2026-09-23 with c8 --all: 94.88 lines/statements, 89.30 branches, and 93.83 functions. There are no coverage exclusions. This repo declares no .ci-siblings; its one cross-repo guard is the xchain-hub coins byte-identity check, which reads files rather than executing src, so the sibling-less CI run measures the same surface.", + "lines": 93.5, + "statements": 93.5, + "branches": 88, + "functions": 92.5 } diff --git a/package.json b/package.json index 7dd05caa..53f5b969 100644 --- a/package.json +++ b/package.json @@ -22,8 +22,8 @@ "scripts": { "test": "mocha 'test/unit/**/*.test.js' --timeout 2000 --recursive", "lint": "eslint src test bin scripts", - "coverage": "c8 --reporter=text --reporter=html --include 'src/**/*.js' mocha 'test/unit/**/*.test.js' --timeout 2000 --recursive --exit", - "coverage:check": "c8 --check-coverage --lines 91 --statements 91 --branches 86.5 --functions 89.5 --reporter=text-summary --include 'src/**/*.js' mocha 'test/unit/**/*.test.js' --timeout 2000 --recursive --exit", + "coverage": "c8 --all --reporter=text --reporter=html --include 'src/**/*.js' mocha 'test/unit/**/*.test.js' --timeout 2000 --recursive --exit", + "coverage:check": "c8 --all --check-coverage --lines 93.5 --statements 93.5 --branches 88 --functions 92.5 --reporter=text-summary --include 'src/**/*.js' mocha 'test/unit/**/*.test.js' --timeout 2000 --recursive --exit", "ci": "mocha 'test/unit/**/*.test.js' --timeout 2000 --recursive --exit && npm run ci:security && npm run ci:regression", "ci:security": "mocha 'test/security/**/*.test.js' --timeout 10000 --recursive --exit", "ci:regression": "mocha 'test/regression/**/*.test.js' --timeout 30000 --recursive --exit", diff --git a/test/unit/coverage_thresholds_sync.test.js b/test/unit/coverage_thresholds_sync.test.js index f7cce6e1..3ef53d35 100644 --- a/test/unit/coverage_thresholds_sync.test.js +++ b/test/unit/coverage_thresholds_sync.test.js @@ -40,4 +40,10 @@ describe('coverage ratchet floors', () => { it('fails the job on a shortfall rather than only reporting it', () => { assert.match(pkg.scripts['coverage:check'], /--check-coverage/); }); + + it('measures source files that the unit suite does not load', () => { + for (const scriptName of ['coverage', 'coverage:check']) { + assert.match(pkg.scripts[scriptName], /(?:^|\s)--all(?:\s|$)/); + } + }); }); From ec0a008105dac06b32c6160f185e24203dce06cb Mon Sep 17 00:00:00 2001 From: J-Dog Date: Thu, 24 Sep 2026 08:01:29 -0700 Subject: [PATCH 06/50] test information schema SQL builders Cover exact schema, table, multi-table, and size query output, including quote escaping. --- test/unit/db/information_schema.test.js | 48 +++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 test/unit/db/information_schema.test.js diff --git a/test/unit/db/information_schema.test.js b/test/unit/db/information_schema.test.js new file mode 100644 index 00000000..eaaa8e6e --- /dev/null +++ b/test/unit/db/information_schema.test.js @@ -0,0 +1,48 @@ +'use strict' + +// Copyright © 2025-2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC - https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +const { expect } = require('chai') + +const { + schemaExistsSql, + tableCountSql, + tableExistsSql, + tablesExistSql, + schemaSizeSql +} = require('../../../src/db/information_schema') + +describe('information_schema SQL builders', () => { + it('builds the schema existence query', () => { + expect(schemaExistsSql('xchain_btc')).to.equal( + "SELECT COUNT(SCHEMA_NAME) FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = 'xchain_btc'" + ) + }) + + it('builds the table count query', () => { + expect(tableCountSql("'db1'")).to.equal( + "SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA = 'db1'" + ) + }) + + it('builds the table existence query', () => { + expect(tableExistsSql("'db1'", "'blocks'")).to.equal( + "SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA = 'db1' AND TABLE_NAME = 'blocks'" + ) + }) + + it('builds the multi-table existence query with escaped literals', () => { + expect(tablesExistSql("'db1'", ['blocks', "o'brien"])).to.equal( + "SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA = 'db1' AND TABLE_NAME IN ('blocks', 'o\\'brien')" + ) + }) + + it('builds the schema size query', () => { + expect(schemaSizeSql('db1')).to.equal( + "SELECT SUM(DATA_LENGTH + INDEX_LENGTH) FROM information_schema.TABLES WHERE TABLE_SCHEMA = 'db1'" + ) + }) +}) From b29acdb2d6a11993e81238c1f2118791aceeb3d9 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Thu, 24 Sep 2026 09:55:01 -0700 Subject: [PATCH 07/50] Test price fence SQL builders Cover chain-wide, network-scoped, manual, and quote escaping outputs. --- test/unit/db/price_fence.test.js | 40 ++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 test/unit/db/price_fence.test.js diff --git a/test/unit/db/price_fence.test.js b/test/unit/db/price_fence.test.js new file mode 100644 index 00000000..881700a6 --- /dev/null +++ b/test/unit/db/price_fence.test.js @@ -0,0 +1,40 @@ +'use strict' + +// Copyright © 2025-2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC - https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +const { expect } = require('chai') + +const { + clearChainFenceSql, + clearNetworkFenceSql, + manualClearStatement +} = require('../../../src/db/price_fence') + +describe('price fence SQL builders', () => { + it('builds the chain-wide fence clear', () => { + expect(clearChainFenceSql('xchain_hub', 'DOGE')).to.equal( + "DELETE FROM `xchain_hub`.price_ingest_watermarks WHERE source_chain = 'DOGE'" + ) + }) + + it('builds the network-scoped fence clear with the legacy bucket', () => { + expect(clearNetworkFenceSql('xchain_hub', 'DOGE', 'dogecoin-mainnet')).to.equal( + "DELETE FROM `xchain_hub`.price_ingest_watermarks WHERE source_chain = 'DOGE' AND network IN ('dogecoin-mainnet', '')" + ) + }) + + it('builds the manual network-scoped fence clear', () => { + expect(manualClearStatement('DOGE', 'dogecoin-mainnet')).to.equal( + "DELETE FROM price_ingest_watermarks WHERE source_chain = 'DOGE' AND network IN ('dogecoin-mainnet', '');" + ) + }) + + it('escapes embedded quotes in the chain ticker', () => { + expect(clearChainFenceSql('xchain_hub', "O'BRIEN")).to.equal( + "DELETE FROM `xchain_hub`.price_ingest_watermarks WHERE source_chain = 'O\\'BRIEN'" + ) + }) +}) From b57c855e09a7a8447156ed16f66927dff96bcde1 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Thu, 24 Sep 2026 15:47:52 -0700 Subject: [PATCH 08/50] Require siblings in coverage tier Run the coverage ratchet with strict sibling checks, matching the CI tier. --- bin/ci-full.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/ci-full.sh b/bin/ci-full.sh index 9534faa0..6234b799 100755 --- a/bin/ci-full.sh +++ b/bin/ci-full.sh @@ -135,7 +135,7 @@ run_tier "drift: coin consensus-pin conformance" node -e ' run_tier "identity pin (vendored coin bytes)" node bin/pin_identity.js --compare bin/pins/identity.json # --- job: coverage ----------------------------------------------------------- -run_tier "coverage ratchet (coverage:check)" npm run coverage:check +run_tier "coverage ratchet (coverage:check)" env XCHAIN_REQUIRE_SIBLINGS=1 npm run coverage:check echo # >>> ci-tier summary (generated) >>> From f4a44548ca863b88b898af5f4907b58067f996c9 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Thu, 24 Sep 2026 15:50:16 -0700 Subject: [PATCH 09/50] Install prototype mixins as non-enumerable methods Preserve source descriptors while hiding mixed-in methods from enumeration. Cover every database and downloader mixin method with descriptor assertions. --- src/db/index.js | 8 +++- src/services/github_downloader.js | 8 +++- .../unit/prototype_method_descriptors.test.js | 42 +++++++++++++++++++ 3 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 test/unit/prototype_method_descriptors.test.js diff --git a/src/db/index.js b/src/db/index.js index 6c58b12b..745ecdba 100644 --- a/src/db/index.js +++ b/src/db/index.js @@ -29,6 +29,12 @@ const mariadb = require('mariadb') const { sleep } = require('../utils/helpers') const { CREATE_MODULES_TABLE_SQL, modulesMixin } = require('./modules') +function installMethods(target, methods) { + const descriptors = Object.getOwnPropertyDescriptors(methods) + for (const key of Reflect.ownKeys(descriptors)) descriptors[key].enumerable = false + Object.defineProperties(target, descriptors) +} + class MariaDbStore { constructor(config = null) { this.config = config @@ -118,6 +124,6 @@ class MariaDbStore { // Every query against the registry table, installed onto the prototype so a // call site cannot tell the split happened. One place to add a table family: // a new file beside this one, and its row here. -Object.assign(MariaDbStore.prototype, modulesMixin) +installMethods(MariaDbStore.prototype, modulesMixin) module.exports = MariaDbStore diff --git a/src/services/github_downloader.js b/src/services/github_downloader.js index 48df96fa..d70b1ada 100644 --- a/src/services/github_downloader.js +++ b/src/services/github_downloader.js @@ -54,6 +54,12 @@ const { getHostArch } = require('./github_downloader/host_arch.js'); const SHA256_RE = /^[a-f0-9]{64}$/i; +function installMethods(target, methods) { + const descriptors = Object.getOwnPropertyDescriptors(methods); + for (const key of Reflect.ownKeys(descriptors)) descriptors[key].enumerable = false; + Object.defineProperties(target, descriptors); +} + // Picks the release asset built for linux on the host architecture, or // throws when the release has none. function selectHostLinuxAsset(release) { @@ -305,6 +311,6 @@ class GitHubDownloader { } } -Object.assign(GitHubDownloader.prototype, require('./github_downloader/hash_verification.js')); +installMethods(GitHubDownloader.prototype, require('./github_downloader/hash_verification.js')); module.exports = GitHubDownloader; diff --git a/test/unit/prototype_method_descriptors.test.js b/test/unit/prototype_method_descriptors.test.js new file mode 100644 index 00000000..519b033b --- /dev/null +++ b/test/unit/prototype_method_descriptors.test.js @@ -0,0 +1,42 @@ +'use strict' + +// Copyright © 2025-2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC - https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +const { expect } = require('chai') + +const MariaDbStore = require('../../src/db') +const { modulesMixin } = require('../../src/db/modules') +const GitHubDownloader = require('../../src/services/github_downloader') +const hashVerification = require('../../src/services/github_downloader/hash_verification') + +describe('prototype method descriptors', function () { + const installations = [ + ['MariaDbStore', MariaDbStore.prototype, modulesMixin], + ['GitHubDownloader', GitHubDownloader.prototype, hashVerification] + ] + + for (const [className, prototype, methods] of installations) { + for (const methodName of Reflect.ownKeys(methods)) { + it(`${className}.${String(methodName)} is non-enumerable and callable`, function () { + const sourceDescriptor = Object.getOwnPropertyDescriptor(methods, methodName) + const installedDescriptor = Object.getOwnPropertyDescriptor(prototype, methodName) + + expect(installedDescriptor).to.include({ + configurable: sourceDescriptor.configurable, + enumerable: false, + writable: sourceDescriptor.writable + }) + expect(installedDescriptor.value).to.equal(sourceDescriptor.value) + expect(prototype[methodName]).to.be.a('function') + }) + } + } +}) From 1da68a096a790ba391463976dcdd738b6ff747b7 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Thu, 24 Sep 2026 16:24:21 -0700 Subject: [PATCH 10/50] Add scheduled Stryker mutation workflow Runs weekly and on demand, and uploads the mutation report as an artifact. --- .github/workflows/mutation.yml | 41 ++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/mutation.yml diff --git a/.github/workflows/mutation.yml b/.github/workflows/mutation.yml new file mode 100644 index 00000000..00323c9e --- /dev/null +++ b/.github/workflows/mutation.yml @@ -0,0 +1,41 @@ +# Scheduled Stryker mutation run over the modules listed in stryker.config.json. +# Too slow for the push gate, so it runs weekly and on demand. The report is +# uploaded even when the break threshold fails the run, so a score drop can be +# inspected from the artifact. +name: mutation + +permissions: + contents: read + +on: + schedule: + - cron: "0 3 * * 0" + workflow_dispatch: + +jobs: + stryker: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + + - name: Use Node.js 22 + uses: actions/setup-node@v4 + with: + node-version: "22" + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Run Stryker + run: npm run test:mutation + + - name: Upload mutation report + if: always() + uses: actions/upload-artifact@v4 + with: + name: mutation-report + path: reports/mutation/ + if-no-files-found: warn + retention-days: 30 From 9f33061d26b2f8820a431033ae04dafbcea92b09 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Thu, 24 Sep 2026 16:43:20 -0700 Subject: [PATCH 11/50] fix(test): isolate config fixtures and surface benchmark failures Load benchmark and smoke inputs from tracked public fixtures, and keep CI and secret environment tests off operator config paths. Fail benchmark runs when a scenario reports an error and keep strict sibling checks armed for coverage. --- bin/ci-full.sh | 14 +++-- config/fixtures/bitcoin-mainnet | 3 + config/fixtures/bitcoin-regtest | 8 +++ config/fixtures/bitcoin-testnet | 3 + config/fixtures/dogecoin-mainnet | 3 + config/fixtures/dogecoin-regtest | 8 +++ config/fixtures/dogecoin-testnet | 3 + config/fixtures/litecoin-mainnet | 3 + config/fixtures/litecoin-regtest | 8 +++ config/fixtures/litecoin-testnet | 3 + test/benchmarks/helpers/harness.js | 57 ++++++++++++------- ...005_config_template_file_integrity.test.js | 13 ++--- .../s_smoke_006_config_composition.test.js | 20 +++++-- test/unit/secret_env.test.js | 52 +++++++++++------ 14 files changed, 141 insertions(+), 57 deletions(-) create mode 100644 config/fixtures/bitcoin-mainnet create mode 100644 config/fixtures/bitcoin-regtest create mode 100644 config/fixtures/bitcoin-testnet create mode 100644 config/fixtures/dogecoin-mainnet create mode 100644 config/fixtures/dogecoin-regtest create mode 100644 config/fixtures/dogecoin-testnet create mode 100644 config/fixtures/litecoin-mainnet create mode 100644 config/fixtures/litecoin-regtest create mode 100644 config/fixtures/litecoin-testnet diff --git a/bin/ci-full.sh b/bin/ci-full.sh index 6234b799..09b33bf7 100755 --- a/bin/ci-full.sh +++ b/bin/ci-full.sh @@ -70,12 +70,16 @@ run_tier() { ci_tier_deferred "$1" && return 0 # ci-tier guard (generated) local name="$1"; shift echo; echo "ci:full ===== $name =====" - if "$@"; then + local root + root="$(mktemp -d "${TMPDIR:-/tmp}/xchain-node-ci-full.XXXXXX")" + mkdir -p "$root/config" "$root/data" + if ( export XCHAIN_NODE_CONFIG_DIR="$root/config" XCHAIN_NODE_DATA_DIR="$root/data"; "$@" ); then echo "ci:full ----- $name PASS" else FAILED="$FAILED [$name]" echo "ci:full ----- $name FAIL" fi + rm -rf -- "$root" } need_sib() { local s @@ -128,10 +132,10 @@ run_tier "drift: coin consensus-pin conformance" node -e ' console.log("consensus pin conformance OK (testnet, regtest)"); ' -# --- identity pin (this gate only; no ci.yml job runs it) -------------------- -# bin/pins/identity.json holds the sha256 of every vendored coin file. Nothing -# else reads it, so this tier re-hashes the tree against it and fails on any -# moved, missing or unreadable file instead of letting the pin go stale. +# --- identity pin (also the drift-guards job's identity pin step) ------------ +# bin/pins/identity.json holds the sha256 of every vendored coin file. This +# tier re-hashes the tree against it and fails on any moved, missing or +# unreadable file instead of letting the pin go stale. run_tier "identity pin (vendored coin bytes)" node bin/pin_identity.js --compare bin/pins/identity.json # --- job: coverage ----------------------------------------------------------- diff --git a/config/fixtures/bitcoin-mainnet b/config/fixtures/bitcoin-mainnet new file mode 100644 index 00000000..ef0a514e --- /dev/null +++ b/config/fixtures/bitcoin-mainnet @@ -0,0 +1,3 @@ +NETWORK=bitcoin-mainnet +NODE_EXPOSED_PORT=3000 +DUST_AMOUNT=546 diff --git a/config/fixtures/bitcoin-regtest b/config/fixtures/bitcoin-regtest new file mode 100644 index 00000000..5af03d0a --- /dev/null +++ b/config/fixtures/bitcoin-regtest @@ -0,0 +1,8 @@ +NETWORK=bitcoin-regtest +NODE_EXPOSED_PORT=3020 +UTXO_TRACKER_PORT=3021 +DECODER_PORT=3022 +ENCODER_PORT=3023 +INDEXER_PORT=3024 +REGTEST_MINER_PORT=3025 +DUST_AMOUNT=546 diff --git a/config/fixtures/bitcoin-testnet b/config/fixtures/bitcoin-testnet new file mode 100644 index 00000000..76b0b898 --- /dev/null +++ b/config/fixtures/bitcoin-testnet @@ -0,0 +1,3 @@ +NETWORK=bitcoin-testnet +NODE_EXPOSED_PORT=3010 +DUST_AMOUNT=546 diff --git a/config/fixtures/dogecoin-mainnet b/config/fixtures/dogecoin-mainnet new file mode 100644 index 00000000..cda22cb5 --- /dev/null +++ b/config/fixtures/dogecoin-mainnet @@ -0,0 +1,3 @@ +NETWORK=dogecoin-mainnet +NODE_EXPOSED_PORT=3030 +DUST_AMOUNT=100000000 diff --git a/config/fixtures/dogecoin-regtest b/config/fixtures/dogecoin-regtest new file mode 100644 index 00000000..d40f2da4 --- /dev/null +++ b/config/fixtures/dogecoin-regtest @@ -0,0 +1,8 @@ +NETWORK=dogecoin-regtest +NODE_EXPOSED_PORT=3050 +UTXO_TRACKER_PORT=3051 +DECODER_PORT=3052 +ENCODER_PORT=3053 +INDEXER_PORT=3054 +REGTEST_MINER_PORT=3055 +DUST_AMOUNT=100000000 diff --git a/config/fixtures/dogecoin-testnet b/config/fixtures/dogecoin-testnet new file mode 100644 index 00000000..6f7aad05 --- /dev/null +++ b/config/fixtures/dogecoin-testnet @@ -0,0 +1,3 @@ +NETWORK=dogecoin-testnet +NODE_EXPOSED_PORT=3040 +DUST_AMOUNT=100000000 diff --git a/config/fixtures/litecoin-mainnet b/config/fixtures/litecoin-mainnet new file mode 100644 index 00000000..325b1075 --- /dev/null +++ b/config/fixtures/litecoin-mainnet @@ -0,0 +1,3 @@ +NETWORK=litecoin-mainnet +NODE_EXPOSED_PORT=3060 +DUST_AMOUNT=546 diff --git a/config/fixtures/litecoin-regtest b/config/fixtures/litecoin-regtest new file mode 100644 index 00000000..28394e16 --- /dev/null +++ b/config/fixtures/litecoin-regtest @@ -0,0 +1,8 @@ +NETWORK=litecoin-regtest +NODE_EXPOSED_PORT=3080 +UTXO_TRACKER_PORT=3081 +DECODER_PORT=3082 +ENCODER_PORT=3083 +INDEXER_PORT=3084 +REGTEST_MINER_PORT=3085 +DUST_AMOUNT=546 diff --git a/config/fixtures/litecoin-testnet b/config/fixtures/litecoin-testnet new file mode 100644 index 00000000..dd934a35 --- /dev/null +++ b/config/fixtures/litecoin-testnet @@ -0,0 +1,3 @@ +NETWORK=litecoin-testnet +NODE_EXPOSED_PORT=3070 +DUST_AMOUNT=546 diff --git a/test/benchmarks/helpers/harness.js b/test/benchmarks/helpers/harness.js index c2c3f624..efb4a9d3 100644 --- a/test/benchmarks/helpers/harness.js +++ b/test/benchmarks/helpers/harness.js @@ -26,6 +26,7 @@ */ const fs = require('fs') +const os = require('os') const path = require('path') const { Readable } = require('stream') const proxyquire = require('proxyquire').noCallThru() @@ -33,25 +34,28 @@ const MetricsCollector = require('./metrics_collector') const BASELINE_PATH = path.join(__dirname, '../baseline.json') +// These keys match the scenario file basenames used by runScenario(). const SCENARIO_FILES = [ - 'config-generation', - 'filter-params', - 'config-parsing-scale', - 'resolve-args', - 'naming-helpers' + 'config_generation', + 'filter_params', + 'config_parsing_scale', + 'resolve_args', + 'naming_helpers' ] -// --- Config file contents for mocking --- -const CONFIG_FILES = { - 'bitcoin-mainnet': 'NETWORK=bitcoin-mainnet\nNODE_EXPOSED_PORT=3000\nDUST_AMOUNT=546\n', - 'bitcoin-testnet': 'NETWORK=bitcoin-testnet\nNODE_EXPOSED_PORT=3010\nDUST_AMOUNT=546\n', - 'bitcoin-regtest': 'NETWORK=bitcoin-regtest\nNODE_EXPOSED_PORT=3020\nUTXO_TRACKER_PORT=3021\nDECODER_PORT=3022\nENCODER_PORT=3023\nINDEXER_PORT=3024\nREGTEST_MINER_PORT=3025\nDUST_AMOUNT=546\n', - 'dogecoin-mainnet': 'NETWORK=dogecoin-mainnet\nNODE_EXPOSED_PORT=3030\nDUST_AMOUNT=100000000\n', - 'dogecoin-testnet': 'NETWORK=dogecoin-testnet\nNODE_EXPOSED_PORT=3040\nDUST_AMOUNT=100000000\n', - 'dogecoin-regtest': 'NETWORK=dogecoin-regtest\nNODE_EXPOSED_PORT=3050\nUTXO_TRACKER_PORT=3051\nDECODER_PORT=3052\nENCODER_PORT=3053\nINDEXER_PORT=3054\nREGTEST_MINER_PORT=3055\nDUST_AMOUNT=100000000\n', - 'litecoin-mainnet': 'NETWORK=litecoin-mainnet\nNODE_EXPOSED_PORT=3060\nDUST_AMOUNT=546\n', - 'litecoin-testnet': 'NETWORK=litecoin-testnet\nNODE_EXPOSED_PORT=3070\nDUST_AMOUNT=546\n', - 'litecoin-regtest': 'NETWORK=litecoin-regtest\nNODE_EXPOSED_PORT=3080\nUTXO_TRACKER_PORT=3081\nDECODER_PORT=3082\nENCODER_PORT=3083\nINDEXER_PORT=3084\nREGTEST_MINER_PORT=3085\nDUST_AMOUNT=546\n' +const BENCH_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), 'xchain-node-benchmark-')) +process.env.XCHAIN_NODE_CONFIG_DIR = path.join(BENCH_ROOT, 'config') +process.env.XCHAIN_NODE_DATA_DIR = path.join(BENCH_ROOT, 'data') +fs.mkdirSync(process.env.XCHAIN_NODE_CONFIG_DIR, { recursive: true }) +fs.mkdirSync(process.env.XCHAIN_NODE_DATA_DIR, { recursive: true }) +process.on('exit', () => { + try { fs.rmSync(BENCH_ROOT, { recursive: true, force: true }) } catch { /* best effort cleanup */ } +}) + +const FIXTURE_CONFIG_DIR = path.join(__dirname, '../../../config/fixtures') +const CONFIG_FILES = {} +for (const basename of fs.readdirSync(FIXTURE_CONFIG_DIR)) { + CONFIG_FILES[basename] = fs.readFileSync(path.join(FIXTURE_CONFIG_DIR, basename), 'utf8') } function parseArgs() { @@ -115,9 +119,9 @@ Scenarios: ${SCENARIO_FILES.join(', ')} */ function createMockedConfigService() { const realFs = require('fs') - const configDir = require('../../../src/config').configDir const fsStub = { + ...realFs, existsSync: (filePath) => { // Check if it's a config file path we know about const basename = path.basename(filePath) @@ -131,9 +135,7 @@ function createMockedConfigService() { return Readable.from(content) } return realFs.createReadStream(filePath) - }, - rmSync: realFs.rmSync, - mkdirSync: realFs.mkdirSync + } } return proxyquire('../../../src/services/config_service', { fs: fsStub }) @@ -149,7 +151,7 @@ async function createContext() { return { ConfigService, constants, CONFIG_FILES } } -async function destroyContext(_context) { +async function destroyContext() { // No persistent resources to release in the current scenarios } @@ -307,6 +309,13 @@ async function main() { } } + const failedScenarios = Object.entries(allResults.scenarios) + .filter(([, result]) => result && result.error) + .map(([name]) => name) + if (failedScenarios.length) { + process.exitCode = 1 + } + if (config.json) { console.log(JSON.stringify(allResults, null, 2)) } @@ -332,7 +341,11 @@ async function main() { } if (!config.json) { - console.log('\n=== Benchmark Complete ===') + if (failedScenarios.length) { + console.log(`\n=== Benchmark Complete (FAILED: ${failedScenarios.join(', ')}) ===`) + } else { + console.log('\n=== Benchmark Complete ===') + } } } diff --git a/test/smoke/smoke.test/s_smoke_005_config_template_file_integrity.test.js b/test/smoke/smoke.test/s_smoke_005_config_template_file_integrity.test.js index 9e900eb7..38a225b5 100644 --- a/test/smoke/smoke.test/s_smoke_005_config_template_file_integrity.test.js +++ b/test/smoke/smoke.test/s_smoke_005_config_template_file_integrity.test.js @@ -16,16 +16,13 @@ const fs = require('fs') const ROOT = path.join(__dirname, '..', '..', '..') -// The per-coin/network main config files are operator-authored and untracked -// (7c03f7a stopped tracking them; ConfigService falls back to defaults when one -// is absent), so this suite checks whichever of the nine exist on this machine -// and marks the rest pending instead of failing a checkout that never ran an -// install for that coin/network. Comment lines are allowed: the reader skips -// any line without a KEY= prefix, and the restored regtest files open with a -// comment block explaining where the credentials live. +// The per-coin/network main config files are operator-authored and untracked, +// so this suite reads the tracked fixture set under config/fixtures instead of +// whatever an operator machine holds. Comment lines are allowed: the reader +// skips any line without a KEY= prefix. describe('S-SMOKE-005 – Config Template File Integrity', function () { - const configDir = path.join(ROOT, 'config') + const configDir = path.join(ROOT, 'config', 'fixtures') const coins = ['bitcoin', 'litecoin', 'dogecoin'] const networks = ['mainnet', 'testnet', 'regtest'] diff --git a/test/smoke/smoke.test/s_smoke_006_config_composition.test.js b/test/smoke/smoke.test/s_smoke_006_config_composition.test.js index e5a0b46a..ca1c951e 100644 --- a/test/smoke/smoke.test/s_smoke_006_config_composition.test.js +++ b/test/smoke/smoke.test/s_smoke_006_config_composition.test.js @@ -15,19 +15,27 @@ const path = require('path') const fs = require('fs') const ROOT = path.join(__dirname, '..', '..', '..') -const ConfigService = require(path.join(ROOT, 'src/services/config_service')) +const { configDir } = require(path.join(ROOT, 'src/config')) +const proxyquire = require('proxyquire').noCallThru() + +let ConfigService describe('S-SMOKE-006 – Config Composition', function () { + // Reloaded with no stubs so the shared sidecar and database helpers are + // configured with the real fs, not one an earlier suite handed them. + before(function () { + ConfigService = proxyquire(path.join(ROOT, 'src/services/config_service'), {}) + }) + it('getDefaultConfig returns populated config for bitcoin/mainnet', async function () { const config = await ConfigService.getDefaultConfig('xchain-decoder', 'bitcoin', 'mainnet') - // NETWORK is computed (coin-prefixed for the decoder), so it holds with or - // without an operator file. NODE_EXPOSED_PORT and DUST_AMOUNT only ever come - // from the untracked config/bitcoin-mainnet file, so they are asserted only - // where that file exists (see S-SMOKE-005 for why it may not). + // NETWORK is computed, so it holds with or without a per-coin file. + // NODE_EXPOSED_PORT and DUST_AMOUNT come only from that file, so they + // are asserted only where the resolved config dir holds one. expect(config).to.have.property('NETWORK', 'bitcoin-mainnet') - if (fs.existsSync(path.join(ROOT, 'config', 'bitcoin-mainnet'))) { + if (fs.existsSync(path.join(configDir, 'bitcoin-mainnet'))) { expect(config).to.have.property('NODE_EXPOSED_PORT') expect(config).to.have.property('DUST_AMOUNT') } diff --git a/test/unit/secret_env.test.js b/test/unit/secret_env.test.js index 003a4dd1..4a45d068 100644 --- a/test/unit/secret_env.test.js +++ b/test/unit/secret_env.test.js @@ -20,13 +20,32 @@ // credential it was supposed to have rotated away from. const { expect } = require('chai') -const { configStub } = require('../helpers/config_stub') const proxyquire = require('proxyquire').noCallThru() const path = require('path') const fs = require('fs') +const os = require('os') const { Readable } = require('stream') +// src/config resolves its config and data dirs from the environment at load +// time; pointing them at a scratch root keeps the sidecar paths this suite +// builds off the operator's real config directory. +const SCRATCH_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), 'xchain-node-secret-env-')) +const PRIOR_DIRS = { + XCHAIN_NODE_CONFIG_DIR: process.env.XCHAIN_NODE_CONFIG_DIR, + XCHAIN_NODE_DATA_DIR: process.env.XCHAIN_NODE_DATA_DIR +} +process.env.XCHAIN_NODE_CONFIG_DIR = path.join(SCRATCH_ROOT, 'config') +process.env.XCHAIN_NODE_DATA_DIR = path.join(SCRATCH_ROOT, 'data') +const { configStub } = require('../helpers/config_stub') const { SEP, configDir } = require('../../src/config') +for (const [key, value] of Object.entries(PRIOR_DIRS)) { + if (value === undefined) delete process.env[key] + else process.env[key] = value +} +process.on('exit', () => { + try { fs.rmSync(SCRATCH_ROOT, { recursive: true, force: true }) } catch { /* best effort cleanup */ } +}) + const secretEnv = require('../../src/config/secret_env') function streamFromString(str) { @@ -70,25 +89,19 @@ const CONTAINER_ID = 'a'.repeat(64) const coinSidecar = path.resolve(configDir, 'bitcoin-mainnet') + '.local' const coinMain = path.resolve(configDir, 'bitcoin-mainnet') -// Hard pin: a missing sibling or table path is a test failure. -const HUB_TABLE = path.join(__dirname, '../../../xchain-hub/src/secret_env.js') +// Local runs may omit sibling repositories. Ratchet tiers require them. +const HUB_TABLE = path.join(__dirname, '../../../xchain-hub/src/secret_env.js') +const HUB_TABLE_PRESENT = fs.existsSync(HUB_TABLE) +const REQUIRE_SIBLINGS = process.env.XCHAIN_REQUIRE_SIBLINGS === '1' // The platform checkout as a directory, not just the one script this suite // reads from it: a present checkout missing the pinned tool is a moved or // renamed file, while an absent checkout is a standalone install with no // platform tree to compare against. // -// The platform tree is not a sibling repo, so XCHAIN_REQUIRE_SIBLINGS cannot -// vouch for it the way it vouches for xchain-hub and xchain-indexer: CI stages -// only the public repos .ci-siblings names, cloned BESIDE this checkout, while -// the platform tree is private and sits as this checkout's PARENT. On a runner -// it is therefore always absent, and a guard that threw there would keep the -// gate red for a layout no roster line can fix (CI run 35020607785). This one -// guard takes its own switch instead: XCHAIN_REQUIRE_PLATFORM=1, exported by -// the venue that runs from the platform root (bin/ci-all.sh), turns an absent -// checkout into a failure; anywhere else the test is reported PENDING with the -// reason printed, never as a pass. XCHAIN_PLATFORM_ROOT relocates the root the -// same way the platform's own bin/ scripts accept it. +// The private platform tree is not a declared sibling. A platform-root hand +// run can require it with XCHAIN_REQUIRE_PLATFORM=1, and XCHAIN_PLATFORM_ROOT +// can relocate the checkout. const PLATFORM_ROOT = process.env.XCHAIN_PLATFORM_ROOT || path.resolve(__dirname, '..', '..', '..') const PLATFORM_DIR = path.join(PLATFORM_ROOT, 'claude') const AUDIT_TOOL = path.join(PLATFORM_DIR, 'bin', 'env-secret-name-audit.js') @@ -124,8 +137,15 @@ describe('secret-env', function () { it('agrees with the xchain-hub table on every key both own', function () { // xchain-node composes the hub container's env, so if the two tables // disagreed on a name the hub would boot without its DB password. - expect(fs.existsSync(HUB_TABLE), - 'xchain-hub secret-env table is missing at pinned path ' + HUB_TABLE).to.equal(true) + if (!HUB_TABLE_PRESENT) { + if (REQUIRE_SIBLINGS) + throw new Error('XCHAIN_REQUIRE_SIBLINGS=1 but xchain-hub secret-env table not found at ' + HUB_TABLE) + console.log('SKIP: secret-env xchain-hub alias-table agreement - xchain-hub is not checked out beside ' + + 'this repo at ' + HUB_TABLE + '; set XCHAIN_REQUIRE_SIBLINGS=1 with the sibling checked out to ' + + 'run this guard') + this.skip() + return + } const hubAliases = require(HUB_TABLE).SECRET_ENV_ALIASES for (const [legacy, preferred] of Object.entries(hubAliases)) { expect(secretEnv.SECRET_ENV_ALIASES[legacy], 'xchain-node is missing ' + legacy) From 8699e4229708abe997d84867fc73369e7a439301 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Thu, 24 Sep 2026 16:44:38 -0700 Subject: [PATCH 12/50] Bring integration tier to the current service contracts Stops go through the per-module stop budget, docker build matching skips the buildx probe, node and decoder credentials are seeded where the config service reads them, and the test env busts the snake_case config and module service modules so patched directories take effect instead of writing into the repo. --- test/integration/docker_commands.test.js | 6 +++--- ...uild_and_up_with_overwrite_container_id.test.js | 10 +++++++--- .../docker_build_working_directory.test.js | 2 +- .../docker_commands.test/support/fixture.js | 2 +- .../xchain_decoder_docker_run_command.test.js | 5 ++++- test/integration/helpers/test-env.js | 2 +- test/integration/hub_config_update.test.js | 6 ++++-- .../hub_config_update.test/support/fixture.js | 3 ++- test/integration/module_lifecycle.test.js | 14 ++++++++------ 9 files changed, 31 insertions(+), 19 deletions(-) diff --git a/test/integration/docker_commands.test.js b/test/integration/docker_commands.test.js index 8d5e9513..cf20d575 100644 --- a/test/integration/docker_commands.test.js +++ b/test/integration/docker_commands.test.js @@ -23,7 +23,7 @@ dockerSuite('xchain-encoder Docker run command', function (fixture) { const { ModuleService } = makeBuildAndUp() await ModuleService.buildAndUp('xchain-encoder', 'bitcoin', 'mainnet', null, true) - const buildCmds = capture.findCommands(/docker build/) + const buildCmds = capture.findCommands(/docker build /) expect(buildCmds).to.have.length(1) expect(buildCmds[0].command).to.include('-t xchain-node-bitcoin-mainnet-xchain-encoder') @@ -55,11 +55,11 @@ dockerSuite('xchain-encoder Docker run command', function (fixture) { expect(runCmd).to.not.include('NETWORK=bitcoin-mainnet') expect(runCmd).to.not.include('NODE_PORT=8332') expect(runCmd).to.not.include('ENCODER_API_PORT=3003') - expect(runCmd).to.not.include('NODE_URL=node') + expect(runCmd).to.not.include('NODE_URL=xchain-node-bitcoin-mainnet-node') expect(runEnv.NETWORK).to.equal('bitcoin-mainnet') expect(runEnv.NODE_PORT).to.equal('8332') expect(runEnv.ENCODER_API_PORT).to.equal('3003') - expect(runEnv.NODE_URL).to.equal('node') + expect(runEnv.NODE_URL).to.equal('xchain-node-bitcoin-mainnet-node') }) }) diff --git a/test/integration/docker_commands.test/build_and_up_with_overwrite_container_id.test.js b/test/integration/docker_commands.test/build_and_up_with_overwrite_container_id.test.js index c744c2c7..85a4e9b4 100644 --- a/test/integration/docker_commands.test/build_and_up_with_overwrite_container_id.test.js +++ b/test/integration/docker_commands.test/build_and_up_with_overwrite_container_id.test.js @@ -26,9 +26,13 @@ dockerSuite('buildAndUp with overwriteContainerId', function (fixture) { await ModuleService.buildAndUp('xchain-encoder', 'bitcoin', 'mainnet', oldContainerId, true) - const killCmds = capture.findCommands(/docker kill/) - expect(killCmds).to.have.length(1) - expect(killCmds[0].command).to.include(oldContainerId) + // The old container is stopped with the per-module stop budget + // (`docker stop -t `), never `docker kill`; the budget itself + // is pinned by test/unit/stop_budget_service.test.js. + expect(capture.findCommands(/docker kill/)).to.have.length(0) + const stopCmds = capture.findCommands(/docker stop -t \d+ /) + expect(stopCmds).to.have.length(1) + expect(stopCmds[0].command).to.include(oldContainerId) // Two `docker rm`s, not one: the explicit overwriteContainerId removal // above is id-keyed, plus buildAndUp also runs a name-keyed diff --git a/test/integration/docker_commands.test/docker_build_working_directory.test.js b/test/integration/docker_commands.test/docker_build_working_directory.test.js index c40c1b1d..acc67fde 100644 --- a/test/integration/docker_commands.test/docker_build_working_directory.test.js +++ b/test/integration/docker_commands.test/docker_build_working_directory.test.js @@ -23,7 +23,7 @@ dockerSuite('docker build working directory', function (fixture) { const { ModuleService } = makeBuildAndUp() await ModuleService.buildAndUp('xchain-encoder', 'bitcoin', 'mainnet', null, true) - const buildCmd = capture.findCommands(/docker build/)[0] + const buildCmd = capture.findCommands(/docker build /)[0] expect(buildCmd.options.cwd).to.include('xchain-encoder') }) }) diff --git a/test/integration/docker_commands.test/support/fixture.js b/test/integration/docker_commands.test/support/fixture.js index dc9592d0..220e3a33 100644 --- a/test/integration/docker_commands.test/support/fixture.js +++ b/test/integration/docker_commands.test/support/fixture.js @@ -29,7 +29,7 @@ function configureCapture(capture) { const parts = cmd.trim().split(/\s+/) return { stdout: parts[parts.length - 1] } } - capture.when(/docker kill/).respondsWith(extractId) + capture.when(/docker (kill|stop)/).respondsWith(extractId) capture.when(/docker rm/).respondsWith(extractId) // `git clone` is mocked out (no real network/process runs), but // buildAndUp's LIBRARY_BUNDLES staging (ea43475) does a REAL diff --git a/test/integration/docker_commands.test/xchain_decoder_docker_run_command.test.js b/test/integration/docker_commands.test/xchain_decoder_docker_run_command.test.js index 36727851..4efab0e5 100644 --- a/test/integration/docker_commands.test/xchain_decoder_docker_run_command.test.js +++ b/test/integration/docker_commands.test/xchain_decoder_docker_run_command.test.js @@ -17,7 +17,10 @@ dockerSuite('xchain-decoder Docker run command', function (fixture) { it('includes database env vars and bootstrap volume', async function () { const { env, capture, makeBuildAndUp } = fixture() - env.writeConfigFile('bitcoin-mainnet', '') + // DECODER_DB_PASS is generated per install unless the config file + // carries it (test/unit/config_service.test/database_credentials.test.js); + // seed it so the value asserted below is fixed. + env.writeConfigFile('bitcoin-mainnet', 'DECODER_DB_PASS=xchain-password\n') env.createFakeModule('xchain-decoder') const { ModuleService } = makeBuildAndUp() diff --git a/test/integration/helpers/test-env.js b/test/integration/helpers/test-env.js index 76079237..85494d97 100644 --- a/test/integration/helpers/test-env.js +++ b/test/integration/helpers/test-env.js @@ -243,7 +243,7 @@ class TestEnv { // This forces them to re-require constants with patched values. this._cachedModules = [] for (const key of Object.keys(require.cache)) { - if (key.includes('ConfigService') || key.includes('ModuleService')) { + if (/[\/](config_service|module_service)([\/.]|$)/.test(key)) { this._cachedModules.push({ key, module: require.cache[key] }) delete require.cache[key] } diff --git a/test/integration/hub_config_update.test.js b/test/integration/hub_config_update.test.js index a3b17c89..f70f915f 100644 --- a/test/integration/hub_config_update.test.js +++ b/test/integration/hub_config_update.test.js @@ -86,7 +86,7 @@ hubConfigSuite('updateHubOrExplorer payload for hub', function (fixture) { // static "rpc"/"rpc" default (a well-known credential left on a // live stack). Seed both here so this test still asserts a known, // fixed value instead of a per-run random one. - env.writeConfigFile('bitcoin-mainnet', 'NODE_USER=rpc\nNODE_PASSWORD=rpc\n') + env.writeConfigFile('bitcoin-mainnet.local', 'NODE_USER=rpc\nNODE_PASSWORD=rpc\n') state.setStatusUpdated(true) state.setLastStatus({ 'bitcoin': { @@ -106,7 +106,9 @@ hubConfigSuite('updateHubOrExplorer payload for hub', function (fixture) { const nodeConfig = config['bitcoin']['mainnet']['node'] expect(nodeConfig).to.exist - expect(nodeConfig.host).to.equal('node') + // NODE_URL is the coin-scoped container name; pinned by + // test/unit/config_service.test/with_coin_and_network.test.js. + expect(nodeConfig.host).to.equal('xchain-node-bitcoin-mainnet-node') expect(nodeConfig.port).to.equal(8332) expect(nodeConfig.user).to.equal('rpc') expect(nodeConfig.pass).to.equal('rpc') diff --git a/test/integration/hub_config_update.test/support/fixture.js b/test/integration/hub_config_update.test/support/fixture.js index f6869cdc..cd7c064f 100644 --- a/test/integration/hub_config_update.test/support/fixture.js +++ b/test/integration/hub_config_update.test/support/fixture.js @@ -51,7 +51,8 @@ function makeHubService(httpCapture, options = {}) { buildAndUp: async () => TestEnv.fakeContainerId('h') }, '../utils/helpers': { - sleep: async () => {} + sleep: async () => {}, + redactSecrets: require('../../../../src/utils/helpers').redactSecrets } }) diff --git a/test/integration/module_lifecycle.test.js b/test/integration/module_lifecycle.test.js index f65f66c5..50b5eaa2 100644 --- a/test/integration/module_lifecycle.test.js +++ b/test/integration/module_lifecycle.test.js @@ -219,10 +219,11 @@ describe('Integration: Module Lifecycle (LevelDB state)', function () { const moduleOps = proxyquire('../../src/operations/module_operations', { '../services/docker_service': { - stopContainer: async (id) => { - capture._history.push({ command: 'docker stop ' + id, type: 'exec', options: {} }) - return true + stopContainerByName: async (id, seconds) => { + capture._history.push({ command: 'docker stop -t ' + seconds + ' ' + id, type: 'exec', options: {} }) + return { stopped: true, seconds: 1, killed: false, exitCode: 0 } }, + getContainerStopSettings: async () => null, createDockerNetwork: async () => true, startContainer: async () => true, restartContainer: async () => true, @@ -262,10 +263,11 @@ describe('Integration: Module Lifecycle (LevelDB state)', function () { const moduleOps = proxyquire('../../src/operations/module_operations', { '../services/docker_service': { - stopContainer: async (id) => { - capture._history.push({ command: 'docker stop ' + id, type: 'exec', options: {} }) - return true + stopContainerByName: async (id, seconds) => { + capture._history.push({ command: 'docker stop -t ' + seconds + ' ' + id, type: 'exec', options: {} }) + return { stopped: true, seconds: 1, killed: false, exitCode: 0 } }, + getContainerStopSettings: async () => null, createDockerNetwork: async () => true, startContainer: async () => true, restartContainer: async () => true, From 9969063d3ad107e69e84e1493c8390ca1a6df639 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Fri, 25 Sep 2026 12:36:09 -0700 Subject: [PATCH 13/50] Repoint node to the renamed SDK encoder helper Use requireEncoder for signer broadcasts and chained stake inputs. Align the validator test doubles with the renamed SDK surface. --- src/services/validator_service/signer.js | 2 +- src/services/validator_stake_service.js | 2 +- .../14_emitted_signer_post_funding_failures.test.js | 2 +- test/unit/validator_stake_service.test/stake_validator.test.js | 2 +- .../unit/validator_stake_service.test/unstake_validator.test.js | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/services/validator_service/signer.js b/src/services/validator_service/signer.js index 899afd5d..8092dcb8 100644 --- a/src/services/validator_service/signer.js +++ b/src/services/validator_service/signer.js @@ -126,7 +126,7 @@ module.exports = { // returned txid is the phase-2 (reveal) txid when two-phase encoding is // used; that is the transaction indexers decode. async broadcast(payload) { - const encoder = sdk._requireEncoder(); + const encoder = sdk.requireEncoder(); const txParams = { data: payload, pubkey: ADDRESS, change: ADDRESS, encoding: 'P2SH' }; if (FEE_PER_KB !== undefined) txParams.feePerKb = FEE_PER_KB; diff --git a/src/services/validator_stake_service.js b/src/services/validator_stake_service.js index d59953c7..6035906f 100644 --- a/src/services/validator_stake_service.js +++ b/src/services/validator_stake_service.js @@ -228,7 +228,7 @@ async function chainedInputs(sdk, address, prevTxid, timeoutMs) { const deadline = Date.now() + (timeoutMs || 90000) for (;;) { let utxos = null - try { utxos = await sdk._requireEncoder().getUTXOs(address) } catch { /* transient; retry */ } + try { utxos = await sdk.requireEncoder().getUTXOs(address) } catch { /* transient; retry */ } const outs = ((utxos && utxos.utxos) || []).filter(o => (o.fullTxid || o.txid) === prevTxid) if (outs.length) return outs const left = deadline - Date.now() diff --git a/test/unit/validator_service.test/14_emitted_signer_post_funding_failures.test.js b/test/unit/validator_service.test/14_emitted_signer_post_funding_failures.test.js index 8199380d..07c7a87b 100644 --- a/test/unit/validator_service.test/14_emitted_signer_post_funding_failures.test.js +++ b/test/unit/validator_service.test/14_emitted_signer_post_funding_failures.test.js @@ -80,7 +80,7 @@ function loadEmittedSigner(source, encoder) { if (id === 'path') return path if (id === 'dotenv') return { config: () => ({}) } if (id === '@dankest-llc/xchain-sdk') return { XChainSDK: function () { - this._requireEncoder = () => encoder + this.requireEncoder = () => encoder this.wallet = { signPsbt: () => ({ txHex: 'hex-1', txid: PHASE1 }), signRevealPsbt: () => ({ txHex: 'hex-2', txid: 'e'.repeat(64) }) diff --git a/test/unit/validator_stake_service.test/stake_validator.test.js b/test/unit/validator_stake_service.test/stake_validator.test.js index 6635d391..13ea8b85 100644 --- a/test/unit/validator_stake_service.test/stake_validator.test.js +++ b/test/unit/validator_stake_service.test/stake_validator.test.js @@ -80,7 +80,7 @@ function makeSdk(chain = {}) { }), // The chain link: each broadcast leaves a change output the next action // is funded from. Keyed by txid so the service's filter is exercised. - _requireEncoder: () => ({ + requireEncoder: () => ({ getUTXOs: async () => ({ utxos: chain.noChange ? [] : [ { txid: prevTxidRef.value || 'seed', fullTxid: prevTxidRef.value || 'seed', vout: 1, value: '150000', confirmations: 0 } ] }) diff --git a/test/unit/validator_stake_service.test/unstake_validator.test.js b/test/unit/validator_stake_service.test/unstake_validator.test.js index fa88e667..ddbdca08 100644 --- a/test/unit/validator_stake_service.test/unstake_validator.test.js +++ b/test/unit/validator_stake_service.test/unstake_validator.test.js @@ -80,7 +80,7 @@ function makeSdk(chain = {}) { }), // The chain link: each broadcast leaves a change output the next action // is funded from. Keyed by txid so the service's filter is exercised. - _requireEncoder: () => ({ + requireEncoder: () => ({ getUTXOs: async () => ({ utxos: chain.noChange ? [] : [ { txid: prevTxidRef.value || 'seed', fullTxid: prevTxidRef.value || 'seed', vout: 1, value: '150000', confirmations: 0 } ] }) From 29f26138a0cf36639f8655d2a7bae36509458968 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Fri, 25 Sep 2026 12:40:06 -0700 Subject: [PATCH 14/50] fix(node): report cross-coin credential update outcomes When a module rebuild succeeds but credential reconciliation fails for another coin, update now reports the partial outcome and exits nonzero. --- src/cli/commands.js | 10 ++ .../module_operations/update_modules.js | 67 ++++++---- test/unit/cli_error_handling.test.js | 13 ++ .../20_update_partial_outcome.test.js | 114 ++++++++++++++++++ 4 files changed, 179 insertions(+), 25 deletions(-) create mode 100644 test/unit/module_operations.test/20_update_partial_outcome.test.js diff --git a/src/cli/commands.js b/src/cli/commands.js index 9fbbb36b..2be4374e 100644 --- a/src/cli/commands.js +++ b/src/cli/commands.js @@ -116,6 +116,16 @@ function registerUpdate(program, deps) { console.error('update failed: nothing was updated' + (why ? ' - ' + why : ' (no requested service matched an installed container)')) return process.exit(1) } + if (outcome && Array.isArray(outcome.failed) && outcome.failed.length > 0) { + const updated = outcome.updated + .map(item => `${item.module} (${item.coin} ${item.network})`) + .join('; ') + const why = outcome.failed + .map(f => `${f.module} (${f.coin} ${f.network}): ${f.reason}`) + .join('; ') + console.error('update: partially failed - updated: ' + updated + '; failed: ' + redactSecrets(why)) + return process.exit(1) + } return process.exit(0) }) diff --git a/src/operations/module_operations/update_modules.js b/src/operations/module_operations/update_modules.js index 30f815fa..7982ee4c 100644 --- a/src/operations/module_operations/update_modules.js +++ b/src/operations/module_operations/update_modules.js @@ -129,18 +129,32 @@ function recordInstallOutcome(outcome, result, module, coin, network) { } } -/** - * Runs the update over every requested module and REPORTS what it did. - * - * The report exists because the old `return true` made "updated three - * containers" and "matched nothing at all" indistinguishable to the caller, so - * a run that changed nothing still exited 0 and read as a landed deploy. The - * caller (cli `update`) turns an empty `updated` list into a non-zero exit. - * - * @returns {Promise<{updated: Array, skipped: Array}>} - */ +function credentialDriftScope(err) { + if (!err || err.code !== 'DB_CREDENTIAL_DRIFT') return null + const match = String(err.message || '').match(/^Refusing to rotate the (\S+) (\S+) MariaDB accounts:/) + return match ? { coin: match[1], network: match[2] } : null +} + +async function recordCredentialPartialOutcome(outcome, err, module, coin, network, previousContainerId) { + const reason = err && err.message ? err.message : String(err) + const driftScope = credentialDriftScope(err) + if (!driftScope || (driftScope.coin === coin && driftScope.network === network)) throw err + + let currentContainerId + try { + currentContainerId = await db.getModuleContainer(module, coin, network) + } catch { + throw err + } + if (!currentContainerId || currentContainerId === previousContainerId) throw err + + console.warn(`update: ${module} (${coin} ${network}) was rebuilt, but credential reconciliation failed for ${driftScope.coin} ${driftScope.network}: ${reason}`) + outcome.updated.push({ module, coin, network }) + outcome.failed.push({ module, coin: driftScope.coin, network: driftScope.network, reason }) +} + async function updateModulesOnBranch(servicesList, branch = null, { skipCurrentNode = false, quietNotInstalled = false } = {}) { - const outcome = { updated: [], skipped: [] } + const outcome = { updated: [], skipped: [], failed: [] } try { await updateModulesInto(outcome, servicesList, branch, { skipCurrentNode, quietNotInstalled }) } finally { @@ -187,21 +201,24 @@ async function updateModulesInto(outcome, servicesList, branch, { skipCurrentNod outcome.skipped.push({ module: nextModule, coin: nextCoin, network: nextNetwork, reason: 'not-installed' }) continue } - let moduleBranch = branch - if (!moduleBranch) { - try { moduleBranch = await getModuleBranch(nextModule) } catch { /* use default */ } - // A pinned checkout is detached and answers `HEAD`, which is not a branch anything can clone. Under a release update the manifest pin decides the ref anyway; for a component the manifest does not carry, null means the default branch, the same thing `install` would do. - if (moduleBranch === 'HEAD') moduleBranch = null + try { + let moduleBranch = branch + if (!moduleBranch) { + try { moduleBranch = await getModuleBranch(nextModule) } catch { /* use default */ } + // A pinned checkout is detached and answers `HEAD`, which is not a branch anything can clone. Under a release update the manifest pin decides the ref anyway; for a component the manifest does not carry, null means the default branch, the same thing `install` would do. + if (moduleBranch === 'HEAD') moduleBranch = null + } + // remoteUpdate=true so installModule actually rebuilds the container. Without it, the `if (!containerNodeVersion || remoteUpdate)` guard short-circuits for any already-installed service and `update` becomes a silent no-op. Version-skew guard: a hub-dependent service whose new source declares `xchainRequiresHub` in its package.json is REFUSED when the installed hub is behind that version, before anything is torn down. Under a pinned update the guard must read the PINNED source's package.json, not the branch tip: it clones into a tmp tree to find `xchainRequiresHub`, and reading that from a different ref than the one about to be installed is how a skew guard blesses a version it never saw. + const { resolveComponentRef } = releaseManifestService + const pin = resolveComponentRef(nextModule, moduleBranch) + await assertHubNotBehind(nextModule, pin.ref) + await assertRequiredMigrationsApplied(nextModule, nextCoin, nextNetwork, pin.ref) + // moduleBranch MUST be threaded through: installModule re-clones the module on the remoteUpdate path (cloneGit with this `branch`), so a null branch here re-clones the default branch and clobbers the branch the operator asked for (the cause of `update ` silently deploying master). installModule does the clone, so no separate cloneGit is needed here. + const rebuilt = await installModule(nextModule, nextCoin, nextNetwork, true, moduleContainerId, false, moduleBranch) + recordInstallOutcome(outcome, rebuilt, nextModule, nextCoin, nextNetwork) + } catch (err) { + await recordCredentialPartialOutcome(outcome, err, nextModule, nextCoin, nextNetwork, moduleContainerId) } - // remoteUpdate=true so installModule actually rebuilds the container. Without it, the `if (!containerNodeVersion || remoteUpdate)` guard short-circuits for any already-installed service and `update` becomes a silent no-op. Version-skew guard: a hub-dependent service whose new source declares `xchainRequiresHub` in its package.json is REFUSED when the installed hub is behind that version, before anything is torn down. Throws out of updateModules so the update fails closed with nothing modified for this module. Under a pinned update the guard must read the PINNED source's package.json, not the branch tip: it clones into a tmp tree to find `xchainRequiresHub`, and reading that from a different ref than the one about to be installed is how a skew guard blesses a version it never saw. - const { resolveComponentRef } = releaseManifestService - const pin = resolveComponentRef(nextModule, moduleBranch) - await assertHubNotBehind(nextModule, pin.ref) - // Migration-precondition guard: a service whose new source asserts a GATED (mode=manual) migration at startup is REFUSED when the database it will use has not applied that migration, before anything is torn down. Without it the only thing that discovers the requirement is the recreated container crash-looping - which is exactly how a routine indexer deploy took all three mainnet indexers down on 2026-08-09. Reads the same PINNED ref as the skew guard above, for the same reason: a precondition read from a different ref than the one being installed is a check that blessed a version it never saw. - await assertRequiredMigrationsApplied(nextModule, nextCoin, nextNetwork, pin.ref) - // moduleBranch MUST be threaded through: installModule re-clones the module on the remoteUpdate path (cloneGit with this `branch`), so a null branch here re-clones the default branch and clobbers the branch the operator asked for (the cause of `update ` silently deploying master). installModule does the clone, so no separate cloneGit is needed here. - const rebuilt = await installModule(nextModule, nextCoin, nextNetwork, true, moduleContainerId, false, moduleBranch) - recordInstallOutcome(outcome, rebuilt, nextModule, nextCoin, nextNetwork) } } } diff --git a/test/unit/cli_error_handling.test.js b/test/unit/cli_error_handling.test.js index ce211f98..66200af1 100644 --- a/test/unit/cli_error_handling.test.js +++ b/test/unit/cli_error_handling.test.js @@ -109,6 +109,19 @@ describe('CLI `update` exit code', function () { expect(exitStub.calledWith(0)).to.be.true expect(exitStub.calledWith(1)).to.be.false }) + + it('reports both halves when part of the run updated and part refused', async function () { + const program = loadUpdateAction(sinon.stub().resolves({ + updated: [{ module: 'xchain-indexer', coin: 'dogecoin', network: 'regtest' }], + skipped: [], + failed: [{ module: 'xchain-indexer', coin: 'bitcoin', network: 'regtest', reason: 'Refusing to rotate the bitcoin regtest MariaDB accounts' }] + })) + await runUpdate(program) + expect(errorStub.calledWithMatch(/updated: xchain-indexer \(dogecoin regtest\)/)).to.be.true + expect(errorStub.calledWithMatch(/failed: xchain-indexer \(bitcoin regtest\).*Refusing to rotate/)).to.be.true + expect(exitStub.calledWith(1)).to.be.true + expect(exitStub.calledWith(0)).to.be.false + }) }) // Same shape as the `update` no-op above: `recreate` printed "recreate does not diff --git a/test/unit/module_operations.test/20_update_partial_outcome.test.js b/test/unit/module_operations.test/20_update_partial_outcome.test.js new file mode 100644 index 00000000..1462cf15 --- /dev/null +++ b/test/unit/module_operations.test/20_update_partial_outcome.test.js @@ -0,0 +1,114 @@ +'use strict' + +// GENERATED +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +const { sinon, expect, makeStubs, loadOperations, registerLifecycleHooks } = require('./helpers/harness') + +describe('moduleOperations updateModules(): partial update continuation', function () { + registerLifecycleHooks(() => {}) + + it('continues after reporting unrelated credential drift from a confirmed rebuild', async function () { + const stubs = makeStubs() + const refusal = new Error( + 'Refusing to rotate the bitcoin regtest MariaDB accounts: a running container uses another password' + ) + refusal.code = 'DB_CREDENTIAL_DRIFT' + stubs.db.getModuleContainer.onFirstCall().resolves('old-dogecoin-container') + stubs.db.getModuleContainer.onSecondCall().resolves('new-dogecoin-container') + stubs.db.getModuleContainer.onThirdCall().resolves('litecoin-container') + stubs.installModule.onFirstCall().rejects(refusal) + stubs.installModule.onSecondCall().resolves('new-litecoin-container') + const ops = loadOperations(stubs) + const warn = sinon.stub(console, 'warn') + let outcome + try { + outcome = await ops.updateModules({ + dogecoin: { regtest: ['xchain-indexer'] }, + litecoin: { regtest: ['xchain-encoder'] } + }) + } finally { warn.restore() } + + expect(outcome.updated).to.deep.equal([ + { module: 'xchain-indexer', coin: 'dogecoin', network: 'regtest' }, + { module: 'xchain-encoder', coin: 'litecoin', network: 'regtest' } + ]) + expect(outcome.failed).to.deep.equal([{ + module: 'xchain-indexer', + coin: 'bitcoin', + network: 'regtest', + reason: refusal.message + }]) + expect(warn.calledWithMatch(/dogecoin regtest.*was rebuilt.*bitcoin regtest/)).to.be.true + expect(stubs.installModule.callCount).to.equal(2) + }) + + it('does not infer a rebuild from an unrelated coin named in the error', async function () { + const stubs = makeStubs() + const refusal = new Error( + 'Refusing to rotate the bitcoin regtest MariaDB accounts: a running container uses another password' + ) + refusal.code = 'DB_CREDENTIAL_DRIFT' + stubs.installModule.rejects(refusal) + const ops = loadOperations(stubs) + + let err + try { + await ops.updateModules({ dogecoin: { regtest: ['xchain-indexer'] } }) + } catch (caught) { + err = caught + } + + expect(err).to.equal(refusal) + expect(stubs.db.getModuleContainer.callCount).to.equal(2) + }) + + it('rethrows ordinary install failures without attempting later coins', async function () { + const stubs = makeStubs() + const failure = new Error('docker: no space left on device') + stubs.installModule.rejects(failure) + const ops = loadOperations(stubs) + + let err + try { + await ops.updateModules({ + bitcoin: { regtest: ['xchain-encoder'] }, + dogecoin: { regtest: ['xchain-encoder'] } + }) + } catch (caught) { + err = caught + } + + expect(err).to.equal(failure) + expect(stubs.installModule.calledOnce).to.be.true + }) + + it('fails closed when the credential refusal belongs to the requested coin', async function () { + const stubs = makeStubs() + const refusal = new Error( + 'Refusing to rotate the dogecoin regtest MariaDB accounts: a running container uses another password' + ) + refusal.code = 'DB_CREDENTIAL_DRIFT' + stubs.installModule.rejects(refusal) + const ops = loadOperations(stubs) + const warn = sinon.stub(console, 'warn') + + let err + try { + await ops.updateModules({ dogecoin: { regtest: ['xchain-indexer'] } }) + } catch (caught) { + err = caught + } finally { warn.restore() } + + expect(err).to.equal(refusal) + expect(stubs.db.getModuleContainer.calledOnce).to.be.true + }) +}) From e29504fe61d69884bfe2b44a6a8c304c6c205be2 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Fri, 25 Sep 2026 12:50:58 -0700 Subject: [PATCH 15/50] build: time each ci:full tier Each tier's PASS or FAIL line in ci:full now carries its elapsed seconds. --- bin/ci-full.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/ci-full.sh b/bin/ci-full.sh index 09b33bf7..758b4507 100755 --- a/bin/ci-full.sh +++ b/bin/ci-full.sh @@ -66,21 +66,24 @@ ci_tier_deferred() { return 1 } # <<< ci-tier <<< +# >>> ci-tier timer (generated block; re-run the tier wirer to update) >>> run_tier() { ci_tier_deferred "$1" && return 0 # ci-tier guard (generated) local name="$1"; shift + local __ci_tier_t0=$SECONDS echo; echo "ci:full ===== $name =====" local root root="$(mktemp -d "${TMPDIR:-/tmp}/xchain-node-ci-full.XXXXXX")" mkdir -p "$root/config" "$root/data" if ( export XCHAIN_NODE_CONFIG_DIR="$root/config" XCHAIN_NODE_DATA_DIR="$root/data"; "$@" ); then - echo "ci:full ----- $name PASS" + echo "ci:full ----- $name PASS ($(( SECONDS - __ci_tier_t0 ))s)" else FAILED="$FAILED [$name]" - echo "ci:full ----- $name FAIL" + echo "ci:full ----- $name FAIL ($(( SECONDS - __ci_tier_t0 ))s)" fi rm -rf -- "$root" } +# <<< ci-tier timer <<< need_sib() { local s for s in "$@"; do From 789e163bce006d50cbd4f0a186474ce9a872ed19 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Fri, 25 Sep 2026 13:30:31 -0700 Subject: [PATCH 16/50] test: stabilize the Stryker dry run over the full suite Stryker now mutates the config service too, runs unit and integration tests with a module shim its sandbox needs, and uses a non-dotted temp directory. --- stryker.config.json | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/stryker.config.json b/stryker.config.json index ffe3e8f6..bfd4ff92 100644 --- a/stryker.config.json +++ b/stryker.config.json @@ -4,6 +4,7 @@ "src/cli.js", "src/config/index.js", "src/services/config_service.js", + "src/services/config_service/**", "src/services/module_service.js", "src/services/docker_service.js", "src/services/database_service.js", @@ -11,10 +12,13 @@ "src/services/status_service.js" ], "testRunner": "mocha", + "testRunnerNodeArgs": [ + "--import=data:text/javascript,process.mainModule??={}" + ], "mochaOptions": { "spec": [ - "test/unit/**/*.test.js", - "test/integration/**/*.test.js" + "test/integration/**/*.test.js", + "test/unit/**/*.test.js" ], "no-config": true, "no-opts": true @@ -37,7 +41,7 @@ "break": 60 }, "concurrency": 4, - "tempDirName": ".stryker-tmp", + "tempDirName": "stryker-tmp", "mutator": { "excludedMutations": [ "StringLiteral" From e453c0bc08f468af9dd934992ed846b06d775be3 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Fri, 25 Sep 2026 14:04:24 -0700 Subject: [PATCH 17/50] fix(node): validate the running indexer migrate CLI status The migration precondition scan now checks the migrate CLI's --status answer from the running indexer instead of assuming it, with tests for the probe. --- .../migration_scan.js | 49 ++++++++---- src/utils/indexer_migrate_cli.js | 8 +- test/unit/utils/indexer_migrate_cli.test.js | 80 ++++++++++++++----- 3 files changed, 92 insertions(+), 45 deletions(-) diff --git a/src/services/migration_precondition_service/migration_scan.js b/src/services/migration_precondition_service/migration_scan.js index 6c5adae1..e3f0e7a5 100644 --- a/src/services/migration_precondition_service/migration_scan.js +++ b/src/services/migration_precondition_service/migration_scan.js @@ -85,26 +85,41 @@ function pendingManualMigrations(dir, applied) { }).map(([f]) => f) } -/** - * Does the build CURRENTLY RUNNING in the target container understand per-file - * migration targeting (`--file`)? - * - * This matters because the remedy an operator is about to run executes inside - * that container, on its build, not on the one being deployed. A build without - * the flag does not reject it: it ignores it and applies every pending manual - * migration, which on a live database can mean a data backfill and a - * dedup-then-unique nobody authorised. - * - * Returns true, false, or null when the container could not be read at all - * (stopped, absent, docker unreachable). Callers must treat null like false: - * an unverified capability is not a capability, and the cost of being wrong is - * asymmetric. - */ +/** Verify the running container's migrate CLI through its read-only status contract. */ async function runningBuildSupportsPerFileMigrations(container, deps = {}) { try { const cat = deps.getDockerContainerFileCat || require('../docker_service').getDockerContainerFileCat const found = await readMigrateCli(cat, container) - return found ? /['"]--file['"]/.test(found.source) : null + if (!found) return null + + const execContainer = deps.execContainer || require('../docker_service').execContainer + let raw + try { + raw = await execContainer(container, ['node', found.cliPath, '--status', '--json']) + } catch { + return false + } + + let status + try { + status = JSON.parse(String(raw)) + } catch { + return false + } + if (!status || Array.isArray(status) || typeof status !== 'object') return false + if (typeof status.database !== 'string' || status.database.length === 0) return false + if (!Array.isArray(status.migrations)) return false + for (const row of status.migrations) { + if (!row || Array.isArray(row) || typeof row !== 'object') return false + if (typeof row.file !== 'string' || row.file.length === 0) return false + if (typeof row.applied !== 'boolean') return false + } + + const counts = ['total', 'applied', 'pending'] + if (!counts.every(k => Number.isInteger(status[k]) && status[k] >= 0)) return false + const applied = status.migrations.filter(row => row.applied).length + return status.total === status.migrations.length && + status.applied === applied && status.pending === status.total - applied } catch { return null } @@ -233,7 +248,7 @@ function refusalMessage(module, coin, network, dbName, missing, remedy = {}) { : missing instructions = 'DO NOT run `node ' + migrateCliPathFor(container) + '` inside ' + container + '. ' + (remedy.supportsPerFile === false - ? 'That container runs a build with no per-file targeting: it ignores --file' + ? 'That container did not return a valid --status --json response, so --file support is not verified' : 'Whether that container\'s build honours --file could not be read, and an unverified capability is not one: it may ignore --file') + ' and apply EVERY pending manual migration on ' + dbName + ', which is ' + wouldApply.length + ' file(s):\n' + diff --git a/src/utils/indexer_migrate_cli.js b/src/utils/indexer_migrate_cli.js index 4ed911a2..df4041fe 100644 --- a/src/utils/indexer_migrate_cli.js +++ b/src/utils/indexer_migrate_cli.js @@ -29,13 +29,7 @@ const MIGRATE_CLI_PATHS = ['src/db/migration/migrate.js', 'src/migrate.js'] // found there rather than the layout of the source about to be deployed. const foundPaths = new Map() -/** - * Read the migrate CLI's source out of `container` through - * `cat(container, path)`, trying each known path in order. A path that throws - * or reads empty counts as absent on that build. Returns { cliPath, source } - * for the first path that answers, or null when none does (a stopped - * container, docker unreachable, or a build with no CLI at a known path). - */ +/** Locate the first readable migrate CLI path known for this container build. */ async function readMigrateCli(cat, container) { for (const cliPath of MIGRATE_CLI_PATHS) { let source = null diff --git a/test/unit/utils/indexer_migrate_cli.test.js b/test/unit/utils/indexer_migrate_cli.test.js index 24942dfd..a2deb926 100644 --- a/test/unit/utils/indexer_migrate_cli.test.js +++ b/test/unit/utils/indexer_migrate_cli.test.js @@ -28,9 +28,17 @@ const GATED = '2026-07-24-pubkeys-widen-uncompressed.sql' const NEWEST_PATH = 'src/db/migration/migrate.js' const OLD_PATH = 'src/migrate.js' -// Source text for a CLI that parses --file, and for one that predates it. -const WITH_FILE = "if(a === '--file' || a === '-f'){ push(argv[i + 1]) }" -const WITHOUT_FILE = 'await db.runMigrations({ includeManual: true })' +const CLI_SOURCE = 'main()' +const VALID_STATUS = JSON.stringify({ + database: 'XChain_BTC_Mainnet_Indexer', + total: 2, + applied: 1, + pending: 1, + migrations: [ + { file: 'a.sql', applied: true, mode: 'auto', appliedAt: '2026-09-24T00:00:00.000Z' }, + { file: GATED, applied: false, mode: 'manual', appliedAt: null } + ] +}) // A stand-in for `docker exec cat `: it answers only at the paths // the fake build carries and rejects elsewhere, the way cat exits non-zero on @@ -42,6 +50,10 @@ function catFor(files) { }) } +function execReturning(output) { + return sinon.stub().resolves(output) +} + function registerPathPrecedenceTests() { // Pins the full candidate list, newest first, so a future indexer layout // move that edits MIGRATE_CLI_PATHS without adding the new path (or drops @@ -54,26 +66,26 @@ function registerPathPrecedenceTests() { describe('readMigrateCli', () => { it('reads the v0.19.0+ db/migration layout before the older path', async () => { - const cat = catFor({ [NEWEST_PATH]: WITH_FILE, [OLD_PATH]: WITHOUT_FILE }) + const cat = catFor({ [NEWEST_PATH]: CLI_SOURCE, [OLD_PATH]: CLI_SOURCE }) const found = await readMigrateCli(cat, 'c-newest') - expect(found).to.deep.equal({ cliPath: NEWEST_PATH, source: WITH_FILE }) + expect(found).to.deep.equal({ cliPath: NEWEST_PATH, source: CLI_SOURCE }) expect(cat.firstCall.args[1]).to.equal(NEWEST_PATH) expect(migrateCliPathFor('c-newest')).to.equal(NEWEST_PATH) }) it('falls back to the pre-move path on a build that still carries it', async () => { - const found = await readMigrateCli(catFor({ [OLD_PATH]: WITH_FILE }), 'c-old') - expect(found).to.deep.equal({ cliPath: OLD_PATH, source: WITH_FILE }) + const found = await readMigrateCli(catFor({ [OLD_PATH]: CLI_SOURCE }), 'c-old') + expect(found).to.deep.equal({ cliPath: OLD_PATH, source: CLI_SOURCE }) expect(migrateCliPathFor('c-old')).to.equal(OLD_PATH) }) it('counts an empty read as absent, as the single-path probe did', async () => { - const found = await readMigrateCli(catFor({ [NEWEST_PATH]: '', [OLD_PATH]: WITH_FILE }), 'c-empty') + const found = await readMigrateCli(catFor({ [NEWEST_PATH]: '', [OLD_PATH]: CLI_SOURCE }), 'c-empty') expect(found.cliPath).to.equal(OLD_PATH) }) it('answers null and forgets what it found before when no known path answers', async () => { - await readMigrateCli(catFor({ [OLD_PATH]: WITH_FILE }), 'c-gone') + await readMigrateCli(catFor({ [OLD_PATH]: CLI_SOURCE }), 'c-gone') expect(await readMigrateCli(catFor({}), 'c-gone')).to.equal(null) expect(migrateCliPathFor('c-gone')).to.equal(MIGRATE_CLI_PATHS[0]) }) @@ -82,24 +94,41 @@ function registerPathPrecedenceTests() { function registerCapabilityDetectionTests() { describe('runningBuildSupportsPerFileMigrations', () => { - it('sees --file support on a build from before the move', async () => { - const deps = { getDockerContainerFileCat: catFor({ [OLD_PATH]: WITH_FILE }) } + it('accepts a valid status response from a build before the path move', async () => { + const execContainer = execReturning(VALID_STATUS) + const deps = { getDockerContainerFileCat: catFor({ [OLD_PATH]: CLI_SOURCE }), execContainer } expect(await runningBuildSupportsPerFileMigrations('c-probe-old', deps)).to.equal(true) + expect(execContainer.calledOnceWith('c-probe-old', ['node', OLD_PATH, '--status', '--json'])).to.equal(true) }) - it('sees --file support on a build from after the move', async () => { - const deps = { getDockerContainerFileCat: catFor({ [NEWEST_PATH]: WITH_FILE }) } + it('accepts a valid status response from a build after the path move', async () => { + const execContainer = execReturning(VALID_STATUS) + const deps = { getDockerContainerFileCat: catFor({ [NEWEST_PATH]: CLI_SOURCE }), execContainer } expect(await runningBuildSupportsPerFileMigrations('c-probe-new', deps)).to.equal(true) + expect(execContainer.calledOnceWith('c-probe-new', ['node', NEWEST_PATH, '--status', '--json'])).to.equal(true) + }) + + it('rejects malformed status JSON', async () => { + const deps = { + getDockerContainerFileCat: catFor({ [OLD_PATH]: CLI_SOURCE }), + execContainer: execReturning('{"database":') + } + expect(await runningBuildSupportsPerFileMigrations('c-probe-malformed', deps)).to.equal(false) }) - it('reports a build whose CLI has no --file as lacking it', async () => { - const deps = { getDockerContainerFileCat: catFor({ [OLD_PATH]: WITHOUT_FILE }) } - expect(await runningBuildSupportsPerFileMigrations('c-probe-bare', deps)).to.equal(false) + it('reports a CLI without --status support as lacking the capability', async () => { + const deps = { + getDockerContainerFileCat: catFor({ [OLD_PATH]: CLI_SOURCE }), + execContainer: sinon.stub().rejects(new Error('unrecognized argument "--status"')) + } + expect(await runningBuildSupportsPerFileMigrations('c-probe-no-status', deps)).to.equal(false) }) it('answers null for a container with no CLI at any known path', async () => { - const deps = { getDockerContainerFileCat: catFor({}) } + const execContainer = execReturning(VALID_STATUS) + const deps = { getDockerContainerFileCat: catFor({}), execContainer } expect(await runningBuildSupportsPerFileMigrations('c-probe-none', deps)).to.equal(null) + expect(execContainer.called).to.equal(false) }) }) } @@ -112,13 +141,14 @@ function registerRefusalBehaviorTests() { // The real probe runs here (no runningBuildSupportsPerFileMigrations stub), // so the printed command comes from what the fake container carries. - async function refusalFor(files) { + async function refusalFor(files, execContainer = execReturning(VALID_STATUS)) { const deps = { cloneGit: sinon.stub().resolves(), listDeployPreconditionMigrations: sinon.stub().returns([GATED]), readAppliedMigrations: sinon.stub().resolves({ state: 'ledger', applied: new Set() }), pendingManualMigrations: sinon.stub().returns([GATED]), - getDockerContainerFileCat: catFor(files) + getDockerContainerFileCat: catFor(files), + execContainer } try { await assertRequiredMigrationsApplied(XChainService.XCHAIN_INDEXER, 'bitcoin', 'mainnet', 'master', deps) @@ -129,18 +159,26 @@ function registerRefusalBehaviorTests() { } it('names the pre-move CLI path when the running build carries only that', async () => { - const message = await refusalFor({ [OLD_PATH]: WITH_FILE }) + const message = await refusalFor({ [OLD_PATH]: CLI_SOURCE }) expect(message, 'the deploy must be refused').to.not.equal(null) expect(message).to.contain('node ' + OLD_PATH + ' --file ' + GATED) }) it('names the v0.19.0+ db/migration CLI path when the running build carries that', async () => { - const message = await refusalFor({ [NEWEST_PATH]: WITH_FILE }) + const message = await refusalFor({ [NEWEST_PATH]: CLI_SOURCE }) expect(message, 'the deploy must be refused').to.not.equal(null) expect(message).to.contain('node ' + NEWEST_PATH + ' --file ' + GATED) expect(message).to.not.contain('node ' + OLD_PATH) }) + it('keeps the refusal and omits a scoped command when the CLI rejects --status', async () => { + const execContainer = sinon.stub().rejects(new Error('unrecognized argument "--status"')) + const message = await refusalFor({ [OLD_PATH]: CLI_SOURCE }, execContainer) + expect(message, 'the deploy must be refused').to.not.equal(null) + expect(message).to.contain('did not return a valid --status --json response') + expect(message).to.not.contain('docker exec') + }) + it('still refuses, naming the newest layout as the remedy, when the container answers at none of the known paths', async () => { const message = await refusalFor({}) expect(message, 'the deploy must be refused').to.not.equal(null) From 85de032f2c234eeedf4a8bb5994a2edb66875410 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Fri, 25 Sep 2026 20:06:20 -0700 Subject: [PATCH 18/50] fix(validator): use regtest namespaces in validator links and encoder URL Regtest mapped the stake chain to BTC and the DOGE publisher to DOGE, so regtest stake and unstake output linked mainnet /BTC/validator pages and the default signer encoder pointed at the mainnet /DOGE namespace. Regtest now maps to RBTC and RDOGE, and the tests assert full URLs on every network. --- .../validator_service/validator_paths.js | 2 +- .../12_coin_wallets.test.js | 31 ++++++++++++------- .../stake_validator.test.js | 19 +++++++++++- .../unstake_validator.test.js | 19 +++++++++++- 4 files changed, 56 insertions(+), 15 deletions(-) diff --git a/src/services/validator_service/validator_paths.js b/src/services/validator_service/validator_paths.js index 549379af..b8625439 100644 --- a/src/services/validator_service/validator_paths.js +++ b/src/services/validator_service/validator_paths.js @@ -134,7 +134,7 @@ const ORACLE_EPOCH_START_BY_NETWORK = { mainnet: 1788220800000, testnet: 1787875 const COIN_NETWORKS = { mainnet: { stake: 'bitcoin-mainnet', doge: 'dogecoin-mainnet', stakeCoin: 'BTC', dogeCoin: 'DOGE' }, testnet: { stake: 'bitcoin-testnet', doge: 'dogecoin-testnet', stakeCoin: 'TBTC', dogeCoin: 'TDOGE' }, - regtest: { stake: 'bitcoin-regtest', doge: 'dogecoin-regtest', stakeCoin: 'BTC', dogeCoin: 'DOGE' } + regtest: { stake: 'bitcoin-regtest', doge: 'dogecoin-regtest', stakeCoin: 'RBTC', dogeCoin: 'RDOGE' } } const PUBLIC_ENCODER_BASE = 'https://encoder.xchain.io/' diff --git a/test/unit/validator_service.test/12_coin_wallets.test.js b/test/unit/validator_service.test/12_coin_wallets.test.js index 2a4c9815..bf4ac944 100644 --- a/test/unit/validator_service.test/12_coin_wallets.test.js +++ b/test/unit/validator_service.test/12_coin_wallets.test.js @@ -158,18 +158,25 @@ describe('ValidatorService', function () { delete process.env.XCHAIN_NODE_HUB_SIGNER_DIR }) - it('points the signer at the DOGE wallet and the public testnet encoder', async function () { - const fs = makeFs() - const vs = loadValidatorService(fs) - await vs.initValidator({ network: 'testnet' }) - const w = writtenWallets(fs) - const env = fs.writeFileSync.getCalls().find(c => c.args[0] === FAKE_SIGNER_ENV).args[1] - expect(env).to.include('DOGE_NETWORK=dogecoin-testnet') - expect(env).to.include('DOGE_ADDRESS=' + w.DOGE_ADDRESS) - expect(env).to.include('DOGE_WIF=' + w.DOGE_WIF_SECRET) - expect(env).to.include('DOGE_ENCODER_URL=https://encoder.xchain.io/TDOGE') - // The stake key is NOT in the mounted directory. - expect(env).to.not.include(w.STAKE_WIF_SECRET) + it('points the signer at the complete public encoder URL for every network', async function () { + const cases = [ + ['mainnet', 'DOGE'], + ['testnet', 'TDOGE'], + ['regtest', 'RDOGE'] + ] + for (const [network, coin] of cases) { + const fs = makeFs() + const vs = loadValidatorService(fs) + await vs.initValidator({ network }) + const w = writtenWallets(fs) + const env = fs.writeFileSync.getCalls().find(c => c.args[0] === FAKE_SIGNER_ENV).args[1] + expect(env, network).to.include('DOGE_NETWORK=dogecoin-' + network) + expect(env, network).to.include('DOGE_ADDRESS=' + w.DOGE_ADDRESS) + expect(env, network).to.include('DOGE_WIF=' + w.DOGE_WIF_SECRET) + expect(env, network).to.include('DOGE_ENCODER_URL=https://encoder.xchain.io/' + coin) + // The stake key is NOT in the mounted directory. + expect(env, network).to.not.include(w.STAKE_WIF_SECRET) + } }) it('fills oracle_publish in the fresh capabilities file from the DOGE wallet', async function () { diff --git a/test/unit/validator_stake_service.test/stake_validator.test.js b/test/unit/validator_stake_service.test/stake_validator.test.js index 13ea8b85..300e317a 100644 --- a/test/unit/validator_stake_service.test/stake_validator.test.js +++ b/test/unit/validator_stake_service.test/stake_validator.test.js @@ -95,9 +95,10 @@ function makeSdk(chain = {}) { function run(opts, chain, settingsExtra = {}) { const { sdk, calls } = makeSdk(chain) const logged = [] + const network = settingsExtra.network || 'testnet' const deps = { settings: { enabled: true, pubkey: PUBKEY, network: 'testnet', P2P_PORT: 10002, ...settingsExtra }, - wallets: { NETWORK: 'testnet', STAKE_ADDRESS: ADDRESS, STAKE_WIF_SECRET: 'cFakeWif' }, + wallets: { NETWORK: network, STAKE_ADDRESS: ADDRESS, STAKE_WIF_SECRET: 'cFakeWif' }, makeSdk: () => sdk, sdk: { XChainSDK: function () { throw new Error('makeSdk should be used') } }, log: m => logged.push(String(m)) @@ -224,6 +225,22 @@ describe('ValidatorStakeService', function () { expect(logged.join('\n')).to.include('Broadcast. Watch it land at') }) + it('prints the complete validator URL for every network', async function () { + const cases = [ + ['mainnet', 10001, 'BTC'], + ['testnet', 10002, 'TBTC'], + ['regtest', 10003, 'RBTC'] + ] + for (const [network, port, coin] of cases) { + const { logged } = await run( + { broadcast: true, wait: false }, + { xchain: 30000, coin: '0.001' }, + { network, P2P_PORT: port }) + expect(logged.join('\n'), network).to.include( + 'Broadcast. Watch it land at https://explorer.xchain.io/' + coin + '/validator/' + PUBKEY) + } + }) + it('passes --fee-per-kb through to the encoder', async function () { const { calls } = await run({ broadcast: true, feePerKb: '0.0001' }, { xchain: 25000, coin: '0.001' }) expect(calls.stake[0].enc).to.deep.equal({ feePerKb: 0.0001 }) diff --git a/test/unit/validator_stake_service.test/unstake_validator.test.js b/test/unit/validator_stake_service.test/unstake_validator.test.js index ddbdca08..21fc02ed 100644 --- a/test/unit/validator_stake_service.test/unstake_validator.test.js +++ b/test/unit/validator_stake_service.test/unstake_validator.test.js @@ -95,9 +95,10 @@ function makeSdk(chain = {}) { function runUnstake(opts, chain, settingsExtra = {}) { const { sdk, calls } = makeSdk(chain) const logged = [] + const network = settingsExtra.network || 'testnet' const deps = { settings: { enabled: true, pubkey: PUBKEY, network: 'testnet', P2P_PORT: 10002, ...settingsExtra }, - wallets: { NETWORK: 'testnet', STAKE_ADDRESS: ADDRESS, STAKE_WIF_SECRET: 'cFakeWif' }, + wallets: { NETWORK: network, STAKE_ADDRESS: ADDRESS, STAKE_WIF_SECRET: 'cFakeWif' }, makeSdk: () => sdk, sdk: {}, log: m => logged.push(String(m)) @@ -165,6 +166,22 @@ describe('ValidatorStakeService', function () { expect(out).to.include('cooldown sweep credits it back') }) + it('prints the complete validator URL for every network', async function () { + const cases = [ + ['mainnet', 10001, 'BTC'], + ['testnet', 10002, 'TBTC'], + ['regtest', 10003, 'RBTC'] + ] + for (const [network, port, coin] of cases) { + const { logged } = await runUnstake( + { broadcast: true }, + { existing: STAKED }, + { network, P2P_PORT: port }) + expect(logged.join('\n'), network).to.include( + 'Watch it at https://explorer.xchain.io/' + coin + '/validator/' + PUBKEY) + } + }) + // The old text promised spendability at the activation delay. Nothing in // either message may say the coins come back on that clock again. it('never claims the XCHAIN is spendable once the stake leaves the set', async function () { From 4d12a8bd2854e0016897f198381197b508621556 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Fri, 25 Sep 2026 20:49:53 -0700 Subject: [PATCH 19/50] Isolate unit test config writes Load the unit suite with a temporary config directory before production configuration is cached. Clean the scratch directory after the run and verify it stays outside the checkout. --- .../00_config_directory_isolation.test.js | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 test/unit/00_config_directory_isolation.test.js diff --git a/test/unit/00_config_directory_isolation.test.js b/test/unit/00_config_directory_isolation.test.js new file mode 100644 index 00000000..4ee6fd55 --- /dev/null +++ b/test/unit/00_config_directory_isolation.test.js @@ -0,0 +1,44 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +const { expect } = require('chai') +const fs = require('fs') +const os = require('os') +const path = require('path') + +const configModulePath = require.resolve('../../src/config') +const configWasLoaded = Boolean(require.cache[configModulePath]) +const unitConfigDir = fs.mkdtempSync(path.join(os.tmpdir(), 'xchain-node-unit-config-')) + +process.env.XCHAIN_NODE_CONFIG_DIR = unitConfigDir + +let cleaned = false +function cleanUnitConfigDir() { + if (cleaned) return + cleaned = true + fs.rmSync(unitConfigDir, { recursive: true, force: true }) +} + +process.once('exit', cleanUnitConfigDir) +after(cleanUnitConfigDir) + +describe('unit config directory isolation', function () { + it('takes effect before the production config module is loaded', function () { + expect(configWasLoaded).to.equal(false) + expect(require('../../src/config').configDir).to.equal(unitConfigDir) + }) + + it('uses an operating-system temporary directory outside the checkout', function () { + expect(unitConfigDir.startsWith(path.resolve(os.tmpdir()) + path.sep)).to.equal(true) + expect(unitConfigDir.startsWith(path.resolve(__dirname, '../..') + path.sep)).to.equal(false) + }) +}) From 933d2cbdb4372e07ef70d4d22658053acc368762 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Sat, 26 Sep 2026 02:57:16 -0700 Subject: [PATCH 20/50] fix(module-service): clone detached local module sources at their checked-out commit A local module source checked out at a detached HEAD was cloned by branch name, so a deploy picked up the branch tip instead of the checked-out commit. Detect a detached local source, clone it without -b, and skip the branch-tip check for it. A unit test behind the --unit-test entry point proves the deployed commit matches the detached commit. --- src/services/module_service/git_checkout.js | 57 +++++++++++++++++++-- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/src/services/module_service/git_checkout.js b/src/services/module_service/git_checkout.js index 31192afc..07862e56 100644 --- a/src/services/module_service/git_checkout.js +++ b/src/services/module_service/git_checkout.js @@ -16,6 +16,9 @@ ********************************************************************/ let { execFile } = require('child_process') +const assert = require('node:assert/strict') +const os = require('node:os') +const test = require('node:test') const { promisify } = require('util') let execFileAsync = promisify(execFile) let fs = require('fs') @@ -35,14 +38,26 @@ function configureDependencies(dependencies) { const CLONE_STAGING_SUFFIX = '.xchain-node-staging' const CLONE_PREVIOUS_SUFFIX = '.xchain-node-previous' +async function localSourceIsDetached(url) { + if (!isLocalPathSource(url)) return false + try { + const { stdout } = await execFileAsync('git', ['-C', url, 'rev-parse', '--abbrev-ref', 'HEAD']) + return String(stdout || '').trim() === 'HEAD' + } catch { + return false + } +} + // Run `git clone` into `destination`. Rejects with the operator-facing string // the callers already surface; performs no filesystem cleanup of its own so // the caller owns the rollback decision. -function runGitClone(module, branch, destination) { +async function runGitClone(module, branch, destination) { + const gitUrl = modulesUrls[module] + const cloneBranch = branch && await localSourceIsDetached(gitUrl) ? null : branch + return new Promise((resolve, reject) => { - const gitUrl = modulesUrls[module] const cloneArgs = ['clone'] - if (branch) cloneArgs.push('-b', branch) + if (cloneBranch) cloneArgs.push('-b', cloneBranch) // Local-path sources (no ':' i.e. not a URL/SCP-style remote) on the // Parallels share can't hardlink between the two trees, so force // a copy instead of git's default object-linking. @@ -237,6 +252,7 @@ async function verifyDeploySource(module, branch, dir, expectedCommit) { if (expectedCommit || !branch) return const url = modulesUrls[module] + if (await localSourceIsDetached(url)) return await warnIfSourceBranchIsBehind(module, url, branch) const tip = await readSourceBranchTip(url, branch) @@ -293,3 +309,38 @@ module.exports = { verifyDeploySource, reportDeployedSource } + +if (require.main === module && process.argv.includes('--unit-test')) { + test('runGitClone deploys the exact detached HEAD of a local source', async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'xchain-node-detached-clone-')) + const source = path.join(root, 'source') + const destination = path.join(root, 'destination') + const module = 'detached-head-unit-test' + const git = (args) => execFileAsync('git', args) + t.after(() => { + delete modulesUrls[module] + fs.rmSync(root, { recursive: true, force: true }) + }) + + await git(['init', '--initial-branch=main', source]) + await git(['-C', source, 'config', 'user.email', 'unit-test@example.invalid']) + await git(['-C', source, 'config', 'user.name', 'Unit Test']) + fs.writeFileSync(path.join(source, 'content.txt'), 'detached commit\n') + await git(['-C', source, 'add', 'content.txt']) + await git(['-C', source, 'commit', '-m', 'detached commit']) + const detachedCommit = String((await git(['-C', source, 'rev-parse', 'HEAD'])).stdout).trim() + + fs.writeFileSync(path.join(source, 'content.txt'), 'branch tip\n') + await git(['-C', source, 'commit', '-am', 'branch tip']) + const branchTip = String((await git(['-C', source, 'rev-parse', 'main'])).stdout).trim() + await git(['-C', source, 'checkout', '--detach', detachedCommit]) + modulesUrls[module] = source + + await runGitClone(module, 'main', destination) + await verifyDeploySource(module, 'main', destination) + + const deployedCommit = String((await git(['-C', destination, 'rev-parse', 'HEAD'])).stdout).trim() + assert.notStrictEqual(detachedCommit, branchTip) + assert.strictEqual(deployedCommit, detachedCommit) + }) +} From 177ff9614f1a347d96ad1a5a2037448471cbd284 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Sat, 26 Sep 2026 06:55:21 -0700 Subject: [PATCH 21/50] fix(node): start module containers with --init and bound healthcheck wget with -T Without an init process the module containers left zombie children behind, and the wget healthcheck had no timeout, so a hung endpoint could pile up probe processes. The run args now pass --init and every wget probe uses the configured healthcheck timeout. --- src/services/module_service/build_and_up.js | 2 +- src/services/module_service/docker_args.js | 4 +- ...3_buildandup_init_and_wget_timeout.test.js | 49 +++++++++++++++++++ 3 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 test/unit/module_service.test/13_buildandup_init_and_wget_timeout.test.js diff --git a/src/services/module_service/build_and_up.js b/src/services/module_service/build_and_up.js index 17562569..92367903 100644 --- a/src/services/module_service/build_and_up.js +++ b/src/services/module_service/build_and_up.js @@ -297,7 +297,7 @@ function buildContainerRunArgs(context) { const healthcheckArgs = onlyExecution ? [] : buildHealthcheckArgs(module, environmentVariables) const logOptArgs = onlyExecution ? [] : ['--log-opt', 'max-size=50m', '--log-opt', 'max-file=4'] return [ - 'run', '-d', ...restartArgs, ...stopBudgetArgs, '--name', containerPrefix, '--hostname', containerPrefix, + 'run', '-d', '--init', ...restartArgs, ...stopBudgetArgs, '--name', containerPrefix, '--hostname', containerPrefix, ...logOptArgs, ...volumeArgs, ...ulimitArgs, ...memoryArgs, ...healthcheckArgs, '--network', getDockerNetwork(coin, network), ...envArgs, ...portArgs, '-t', containerPrefix, ...(dockerCmdArgs ?? []) diff --git a/src/services/module_service/docker_args.js b/src/services/module_service/docker_args.js index b2c30e90..fc562ec0 100644 --- a/src/services/module_service/docker_args.js +++ b/src/services/module_service/docker_args.js @@ -270,11 +270,11 @@ function buildHealthcheckArgs(module, environmentVariables) { // dead port. Pick per descriptor: a service whose ping already carries the // real verdict (explorer) stays on ping. const method = hc.probe === 'jsonrpc_health' ? 'health' : 'ping' - cmd = `wget -qO- --post-data='{"jsonrpc":"2.0","method":"${method}","id":1}' --header='Content-Type: application/json' http://localhost:${port}/ || exit 1` + cmd = `wget -T ${parseInt(hc.timeout, 10)} -qO- --post-data='{"jsonrpc":"2.0","method":"${method}","id":1}' --header='Content-Type: application/json' http://localhost:${port}/ || exit 1` } else { // Default: plain HTTP GET on /status; descriptors override via `path` // where /status is too expensive to double as a liveness probe (sync). - cmd = `wget -qO- http://localhost:${port}${hc.path || '/status'} || exit 1` + cmd = `wget -T ${parseInt(hc.timeout, 10)} -qO- http://localhost:${port}${hc.path || '/status'} || exit 1` } return [ diff --git a/test/unit/module_service.test/13_buildandup_init_and_wget_timeout.test.js b/test/unit/module_service.test/13_buildandup_init_and_wget_timeout.test.js new file mode 100644 index 00000000..22b6d586 --- /dev/null +++ b/test/unit/module_service.test/13_buildandup_init_and_wget_timeout.test.js @@ -0,0 +1,49 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +const { + sinon, expect, makeStubs, makeConfigServiceStub, loadModuleService, + stubDockerCreate, runArgsOf, moduleSuite +} = require('./support/helpers') + +function createFixture(configService = makeConfigServiceStub()) { + const stubs = makeStubs() + const seen = stubDockerCreate(stubs) + const service = loadModuleService(stubs, null, { './config_service': configService }) + return { + create: async () => { + await service.buildAndUp('xchain-encoder', 'bitcoin', 'mainnet') + return runArgsOf(seen) + } + } +} + +const healthcheckedEncoder = createFixture() + +moduleSuite('buildAndUp() init and wget timeout', function () { + it('passes init and a bounded wget command for a healthchecked module', async function () { + const args = await healthcheckedEncoder.create() + expect(args).to.include('--init') + const healthCommand = args[args.indexOf('--health-cmd') + 1] + expect(healthCommand).to.match(/^wget -T 5\b/) + }) + + it('passes init when healthcheck argument construction returns no arguments', async function () { + const configService = makeConfigServiceStub() + const environmentVariables = await configService.getDefaultConfig() + delete environmentVariables.ENCODER_API_PORT + configService.getDefaultConfig = sinon.stub().resolves(environmentVariables) + const args = await createFixture(configService).create() + expect(args).to.include('--init') + expect(args).to.not.include('--health-cmd') + }) +}) From ffac2986f85578dd50ffd0390ffd9a4a50738db0 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Sat, 26 Sep 2026 07:01:15 -0700 Subject: [PATCH 22/50] fix(node): check BuildKit availability before clone and database setup buildAndUp now probes for BuildKit before reading module configuration or cloning bundled libraries, so a host without buildx fails at once instead of after the slow setup steps. The image build keeps pinning DOCKER_BUILDKIT=1. --- src/services/module_service/build_and_up.js | 34 ++++----- ...dup_buildkit_precedes_clone_and_db.test.js | 71 +++++++++++++++++++ 2 files changed, 88 insertions(+), 17 deletions(-) create mode 100644 test/unit/module_service.test/13_buildandup_buildkit_precedes_clone_and_db.test.js diff --git a/src/services/module_service/build_and_up.js b/src/services/module_service/build_and_up.js index 92367903..a30a9d2b 100644 --- a/src/services/module_service/build_and_up.js +++ b/src/services/module_service/build_and_up.js @@ -344,8 +344,8 @@ async function createContainer(context, resolve, reject) { } } -// The module Dockerfiles only build under BuildKit. Refuse before the build -// and pin DOCKER_BUILDKIT=1 so a host override cannot select the legacy builder. +// The module Dockerfiles only build under BuildKit (refused earlier, in +// buildAndUp); pin DOCKER_BUILDKIT=1 so a host override cannot select it. function buildOrCreateContainer(context, resolve, reject) { const { reuseImage, module, coin, network, sourceLabels, buildLabelArgs, containerPrefix, dir } = context @@ -354,21 +354,16 @@ function buildOrCreateContainer(context, resolve, reject) { createContainer(context, resolve, reject) return } - const buildKitProbe = typeof checkBuildKitAvailable === 'function' - ? checkBuildKitAvailable() - : Promise.resolve(true) - buildKitProbe.then(() => { - logger.info("Building image of module " + module + (coin && network ? " in " + coin + " " + network : "") - + (sourceLabels.commit ? " from " + sourceLabels.commit.slice(0, 12) + " (" + (sourceLabels.ref || 'detached') + ")" : "")) - const buildEnv = { ...config.childProcessEnv(), DOCKER_BUILDKIT: '1' } - execFile('docker', ['build', ...buildLabelArgs, '.', '-t', containerPrefix], { cwd: dir, env: buildEnv }, (error) => { - if (error) { - reject("Error creating Docker image: " + redactSecrets(error.message)) - return - } - createContainer(context, resolve, reject) - }) - }).catch((err) => reject("Error creating Docker image: " + err)) + logger.info("Building image of module " + module + (coin && network ? " in " + coin + " " + network : "") + + (sourceLabels.commit ? " from " + sourceLabels.commit.slice(0, 12) + " (" + (sourceLabels.ref || 'detached') + ")" : "")) + const buildEnv = { ...config.childProcessEnv(), DOCKER_BUILDKIT: '1' } + execFile('docker', ['build', ...buildLabelArgs, '.', '-t', containerPrefix], { cwd: dir, env: buildEnv }, (error) => { + if (error) { + reject("Error creating Docker image: " + redactSecrets(error.message)) + return + } + createContainer(context, resolve, reject) + }) } function launchContainer(context) { @@ -377,6 +372,11 @@ function launchContainer(context) { async function buildAndUp(module, coin, network, overwriteContainerId = null, onlyExecution = false, dockerCmdArgs = null, options = {}) { if (!checkIfModuleExists(module)) throw "module not found" const reuseImage = options.reuseImage === true + if (!reuseImage && typeof checkBuildKitAvailable === 'function') { + try { + await checkBuildKitAvailable() + } catch (err) { throw "Error creating Docker image: " + err } + } const environmentVariables = await getDefaultConfig(module, coin, network) const dir = getModuleDir(module) await assertDeploymentReady(module, coin, network, environmentVariables, dir, onlyExecution) diff --git a/test/unit/module_service.test/13_buildandup_buildkit_precedes_clone_and_db.test.js b/test/unit/module_service.test/13_buildandup_buildkit_precedes_clone_and_db.test.js new file mode 100644 index 00000000..4ca5fda7 --- /dev/null +++ b/test/unit/module_service.test/13_buildandup_buildkit_precedes_clone_and_db.test.js @@ -0,0 +1,71 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +// A buildx-less host must fail on the BuildKit probe alone: before +// buildAndUp reads module configuration/database state and before it clones +// any bundled library into the build context. + +const { + sinon, expect, makeStubs, makeConfigServiceStub, loadModuleService, moduleSuite +} = require('./support/helpers') + +moduleSuite('buildAndUp(): BuildKit probe precedes clone and database work', function () { + // proxyquire's module_service load sits under the 2s CLI default only while + // the runner is idle; concurrent gate runs push it over (same budget as + // 06_buildandup_module_specific_port_volume_branches.test.js). + this.timeout(10000) + + it('refuses on the buildx probe before configuration/database lookup or the bundled-library clone', async function () { + const stubs = makeStubs() + const calls = [] + const hint = "Docker's buildx plugin is not installed: sudo apt install docker-buildx-plugin" + stubs.checkBuildKitAvailable = sinon.stub().callsFake(async () => { + calls.push('buildx') + throw hint + }) + + const configService = makeConfigServiceStub() + configService.getDefaultConfig = sinon.stub().callsFake(async () => { + calls.push('configuration/database') + return {} + }) + + const cloneGit = sinon.stub().callsFake(async () => { calls.push('clone') }) + const resolveBundledLibRef = sinon.stub().resolves( + { ref: 'develop', commit: null, pinned: false, reason: 'default branch fallback' }) + + const ms = loadModuleService(stubs, null, { + './config_service': configService, + './module_service/clone_and_refs.js': { + configureDependencies: sinon.stub(), + cloneGit, resolveBundledLibRef, + getModuleBranch: sinon.stub().resolves('develop'), + getModuleCommit: sinon.stub().resolves(null) + } + }) + + let thrown = null + try { + // xchain-indexer bundles xchain-vm, so a fix that lets configuration + // or the clone run first would show up here. + await ms.buildAndUp('xchain-indexer', 'bitcoin', 'mainnet') + expect.fail('Should have rejected without buildx') + } catch (err) { thrown = err } + + expect(String(thrown)).to.include('Error creating Docker image') + expect(String(thrown)).to.include('docker-buildx-plugin') + expect(calls).to.deep.equal(['buildx']) + expect(configService.getDefaultConfig.called).to.equal(false) + expect(cloneGit.called).to.equal(false) + expect(resolveBundledLibRef.called).to.equal(false) + }) +}) From 1127268af11800a7f93a71f0efe99c416b4ab7fb Mon Sep 17 00:00:00 2001 From: J-Dog Date: Sat, 26 Sep 2026 14:04:26 -0700 Subject: [PATCH 23/50] ci(node): check out coverage siblings through the shared action The coverage job carried its own copy of the sibling checkout; it now calls the shared checkout-siblings action every other repo uses, which exposes the same checked-out output the strict-sibling flag reads. --- .github/workflows/ci.yml | 29 ++--------------------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 15492622..05433556 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,35 +89,10 @@ jobs: steps: - uses: actions/checkout@v4 - # Re-runs the same suite the `ci` job runs, so it needs the same - # sibling checkout that job's shared workflow does elsewhere. - # Mirrored verbatim (same roster, same clone) so they cannot drift. + # Re-runs the same suite the `ci` job runs, so it needs the same siblings. - name: Check out declared sibling repositories id: siblings - env: - SIBLINGS_REF: ${{ github.ref == 'refs/heads/master' && 'master' || 'develop' }} - OWNER: ${{ github.repository_owner }} - run: | - set -euo pipefail - if [ ! -f .ci-siblings ]; then - echo "no .ci-siblings in this repo; single-repo layout" - exit 0 - fi - repos=$(sed 's/#.*//' .ci-siblings | tr -d '\r' | awk 'NF') - cd "$GITHUB_WORKSPACE/.." - for repo in $repos; do - rm -rf "$repo" - url="https://github.com/$OWNER/$repo.git" - git clone --quiet --depth 1 --branch "$SIBLINGS_REF" "$url" "$repo" 2>/dev/null \ - || git clone --quiet --depth 1 "$url" "$repo" - echo "sibling $repo @ $(git -C "$repo" rev-parse --abbrev-ref HEAD) $(git -C "$repo" rev-parse --short HEAD)" - if [ -f "$repo/package.json" ]; then - ( cd "$repo" && npm ci --omit=dev --ignore-scripts --no-audit --no-fund >/dev/null 2>&1 ) \ - || ( cd "$repo" && npm install --omit=dev --ignore-scripts --no-audit --no-fund >/dev/null 2>&1 ) \ - || echo "sibling $repo: dependency install failed, guards needing its modules will say so" - fi - done - echo "checked-out=1" >> "$GITHUB_OUTPUT" + uses: XChain-Platform/.github/actions/checkout-siblings@master # one definition for every call site; see the action for why this tracks master - name: Use Node.js 22 uses: actions/setup-node@v4 From e4c0c9ff6fc1436dce49b983de1d1e8b11628982 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Sat, 26 Sep 2026 14:07:26 -0700 Subject: [PATCH 24/50] test(ci): require the shared sibling checkout in the coverage job The coverage job must call the shared checkout-siblings action rather than carry its own copy of the clone loop. --- test/unit/ci_full_sibling_roster.test/sibling_roster.test.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/unit/ci_full_sibling_roster.test/sibling_roster.test.js b/test/unit/ci_full_sibling_roster.test/sibling_roster.test.js index d868401d..b3b0a30e 100644 --- a/test/unit/ci_full_sibling_roster.test/sibling_roster.test.js +++ b/test/unit/ci_full_sibling_roster.test/sibling_roster.test.js @@ -102,8 +102,9 @@ describe('.github/workflows/ci.yml coverage job', () => { it('checks declared siblings out before re-running the unit suite for coverage', () => { const siblingStep = steps.find((s) => s.id === 'siblings') expect(siblingStep, 'coverage job has no sibling-checkout step').to.exist - expect(siblingStep.run).to.include('.ci-siblings') - expect(siblingStep.run).to.include('GITHUB_OUTPUT') + // The shared action reads .ci-siblings and publishes checked-out; an inline + // copy here would drift from the roster every other repo checks out. + expect(siblingStep.uses).to.match(/^XChain-Platform\/\.github\/actions\/checkout-siblings@/) }) it('arms XCHAIN_REQUIRE_SIBLINGS for coverage:check only when the checkout actually happened', () => { From 830b2f273bca4b2bd55e2a67f1632074a2ab5353 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Sat, 26 Sep 2026 19:55:04 -0700 Subject: [PATCH 25/50] Refresh CI sibling dispatcher wording Describe sibling shipment using the canonical origin head while retaining the local HEAD fallback. --- .ci-siblings | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci-siblings b/.ci-siblings index 705a615b..a018ad4c 100644 --- a/.ci-siblings +++ b/.ci-siblings @@ -6,9 +6,9 @@ # checks against it. An undeclared sibling is not a neutral omission - the gate # would fail loud (bin/ci-full.sh need_sib) rather than silently skip. # -# ci-dispatch.sh ships each of these at its origin/master (falling back to local -# HEAD) and REFUSES the push if one is not a checkout beside this repo, so a -# declared sibling can never be silently absent. +# ci-dispatch.sh ships each of these at its canonical origin head (falling back +# to local HEAD) and REFUSES the push if one is not a checkout beside this repo, +# so a declared sibling can never be silently absent. # Vendored coins-registry byte-identity: BTC/LTC/DOGE/index/consensus_pin. xchain-hub From 53fd827dc4b9a51cb6296a0f174e6c9f11db392f Mon Sep 17 00:00:00 2001 From: J-Dog Date: Sat, 26 Sep 2026 19:56:36 -0700 Subject: [PATCH 26/50] Align config and network chaos contracts Update chaos expectations to match strict argument parsing and current config defaults. Complete filesystem doubles and pin connector reachability and timeout behavior. --- test/chaos/config_resilience_chaos.test.js | 2 +- .../argument_parsing_resilience_chaos.test.js | 7 +++---- .../config_path_traversal_chaos.test.js | 4 ++-- .../malformed_config_content_chaos.test.js | 2 ++ .../shared_module_config_chaos.test.js | 2 +- test/chaos/network_resilience_chaos.test.js | 6 +++--- 6 files changed, 12 insertions(+), 11 deletions(-) diff --git a/test/chaos/config_resilience_chaos.test.js b/test/chaos/config_resilience_chaos.test.js index e67dd333..989b2337 100644 --- a/test/chaos/config_resilience_chaos.test.js +++ b/test/chaos/config_resilience_chaos.test.js @@ -55,7 +55,7 @@ describe('Chaos: Config Resilience', function () { const config = await cs.getDefaultConfig('xchain-encoder', 'bitcoin', 'regtest') - expect(config).to.have.property('NETWORK', 'regtest') + expect(config).to.have.property('NETWORK', 'bitcoin-regtest') expect(config).to.have.property('NODE_PORT', 18444) expect(config).to.have.property('DECODER_DB_HOST', 'mariadb') expect(warnSpy.calledOnce).to.be.true diff --git a/test/chaos/config_resilience_chaos.test/argument_parsing_resilience_chaos.test.js b/test/chaos/config_resilience_chaos.test/argument_parsing_resilience_chaos.test.js index c4fa605b..46abc5ae 100644 --- a/test/chaos/config_resilience_chaos.test/argument_parsing_resilience_chaos.test.js +++ b/test/chaos/config_resilience_chaos.test/argument_parsing_resilience_chaos.test.js @@ -120,12 +120,11 @@ describe('Chaos: Config Resilience', function () { expect(result.branch).to.equal('feature/my-branch_v1.0') }) - it('handles extremely long argument strings without crashing', function () { + it('refuses extremely long unrecognized argument strings without hanging', function () { const cs = makeConfigService() const longArg = 'a'.repeat(10000) - // Should not throw or hang; just treated as unknown arg - const result = cs.resolveArgs([longArg]) - expect(result.service).to.equal('all') + expect(() => cs.resolveArgs([longArg])) + .to.throw("Unrecognized argument '" + longArg + "'") }) }) }) diff --git a/test/chaos/config_resilience_chaos.test/config_path_traversal_chaos.test.js b/test/chaos/config_resilience_chaos.test/config_path_traversal_chaos.test.js index 1595fcbf..2c83a3bc 100644 --- a/test/chaos/config_resilience_chaos.test/config_path_traversal_chaos.test.js +++ b/test/chaos/config_resilience_chaos.test/config_path_traversal_chaos.test.js @@ -54,7 +54,7 @@ describe('Chaos: Config Resilience', function () { await cs.getDefaultConfig('xchain-encoder', '../../etc', 'passwd') expect.fail('should have thrown') } catch (err) { - expect(err.message).to.include('path traversal') + expect(err.message).to.equal("Unknown coin '../../etc'; cannot derive a database name") } }) @@ -71,7 +71,7 @@ describe('Chaos: Config Resilience', function () { await cs.getDefaultConfig('xchain-encoder', '../..', 'etc/passwd') expect.fail('should have thrown') } catch (err) { - expect(err.message).to.include('path traversal') + expect(err.message).to.equal("Unknown coin '../..'; cannot derive a database name") } }) }) diff --git a/test/chaos/config_resilience_chaos.test/malformed_config_content_chaos.test.js b/test/chaos/config_resilience_chaos.test/malformed_config_content_chaos.test.js index 0e7be0bc..bea14ae6 100644 --- a/test/chaos/config_resilience_chaos.test/malformed_config_content_chaos.test.js +++ b/test/chaos/config_resilience_chaos.test/malformed_config_content_chaos.test.js @@ -105,6 +105,7 @@ describe('Chaos: Config Resilience', function () { const fsStub = { existsSync: sinon.stub().returns(true), createReadStream: sinon.stub().callsFake(() => Readable.from(configContent + '\n')), + appendFileSync: sinon.stub(), writeFileSync: sinon.stub(), rmSync: sinon.stub(), mkdirSync: sinon.stub() @@ -130,6 +131,7 @@ describe('Chaos: Config Resilience', function () { const fsStub = { existsSync: sinon.stub().returns(true), createReadStream: sinon.stub().callsFake(() => Readable.from(configContent + '\n')), + appendFileSync: sinon.stub(), writeFileSync: sinon.stub(), rmSync: sinon.stub(), mkdirSync: sinon.stub() diff --git a/test/chaos/config_resilience_chaos.test/shared_module_config_chaos.test.js b/test/chaos/config_resilience_chaos.test/shared_module_config_chaos.test.js index 1837ac7e..300271f2 100644 --- a/test/chaos/config_resilience_chaos.test/shared_module_config_chaos.test.js +++ b/test/chaos/config_resilience_chaos.test/shared_module_config_chaos.test.js @@ -44,7 +44,7 @@ describe('Chaos: Config Resilience', function () { const cs = makeConfigService() const config = await cs.getDefaultConfig('xchain-hub', null, null) - expect(config).to.have.property('HUB_HOST', '127.0.0.1') + expect(config).to.have.property('HUB_HOST', '0.0.0.0') expect(config).to.have.property('HUB_PORT', 10000) expect(config).to.not.have.property('DECODER_DB_HOST') }) diff --git a/test/chaos/network_resilience_chaos.test.js b/test/chaos/network_resilience_chaos.test.js index 8aa07558..e3853fa7 100644 --- a/test/chaos/network_resilience_chaos.test.js +++ b/test/chaos/network_resilience_chaos.test.js @@ -160,7 +160,7 @@ describe('Chaos: Network Resilience (Hub/Explorer)', function () { expect(data.jsonrpc).to.equal('2.0') expect(data.method).to.equal('updateconfig') expect(data.params.config).to.deep.equal(configPayload) - expect(opts.timeout).to.equal(10000) + expect(opts.timeout).to.equal(60000) }) }) }) @@ -293,7 +293,7 @@ describe('Chaos: Network Resilience (Hub/Explorer)', function () { expect(result).to.be.false }) - it('hub returns false when result is false', async function () { + it('reports the hub reachable when a false result is present', async function () { const axiosStub = { post: sinon.stub().resolves({ data: { result: false } }) } @@ -301,7 +301,7 @@ describe('Chaos: Network Resilience (Hub/Explorer)', function () { const hub = new HubConnector('127.0.0.1', 10000) const result = await hub.ping() - expect(result).to.be.false + expect(result).to.be.true }) it('hub returns false when result is null', async function () { From 8f9e146743095ebfc468255598c32b3c085c3baa Mon Sep 17 00:00:00 2001 From: J-Dog Date: Sat, 26 Sep 2026 20:10:11 -0700 Subject: [PATCH 27/50] test: wire service delivery and privacy coverage Exercise hub retry exhaustion and explorer config writes in the unit tier.\n\nPin telemetry opt-out ordering and the payload field allowlist. --- test/unit/hub_service.test.js | 75 ++++++++++++++++++ test/unit/telemetry_service.test.js | 117 +++++++++++++++++++++++++--- 2 files changed, 182 insertions(+), 10 deletions(-) diff --git a/test/unit/hub_service.test.js b/test/unit/hub_service.test.js index e2d1df1c..e788431c 100644 --- a/test/unit/hub_service.test.js +++ b/test/unit/hub_service.test.js @@ -30,6 +30,7 @@ const { expect } = require('chai') const proxyquire = require('proxyquire') const { HUB_MODULE_NAME, SYNC_MODULE_NAME } = require('../../src/config') +const { updateHubOrExplorer } = require('../../src/services/hub_service/update_hub_or_explorer') function loadHubService({ addContainerToNetwork } = {}) { const getModuleContainer = sinon.stub() @@ -48,6 +49,31 @@ function loadHubService({ addContainerToNetwork } = {}) { return { svc, attach } } +function configDeliveryDependencies({ updateConfig = sinon.stub().resolves(true), lastStatus = {} } = {}) { + return { + HUB_MODULE_NAME: 'xchain-hub', + EXPLORER_MODULE_NAME: 'xchain-explorer', + EXTERNAL_DB: false, + XChainService: { XCHAIN_INDEXER: 'xchain-indexer' }, + db: { getModuleContainer: sinon.stub().resolves('explorer-container') }, + getLastStatus: () => lastStatus, + isStatusUpdated: () => true, + sleep: sinon.stub().resolves(), + redactSecrets: value => String(value && value.message ? value.message : value), + getDefaultConfig: sinon.stub().resolves({ HUB_PORT: 10000, EXPLORER_PORT: 10001 }), + getStatus: sinon.stub().resolves(), + getExternalDbConfig: sinon.stub().resolves(null), + readContainerEnv: sinon.stub().resolves(null), + HubConnector: class { constructor() { this.updateConfig = updateConfig } }, + ExplorerConnector: class {}, + dockerService: { stringToDockerContainerFile: sinon.stub().resolves(true) }, + buildHubModuleConfig: sinon.stub().returns(null), + buildCheckpointConfig: sinon.stub().returns({}), + isCheckpointSelfSyncEnabled: sinon.stub().resolves(false), + logger: { info: sinon.stub() } + } +} + describe('HubService.updateHub network attachment', function () { it('returns true when every shared-container attach succeeds', async function () { @@ -80,6 +106,55 @@ describe('HubService.updateHub network attachment', function () { }) }) +describe('HubService.updateHubOrExplorer config delivery', function () { + it('retries a rejected hub update and succeeds on the next attempt', async function () { + const updateConfig = sinon.stub() + updateConfig.onFirstCall().rejects(new Error('connection refused')) + updateConfig.onSecondCall().resolves(true) + const deps = configDeliveryDependencies({ updateConfig }) + + expect(await updateHubOrExplorer('xchain-hub', deps)).to.be.true + expect(updateConfig.callCount).to.equal(2) + sinon.assert.calledOnceWithExactly(deps.sleep, 3000) + }) + + it('exhausts ten attempts and reports the last hub error', async function () { + const updateConfig = sinon.stub().rejects(new Error('earlier refusal')) + updateConfig.onCall(9).rejects(new Error('final refusal')) + const deps = configDeliveryDependencies({ updateConfig }) + + let thrown + try { + await updateHubOrExplorer('xchain-hub', deps) + } catch (err) { + thrown = err + } + + expect(updateConfig.callCount).to.equal(10) + expect(deps.sleep.callCount).to.equal(9) + expect(thrown).to.include('last error: final refusal') + expect(thrown).not.to.include('earlier refusal') + }) + + it('writes the explorer payload to config.json in its registered container', async function () { + const lastStatus = { bitcoin: { mainnet: { 'xchain-encoder': {} } } } + const deps = configDeliveryDependencies({ lastStatus }) + deps.buildHubModuleConfig.returns({ enabled: true }) + + expect(await updateHubOrExplorer('xchain-explorer', deps)).to.be.true + sinon.assert.calledOnceWithExactly( + deps.dockerService.stringToDockerContainerFile, + 'explorer-container', + JSON.stringify([{ + coin: 'bitcoin', + network: 'mainnet', + 'xchain-encoder': { enabled: true } + }]), + '/XChainExplorer/src/config.json' + ) + }) +}) + // A hub that is crash-looping answers nothing, so the config push spends ten // attempts on connection refusals and then aborts whatever command asked for it. // `skipConfigPush` lets a caller that has already established the hub is down diff --git a/test/unit/telemetry_service.test.js b/test/unit/telemetry_service.test.js index c95f3f26..a5d3b965 100644 --- a/test/unit/telemetry_service.test.js +++ b/test/unit/telemetry_service.test.js @@ -31,18 +31,85 @@ const { expect } = require('chai') const sinon = require('sinon') const proxyquire = require('proxyquire').noCallThru() -describe('TelemetryService.gatherModules() (via gatherPayload)', function () { - - function loadWithStatus(statusObj) { - const getStatus = sinon.stub().resolves(statusObj) - const svc = proxyquire('../../src/services/telemetry_service', { - './status_service': { getStatus }, - // avoid spawning a real `docker version` - 'child_process': { execFile: (cmd, args, cb) => cb(null, '') } - }) - return svc +function loadTelemetry({ env = '', pref = null, status = {} } = {}) { + let savedPref = pref + const report = sinon.stub().resolves() + const fs = { + readFileSync: sinon.stub().callsFake(() => { + if (savedPref === null) throw new Error('missing preference') + return JSON.stringify(savedPref) + }), + existsSync: sinon.stub().returns(true), + mkdirSync: sinon.stub(), + writeFileSync: sinon.stub().callsFake((filePath, value) => { + savedPref = JSON.parse(value) + }), + chmodSync: sinon.stub() } + const svc = proxyquire('../../src/services/telemetry_service', { + fs, + os: { + homedir: () => '/operator', + platform: () => 'linux', + release: () => '6.12.0', + arch: () => 'x64' + }, + crypto: { randomUUID: () => 'generated-install-id' }, + child_process: { execFile: (command, args, options, callback) => callback(null, '27.3.1\n') }, + './status_service': { getStatus: sinon.stub().resolves(status) }, + './telemetry_connector': class { constructor() { this.report = report } }, + '../config': { XCHAIN_NODE_NO_TELEMETRY: env } + }) + return { svc, report, fs, getSavedPref: () => savedPref } +} + +function loadWithStatus(statusObj) { + const getStatus = sinon.stub().resolves(statusObj) + return proxyquire('../../src/services/telemetry_service', { + './status_service': { getStatus }, + 'child_process': { execFile: (cmd, args, cb) => cb(null, '') } + }) +} + +describe('TelemetryService opt-out precedence', function () { + + it('gives the CLI flag priority and persists that choice', async function () { + const harness = loadTelemetry({ env: 'false', pref: { optOut: false } }) + + await harness.svc.maybeReportTelemetry('install', true) + sinon.assert.notCalled(harness.report) + expect(harness.getSavedPref().optOut).to.equal(true) + }) + + it('honours the environment when the CLI flag is absent', async function () { + const harness = loadTelemetry({ env: 'yes', pref: { optOut: false } }) + + await harness.svc.maybeReportTelemetry('install', false) + + sinon.assert.notCalled(harness.report) + expect(harness.getSavedPref().optOut).to.equal(true) + }) + + it('honours the saved preference when flag and environment are absent', async function () { + const harness = loadTelemetry({ pref: { optOut: true } }) + + await harness.svc.maybeReportTelemetry('install', false) + + sinon.assert.notCalled(harness.report) + sinon.assert.notCalled(harness.fs.writeFileSync) + }) + + it('reports when flag, environment, and preference all allow it', async function () { + const harness = loadTelemetry({ pref: { optOut: false, installId: 'existing-install-id' } }) + + await harness.svc.maybeReportTelemetry('install', false) + + sinon.assert.calledOnce(harness.report) + }) +}) + +describe('TelemetryService.gatherModules() (via gatherPayload)', function () { it('adds a null-safe `health` field and leaves `running` a pure liveness signal', async function () { const svc = loadWithStatus({ btc: { @@ -83,4 +150,34 @@ describe('TelemetryService.gatherModules() (via gatherPayload)', function () { const payload = await svc.gatherPayload('heartbeat', 'install-abc') expect(payload.modules[0]).to.include({ module: 'node', running: false, health: null }) }) + +}) + +describe('TelemetryService payload privacy', function () { + + it('emits only allowlisted payload keys with no IP or host field', async function () { + const { svc } = loadTelemetry({ + status: { + btc: { + mainnet: { + node: { + container_version: '1.2.3', + status: { State: { Status: 'running', Health: { Status: 'healthy' } } } + } + } + } + } + }) + + const payload = await svc.gatherPayload('heartbeat', 'install-abc') + expect(Object.keys(payload).sort()).to.deep.equal([ + 'arch', 'docker_version', 'event', 'install_id', 'modules', + 'node_version', 'os_platform', 'os_release' + ]) + expect(Object.keys(payload.modules[0]).sort()).to.deep.equal([ + 'coin', 'health', 'module', 'network', 'running', 'version' + ]) + expect(Object.keys(payload).concat(Object.keys(payload.modules[0]))) + .not.to.satisfy(keys => keys.some(key => /(^|_)(host|ip)($|_)/i.test(key))) + }) }) From 398fdd04efe7c4b4b637704186c519cd3fc3f226 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Sat, 26 Sep 2026 20:19:26 -0700 Subject: [PATCH 28/50] fix(update): require explicit node data directory Refuse node updates before any rebuild work when the data root is absent. Cover direct and mixed updates plus the configured success path. --- .../module_operations/update_modules.js | 11 +++ .../update_datadir_unset.test.js | 77 +++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 test/unit/node_service.test/update_datadir_unset.test.js diff --git a/src/operations/module_operations/update_modules.js b/src/operations/module_operations/update_modules.js index 7982ee4c..49ff850a 100644 --- a/src/operations/module_operations/update_modules.js +++ b/src/operations/module_operations/update_modules.js @@ -34,6 +34,8 @@ async function updateModules(servicesList, ref = null, opts = {}) { const list = opts.all ? includeSharedServicesForUpdate(servicesList) : servicesList const runOpts = { skipCurrentNode: !!opts.all, quietNotInstalled: !!opts.all } + assertNodeDataDirSetForUpdate(list) + await repairValidatorConfigBeforeHubUpdate(list) if (isReleaseRef(ref)) { @@ -62,6 +64,15 @@ async function updateModules(servicesList, ref = null, opts = {}) { return withInstallTarget(releaseRef, async () => updateModulesOnBranch(list, null, runOpts), { fallbackToBranch: false }) } +function assertNodeDataDirSetForUpdate(servicesList) { + const includesNode = Object.values(servicesList).some(networks => + Object.values(networks).some(modules => modules.includes(NODE_MODULE_NAME))) + const dataDir = config.XCHAIN_NODE_DATA_DIR + if (includesNode && (!dataDir || dataDir.trim() === '')) { + throw new Error('Refusing to update a node while XCHAIN_NODE_DATA_DIR is unset. Set it to the existing stack data root before retrying.') + } +} + /** * On a validator, an update that rebuilds the hub first re-runs the validator * repair path (`validator init` over an initialized node): additive only, it diff --git a/test/unit/node_service.test/update_datadir_unset.test.js b/test/unit/node_service.test/update_datadir_unset.test.js new file mode 100644 index 00000000..023f7c54 --- /dev/null +++ b/test/unit/node_service.test/update_datadir_unset.test.js @@ -0,0 +1,77 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +const { expect, makeStubs, loadOperations, registerLifecycleHooks } = require('../module_operations.test/helpers/harness') + +describe('Node update data directory preflight', function () { + let originalDataDir + + registerLifecycleHooks(() => {}) + + beforeEach(function () { + originalDataDir = process.env.XCHAIN_NODE_DATA_DIR + delete process.env.XCHAIN_NODE_DATA_DIR + }) + + afterEach(function () { + if (originalDataDir === undefined) delete process.env.XCHAIN_NODE_DATA_DIR + else process.env.XCHAIN_NODE_DATA_DIR = originalDataDir + }) + + it('refuses a node update with no configured data directory before replacement begins', async function () { + const stubs = makeStubs() + const ops = loadOperations(stubs) + + let error + try { + await ops.updateModules({ bitcoin: { mainnet: ['node'] } }) + } catch (err) { + error = err + } + + expect(error).to.be.an('error') + expect(error.message).to.include('XCHAIN_NODE_DATA_DIR is unset') + expect(stubs.db.getModuleContainer.called).to.equal(false) + expect(stubs.installModule.called).to.equal(false) + expect(stubs.forceRemoveContainerByName.called).to.equal(false) + expect(stubs.stopContainerByName.called).to.equal(false) + }) + + it('preflights the complete request before updating a service listed ahead of the node', async function () { + const stubs = makeStubs() + const ops = loadOperations(stubs) + + let error + try { + await ops.updateModules({ bitcoin: { mainnet: ['xchain-encoder', 'node'] } }) + } catch (err) { + error = err + } + + expect(error).to.be.an('error') + expect(stubs.db.getModuleContainer.called).to.equal(false) + expect(stubs.installModule.called).to.equal(false) + }) + + it('updates the node when the data directory is explicitly configured', async function () { + process.env.XCHAIN_NODE_DATA_DIR = '/srv/xchain/data' + const stubs = makeStubs() + const ops = loadOperations(stubs) + + const outcome = await ops.updateModules({ bitcoin: { mainnet: ['node'] } }) + + expect(outcome.updated).to.deep.equal([ + { module: 'node', coin: 'bitcoin', network: 'mainnet' } + ]) + expect(stubs.installModule.calledOnce).to.equal(true) + }) +}) From a36504a607cd94ba4375300673111b511a1e3d70 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Sat, 26 Sep 2026 20:28:20 -0700 Subject: [PATCH 29/50] test(boundary): align cases with v2 contracts Assert Docker env names stay bare while the child env carries exact values, matching "passes environment variables via the child env (bare --env NAME), not as argv values". Expect unknown argument rejection, matching "refuses an unrecognized service name instead of expanding to every service". Complete the config parsing filesystem stub for sidecar persistence. --- test/boundary/boundary.test.js | 1 + .../docker_env_var_passing.test.js | 18 ++++++++++++------ .../resolve_args_boundaries.test.js | 13 ++++++------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/test/boundary/boundary.test.js b/test/boundary/boundary.test.js index ea5c6ec2..aabed866 100644 --- a/test/boundary/boundary.test.js +++ b/test/boundary/boundary.test.js @@ -49,6 +49,7 @@ describeBoundaryTests('ConfigService: config file parsing', function () { const fsStub = { existsSync: sinon.stub().returns(true), createReadStream: sinon.stub().callsFake(() => streamFromString('NODE_PASSWORD=p@ss=word=123\n')), + appendFileSync: sinon.stub(), writeFileSync: sinon.stub(), rmSync: sinon.stub(), mkdirSync: sinon.stub() diff --git a/test/boundary/boundary.test/docker_env_var_passing.test.js b/test/boundary/boundary.test/docker_env_var_passing.test.js index 90257142..e4ea87cf 100644 --- a/test/boundary/boundary.test/docker_env_var_passing.test.js +++ b/test/boundary/boundary.test/docker_env_var_passing.test.js @@ -89,6 +89,12 @@ function describeBoundaryTests(title, defineTests) { }) } +function expectEnvPassedByName(args, opts, name, value) { + expect(args.some((arg, index) => arg === name && args[index - 1] === '--env')).to.be.true + expect(args.some(arg => String(arg).startsWith(`${name}=`))).to.be.false + expect(opts.env[name]).to.equal(String(value)) +} + // 4. Docker env var escaping (Fix 3) describeBoundaryTests('ModuleService: Docker env var passing (execFile)', function () { it('passes double quotes in environment variable values unescaped', async function () { @@ -106,7 +112,7 @@ describeBoundaryTests('ModuleService: Docker env var passing (execFile)', functi if (args[0] === 'build') { cb(null, '') } else if (args[0] === 'run') { - expect(args).to.include('TEST_VAR=hello"world') + expectEnvPassedByName(args, opts, 'TEST_VAR', 'hello"world') cb(null, 'a'.repeat(64) + '\n') } }) @@ -129,7 +135,7 @@ describeBoundaryTests('ModuleService: Docker env var passing (execFile)', functi if (args[0] === 'build') { cb(null, '') } else if (args[0] === 'run') { - expect(args).to.include('PRICE=costs_$100') + expectEnvPassedByName(args, opts, 'PRICE', 'costs_$100') cb(null, 'a'.repeat(64) + '\n') } }) @@ -153,7 +159,7 @@ describeBoundaryTests('ModuleService: Docker env var passing (execFile)', functi if (args[0] === 'build') { cb(null, '') } else if (args[0] === 'run') { - expect(args).to.include('CMD=run `whoami`') + expectEnvPassedByName(args, opts, 'CMD', 'run `whoami`') cb(null, 'a'.repeat(64) + '\n') } }) @@ -176,7 +182,7 @@ describeBoundaryTests('ModuleService: Docker env var passing (execFile)', functi if (args[0] === 'build') { cb(null, '') } else if (args[0] === 'run') { - expect(args).to.include('PATH_VAR=C:\\Users\\test') + expectEnvPassedByName(args, opts, 'PATH_VAR', 'C:\\Users\\test') cb(null, 'a'.repeat(64) + '\n') } }) @@ -201,7 +207,7 @@ describeBoundaryTests('ModuleService: Docker env var passing (execFile)', functi if (args[0] === 'build') { cb(null, '') } else if (args[0] === 'run') { - expect(args).to.include('PORT=3002') + expectEnvPassedByName(args, opts, 'PORT', 3002) cb(null, 'a'.repeat(64) + '\n') } }) @@ -224,7 +230,7 @@ describeBoundaryTests('ModuleService: Docker env var passing (execFile)', functi if (args[0] === 'build') { cb(null, '') } else if (args[0] === 'run') { - expect(args).to.include('EXPLORER_API_USER=false') + expectEnvPassedByName(args, opts, 'EXPLORER_API_USER', false) cb(null, 'a'.repeat(64) + '\n') } }) diff --git a/test/boundary/boundary.test/resolve_args_boundaries.test.js b/test/boundary/boundary.test/resolve_args_boundaries.test.js index a8a3b871..d30cf3c1 100644 --- a/test/boundary/boundary.test/resolve_args_boundaries.test.js +++ b/test/boundary/boundary.test/resolve_args_boundaries.test.js @@ -72,15 +72,14 @@ describeBoundaryTests('ConfigService: resolveArgs boundaries', function () { }) }) describeBoundaryTests('ConfigService: resolveArgs boundaries', function () { - it('does not set branch when expectBranch is false', function () { - const result = resolveArgs(['unknownarg', 'bitcoin', 'mainnet'], { expectBranch: false }) - expect(result.branch).to.be.null + it('rejects an unrecognized argument when expectBranch is false', function () { + expect(() => resolveArgs(['unknownarg', 'bitcoin', 'mainnet'], { expectBranch: false })) + .to.throw("Unrecognized argument 'unknownarg'") }) - it('only takes the first unrecognized arg as branch', function () { - const result = resolveArgs(['mybranch', 'otherbranch', 'bitcoin'], { expectBranch: true }) - expect(result.branch).to.equal('mybranch') - // 'otherbranch' is silently ignored + it('rejects a second unrecognized argument after filling the branch slot', function () { + expect(() => resolveArgs(['mybranch', 'otherbranch', 'bitcoin'], { expectBranch: true })) + .to.throw("Unrecognized argument 'otherbranch'") }) it('handles empty args array', function () { From cd25e5cb6ca5e994d027e501f871d1bf7fd7c611 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Sat, 26 Sep 2026 20:28:49 -0700 Subject: [PATCH 30/50] Document hub result-presence contract Name the unit coverage that pins a present false JSON-RPC result as evidence the hub is reachable. --- test/chaos/network_resilience_chaos.test.js | 1 + 1 file changed, 1 insertion(+) diff --git a/test/chaos/network_resilience_chaos.test.js b/test/chaos/network_resilience_chaos.test.js index e3853fa7..c7eb6493 100644 --- a/test/chaos/network_resilience_chaos.test.js +++ b/test/chaos/network_resilience_chaos.test.js @@ -301,6 +301,7 @@ describe('Chaos: Network Resilience (Hub/Explorer)', function () { const hub = new HubConnector('127.0.0.1', 10000) const result = await hub.ping() + // Pinned by hub_connector.test.js: returns true when a response contains a result. expect(result).to.be.true }) From 4b1715f6b144b44c0d5e3296a27f8bba9e4890b6 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Sun, 27 Sep 2026 00:39:01 -0700 Subject: [PATCH 31/50] test: keep update preflight suite within limits --- .../update_datadir_unset.test.js | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/test/unit/node_service.test/update_datadir_unset.test.js b/test/unit/node_service.test/update_datadir_unset.test.js index 023f7c54..21b9aa5d 100644 --- a/test/unit/node_service.test/update_datadir_unset.test.js +++ b/test/unit/node_service.test/update_datadir_unset.test.js @@ -12,20 +12,23 @@ const { expect, makeStubs, loadOperations, registerLifecycleHooks } = require('../module_operations.test/helpers/harness') -describe('Node update data directory preflight', function () { - let originalDataDir +let originalDataDir - registerLifecycleHooks(() => {}) +function clearConfiguredDataDir() { + originalDataDir = process.env.XCHAIN_NODE_DATA_DIR + delete process.env.XCHAIN_NODE_DATA_DIR +} - beforeEach(function () { - originalDataDir = process.env.XCHAIN_NODE_DATA_DIR - delete process.env.XCHAIN_NODE_DATA_DIR - }) +function restoreConfiguredDataDir() { + if (originalDataDir === undefined) delete process.env.XCHAIN_NODE_DATA_DIR + else process.env.XCHAIN_NODE_DATA_DIR = originalDataDir +} - afterEach(function () { - if (originalDataDir === undefined) delete process.env.XCHAIN_NODE_DATA_DIR - else process.env.XCHAIN_NODE_DATA_DIR = originalDataDir - }) +describe('Node update data directory preflight', function () { + registerLifecycleHooks(() => {}) + + beforeEach(clearConfiguredDataDir) + afterEach(restoreConfiguredDataDir) it('refuses a node update with no configured data directory before replacement begins', async function () { const stubs = makeStubs() From a66f891882f453a57b4a09f8112dbd205141b8ca Mon Sep 17 00:00:00 2001 From: J-Dog Date: Sun, 27 Sep 2026 01:02:27 -0700 Subject: [PATCH 32/50] Narrow the fast CI test selection Select mapped unit, security, and regression tests for ordinary changes while widening consensus-sensitive changes to the existing full CI tier. Preserve the full and unset tier behavior and fall back safely when change selection is unavailable. --- bin/ci-full.sh | 49 ++++++- bin/ci_fast_select.js | 233 ++++++++++++++++++++++++++++++++ bin/test/ci_fast_select.test.js | 134 ++++++++++++++++++ 3 files changed, 413 insertions(+), 3 deletions(-) create mode 100644 bin/ci_fast_select.js create mode 100644 bin/test/ci_fast_select.test.js diff --git a/bin/ci-full.sh b/bin/ci-full.sh index 758b4507..005b640a 100755 --- a/bin/ci-full.sh +++ b/bin/ci-full.sh @@ -66,6 +66,11 @@ ci_tier_deferred() { return 1 } # <<< ci-tier <<< +fast_defer() { + DEFERRED="$DEFERRED [$1]" + echo; echo "ci:full ===== $1 DEFERRED (CI_TIER=fast, runs in the full sweep) =====" +} + # >>> ci-tier timer (generated block; re-run the tier wirer to update) >>> run_tier() { ci_tier_deferred "$1" && return 0 # ci-tier guard (generated) @@ -111,14 +116,52 @@ if [ "${#DECLARED_SIBLINGS[@]}" -gt 0 ]; then need_sib "${DECLARED_SIBLINGS[@]}" fi +FAST_SELECTOR_MODE="full" +if [ "${CI_TIER:-full}" = "fast" ]; then + FAST_PLAN="" + FAST_SELECTOR_REASON="" + if [ ! -f bin/ci_fast_select.js ]; then + FAST_SELECTOR_REASON="helper missing" + elif FAST_PLAN="$(node bin/ci_fast_select.js --plan 2>&1)"; then + FAST_PLAN_HEADER="${FAST_PLAN%%$'\n'*}" + case "$FAST_PLAN_HEADER" in + "consensus 0") FAST_SELECTOR_MODE="changed" ;; + "consensus 1") FAST_SELECTOR_MODE="consensus" ;; + *) FAST_SELECTOR_REASON="selector returned no consensus decision" ;; + esac + if [ -z "$FAST_SELECTOR_REASON" ]; then + printf '%s\n' "$FAST_PLAN" + fi + else + FAST_SELECTOR_STATUS=$? + FAST_SELECTOR_REASON="exit $FAST_SELECTOR_STATUS: ${FAST_PLAN//$'\n'/; }" + fi + if [ -n "$FAST_SELECTOR_REASON" ]; then + echo "ci:full: fast selector unavailable ($FAST_SELECTOR_REASON); running the full unit tier" + fi +fi + # --- job: ci (XChain-Platform/.github ci-reusable.yml -> npm run ci) ------- # ci-reusable.yml arms XCHAIN_REQUIRE_SIBLINGS whenever it checked # siblings out, so every sibling guard fails loud on a miss instead of # skipping; match that here for a true local twin. -if [ "${#DECLARED_SIBLINGS[@]}" -gt 0 ]; then - run_tier "ci" env XCHAIN_REQUIRE_SIBLINGS=1 npm run ci +if [ "${CI_TIER:-full}" != "fast" ] || [ "$FAST_SELECTOR_MODE" != "changed" ]; then + if [ "${#DECLARED_SIBLINGS[@]}" -gt 0 ]; then + run_tier "ci" env XCHAIN_REQUIRE_SIBLINGS=1 npm run ci + else + run_tier "ci" npm run ci + fi else - run_tier "ci" npm run ci + if [ "${#DECLARED_SIBLINGS[@]}" -gt 0 ]; then + run_tier "ci (changed tests)" env XCHAIN_REQUIRE_SIBLINGS=1 node bin/ci_fast_select.js --run + else + run_tier "ci (changed tests)" node bin/ci_fast_select.js --run + fi + fast_defer "ci" +fi + +if [ "${CI_TIER:-full}" = "fast" ]; then + run_tier "fast-tier selector self-test" ./node_modules/.bin/mocha --no-config --timeout 20000 --exit bin/test/ci_fast_select.test.js fi # --- job: drift-guards ------------------------------------------------------- diff --git a/bin/ci_fast_select.js b/bin/ci_fast_select.js new file mode 100644 index 00000000..16885379 --- /dev/null +++ b/bin/ci_fast_select.js @@ -0,0 +1,233 @@ +'use strict' + +const fs = require('node:fs') +const path = require('node:path') +const { execFileSync, spawnSync } = require('node:child_process') + +const GROUPS = [ + { name: 'unit', prefix: 'test/unit/', args: ['--timeout', '2000', '--recursive', '--exit'] }, + { name: 'security', prefix: 'test/security/', args: ['--timeout', '10000', '--recursive', '--exit'] }, + { name: 'regression', prefix: 'test/regression/', args: ['--timeout', '30000', '--recursive', '--exit'] }, +] + +const CONSENSUS = [ + 'src/coins/', + 'src/github_hashes.json', + 'src/release-manifest.json', + 'src/config/', + 'src/services/bootstrap_service.js', + 'src/services/github_downloader.js', + 'src/services/validator_service.js', + 'src/services/go_live_gate.js', + 'src/services/skew_guard_service.js', + 'bin/pins/', + 'bin/pin_identity.js', +] + +const WIDEN = ['test/helpers/'] +const ALWAYS = [] + +function runGit(git, args) { + const output = git(args) + return Buffer.isBuffer(output) ? output.toString('utf8') : String(output || '') +} + +function resolveBase({ env, git }) { + const requested = env.PROM_CI_BASE_SHA + if (requested) { + try { + runGit(git, ['cat-file', '-e', `${requested}^{commit}`]) + return requested + } catch (_) {} + } + try { + return runGit(git, ['merge-base', 'HEAD', 'origin/develop']).trim() || null + } catch (_) { + return null + } +} + +function splitLines(value) { + if (Array.isArray(value)) return value.filter(Boolean) + return String(value || '').split(/\r?\n/).filter(Boolean) +} + +function groupFor(file) { + return GROUPS.find((group) => file.startsWith(group.prefix) && file.endsWith('.test.js')) +} + +function startsWithAny(file, entries) { + return entries.some((entry) => file.startsWith(entry)) +} + +function sourceText(file) { + try { + return fs.readFileSync(file, 'utf8') + } catch (_) { + return '' + } +} + +function requireTargets(source, importer) { + const targets = [] + const expression = /\brequire\s*\(\s*(['"])(\.[^'"]+)\1\s*\)/g + let match + while ((match = expression.exec(source))) { + const resolved = path.resolve(path.dirname(importer), match[2]) + targets.push(resolved, `${resolved}.js`, path.join(resolved, 'index.js')) + } + return targets +} + +function requiresFile(importer, target) { + const absoluteTarget = path.resolve(target) + return requireTargets(sourceText(importer), importer).includes(absoluteTarget) +} + +function namedModuleTest(importer, target) { + const tail = target.replace(/\.js$/, '') + return sourceText(importer).includes(tail) +} + +function sourceMatches(file, tests, requirers) { + const name = path.basename(file, '.js') + const relativeDirectory = path.dirname(file.slice('src/'.length)) + const matches = new Set() + for (const test of tests) { + const testName = path.basename(test) + const parts = test.split('/') + if (name !== 'index' && testName === `${name}.test.js`) matches.add(test) + if (parts.includes(`${name}.test`)) matches.add(test) + const group = groupFor(test) + const directDirectory = relativeDirectory === '.' + ? group && group.prefix.slice(0, -1) + : group && `${group.prefix}${relativeDirectory}` + if (directDirectory && path.dirname(test) === directDirectory) matches.add(test) + if (requirers.includes(test) && namedModuleTest(test, file)) matches.add(test) + } + return matches +} + +function collectSourceData(changedSources, findRequirers) { + return changedSources.map((file) => ({ + file, + requirers: splitLines(findRequirers(path.basename(file, '.js'))), + })) +} + +function consensusReasons(changedFiles, sourceData) { + const reasons = [] + for (const file of changedFiles) { + if (file === 'package.json' || startsWithAny(file, CONSENSUS) || startsWithAny(file, WIDEN)) { + reasons.push(`consensus: ${file}`) + } + } + for (const { file, requirers } of sourceData) { + for (const importer of requirers) { + if (startsWithAny(importer, CONSENSUS) && requiresFile(importer, file)) { + reasons.push(`consensus importer: ${importer} requires ${file}`) + } + } + } + return reasons +} + +function selectedTestFiles(changedFiles, tests, sourceData, reasons) { + const selected = new Set(ALWAYS) + for (const file of changedFiles) { + const group = groupFor(file) + if (group) selected.add(file) + else if (file.startsWith('test/') && file.endsWith('.test.js')) reasons.push(`deferred: ${file}`) + } + for (const { file, requirers } of sourceData) { + for (const test of sourceMatches(file, tests, requirers)) selected.add(test) + } + return selected +} + +function selectFastTests(changedFiles, { listTests, findRequirers }) { + const changed = splitLines(changedFiles) + const tests = splitLines(listTests()).filter((file) => groupFor(file) && fs.existsSync(file)) + const sources = changed.filter((file) => file.startsWith('src/') && file.endsWith('.js')) + const sourceData = collectSourceData(sources, findRequirers) + const reasons = consensusReasons(changed, sourceData) + const selected = selectedTestFiles(changed, tests, sourceData, reasons) + const consensus = reasons.some((reason) => reason.startsWith('consensus')) + const planned = consensus ? [] : [...selected] + .filter((file) => tests.includes(file)) + .sort() + .map((file) => ({ group: groupFor(file).name, file })) + return { consensus, reasons: [...new Set(reasons)].sort(), tests: planned } +} + +function git(args) { + return execFileSync('git', args, { encoding: 'utf8' }) +} + +function listTrackedFiles() { + return splitLines(git(['ls-files'])) +} + +function findRequirers(basename) { + const result = spawnSync('git', ['grep', '-l', '--fixed-strings', basename, '--', '*.js'], { + encoding: 'utf8', + }) + if (result.status === 1) return [] + if (result.status !== 0) throw new Error((result.stderr || 'git grep failed').trim()) + return splitLines(result.stdout) +} + +function resolvePlan() { + const base = resolveBase({ env: process.env, git }) + if (!base) return { error: 'git could not resolve a usable push base' } + const changed = splitLines(git(['diff', '--name-only', `${base}...HEAD`])) + return { plan: selectFastTests(changed, { listTests: listTrackedFiles, findRequirers }) } +} + +function printPlan(plan) { + console.log(`consensus ${plan.consensus ? 1 : 0}`) + for (const reason of plan.reasons) console.log(`reason ${reason}`) + for (const test of plan.tests) console.log(`test ${test.group} ${test.file}`) +} + +function runPlan(plan) { + if (plan.tests.length === 0) { + console.log('ci:fast: no test maps to this push') + return 0 + } + let failed = false + for (const group of GROUPS) { + const files = plan.tests.filter((test) => test.group === group.name).map((test) => test.file) + if (files.length === 0) continue + const result = spawnSync('./node_modules/.bin/mocha', ['--no-config', ...group.args, ...files], { + stdio: 'inherit', + }) + if (result.status !== 0) failed = true + } + return failed ? 1 : 0 +} + +function main() { + const mode = process.argv[2] + if (mode !== '--plan' && mode !== '--run') { + console.error('usage: node bin/ci_fast_select.js --plan|--run') + return 2 + } + let result + try { + result = resolvePlan() + } catch (error) { + console.log(`selector-error ${error.message}`) + return 2 + } + if (result.error) { + console.log(`no-base ${result.error}`) + return 3 + } + if (mode === '--plan') printPlan(result.plan) + return mode === '--run' ? runPlan(result.plan) : 0 +} + +module.exports = { resolveBase, selectFastTests } + +if (require.main === module) process.exitCode = main() diff --git a/bin/test/ci_fast_select.test.js b/bin/test/ci_fast_select.test.js new file mode 100644 index 00000000..6d2f2310 --- /dev/null +++ b/bin/test/ci_fast_select.test.js @@ -0,0 +1,134 @@ +'use strict' + +const assert = require('node:assert') +const fs = require('node:fs') +const path = require('node:path') +const { execFileSync, spawnSync } = require('node:child_process') + +const { resolveBase, selectFastTests } = require('../ci_fast_select') + +const ROOT = path.resolve(__dirname, '../..') + +function git(args) { + return execFileSync('git', args, { cwd: ROOT, encoding: 'utf8' }) +} + +function listTests() { + return git(['ls-files']).split(/\r?\n/).filter(Boolean) +} + +function findRequirers(basename) { + const result = spawnSync('git', ['grep', '-l', '--fixed-strings', basename, '--', '*.js'], { + cwd: ROOT, + encoding: 'utf8', + }) + if (result.status === 1) return [] + assert.strictEqual(result.status, 0, result.stderr) + return result.stdout.split(/\r?\n/).filter(Boolean) +} + +function select(changedFiles) { + const previous = process.cwd() + process.chdir(ROOT) + try { + return selectFastTests(changedFiles, { listTests, findRequirers }) + } finally { + process.chdir(previous) + } +} + +function files(plan) { + return plan.tests.map((test) => test.file) +} + +describe('bin/ci_fast_select.js', () => { + it('maps command_lock to its unit test without widening the suite', () => { + const plan = select(['src/utils/command_lock.js']) + assert.strictEqual(plan.consensus, false) + assert.ok(files(plan).includes('test/unit/command_lock.test.js')) + }) + + it('maps autoheal_service to its unit test family', () => { + const plan = select(['src/services/autoheal_service.js']) + assert.strictEqual(plan.consensus, false) + assert.ok(files(plan).includes('test/unit/autoheal_service.test.js')) + assert.ok(files(plan).includes('test/unit/autoheal_service.test/01_health_state_guards.test.js')) + }) + + it('widens for the hashes manifest and validator service', () => { + for (const changed of ['src/github_hashes.json', 'src/services/validator_service.js']) { + const plan = select([changed]) + assert.strictEqual(plan.consensus, true) + assert.ok(plan.reasons.some((reason) => reason.includes(changed))) + assert.deepStrictEqual(plan.tests, []) + } + }) + + it('widens when a consensus entry point requires the changed module', () => { + const plan = select(['src/state.js']) + assert.strictEqual(plan.consensus, true) + assert.ok(plan.reasons.some((reason) => reason.includes('src/services/bootstrap_service.js'))) + }) + + it('selects nothing for documentation alone', () => { + const plan = select(['README.md']) + assert.strictEqual(plan.consensus, false) + assert.deepStrictEqual(plan.tests, []) + assert.deepStrictEqual(plan.reasons, []) + }) + + it('widens for package metadata', () => { + const plan = select(['package.json']) + assert.strictEqual(plan.consensus, true) + assert.ok(plan.reasons.some((reason) => reason.includes('package.json'))) + }) + + it('defers a changed integration test instead of selecting it', () => { + const changed = 'test/integration/config_pipeline.test.js' + const plan = select([changed]) + assert.strictEqual(plan.consensus, false) + assert.deepStrictEqual(plan.tests, []) + assert.ok(plan.reasons.includes(`deferred: ${changed}`)) + }) + + it('returns null when neither requested nor fallback base resolves', () => { + const rejectingGit = () => { throw new Error('unknown revision') } + assert.strictEqual(resolveBase({ + env: { PROM_CI_BASE_SHA: 'missing' }, + git: rejectingGit, + }), null) + }) + + it('returns the requested base when git accepts it as a commit', () => { + const calls = [] + const acceptingGit = (args) => { + calls.push(args) + assert.deepStrictEqual(args, ['cat-file', '-e', 'abc123^{commit}']) + return '' + } + assert.strictEqual(resolveBase({ + env: { PROM_CI_BASE_SHA: 'abc123' }, + git: acceptingGit, + }), 'abc123') + assert.strictEqual(calls.length, 1) + }) + + it('falls back to the develop merge base after a stale requested base', () => { + const fallbackGit = (args) => { + if (args[0] === 'cat-file') throw new Error('stale') + assert.deepStrictEqual(args, ['merge-base', 'HEAD', 'origin/develop']) + return 'def456\n' + } + assert.strictEqual(resolveBase({ + env: { PROM_CI_BASE_SHA: 'stale' }, + git: fallbackGit, + }), 'def456') + }) + + it('keeps the fast selector guarded and the original strict ci tier available', () => { + const script = fs.readFileSync(path.join(ROOT, 'bin/ci-full.sh'), 'utf8') + assert.ok(script.includes('ci_fast_select.js --plan')) + assert.match(script, /CI_TIER:-full[^\n]+fast[\s\S]+ci_fast_select\.js --plan/) + assert.ok(script.includes('run_tier "ci" env XCHAIN_REQUIRE_SIBLINGS=1 npm run ci')) + }) +}) From 127ce19447523af85547b9e4b35d70613a354944 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Sun, 27 Sep 2026 02:28:58 -0700 Subject: [PATCH 33/50] Guard migrate status probe by source contract Refuse to execute migration CLIs unless their source advertises a quoted status flag. Cover legacy and strict parsers while preserving validated status execution. --- .../migration_scan.js | 1 + test/unit/utils/indexer_migrate_cli.test.js | 2 +- .../utils/migrate_cli_status_guard.test.js | 95 +++++++++++++++++++ 3 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 test/unit/utils/migrate_cli_status_guard.test.js diff --git a/src/services/migration_precondition_service/migration_scan.js b/src/services/migration_precondition_service/migration_scan.js index e3f0e7a5..d0db1a6e 100644 --- a/src/services/migration_precondition_service/migration_scan.js +++ b/src/services/migration_precondition_service/migration_scan.js @@ -91,6 +91,7 @@ async function runningBuildSupportsPerFileMigrations(container, deps = {}) { const cat = deps.getDockerContainerFileCat || require('../docker_service').getDockerContainerFileCat const found = await readMigrateCli(cat, container) if (!found) return null + if (!/(['"])--status\1/.test(found.source)) return false const execContainer = deps.execContainer || require('../docker_service').execContainer let raw diff --git a/test/unit/utils/indexer_migrate_cli.test.js b/test/unit/utils/indexer_migrate_cli.test.js index a2deb926..61aed71e 100644 --- a/test/unit/utils/indexer_migrate_cli.test.js +++ b/test/unit/utils/indexer_migrate_cli.test.js @@ -28,7 +28,7 @@ const GATED = '2026-07-24-pubkeys-widen-uncompressed.sql' const NEWEST_PATH = 'src/db/migration/migrate.js' const OLD_PATH = 'src/migrate.js' -const CLI_SOURCE = 'main()' +const CLI_SOURCE = "const STATUS = '--status'; const JSON_OUTPUT = '--json'; main()" const VALID_STATUS = JSON.stringify({ database: 'XChain_BTC_Mainnet_Indexer', total: 2, diff --git a/test/unit/utils/migrate_cli_status_guard.test.js b/test/unit/utils/migrate_cli_status_guard.test.js new file mode 100644 index 00000000..2daeb6cc --- /dev/null +++ b/test/unit/utils/migrate_cli_status_guard.test.js @@ -0,0 +1,95 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. + +const sinon = require('sinon') +const { expect } = require('chai') + +const { + runningBuildSupportsPerFileMigrations +} = require('../../../src/services/migration_precondition_service') + +const NEWEST_PATH = 'src/db/migration/migrate.js' +const OLD_PATH = 'src/migrate.js' +const VALID_STATUS = JSON.stringify({ + database: 'XChain_BTC_Mainnet_Indexer', + total: 2, + applied: 1, + pending: 1, + migrations: [ + { file: 'a.sql', applied: true, mode: 'auto', appliedAt: '2026-09-24T00:00:00.000Z' }, + { file: 'b.sql', applied: false, mode: 'manual', appliedAt: null } + ] +}) + +const FILE_ONLY_SOURCE = ` +const args = process.argv.slice(2) +const fileIndex = args.findIndex(arg => arg === '--file' || arg === '-f') +main(fileIndex >= 0 ? args[fileIndex + 1] : null) +` + +const STRICT_SOURCE = ` +const args = process.argv.slice(2) +for (const arg of args) { + if (arg === '--file' || arg === '-f' || arg === '--help') continue + process.exit(2) +} +main() +` + +const STATUS_SOURCE = ` +const STATUS = '--status' +const JSON_OUTPUT = '--json' +main() +` + +function catServing(filePath, source) { + return sinon.stub().callsFake(async (container, candidate) => { + if (candidate === filePath) return source + throw new Error('missing') + }) +} + +async function probe(filePath, source, output = VALID_STATUS) { + const execContainer = sinon.stub().resolves(output) + const deps = { + getDockerContainerFileCat: catServing(filePath, source), + execContainer + } + const result = await runningBuildSupportsPerFileMigrations('indexer', deps) + return { result, execContainer } +} + +describe('migrate CLI status contract guard', () => { + it('does not execute a file-only CLI at the current path', async () => { + const { result, execContainer } = await probe(NEWEST_PATH, FILE_ONLY_SOURCE) + expect(result).to.equal(false) + expect(execContainer.notCalled).to.equal(true) + }) + + it('does not execute a strict CLI without the status contract', async () => { + const { result, execContainer } = await probe(NEWEST_PATH, STRICT_SOURCE) + expect(result).to.equal(false) + expect(execContainer.notCalled).to.equal(true) + }) + + it('does not execute a file-only CLI at the old path', async () => { + const { result, execContainer } = await probe(OLD_PATH, FILE_ONLY_SOURCE) + expect(result).to.equal(false) + expect(execContainer.notCalled).to.equal(true) + }) + + it('executes a CLI that advertises the status contract', async () => { + const { result, execContainer } = await probe(NEWEST_PATH, STATUS_SOURCE) + expect(result).to.equal(true) + expect(execContainer.calledOnceWith( + 'indexer', ['node', NEWEST_PATH, '--status', '--json'] + )).to.equal(true) + }) +}) From 36620383c17b0921334717a3d4aa0b4146601a68 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Sun, 27 Sep 2026 02:30:14 -0700 Subject: [PATCH 34/50] Support both SDK encoder accessors Prefer the public encoder accessor when available and fall back to the legacy underscore method used by the locked SDK. Cover emitted signer broadcasts and chained stake funding for both supported SDK shapes. --- src/services/validator_service/signer.js | 6 +- src/services/validator_stake_service.js | 6 +- .../unit/validator_sdk_encoder_compat.test.js | 172 ++++++++++++++++++ 3 files changed, 182 insertions(+), 2 deletions(-) create mode 100644 test/unit/validator_sdk_encoder_compat.test.js diff --git a/src/services/validator_service/signer.js b/src/services/validator_service/signer.js index 8092dcb8..fc5d68cf 100644 --- a/src/services/validator_service/signer.js +++ b/src/services/validator_service/signer.js @@ -121,12 +121,16 @@ for (const [name, value] of [['DOGE_NETWORK', NETWORK], ['DOGE_WIF', WIF], ['DOG const sdk = new XChainSDK({ network: NETWORK, encoderUrl: ENCODER }); +function requireEncoder() { + return typeof sdk.requireEncoder === 'function' ? sdk.requireEncoder() : sdk['_requireEncoder'](); +} + module.exports = { // Full publish pipeline for a raw wire payload string -> { txid }. The // returned txid is the phase-2 (reveal) txid when two-phase encoding is // used; that is the transaction indexers decode. async broadcast(payload) { - const encoder = sdk.requireEncoder(); + const encoder = requireEncoder(); const txParams = { data: payload, pubkey: ADDRESS, change: ADDRESS, encoding: 'P2SH' }; if (FEE_PER_KB !== undefined) txParams.feePerKb = FEE_PER_KB; diff --git a/src/services/validator_stake_service.js b/src/services/validator_stake_service.js index 6035906f..9900c8b3 100644 --- a/src/services/validator_stake_service.js +++ b/src/services/validator_stake_service.js @@ -200,6 +200,10 @@ function explorerUrl(coins, pathPart) { const sleep = (ms) => new Promise(r => setTimeout(r, ms)) +function requireEncoder(sdk) { + return typeof sdk.requireEncoder === 'function' ? sdk.requireEncoder() : sdk['_requireEncoder']() +} + /** * The outputs of `prevTxid` that belong to this address, once the encoder can * see them. Seconds (mempool visibility), not a block. @@ -228,7 +232,7 @@ async function chainedInputs(sdk, address, prevTxid, timeoutMs) { const deadline = Date.now() + (timeoutMs || 90000) for (;;) { let utxos = null - try { utxos = await sdk.requireEncoder().getUTXOs(address) } catch { /* transient; retry */ } + try { utxos = await requireEncoder(sdk).getUTXOs(address) } catch { /* transient; retry */ } const outs = ((utxos && utxos.utxos) || []).filter(o => (o.fullTxid || o.txid) === prevTxid) if (outs.length) return outs const left = deadline - Date.now() diff --git a/test/unit/validator_sdk_encoder_compat.test.js b/test/unit/validator_sdk_encoder_compat.test.js new file mode 100644 index 00000000..eb89c8e7 --- /dev/null +++ b/test/unit/validator_sdk_encoder_compat.test.js @@ -0,0 +1,172 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +const { expect } = require('chai') +const sinon = require('sinon') +const proxyquire = require('proxyquire').noCallThru() +const path = require('path') +const vm = require('vm') +const { configStub } = require('../helpers/config_stub') +const { stakeValidator } = require('../../src/services/validator_stake_service') + +const FAKE_CONFIG_DIR = '/tmp/test-xchain-config' +const FAKE_VALIDATOR_DIR = path.join(FAKE_CONFIG_DIR, 'validator') +const FAKE_SIGNER_DIR = path.join(FAKE_VALIDATOR_DIR, 'signer') +const FAKE_SIGNER_FILE = path.join(FAKE_SIGNER_DIR, 'signer.js') +const FAKE_HUB_SIDECAR = path.join(FAKE_CONFIG_DIR, 'hub.local') +const PHASE1 = 'f'.repeat(64) +const PUBKEY = 'ab'.repeat(32) +const ADDRESS = 'mStakeAddress' + +function makeHubApiKeyStub() { + return sinon.stub().resolves({ path: FAKE_HUB_SIDECAR, generated: true }) +} + +function makeHubApiKeyReadStub() { + return sinon.stub().resolves({ path: FAKE_HUB_SIDECAR, present: false }) +} + +function loadValidatorService(fsStub) { + return proxyquire('../../src/services/validator_service', { + 'fs': fsStub, + './config_service': { + ensureHubApiKey: makeHubApiKeyStub(), + readHubApiKey: makeHubApiKeyReadStub() + }, + '../config': configStub({ configDir: FAKE_CONFIG_DIR }) + }) +} + +function makeFs() { + return { + existsSync: sinon.stub().returns(false), + readFileSync: sinon.stub().returns('{}'), + writeFileSync: sinon.stub(), + mkdirSync: sinon.stub(), + chmodSync: sinon.stub(), + lstatSync: sinon.stub().throws(Object.assign(new Error('ENOENT'), { code: 'ENOENT' })), + renameSync: sinon.stub(), + readdirSync: sinon.stub().returns(['capabilities.json']) + } +} + +function loadEmittedSigner(source, encoder, encoderMethod) { + const mod = { exports: {} } + vm.runInNewContext(source, { + require: (id) => { + if (id === 'path') return path + if (id === 'dotenv') return { config: () => ({}) } + if (id === '@dankest-llc/xchain-sdk') return { XChainSDK: function () { + this[encoderMethod] = () => encoder + this.wallet = { + signPsbt: () => ({ txHex: 'hex-1', txid: PHASE1 }), + signRevealPsbt: () => ({ txHex: 'hex-2', txid: 'e'.repeat(64) }) + } + } } + throw new Error('unexpected require in the emitted signer: ' + id) + }, + module: mod, + exports: mod.exports, + __dirname: FAKE_SIGNER_DIR, + console, + process: { env: { + DOGE_NETWORK: 'dogecoin-testnet', + DOGE_WIF: 'test-wif', + DOGE_ADDRESS: 'test-address', + DOGE_ENCODER_URL: 'http://encoder.invalid' + } }, + Number, String, Error, Promise, Object + }, { filename: 'signer.js' }) + return mod.exports +} + +async function emitSigner() { + const fs = makeFs() + await loadValidatorService(fs).initValidator({ network: 'testnet' }) + return fs.writeFileSync.getCalls().find(c => c.args[0] === FAKE_SIGNER_FILE).args[1] +} + +function stubEncoder() { + return { + createTx: async () => ({ psbt: 'psbt-1', encoding: 'P2SH' }), + broadcastTx: async () => ({ txid: PHASE1 }), + spendP2sh: async () => ({ psbt: 'psbt-2' }) + } +} + +function makeExplorer(chain) { + return { + getAddress: sinon.stub().resolves({ balances: { confirmed: chain.coin, pending: '0' } }), + getToken: sinon.stub().resolves({ mints: { max: 10000, address_max: 50000 } }), + getValidators: sinon.stub().resolves({ data: [] }) + } +} + +function makeSdk(chain = {}) { + const calls = { mint: [], stake: [] } + const prevTxidRef = { value: null } + const sdk = { + explorer: makeExplorer(chain), + getBalances: async () => ({ data: [{ tick: 'XCHAIN', amount: String(chain.xchain) }] }), + session: () => ({ + address: ADDRESS, + submit: async (actionData, enc, opts) => { + const rec = { params: actionData.params, enc, opts } + if (actionData.action === 'MINT') calls.mint.push(rec) + if (actionData.action === 'STAKE') calls.stake.push(rec) + const txid = actionData.action === 'STAKE' ? 'staketx' : 'mint' + calls.mint.length + prevTxidRef.value = txid + return { txid, spentInputs: (enc.utxos || []).map(u => ({ txid: u.txid, vout: u.vout })) } + } + }), + _requireEncoder: () => ({ + getUTXOs: async () => ({ utxos: [{ + txid: prevTxidRef.value || 'seed', + fullTxid: prevTxidRef.value || 'seed', + vout: 1, + value: '150000', + confirmations: 0 + }] }) + }) + } + return { sdk, calls } +} + +function run(opts, chain) { + const { sdk, calls } = makeSdk(chain) + const deps = { + settings: { enabled: true, pubkey: PUBKEY, network: 'testnet', P2P_PORT: 10002 }, + wallets: { NETWORK: 'testnet', STAKE_ADDRESS: ADDRESS, STAKE_WIF_SECRET: 'cFakeWif' }, + makeSdk: () => sdk, + sdk: { XChainSDK: function () { throw new Error('makeSdk should be used') } }, + log: () => {} + } + return stakeValidator(opts, deps).then(result => ({ result, calls })) +} + +describe('SDK encoder compatibility', function () { + for (const encoderMethod of ['_requireEncoder', 'requireEncoder']) { + it('broadcasts from the emitted signer with ' + encoderMethod, async function () { + const signer = loadEmittedSigner(await emitSigner(), stubEncoder(), encoderMethod) + const result = await signer.broadcast('payload') + expect(result.phase1_txid).to.equal(PHASE1) + }) + } + + it('chains stake actions with the underscore encoder method', async function () { + const { calls } = await run( + { broadcast: true, chainTimeoutMs: 50, balancePollMs: 5 }, + { xchain: 0, coin: '0.001' }) + expect(calls.mint[1].enc.utxos.map(u => u.txid)).to.deep.equal(['mint1']) + expect(calls.stake[0].enc.utxos.map(u => u.txid)).to.deep.equal(['mint3']) + }) +}) From 29f714ea7fd4796692889d86b971da306849f013 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Mon, 28 Sep 2026 05:44:10 -0700 Subject: [PATCH 35/50] Pass the anchor fold arming variable to regtest indexer and hub The fold gate is unpinned on regtest, so a deployed venue could not arm it from the host. Carry XC_ANCHOR_FOLD_REGTEST_ACTIVATION to the regtest indexer and the hub beside the mirror admission variable, never to mainnet or testnet. --- src/config/env_views.js | 4 +- src/services/config_service/networks.js | 4 +- src/services/config_service/services.js | 5 +- .../anchor_fold_passthrough.test.js | 68 +++++++++++++++++++ .../get_default_config_regtest.test.js | 3 +- 5 files changed, 79 insertions(+), 5 deletions(-) create mode 100644 test/unit/config_service.test/anchor_fold_passthrough.test.js diff --git a/src/config/env_views.js b/src/config/env_views.js index 2581dbb0..5099372e 100644 --- a/src/config/env_views.js +++ b/src/config/env_views.js @@ -59,7 +59,7 @@ const LIST_VIEWS = { "XCHAIN_PRICE_CONFIRMATION_BUFFER", "XCHAIN_PRICE_BOOTSTRAP_SATS", "XCHAIN_PRICE_MIN_BTC_VOLUME", "HUB_ALLOW_UNAUTHENTICATED", "XC_ROLLCALL_REGTEST_ACTIVATION", "XC_ROLLCALL_GATES_REGTEST_ACTIVATION", - "XC_MIRROR_ADMISSION_ACTIVATION", + "XC_MIRROR_ADMISSION_ACTIVATION", "XC_ANCHOR_FOLD_REGTEST_ACTIVATION", "XCHAIN_PRICE_INDEXER_DB_SECRET" ]], // The indexer's regtest genesis bootstrap (ConfigService genesisPassthroughVars). @@ -75,7 +75,7 @@ const LIST_VIEWS = { "DOGE_INDEXER_API_URL", "DOGE_INDEXER_API_KEY", "XC_ROLLCALL_REGTEST_ACTIVATION", "XC_ROLLCALL_GATES_REGTEST_ACTIVATION", "HUB_SYNC_ANCHOR_ATTEST_GRACE_S", "HUB_PRICE_SYNC_TIMEOUT_MS", "XCHAIN_COINPAY_EXPIRATION_S", - "XC_MIRROR_ADMISSION_ACTIVATION" + "XC_MIRROR_ADMISSION_ACTIVATION", "XC_ANCHOR_FOLD_REGTEST_ACTIVATION" ]], // The hub-sync watermark graces a regtest indexer is handed (hubSyncRegtestGraceVars). HUB_SYNC_GRACE_ENV: ['hub sync grace', [ diff --git a/src/services/config_service/networks.js b/src/services/config_service/networks.js index 88d309dc..bafa8fc2 100644 --- a/src/services/config_service/networks.js +++ b/src/services/config_service/networks.js @@ -244,6 +244,7 @@ async function hasBitcoinIndexer(network) { // (D84 precedent): it arms the admission-map mirror and its consumer and // barrier gates together, so a venue arms as a unit; the hub's own // mirror-admission gate module gates it for real. +// XC_ANCHOR_FOLD_REGTEST_ACTIVATION rides beside it: the hub and the regtest indexer arm the fold together. // Secret-bearing names in this list (XCHAIN_PRICE_INDEXER_DB_PASS) are also // accepted from the host env under their redaction-safe `*_SECRET` spelling; // everything else resolves to a plain process.env read. @@ -288,7 +289,8 @@ function configureHubBeforeKey(defaultValues, module) { "HUB_API_KEY", "HUB_ALLOW_UNAUTHENTICATED", "XC_ROLLCALL_REGTEST_ACTIVATION", "XC_ROLLCALL_GATES_REGTEST_ACTIVATION", - "XC_MIRROR_ADMISSION_ACTIVATION" + "XC_MIRROR_ADMISSION_ACTIVATION", + "XC_ANCHOR_FOLD_REGTEST_ACTIVATION" ] for (const varName of hubPassthroughVars) { const value = readSecretHostEnv(varName) diff --git a/src/services/config_service/services.js b/src/services/config_service/services.js index 4205d626..5a9558b4 100644 --- a/src/services/config_service/services.js +++ b/src/services/config_service/services.js @@ -179,6 +179,8 @@ function configureE2e(defaultValues, module, coin) { // path here the indexer side of the admission-map mirror can never be armed on // regtest (row 24x), and it must arm together with the hub's copy above or the // admission-era canonical refuses a legacy-map row and halts the block loop. +// XC_ANCHOR_FOLD_REGTEST_ACTIVATION arms the ANCHOR v3 fold gate the same way; unarmed, +// a deployed regtest indexer rejects every v3 bundle as an unknown version. // The indexer pushes chain tips to HUB_API_URL; when that hub enforces // HUB_API_KEY, the indexer must present the same key or its writes 401. // Sourced from host env so it persists across `update`, then from the shared @@ -205,7 +207,8 @@ function configureIndexerBeforeHubKey(defaultValues, module, network) { "HUB_SYNC_ANCHOR_ATTEST_GRACE_S", "HUB_PRICE_SYNC_TIMEOUT_MS", "XCHAIN_COINPAY_EXPIRATION_S", - "XC_MIRROR_ADMISSION_ACTIVATION") + "XC_MIRROR_ADMISSION_ACTIVATION", + "XC_ANCHOR_FOLD_REGTEST_ACTIVATION") for (const varName of rollcallPassthroughVars) { if (config.INDEXER_ROLLCALL_ENV[varName] !== undefined && config.INDEXER_ROLLCALL_ENV[varName] !== "") { defaultValues[varName] = config.INDEXER_ROLLCALL_ENV[varName] diff --git a/test/unit/config_service.test/anchor_fold_passthrough.test.js b/test/unit/config_service.test/anchor_fold_passthrough.test.js new file mode 100644 index 00000000..2d6f2b4b --- /dev/null +++ b/test/unit/config_service.test/anchor_fold_passthrough.test.js @@ -0,0 +1,68 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + + +const { expect, makeServiceWithConfig } = require('./helpers.test') + +// The fold gate is unpinned on regtest, so a deployed venue arms it only +// through the container env, and the hub and indexer must arm as a unit. +function anchorFoldPassthrough() { + const VAR = 'XC_ANCHOR_FOLD_REGTEST_ACTIVATION' + + let saved + + beforeEach(function () { + saved = process.env[VAR] + delete process.env[VAR] + }) + + afterEach(function () { + if (saved === undefined) delete process.env[VAR]; else process.env[VAR] = saved + }) + + it('carries a host value of 0 to a regtest indexer as the string 0', async function () { + process.env[VAR] = '0' + const cs = makeServiceWithConfig('') + const config = await cs.getDefaultConfig('xchain-indexer', 'dogecoin', 'regtest') + expect(config[VAR]).to.equal('0') + }) + + it('NEVER reaches a shared-ledger indexer, whatever the host env says', async function () { + process.env[VAR] = '0' + const cs = makeServiceWithConfig('') + for (const net of ['mainnet', 'testnet']) { + const config = await cs.getDefaultConfig('xchain-indexer', 'dogecoin', net) + expect(config, net).to.not.have.property(VAR) + } + }) + + it('reaches the container hub, so the venue arms as a unit', async function () { + process.env[VAR] = '0' + const cs = makeServiceWithConfig('') + const config = await cs.getDefaultConfig('xchain-hub', null, null) + expect(config[VAR]).to.equal('0') + }) + + it('is absent from both configs when the host leaves it unset', async function () { + const cs = makeServiceWithConfig('') + const indexer = await cs.getDefaultConfig('xchain-indexer', 'dogecoin', 'regtest') + const hub = await cs.getDefaultConfig('xchain-hub', null, null) + expect(indexer).to.not.have.property(VAR) + expect(hub).to.not.have.property(VAR) + }) +} + +describe('ConfigService', function () { + describe('getDefaultConfig()', function () { + describe('ANCHOR fold passthrough', anchorFoldPassthrough) + }) +}) diff --git a/test/unit/config_service.test/get_default_config_regtest.test.js b/test/unit/config_service.test/get_default_config_regtest.test.js index e326988f..9e0c1f46 100644 --- a/test/unit/config_service.test/get_default_config_regtest.test.js +++ b/test/unit/config_service.test/get_default_config_regtest.test.js @@ -34,7 +34,8 @@ function regtestOnlyPassthrough() { 'HUB_SYNC_ANCHOR_ATTEST_GRACE_S', 'HUB_PRICE_SYNC_TIMEOUT_MS', 'XCHAIN_COINPAY_EXPIRATION_S', - 'XC_MIRROR_ADMISSION_ACTIVATION' + 'XC_MIRROR_ADMISSION_ACTIVATION', + 'XC_ANCHOR_FOLD_REGTEST_ACTIVATION' ] let saved From cab0f995cec90923aa4565df8156b9f10d5d1fd7 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Mon, 28 Sep 2026 19:39:09 -0700 Subject: [PATCH 36/50] test: provide data dir for module updates Create and clean up a fresh temporary node data directory around each update suite while preserving the caller environment. --- .../01_update_modules.test.js | 25 ++++++++++++++++++ .../07_update_modules_all_expansion.test.js | 26 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/test/unit/module_operations.test/01_update_modules.test.js b/test/unit/module_operations.test/01_update_modules.test.js index 1c08b4cc..693ba89d 100644 --- a/test/unit/module_operations.test/01_update_modules.test.js +++ b/test/unit/module_operations.test/01_update_modules.test.js @@ -11,6 +11,26 @@ // contact legal@dankest.llc. const { sinon, expect, makeStubs, loadOperations, registerLifecycleHooks, requireFromUnit } = require('./helpers/harness') +const fs = require('fs') +const os = require('os') +const path = require('path') + +function registerNodeDataDirHooks() { + let dataDir, previousDataDir + beforeEach(function () { + previousDataDir = process.env.XCHAIN_NODE_DATA_DIR + dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'xchain-node-update-')) + process.env.XCHAIN_NODE_DATA_DIR = dataDir + }) + afterEach(function () { + try { + fs.rmSync(dataDir, { recursive: true, force: true }) + } finally { + if (previousDataDir === undefined) delete process.env.XCHAIN_NODE_DATA_DIR + else process.env.XCHAIN_NODE_DATA_DIR = previousDataDir + } + }) +} @@ -20,6 +40,7 @@ const { sinon, expect, makeStubs, loadOperations, registerLifecycleHooks, requir describe('moduleOperations', function () { + registerNodeDataDirHooks() let resolveInstallTargetStub, recordInstallTargetStub, resolveUpdateTargetStub registerLifecycleHooks(stubs => { ({ resolveInstallTargetStub, recordInstallTargetStub, resolveUpdateTargetStub } = stubs) @@ -69,6 +90,7 @@ describe('moduleOperations', function () { }) describe('moduleOperations', function () { + registerNodeDataDirHooks() let resolveInstallTargetStub, recordInstallTargetStub, resolveUpdateTargetStub registerLifecycleHooks(stubs => { ({ resolveInstallTargetStub, recordInstallTargetStub, resolveUpdateTargetStub } = stubs) @@ -117,6 +139,7 @@ describe('moduleOperations', function () { }) describe('moduleOperations', function () { + registerNodeDataDirHooks() let resolveInstallTargetStub, recordInstallTargetStub, resolveUpdateTargetStub registerLifecycleHooks(stubs => { ({ resolveInstallTargetStub, recordInstallTargetStub, resolveUpdateTargetStub } = stubs) @@ -162,6 +185,7 @@ describe('moduleOperations', function () { }) describe('moduleOperations', function () { + registerNodeDataDirHooks() let resolveInstallTargetStub, recordInstallTargetStub, resolveUpdateTargetStub registerLifecycleHooks(stubs => { ({ resolveInstallTargetStub, recordInstallTargetStub, resolveUpdateTargetStub } = stubs) @@ -211,6 +235,7 @@ describe('moduleOperations', function () { }) describe('moduleOperations', function () { + registerNodeDataDirHooks() let resolveInstallTargetStub, recordInstallTargetStub, resolveUpdateTargetStub registerLifecycleHooks(stubs => { ({ resolveInstallTargetStub, recordInstallTargetStub, resolveUpdateTargetStub } = stubs) diff --git a/test/unit/module_operations.test/07_update_modules_all_expansion.test.js b/test/unit/module_operations.test/07_update_modules_all_expansion.test.js index f86eb600..3804a43a 100644 --- a/test/unit/module_operations.test/07_update_modules_all_expansion.test.js +++ b/test/unit/module_operations.test/07_update_modules_all_expansion.test.js @@ -11,6 +11,26 @@ // contact legal@dankest.llc. const { sinon, expect, makeStubs, loadOperations, registerLifecycleHooks, requireFromUnit } = require('./helpers/harness') +const fs = require('fs') +const os = require('os') +const path = require('path') + +function registerNodeDataDirHooks() { + let dataDir, previousDataDir + beforeEach(function () { + previousDataDir = process.env.XCHAIN_NODE_DATA_DIR + dataDir = fs.mkdtempSync(path.join(os.tmpdir(), 'xchain-node-update-all-')) + process.env.XCHAIN_NODE_DATA_DIR = dataDir + }) + afterEach(function () { + try { + fs.rmSync(dataDir, { recursive: true, force: true }) + } finally { + if (previousDataDir === undefined) delete process.env.XCHAIN_NODE_DATA_DIR + else process.env.XCHAIN_NODE_DATA_DIR = previousDataDir + } + }) +} @@ -24,6 +44,7 @@ const { sinon, expect, makeStubs, loadOperations, registerLifecycleHooks, requir return requireFromUnit('../../src/services/config_service').filterCommandParameters(null, 'all', 'bitcoin', 'mainnet') } describe('moduleOperations', function () { + registerNodeDataDirHooks() let resolveInstallTargetStub, recordInstallTargetStub, resolveUpdateTargetStub registerLifecycleHooks(stubs => { ({ resolveInstallTargetStub, recordInstallTargetStub, resolveUpdateTargetStub } = stubs) @@ -70,6 +91,7 @@ describe('moduleOperations', function () { }) describe('moduleOperations', function () { + registerNodeDataDirHooks() let resolveInstallTargetStub, recordInstallTargetStub, resolveUpdateTargetStub registerLifecycleHooks(stubs => { ({ resolveInstallTargetStub, recordInstallTargetStub, resolveUpdateTargetStub } = stubs) @@ -119,6 +141,7 @@ describe('moduleOperations', function () { }) describe('moduleOperations', function () { + registerNodeDataDirHooks() let resolveInstallTargetStub, recordInstallTargetStub, resolveUpdateTargetStub registerLifecycleHooks(stubs => { ({ resolveInstallTargetStub, recordInstallTargetStub, resolveUpdateTargetStub } = stubs) @@ -150,6 +173,7 @@ describe('moduleOperations', function () { }) describe('moduleOperations', function () { + registerNodeDataDirHooks() let resolveInstallTargetStub, recordInstallTargetStub, resolveUpdateTargetStub registerLifecycleHooks(stubs => { ({ resolveInstallTargetStub, recordInstallTargetStub, resolveUpdateTargetStub } = stubs) @@ -185,6 +209,7 @@ describe('moduleOperations', function () { }) describe('moduleOperations', function () { + registerNodeDataDirHooks() let resolveInstallTargetStub, recordInstallTargetStub, resolveUpdateTargetStub registerLifecycleHooks(stubs => { ({ resolveInstallTargetStub, recordInstallTargetStub, resolveUpdateTargetStub } = stubs) @@ -235,6 +260,7 @@ describe('moduleOperations', function () { }) describe('moduleOperations', function () { + registerNodeDataDirHooks() let resolveInstallTargetStub, recordInstallTargetStub, resolveUpdateTargetStub registerLifecycleHooks(stubs => { ({ resolveInstallTargetStub, recordInstallTargetStub, resolveUpdateTargetStub } = stubs) From 8371b5d1d3bdb07cfdb10be456c237750ba2dadb Mon Sep 17 00:00:00 2001 From: J-Dog Date: Tue, 29 Sep 2026 09:02:47 -0700 Subject: [PATCH 37/50] Fix 27 issues found in a code review round MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each fix was re-derived from the code, verified against its own tests, and adversarially re-checked before it was accepted. - Node's migrationMode twin has drifted from the indexer's: untagged or unknown mode reads as null, not 'manual' - The twin cross-reference in node's migrationMode names an indexer symbol that does not exist - Twin cross-references between node and indexer migration parsers are stale or missing - clear-reorg-halt relay reads err.stdout/err.stderr, but execContainer rejects with a bare callback-style ... - A --reason starting with '-' passes the node's check but the decoder's parseArgs refuses it as a missing ... - Tracker archive height may understate actual snapshot data height - Archive metadata trusted before cryptographic verification - Archive identity (coin/network/module) is not verified at restore time - Stake pre-check doesn't check delegations, so a plan the indexer will reject as 'already delegated' is shown ... - Stake pre-check refuses re-staking a key whose stake has fully cooled down, but the indexer accepts it once ... - Both stake and unstake read only the first 100-row page of getValidators, so a stake off that page reads as ... - Unstake CLI treats any status='valid' stake row as active, so it broadcasts UNSTAKE for stakes already ... - A funding chain that breaks at the final MINT→STAKE hop still broadcasts an unchained STAKE instead of ... - Unstake plan prints one stakes row's amount as the whole stake to be withdrawn and locked - stakeTiming doc cites xchain-indexer/src/actions/unstake.js, which is now the actions/unstake/ directory - reset: decoder/indexer bounce swallows restartContainer failures, so reset exits 0 with the service down - reset (docker DB mode): MariaDB root credential is first resolved after node data and tracker volume are wiped - Stake CLI's 'already staked' check drifts from indexer validateFreeKey: ignores delegations and the key-reuse ... - DEFAULT_STAKE_AMOUNT comment attributes the 25000 floor to the indexer's STAKING.CAPABILITIES; it is the ... - Node execs only src/clear_reorg_halt.js, which decoders v0.16.0–v0.18.x do not have; the shim covers ... - Gate ignores decoder's reorg_halt_parked, so a parked decoder whose halt marker was never written passes the ... - Gate header says an indexer database cannot hold sync_halt, but xchain-sync creates and writes sync_halt in ... - migration_files.js header describes an indexer migrations layout the indexer's runner does not read - clear-reorg-halt is missing from mutatingCommands, so its audited DB write runs without the command lock - skipsPreAction exempts every validator subcommand from the command lock, including key-regenerating init ... - When the UTXO tracker is unreachable, the balance reads 0 and the CLI blocks with 'fund it first' instead of ... - readChainState reads token.mints without unwrapping the one-element array envelope the SDK documents, so a ... --- .ci-siblings | 4 + bin/pins/suite-title-renames.json | 3 + src/cli/dispatch.js | 49 +++- src/cli/output.js | 8 +- src/operations/module_operations.js | 3 +- .../module_operations/module_controls.js | 61 ++++- .../module_operations/reset_modules.js | 12 +- src/services/bootstrap_archive_meta.js | 30 ++- src/services/bootstrap_health_gate.js | 11 +- .../bootstrap_health_gate/halt_markers.js | 4 +- .../status_evaluation.js | 14 +- src/services/bootstrap_service.js | 4 +- .../bootstrap_service/ensure_restored.js | 23 +- .../bootstrap_service/restore_archive.js | 24 ++ .../bootstrap_service/tracker_archive.js | 55 ++-- .../bootstrap_service/tracker_height.js | 102 +++++++ src/services/docker_service.js | 5 +- .../migration_scan.js | 27 +- src/services/stop_budget_service.js | 38 ++- src/services/validator_stake_service.js | 80 ++++-- .../validator_stake_service/free_key.js | 251 +++++++++++++++++ .../validator_stake_service/send_lock.js | 47 ++++ .../stake_operations.js | 185 ++++++++++--- .../unstake_operations.js | 142 ++++++++-- .../validator_set_read.js | 65 +++++ src/utils/migration_files.js | 20 +- test/helpers/stake_harness.js | 120 +++++++++ test/unit/bootstrap_archive_meta.test.js | 73 +++++ .../durable_halt_markers.test.js | 17 +- .../evaluate_status_payload.test.js | 27 ++ .../bootstrap_service.test/helpers/support.js | 17 +- ...ive_identity_and_refusal_signature.test.js | 210 +++++++++++++++ ...acker_archive_height_after_restart.test.js | 145 ++++++++++ .../sibling_roster.test.js | 5 +- test/unit/cli/dispatch_lock.test.js | 170 ++++++++++++ test/unit/cli/validator_send_lock.test.js | 95 +++++++ .../13_exec_container_error_branch.test.js | 9 +- .../01_migration_metadata.test.js | 46 +++- .../02_migration_inventory.test.js | 9 +- .../03_module_controls.test.js | 63 ++++- .../11_reset_external_db_reachability.test.js | 74 +++++ .../17_module_error_paths.test.js | 68 +++++ .../module_operations.test/helpers/harness.js | 3 + test/unit/stop_budget_service.test.js | 46 ++++ test/unit/utils/migration_files.test.js | 12 +- .../unit/validator_sdk_encoder_compat.test.js | 4 +- .../free_key.test.js | 126 +++++++++ .../send_lock.test.js | 253 ++++++++++++++++++ .../stake_free_key_check.test.js | 113 ++++++++ .../stake_validator.test.js | 135 +++------- .../unstake_validator.test.js | 109 +++++++- .../upstream_reads.test.js | 88 ++++++ .../validator_set_paging.test.js | 165 ++++++++++++ 53 files changed, 3160 insertions(+), 309 deletions(-) create mode 100644 src/services/bootstrap_service/tracker_height.js create mode 100644 src/services/validator_stake_service/free_key.js create mode 100644 src/services/validator_stake_service/send_lock.js create mode 100644 src/services/validator_stake_service/validator_set_read.js create mode 100644 test/helpers/stake_harness.js create mode 100644 test/unit/bootstrap_service.test/restore_checks_archive_identity_and_refusal_signature.test.js create mode 100644 test/unit/bootstrap_service.test/tracker_archive_height_after_restart.test.js create mode 100644 test/unit/cli/dispatch_lock.test.js create mode 100644 test/unit/cli/validator_send_lock.test.js create mode 100644 test/unit/validator_stake_service.test/free_key.test.js create mode 100644 test/unit/validator_stake_service.test/send_lock.test.js create mode 100644 test/unit/validator_stake_service.test/stake_free_key_check.test.js create mode 100644 test/unit/validator_stake_service.test/upstream_reads.test.js create mode 100644 test/unit/validator_stake_service.test/validator_set_paging.test.js diff --git a/.ci-siblings b/.ci-siblings index a018ad4c..fab14c6f 100644 --- a/.ci-siblings +++ b/.ci-siblings @@ -18,3 +18,7 @@ xchain-hub # src/rollcall_activation.js). Both guards throw under XCHAIN_REQUIRE_SIBLINGS=1 # when this checkout is absent, which is exactly what CI run 35020607785 did. xchain-indexer +# The migrationMode parity guard (migration_precondition_service.test) holds +# this tool's copy to the decoder runner's src/db/migration_statements.js, and +# throws under XCHAIN_REQUIRE_SIBLINGS=1 when this checkout is absent. +xchain-decoder diff --git a/bin/pins/suite-title-renames.json b/bin/pins/suite-title-renames.json index 45a1d2c8..d98067a4 100644 --- a/bin/pins/suite-title-renames.json +++ b/bin/pins/suite-title-renames.json @@ -188,6 +188,9 @@ "MariaDbStore removeModuleContainer() returns false when pool is not ready": "MariaDbStore deleteModuleContainer() returns false when pool is not ready", "MariaDbStore removeModuleContainer() returns true for non-existent key (idempotent delete)": "MariaDbStore deleteModuleContainer() returns true for non-existent key (idempotent delete)" }, + "test/unit/migration_precondition_service.test.js": { + "MigrationPreconditionService migrationMode returns null when no mode is declared": "MigrationPreconditionService migrationMode defaults to manual when no recognized mode is declared" + }, "test/unit/module_service.test.js": { "ModuleService buildAndUp(): module-specific port/volume branches rejects when insertModuleContainer returns false": "ModuleService buildAndUp(): module-specific port/volume branches rejects when setModuleContainer returns false", "ModuleService uninstallModule(): removeModuleContainer returns false throws when removeModuleContainer returns false after successful container removal": "ModuleService uninstallModule(): deleteModuleContainer returns false throws when deleteModuleContainer returns false after successful container removal" diff --git a/src/cli/dispatch.js b/src/cli/dispatch.js index d9234ed7..ace37e57 100644 --- a/src/cli/dispatch.js +++ b/src/cli/dispatch.js @@ -30,7 +30,9 @@ function dispatchSettings(config) { // invocation is refused with a clear message instead of interleaving. // `e2etest` is included: its action does its own docker build/run/rm, so it // must stay serialized against install/update the same way the others are. - const mutatingCommands = ['install', 'update', 'recreate', 'reinstall', 'uninstall', 'reset', 'bootstrap', 'sync', 'start', 'stop', 'restart', 'rollback', 'autoheal', 'e2etest'] + // `clear-reorg-halt` is included, dry runs too: its database-intact check and + // the audited clear it writes must not interleave with reset, restore or recreate. + const mutatingCommands = ['install', 'update', 'recreate', 'reinstall', 'uninstall', 'reset', 'bootstrap', 'sync', 'start', 'stop', 'restart', 'rollback', 'autoheal', 'e2etest', 'clear-reorg-halt'] // How long a non-mutating command blocks for a lock-holding mutator before // giving up (bounded so a read-only command pauses, then errors clearly, // rather than corrupting the stack by provisioning concurrently). Tunable. @@ -47,6 +49,7 @@ function skipsPreAction(actionCommand) { // `validator` subcommands are offline (key generation + local config // file writes). They must NOT trigger the Docker/MariaDB precheck, so an // operator can prepare their validator identity before any stack is up. + // Writers still lock: init to exit (validatorLockLabel), stake/unstake sends (validatorSendLock). const parentName = actionCommand.parent && actionCommand.parent.name() if (commandName === 'validator' || parentName === 'validator') return true // `rollback` is declared but unimplemented: its action only names the @@ -67,6 +70,38 @@ function skipsPreAction(actionCommand) { return commandName === 'bootstrap-republish-due' } +// Name the lock for a validator subcommand that must serialize, or null for one +// that need not. `init` writes the signing key, settings, capabilities and wallets +// one file at a time, and a concurrent `install`/`update xchain-hub` mounts them. +function validatorLockLabel(actionCommand) { + const parentName = actionCommand.parent && actionCommand.parent.name() + if (parentName !== 'validator') return null + return actionCommand.name() === 'init' ? 'validator init' : null +} + +// Build the lock `validator stake|unstake --broadcast` holds only while it sends. +// The service hands it back before the indexer wait (up to --timeout), so a long +// wait blocks no deploy; it refuses a held lock the way any mutator does. +function validatorSendLock(command, deps) { + const { MUTATING_LOCK_WAIT_MS } = dispatchSettings(deps.config || {}) + let release = null + const onExit = () => { if (release) release() } + return { + hold() { + if (release) return + release = deps.acquireCommandLock({ command, waitMs: MUTATING_LOCK_WAIT_MS }) + process.on('exit', onExit) + }, + release() { + if (!release) return + const drop = release + release = null + process.removeListener('exit', onExit) + drop() + } + } +} + // preCheck provisions shared containers/DB/hub (buildDatabaseModule, // ensureXchainNodeAccess, scanAndRegisterModules, installHubModule) for // EVERY non-validator command, not just the mutating ones. Running that @@ -80,8 +115,8 @@ function skipsPreAction(actionCommand) { // the lock only across preCheck, releasing it right after, so a // long-running `monitor`/`tail`/`logs` does not pin the lock for its // lifetime; it waits a bounded time for a busy mutator, then errors. -function commandLock(commandName, settings, acquireCommandLock) { - const holdThroughAction = settings.mutatingCommands.includes(commandName) +function commandLock(commandName, settings, acquireCommandLock, + holdThroughAction = settings.mutatingCommands.includes(commandName)) { let release try { release = acquireCommandLock({ @@ -135,6 +170,12 @@ async function beforeAction(thisCommand, actionCommand, settings, deps) { setVerbose(thisCommand.opts().verbose ?? false) if (thisCommand.opts().verbose) console.log("Checking xchain-node structure") const commandName = actionCommand.name() + // Hold the lock to exit for a validator file writer, still skipping preCheck. + const validatorLabel = validatorLockLabel(actionCommand) + if (validatorLabel) { + commandLock(validatorLabel, settings, acquireCommandLock, true) + return + } if (skipsPreAction(actionCommand)) return if (commandName === 'update') { @@ -175,4 +216,4 @@ function installDispatch(program, deps) { beforeAction(thisCommand, actionCommand, settings, deps)) } -module.exports = { installDispatch } +module.exports = { installDispatch, validatorSendLock } diff --git a/src/cli/output.js b/src/cli/output.js index 73f541e9..489724e9 100644 --- a/src/cli/output.js +++ b/src/cli/output.js @@ -14,6 +14,8 @@ * XChain Node - CLI * Commander setup and command definitions ********************************************************************/ +const { validatorSendLock } = require('./dispatch') + function registerPs(program, deps) { const { getStatus } = deps program @@ -121,7 +123,7 @@ function registerValidatorStake(validator, deps) { validator .command('stake') .description('Mint XCHAIN if short (testnet) and broadcast the STAKE naming this validator\'s pubkey; dry run without --broadcast') - .option('--amount ', 'amount to stake (default 25000, clears every capability floor)') + .option('--amount ', 'amount to stake (default 25000, clears every capability floor under default governance settings)') .option('--broadcast', 'actually send the transactions (default: print the plan only)') .option('--no-wait', 'return once the STAKE is broadcast instead of waiting for it to index') .option('--serialize', 'send one action per block (default: chained back to back into one block)') @@ -129,7 +131,7 @@ function registerValidatorStake(validator, deps) { .option('--timeout ', 'how long to wait for the stake to index (default 120)') .action(async (opts) => { try { - await stakeValidator(opts) + await stakeValidator(opts, { sendLock: validatorSendLock('validator stake', deps) }) } catch (e) { console.error('\nERROR: ' + e.message + '\n') return process.exit(1) @@ -150,7 +152,7 @@ function registerValidatorUnstake(validator, deps) { .option('--timeout ', 'how long to wait for it to index (default 120)') .action(async (opts) => { try { - await unstakeValidator(opts) + await unstakeValidator(opts, { sendLock: validatorSendLock('validator unstake', deps) }) } catch (e) { console.error('\nERROR: ' + e.message + '\n') return process.exit(1) diff --git a/src/operations/module_operations.js b/src/operations/module_operations.js index 266dd205..27f14de1 100644 --- a/src/operations/module_operations.js +++ b/src/operations/module_operations.js @@ -30,7 +30,7 @@ const { sleep } = require('../utils/helpers') const { getDockerContainerImageName, getUtxoTrackerVolumeName, getDockerNetwork } = require('../services/config_service') const { createDockerNetwork, probeContainerPresenceByName, stopContainer, stopContainerByName, startContainer, restartContainer, execContainer, shellContainer, logContainer, startDockerMonitor, waitContainer, saveContainerLogs, getContainerBindMounts, getContainerStopSettings, removeContainer } = require('../services/docker_service') const { stopModuleContainer } = require('../services/stop_budget_service') -const { buildDatabaseModule, resetDatabases, clearHubPriceIngestWatermark, purgeHubCrossChainRows, manualHubCrossChainPurgeStatements, getDatabaseContainerId, pingExternalDatabase } = require('../services/database_service') +const { buildDatabaseModule, resetDatabases, clearHubPriceIngestWatermark, purgeHubCrossChainRows, manualHubCrossChainPurgeStatements, getDatabaseContainerId, pingExternalDatabase, askMariadbRootPassword } = require('../services/database_service') const { getModuleBranch, installModule, uninstallModule } = require('../services/module_service') const { assertHubNotBehind } = require('../services/skew_guard_service') const { assertRequiredMigrationsApplied } = require('../services/migration_precondition_service') @@ -164,6 +164,7 @@ const dependencies = { stopModuleContainer, buildDatabaseModule, resetDatabases, clearHubPriceIngestWatermark, purgeHubCrossChainRows, manualHubCrossChainPurgeStatements, getDatabaseContainerId, + askMariadbRootPassword, pingExternalDatabase, getModuleBranch, installModule: installModuleWithProgress, uninstallModule, assertHubNotBehind, assertRequiredMigrationsApplied, statusChanged, reindexAffectedModules, recordReindex, config, bootstrapService, diff --git a/src/operations/module_operations/module_controls.js b/src/operations/module_operations/module_controls.js index f1c9fe21..35731f9f 100644 --- a/src/operations/module_operations/module_controls.js +++ b/src/operations/module_operations/module_controls.js @@ -127,6 +127,29 @@ async function startModules(servicesList) { return true } +// The decoder's clear tool, relative to the decoder container's working directory. +// Decoder releases before v0.19.0 ship it only as src/clear-reorg-halt.js. +const CLEAR_TOOL_PATH = 'src/clear_reorg_halt.js' +// Exit code of the probe below when the container is up and the tool file is absent. +const CLEAR_TOOL_ABSENT = 3 + +// Print why a decoder's clear failed, and name an update when the tool itself is missing. +async function reportClearFailure(containerId, coin, network, err) { + // execContainer attaches the child's stdout (the live halt it measured) and stderr (the refusal). + const text = [err && err.stdout, err && err.stderr].filter(Boolean).join('\n').trim() + console.log(text || ('clear-reorg-halt: failed for ' + coin + ' ' + network + ': ' + (err && err.message))) + // Exit 2-5 are the tool's own refusals, so only a generic exit 1 can mean the tool is not there. + if (!err || err.code !== 1) return + const probe = 'process.exit(require("fs").existsSync(' + JSON.stringify(CLEAR_TOOL_PATH) + ') ? 0 : ' + CLEAR_TOOL_ABSENT + ')' + try { + await execContainer(containerId, ['node', '-e', probe]) + } catch (probeErr) { + if (!probeErr || probeErr.code !== CLEAR_TOOL_ABSENT) return + console.log('clear-reorg-halt: the xchain-decoder for ' + coin + ' ' + network + ' has no ' + CLEAR_TOOL_PATH + + ', so it predates decoder v0.19.0; update the decoder, then run this command again.') + } +} + // Audited clear of a decoder's durable REORG_HALT marker, run inside the decoder // container so it uses the service's own DB credentials and code // (xchain-decoder/src/clear_reorg_halt.js checks the database is intact, then @@ -140,8 +163,9 @@ async function clearDecoderReorgHalt(servicesList, { reason, force = false, dryR console.log('clear-reorg-halt: --reason must say, in at least 8 characters, why this database is known good; it is recorded with the clear.') return false } - const args = ['node', 'src/clear_reorg_halt.js'] - if (reasonText) args.push('--reason', reasonText) + const args = ['node', CLEAR_TOOL_PATH] + // Send the reason as one argument: the decoder refuses a separate --reason value that starts with '-'. + if (reasonText) args.push('--reason=' + reasonText) if (force) args.push('--force') if (dryRun) args.push('--dry-run') let targeted = 0 @@ -160,10 +184,8 @@ async function clearDecoderReorgHalt(servicesList, { reason, force = false, dryR const out = await execContainer(containerId, args) if (out) console.log(out) } catch (err) { - // The script prints its refusal on stderr and exits non-zero; docker - // exec surfaces that as an error whose stdout/stderr carry the text. - const text = [err && err.stdout, err && err.stderr].filter(Boolean).join('\n').trim() - console.log(text || ('clear-reorg-halt: failed for ' + nextCoin + ' ' + nextNetwork + ': ' + (err && err.message))) + // The script prints its refusal on stderr and exits non-zero. + await reportClearFailure(containerId, nextCoin, nextNetwork, err) ok = false } } @@ -256,13 +278,30 @@ async function bounceResetModules(context) { if (bounceCandidates.length > 0) { await sleep(5000) for (const module of bounceCandidates) { + // Skip a module the start pass already reported, so one failure is not listed twice. + if (startFailures.some((f) => f.module === module)) continue + let containerId = null + try { + containerId = await db.getModuleContainerStrict(module, coin, network) + } catch (err) { + // A failed read is not "not installed": the bounce did not happen, so report it like a start failure. + startFailures.push({ module, error: `registry lookup failed before the post-reset bounce (${failureReason(err)})` }) + continue + } + if (!containerId) continue /* not installed, skip */ + // restartContainer = docker stop + docker start; sufficient to re-enter Node's bootstrap with the freshly-created DB ready. + // A failed restart can leave the wiped service stopped, so it gets the start pass's one retry and is then reported, never swallowed. try { - const containerId = await db.getModuleContainer(module, coin, network) - // Latent today only because the catch below hides a null-arg failure; guard explicitly so a future narrower catch stays correct (uuid:fd7cc224 sibling site). - if (!containerId) continue - // restartContainer = docker stop + docker start; sufficient to re-enter Node's bootstrap with the freshly-created DB ready. await restartContainer(containerId) - } catch { /* not installed, skip */ } + } catch (firstErr) { + console.warn(`Failed to restart ${module} (${firstErr && firstErr.message}); retrying in 3s...`) + await sleep(3000) + try { + await restartContainer(containerId) + } catch (retryErr) { + startFailures.push({ module, error: `post-reset restart failed (${(retryErr && retryErr.message) || String(retryErr)})` }) + } + } } } diff --git a/src/operations/module_operations/reset_modules.js b/src/operations/module_operations/reset_modules.js index c9be512d..bb0d3dc3 100644 --- a/src/operations/module_operations/reset_modules.js +++ b/src/operations/module_operations/reset_modules.js @@ -1,10 +1,10 @@ 'use strict' -let confirmDestructiveReset, failureReason, isNoSuchContainerError, isNoSuchVolumeError, resolveNodeDataPath, restartResetModules, restartStoppedModules, Coin, CoinTickerSymbol, EXTERNAL_DB, HUB_MODULE_NAME, Network, NODE_MODULE_NAME, XChainService, clearHubPriceIngestWatermark, config, dataDir, db, execFileAsync, fs, getContainerBindMounts, getDatabaseContainerId, getDockerContainerImageName, getUtxoTrackerVolumeName, manualHubCrossChainPurgeStatements, nodeService, path, pingExternalDatabase, purgeHubCrossChainRows, readline, recordReindex, reindexAffectedModules, resetDatabases, restartContainer, sleep, startContainer, statusChanged, stopContainer +let askMariadbRootPassword, confirmDestructiveReset, failureReason, isNoSuchContainerError, isNoSuchVolumeError, resolveNodeDataPath, restartResetModules, restartStoppedModules, Coin, CoinTickerSymbol, EXTERNAL_DB, HUB_MODULE_NAME, Network, NODE_MODULE_NAME, XChainService, clearHubPriceIngestWatermark, config, dataDir, db, execFileAsync, fs, getContainerBindMounts, getDatabaseContainerId, getDockerContainerImageName, getUtxoTrackerVolumeName, manualHubCrossChainPurgeStatements, nodeService, path, pingExternalDatabase, purgeHubCrossChainRows, readline, recordReindex, reindexAffectedModules, resetDatabases, restartContainer, sleep, startContainer, statusChanged, stopContainer let RESETTABLE_SERVICES function configure(dependencies) { - ({ confirmDestructiveReset, failureReason, isNoSuchContainerError, isNoSuchVolumeError, resolveNodeDataPath, restartResetModules, restartStoppedModules, Coin, CoinTickerSymbol, EXTERNAL_DB, HUB_MODULE_NAME, Network, NODE_MODULE_NAME, XChainService, clearHubPriceIngestWatermark, config, dataDir, db, execFileAsync, fs, getContainerBindMounts, getDatabaseContainerId, getDockerContainerImageName, getUtxoTrackerVolumeName, manualHubCrossChainPurgeStatements, nodeService, path, pingExternalDatabase, purgeHubCrossChainRows, readline, recordReindex, reindexAffectedModules, resetDatabases, restartContainer, sleep, startContainer, statusChanged, stopContainer } = dependencies) + ({ askMariadbRootPassword, confirmDestructiveReset, failureReason, isNoSuchContainerError, isNoSuchVolumeError, resolveNodeDataPath, restartResetModules, restartStoppedModules, Coin, CoinTickerSymbol, EXTERNAL_DB, HUB_MODULE_NAME, Network, NODE_MODULE_NAME, XChainService, clearHubPriceIngestWatermark, config, dataDir, db, execFileAsync, fs, getContainerBindMounts, getDatabaseContainerId, getDockerContainerImageName, getUtxoTrackerVolumeName, manualHubCrossChainPurgeStatements, nodeService, path, pingExternalDatabase, purgeHubCrossChainRows, readline, recordReindex, reindexAffectedModules, resetDatabases, restartContainer, sleep, startContainer, statusChanged, stopContainer } = dependencies) RESETTABLE_SERVICES = [ 'all', NODE_MODULE_NAME, @@ -174,6 +174,14 @@ async function checkResetDatabase(context) { console.log('Aborted: MariaDB container not found; install the database first. No data was touched.') return false } + // Resolve the root password now, not in resetDatabases after the wipes: a stale root after a rotation, or a + // non-TTY run with no usable copy, must refuse here. The value is cached, so resetDatabases asks nothing again. + try { + await askMariadbRootPassword(coin, network) + } catch (err) { + console.log(`Aborted: cannot resolve the MariaDB root password (${failureReason(err)}). No data was touched.`) + return false + } } } diff --git a/src/services/bootstrap_archive_meta.js b/src/services/bootstrap_archive_meta.js index 0c0d8891..d651450c 100644 --- a/src/services/bootstrap_archive_meta.js +++ b/src/services/bootstrap_archive_meta.js @@ -31,9 +31,11 @@ * size, so the restore learns the height before it touches the store. * * The member sits inside the signed wrapper, so its provenance is the - * archive's. An archive published before this member existed reads as - * "height unknown", never as an error: the restore still works, it just cannot - * compare. + * archive's only once the caller has verified the wrapper's signature: a read + * before that is a claim, and no outcome may rest on it until the signature + * check has run (the restore paths verify before they act on it). An archive + * published before this member existed reads as "height unknown", never as an + * error: the restore still works, it just cannot compare. ********************************************************************/ const fs = require('fs') @@ -129,7 +131,9 @@ function readLeadingArchiveMember(archivePath, memberName, { maxBytes = META_MAX const header = parseTarHeader(buffered.subarray(offset, offset + 512)) if (!header) return finish(null) if (TAR_EXTENSION_TYPES.has(header.typeflag)) { - if (++skipped > maxSkippedEntries) return finish(null) + // Cap the declared size too, not just the count: this runs before the signature check, and an + // uncapped size makes the walk buffer whatever the gunzip emits (real pax headers are bytes). + if (++skipped > maxSkippedEntries || header.size > maxBytes) return finish(null) offset += 512 + Math.ceil(header.size / 512) * 512 continue } @@ -165,12 +169,30 @@ async function readBootstrapArchiveMeta(archivePath) { } } +const ARCHIVE_IDENTITY_FIELDS = ['module', 'coin', 'network'] + +// Compare an archive's embedded identity with the restore target: { status: 'match' | 'unchecked' | 'mismatch', mismatches, unchecked }. +// Only a present field that differs is a mismatch. A null meta or field is unchecked, not a mismatch: legacy archives carry none, and +// the tracker's convert-bootstrap-to-classiclevel.sh writes coin/network null when BOOTSTRAP_COIN/BOOTSTRAP_NETWORK are unset. +function compareArchiveIdentity(meta, target) { + const mismatches = [] + const unchecked = [] + for (const field of ARCHIVE_IDENTITY_FIELDS) { + const value = meta && typeof meta[field] === 'string' ? meta[field] : null + if (value === null) unchecked.push(field) + else if (value !== target[field]) mismatches.push({ field, archive: value, target: target[field] }) + } + const status = mismatches.length > 0 ? 'mismatch' : unchecked.length > 0 ? 'unchecked' : 'match' + return { status, mismatches, unchecked } +} + module.exports = { BOOTSTRAP_META_MEMBER, BOOTSTRAP_META_FORMAT, buildBootstrapMeta, writeBootstrapMeta, readBootstrapArchiveMeta, + compareArchiveIdentity, // Exported for tests parseTarHeader, readLeadingArchiveMember diff --git a/src/services/bootstrap_health_gate.js b/src/services/bootstrap_health_gate.js index e6f83ebc..11047f0f 100644 --- a/src/services/bootstrap_health_gate.js +++ b/src/services/bootstrap_health_gate.js @@ -32,9 +32,10 @@ * - reporting an unhealthy/halted status on its own health surface * - carrying a durable halt marker in its database: a decoder REORG_HALT row * (events.code = 'REORG_HALT') or an uncleared xchain-sync divergence halt - * (sync_halt with cleared_at IS NULL). For an indexer source that means the - * PAIRED DECODER's database, which is the only place either marker is written; - * an indexer's own events table only ever carries code='REORG'. + * (sync_halt with cleared_at IS NULL). For an indexer source both its own + * database and the PAIRED DECODER's are read: REORG_HALT is written only in the + * decoder's (an indexer's own events table only ever carries code='REORG'), + * while xchain-sync writes sync_halt into either replica. * - materially behind its node's tip * * FAIL CLOSED throughout. A probe that cannot be run, cannot be parsed, or @@ -298,8 +299,8 @@ async function haltMarkerReasons(coin, network, module, dbDeps, since) { if (markers.syncHalt > 0) reasons.push('the database carries an uncleared xchain-sync divergence halt ' + '(sync_halt with cleared_at IS NULL): its contents are known to diverge from the source of truth.') - // An indexer's own database cannot hold these rows; the paired decoder's can, - // and an indexer frozen behind a halted decoder is exactly as unfit to publish. + // Refuse on the paired decoder's markers too: REORG_HALT lives only there, and an + // indexer frozen behind a halted decoder is exactly as unfit to publish. if (markers.upstream && markers.upstream.reorgHalt > 0) reasons.push(`the paired decoder database ${markers.upstream.dbName} carries a durable REORG_HALT ` + "marker (events.code='REORG_HALT'), so this indexer is frozen behind a decoder that aborted " + diff --git a/src/services/bootstrap_health_gate/halt_markers.js b/src/services/bootstrap_health_gate/halt_markers.js index 15cf3458..7c9ae1a9 100644 --- a/src/services/bootstrap_health_gate/halt_markers.js +++ b/src/services/bootstrap_health_gate/halt_markers.js @@ -223,7 +223,7 @@ async function readRaisedInWindow(readCount, name, since, found, hasSyncHalt) { // quietly carrying a stale halt marker, and the one that does not depend on // the running image being new enough to report the marker on its health // surface: read the marker rows straight out of the database being dumped - and, -// for an indexer source, out of the paired decoder database that owns them. +// for an indexer source, out of the paired decoder database that owns REORG_HALT. async function readHaltMarkers(coin, network, module, deps, since) { const run = sqlRunner(coin, network, deps) const readCount = countReader(run) @@ -232,7 +232,7 @@ async function readHaltMarkers(coin, network, module, deps, since) { const markers = await probeDatabase(run, readCount, dbName, since && since.own) // An xchain-indexer database structurally CANNOT carry the REORG_HALT marker, so - // the probe above is a guaranteed zero for an indexer source and this backstop had + // the REORG_HALT count above is a guaranteed zero for an indexer and this backstop had // no reach there at all: the indexer only ever writes code='REORG' into its own // events table and reads the halt marker out of the DECODER's connection, while // the marker row is written solely into the decoder database. Nothing else in the diff --git a/src/services/bootstrap_health_gate/status_evaluation.js b/src/services/bootstrap_health_gate/status_evaluation.js index 75f68803..7ca0edbb 100644 --- a/src/services/bootstrap_health_gate/status_evaluation.js +++ b/src/services/bootstrap_health_gate/status_evaluation.js @@ -65,7 +65,8 @@ function evaluateContainerState(raw, { now = Date.now() } = {}) { // Interpret a decoder/indexer/utxo-tracker health payload. Pure, so the policy // is unit-testable without docker. Field names differ per service, which is why // every known spelling is checked rather than one canonical key: -// decoder health: status, lag_blocks/blockLag, reorg_halted, reorg_halt_checked_at +// decoder health: status, lag_blocks/blockLag, reorg_halted, reorg_halt_checked_at, +// reorg_halt_parked // indexer health: status, lag, decoderReorgHalted, stallClass // tracker health: lag, synced, halted // any /status: status ('ok'|'healthy'|'halted'|'degraded'|'unhealthy') @@ -113,6 +114,7 @@ function statusFlagReasons(payload) { && (payload.reorg_halt_checked_at === null || payload.reorg_halt_checked_at === undefined)) reasons.push('the decoder has never completed a REORG_HALT marker probe (reorg_halt_checked_at is ' + 'null), so its "not halted" report is an untested default rather than a reading') + reasons.push(...reorgParkReasons(payload)) if (payload.decoderReorgHalted === true) reasons.push('the upstream decoder carries a durable REORG_HALT marker, so this database is frozen behind it') // The indexer's own single-field verdict on its block counter: @@ -136,6 +138,16 @@ function statusFlagReasons(payload) { return reasons } +// Refuse a decoder parked on a REORG_HALT whatever reorg_halted says: a failed marker +// write leaves no row, so the next probe reports false and only reorg_halt_parked tells +// the truth. Strict equality, so an image publishing no such field keeps today's behavior. +function reorgParkReasons(payload) { + if (payload.reorg_halt_parked !== true) return [] + return ['the decoder\'s parse loop is PARKED on a REORG_HALT (reorg_halt_parked is true), so nothing is ' + + 'being parsed and this database is frozen at the halt' + + (payload.reorg_halt_parked_at ? ` since ${payload.reorg_halt_parked_at}` : '')] +} + // Refusal reasons from the first lag field the payload publishes, checked // against `maxLag`. function lagReasons(payload, maxLag) { diff --git a/src/services/bootstrap_service.js b/src/services/bootstrap_service.js index 77150536..791188d5 100644 --- a/src/services/bootstrap_service.js +++ b/src/services/bootstrap_service.js @@ -113,12 +113,12 @@ configureMariadbArchive({ }) configureRestoreArchive({ fs, zlib, childProcess, config, state, configService, dockerService, - databaseService, archiveSigning, workspace, logger + databaseService, archiveSigning, archiveMeta, workspace, logger }) configureFreshness({ childProcess, config, configService, databaseService, logger }) configureDownload({ fs, axios, config, workspace, logger, staleAfterDays: BOOTSTRAP_STALE_AFTER_DAYS }) configureEnsureRestored({ - fs, config, configService, nodeTipGuard, restoreArchive, download, logger + fs, config, configService, nodeTipGuard, archiveSigning, restoreArchive, download, logger }) const { XChainService } = config diff --git a/src/services/bootstrap_service/ensure_restored.js b/src/services/bootstrap_service/ensure_restored.js index 3d0b6287..139e5841 100644 --- a/src/services/bootstrap_service/ensure_restored.js +++ b/src/services/bootstrap_service/ensure_restored.js @@ -25,6 +25,7 @@ let { getDefaultConfig } = require('../config_service') let { assessNodeTipForRestore } = require('../bootstrap_node_tip_guard') const { redactSecrets } = require('../../utils/helpers') const { BOOTSTRAP_SIG_SUFFIX } = require('./archive_signing') +let { checkBootstrapSignature } = require('./archive_signing') let { restoreBootstrap } = require('./restore_archive') let { downloadBootstrap } = require('./download') const { getLogger } = require('../../observability/logger') @@ -36,6 +37,7 @@ function configureDependencies(dependencies) { ;({ XChainService } = config) ;({ getDefaultConfig } = dependencies.configService) ;({ assessNodeTipForRestore } = dependencies.nodeTipGuard) + ;({ checkBootstrapSignature } = dependencies.archiveSigning) ;({ restoreBootstrap } = dependencies.restoreArchive) ;({ downloadBootstrap } = dependencies.download) logger = dependencies.logger @@ -114,6 +116,13 @@ async function assessNodeTipBeforeRestore(coin, network, module, archivePath) { return assessNodeTipForRestore({ coin, network, module, archivePath }) } +// Verify the signature before a refusal retires the archive: the refusal rests on the unverified bootstrap.json height. +// A tampered archive then throws into the caller's catch, which records the integrity failure and KEEPS the archive, +// instead of being reported as node lag and deleted. Costs one hash pass, on the refuse branch only. +async function verifyRefusedArchive(archivePath) { + await checkBootstrapSignature(archivePath) +} + // A restore that went ahead with the node still below the archive is reported // as such, so the summary identifies the service's wait state. function recordRestoredOutcome(module, tip, archive) { @@ -212,9 +221,10 @@ async function ensureBootstrapUtxoTracker(coin, network) { archivePath = path.join(bootstrapDir, fileName) const tip = await assessNodeTipBeforeRestore(coin, network, XChainService.XCHAIN_UTXO_TRACKER, archivePath) if (tip.refuse) { - // Refused before any restore was attempted, so nothing needs the - // archive as evidence; see retireBootstrapArchive for why it goes - // anyway rather than waiting around for a re-run. + // Refused before any restore was attempted, and the archive is + // signature-verified first, so retiring it destroys no evidence; + // see retireBootstrapArchive for why it goes anyway. + await verifyRefusedArchive(archivePath) const bytes = await statBootstrapArchiveBytes(archivePath) const archive = retireBootstrapArchive(archivePath, bytes, 'restore refused, node behind; a re-run downloads a fresh copy anyway') recordBootstrapOutcome(XChainService.XCHAIN_UTXO_TRACKER, 'node-behind', tip.detail, archive) @@ -276,9 +286,10 @@ async function ensureBootstrapMariaDb(coin, network, module) { archivePath = path.join(bootstrapDir, fileName) const tip = await assessNodeTipBeforeRestore(coin, network, module, archivePath) if (tip.refuse) { - // Refused before any restore was attempted, so nothing needs the - // archive as evidence; see retireBootstrapArchive for why it goes - // anyway rather than waiting around for a re-run. + // Refused before any restore was attempted, and the archive is + // signature-verified first, so retiring it destroys no evidence; + // see retireBootstrapArchive for why it goes anyway. + await verifyRefusedArchive(archivePath) const bytes = await statBootstrapArchiveBytes(archivePath) const archive = retireBootstrapArchive(archivePath, bytes, 'restore refused, node behind; a re-run downloads a fresh copy anyway') recordBootstrapOutcome(module, 'node-behind', tip.detail, archive) diff --git a/src/services/bootstrap_service/restore_archive.js b/src/services/bootstrap_service/restore_archive.js index 6a1c171d..40ecacf8 100644 --- a/src/services/bootstrap_service/restore_archive.js +++ b/src/services/bootstrap_service/restore_archive.js @@ -32,6 +32,8 @@ let { getDatabaseContainerId, ensureDatabasePool, getExternalDbConfig, executeNa let databaseService = require('../database_service') const { dockerMariadbArgs, mariadbEnv } = require('../../utils/docker_mariadb') let { checkBootstrapSignature, ensureVerifiedInnerArchive } = require('./archive_signing') +const { BootstrapIntegrityError } = require('./archive_signing') +let { readBootstrapArchiveMeta, compareArchiveIdentity } = require('../bootstrap_archive_meta') let { startProgress, getWorkDir } = require('./workspace') const { getLogger } = require('../../observability/logger') let logger = getLogger() @@ -48,6 +50,7 @@ function configureDependencies(dependencies) { databaseService = dependencies.databaseService ;({ getDatabaseContainerId, ensureDatabasePool, getExternalDbConfig, executeNativeMariaDbCommand } = databaseService) ;({ checkBootstrapSignature, ensureVerifiedInnerArchive } = dependencies.archiveSigning) + ;({ readBootstrapArchiveMeta, compareArchiveIdentity } = dependencies.archiveMeta) ;({ startProgress, getWorkDir } = dependencies.workspace) logger = dependencies.logger } @@ -64,6 +67,25 @@ async function restoreBootstrap(coin, network, module, fileName) { } } +// Refuse an archive whose bootstrap.json names another module, coin or network than the restore target. +// Runs after the signature check, so the identity compared is authenticated, and before any extract, stop or wipe. The signature +// covers only the archive digest under one publisher key, so it proves who made the archive, not which slot it belongs to. +async function assertArchiveIdentity(archivePath, target) { + const identity = compareArchiveIdentity(await readBootstrapArchiveMeta(archivePath), target) + const label = (id) => `${id.module} ${id.coin}/${id.network}` + if (identity.status === 'mismatch') { + const found = identity.mismatches.map(m => `${m.field} ${m.archive}`).join(', ') + throw new BootstrapIntegrityError( + `Bootstrap archive identity mismatch for ${archivePath}: the archive declares ${found} but the restore target is ${label(target)}. ` + + 'Refusing to restore; nothing was stopped or wiped.' + ) + } + // Legacy archives carry no identity: say it could not be checked, then restore as before. + if (identity.status === 'unchecked') { + logger.info(`Note: the bootstrap archive does not declare its ${identity.unchecked.join('/')}, so that part of its identity cannot be checked against ${label(target)}.`) + } +} + async function restoreBootstrapUtxoTracker(coin, network, fileName) { const context = await prepareTrackerRestore(coin, network, fileName) @@ -93,6 +115,7 @@ async function prepareTrackerRestore(coin, network, fileName) { // corruption (it ships inside the same archive). Provenance comes from the // detached signature checked here. await checkBootstrapSignature(archivePath) + await assertArchiveIdentity(archivePath, { module: XChainService.XCHAIN_UTXO_TRACKER, coin, network }) // Extract and verify the inner archive against the checksum that // shipped inside the signature-verified outer archive (resumable, but the @@ -196,6 +219,7 @@ async function prepareMariaRestore(coin, network, module, fileName) { // corruption (it ships inside the same archive). Provenance comes from the // detached signature checked here. await checkBootstrapSignature(archivePath) + await assertArchiveIdentity(archivePath, { module, coin, network }) // Extract and verify the inner archive against the checksum that // shipped inside the signature-verified outer archive (resumable, but the diff --git a/src/services/bootstrap_service/tracker_archive.js b/src/services/bootstrap_service/tracker_archive.js index e3a26151..92e68295 100644 --- a/src/services/bootstrap_service/tracker_archive.js +++ b/src/services/bootstrap_service/tracker_archive.js @@ -36,6 +36,8 @@ let { declareEncoderMaintenance, clearEncoderMaintenance } = require('../encoder const { redactSecrets } = require('../../utils/helpers') let { maybeSignBootstrap } = require('./archive_signing') let { startProgress, buildDateTimeString, getWorkDir, assertBootstrapCapacity, ensureDir, ensureDirWritable } = require('./workspace') +const trackerHeight = require('./tracker_height') +const { readTrackerCommittedHeight, readTrackerHeightAfterRestart, chooseArchiveHeight } = trackerHeight const { getLogger } = require('../../observability/logger') let logger = getLogger() @@ -55,6 +57,7 @@ function configureDependencies(dependencies) { ;({ maybeSignBootstrap } = dependencies.archiveSigning) ;({ startProgress, buildDateTimeString, getWorkDir, assertBootstrapCapacity, ensureDir, ensureDirWritable } = dependencies.workspace) logger = dependencies.logger + trackerHeight.configureDependencies(dependencies) } // Where the pre-compress hardlink snapshot of the tracker volume lives. @@ -183,6 +186,9 @@ async function makeBootstrapUtxoTracker(coin, network) { try { const snapshotTaken = await takeTrackerSnapshot(context, endMaintenanceWindow) await compressTrackerArchive(context, snapshotTaken) + // Fallback path: the store is archived, so bring the tracker back before the wrap and sign, and read its height there. + if (!snapshotTaken) await restartTrackerAndReadHeight(context, endMaintenanceWindow) + context.archiveHeight = chooseArchiveHeight(context.preStopHeight, context.postRestartHeight) await finalizeTrackerArchive(coin, network, context) } finally { await cleanupTrackerArchive(context.volumeName, context.containerId, context.containerRestored, endMaintenanceWindow) @@ -206,9 +212,11 @@ async function prepareTrackerArchive(coin, network) { context.containerId = await db.getModuleContainer(XChainService.XCHAIN_UTXO_TRACKER, coin, network) if (!context.containerId) throw new Error(`utxo-tracker container not found for ${coin}/${network}`) - // The height the archive will end at, read from the tracker's own status while it is still running (the store is LevelDB, nothing else can tell). - // Best-effort: an archive without a height still restores, it just cannot be compared with the coin node at restore time (BootstrapNodeTipGuard). - context.archiveHeight = await readTrackerCommittedHeight(coin, network, context.containerId) + // A floor for the archive height, read while the tracker still runs: blocks it commits before the stop completes are in the archive + // but not in this reading, so the recorded height is taken again after the restart (chooseArchiveHeight). Best-effort: an archive + // without a height still restores, it just cannot be compared with the coin node at restore time (BootstrapNodeTipGuard). + context.preStopHeight = await readTrackerCommittedHeight(coin, network, context.containerId) + context.postRestartHeight = null // Staging and output dirs are prepared before the stop for the same reason as the capacity check: a read-only mount or a missing parent should not be // discovered with the tracker already dark. @@ -292,14 +300,21 @@ async function takeTrackerSnapshot(context, endMaintenanceWindow) { // encoder correctly published as tracker_reachable:false. if (snapshotTaken) { logger.info(`Starting ${XChainService.XCHAIN_UTXO_TRACKER} container (compressing from the snapshot)...`) - await startContainer(context.containerId) - context.containerRestored = true - await endMaintenanceWindow() + await restartTrackerAndReadHeight(context, endMaintenanceWindow) } return snapshotTaken } +// Restart the tracker, end the maintenance window, and read the committed height it comes back at. +// The restarted store is the archived store plus whatever it commits since, so this reading sits at or above the archived data. +async function restartTrackerAndReadHeight(context, endMaintenanceWindow) { + await startContainer(context.containerId) + context.containerRestored = true + await endMaintenanceWindow() + context.postRestartHeight = await readTrackerHeightAfterRestart(context.coin, context.network, context.containerId) +} + async function compressTrackerArchive(context, snapshotTaken) { const tarSource = snapshotTaken ? `/data/${TRACKER_SNAPSHOT_DIR}` : '/data' @@ -369,31 +384,11 @@ async function cleanupTrackerArchive(volumeName, containerId, containerRestored, await endMaintenanceWindow() } -// The tracker's committed height from its status surface, or null. Asked -// before the container is stopped for the compress. -async function readTrackerCommittedHeight(coin, network, containerId) { - try { - const { probeServiceStatus, MODULE_API_PORT_KEY } = bootstrapHealthGate - if (typeof probeServiceStatus !== 'function') return null - const config = await getDefaultConfig(XChainService.XCHAIN_UTXO_TRACKER, coin, network) - const port = config && config[MODULE_API_PORT_KEY[XChainService.XCHAIN_UTXO_TRACKER]] - if (!port) return null - const runner = (cmd, args) => execFileAsync(cmd, args, { timeout: 15000 }) - const payload = await probeServiceStatus(containerId, port, runner) - const candidates = ['committed_height', 'tracker_height'] - for (const key of candidates) { - const value = Number(payload && payload[key]) - if (Number.isInteger(value) && value >= 0) return value - } - return null - } catch (err) { - logger.info(`Could not read the tracker height for the archive metadata (${redactSecrets(err.message)}); the archive will carry no height.`) - return null - } -} - module.exports = { configureDependencies, makeBootstrapUtxoTracker, - readTrackerCommittedHeight + readTrackerCommittedHeight, + // Exported for tests + chooseArchiveHeight, + readTrackerHeightAfterRestart } diff --git a/src/services/bootstrap_service/tracker_height.js b/src/services/bootstrap_service/tracker_height.js new file mode 100644 index 00000000..d87f68c8 --- /dev/null +++ b/src/services/bootstrap_service/tracker_height.js @@ -0,0 +1,102 @@ +/********************************************************************* + * + * Copyright © 2025–2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC – https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ********************************************************************** + * XChain Node - Bootstrap Service + * The end height a utxo-tracker bootstrap archive records. The store is + * LevelDB, so the tracker's own status surface is the only thing that can + * tell; the restore-time node tip guard trusts the recorded height as a + * bound the archived data never exceeds. + ********************************************************************/ + +let { execFile } = require('child_process') +const { promisify } = require('util') +let execFileAsync = promisify(execFile) + +let { XChainService } = require('../../config') +let { getDefaultConfig } = require('../config_service') +let bootstrapHealthGate = require('../bootstrap_health_gate') +const { redactSecrets } = require('../../utils/helpers') +const { getLogger } = require('../../observability/logger') +let logger = getLogger() + +// Takes the same dependency bag tracker_archive.js is configured with. +function configureDependencies(dependencies) { + ;({ execFile } = dependencies.childProcess) + execFileAsync = promisify(execFile) + ;({ XChainService } = dependencies.config) + ;({ getDefaultConfig } = dependencies.configService) + bootstrapHealthGate = dependencies.bootstrapHealthGate + logger = dependencies.logger +} + +// How long a just-restarted tracker gets to answer its status surface before the pre-stop floor is used instead. +const POST_RESTART_PROBE_ATTEMPTS = 12 +const POST_RESTART_PROBE_DELAY_MS = 5000 + +// The tracker's committed height from its status surface, or null. Asked +// before the stop (a floor) and again after the restart (the recorded height). +async function readTrackerCommittedHeight(coin, network, containerId, { quiet = false } = {}) { + try { + const { probeServiceStatus, MODULE_API_PORT_KEY } = bootstrapHealthGate + if (typeof probeServiceStatus !== 'function') return null + const config = await getDefaultConfig(XChainService.XCHAIN_UTXO_TRACKER, coin, network) + const port = config && config[MODULE_API_PORT_KEY[XChainService.XCHAIN_UTXO_TRACKER]] + if (!port) return null + const runner = (cmd, args) => execFileAsync(cmd, args, { timeout: 15000 }) + const payload = await probeServiceStatus(containerId, port, runner) + const candidates = ['committed_height', 'tracker_height'] + for (const key of candidates) { + const value = Number(payload && payload[key]) + if (Number.isInteger(value) && value >= 0) return value + } + return null + } catch (err) { + if (!quiet) logger.info(`Could not read the tracker height for the archive metadata (${redactSecrets(err.message)}); the archive will carry no height.`) + return null + } +} + +// The restarted tracker's committed height, retried while its API comes up, or null. +async function readTrackerHeightAfterRestart(coin, network, containerId, { attempts = POST_RESTART_PROBE_ATTEMPTS, delayMs = POST_RESTART_PROBE_DELAY_MS } = {}) { + // No status probe wired in: nothing a retry could change. + if (typeof bootstrapHealthGate.probeServiceStatus !== 'function') return null + for (let i = 0; i < attempts; i++) { + const height = await readTrackerCommittedHeight(coin, network, containerId, { quiet: true }) + if (height !== null) return height + if (i + 1 < attempts) await new Promise(resolve => setTimeout(resolve, delayMs)) + } + return null +} + +// Pick the height the archive records: never below the data it holds, since the restore guard treats nodeHeight >= archiveHeight as safe. +// max() keeps a rollback during the restart from lowering it. With no post-restart reading, the pre-stop floor is kept (a height off by +// the stop window still guards more than none, which the guard reads as "unknown" and restores unchecked). +function chooseArchiveHeight(preStopHeight, postRestartHeight) { + const pre = Number.isInteger(preStopHeight) ? preStopHeight : null + const post = Number.isInteger(postRestartHeight) ? postRestartHeight : null + const chosen = post === null ? pre : pre === null ? post : Math.max(pre, post) + if (post === null && pre !== null) { + logger.info(`WARNING: could not read the tracker height after the restart; the archive records the pre-stop height ${pre}, ` + + 'which may understate the archived data by the blocks committed while the tracker was stopping.') + } else { + logger.info(`Archive height: ${chosen === null ? 'unknown' : chosen} (pre-stop ${pre === null ? 'unknown' : pre}, after restart ${post === null ? 'unknown' : post}).`) + } + return chosen +} + +module.exports = { + configureDependencies, + readTrackerCommittedHeight, + readTrackerHeightAfterRestart, + chooseArchiveHeight +} diff --git a/src/services/docker_service.js b/src/services/docker_service.js index 0a792ef9..12f7bd02 100644 --- a/src/services/docker_service.js +++ b/src/services/docker_service.js @@ -144,8 +144,11 @@ async function startContainer(containerId) { async function execContainer(containerId, commandArgs) { return new Promise((resolve, reject) => { - execFile('docker', ['exec', '-i', containerId, ...commandArgs], (error, stdout) => { + execFile('docker', ['exec', '-i', containerId, ...commandArgs], (error, stdout, stderr) => { if (error) { + // Attach the child's output, which the callback form of execFile leaves off the error. + error.stdout = stdout + error.stderr = stderr reject(error) } else { resolve(stdout.trim()) diff --git a/src/services/migration_precondition_service/migration_scan.js b/src/services/migration_precondition_service/migration_scan.js index d0db1a6e..fc948669 100644 --- a/src/services/migration_precondition_service/migration_scan.js +++ b/src/services/migration_precondition_service/migration_scan.js @@ -32,10 +32,10 @@ const LEDGER_TABLE = 'schema_migrations' * body prose or a data literal. Widening that to the whole file is how a * migration that merely DISCUSSES the convention would start refusing deploys. * - * Twin of xchain-indexer's Database.migrationDeclaresDeployPrecondition. It is - * duplicated rather than shared because this tool reads these files out of a - * source tree it has only cloned, with that tree's dependencies uninstalled, so - * requiring the module is not available to it. Keep the two in step. + * Twin of xchain-indexer src/db/database/migration_registry.js and xchain-decoder + * src/db/migration_preconditions.js (both Database.migrationDeclaresDeployPrecondition). + * Duplicated, not shared: this tool reads a cloned source tree whose dependencies are + * not installed, so it cannot require either module. Keep all three in step. */ function migrationDeclaresDeployPrecondition(raw) { const prologue = [] @@ -48,13 +48,14 @@ function migrationDeclaresDeployPrecondition(raw) { } /** - * The `mode=` a migration header declares, or null when it declares none. - * Prologue-anchored exactly like migrationDeclaresDeployPrecondition, so a token - * in body prose or a data literal cannot answer for the file. + * The `mode=` a migration header declares: 'auto', or 'manual' for anything else, + * because the runners gate every non-auto file and an unscoped operator run applies it. + * Prologue-anchored like migrationDeclaresDeployPrecondition, so a token in body prose + * or a data literal cannot answer for the file. * - * Twin of the modules' own Database._migrationMode, duplicated for the reason - * given above: this tool reads a cloned tree whose dependencies are not - * installed. Keep them in step. + * Twin of Database.prototype.migrationMode in xchain-indexer src/db/database/migration_scan.js + * and xchain-decoder src/db/migration_statements.js, duplicated for the reason given + * above. Keep the scan, the tag regex and the default in step. */ function migrationMode(raw) { const prologue = [] @@ -63,12 +64,12 @@ function migrationMode(raw) { if (trimmed === '' || trimmed.startsWith('--')) { prologue.push(line); continue } break } - const m = prologue.join('\n').match(/^\s*--\s*xchain:migration\b[^\n]*\bmode\s*=\s*([A-Za-z]+)/im) - return m ? m[1].toLowerCase() : null + const m = prologue.join('\n').match(/^\s*--\s*xchain:migration\b[^\n]*\bmode\s*=\s*(auto|manual)\b/im) + return m ? m[1].toLowerCase() : 'manual' } /** - * Every gated (mode=manual) migration in `dir` that the ledger has not recorded, + * Every gated (not mode=auto) migration in `dir` that the ledger has not recorded, * sorted. This is the blast radius of an UNSCOPED migrate run against that * database: the runner applies every pending manual file, not just the one an * operator names. The refusal names that whole set, so the consequence is on diff --git a/src/services/stop_budget_service.js b/src/services/stop_budget_service.js index 66f8154d..a92a2a7a 100644 --- a/src/services/stop_budget_service.js +++ b/src/services/stop_budget_service.js @@ -98,21 +98,47 @@ function shutdownTimeoutMsForBudget(module, seconds) { return Math.max(budgetMs - STOP_DRAIN_MARGIN_MS, Math.floor(budgetMs / 2)) } +// A SHUTDOWN_TIMEOUT_MS the service would honour (parsed as the services parse +// it) that is at or above the stop budget, so docker's kill always lands first. +// Null when the value is under the budget or one the service would ignore. +function overBudgetShutdownMs(value, budgetSeconds) { + if (value === undefined || value === null) return null + const ms = parseInt(value, 10) + return Number.isFinite(ms) && ms > 0 && ms >= budgetSeconds * 1000 ? ms : null +} + // The container env entry that carries the derived drain budget. An explicit // SHUTDOWN_TIMEOUT_MS in the module config wins and nothing is added, and so // does a null module (a one-shot run has no stop budget to derive from). +// Warn when that explicit value is not under the budget (docker kills first). function shutdownTimeoutEnv(module, moduleConfig, env = MODULE_STOP_TIMEOUT_ENV) { if (!module) return {} const configured = moduleConfig ? moduleConfig.SHUTDOWN_TIMEOUT_MS : undefined - if (configured !== undefined && configured !== null && String(configured).trim() !== '') return {} + if (configured !== undefined && configured !== null && String(configured).trim() !== '') { + if (module !== 'node') { + const budget = moduleStopTimeoutSeconds(module, env) + const over = overBudgetShutdownMs(configured, budget) + if (over !== null) logger.warn(overBudgetShutdownLine(module, '', over, budget, 'the module config sets')) + } + return {} + } const derived = moduleShutdownTimeoutMs(module, env) return derived === null ? {} : { SHUTDOWN_TIMEOUT_MS: String(derived) } } +// The operator line for a drain timer at or above its stop budget. +function overBudgetShutdownLine(module, where, ms, budgetSeconds, source) { + return `WARNING: ${module}${where}: ${source} SHUTDOWN_TIMEOUT_MS=${ms}, at or above its ${budgetSeconds} s stop ` + + 'budget, so a drain that runs long is killed by docker before the service\'s own timer can end it. Lower ' + + `SHUTDOWN_TIMEOUT_MS in the module config below ${budgetSeconds * 1000} (or remove it so the node derives it), ` + + `or raise ${moduleStopTimeoutEnvName(module)}, then recreate the container (xchain-node recreate).` +} + // Why a running service's own drain may not follow its current budget: the -// container carries a different stamped budget, or it predates the forwarded -// SHUTDOWN_TIMEOUT_MS while an override is set. Null when nothing drifted, the -// container could not be read, or neither side involves a forwarded drain. +// container carries a different stamped budget, it predates the forwarded +// SHUTDOWN_TIMEOUT_MS while an override is set, or it carries a drain timer at +// or above the budget. Null when nothing drifted, the container could not be +// read, or neither side involves a forwarded drain. function describeStopBudgetDrift(module, coin, network, settings, budgetSeconds) { if (!settings || module === 'node') return null const forwards = shutdownTimeoutMsForBudget(module, budgetSeconds) !== null @@ -125,7 +151,9 @@ function describeStopBudgetDrift(module, coin, network, settings, budgetSeconds) } else if (settings.shutdownTimeoutMs === null && !isDefaultStopBudget(module, budgetSeconds)) { created = 'before the node forwarded SHUTDOWN_TIMEOUT_MS' } else { - return null + // Flag a drain timer docker's kill would beat, before the stop rather than after it + const over = overBudgetShutdownMs(settings.shutdownTimeoutMs, budgetSeconds) + return over === null ? null : overBudgetShutdownLine(module, where, over, budgetSeconds, 'the container carries') } return `WARNING: ${module}${where} was created ${created}, so its own drain timer does not follow ` + `${moduleStopTimeoutEnvName(module)} (now ${budgetSeconds} s). Recreate it (xchain-node recreate) so the ` + diff --git a/src/services/validator_stake_service.js b/src/services/validator_stake_service.js index 9900c8b3..7d296910 100644 --- a/src/services/validator_stake_service.js +++ b/src/services/validator_stake_service.js @@ -35,10 +35,12 @@ const { } = require('./validator_service') const { getCoinConfigByFullName } = require('../coins') const config = require('../config'); +const { readKeyHolders } = require('./validator_stake_service/free_key') const STAKE_TICK = 'XCHAIN' -// One stake that clears every capability floor at once (llm attestation -// provider is the highest at 25000); see the indexer's STAKING.CAPABILITIES. +// One stake that clears every capability floor at once. The highest is the hub's llm attestation +// provider floor (min_stake_xchain in xchain-hub/src/validators/provider_registry/defaults.js), not the +// indexer's STAKING.CAPABILITIES (top 5000); governance can raise it per block, so this is a snapshot. const DEFAULT_STAKE_AMOUNT = 25000 const PUBLIC_EXPLORER = 'https://explorer.xchain.io' @@ -74,9 +76,10 @@ function paren(text) { * * They are separate clocks and they differ by more than two orders of magnitude, * which is exactly why both have to be printed. Both count from the same block, - * the one the action lands in (xchain-indexer/src/actions/unstake.js: - * deactivation_block = BLOCK_INDEX + ACTIVATION_DELAY_BLOCKS, COOLDOWN_END_BLOCK - * = BLOCK_INDEX + COOLDOWN_BLOCKS): + * the one the action lands in (xchain-indexer/src/actions/unstake/index.js: + * deactivation_block = BLOCK_INDEX + ACTIVATION_DELAY_BLOCKS in sweepCapabilityStake, + * COOLDOWN_END_BLOCK = BLOCK_INDEX + COOLDOWN_BLOCKS; the STAKE side's activation + * delay is in xchain-indexer/src/actions/stake/capability_stake.js): * activation: how long a STAKE waits before it counts, and how long an * UNSTAKEd one keeps counting toward every capability before it drops out. * cooldown: how long the escrowed XCHAIN stays locked afterwards, until the @@ -115,6 +118,23 @@ function num(x) { return Number.isFinite(n) ? n : 0 } +// Unwrap an explorer entity read: some deployments answer a one-element array +// of the record rather than the record itself (the SDK's findToken handles both). +function entity(body) { + return Array.isArray(body) ? body[0] : body +} + +// Read the address's native balance, or null when the explorer could not. +// A tracker outage answers confirmed: null with tracker_available: false, and +// Number(null) is 0, so reading it as a number would tell a funded operator to fund it. +function nativeBalance(addr) { + const balances = addr && addr.balances + if (!balances || addr.tracker_available === false) return null + const confirmed = balances.confirmed + if (confirmed === null || confirmed === undefined || !Number.isFinite(Number(confirmed))) return null + return { confirmed: Number(confirmed), pending: num(balances.pending) } +} + // getToken() throws a raw HTTP error when the tick has no token record at // all, which is exactly the state a freshly reset regtest venue is in (a // reset does not reissue XCHAIN). Left uncaught that surfaces as an @@ -136,18 +156,23 @@ async function getStakeToken(sdk) { } // Read everything the plan needs from the explorer. Split out so the -// broadcast path and the tests share one shape. -async function readChainState(sdk, address, pubkey) { - const addr = await sdk.explorer.getAddress(address) - const coinBal = num(addr && addr.balances && addr.balances.confirmed) - const coinPending = num(addr && addr.balances && addr.balances.pending) +// broadcast path and the tests share one shape. `keyCtx` ({ network, coins, +// cooldownBlocks }) is what the free-key rules are judged against. +async function readChainState(sdk, address, pubkey, keyCtx) { + const native = nativeBalance(entity(await sdk.explorer.getAddress(address))) + const coinBal = native ? native.confirmed : null + const coinPending = native ? native.pending : null const bals = await sdk.getBalances(address) const row = ((bals && bals.data) || []).find(b => b && b.tick === STAKE_TICK) const tokenBal = num(row && row.amount) - const token = await getStakeToken(sdk) - const mints = (token && token.mints) || {} + // Read the faucet caps from the token's mints group; a token with no such + // group is an unreadable answer, not a MAX_MINT of 0. + const token = entity(await getStakeToken(sdk)) + const mints = (token && token.mints && typeof token.mints === 'object') ? token.mints : null + const mintUnreadable = mints ? null : 'the ' + STAKE_TICK + ' token read returned an unexpected shape (no mints group), ' + + 'so its faucet caps are unknown' // An existing stake on this pubkey means v1 would be rejected (the indexer // refuses a pubkey that already carries one); say so instead of spending. @@ -157,29 +182,27 @@ async function readChainState(sdk, address, pubkey) { // and calling one that does not exist throws a TypeError that a catch here // would turn into "no existing stake" - a reassuring null that green-lights // a duplicate STAKE. A failed read is reported, never silently treated as - // an answer. - let existing = null - let existingUnknown = null - try { - const v = await sdk.explorer.getValidators() - const rows = (v && v.data) || [] - existing = rows.find(r => r && r.status === 'valid' && - String(r.signing_pubkey || '').toLowerCase() === pubkey) || null - } catch (e) { - existingUnknown = e.message - } + // an answer. The set is read page by page to the end (readValidatorSet), and + // a truncated read throws into the same catch rather than reading as absent. + // readKeyHolders also applies the reuse gate and reads the key's delegations. + const holders = await readKeyHolders(sdk, pubkey, keyCtx) - return { coinBal, coinPending, tokenBal, mintMax: num(mints.max), mintAddressMax: num(mints.address_max), existing, existingUnknown } + return { coinBal, coinPending, tokenBal, mintMax: num(mints && mints.max), + mintAddressMax: num(mints && mints.address_max), mintUnreadable, ...holders } } // Plan the mints: how many transactions, at what amount each, to lift the // balance to `amount`. Pure, so it is testable without a network. -function planMints(network, tokenBal, amount, mintMax, mintAddressMax) { +// `mintUnreadable` names why the caps could not be read, so it is not reported as MAX_MINT 0. +function planMints(network, tokenBal, amount, mintMax, mintAddressMax, mintUnreadable) { const short = Math.max(0, amount - tokenBal) if (short === 0) return { short, mints: [], reason: null } if (network === 'mainnet') { return { short, mints: [], reason: 'XCHAIN is not mintable on mainnet: acquire ' + short + ' more and re-run.' } } + if (mintUnreadable) { + return { short, mints: [], reason: mintUnreadable + '; cannot plan the mints for the shortfall. Retry, or check the explorer.' } + } if (!(mintMax > 0)) { return { short, mints: [], reason: 'the ' + STAKE_TICK + ' token reports no open mint (MAX_MINT is 0); cannot mint the shortfall.' } } @@ -210,8 +233,9 @@ function requireEncoder(sdk) { * * Handing these to the encoder as the ONLY candidate inputs is what lets the * whole run go out at once safely. The indexer resolves a STAKE's balance from - * every ledger entry with a LOWER ACTION INDEX (`m.action_index < ?` in db.js - * getAddressCreditDebit), and that index is global, so the mints count whether + * every ledger entry with a LOWER ACTION INDEX (`m.action_index < ?` in + * xchain-indexer/src/db/misc/index.js getAddressCreditDebit), and that index + * is global, so the mints count whether * they share the STAKE's block or sit in an earlier one. All that matters is * that they come FIRST, which the funding chain guarantees by construction: * consensus requires a parent transaction to precede its child, so a chain @@ -306,5 +330,5 @@ const stakeValidator = createStakeValidator(operationHelpers) const unstakeValidator = createUnstakeValidator(operationHelpers) module.exports = { - stakeValidator, unstakeValidator, planMints, stakeTiming + stakeValidator, unstakeValidator, planMints, stakeTiming, readChainState } diff --git a/src/services/validator_stake_service/free_key.js b/src/services/validator_stake_service/free_key.js new file mode 100644 index 00000000..941ec836 --- /dev/null +++ b/src/services/validator_stake_service/free_key.js @@ -0,0 +1,251 @@ +/********************************************************************* + * + * Copyright © 2025–2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC – https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ********************************************************************** + * XChain Node - Validator Stake Free-Key Check + * + * Whether a STAKE v1 naming this signing pubkey would be admitted, judged the + * way the indexer's validateFreeKey judges it + * (xchain-indexer/src/actions/stake/capability_stake.js), so the CLI refuses + * before any MINT fee is spent rather than after the STAKE is rejected: + * - a stakes row holds the key; once stake_key_reuse_activation is armed, + * a row that is deactivated AND past cooldown no longer does + * (active_stake.js reuseBlockingOnly); + * - a valid delegation holds the key until its deactivation_block has + * passed (delegations getDelegationByPubkey): 'already delegated'. + * The CLI only reads these rules; consensus stays in the indexer. + ********************************************************************/ + +const { readValidatorSet } = require('./validator_set_read') + +// stake_key_reuse_activation heights, a read-only mirror of the indexer's +// gate row (xchain-indexer/src/protocol_changes/gates_3.js). The unit suite +// holds this copy to that row whenever the sibling checkout is present. +const STAKE_KEY_REUSE_ACTIVATION = Object.freeze({ + 'BTC:mainnet': null, + 'LTC:mainnet': null, + 'DOGE:mainnet': null, + mainnet: null, + 'BTC:testnet': 156000, + 'LTC:testnet': 4897000, + 'DOGE:testnet': 67920000, + testnet: null, + regtest: 0 +}) + +// The coin a stake chain's full name stands for, as the indexer keys its gates. +const COIN_BY_FULL_NAME = Object.freeze({ bitcoin: 'BTC', litecoin: 'LTC', dogecoin: 'DOGE' }) + +// The explorer's per-method row cap, and the OFFSET ceiling that bounds a sweep. +const PAGE_LIMIT = 100 +const MAX_PAGES = 1001 + +// XCHAIN amounts carry at most 8 decimals, so sums are exact in 1e-8 units. +const UNIT_DIGITS = 8 + +/** + * The reuse gate's height for this network and coin, resolved as the indexer + * resolves it: ':' first, then the bare network. Null is inert. + */ +function stakeKeyReuseHeight(network, coin) { + const own = (k) => Object.prototype.hasOwnProperty.call(STAKE_KEY_REUSE_ACTIVATION, k) + const key = coin && own(coin + ':' + network) ? coin + ':' + network : network + const height = own(key) ? STAKE_KEY_REUSE_ACTIVATION[key] : null + return Number.isInteger(height) ? height : null +} + +// The coin the indexer names this stake chain by ('bitcoin-testnet' is BTC). +function stakeCoin(coins) { + return COIN_BY_FULL_NAME[String(coins.stake).split('-')[0]] || null +} + +// An empty deactivation_block is a row no UNSTAKE, REVOKE or eviction has touched. +function isUndeactivated(row) { + return row.deactivation_block === null || row.deactivation_block === undefined || row.deactivation_block === '' +} + +// The deactivation block as a number, or null when it is missing or not an integer. +function deactivationBlock(row) { + if (isUndeactivated(row)) return null + const n = Number(row.deactivation_block) + return Number.isInteger(n) ? n : null +} + +/** + * The stakes rows that still hold the key for a STAKE landing at landBlock. + * Before the gate every valid row holds it forever; after it, a row stops + * holding once deactivation_block + cooldownBlocks <= landBlock. A row whose + * deactivation block cannot be read keeps holding, the refusing side. + */ +function stakeRowsHoldingKey(rows, { reuseActive, landBlock, cooldownBlocks }) { + if (!reuseActive) return rows.slice() + return rows.filter(r => { + const d = deactivationBlock(r) + if (isUndeactivated(r) || d === null) return true + return !(d + cooldownBlocks <= landBlock) + }) +} + +/** + * The delegation rows that hold the key for a STAKE landing at landBlock: + * valid, and not yet deactivated at that block. A pending-activation + * delegation holds it too. With no landBlock known, any deactivated row + * still holds, the refusing side. + */ +function delegationRowsHoldingKey(rows, landBlock) { + return rows.filter(r => { + if (!r || r.status !== 'valid') return false + if (isUndeactivated(r)) return true + const d = deactivationBlock(r) + return d === null || !Number.isInteger(landBlock) || d > landBlock + }) +} + +// One decimal amount string as 1e-8 units, or null when it is not a plain decimal. +function toUnits(amount) { + const m = /^(\d+)(?:\.(\d{1,8}))?$/.exec(String(amount).trim()) + if (!m) return null + return BigInt(m[1]) * (10n ** BigInt(UNIT_DIGITS)) + BigInt((m[2] || '').padEnd(UNIT_DIGITS, '0')) +} + +/** + * The exact decimal sum of the rows' amounts, trailing zeros trimmed, or null + * when any amount is not a plain decimal (so nothing false is printed). + */ +function sumAmounts(rows) { + let total = 0n + for (const r of rows) { + const u = toUnits(r && r.amount) + if (u === null) return null + total += u + } + const scale = 10n ** BigInt(UNIT_DIGITS) + const frac = (total % scale).toString().padStart(UNIT_DIGITS, '0').replace(/0+$/, '') + return (total / scale).toString() + (frac ? '.' + frac : '') +} + +// The explorer's indexed tip for the stake chain; throws when it is unreadable. +async function readTip(sdk, coins) { + const status = await sdk.explorer.getStatus() + const tip = Number(status && status.last_block && status.last_block[coins.stake]) + if (!Number.isInteger(tip) || tip < 0) throw new Error('the explorer reported no last block for ' + coins.stake) + return tip +} + +// Memoize readTip so the stake and delegation judgements share one read. +function tipOnce(sdk, coins) { + let pending = null + return () => (pending = pending || readTip(sdk, coins)) +} + +/** + * Every delegation row the explorer holds for this signing pubkey, paged to + * the end. Throws on anything short of a complete answer, including an + * explorer that predates the pubkey lookup, so a failure never reads as + * "not delegated". + */ +async function readDelegationsByPubkey(sdk, pubkey) { + if (!sdk.explorer || typeof sdk.explorer.getDelegations !== 'function') { + throw new Error('this SDK has no getDelegations') + } + const rows = [] + for (let page = 1; ; page++) { + if (page > MAX_PAGES) throw new Error('delegation read incomplete: more than ' + (MAX_PAGES * PAGE_LIMIT) + ' rows') + const v = await sdk.explorer.getDelegations(pubkey, 'pubkey', { page, limit: PAGE_LIMIT, sortorder: 'ASC' }) + if (!v || !Array.isArray(v.data)) throw new Error('the delegation lookup by pubkey returned no row list') + rows.push(...v.data) + const total = Number(v.total) + const known = v.total !== undefined && v.total !== null && Number.isInteger(total) && total >= 0 + if (known ? rows.length >= total : v.data.length < PAGE_LIMIT) break + if (!v.data.length) throw new Error('delegation read incomplete: saw ' + rows.length + ' of ' + total + ' rows') + } + return rows.filter(r => r && String(r.signing_pubkey || '').toLowerCase() === pubkey) +} + +/** + * Judge this pubkey's stakes rows: which still hold it, and, when the key is + * held only by withdrawn stake, why. `landBlock` is the explorer tip + 1, the + * earliest block a STAKE sent now can land in. + */ +async function judgeStakeRows(rows, { network, coins, cooldownBlocks, tip }) { + const reuseHeight = stakeKeyReuseHeight(network, stakeCoin(coins)) + const judged = { reuseHeight, reuseActive: false, tipBlock: null, tipError: null } + // The gate can only free a key whose every row is withdrawn, so only then is the tip worth reading. + if (reuseHeight !== null && rows.every(r => !isUndeactivated(r))) { + try { + judged.tipBlock = await tip() + judged.reuseActive = judged.tipBlock + 1 >= reuseHeight + } catch (e) { + judged.tipError = e.message + } + } + const landBlock = judged.tipBlock === null ? null : judged.tipBlock + 1 + judged.holding = stakeRowsHoldingKey(rows, { reuseActive: judged.reuseActive, landBlock, cooldownBlocks }) + // The undeactivated rows are the stake UNSTAKE v0 would sweep, so their sum is the one to print. + const live = rows.filter(isUndeactivated) + judged.liveCount = live.length + judged.liveAmount = live.length ? sumAmounts(live) : null + const ends = judged.holding.map(deactivationBlock).filter(d => d !== null) + judged.freeFromBlock = judged.reuseActive && ends.length === judged.holding.length && ends.length + ? Math.max(...ends) + cooldownBlocks + : null + return judged +} + +/** + * Read everything that decides whether this pubkey is free for a STAKE v1. + * Returns { existing, stakeJudgement, existingUnknown, delegated, + * delegationUnknown }: `existing` is the newest stakes row still holding the + * key, `delegated` the delegation rows holding it; each *Unknown is the + * message of a read that failed, never a silent "free". + */ +async function readKeyHolders(sdk, pubkey, { network, coins, cooldownBlocks }) { + const tip = tipOnce(sdk, coins) + const out = { existing: null, stakeJudgement: null, existingUnknown: null, delegated: null, delegationUnknown: null } + try { + const v = await readValidatorSet(sdk) + const rows = ((v && v.data) || []).filter(r => r && r.status === 'valid' && + String(r.signing_pubkey || '').toLowerCase() === pubkey) + if (rows.length) { + const judged = await judgeStakeRows(rows, { network, coins, cooldownBlocks, tip }) + if (judged.holding.length) { + out.stakeJudgement = judged + out.existing = judged.holding.reduce((a, b) => (Number(b.action_index) > Number(a.action_index) ? b : a)) + } + } + } catch (e) { + out.existingUnknown = e.message + } + if (out.existing) return out + try { + const rows = await readDelegationsByPubkey(sdk, pubkey) + let landBlock = null + if (rows.some(r => r && r.status === 'valid' && !isUndeactivated(r))) { + try { landBlock = (await tip()) + 1 } catch { landBlock = null } + } + const holding = delegationRowsHoldingKey(rows, landBlock) + if (holding.length) out.delegated = holding + } catch (e) { + out.delegationUnknown = e.message + } + return out +} + +module.exports = { + STAKE_KEY_REUSE_ACTIVATION, + stakeKeyReuseHeight, + stakeRowsHoldingKey, + delegationRowsHoldingKey, + sumAmounts, + readDelegationsByPubkey, + readKeyHolders +} diff --git a/src/services/validator_stake_service/send_lock.js b/src/services/validator_stake_service/send_lock.js new file mode 100644 index 00000000..6afcf048 --- /dev/null +++ b/src/services/validator_stake_service/send_lock.js @@ -0,0 +1,47 @@ +/********************************************************************* + * + * Copyright © 2025–2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC – https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ********************************************************************** + * XChain Node - Validator Stake Send Lock + * Holds the command lock across a broadcast's send phase only + ********************************************************************/ + +// Stand in for the lock when the caller passed none, or the run is a dry run. +const NO_LOCK = Object.freeze({ hold() {}, release() {} }) + +// Pick the lock a run serializes its sends on. Only --broadcast sends anything, +// so a dry run never takes it and can inspect the plan during a deploy. +function sendLockFor(opts, deps) { + return (opts && opts.broadcast && deps && deps.sendLock) ? deps.sendLock : NO_LOCK +} + +// Hand the lock back when the SDK starts its indexer wait, which follows every +// broadcast of the action. A wait of up to --timeout must not block other commands. +function releaseOnIndexWait(lock) { + return (step) => { if (step === 'waiting') lock.release() } +} + +// Take the lock before the next send, naming what already went out when refused. +// A refusal mid-run leaves the earlier transactions broadcast, and the operator re-runs. +function holdForSend(lock, sent, kind) { + try { + lock.hold() + } catch (err) { + if (!sent.length) throw err + const stopped = new Error(err.message + ' Stopped before the ' + kind + '; already broadcast: ' + + sent.map(s => s.step + ' ' + s.txid).join(', ') + '. Re-run this command once that one finishes.') + stopped.code = err.code + throw stopped + } +} + +module.exports = { NO_LOCK, sendLockFor, releaseOnIndexWait, holdForSend } diff --git a/src/services/validator_stake_service/stake_operations.js b/src/services/validator_stake_service/stake_operations.js index d71deb2c..3d3e39a6 100644 --- a/src/services/validator_stake_service/stake_operations.js +++ b/src/services/validator_stake_service/stake_operations.js @@ -15,6 +15,7 @@ ********************************************************************/ const { getLogger } = require('../../observability/logger') +const { sendLockFor, releaseOnIndexWait, holdForSend } = require('./send_lock') function defaultLog() { const logger = getLogger() @@ -26,8 +27,11 @@ function logStakeBalances(log, network, coins, pubkey, address, amount, state, p log('Validator stake plan (' + network + ')') log(' signing pubkey : ' + pubkey) log(' stake address : ' + address) - log(' ' + coins.stakeCoin.padEnd(15) + ': ' + state.coinBal + ' confirmed' + - (state.coinPending ? ' (+' + state.coinPending + ' pending)' : '') + ' (pays the transaction fees)') + // Print an unreadable native balance as unavailable, never as a number. + const coinLine = state.coinBal === null + ? 'unavailable (the explorer could not read it from its UTXO tracker)' + : state.coinBal + ' confirmed' + (state.coinPending ? ' (+' + state.coinPending + ' pending)' : '') + log(' ' + coins.stakeCoin.padEnd(15) + ': ' + coinLine + ' (pays the transaction fees)') log(' ' + STAKE_TICK.padEnd(15) + ': ' + state.tokenBal + ' held, ' + amount + ' to stake' + (plan.short ? ', short ' + plan.short : '')) if (state.mintMax) log(' faucet caps : ' + state.mintMax + ' per MINT, ' + (state.mintAddressMax || 'no') + ' per address') @@ -56,30 +60,96 @@ function logStakeSteps(log, amount, pubkey, plan, timing, STAKE_TICK, paren) { function stakeBlockers(state, plan, coins, address) { const blockers = [] if (plan.reason) blockers.push(plan.reason) - if (state.coinBal <= 0) blockers.push('no confirmed ' + coins.stakeCoin + ' at ' + address + ' to pay fees; fund it first.') + // Refuse when the fee balance is unreadable, and say it is an outage rather than an empty address. + if (state.coinBal === null) { + blockers.push('the ' + coins.stakeCoin + ' balance at ' + address + ' is unavailable (the explorer\'s UTXO ' + + 'tracker did not answer), so this run cannot confirm it can pay fees. Retry, or check the explorer; ' + + 'this does not mean the address is unfunded.') + } else if (state.coinBal <= 0) { + blockers.push('no confirmed ' + coins.stakeCoin + ' at ' + address + ' to pay fees; fund it first.') + } + // Refuse when the delegation check could not run: the indexer rejects a STAKE on a + // delegated key only after every MINT fee is spent, so an unknown is not a pass. + if (state.delegationUnknown) { + blockers.push('could not confirm this signing pubkey is not delegated (' + state.delegationUnknown + '). ' + + 'An explorer that predates the /delegations//pubkey lookup cannot answer it, and ' + + 'nothing is sent until one can.') + } return blockers } -async function sendStakeAction(context, progress, kind, params, isLast) { - const { sdk, session, address, opts, log, timeoutMs, baseEncoder, chainedInputs } = context - const enc = Object.assign({}, baseEncoder) - if (progress.chain && progress.prevTxid) { - const inputs = await chainedInputs(sdk, address, progress.prevTxid, opts.chainTimeoutMs) - if (inputs) enc.utxos = inputs - else { - // Nothing spendable came back from the previous transaction, so - // the ordering guarantee is gone. Say so rather than broadcast a - // STAKE that a miner may place ahead of its own funding. - progress.chainBroken = true - log(' (no spendable output from ' + progress.prevTxid.slice(0, 16) + '..., cannot chain)') +// Why a key held only by withdrawn stake cannot be staked yet, in the operator's terms. +function heldKeyReason(judged, timing) { + if (judged.tipError) { + return 'could not read the chain tip (' + judged.tipError + '), so this run cannot tell whether the key has been released.' + } + if (judged.reuseHeight === null) { + return 'this network does not yet release a signing key that has staked; stake under a new signing key.' + } + if (!judged.reuseActive) { + return 'this network releases a withdrawn key only from block ' + judged.reuseHeight + + ' (the explorer is at block ' + judged.tipBlock + '); re-run then, or stake under a new signing key.' + } + if (judged.freeFromBlock === null) { + return 'a stake row for it has no readable deactivation block, so this run cannot tell when the key is released.' + } + return 'its ' + timing.cooldownBlocks + '-block cooldown still holds the key; a new STAKE on it is admitted from block ' + + judged.freeFromBlock + ' (the explorer is at block ' + judged.tipBlock + '). Re-run then.' +} + +// Say why the pubkey is not free for a STAKE v1, naming the stake or delegation holding it. +function logKeyHeld(log, state, ctx) { + const { coins, pubkey, timing, explorerUrl, STAKE_TICK } = ctx + log('') + if (state.existing) { + const judged = state.stakeJudgement + if (judged.liveCount > 0) { + const amount = judged.liveAmount === null ? '' : ' of ' + judged.liveAmount + ' ' + STAKE_TICK + log(' This pubkey already carries a valid STAKE' + amount + + ' (newest action ' + state.existing.action_index + ', activates at block ' + state.existing.activation_block + ').') + log(' Nothing to do. Check it at ' + explorerUrl(coins, 'validator/' + pubkey)) + } else { + log(' This pubkey\'s stake is withdrawn (an UNSTAKE or a ROLLCALL eviction), but the key is not free') + log(' for a new STAKE: ' + heldKeyReason(judged, timing) + ' Nothing was sent.') } + } else { + const d = state.delegated[state.delegated.length - 1] + log(' This pubkey is held by a delegation (action ' + d.action_index + ' from ' + d.source + '), so the indexer') + log(' would reject a STAKE on it as already delegated. Revoke the delegation, or stake under a') + log(' new signing key. Nothing was sent.') } + log('') +} + +// The previous transaction's outputs for the next action to spend, or null +// when there is no chain to extend or the link could not be formed. +async function linkInputs(context, progress) { + const { sdk, address, opts, log, chainedInputs } = context + if (!(progress.chain && progress.prevTxid)) return null + const inputs = await chainedInputs(sdk, address, progress.prevTxid, opts.chainTimeoutMs) + if (!inputs) { + // Nothing spendable came back from the previous transaction, so the + // ordering guarantee is gone. Say so; broadcastStake then waits the + // mints out rather than send a STAKE a miner may place ahead of them. + progress.chainBroken = true + log(' (no spendable output from ' + progress.prevTxid.slice(0, 16) + '..., cannot chain)') + } + return inputs +} + +// Submit one action, funded from `inputs` when the funding chain supplied them. +async function sendStakeAction(context, progress, kind, params, isLast, inputs) { + const { session, opts, log, timeoutMs, baseEncoder, sendLock } = context + const enc = Object.assign({}, baseEncoder) + if (inputs) enc.utxos = inputs // Only the final action waits for the indexer: the ones before it are // ordered by the funding chain, so waiting on them buys nothing but a // block of latency. const wait = isLast && opts.wait !== false + // Hold the command lock while this action is sent; the SDK's indexer wait hands it back. + holdForSend(sendLock, progress.sent, kind) const r = await session.submit({ action: kind, params }, enc, - { waitForIndexer: wait, timeout: timeoutMs, pollInterval: 15000 }) + { waitForIndexer: wait, timeout: timeoutMs, pollInterval: 15000, onProgress: releaseOnIndexWait(sendLock) }) if (progress.chain && progress.prevTxid && !(r.spentInputs || []).some(i => i.txid === progress.prevTxid)) { progress.chainBroken = true } @@ -109,10 +179,16 @@ async function broadcastStake(context) { for (let i = 0; i < plan.mints.length; i++) { log('') log(' Sending MINT ' + (i + 1) + '/' + plan.mints.length + ' (' + plan.mints[i] + ' ' + STAKE_TICK + ')...') + const inputs = await linkInputs(context, progress) await sendStakeAction(context, progress, 'MINT', - { VERSION: 0, TICK: STAKE_TICK, AMOUNT: String(plan.mints[i]) }, opts.serialize === true) + { VERSION: 0, TICK: STAKE_TICK, AMOUNT: String(plan.mints[i]) }, opts.serialize === true, inputs) } + // Form the STAKE's own link to the last mint BEFORE deciding whether to + // wait: a miss on this final hop breaks the ordering exactly as a miss + // between mints does, and must reach the same wait below. + const stakeInputs = progress.chainBroken ? null : await linkInputs(context, progress) + // A broken chain means in-block order is the miner's choice, and a STAKE // evaluated before its own mints is rejected for insufficient funds. Wait // the mints out instead: once they are indexed, ordering stops mattering. @@ -120,6 +196,8 @@ async function broadcastStake(context) { log('') log(' Waiting for the mints to be indexed before staking' + (progress.chainBroken ? ' (the funding chain broke, so in-block order is not guaranteed)' : '') + '...') + // Hand the lock back for the wait; the STAKE takes it again before it is sent. + context.sendLock.release() const ok = await waitForBalance(sdk, address, amount, context.timeoutMs, log, opts.balancePollMs) if (!ok) { log('') @@ -129,17 +207,18 @@ async function broadcastStake(context) { return { staked: false, sent: progress.sent, pendingMints: true } } progress.prevTxid = null // fund the STAKE freely; ordering no longer matters + return finishStake(context, progress, null) } - return finishStake(context, progress) + return finishStake(context, progress, stakeInputs) } -async function finishStake(context, progress) { +async function finishStake(context, progress, stakeInputs) { const { opts, coins, pubkey, amount, log, timing, explorerUrl, STAKE_TICK, paren } = context log('') log(' Sending STAKE v1 (' + amount + ' ' + STAKE_TICK + ' to ' + pubkey + ')...') const r = await sendStakeAction(context, progress, 'STAKE', - { VERSION: 1, AMOUNT: String(amount), SIGNING_PUBKEY: pubkey }, true) + { VERSION: 1, AMOUNT: String(amount), SIGNING_PUBKEY: pubkey }, true, stakeInputs) log('') if (opts.wait === false) { log(' Broadcast. Watch it land at ' + explorerUrl(coins, 'validator/' + pubkey)) @@ -154,37 +233,60 @@ async function finishStake(context, progress) { return { staked: true, sent: progress.sent, txid: r.txid, chained: progress.chain && !progress.chainBroken } } +// Warn that the validator set could not be read, so an existing stake is unknown. +function logSetUnreadable(log, reason, validatorUrl) { + log('') + log(' WARNING: could not read the validator set (' + reason + '),') + log(' so this run cannot tell whether the pubkey is already staked. Check') + log(' ' + validatorUrl + ' before broadcasting.') +} + +// Say that a dry run sent nothing and how --broadcast would send the steps. +function logDryRun(log, stepCount) { + log('') + log(' Dry run: nothing sent. Re-run with --broadcast to send the ' + stepCount + ' transaction(s) above.') + log(' They go out back to back, each funded by the one before it, so the run confirms in the') + log(' next block or two rather than costing a block per step. (--serialize sends them a block') + log(' apart instead.)') + log('') +} + function createStakeValidator(helpers) { const { openValidatorSession, stakeTiming, readChainState, planMints, explorerUrl, chainedInputs, waitForBalance, paren, STAKE_TICK, DEFAULT_STAKE_AMOUNT } = helpers /** * Run the stake command. `deps` lets tests inject an SDK factory and a - * logger; production uses the real SDK and service logger. + * logger; production uses the real SDK and service logger. `deps.sendLock` + * ({ hold, release }) serializes a --broadcast run's sends on the command lock. */ return async function stakeValidator(opts = {}, deps = {}) { + const sendLock = sendLockFor(opts, deps) + try { + return await planAndStake(opts, deps, sendLock) + } finally { + sendLock.release() + } + } + + async function planAndStake(opts, deps, sendLock) { const log = deps.log || defaultLog() const { network, coins, pubkey, sdk, session, address } = openValidatorSession(opts, deps) const amount = parseInt(opts.amount) || DEFAULT_STAKE_AMOUNT const timing = stakeTiming(coins, network) - const state = await readChainState(sdk, address, pubkey) - const plan = planMints(network, state.tokenBal, amount, state.mintMax, state.mintAddressMax) + // Take the lock before reading the plan, so no other broadcast sends between this read and these sends. + sendLock.hold() + const state = await readChainState(sdk, address, pubkey, { network, coins, cooldownBlocks: timing.cooldownBlocks }) + const plan = planMints(network, state.tokenBal, amount, state.mintMax, state.mintAddressMax, state.mintUnreadable) logStakeBalances(log, network, coins, pubkey, address, amount, state, plan, STAKE_TICK) - if (state.existing) { - log('') - log(' This pubkey already carries a valid STAKE of ' + state.existing.amount + - ' (action ' + state.existing.action_index + ', activates at block ' + state.existing.activation_block + ').') - log(' Nothing to do. Check it at ' + explorerUrl(coins, 'validator/' + pubkey)) - log('') - return { staked: false, existing: state.existing } - } - if (state.existingUnknown) { - log('') - log(' WARNING: could not read the validator set (' + state.existingUnknown + '),') - log(' so this run cannot tell whether the pubkey is already staked. Check') - log(' ' + explorerUrl(coins, 'validator/' + pubkey) + ' before broadcasting.') + if (state.existing || state.delegated) { + logKeyHeld(log, state, { coins, pubkey, timing, explorerUrl, STAKE_TICK }) + return state.existing + ? { staked: false, existing: state.existing } + : { staked: false, delegated: state.delegated } } + if (state.existingUnknown) logSetUnreadable(log, state.existingUnknown, explorerUrl(coins, 'validator/' + pubkey)) const steps = logStakeSteps(log, amount, pubkey, plan, timing, STAKE_TICK, paren) const blockers = stakeBlockers(state, plan, coins, address) @@ -195,18 +297,13 @@ function createStakeValidator(helpers) { return { staked: false, plan, blockers } } if (!opts.broadcast) { - log('') - log(' Dry run: nothing sent. Re-run with --broadcast to send the ' + steps.length + ' transaction(s) above.') - log(' They go out back to back, each funded by the one before it, so the run confirms in the') - log(' next block or two rather than costing a block per step. (--serialize sends them a block') - log(' apart instead.)') - log('') + logDryRun(log, steps.length) return { staked: false, plan, dryRun: true } } return broadcastStake({ opts, sdk, session, address, coins, pubkey, amount, plan, log, timing, - explorerUrl, chainedInputs, waitForBalance, paren, STAKE_TICK }) + explorerUrl, chainedInputs, waitForBalance, paren, STAKE_TICK, sendLock }) } } -module.exports = { createStakeValidator } +module.exports = { createStakeValidator, stakeBlockers, logStakeBalances } diff --git a/src/services/validator_stake_service/unstake_operations.js b/src/services/validator_stake_service/unstake_operations.js index 24a06020..7d0445f4 100644 --- a/src/services/validator_stake_service/unstake_operations.js +++ b/src/services/validator_stake_service/unstake_operations.js @@ -15,6 +15,10 @@ ********************************************************************/ const { getLogger } = require('../../observability/logger') +// Reads /validators to the end: one unpaged read drops every stake past the newest 100 rows. +const { readValidatorSet } = require('./validator_set_read') +const { sumAmounts } = require('./free_key') +const { sendLockFor, releaseOnIndexWait } = require('./send_lock') function defaultLog() { const logger = getLogger() @@ -61,6 +65,104 @@ function logUnstakeSuccess(log, active, timing, coins, pubkey, STAKE_TICK, paren log('') } +// An empty deactivation_block is a stake no UNSTAKE or ROLLCALL eviction has touched. +function isUndeactivated(row) { + return row.deactivation_block === null || row.deactivation_block === undefined || row.deactivation_block === '' +} + +// Sort rows as the indexer admits an UNSTAKE landing at landBlock: undeactivated AND activation_block <= landBlock. +// The explorer keeps status='valid' on deactivated rows, so status alone never means "active". +function classifyUnstakeRows(rows, landBlock) { + const live = rows.filter(isUndeactivated) + return { + deactivated: rows.filter(r => !isUndeactivated(r)), + admissible: live.filter(r => Number(r.activation_block) <= landBlock), + pending: live.filter(r => !(Number(r.activation_block) <= landBlock)) + } +} + +// The stake being withdrawn: the one admissible row as-is, or several summed, since the indexer withdraws them all. +function withdrawnStake(admissible) { + if (admissible.length === 1) return admissible[0] + // Sum in exact 1e-8 units; a float sum drifts and prints tiny totals as '1e-8'. + const exact = sumAmounts(admissible) + const total = admissible.reduce((s, r) => s + Number(r.amount), 0) + return { + amount: exact !== null ? exact : String(Math.round(total * 1e8) / 1e8), + action_index: admissible.map(r => r.action_index).join(', '), + activation_block: String(Math.max(...admissible.map(r => Number(r.activation_block)))) + } +} + +// The explorer's indexed tip for the stake coin; unreadable means refuse, never guess. +async function readTip(sdk, coins, fail) { + let tip + try { + const status = await sdk.explorer.getStatus() + tip = Number(status && status.last_block && status.last_block[coins.stake]) + } catch (e) { + throw fail('could not read the chain tip (' + e.message + '), so this run cannot tell ' + + 'whether the stake is active yet. Nothing was sent.') + } + if (!Number.isInteger(tip) || tip < 0) { + throw fail('the explorer reported no last block for ' + coins.stake + ', so this run cannot tell ' + + 'whether the stake is active yet. Nothing was sent.') + } + return tip +} + +/** + * Decide what an UNSTAKE would withdraw, before anything is sent. Returns + * { active } when the indexer would admit one, or { done } with the result to + * return when there is nothing to withdraw; throws when it would reject. + */ +async function resolveUnstakeTarget({ sdk, coins, pubkey, address, timing, log, fail, paren, STAKE_TICK }) { + // Read the set rather than a per-pubkey lookup (see readChainState): the + // SDK exposes no getValidator, and a lookup failure must not read as + // "nothing staked" when that is the very thing being acted on. + let rows = [] + try { + const v = await readValidatorSet(sdk) + rows = ((v && v.data) || []).filter(r => r && r.status === 'valid' && + String(r.signing_pubkey || '').toLowerCase() === pubkey) + } catch (e) { + throw fail('could not read the validator set (' + e.message + '), so this run cannot tell ' + + 'whether there is a stake to withdraw. Nothing was sent.') + } + + if (!rows.length) { + logUnstakePlan(log, pubkey, address, null, timing, STAKE_TICK, paren) + log('') + log(' This pubkey carries no valid stake. Nothing to withdraw.') + log('') + return { done: { unstaked: false, nothingStaked: true } } + } + + // Stop when every row is already deactivated: the indexer rejects a second UNSTAKE. + if (!rows.some(isUndeactivated)) { + const deactivationBlock = Math.max(...rows.map(r => Number(r.deactivation_block))) + const cooldownEndBlock = deactivationBlock - timing.activationBlocks + timing.cooldownBlocks + logUnstakePlan(log, pubkey, address, null, timing, STAKE_TICK, paren) + log('') + log(' This stake is already withdrawn (an earlier UNSTAKE, or a ROLLCALL eviction).') + log(' It drops out of the active set at block ' + deactivationBlock + + ' and its cooldown ends at block ' + cooldownEndBlock + '. Nothing was sent.') + log('') + return { done: { unstaked: false, alreadyUnstaking: true, deactivationBlock, cooldownEndBlock } } + } + + // The UNSTAKE lands no earlier than the next block, so that is the block admission is judged at. + const tip = await readTip(sdk, coins, fail) + const { admissible, pending } = classifyUnstakeRows(rows, tip + 1) + if (!admissible.length) { + const from = Math.min(...pending.map(r => Number(r.activation_block))) + throw fail('this stake is not active yet: an UNSTAKE can land from block ' + + (Number.isFinite(from) ? from : '(unknown)') + + ' (the explorer is at block ' + tip + '). Re-run then. Nothing was sent.') + } + return { active: withdrawnStake(admissible) } +} + function createUnstakeValidator({ openValidatorSession, stakeTiming, fail, paren, explorerUrl, STAKE_TICK }) { @@ -73,32 +175,30 @@ function createUnstakeValidator({ * raises the federation's quorum threshold (CapabilitySnapshot.getQuorum) and * puts a hub that cannot answer into publisher elections. Standing down is how * an operator stops being that. + * + * `deps.sendLock` ({ hold, release }) serializes a --broadcast run's send on the command lock. */ return async function unstakeValidator(opts = {}, deps = {}) { + const sendLock = sendLockFor(opts, deps) + try { + return await planAndUnstake(opts, deps, sendLock) + } finally { + sendLock.release() + } + } + + async function planAndUnstake(opts, deps, sendLock) { const log = deps.log || defaultLog() const { network, coins, pubkey, sdk, session, address } = openValidatorSession(opts, deps) const timing = stakeTiming(coins, network) - // Read the set rather than a per-pubkey lookup (see readChainState): the - // SDK exposes no getValidator, and a lookup failure must not read as - // "nothing staked" when that is the very thing being acted on. - let active = null - try { - const v = await sdk.explorer.getValidators() - active = ((v && v.data) || []).find(r => r && r.status === 'valid' && - String(r.signing_pubkey || '').toLowerCase() === pubkey) || null - } catch (e) { - throw fail('could not read the validator set (' + e.message + '), so this run cannot tell ' + - 'whether there is a stake to withdraw. Nothing was sent.') - } + // Take the lock before reading the stake, so no other broadcast sends between this read and this send. + sendLock.hold() + const target = await resolveUnstakeTarget({ sdk, coins, pubkey, address, timing, log, fail, paren, STAKE_TICK }) + if (target.done) return target.done + const active = target.active logUnstakePlan(log, pubkey, address, active, timing, STAKE_TICK, paren) - if (!active) { - log('') - log(' This pubkey carries no valid stake. Nothing to withdraw.') - log('') - return { unstaked: false, nothingStaked: true } - } if (!opts.broadcast) { log('') log(' Dry run: nothing sent. Re-run with --broadcast to withdraw.') @@ -113,12 +213,14 @@ function createUnstakeValidator({ log('') log(' Sending UNSTAKE v0...') + // Hand the command lock back once the UNSTAKE is broadcast and the SDK starts its indexer wait. const r = await session.submit({ action: 'UNSTAKE', params: { VERSION: 0, SIGNING_PUBKEY: pubkey } }, enc, - { waitForIndexer: opts.wait !== false, timeout: timeoutMs, pollInterval: 15000 }) + { waitForIndexer: opts.wait !== false, timeout: timeoutMs, pollInterval: 15000, + onProgress: releaseOnIndexWait(sendLock) }) log(' txid ' + r.txid + (opts.wait !== false ? ' (indexed)' : ' (broadcast)')) logUnstakeSuccess(log, active, timing, coins, pubkey, STAKE_TICK, paren, explorerUrl) return { unstaked: true, txid: r.txid } } } -module.exports = { createUnstakeValidator } +module.exports = { createUnstakeValidator, classifyUnstakeRows } diff --git a/src/services/validator_stake_service/validator_set_read.js b/src/services/validator_stake_service/validator_set_read.js new file mode 100644 index 00000000..63efa2bc --- /dev/null +++ b/src/services/validator_stake_service/validator_set_read.js @@ -0,0 +1,65 @@ +/********************************************************************* + * + * Copyright © 2025–2026 Dankest, LLC + * Based on XChain Platform by Dankest, LLC – https://dankest.llc + * + * SPDX-License-Identifier: AGPL-3.0-or-later + * + * This file is part of XChain Platform. Licensed under the GNU Affero + * General Public License v3.0 or later; see LICENSE.md. A commercial + * license (without AGPL source-disclosure terms) is available - + * contact legal@dankest.llc. + * + ********************************************************************** + * XChain Node - Validator Set Read + * + * The whole /validators set, read page by page. The explorer serves at most + * 100 rows a page, newest first, and counts every valid stakes row (top-ups + * and partial-unstake residuals included), so one unpaged read silently drops + * every older validator once the set passes 100 rows, and a stake that is on + * chain reads as "nothing staked". + ********************************************************************/ + +// The explorer's per-method row cap for /validators. +const PAGE_LIMIT = 100 +// The explorer caps the API OFFSET at 100000, so page 1002 would repeat page 1001. +const MAX_PAGES = 1001 + +function incomplete(msg) { + return new Error('validator set read incomplete: ' + msg) +} + +// A usable `total`: a finite, non-negative integer, or null when the response has none. +function readTotal(v) { + if (!v || v.total === undefined || v.total === null || v.total === '') return null + const t = Number(v.total) + return Number.isInteger(t) && t >= 0 ? t : null +} + +/** + * Every /validators row, oldest first (action_index ASC), so a stake landing mid-sweep appends at the + * end instead of shifting rows across a page; any partial read throws, so it never reads as "not staked". + */ +async function readValidatorSet(sdk) { + const rows = [] + let total = null + for (let page = 1; ; page++) { + if (page > MAX_PAGES) throw incomplete('more than ' + (MAX_PAGES * PAGE_LIMIT) + ' rows') + const v = await sdk.explorer.getValidators({ page, limit: PAGE_LIMIT, sortorder: 'ASC' }) + if (!v || !Array.isArray(v.data)) throw incomplete('page ' + page + ' returned no row list') + const pageTotal = readTotal(v) + // A shrinking total means rows left the set mid-sweep, so offsets shifted under us. + if (pageTotal !== null && total !== null && pageTotal < total) { + throw incomplete('the set shrank from ' + total + ' to ' + pageTotal + ' rows mid-read') + } + if (pageTotal !== null) total = pageTotal + rows.push(...v.data) + // Done when every counted row is in hand, or, with no total, on a short page. + if (total !== null ? rows.length >= total : v.data.length < PAGE_LIMIT) break + // An empty page before the total is reached is a truncated read, not the end. + if (!v.data.length) throw incomplete('saw ' + rows.length + ' of ' + total + ' rows') + } + return { total: rows.length, data: rows } +} + +module.exports = { readValidatorSet } diff --git a/src/utils/migration_files.js b/src/utils/migration_files.js index dde47b23..65ee04e2 100644 --- a/src/utils/migration_files.js +++ b/src/utils/migration_files.js @@ -13,11 +13,14 @@ ********************************************************************** * Where a cloned module tree keeps its tracked migrations, and what they are. * - * The indexer keeps them under src/db/sql/migrations/, grouped into bucket - * directories of at most twenty files. The decoder, and every indexer ref cut - * before that move, keep them flat under src/sql/migrations/. One xchain-node - * release has to deploy refs from both sides of the move, so the cloned tree - * answers for itself rather than this tool assuming one layout. + * The indexer and the decoder both keep them flat under src/sql/migrations/, and + * the indexer's three readers (src/db/database/migration_runner.js, + * src/db/database/schema_setup.js and src/db/migration/migrate.js) list only that + * directory's top-level .sql files, so a subdirectory is invisible to them. The + * src/db/sql/migrations/ preference and the one-bucket-down walk below are + * forward-compat only: no indexer ref ships that layout. If the indexer ever moves + * or buckets its migrations, all three readers must change in the same release, + * or this listing and the set its runner applies silently diverge. * * A schema_migrations ledger row is keyed by the migration's basename alone, * so the listing below returns basenames and keeps the path beside each one; @@ -30,9 +33,10 @@ const fs = require('fs') const path = require('path') /** - * The migrations directory of a module checkout at `root`: the moved home when - * the tree carries it, otherwise the flat one. The flat path is returned even - * when it does not exist, so a caller's missing-directory handling still runs. + * The migrations directory of a module checkout at `root`: a future moved home + * when the tree carries it, otherwise the flat one every ref uses today. The flat + * path is returned even when it does not exist, so a caller's missing-directory + * handling still runs. */ function migrationsDirOf(root) { const moved = path.join(root, 'src', 'db', 'sql', 'migrations') diff --git a/test/helpers/stake_harness.js b/test/helpers/stake_harness.js new file mode 100644 index 00000000..87d493ff --- /dev/null +++ b/test/helpers/stake_harness.js @@ -0,0 +1,120 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +// Shared fake SDK and runner for the stakeValidator() test files. + +const sinon = require('sinon') + +const { stakeValidator } = require('../../src/services/validator_stake_service') + +const PUBKEY = 'ab'.repeat(32) +const ADDRESS = 'mStakeAddress' + +function makeExplorer(chain) { + return { + getAddress: sinon.stub().resolves({ balances: { confirmed: chain.coin ?? '0.001', pending: '0' } }), + getToken: chain.tokenMissing + ? sinon.stub().rejects(Object.assign( + new Error('Explorer returned HTTP 404 for /RBTC/api/token/XCHAIN'), + { code: 'EXPLORER_HTTP_404', details: { status: 404 } })) + : sinon.stub().resolves({ mints: { max: chain.mintMax ?? 10000, address_max: chain.addressMax ?? 50000 } }), + // The whole validator set, which is the method the SDK actually has. + // Carries an unrelated validator too, so the pubkey filter is exercised + // rather than "the only row wins". + getValidators: chain.validatorsThrow + ? sinon.stub().rejects(new Error(chain.validatorsThrow)) + : sinon.stub().resolves({ data: [ + { status: 'valid', signing_pubkey: 'ff'.repeat(32), amount: '25000', action_index: '1', activation_block: '1' }, + ...(chain.existing ? [chain.existing] : []), + ...(chain.stakeRows || []) + ] }), + // The delegations holding a pubkey, as the explorer's pubkey lane answers them. + getDelegations: chain.delegationsThrow + ? sinon.stub().rejects(new Error(chain.delegationsThrow)) + : sinon.stub().resolves(chain.delegationsBody || { + total: (chain.delegations || []).length, data: chain.delegations || [] }), + // The explorer's indexed tip, under every stake chain. + getStatus: sinon.stub().resolves({ last_block: { + 'bitcoin-mainnet': chain.tip ?? 200000, 'bitcoin-testnet': chain.tip ?? 200000, 'bitcoin-regtest': chain.tip ?? 200000 } }) + } +} + +function trackPreviousOutput(session, prevTxidRef) { + return () => { + const s = session() + const inner = s.submit + s.submit = async (a, e, o) => { const r = await inner(a, e, o); prevTxidRef.value = r.txid; return r } + return s + } +} + +// A fake SDK shaped like the parts the command touches: explorer reads, +// balances, and a session whose mint/stake record what they were asked. +function makeSdk(chain = {}) { + const calls = { mint: [], stake: [] } + const sdk = { + explorer: makeExplorer(chain), + // The mints credit once they index. Modelled by reporting the post-mint + // balance after they have been sent, so the fallback path can finish. + getBalances: async () => { + const base = chain.xchain !== undefined ? chain.xchain : 0 + const credited = calls.mint.length && chain.mintsNeverIndex !== true + ? base + calls.mint.reduce((s, c) => s + Number(c.params.AMOUNT), 0) + : base + return { data: [{ tick: 'XCHAIN', amount: String(credited) }] } + }, + // Every action goes through session.submit; the convenience wrappers + // (mint/stake) are sugar over it, and the service calls submit directly + // so one code path handles the funding chain. + session: () => ({ + address: ADDRESS, + submit: async (actionData, enc, opts) => { + const rec = { params: actionData.params, enc, opts } + if (actionData.action === 'MINT') calls.mint.push(rec) + if (actionData.action === 'STAKE') calls.stake.push(rec) + const txid = actionData.action === 'STAKE' ? 'staketx' : 'mint' + calls.mint.length + // Report the inputs the encoder was told to use, so the chain + // assertion in the service sees a real answer. + const spentInputs = (enc.utxos || []).map(u => ({ txid: u.txid, vout: u.vout })) + return { txid, spentInputs } + } + }), + // The chain link: each broadcast leaves a change output the next action + // is funded from. Keyed by txid so the service's filter is exercised. + // noChange breaks every hop; noChangeFor breaks only the hop after the named txids. + requireEncoder: () => ({ + getUTXOs: async () => ({ utxos: chain.noChange || (chain.noChangeFor || []).includes(prevTxidRef.value) ? [] : [ + { txid: prevTxidRef.value || 'seed', fullTxid: prevTxidRef.value || 'seed', vout: 1, value: '150000', confirmations: 0 } + ] }) + }) + } + // The fake encoder answers with an output of whatever was broadcast last. + const prevTxidRef = { value: null } + sdk.session = trackPreviousOutput(sdk.session, prevTxidRef) + return { sdk, calls } +} + +function run(opts, chain, settingsExtra = {}) { + const { sdk, calls } = makeSdk(chain) + const logged = [] + const network = settingsExtra.network || 'testnet' + const deps = { + settings: { enabled: true, pubkey: PUBKEY, network: 'testnet', P2P_PORT: 10002, ...settingsExtra }, + wallets: { NETWORK: network, STAKE_ADDRESS: ADDRESS, STAKE_WIF_SECRET: 'cFakeWif' }, + makeSdk: () => sdk, + sdk: { XChainSDK: function () { throw new Error('makeSdk should be used') } }, + log: m => logged.push(String(m)) + } + return stakeValidator(opts, deps).then(result => ({ result, calls, logged })) +} + +module.exports = { PUBKEY, ADDRESS, makeSdk, run } diff --git a/test/unit/bootstrap_archive_meta.test.js b/test/unit/bootstrap_archive_meta.test.js index 8e6906bb..703de388 100644 --- a/test/unit/bootstrap_archive_meta.test.js +++ b/test/unit/bootstrap_archive_meta.test.js @@ -193,3 +193,76 @@ describe('BootstrapArchiveMeta', function () { }) }) }) + +// One raw 512-byte tar header block: only the fields parseTarHeader reads. +function tarHeaderBlock(name, size, typeflag) { + const block = Buffer.alloc(512) + block.write(name, 0) + block.write(size.toString(8).padStart(11, '0') + ' ', 124) + block[156] = typeflag.charCodeAt(0) + return block +} + +function padTo512(body) { + return Buffer.concat([body, Buffer.alloc((512 - (body.length % 512)) % 512)]) +} + +describe('BootstrapArchiveMeta', function () { + beforeEach(setupWorkDir) + afterEach(cleanupWorkDir) + + describe('readLeadingArchiveMember(): extension header size cap', function () { + it('gives up at once on an extension header declaring more than the ceiling, without buffering the stream', async function () { + // The reader runs before the signature check, so a crafted pax header must not make it buffer the whole gunzip output. + const raw = Buffer.concat([tarHeaderBlock('PaxHeader', 1024 * 1024 * 1024, 'x'), Buffer.alloc(48 * 1024 * 1024, 0x41)]) + const archive = path.join(dir, 'crafted.tar.gz') + fs.writeFileSync(archive, require('zlib').gzipSync(raw, { level: 1 })) + const timedOut = Symbol('timed out') + const result = await Promise.race([ + meta.readLeadingArchiveMember(archive, 'bootstrap.json'), + new Promise(resolve => setTimeout(() => resolve(timedOut), 1000)) + ]) + expect(result).to.equal(null) + }) + + it('still skips a small pax header ahead of bootstrap.json', async function () { + const body = Buffer.from(JSON.stringify(meta.buildBootstrapMeta({ module: 'xchain-decoder', coin: 'bitcoin', network: 'mainnet', height: 42 }))) + const pax = Buffer.from('30 mtime=1759000000.000000000\n') + const raw = Buffer.concat([ + tarHeaderBlock('PaxHeader', pax.length, 'x'), padTo512(pax), + tarHeaderBlock('bootstrap.json', body.length, '0'), padTo512(body), + Buffer.alloc(1024) + ]) + const archive = path.join(dir, 'pax.tar.gz') + fs.writeFileSync(archive, require('zlib').gzipSync(raw)) + expect((await meta.readBootstrapArchiveMeta(archive)).height).to.equal(42) + }) + }) + + describe('compareArchiveIdentity()', function () { + const target = { module: 'xchain-decoder', coin: 'bitcoin', network: 'mainnet' } + const full = { format: 1, module: 'xchain-decoder', coin: 'bitcoin', network: 'mainnet', height: 1, created: null } + + it('matches an archive that declares the target identity', function () { + expect(meta.compareArchiveIdentity(full, target)).to.deep.equal({ status: 'match', mismatches: [], unchecked: [] }) + }) + + it('reports a legacy archive with no metadata as unchecked, not as a mismatch', function () { + expect(meta.compareArchiveIdentity(null, target).status).to.equal('unchecked') + }) + + it('reports null coin/network (the classic-level converter shape) as unchecked while still checking the module', function () { + const converted = { ...full, coin: null, network: null } + expect(meta.compareArchiveIdentity(converted, target)).to.deep.equal({ status: 'unchecked', mismatches: [], unchecked: ['coin', 'network'] }) + expect(meta.compareArchiveIdentity({ ...converted, module: 'xchain-utxo-tracker' }, target).status).to.equal('mismatch') + }) + + it('names each field that differs', function () { + for (const [field, value] of [['module', 'xchain-indexer'], ['coin', 'dogecoin'], ['network', 'testnet']]) { + const result = meta.compareArchiveIdentity({ ...full, [field]: value }, target) + expect(result.status).to.equal('mismatch') + expect(result.mismatches).to.deep.equal([{ field, archive: value, target: target[field] }]) + } + }) + }) +}) diff --git a/test/unit/bootstrap_health_gate.test/durable_halt_markers.test.js b/test/unit/bootstrap_health_gate.test/durable_halt_markers.test.js index c4b87da5..61fede46 100644 --- a/test/unit/bootstrap_health_gate.test/durable_halt_markers.test.js +++ b/test/unit/bootstrap_health_gate.test/durable_halt_markers.test.js @@ -12,7 +12,8 @@ const { expect, installEnvironmentHooks, loadGate, - makeRunner + makeRunner, + refusal } = require('./support/bootstrap_health_gate') describe('BootstrapHealthGate', function () { @@ -36,5 +37,19 @@ describe('BootstrapHealthGate', function () { const sqls = runner.getCalls().map(c => (c.args[1] || []).join(' ')) expect(sqls.some(s => /FROM `[^`]+`\.events/.test(s))).to.equal(false) }) + + // xchain-sync provisions and writes sync_halt in an indexer replica's own + // database, so the gate's own-database probe is live for an indexer, not dead code. + it('REFUSES an indexer whose OWN database carries an uncleared sync halt', async function () { + const gate = loadGate() + const runner = makeRunner({ + status: { status: 'healthy', lag: 0, decoderReorgHalted: false }, + syncHaltRows: '1', + decoder: { syncHaltRows: '0' } + }) + const err = await refusal(callGate(gate, { module: XChainService.XCHAIN_INDEXER, runner })) + expect(err.message).to.match(/the database carries an uncleared xchain-sync divergence halt/) + expect(err.message).to.not.match(/paired decoder database[\s\S]*sync_halt/) + }) }) }) diff --git a/test/unit/bootstrap_health_gate.test/evaluate_status_payload.test.js b/test/unit/bootstrap_health_gate.test/evaluate_status_payload.test.js index f0ac8987..197b5fe1 100644 --- a/test/unit/bootstrap_health_gate.test/evaluate_status_payload.test.js +++ b/test/unit/bootstrap_health_gate.test/evaluate_status_payload.test.js @@ -168,3 +168,30 @@ describe('BootstrapHealthGate', function () { }) }) }) + +describe('BootstrapHealthGate', function () { + + installEnvironmentHooks() + + describe('evaluateStatusPayload()', function () { + + // A parked decoder whose marker write failed reads reorg_halted false after its + // next probe; the lag ceiling is operator-raisable, so it must not be what refuses. + it('REFUSES a decoder parked on a REORG_HALT whose marker was never written, even under a raised lag ceiling', function () { + const gate = loadGate() + const parked = { + status: 'healthy', lag_blocks: 130, reorg_halted: false, + reorg_halt_checked_at: '2026-09-29T00:00:00.000Z', + reorg_halt_parked: true, reorg_halt_parked_at: '2026-09-28T23:59:00.000Z' + } + const reasons = gate.evaluateStatusPayload(parked, { maxLag: 1000 }) + expect(reasons).to.have.lengthOf(1) + expect(reasons[0]).to.match(/PARKED on a REORG_HALT/) + expect(reasons[0]).to.match(/since 2026-09-28T23:59:00\.000Z/) + for (const notParked of [false, undefined, null, 'true', 1]) { + expect(gate.evaluateStatusPayload({ ...parked, reorg_halt_parked: notParked }, { maxLag: 1000 }), + `reorg_halt_parked=${JSON.stringify(notParked)}`).to.deep.equal([]) + } + }) + }) +}) diff --git a/test/unit/bootstrap_service.test/helpers/support.js b/test/unit/bootstrap_service.test/helpers/support.js index 3d8606b4..2ab23b61 100644 --- a/test/unit/bootstrap_service.test/helpers/support.js +++ b/test/unit/bootstrap_service.test/helpers/support.js @@ -194,9 +194,13 @@ function makeBootstrapPolicyStubs() { // guard defaults to "unknown, do not refuse" so restore MECHANICS stay the // subject here. Both have their own suites (BootstrapArchiveMeta.test.js, // BootstrapNodeTipGuard.test.js). + // The restore-time reader defaults to "no metadata" (a legacy archive), so the identity check passes through and restore + // MECHANICS stay the subject; the comparison itself is the real one. const archiveMetaStub = { buildBootstrapMeta: require('../../../../src/services/bootstrap_archive_meta').buildBootstrapMeta, - writeBootstrapMeta: sinon.stub().resolves('bootstrap.json') + writeBootstrapMeta: sinon.stub().resolves('bootstrap.json'), + readBootstrapArchiveMeta: sinon.stub().resolves(null), + compareArchiveIdentity: require('../../../../src/services/bootstrap_archive_meta').compareArchiveIdentity } const nodeTipGuardStub = { assessNodeTipForRestore: sinon.stub().resolves({ verdict: 'unknown', refuse: false, detail: 'not compared in this test' }) @@ -326,8 +330,11 @@ function makeBootstrapOverrides(stubs, execFileCb) { ensureDatabasePool: stubs.databaseService.ensureDatabasePool, askMariadbRootPassword: stubs.databaseService.askMariadbRootPassword }, + // The status probe is absent unless a suite sets one, so the tracker height reads null by default. './bootstrap_health_gate': { - assertBootstrapSourceHealthy: stubs.healthGate.assertBootstrapSourceHealthy + assertBootstrapSourceHealthy: stubs.healthGate.assertBootstrapSourceHealthy, + probeServiceStatus: stubs.healthGate.probeServiceStatus, + MODULE_API_PORT_KEY: stubs.healthGate.MODULE_API_PORT_KEY }, './bootstrap_republish_ledger': { recordBootstrapPublished: stubs.republishLedger.recordBootstrapPublished @@ -337,8 +344,10 @@ function makeBootstrapOverrides(stubs, execFileCb) { clearEncoderMaintenance: stubs.encoderMaintenance.clearEncoderMaintenance }, './bootstrap_archive_meta': { - buildBootstrapMeta: stubs.archiveMeta.buildBootstrapMeta, - writeBootstrapMeta: stubs.archiveMeta.writeBootstrapMeta + buildBootstrapMeta: stubs.archiveMeta.buildBootstrapMeta, + writeBootstrapMeta: stubs.archiveMeta.writeBootstrapMeta, + readBootstrapArchiveMeta: stubs.archiveMeta.readBootstrapArchiveMeta, + compareArchiveIdentity: stubs.archiveMeta.compareArchiveIdentity }, './bootstrap_node_tip_guard': { assessNodeTipForRestore: stubs.nodeTipGuard.assessNodeTipForRestore diff --git a/test/unit/bootstrap_service.test/restore_checks_archive_identity_and_refusal_signature.test.js b/test/unit/bootstrap_service.test/restore_checks_archive_identity_and_refusal_signature.test.js new file mode 100644 index 00000000..6fd2d355 --- /dev/null +++ b/test/unit/bootstrap_service.test/restore_checks_archive_identity_and_refusal_signature.test.js @@ -0,0 +1,210 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +const { + sinon, + expect, + PassThrough, + XChainService, + COIN, + NETWORK, + FAKE_CONTAINER_ID, + FAKE_DB_CONTAINER, + makeSpawnProc, + drainPassThrough, + makeStubs, + stubVerifiedInner, + loadBootstrapService +} = require('./helpers/support') + +let savedRequireSigned + +function allowUnsignedBootstrap() { + savedRequireSigned = process.env.XCHAIN_NODE_REQUIRE_SIGNED_BOOTSTRAP + process.env.XCHAIN_NODE_REQUIRE_SIGNED_BOOTSTRAP = '0' +} + +function restoreRequireSignedBootstrapSetting() { + if (savedRequireSigned === undefined) delete process.env.XCHAIN_NODE_REQUIRE_SIGNED_BOOTSTRAP + else process.env.XCHAIN_NODE_REQUIRE_SIGNED_BOOTSTRAP = savedRequireSigned +} + +const TRACKER_ARCHIVE = '/data/bitcoin/mainnet/xchain-utxo-tracker/bootstrap/latest.tgz' +const DECODER_ARCHIVE = '/data/bitcoin/mainnet/xchain-decoder/bootstrap/latest.tgz' + +function declared(module, coin = COIN, network = NETWORK) { + return { format: 1, module, coin, network, height: 100, created: '2026-09-01T00:00:00Z' } +} + +function anyWipeOrExtract(stubs) { + return stubs.execFile.getCalls().some(c => { + const argv = Array.isArray(c.args[1]) ? c.args[1].join(' ') : '' + return argv.includes('-delete') || argv.includes('DROP DATABASE') || (c.args[0] === 'tar' && c.args[1][0] === 'xzf') + }) +} + +function mysqlSpawn(stubs) { + const mysqlProc = makeSpawnProc() + stubs.spawn = sinon.stub().callsFake(() => { + setImmediate(() => { drainPassThrough(mysqlProc.stdin); mysqlProc.emit('close', 0) }) + return mysqlProc + }) +} + +function stubDownloadedArchive(stubs) { + stubs.fs.existsSync.callsFake(p => !/\.(pem|sig)$/.test(String(p))) + const dataStream = new PassThrough() + const writeStream = new PassThrough() + drainPassThrough(writeStream) + stubs.fs.createWriteStream.returns(writeStream) + stubs.axios.resolves({ status: 200, headers: {}, data: dataStream }) + return () => { dataStream.end(); writeStream.emit('finish') } +} + +function archiveRemoved(stubs, archivePath) { + return stubs.fs.rmSync.getCalls().some(c => String(c.args[0]) === archivePath) +} + +describe('BootstrapService', function () { + beforeEach(allowUnsignedBootstrap) + afterEach(restoreRequireSignedBootstrapSetting) + describe('restoreBootstrap(): archive identity', function () { + it('refuses a tracker archive declaring another coin/network before anything is stopped, wiped or extracted', async function () { + const stubs = makeStubs() + stubVerifiedInner(stubs, { archivePath: TRACKER_ARCHIVE }) + stubs.archiveMeta.readBootstrapArchiveMeta.resolves(declared(XChainService.XCHAIN_UTXO_TRACKER, 'dogecoin', 'testnet')) + const bs = loadBootstrapService(stubs) + + const err = await bs.restoreBootstrap(COIN, NETWORK, XChainService.XCHAIN_UTXO_TRACKER, 'latest.tgz').then(() => null, e => e) + expect(err).to.not.equal(null) + expect(err.name).to.equal('BootstrapIntegrityError') + expect(err.message).to.match(/coin dogecoin, network testnet/) + expect(stubs.dockerService.stopContainer.called).to.equal(false) + expect(anyWipeOrExtract(stubs)).to.equal(false) + }) + + it('refuses an indexer archive offered to the decoder (module-only mismatch) before the DROP', async function () { + const stubs = makeStubs() + stubVerifiedInner(stubs, { archivePath: DECODER_ARCHIVE, innerName: 'dump.sql.gz', checksumName: 'dump.sha256' }) + stubs.archiveMeta.readBootstrapArchiveMeta.resolves(declared(XChainService.XCHAIN_INDEXER)) + mysqlSpawn(stubs) + const bs = loadBootstrapService(stubs) + + const err = await bs.restoreBootstrap(COIN, NETWORK, XChainService.XCHAIN_DECODER, 'latest.tgz').then(() => null, e => e) + expect(err && err.name).to.equal('BootstrapIntegrityError') + expect(err.message).to.match(/declares module xchain-indexer but the restore target is xchain-decoder/) + expect(stubs.dockerService.stopContainer.called).to.equal(false) + expect(stubs.spawn.called).to.equal(false) + expect(anyWipeOrExtract(stubs)).to.equal(false) + }) + }) +}) + +describe('BootstrapService', function () { + beforeEach(allowUnsignedBootstrap) + afterEach(restoreRequireSignedBootstrapSetting) + describe('restoreBootstrap(): archive identity', function () { + it('restores a matching archive exactly as before', async function () { + const stubs = makeStubs() + stubVerifiedInner(stubs, { archivePath: DECODER_ARCHIVE, innerName: 'dump.sql.gz', checksumName: 'dump.sha256' }) + stubs.archiveMeta.readBootstrapArchiveMeta.resolves(declared(XChainService.XCHAIN_DECODER)) + stubs.databaseService.getDatabaseContainerId.resolves(FAKE_DB_CONTAINER) + stubs.db.getModuleContainer.resolves('svc-cid') + stubs.fs.promises.stat.resolves({ size: 512 }) + mysqlSpawn(stubs) + const bs = loadBootstrapService(stubs) + + expect(await bs.restoreBootstrap(COIN, NETWORK, XChainService.XCHAIN_DECODER, 'latest.tgz')).to.equal(true) + expect(stubs.spawn.called).to.equal(true) + }) + + it('restores a converted tracker archive whose coin/network are null, checking only its module', async function () { + const stubs = makeStubs() + stubVerifiedInner(stubs, { archivePath: TRACKER_ARCHIVE }) + stubs.archiveMeta.readBootstrapArchiveMeta.resolves(declared(XChainService.XCHAIN_UTXO_TRACKER, null, null)) + stubs.fs.promises.stat.resolves({ size: 2048, isFile: () => true }) + stubs.db.getModuleContainer.resolves(FAKE_CONTAINER_ID) + const tarProc = makeSpawnProc() + stubs.spawn = sinon.stub().callsFake(() => { + setImmediate(() => { drainPassThrough(tarProc.stdin); tarProc.emit('close', 0) }) + return tarProc + }) + const bs = loadBootstrapService(stubs) + + expect(await bs.restoreBootstrap(COIN, NETWORK, XChainService.XCHAIN_UTXO_TRACKER, 'latest.tgz')).to.equal(true) + expect(stubs.dockerService.stopContainer.called).to.equal(true) + }) + }) +}) + +describe('BootstrapService', function () { + beforeEach(allowUnsignedBootstrap) + afterEach(restoreRequireSignedBootstrapSetting) + describe('restoreBootstrap(): archive identity', function () { + it('records an identity mismatch on the install path as a failure that kept the archive, with no wipe', async function () { + const stubs = makeStubs() + const finishDownload = stubDownloadedArchive(stubs) + stubVerifiedInner(stubs, { manageExistsSync: false }) + stubs.archiveMeta.readBootstrapArchiveMeta.resolves(declared(XChainService.XCHAIN_UTXO_TRACKER, 'litecoin')) + const bs = loadBootstrapService(stubs) + bs.resetBootstrapOutcomes() + + const promise = bs.ensureBootstrapUtxoTracker(COIN, NETWORK) + setImmediate(finishDownload) + expect(await promise).to.equal(false) + expect(stubs.dockerService.stopContainer.called).to.equal(false) + expect(archiveRemoved(stubs, TRACKER_ARCHIVE)).to.equal(false) + + const lines = [] + const log = sinon.stub(console, 'log').callsFake((...a) => lines.push(a.join(' '))) + try { bs.reportBootstrapOutcomes() } finally { log.restore() } + expect(lines.join('\n')).to.match(/xchain-utxo-tracker: NOT restored: Bootstrap archive identity mismatch .* declares coin litecoin/) + }) + }) +}) + +describe('BootstrapService', function () { + afterEach(restoreRequireSignedBootstrapSetting) + describe('the node tip guard refusal is signature-verified before the archive is retired', function () { + it('keeps a refused archive that fails its signature check and records the integrity failure, not node lag', async function () { + // Signatures required (the default) and none published: checkBootstrapSignature throws, as it does for a tampered archive. + savedRequireSigned = process.env.XCHAIN_NODE_REQUIRE_SIGNED_BOOTSTRAP + delete process.env.XCHAIN_NODE_REQUIRE_SIGNED_BOOTSTRAP + const stubs = makeStubs() + const finishDownload = stubDownloadedArchive(stubs) + stubs.nodeTipGuard.assessNodeTipForRestore.resolves({ verdict: 'behind-refuse', refuse: true, detail: 'node behind a forged height' }) + const bs = loadBootstrapService(stubs) + bs.resetBootstrapOutcomes() + + const promise = bs.ensureBootstrapMariaDb(COIN, NETWORK, XChainService.XCHAIN_DECODER) + setImmediate(finishDownload) + expect(await promise).to.equal(false) + expect(archiveRemoved(stubs, DECODER_ARCHIVE), 'the unverified archive is evidence and must be kept').to.equal(false) + + const lines = [] + const log = sinon.stub(console, 'log').callsFake((...a) => lines.push(a.join(' '))) + try { bs.reportBootstrapOutcomes() } finally { log.restore() } + const summary = lines.join('\n') + expect(summary).to.match(/xchain-decoder: NOT restored: Refusing unsigned bootstrap/) + expect(summary).to.not.match(/the coin node is behind the archive/) + }) + + it('retires a refused archive once its signature check passes, recording node-behind as before', async function () { + allowUnsignedBootstrap() + const stubs = makeStubs() + const finishDownload = stubDownloadedArchive(stubs) + stubs.nodeTipGuard.assessNodeTipForRestore.resolves({ verdict: 'behind-refuse', refuse: true, detail: 'node behind' }) + const bs = loadBootstrapService(stubs) + bs.resetBootstrapOutcomes() + + const promise = bs.ensureBootstrapUtxoTracker(COIN, NETWORK) + setImmediate(finishDownload) + expect(await promise).to.equal(false) + expect(archiveRemoved(stubs, TRACKER_ARCHIVE)).to.equal(true) + }) + }) +}) diff --git a/test/unit/bootstrap_service.test/tracker_archive_height_after_restart.test.js b/test/unit/bootstrap_service.test/tracker_archive_height_after_restart.test.js new file mode 100644 index 00000000..64710817 --- /dev/null +++ b/test/unit/bootstrap_service.test/tracker_archive_height_after_restart.test.js @@ -0,0 +1,145 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +const { + sinon, + expect, + XChainService, + COIN, + NETWORK, + FAKE_CONTAINER_ID, + makeAutoSpawn, + makeStubs, + loadBootstrapService +} = require('./helpers/support') + +const PORT_KEY = 'UTXO_TRACKER_API_PORT' + +let savedRequireSigned + +function saveRequireSignedBootstrapSetting() { + savedRequireSigned = process.env.XCHAIN_NODE_REQUIRE_SIGNED_BOOTSTRAP + process.env.XCHAIN_NODE_REQUIRE_SIGNED_BOOTSTRAP = '0' +} + +function restoreRequireSignedBootstrapSetting() { + if (savedRequireSigned === undefined) delete process.env.XCHAIN_NODE_REQUIRE_SIGNED_BOOTSTRAP + else process.env.XCHAIN_NODE_REQUIRE_SIGNED_BOOTSTRAP = savedRequireSigned +} + +// Wire a tracker status probe that answers `heights` in order, noting whether the tracker had been restarted at each call. +function trackerStubsWithProbe(heights, { snapshotFails = false } = {}) { + const stubs = makeStubs() + stubs.fs.existsSync.returns(false) + stubs.db.getModuleContainer.resolves(FAKE_CONTAINER_ID) + stubs.fs.promises.stat.resolves({ size: 1024 * 1024 }) + stubs.execFile = sinon.stub().callsFake((cmd, args) => { + if (snapshotFails && cmd === 'docker' && Array.isArray(args) && args.includes('sh')) { + return Promise.reject(new Error('cp: cannot create hard link')) + } + return Promise.resolve({ stdout: '104857600\t/data\n' }) + }) + stubs.configService.getDefaultConfig.resolves({ + UTXO_TRACKER_BOOTSTRAP_VOLUME: '/data/bitcoin/mainnet/xchain-utxo-tracker/bootstrap/', + [PORT_KEY]: '8080' + }) + const probeLog = [] + stubs.healthGate.MODULE_API_PORT_KEY = { [XChainService.XCHAIN_UTXO_TRACKER]: PORT_KEY } + stubs.healthGate.probeServiceStatus = sinon.stub().callsFake(async () => { + const answer = heights[Math.min(probeLog.length, heights.length - 1)] + probeLog.push({ restarted: stubs.dockerService.startContainer.called }) + if (answer instanceof Error) throw answer + return { committed_height: answer } + }) + return { stubs, probeLog } +} + +function recordedHeight(stubs) { + return stubs.archiveMeta.writeBootstrapMeta.firstCall.args[1].height +} + +describe('BootstrapService', function () { + beforeEach(saveRequireSignedBootstrapSetting) + afterEach(restoreRequireSignedBootstrapSetting) + describe('makeBootstrapUtxoTracker(): archive height is read after the restart', function () { + it('records the post-restart height on the snapshot path, not the pre-stop one', async function () { + const { stubs, probeLog } = trackerStubsWithProbe([100, 101]) + makeAutoSpawn(stubs) + + const bs = loadBootstrapService(stubs) + expect(await bs.makeBootstrap(COIN, NETWORK, XChainService.XCHAIN_UTXO_TRACKER)).to.be.true + + expect(probeLog.map(p => p.restarted)).to.deep.equal([false, true]) + expect(recordedHeight(stubs)).to.equal(101) + }) + + it('restarts the tracker after the compress and before the wrap on the stopped-tracker fallback', async function () { + const { stubs, probeLog } = trackerStubsWithProbe([100, 104], { snapshotFails: true }) + const spawnCalls = makeAutoSpawn(stubs) + const rawSpawn = stubs.spawn + const startedAtSpawn = [] + stubs.spawn = sinon.stub().callsFake((cmd, args) => { + startedAtSpawn.push(stubs.dockerService.startContainer.callCount) + return rawSpawn(cmd, args) + }) + + const bs = loadBootstrapService(stubs) + expect(await bs.makeBootstrap(COIN, NETWORK, XChainService.XCHAIN_UTXO_TRACKER)).to.be.true + + // spawn #0 is the docker tar compress (tracker still stopped), spawn #1 the outer wrap (tracker back). + expect(spawnCalls[0].cmd).to.equal('docker') + expect(spawnCalls[1].cmd).to.equal('tar') + expect(startedAtSpawn).to.deep.equal([0, 1]) + expect(stubs.dockerService.startContainer.callCount).to.equal(1) + expect(stubs.encoderMaintenance.clearEncoderMaintenance.callCount).to.equal(1) + expect(probeLog.map(p => p.restarted)).to.deep.equal([false, true]) + expect(recordedHeight(stubs)).to.equal(104) + }) + + it('keeps the pre-stop height when a rollback leaves the post-restart reading lower', async function () { + const { stubs } = trackerStubsWithProbe([100, 98]) + makeAutoSpawn(stubs) + + const bs = loadBootstrapService(stubs) + expect(await bs.makeBootstrap(COIN, NETWORK, XChainService.XCHAIN_UTXO_TRACKER)).to.be.true + expect(recordedHeight(stubs)).to.equal(100) + }) + }) +}) + +describe('BootstrapService', function () { + beforeEach(saveRequireSignedBootstrapSetting) + afterEach(restoreRequireSignedBootstrapSetting) + describe('tracker archive height helpers', function () { + const trackerArchive = require('../../../src/services/bootstrap_service/tracker_archive') + + it('chooseArchiveHeight never records below either reading and keeps the floor when the post-read fails', function () { + loadBootstrapService(makeStubs()) + expect(trackerArchive.chooseArchiveHeight(null, null)).to.equal(null) + expect(trackerArchive.chooseArchiveHeight(100, null)).to.equal(100) + expect(trackerArchive.chooseArchiveHeight(null, 101)).to.equal(101) + expect(trackerArchive.chooseArchiveHeight(100, 101)).to.equal(101) + expect(trackerArchive.chooseArchiveHeight(100, 98)).to.equal(100) + }) + + it('readTrackerHeightAfterRestart retries a tracker that is still coming up', async function () { + const { stubs } = trackerStubsWithProbe([new Error('connection refused'), new Error('connection refused'), 107]) + loadBootstrapService(stubs) + const height = await trackerArchive.readTrackerHeightAfterRestart(COIN, NETWORK, FAKE_CONTAINER_ID, { attempts: 5, delayMs: 0 }) + expect(height).to.equal(107) + expect(stubs.healthGate.probeServiceStatus.callCount).to.equal(3) + }) + + it('readTrackerHeightAfterRestart gives up with null after its attempts', async function () { + const { stubs } = trackerStubsWithProbe([new Error('connection refused')]) + loadBootstrapService(stubs) + const height = await trackerArchive.readTrackerHeightAfterRestart(COIN, NETWORK, FAKE_CONTAINER_ID, { attempts: 3, delayMs: 0 }) + expect(height).to.equal(null) + expect(stubs.healthGate.probeServiceStatus.callCount).to.equal(3) + }) + }) +}) diff --git a/test/unit/ci_full_sibling_roster.test/sibling_roster.test.js b/test/unit/ci_full_sibling_roster.test/sibling_roster.test.js index b3b0a30e..4c7dd1e6 100644 --- a/test/unit/ci_full_sibling_roster.test/sibling_roster.test.js +++ b/test/unit/ci_full_sibling_roster.test/sibling_roster.test.js @@ -38,7 +38,10 @@ function makeFixture() { fs.copyFileSync(CI_FULL, path.join(nodeDir, 'ci-full.sh')) fs.chmodSync(path.join(nodeDir, 'ci-full.sh'), 0o755) fs.writeFileSync(path.join(root, 'xchain-node', '.ci-siblings'), REAL_CI_SIBLINGS) - for (const sib of ['xchain-hub', 'xchain-indexer']) fs.mkdirSync(path.join(root, sib)) + // Every sibling the roster declares, read from the roster itself so the fixture + // cannot drift from it the way a second hard-coded list would. + const declared = REAL_CI_SIBLINGS.split('\n').map(l => l.trim()).filter(l => l && !l.startsWith('#')) + for (const sib of declared) fs.mkdirSync(path.join(root, sib)) const fakebin = path.join(root, 'fakebin') fs.mkdirSync(fakebin) diff --git a/test/unit/cli/dispatch_lock.test.js b/test/unit/cli/dispatch_lock.test.js new file mode 100644 index 00000000..a83eafc7 --- /dev/null +++ b/test/unit/cli/dispatch_lock.test.js @@ -0,0 +1,170 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. +// +// Pins which commands keep the command lock through their action. A command +// that writes state a concurrent deploy reads must hold it to exit; a command +// that only reads hands it back once preCheck is done. + +const sinon = require('sinon') +const { expect } = require('chai') +const { Command } = require('commander') + +const { installDispatch } = require('../../../src/cli/dispatch') + +const SIGNALS = ['exit', 'SIGINT', 'SIGTERM'] + +// Build a real Commander tree with do-nothing actions, so the hook sees the +// same parent/child shape the shipped CLI gives it. +function buildProgram(deps) { + const program = new Command() + program.exitOverride() + program.option('--verbose').option('--no-telemetry') + installDispatch(program, deps) + for (const name of ['install', 'update', 'reset', 'bootstrap', 'ps', 'logs', 'exec', 'shell']) + program.command(name).allowUnknownOption().argument('[args...]').action(deps.action) + program.command('clear-reorg-halt').argument('').argument('') + .option('--reason ').option('--dry-run').action(deps.action) + const validator = program.command('validator').action(deps.action) + validator.command('init').option('--force').option('--force-wallets').action(deps.action) + validator.command('stake').option('--broadcast').action(deps.action) + validator.command('unstake').option('--broadcast').action(deps.action) + validator.command('status').action(deps.action) + validator.command('drift').action(deps.action) + return program +} + +function makeDeps() { + const release = sinon.stub() + return { + release, + config: {}, + action: sinon.stub(), + setVerbose: sinon.stub(), + maybeSelfUpdateBeforeUpdate: sinon.stub().resolves(), + redactSecrets: (s) => s, + acquireCommandLock: sinon.stub().returns(release), + preCheck: sinon.stub().resolves(), + refForPreCheck: sinon.stub().returns(null), + commandRepairsHub: sinon.stub().returns(false), + maybeReportTelemetry: sinon.stub().resolves(), + noticeNewerRelease: sinon.stub().resolves() + } +} + +let listenersBefore + +// Drop the exit/signal handlers a hold-through lock registers, so no stub +// release outlives its test. +function snapshotListeners() { + listenersBefore = {} + for (const ev of SIGNALS) listenersBefore[ev] = process.listeners(ev) +} + +function restoreListeners() { + for (const ev of SIGNALS) { + for (const fn of process.listeners(ev)) { + if (!listenersBefore[ev].includes(fn)) process.removeListener(ev, fn) + } + } + sinon.restore() +} + +async function run(argv) { + const deps = makeDeps() + await buildProgram(deps).parseAsync(argv, { from: 'user' }) + return deps +} + +describe('CLI dispatch: commands that hold the lock through their action', function () { + beforeEach(snapshotListeners) + afterEach(restoreListeners) + + for (const argv of [['install', 'xchain-hub'], ['update', 'xchain-hub'], ['reset', 'xchain-decoder'], ['bootstrap', 'restore']]) { + it(`${argv[0]} keeps the lock and refuses a held one at once`, async function () { + const deps = await run(argv) + expect(deps.acquireCommandLock.calledOnceWith({ command: argv[0], waitMs: 0 })).to.be.true + expect(deps.release.called).to.be.false + expect(deps.action.calledOnce).to.be.true + }) + } + + it('clear-reorg-halt keeps the lock through its database check and audited write', async function () { + const deps = await run(['clear-reorg-halt', 'bitcoin', 'regtest', '--reason', 'known good replica']) + expect(deps.acquireCommandLock.calledOnceWith({ command: 'clear-reorg-halt', waitMs: 0 })).to.be.true + expect(deps.preCheck.calledOnce).to.be.true + expect(deps.release.called).to.be.false + }) + + it('clear-reorg-halt --dry-run is serialized too, so its verdict cannot describe a DB mid-reset', async function () { + const deps = await run(['clear-reorg-halt', 'bitcoin', 'regtest', '--dry-run']) + expect(deps.acquireCommandLock.calledOnceWith({ command: 'clear-reorg-halt', waitMs: 0 })).to.be.true + expect(deps.release.called).to.be.false + }) +}) + +describe('CLI dispatch: commands that hand the lock back after preCheck', function () { + beforeEach(snapshotListeners) + afterEach(restoreListeners) + + for (const name of ['ps', 'logs', 'exec', 'shell']) { + it(`${name} releases the lock once preCheck returns`, async function () { + const deps = await run([name]) + expect(deps.acquireCommandLock.calledOnceWith({ command: name, waitMs: 15000 })).to.be.true + expect(deps.release.calledOnce).to.be.true + expect(deps.release.calledAfter(deps.preCheck)).to.be.true + }) + } +}) + +describe('CLI dispatch: validator subcommands', function () { + beforeEach(snapshotListeners) + afterEach(restoreListeners) + + for (const flags of [[], ['--force'], ['--force-wallets']]) { + it(`${['validator init', ...flags].join(' ')} holds the lock to exit and still skips preCheck`, async function () { + const deps = await run(['validator', 'init', ...flags]) + expect(deps.acquireCommandLock.calledOnceWith({ command: 'validator init', waitMs: 0 })).to.be.true + expect(deps.release.called).to.be.false + expect(deps.preCheck.called).to.be.false + expect(deps.action.calledOnce).to.be.true + }) + } + + it('validator init --force is refused, and never regenerates the key, while another run holds the lock', async function () { + const exitStub = sinon.stub(process, 'exit') + sinon.stub(console, 'error') + const deps = makeDeps() + const held = new Error('Another xchain-node instance (running "update") holds the command lock') + held.code = 'ELOCKHELD' + deps.acquireCommandLock = sinon.stub().throws(held) + // Stand in for the real exit: a refused lock must end the run before the action. + exitStub.callsFake((code) => { throw Object.assign(new Error('exit'), { exitCode: code }) }) + let exited = null + try { + await buildProgram(deps).parseAsync(['validator', 'init', '--force'], { from: 'user' }) + } catch (err) { + exited = err.exitCode + } + expect(exited).to.equal(1) + expect(deps.action.called).to.be.false + }) + + // --broadcast too: its service locks only the sends, and a dispatch hold would refuse that service's own lock. + for (const argv of [['validator', 'status'], ['validator', 'drift'], ['validator', 'stake'], ['validator', 'unstake'], + ['validator', 'stake', '--broadcast'], ['validator', 'unstake', '--broadcast'], ['validator']]) { + it(`${argv.join(' ')} takes no lock and runs no preCheck`, async function () { + const deps = await run(argv) + expect(deps.acquireCommandLock.called).to.be.false + expect(deps.preCheck.called).to.be.false + }) + } +}) diff --git a/test/unit/cli/validator_send_lock.test.js b/test/unit/cli/validator_send_lock.test.js new file mode 100644 index 00000000..791867be --- /dev/null +++ b/test/unit/cli/validator_send_lock.test.js @@ -0,0 +1,95 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. +// +// Pins the lock `validator stake|unstake` hand their service: the CLI passes +// one to every run, and it takes the command lock once per hold and gives it back once. + +const sinon = require('sinon') +const { expect } = require('chai') +const { Command } = require('commander') + +const { validatorSendLock } = require('../../../src/cli/dispatch') +const { registerValidatorCommands } = require('../../../src/cli/output') + +function lockDeps(config = {}) { + const release = sinon.stub() + return { release, config, acquireCommandLock: sinon.stub().returns(release) } +} + +describe('CLI: validatorSendLock', function () { + afterEach(function () { sinon.restore() }) + + it('takes the lock once per hold, under the command\'s name, refusing a held one at once', function () { + const deps = lockDeps() + const lock = validatorSendLock('validator stake', deps) + lock.hold() + lock.hold() + expect(deps.acquireCommandLock.calledOnceWith({ command: 'validator stake', waitMs: 0 })).to.be.true + lock.release() + lock.release() + expect(deps.release.calledOnce).to.be.true + }) + + it('takes it again after a release, for the send that follows an index wait', function () { + const deps = lockDeps() + const lock = validatorSendLock('validator stake', deps) + lock.hold() + lock.release() + lock.hold() + expect(deps.acquireCommandLock.calledTwice).to.be.true + lock.release() + }) + + it('waits as long as any mutator is told to wait for a held lock', function () { + const deps = lockDeps({ XCHAIN_NODE_MUTATING_LOCK_WAIT_MS: '30000' }) + const lock = validatorSendLock('validator unstake', deps) + lock.hold() + expect(deps.acquireCommandLock.calledOnceWith({ command: 'validator unstake', waitMs: 30000 })).to.be.true + lock.release() + }) + + it('releases on process exit while held, and drops that handler once released', function () { + const deps = lockDeps() + const before = process.listeners('exit').length + const lock = validatorSendLock('validator stake', deps) + lock.hold() + expect(process.listeners('exit').length).to.equal(before + 1) + lock.release() + expect(process.listeners('exit').length).to.equal(before) + }) +}) + +describe('CLI: validator stake|unstake hand their service a send lock', function () { + afterEach(function () { sinon.restore() }) + + for (const [name, service, command] of [ + ['stake', 'stakeValidator', 'validator stake'], + ['unstake', 'unstakeValidator', 'validator unstake'] + ]) { + it(`validator ${name} --broadcast passes a lock named "${command}"`, async function () { + sinon.stub(process, 'exit') + const deps = lockDeps() + deps[service] = sinon.stub().resolves({}) + const program = new Command() + program.exitOverride() + registerValidatorCommands(program, deps) + await program.parseAsync(['validator', name, '--broadcast'], { from: 'user' }) + + expect(deps[service].calledOnce).to.be.true + const [opts, serviceDeps] = deps[service].firstCall.args + expect(opts.broadcast).to.be.true + serviceDeps.sendLock.hold() + expect(deps.acquireCommandLock.calledOnceWith({ command, waitMs: 0 })).to.be.true + serviceDeps.sendLock.release() + }) + } +}) diff --git a/test/unit/docker_service.test/13_exec_container_error_branch.test.js b/test/unit/docker_service.test/13_exec_container_error_branch.test.js index 9001e4ca..747e04bc 100644 --- a/test/unit/docker_service.test/13_exec_container_error_branch.test.js +++ b/test/unit/docker_service.test/13_exec_container_error_branch.test.js @@ -55,16 +55,21 @@ describe('DockerService', function () { it('rejects when docker exec fails', async function () { const stubs = makeStubs() + // The callback form of execFile hands stdout and stderr to the callback, never to the error. + const failure = Object.assign(new Error('exec failed'), { code: 4 }) stubs.execFile.callsFake((cmd, args, ...rest) => { const cb = typeof rest[0] === 'function' ? rest[0] : rest[1] - cb(new Error('exec failed')) + cb(failure, 'live marker line\n', 'REFUSED line\n') }) const ds = loadDockerService(stubs) try { await ds.execContainer('abc123', ['ls']) expect.fail() } catch (err) { - expect(err).to.be.an.instanceOf(Error) + expect(err).to.equal(failure) + expect(err.code).to.equal(4) + expect(err.stdout).to.equal('live marker line\n') + expect(err.stderr).to.equal('REFUSED line\n') } }) }) diff --git a/test/unit/migration_precondition_service.test/01_migration_metadata.test.js b/test/unit/migration_precondition_service.test/01_migration_metadata.test.js index b720f146..4eabed64 100644 --- a/test/unit/migration_precondition_service.test/01_migration_metadata.test.js +++ b/test/unit/migration_precondition_service.test/01_migration_metadata.test.js @@ -10,6 +10,27 @@ // license (without AGPL source-disclosure terms) is available - // contact legal@dankest.llc. +const fs = require('fs') +const path = require('path') + +// The three copies of migrationMode: this tool's, and the two runners' it mirrors. +const MODE_TWINS = { + node: path.join(__dirname, '../../../src/services/migration_precondition_service/migration_scan.js'), + indexer: path.join(__dirname, '../../../../xchain-indexer/src/db/database/migration_scan.js'), + decoder: path.join(__dirname, '../../../../xchain-decoder/src/db/migration_statements.js') +} +const REQUIRE_SIBLINGS = process.env.XCHAIN_REQUIRE_SIBLINGS === '1' + +// Pull the mode tag regex and the no-match fallback out of one copy's source text. +function modeRule(source, file) { + const regex = source.match(/\.match\((\/[^\n]*\\bmode\\s\*=[^\n]*\/[a-z]*)\)/) + const fallback = source.match(/return m \? m\[1\]\.toLowerCase\(\) : ([^;\n]+?);?[ \t]*$/m) + if (!regex || !fallback) + throw new Error('could not find the migrationMode tag regex or fallback in ' + file + + '; re-read that copy and update this parity test rather than skipping it') + return { regex: regex[1], fallback: fallback[1].trim() } +} + function registerMigrationDirectiveTests({ expect, migrationDeclaresDeployPrecondition, TAGGED, UNTAGGED }) { describe('migrationDeclaresDeployPrecondition', () => { it('reads the tag off the xchain:migration directive line', () => { @@ -45,14 +66,33 @@ function registerMigrationModeTests({ expect, migrationMode, TAGGED }) { expect(migrationMode('-- xchain:migration mode=auto\nSELECT 1;\n')).to.equal('auto') }) - it('returns null when no mode is declared', () => { - expect(migrationMode('-- just a comment\nSELECT 1;\n')).to.equal(null) + it('defaults to manual when no recognized mode is declared', () => { + // The runners gate every non-auto file, so an unscoped run applies these too. + expect(migrationMode('-- just a comment\nSELECT 1;\n')).to.equal('manual') + expect(migrationMode('-- xchain:migration deploy-precondition=required\nSELECT 1;\n')).to.equal('manual') + expect(migrationMode('-- xchain:migration mode=automatic\nSELECT 1;\n')).to.equal('manual') + expect(migrationMode('-- xchain:migration mode=auto2\nSELECT 1;\n')).to.equal('manual') + expect(migrationMode('-- xchain:migration mode=bogus\n-- xchain:migration mode=auto\nSELECT 1;\n')).to.equal('auto') }) it('ignores a mode token that appears after the prologue', () => { // Body prose and data literals must not be able to answer for the file. const body = '-- header\nSELECT 1;\n-- xchain:migration mode=auto\n' - expect(migrationMode(body)).to.equal(null) + expect(migrationMode(body)).to.equal('manual') + }) + + it('matches the indexer and decoder runners\' rule', function () { + // The copies are duplicated by necessity; this is what keeps them in step. + const twins = { indexer: MODE_TWINS.indexer, decoder: MODE_TWINS.decoder } + const present = Object.entries(twins).filter(([, file]) => fs.existsSync(file)) + if (present.length < 2) { + if (REQUIRE_SIBLINGS) + throw new Error('XCHAIN_REQUIRE_SIBLINGS=1 but a runner twin is missing: ' + Object.values(twins).join(', ')) + return this.skip() // sibling repos not checked out + } + const ours = modeRule(fs.readFileSync(MODE_TWINS.node, 'utf8'), MODE_TWINS.node) + for (const [, file] of present) + expect(modeRule(fs.readFileSync(file, 'utf8'), file), file).to.deep.equal(ours) }) }) } diff --git a/test/unit/migration_precondition_service.test/02_migration_inventory.test.js b/test/unit/migration_precondition_service.test/02_migration_inventory.test.js index dec28174..d6f88e59 100644 --- a/test/unit/migration_precondition_service.test/02_migration_inventory.test.js +++ b/test/unit/migration_precondition_service.test/02_migration_inventory.test.js @@ -19,15 +19,20 @@ function registerPendingManualMigrations({ fs, os, path, expect, pendingManualMi fs.writeFileSync(path.join(dir, 'a-manual.sql'), TAGGED) fs.writeFileSync(path.join(dir, 'b-auto.sql'), '-- xchain:migration mode=auto\nSELECT 1;\n') fs.writeFileSync(path.join(dir, 'c-manual.sql'), '-- xchain:migration mode=manual\nSELECT 1;\n') + // The runners gate an untagged or misspelt file as manual, so an unscoped run applies it. + fs.writeFileSync(path.join(dir, 'd-untagged.sql'), '-- plain comment\nSELECT 1;\n') + fs.writeFileSync(path.join(dir, 'e-typo.sql'), '-- xchain:migration mode=manually\nSELECT 1;\n') }) afterEach(() => { fs.rmSync(dir, { recursive: true, force: true }) }) it('lists only manual migrations the ledger has not recorded', () => { - expect(pendingManualMigrations(dir, new Set())).to.deep.equal(['a-manual.sql', 'c-manual.sql']) + expect(pendingManualMigrations(dir, new Set())) + .to.deep.equal(['a-manual.sql', 'c-manual.sql', 'd-untagged.sql', 'e-typo.sql']) }) it('excludes what the ledger already carries', () => { - expect(pendingManualMigrations(dir, new Set(['a-manual.sql']))).to.deep.equal(['c-manual.sql']) + expect(pendingManualMigrations(dir, new Set(['a-manual.sql']))) + .to.deep.equal(['c-manual.sql', 'd-untagged.sql', 'e-typo.sql']) }) it('yields nothing for a missing directory rather than throwing', () => { diff --git a/test/unit/module_operations.test/03_module_controls.test.js b/test/unit/module_operations.test/03_module_controls.test.js index c3f33b8e..9fb2a82a 100644 --- a/test/unit/module_operations.test/03_module_controls.test.js +++ b/test/unit/module_operations.test/03_module_controls.test.js @@ -221,7 +221,11 @@ describe('moduleOperations', function () { expect(ok).to.be.true expect(stubs.db.getModuleContainer.calledWith('xchain-decoder', 'bitcoin', 'mainnet')).to.be.true expect(stubs.execContainer.calledWith('container-id-123', - ['node', 'src/clear_reorg_halt.js', '--reason', REASON])).to.be.true + ['node', 'src/clear_reorg_halt.js', '--reason=' + REASON])).to.be.true + // A reason that starts with '-' travels as one argument, so the decoder cannot read it as a flag. + await ops.clearDecoderReorgHalt({ bitcoin: { mainnet: ['xchain-decoder'] } }, { reason: '-- verified against replica' }) + expect(stubs.execContainer.secondCall.args[1]).to.deep.equal( + ['node', 'src/clear_reorg_halt.js', '--reason=-- verified against replica']) }) it('passes --force and --dry-run through', async function () { @@ -229,7 +233,7 @@ describe('moduleOperations', function () { const ops = loadOperations(stubs) await ops.clearDecoderReorgHalt({ bitcoin: { mainnet: ['xchain-decoder'] } }, { reason: REASON, force: true, dryRun: true }) expect(stubs.execContainer.firstCall.args[1]).to.deep.equal( - ['node', 'src/clear_reorg_halt.js', '--reason', REASON, '--force', '--dry-run']) + ['node', 'src/clear_reorg_halt.js', '--reason=' + REASON, '--force', '--dry-run']) }) it('refuses a trivial reason without touching any container', async function () { @@ -272,7 +276,12 @@ describe('moduleOperations', function () { it('reports false when the script refuses (non-zero exit) and prints its text', async function () { const stubs = makeStubs() - stubs.execContainer.rejects(Object.assign(new Error('exit 4'), { stderr: 'clear-reorg-halt: REFUSED. dispenser state' })) + // The shape execContainer rejects with: execFile's error plus the child's attached output. + stubs.execContainer.rejects(Object.assign(new Error('Command failed: docker exec -i container-id-123 node src/clear_reorg_halt.js'), { + code: 4, + stdout: 'clear-reorg-halt: live REORG_HALT marker since 2026-09-17T00:00:00Z: reorg at 900000\n', + stderr: 'clear-reorg-halt: REFUSED. dispenser state\n' + })) const ops = loadOperations(stubs) const logged = [] const orig = console.log @@ -281,7 +290,9 @@ describe('moduleOperations', function () { try { ok = await ops.clearDecoderReorgHalt({ bitcoin: { mainnet: ['xchain-decoder'] } }, { reason: REASON }) } finally { console.log = orig } expect(ok).to.be.false - expect(logged.some(l => /REFUSED/.test(l))).to.be.true + expect(logged.some(l => /live REORG_HALT marker since/.test(l) && /REFUSED/.test(l))).to.be.true + expect(logged.some(l => /Command failed/.test(l))).to.be.false + expect(stubs.execContainer.calledOnce).to.be.true }) it('reports false when no decoder container is installed for the target', async function () { @@ -310,3 +321,47 @@ describe('moduleOperations', function () { }) }) }) + +// Runs a clear whose tool exec fails with exit 1, the probe answering with probeResult. +async function clearWithMissingTool(probeResult) { + const stubs = makeStubs() + const missing = Object.assign(new Error('Command failed: docker exec'), { + code: 1, stdout: '', stderr: "Error: Cannot find module '/app/src/clear_reorg_halt.js'" + }) + stubs.execContainer.onFirstCall().rejects(missing) + if (probeResult === 'present') stubs.execContainer.onSecondCall().resolves('') + else stubs.execContainer.onSecondCall().rejects(Object.assign(new Error('probe'), { code: probeResult })) + const ops = loadOperations(stubs) + const logged = [] + const orig = console.log + console.log = (l) => logged.push(String(l)) + let ok + try { ok = await ops.clearDecoderReorgHalt({ bitcoin: { mainnet: ['xchain-decoder'] } }, { reason: REASON }) } + finally { console.log = orig } + return { ok, logged, stubs } +} + +describe('moduleOperations', function () { + registerLifecycleHooks(() => {}) + + describe('clearDecoderReorgHalt(): decoder without the clear tool', function () { + + it('names a decoder update when the container has no src/clear_reorg_halt.js', async function () { + const { ok, logged, stubs } = await clearWithMissingTool(3) + expect(ok).to.be.false + expect(stubs.execContainer.secondCall.args[1].slice(0, 2)).to.deep.equal(['node', '-e']) + expect(stubs.execContainer.secondCall.args[1][2]).to.include('"src/clear_reorg_halt.js"') + expect(logged.some(l => /Cannot find module/.test(l))).to.be.true + expect(logged.some(l => /predates decoder v0\.19\.0; update the decoder/.test(l))).to.be.true + }) + + it('adds no update hint when the probe finds the tool or cannot tell', async function () { + for (const probeResult of ['present', 1]) { + const { ok, logged, stubs } = await clearWithMissingTool(probeResult) + expect(ok).to.be.false + expect(stubs.execContainer.calledTwice).to.be.true + expect(logged.some(l => /update the decoder/.test(l))).to.be.false + } + }) + }) +}) diff --git a/test/unit/module_operations.test/11_reset_external_db_reachability.test.js b/test/unit/module_operations.test/11_reset_external_db_reachability.test.js index f5203f40..62135dfb 100644 --- a/test/unit/module_operations.test/11_reset_external_db_reachability.test.js +++ b/test/unit/module_operations.test/11_reset_external_db_reachability.test.js @@ -135,3 +135,77 @@ describe('moduleOperations', function () { }) }) }) + +// A docker-mode DB reset first needed the MariaDB root password inside resetDatabases, after the +// node datadir and tracker volume were already wiped, so a root it could not resolve left a half-reset stack down. + +// Every `docker run --rm -v :/data` wipe this reset issued. +function rootPrecheckWipeRuns(execFileStub) { + return execFileStub.getCalls() + .filter(c => c.args[0] === 'docker' && Array.isArray(c.args[1]) && c.args[1][0] === 'run') +} + +// Run a reset with console.log captured, fake timers carrying it past the restart pass. +async function runRootPrecheckReset(stubs, args, constantsOverrides = null) { + const ops = loadOperations(stubs, constantsOverrides) + const lines = [] + const logStub = sinon.stub(console, 'log').callsFake((...a) => lines.push(a.join(' '))) + const clock = sinon.useFakeTimers() + try { + const promise = ops.resetModules(...args) + await clock.tickAsync(20000) + return { result: await promise, output: lines.join('\n') } + } finally { + clock.restore() + logStub.restore() + } +} + +describe('moduleOperations', function () { + registerLifecycleHooks(() => {}) + + describe('resetModules()', function () { + describe('docker-mode MariaDB root password precheck', function () { + + it('aborts before anything is stopped or wiped when root cannot be resolved', async function () { + const stubs = makeStubs() + stubs.execFile.callsFake((cmd, args, cb) => cb(null, '', '')) + stubs.askMariadbRootPassword.rejects(new Error('MariaDB root password required but no TTY to prompt on')) + const { result, output } = await runRootPrecheckReset(stubs, ['all', 'bitcoin', 'mainnet', true]) + expect(result).to.equal(false) + expect(stubs.stopContainer.called).to.equal(false) + expect(stubs.resetDatabases.called).to.equal(false) + expect(rootPrecheckWipeRuns(stubs.execFile)).to.be.empty + expect(output).to.include('cannot resolve the MariaDB root password') + expect(output).to.include('No data was touched.') + }) + + it('resolves root before the first stop when it succeeds', async function () { + const stubs = makeStubs() + stubs.execFile.callsFake((cmd, args, cb) => cb(null, '', '')) + const { result } = await runRootPrecheckReset(stubs, ['all', 'bitcoin', 'mainnet', true]) + expect(result).to.equal(true) + expect(stubs.askMariadbRootPassword.calledOnceWith('bitcoin', 'mainnet')).to.equal(true) + expect(stubs.askMariadbRootPassword.calledBefore(stubs.stopContainer)).to.equal(true) + }) + + it('asks for no root password when no database is being reset', async function () { + const stubs = makeStubs() + stubs.execFile.callsFake((cmd, args, cb) => cb(null, '', '')) + const { result } = await runRootPrecheckReset(stubs, ['xchain-utxo-tracker', 'bitcoin', 'mainnet', true]) + expect(result, 'the tracker-only reset must run to completion').to.equal(true) + expect(stubs.stopContainer.called).to.equal(true) + expect(stubs.askMariadbRootPassword.called).to.equal(false) + }) + + it('leaves EXTERNAL_DB mode on its own reachability probe', async function () { + const stubs = makeStubs() + stubs.execFile.callsFake((cmd, args, cb) => cb(null, '', '')) + const { result } = await runRootPrecheckReset(stubs, ['all', 'bitcoin', 'mainnet', true], { EXTERNAL_DB: true }) + expect(result).to.equal(true) + expect(stubs.pingExternalDatabase.called).to.equal(true) + expect(stubs.askMariadbRootPassword.called).to.equal(false) + }) + }) + }) +}) diff --git a/test/unit/module_operations.test/17_module_error_paths.test.js b/test/unit/module_operations.test/17_module_error_paths.test.js index eeef1d0b..ff57812a 100644 --- a/test/unit/module_operations.test/17_module_error_paths.test.js +++ b/test/unit/module_operations.test/17_module_error_paths.test.js @@ -191,3 +191,71 @@ describe('moduleOperations', function () { }) }) }) + +// The post-reset bounce of decoder + indexer is the last thing reset does to them, so a +// restart failure swallowed there leaves a freshly wiped service stopped while reset exits 0. + +// Run a joint decoder/indexer reset to completion and return its outcome, settled or thrown. +// `duringSettle` runs while the bounce waits out its 5s settle, after the start pass is done. +async function runBounceReset(stubs, { duringSettle = () => {} } = {}) { + const ops = loadOperations(stubs) + const clock = sinon.useFakeTimers() + const warn = sinon.stub(console, 'warn') + try { + const promise = ops.resetModules('xchain-decoder', 'bitcoin', 'mainnet', true, true) + .then((value) => ({ value }), (error) => ({ error })) + await clock.tickAsync(1000) + duringSettle() + await clock.tickAsync(20000) + return await promise + } finally { + clock.restore() + warn.restore() + } +} + +describe('moduleOperations', function () { + registerLifecycleHooks(() => {}) + + describe('resetModules()', function () { + describe('the post-reset decoder/indexer bounce', function () { + + it('reports a bounce restart that fails twice instead of returning success', async function () { + const stubs = makeStubs() + stubs.execFile.callsFake((cmd, args, cb) => cb(null, '', '')) + stubs.restartContainer.rejects(new Error('docker restart: container failed to start')) + const outcome = await runBounceReset(stubs) + expect(outcome.error, 'reset must not resolve with a bounced service down').to.be.an('error') + expect(outcome.error.message).to.match(/failed to restart/) + expect(outcome.error.message).to.match(/post-reset restart failed \(docker restart: container failed to start\)/) + // One try plus one retry per bounced module (decoder and indexer). + expect(stubs.restartContainer.callCount).to.equal(4) + }) + + it('recovers silently when the retry succeeds', async function () { + const stubs = makeStubs() + stubs.execFile.callsFake((cmd, args, cb) => cb(null, '', '')) + stubs.restartContainer.onCall(0).rejects(new Error('transient')) + stubs.restartContainer.resolves(true) + const outcome = await runBounceReset(stubs) + expect(outcome.error).to.equal(undefined) + expect(outcome.value).to.equal(true) + }) + + it('reports a registry read that fails at bounce time rather than skipping it as not installed', async function () { + const stubs = makeStubs() + stubs.execFile.callsFake((cmd, args, cb) => cb(null, '', '')) + let bouncing = false + stubs.db.getModuleContainerStrict.callsFake(async () => { + if (bouncing) throw new Error('ER_LOCK_WAIT_TIMEOUT') + return 'container-id-123' + }) + const outcome = await runBounceReset(stubs, { duringSettle: () => { bouncing = true } }) + expect(stubs.startContainer.called, 'the start pass must finish before the flip').to.equal(true) + expect(outcome.error).to.be.an('error') + expect(outcome.error.message).to.match(/registry lookup failed before the post-reset bounce/) + expect(stubs.restartContainer.called).to.equal(false) + }) + }) + }) +}) diff --git a/test/unit/module_operations.test/helpers/harness.js b/test/unit/module_operations.test/helpers/harness.js index 7e722555..1e2e9e73 100644 --- a/test/unit/module_operations.test/helpers/harness.js +++ b/test/unit/module_operations.test/helpers/harness.js @@ -71,6 +71,8 @@ function makeStubs() { saveContainerLogs: sinon.stub().resolves(true), buildDatabaseModule: sinon.stub().resolves(true), resetDatabases: sinon.stub().resolves(true), + // Docker-mode reset resolves root before any stop; resolves by default so only the tests about it fail it. + askMariadbRootPassword: sinon.stub().resolves('stub-root'), clearHubPriceIngestWatermark: sinon.stub().resolves(true), // The regtest re-genesis purge of the hub's cross-chain relic rows. The // statements helper feeds the never-fatal catch's operator message. @@ -158,6 +160,7 @@ function loadOperations(stubs, constantsOverrides = null) { '../services/database_service': { buildDatabaseModule: stubs.buildDatabaseModule, resetDatabases: stubs.resetDatabases, + askMariadbRootPassword: stubs.askMariadbRootPassword, clearHubPriceIngestWatermark: stubs.clearHubPriceIngestWatermark, purgeHubCrossChainRows: stubs.purgeHubCrossChainRows, manualHubCrossChainPurgeStatements: stubs.manualHubCrossChainPurgeStatements, diff --git a/test/unit/stop_budget_service.test.js b/test/unit/stop_budget_service.test.js index a0e4075a..53f5dbba 100644 --- a/test/unit/stop_budget_service.test.js +++ b/test/unit/stop_budget_service.test.js @@ -232,3 +232,49 @@ describe('StopBudgetService', function () { }) }) }) + +// An explicit drain timer the node cannot derive must still land under the budget. +describe('StopBudgetService', function () { + beforeEach(prepareConsoleStubs) + afterEach(restoreConsoleStubs) + + const DECODER_300 = { XCHAIN_NODE_MODULE_STOP_TIMEOUT_SECONDS_XCHAIN_DECODER: '300' } + const overLine = () => warnStub.args.map(a => String(a[0])).find(l => /at or above/.test(l)) + const cleanStop = () => sinon.stub().resolves({ stopped: true, seconds: 5, killed: false, exitCode: 0 }) + + it('keeps an explicit value at or above the budget but warns that docker will kill first', function () { + expect(sbs.shutdownTimeoutEnv('xchain-decoder', { SHUTDOWN_TIMEOUT_MS: '150000' }, {})).to.deep.equal({}) + expect(overLine()).to.match(/xchain-decoder: the module config sets SHUTDOWN_TIMEOUT_MS=150000, at or above its 120 s stop budget/) + expect(overLine()).to.match(/XCHAIN_NODE_MODULE_STOP_TIMEOUT_SECONDS_XCHAIN_DECODER/) + warnStub.resetHistory() + sbs.shutdownTimeoutEnv('xchain-decoder', { SHUTDOWN_TIMEOUT_MS: '120000' }, {}) + expect(overLine(), 'exactly at the budget').to.match(/120000/) + warnStub.resetHistory() + const lowered = { XCHAIN_NODE_MODULE_STOP_TIMEOUT_SECONDS_XCHAIN_DECODER: '60' } + sbs.shutdownTimeoutEnv('xchain-decoder', { SHUTDOWN_TIMEOUT_MS: '100000' }, lowered) + expect(overLine(), 'a budget lowered under a pinned value').to.match(/60 s stop budget/) + warnStub.resetHistory() + sbs.shutdownTimeoutEnv('xchain-explorer', { SHUTDOWN_TIMEOUT_MS: '45000' }, {}) + expect(overLine(), 'a 30 s default-budget service').to.match(/xchain-explorer.*30 s stop budget/) + }) + + it('stays quiet for an explicit value under the budget or one the service would ignore', function () { + sbs.shutdownTimeoutEnv('xchain-decoder', { SHUTDOWN_TIMEOUT_MS: '119000' }, {}) + sbs.shutdownTimeoutEnv('xchain-decoder', { SHUTDOWN_TIMEOUT_MS: '45000' }, DECODER_300) + sbs.shutdownTimeoutEnv('xchain-explorer', { SHUTDOWN_TIMEOUT_MS: '8000' }, {}) + sbs.shutdownTimeoutEnv('xchain-decoder', { SHUTDOWN_TIMEOUT_MS: 'slow' }, {}) + expect(warnStub.called).to.be.false + }) + + it('warns before the stop when the container carries a drain timer at or above its budget', async function () { + const read = sinon.stub().resolves({ stopTimeout: 120, shutdownTimeoutMs: '150000' }) + const stop = cleanStop() + await sbs.stopModuleContainer(stop, 'xchain-decoder', 'bitcoin', 'mainnet', 'abc123', {}, read) + expect(read.calledBefore(stop)).to.be.true + expect(overLine()).to.match(/xchain-decoder \(bitcoin mainnet\): the container carries SHUTDOWN_TIMEOUT_MS=150000, at or above its 120 s stop budget/) + warnStub.resetHistory() + const explorer = sinon.stub().resolves({ stopTimeout: 30, shutdownTimeoutMs: '45000' }) + await sbs.stopModuleContainer(cleanStop(), 'xchain-explorer', 'bitcoin', 'mainnet', 'abc123', {}, explorer) + expect(overLine()).to.match(/xchain-explorer \(bitcoin mainnet\).*30 s stop budget/) + }) +}) diff --git a/test/unit/utils/migration_files.test.js b/test/unit/utils/migration_files.test.js index 305ff545..d0aa0472 100644 --- a/test/unit/utils/migration_files.test.js +++ b/test/unit/utils/migration_files.test.js @@ -5,11 +5,11 @@ // // SPDX-License-Identifier: AGPL-3.0-or-later // -// The deploy-precondition guard reads migrations out of a cloned module tree -// whose layout depends on the ref: flat src/sql/migrations/ for the decoder and -// for indexer refs cut before the SQL home moved, bucketed src/db/sql/migrations/ -// after. A reader that assumes one layout reads an empty directory on the other -// and waves a gated deploy through, so both layouts are pinned here. +// The deploy-precondition guard reads migrations out of a cloned module tree. +// The indexer and decoder keep them flat under src/sql/migrations/ today; the +// bucketed src/db/sql/migrations/ layout is pinned as forward-compat, so a reader +// that assumed one layout never reads an empty directory and waves a gated deploy +// through if the indexer moves. const fs = require('fs') const os = require('os') @@ -42,7 +42,7 @@ describe('migration_files', () => { expect(migrationsDirOf(root)).to.equal(path.join(root, 'src', 'db', 'sql', 'migrations')) }) - it('falls back to the flat home for a tree cut before the move, or the decoder', () => { + it('reads the flat home the indexer and decoder use today', () => { write(root, 'src/sql/migrations/2026-05-30-a.sql', AUTO) expect(migrationsDirOf(root)).to.equal(path.join(root, 'src', 'sql', 'migrations')) }) diff --git a/test/unit/validator_sdk_encoder_compat.test.js b/test/unit/validator_sdk_encoder_compat.test.js index eb89c8e7..6838f0f7 100644 --- a/test/unit/validator_sdk_encoder_compat.test.js +++ b/test/unit/validator_sdk_encoder_compat.test.js @@ -107,7 +107,9 @@ function makeExplorer(chain) { return { getAddress: sinon.stub().resolves({ balances: { confirmed: chain.coin, pending: '0' } }), getToken: sinon.stub().resolves({ mints: { max: 10000, address_max: 50000 } }), - getValidators: sinon.stub().resolves({ data: [] }) + getValidators: sinon.stub().resolves({ data: [] }), + // No delegation holds the key; without an answer the stake command refuses to send. + getDelegations: sinon.stub().resolves({ total: 0, data: [] }) } } diff --git a/test/unit/validator_stake_service.test/free_key.test.js b/test/unit/validator_stake_service.test/free_key.test.js new file mode 100644 index 00000000..1164e473 --- /dev/null +++ b/test/unit/validator_stake_service.test/free_key.test.js @@ -0,0 +1,126 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +// The free-key rules the stake CLI mirrors from the indexer, and the reuse +// gate heights' agreement with the indexer row they copy. + +const fs = require('fs') +const path = require('path') +const { expect } = require('chai') + +const { + STAKE_KEY_REUSE_ACTIVATION, stakeKeyReuseHeight, stakeRowsHoldingKey, + delegationRowsHoldingKey, sumAmounts, readDelegationsByPubkey +} = require('../../../src/services/validator_stake_service/free_key') + +describe('validator stake free-key rules', function () { + + it('resolves the reuse gate COIN:network first, then the bare network', () => { + expect(stakeKeyReuseHeight('testnet', 'BTC')).to.equal(156000) + expect(stakeKeyReuseHeight('testnet', 'LTC')).to.equal(4897000) + expect(stakeKeyReuseHeight('regtest', 'BTC')).to.equal(0) + expect(stakeKeyReuseHeight('mainnet', 'BTC')).to.equal(null) + expect(stakeKeyReuseHeight('testnet', 'XYZ')).to.equal(null) + expect(stakeKeyReuseHeight('constructor', null)).to.equal(null) + }) + + it('before the gate every valid row holds the key, withdrawn or not', () => { + const rows = [{ deactivation_block: '10' }, { deactivation_block: null }] + expect(stakeRowsHoldingKey(rows, { reuseActive: false, landBlock: 99999, cooldownBlocks: 1000 })).to.have.length(2) + }) + + it('after the gate a row releases the key at deactivation_block + cooldown, not a block before', () => { + const opts = (landBlock) => ({ reuseActive: true, landBlock, cooldownBlocks: 1000 }) + const rows = [{ deactivation_block: '5000' }] + expect(stakeRowsHoldingKey(rows, opts(5999))).to.have.length(1) + expect(stakeRowsHoldingKey(rows, opts(6000))).to.have.length(0) + expect(stakeRowsHoldingKey([{ deactivation_block: null }], opts(1e9))).to.have.length(1) + expect(stakeRowsHoldingKey([{ deactivation_block: 'junk' }], opts(1e9)), 'unreadable holds').to.have.length(1) + }) + + it('a delegation holds the key until its deactivation block has passed', () => { + const d = (deactivation_block, status = 'valid') => [{ status, deactivation_block }] + expect(delegationRowsHoldingKey(d(null), 10)).to.have.length(1) + expect(delegationRowsHoldingKey(d('11'), 10)).to.have.length(1) + expect(delegationRowsHoldingKey(d('10'), 10)).to.have.length(0) + expect(delegationRowsHoldingKey(d('10'), null), 'no tip means the refusing side').to.have.length(1) + expect(delegationRowsHoldingKey(d(null, 'invalid: x'), 10)).to.have.length(0) + }) + + it('sums amounts exactly in 1e-8 units and refuses to guess on a malformed one', () => { + expect(sumAmounts([{ amount: '0.1' }, { amount: '0.2' }])).to.equal('0.3') + expect(sumAmounts([{ amount: '12345678.12345678' }, { amount: '87654321.87654321' }])).to.equal('99999999.99999999') + expect(sumAmounts([{ amount: '0.00000001' }])).to.equal('0.00000001') + expect(sumAmounts([{ amount: '900000000.00000001' }, { amount: '0.00000001' }])).to.equal('900000000.00000002') + expect(sumAmounts([{ amount: '25000' }, { amount: '1e3' }])).to.equal(null) + }) + + it('pages the pubkey lookup to the end and refuses a response with no row list', async () => { + const pubkey = 'ab'.repeat(32) + const page = (n) => Array.from({ length: n }, (_, i) => ({ status: 'valid', signing_pubkey: pubkey, action_index: String(i) })) + const calls = [] + const sdk = { explorer: { getDelegations: async (q, t, o) => { + calls.push(o.page) + return o.page === 1 ? { total: 101, data: page(100) } : { total: 101, data: page(1) } + } } } + expect(await readDelegationsByPubkey(sdk, pubkey)).to.have.length(101) + expect(calls).to.deep.equal([1, 2]) + const bad = { explorer: { getDelegations: async () => ({ error: 'no route' }) } } + let err = null + try { await readDelegationsByPubkey(bad, pubkey) } catch (e) { err = e } + expect(err && err.message).to.include('no row list') + }) +}) + +// The heights are a copy of consensus data the CLI only reads. With the +// sibling indexer checkout present, hold the copy to the indexer's +// stake_key_reuse_activation row; a standalone deploy skips, and +// XCHAIN_REQUIRE_SIBLINGS=1 turns every miss into a failure. +describe('stake_key_reuse_activation mirror agrees with the xchain-indexer gate row', function () { + const INDEXER_DIR = process.env.XCHAIN_INDEXER_DIR || path.join(__dirname, '..', '..', '..', '..', 'xchain-indexer') + const PROTOCOL_CHANGES_DIR = path.join(INDEXER_DIR, 'src', 'protocol_changes') + const REQUIRE_SIBLINGS = process.env.XCHAIN_REQUIRE_SIBLINGS === '1' + const ROW = "addGate('stake_key_reuse_activation.STAKE_KEY_REUSE_ACTIVATION'" + let body = null + + before(function () { + const miss = (why) => { + if (REQUIRE_SIBLINGS) throw new Error('XCHAIN_REQUIRE_SIBLINGS=1 but ' + why) + this.skip() + } + if (!fs.existsSync(PROTOCOL_CHANGES_DIR)) return miss(PROTOCOL_CHANGES_DIR + ' is absent') + // Find the row in whichever part file carries it, so a file split does not skip the check. + const src = fs.readdirSync(PROTOCOL_CHANGES_DIR).filter(f => f.endsWith('.js')) + .map(f => fs.readFileSync(path.join(PROTOCOL_CHANGES_DIR, f), 'utf8')) + .find(s => s.includes(ROW)) + if (!src) return miss('no ' + ROW + ", ...) row was found under " + PROTOCOL_CHANGES_DIR) + const start = src.indexOf('{', src.indexOf(ROW)) + body = src.slice(start + 1, src.indexOf('}', start)) + }) + + // Read the literals from the source text: requiring the registry would resolve it, + // and a mismatch there is exactly what this test exists to catch. + function indexerRow() { + const row = {} + for (const line of body.split('\n')) { + const m = /^\s*'?([A-Za-z]+(?::[a-z]+)?)'?\s*:\s*(null|\d+)\s*,?/.exec(line) + if (m) row[m[1]] = m[2] === 'null' ? null : parseInt(m[2], 10) + } + return row + } + + it('carries every key and height the indexer row carries, and nothing else', () => { + const row = indexerRow() + expect(Object.keys(row), 'parsed no keys from the indexer row').to.have.length.above(0) + expect(row).to.deep.equal({ ...STAKE_KEY_REUSE_ACTIVATION }) + }) +}) diff --git a/test/unit/validator_stake_service.test/send_lock.test.js b/test/unit/validator_stake_service.test/send_lock.test.js new file mode 100644 index 00000000..179a4ded --- /dev/null +++ b/test/unit/validator_stake_service.test/send_lock.test.js @@ -0,0 +1,253 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. +// +// Pins the send-phase command lock on `validator stake|unstake --broadcast`: a +// second broadcast is refused while a send is in flight, and the indexer wait +// that follows holds nothing. Drives the real file lock in a scratch directory. + +const fs = require('fs') +const os = require('os') +const path = require('path') +const sinon = require('sinon') +const { expect } = require('chai') + +const { stakeValidator, unstakeValidator } = require('../../../src/services/validator_stake_service') +const { acquireCommandLock } = require('../../../src/utils/command_lock') +const { validatorSendLock } = require('../../../src/cli/dispatch') +const { COIN_NETWORKS } = require('../../../src/services/validator_service') + +const PUBKEY = 'ab'.repeat(32) +const ADDRESS = 'mStakeAddress' +const STAKED = { status: 'valid', signing_pubkey: PUBKEY, amount: '25000', action_index: '44', activation_block: '150313' } + +let lockDir + +// Read who holds the lock file right now, or null when it is free. +function lockHolder() { + const file = path.join(lockDir, 'command.lock') + return fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, 'utf8')).command : null +} + +// Try the lock as another command would; true when it was free and is handed straight back. +function otherCommandGetsLock(command = 'update') { + try { + acquireCommandLock({ command })() + return true + } catch (err) { + if (err.code === 'ELOCKHELD') return false + throw err + } +} + +function makeExplorer(chain) { + return { + getAddress: sinon.stub().resolves({ balances: { confirmed: '0.001', pending: '0' } }), + getToken: sinon.stub().resolves({ mints: { max: 10000, address_max: 50000 } }), + getValidators: sinon.stub().resolves({ data: chain.existing ? [chain.existing] : [] }), + getDelegations: sinon.stub().resolves({ total: 0, data: [] }), + getStatus: sinon.stub().resolves({ last_block: Object.fromEntries( + Object.values(COIN_NETWORKS).map(c => [c.stake, 200000])) }) + } +} + +// A fake SDK whose submit follows the real lifecycle order: broadcast, then the +// 'waiting' progress step, then the indexer wait. Hooks run inside each phase. +function makeSdk(chain, hooks = {}) { + const sent = [] + let prevTxid = null + const sdk = { + explorer: makeExplorer(chain), + getBalances: async () => ({ data: [{ tick: 'XCHAIN', amount: String(chain.xchain || 0) }] }), + requireEncoder: () => ({ getUTXOs: async () => ({ utxos: [ + { txid: prevTxid || 'seed', fullTxid: prevTxid || 'seed', vout: 1, value: '150000', confirmations: 0 }] }) }), + session: () => ({ + address: ADDRESS, + submit: async (actionData, enc, opts) => { + const txid = actionData.action.toLowerCase() + (sent.length + 1) + const progress = opts.onProgress || (() => {}) + progress('broadcasting', { txid }) + if (hooks.duringSend) await hooks.duringSend(actionData.action) + sent.push({ action: actionData.action, txid }) + prevTxid = txid + if (opts.waitForIndexer) { + progress('waiting', { txid }) + if (hooks.duringWait) await hooks.duringWait(actionData.action) + } + return { txid, spentInputs: (enc.utxos || []).map(u => ({ txid: u.txid, vout: u.vout })) } + } + }) + } + return { sdk, sent } +} + +function runDeps(sdk, command) { + return { + settings: { enabled: true, pubkey: PUBKEY, network: 'testnet', P2P_PORT: 10002 }, + wallets: { NETWORK: 'testnet', STAKE_ADDRESS: ADDRESS, STAKE_WIF_SECRET: 'cFakeWif' }, + makeSdk: () => sdk, + sdk: {}, + log: () => {}, + sendLock: validatorSendLock(command, { acquireCommandLock, config: {} }) + } +} + +function stake(opts, chain, hooks) { + const { sdk, sent } = makeSdk(chain, hooks) + return stakeValidator(opts, runDeps(sdk, 'validator stake')).then(result => ({ result, sent })) +} + +function unstake(opts, chain, hooks) { + const { sdk, sent } = makeSdk(chain, hooks) + return unstakeValidator(opts, runDeps(sdk, 'validator unstake')).then(result => ({ result, sent })) +} + +// Run a second broadcast from inside the first and report how it ended and what it sent. +async function secondRun(run, command, chain) { + const { sdk, sent } = makeSdk(chain) + try { + await run({ broadcast: true }, runDeps(sdk, command)) + return { refused: false, sent } + } catch (err) { + return { refused: err.code === 'ELOCKHELD', message: err.message, sent } + } +} + +function useScratchLockDir() { + beforeEach(function () { + lockDir = fs.mkdtempSync(path.join(os.tmpdir(), 'xchain-node-sendlock-')) + process.env.XCHAIN_NODE_LOCK_DIR = lockDir + }) + afterEach(function () { + delete process.env.XCHAIN_NODE_LOCK_DIR + fs.rmSync(lockDir, { recursive: true, force: true }) + sinon.restore() + }) +} + +describe('validator stake --broadcast: send-phase command lock', function () { + useScratchLockDir() + + it('refuses a second stake --broadcast while the first is sending, and it sends nothing', async function () { + let second = null + let holder = null + const { result } = await stake({ broadcast: true }, { xchain: 30000 }, { + duringSend: async () => { + holder = lockHolder() + second = await secondRun(stakeValidator, 'validator stake', { xchain: 30000 }) + } + }) + expect(result.staked).to.be.true + expect(second.sent, 'the second run sent').to.deep.equal([]) + expect(second.refused, second.message).to.be.true + expect(holder).to.equal('validator stake') + expect(second.message).to.include('running "validator stake"') + expect(lockHolder()).to.equal(null) + }) + + it('hands the lock back for the STAKE\'s index wait, so another command runs meanwhile', async function () { + const seen = {} + await stake({ broadcast: true }, { xchain: 30000 }, { + duringSend: async () => { seen.send = otherCommandGetsLock() }, + duringWait: async () => { seen.wait = otherCommandGetsLock() } + }) + expect(seen.send, 'another command during the send').to.be.false + expect(seen.wait, 'another command during the index wait').to.be.true + }) + + it('holds the lock across every chained mint and the STAKE, which send without a wait between', async function () { + const during = [] + const { sent } = await stake({ broadcast: true }, { xchain: 0 }, { + duringSend: async (action) => { during.push(action + ':' + otherCommandGetsLock()) } + }) + expect(sent.map(s => s.action)).to.deep.equal(['MINT', 'MINT', 'MINT', 'STAKE']) + expect(during).to.deep.equal(['MINT:false', 'MINT:false', 'MINT:false', 'STAKE:false']) + }) +}) + +describe('validator stake --broadcast: send lock across waits, dry runs and early stops', function () { + useScratchLockDir() + + it('--serialize retakes the lock after each mint\'s wait and stops, naming what went out, if it is taken', async function () { + let taken = null + let failure = null + try { + await stake({ broadcast: true, serialize: true }, { xchain: 0 }, { + duringWait: async (action) => { + if (action === 'MINT' && !taken) taken = acquireCommandLock({ command: 'update' }) + } + }) + } catch (err) { + failure = err + } finally { + if (taken) taken() + } + expect(failure, 'the run must stop at the second mint').to.not.equal(null) + expect(failure.code).to.equal('ELOCKHELD') + expect(failure.message).to.include('Stopped before the MINT; already broadcast: MINT mint1') + }) + + it('a dry run takes no lock, so it plans while a deploy holds it', async function () { + const release = acquireCommandLock({ command: 'update' }) + try { + const { result } = await stake({}, { xchain: 30000 }) + expect(result.dryRun).to.be.true + } finally { + release() + } + }) + + it('releases the lock and its exit handler when the run stops short of sending', async function () { + const exitListeners = process.listeners('exit').length + const { result } = await stake({ broadcast: true }, { xchain: 30000, existing: STAKED }) + expect(result.staked).to.be.false + expect(lockHolder()).to.equal(null) + expect(process.listeners('exit').length).to.equal(exitListeners) + }) +}) + +describe('validator unstake --broadcast: send-phase command lock', function () { + useScratchLockDir() + + it('refuses a second unstake --broadcast while the first is sending, and it sends nothing', async function () { + let second = null + let holder = null + const { result } = await unstake({ broadcast: true }, { existing: STAKED }, { + duringSend: async () => { + holder = lockHolder() + second = await secondRun(unstakeValidator, 'validator unstake', { existing: STAKED }) + } + }) + expect(result.unstaked).to.be.true + expect(second.sent, 'the second run sent').to.deep.equal([]) + expect(second.refused, second.message).to.be.true + expect(holder).to.equal('validator unstake') + expect(lockHolder()).to.equal(null) + }) + + it('hands the lock back for the UNSTAKE\'s index wait, so another command runs meanwhile', async function () { + const seen = {} + await unstake({ broadcast: true }, { existing: STAKED }, { + duringSend: async () => { seen.send = otherCommandGetsLock() }, + duringWait: async () => { seen.wait = otherCommandGetsLock() } + }) + expect(seen.send, 'another command during the send').to.be.false + expect(seen.wait, 'another command during the index wait').to.be.true + }) + + it('--no-wait keeps the lock to the broadcast and frees it when the run returns', async function () { + const { result } = await unstake({ broadcast: true, wait: false }, { existing: STAKED }, { + duringSend: async () => { expect(otherCommandGetsLock()).to.be.false } + }) + expect(result.unstaked).to.be.true + expect(lockHolder()).to.equal(null) + }) +}) diff --git a/test/unit/validator_stake_service.test/stake_free_key_check.test.js b/test/unit/validator_stake_service.test/stake_free_key_check.test.js new file mode 100644 index 00000000..5de9c9e6 --- /dev/null +++ b/test/unit/validator_stake_service.test/stake_free_key_check.test.js @@ -0,0 +1,113 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +const { expect } = require('chai') + +const { stakeValidator } = require('../../../src/services/validator_stake_service') +const { PUBKEY, ADDRESS, makeSdk, run } = require('../../helpers/stake_harness') + +// The free-key check mirrors the indexer's validateFreeKey: withdrawn stake +// frees the key only once stake_key_reuse_activation is armed and the cooldown +// has passed, and a delegation holding the key refuses the STAKE outright. +describe('ValidatorStakeService free-key check', function () { + + const row = (over) => Object.assign({ status: 'valid', signing_pubkey: PUBKEY, amount: '25000', + action_index: '33', activation_block: '150061', deactivation_block: null }, over) + + it('stakes again on testnet past the reuse gate once every row is withdrawn and cooled down', async function () { + const stakeRows = [row({ deactivation_block: '150000' }), row({ action_index: '40', deactivation_block: '150100' })] + const { result, logged } = await run({}, { xchain: 25000, coin: '0.001', stakeRows, tip: 199999 }) + expect(result.dryRun).to.be.true + expect(logged.join('\n')).to.not.include('already carries') + }) + + it('refuses inside the cooldown and names the block the key is released at', async function () { + const stakeRows = [row({ deactivation_block: '199500' })] + const { result, calls, logged } = await run({ broadcast: true }, { xchain: 25000, coin: '0.001', stakeRows, tip: 200000 }) + expect(result.staked).to.be.false + expect(result.existing).to.equal(stakeRows[0]) + expect(calls.stake).to.have.length(0) + const out = logged.join('\n') + expect(out).to.include('stake is withdrawn') + expect(out).to.include('admitted from block 200500') + expect(out).to.not.include('already carries a valid STAKE') + }) + + it('refuses a withdrawn key on testnet below the gate height, naming that height', async function () { + const stakeRows = [row({ deactivation_block: '100000' })] + const { result, logged } = await run({}, { xchain: 25000, coin: '0.001', stakeRows, tip: 150000 }) + expect(result.existing).to.equal(stakeRows[0]) + expect(logged.join('\n')).to.include('releases a withdrawn key only from block 156000') + }) + + it('refuses a withdrawn key on mainnet, where the gate is not armed', async function () { + const stakeRows = [row({ deactivation_block: '100' })] + const { result, logged } = await run({}, { xchain: 25000, coin: '0.01', stakeRows, tip: 900000 }, + { network: 'mainnet', P2P_PORT: 10001 }) + expect(result.existing).to.equal(stakeRows[0]) + expect(logged.join('\n')).to.include('does not yet release a signing key') + }) + + it('prints the exact sum of the undeactivated rows, not one row and not a float', async function () { + const stakeRows = [ + row({ amount: '900000000.00000001' }), + row({ action_index: '41', amount: '0.00000001' }), + row({ action_index: '42', amount: '500', deactivation_block: '150000' }) + ] + const { logged } = await run({}, { xchain: 25000, coin: '0.001', stakeRows }) + expect(logged.join('\n')).to.include('already carries a valid STAKE of 900000000.00000002 XCHAIN (newest action 42') + }) +}) + +describe('ValidatorStakeService free-key check, delegations', function () { + + const row = (over) => Object.assign({ status: 'valid', signing_pubkey: PUBKEY, amount: '25000', + action_index: '33', activation_block: '150061', deactivation_block: null }, over) + + it('refuses a key an active delegation holds, before any MINT is sent', async function () { + const delegations = [{ status: 'valid', signing_pubkey: PUBKEY, source: 'mDelegator', action_index: '50', deactivation_block: null }] + const { result, calls, logged } = await run({ broadcast: true }, { xchain: 0, coin: '0.001', delegations }) + expect(result.staked).to.be.false + expect(result.delegated).to.deep.equal(delegations) + expect(calls.mint).to.have.length(0) + expect(calls.stake).to.have.length(0) + expect(logged.join('\n')).to.include('held by a delegation (action 50 from mDelegator)') + }) + + it('asks the explorer for the delegations by pubkey', async function () { + const { sdk } = makeSdk({ xchain: 25000, coin: '0.001' }) + await stakeValidator({}, { settings: { enabled: true, pubkey: PUBKEY, network: 'testnet' }, + wallets: { STAKE_ADDRESS: ADDRESS, STAKE_WIF_SECRET: 'cFakeWif' }, makeSdk: () => sdk, sdk: {}, log: () => {} }) + expect(sdk.explorer.getDelegations.firstCall.args.slice(0, 2)).to.deep.equal([PUBKEY, 'pubkey']) + }) + + it('does not count a delegation whose deactivation block has passed', async function () { + const delegations = [{ status: 'valid', signing_pubkey: PUBKEY, source: 'mDelegator', action_index: '50', deactivation_block: '1000' }] + const { result } = await run({}, { xchain: 25000, coin: '0.001', delegations, tip: 200000 }) + expect(result.dryRun).to.be.true + }) + + it('blocks, rather than passes, when the explorer cannot answer the pubkey lookup', async function () { + const { result, calls, logged } = await run({ broadcast: true }, + { xchain: 0, coin: '0.001', delegationsThrow: 'Explorer returned HTTP 404 for /TBTC/api/delegations/x/pubkey' }) + expect(result.blockers.join(' ')).to.include('could not confirm this signing pubkey is not delegated') + expect(calls.mint).to.have.length(0) + expect(logged.join('\n')).to.include('BLOCKED') + }) + + it('blocks when the pubkey lookup answers without a row list', async function () { + const { result, calls } = await run({ broadcast: true }, + { xchain: 25000, coin: '0.001', delegationsBody: { error: 'unknown type' } }) + expect(result.blockers.join(' ')).to.include('returned no row list') + expect(calls.stake).to.have.length(0) + }) +}) diff --git a/test/unit/validator_stake_service.test/stake_validator.test.js b/test/unit/validator_stake_service.test/stake_validator.test.js index 300e317a..0f988bbd 100644 --- a/test/unit/validator_stake_service.test/stake_validator.test.js +++ b/test/unit/validator_stake_service.test/stake_validator.test.js @@ -13,98 +13,7 @@ const { expect } = require('chai') const { stakeValidator } = require('../../../src/services/validator_stake_service') -const sinon = require('sinon') - -const PUBKEY = 'ab'.repeat(32) -const ADDRESS = 'mStakeAddress' - -function makeExplorer(chain) { - return { - getAddress: sinon.stub().resolves({ balances: { confirmed: chain.coin ?? '0.001', pending: '0' } }), - getToken: chain.tokenMissing - ? sinon.stub().rejects(Object.assign( - new Error('Explorer returned HTTP 404 for /RBTC/api/token/XCHAIN'), - { code: 'EXPLORER_HTTP_404', details: { status: 404 } })) - : sinon.stub().resolves({ mints: { max: chain.mintMax ?? 10000, address_max: chain.addressMax ?? 50000 } }), - // The whole validator set, which is the method the SDK actually has. - // Carries an unrelated validator too, so the pubkey filter is exercised - // rather than "the only row wins". - getValidators: chain.validatorsThrow - ? sinon.stub().rejects(new Error(chain.validatorsThrow)) - : sinon.stub().resolves({ data: [ - { status: 'valid', signing_pubkey: 'ff'.repeat(32), amount: '25000', action_index: '1', activation_block: '1' }, - ...(chain.existing ? [chain.existing] : []) - ] }) - } -} - -function trackPreviousOutput(session, prevTxidRef) { - return () => { - const s = session() - const inner = s.submit - s.submit = async (a, e, o) => { const r = await inner(a, e, o); prevTxidRef.value = r.txid; return r } - return s - } -} - -// A fake SDK shaped like the parts the command touches: explorer reads, -// balances, and a session whose mint/stake record what they were asked. -function makeSdk(chain = {}) { - const calls = { mint: [], stake: [] } - const sdk = { - explorer: makeExplorer(chain), - // The mints credit once they index. Modelled by reporting the post-mint - // balance after they have been sent, so the fallback path can finish. - getBalances: async () => { - const base = chain.xchain !== undefined ? chain.xchain : 0 - const credited = calls.mint.length && chain.mintsNeverIndex !== true - ? base + calls.mint.reduce((s, c) => s + Number(c.params.AMOUNT), 0) - : base - return { data: [{ tick: 'XCHAIN', amount: String(credited) }] } - }, - // Every action goes through session.submit; the convenience wrappers - // (mint/stake) are sugar over it, and the service calls submit directly - // so one code path handles the funding chain. - session: () => ({ - address: ADDRESS, - submit: async (actionData, enc, opts) => { - const rec = { params: actionData.params, enc, opts } - if (actionData.action === 'MINT') calls.mint.push(rec) - if (actionData.action === 'STAKE') calls.stake.push(rec) - const txid = actionData.action === 'STAKE' ? 'staketx' : 'mint' + calls.mint.length - // Report the inputs the encoder was told to use, so the chain - // assertion in the service sees a real answer. - const spentInputs = (enc.utxos || []).map(u => ({ txid: u.txid, vout: u.vout })) - return { txid, spentInputs } - } - }), - // The chain link: each broadcast leaves a change output the next action - // is funded from. Keyed by txid so the service's filter is exercised. - requireEncoder: () => ({ - getUTXOs: async () => ({ utxos: chain.noChange ? [] : [ - { txid: prevTxidRef.value || 'seed', fullTxid: prevTxidRef.value || 'seed', vout: 1, value: '150000', confirmations: 0 } - ] }) - }) - } - // The fake encoder answers with an output of whatever was broadcast last. - const prevTxidRef = { value: null } - sdk.session = trackPreviousOutput(sdk.session, prevTxidRef) - return { sdk, calls } -} - -function run(opts, chain, settingsExtra = {}) { - const { sdk, calls } = makeSdk(chain) - const logged = [] - const network = settingsExtra.network || 'testnet' - const deps = { - settings: { enabled: true, pubkey: PUBKEY, network: 'testnet', P2P_PORT: 10002, ...settingsExtra }, - wallets: { NETWORK: network, STAKE_ADDRESS: ADDRESS, STAKE_WIF_SECRET: 'cFakeWif' }, - makeSdk: () => sdk, - sdk: { XChainSDK: function () { throw new Error('makeSdk should be used') } }, - log: m => logged.push(String(m)) - } - return stakeValidator(opts, deps).then(result => ({ result, calls, logged })) -} +const { PUBKEY, ADDRESS, makeSdk, run } = require('../../helpers/stake_harness') describe('ValidatorStakeService', function () { @@ -175,7 +84,12 @@ describe('ValidatorStakeService', function () { expect(calls.mint[2].enc.utxos.map(u => u.txid)).to.deep.equal(['mint2']) expect(calls.stake[0].enc.utxos.map(u => u.txid)).to.deep.equal(['mint3']) }) + }) +}) +describe('ValidatorStakeService', function () { + + describe('stakeValidator()', function () { it('waits for the mints to index when no chain can be formed, rather than racing the STAKE', async function () { const { calls, logged, result } = await run( { broadcast: true, chainTimeoutMs: 5, balancePollMs: 5 }, @@ -186,6 +100,41 @@ describe('ValidatorStakeService', function () { expect(calls.stake[0].enc.utxos, 'funded freely once ordering stops mattering').to.be.undefined expect(result.chained).to.be.false }) + + // The STAKE's own link is the only chained hop of a single-mint run, so a + // miss there must wait the mint out rather than broadcast an unchained STAKE. + it('waits for a single mint to index when only the STAKE hop cannot chain', async function () { + const { calls, logged, result } = await run( + { broadcast: true, chainTimeoutMs: 5, balancePollMs: 5 }, + { xchain: 15000, coin: '0.001', noChangeFor: ['mint1'] }) + expect(calls.mint).to.have.length(1) + expect(logged.join('\n')).to.include('cannot chain') + expect(logged.join('\n')).to.include('the funding chain broke') + expect(calls.stake).to.have.length(1) + expect(calls.stake[0].enc.utxos, 'funded freely once the mint indexed').to.be.undefined + expect(result.staked).to.be.true + expect(result.chained).to.be.false + }) + + it('keeps the mints chained and waits when only the last MINT to STAKE hop breaks', async function () { + const { calls, logged, result } = await run( + { broadcast: true, chainTimeoutMs: 5, balancePollMs: 5 }, + { xchain: 0, coin: '0.001', noChangeFor: ['mint3'] }) + expect(calls.mint[1].enc.utxos.map(u => u.txid)).to.deep.equal(['mint1']) + expect(calls.mint[2].enc.utxos.map(u => u.txid)).to.deep.equal(['mint2']) + expect(logged.join('\n')).to.include('the funding chain broke') + expect(calls.stake[0].enc.utxos).to.be.undefined + expect(result.chained).to.be.false + }) + + it('sends no STAKE when the STAKE hop breaks and the mint never indexes', async function () { + const { calls, logged, result } = await run( + { broadcast: true, chainTimeoutMs: 5, balancePollMs: 5, timeout: 0.0001 }, + { xchain: 15000, coin: '0.001', noChangeFor: ['mint1'], mintsNeverIndex: true }) + expect(calls.stake, 'never broadcast a STAKE that can land ahead of its funding').to.have.length(0) + expect(result.pendingMints).to.be.true + expect(logged.join('\n')).to.include('re-run this command to send the STAKE') + }) }) }) @@ -326,7 +275,7 @@ describe('ValidatorStakeService', function () { it('calls explorer methods that the published SDK actually exposes', function () { const { XChainSDK } = require('@dankest-llc/xchain-sdk') const sdk = new XChainSDK({ network: 'bitcoin-testnet' }) // offline: no network I/O in the constructor - for (const m of ['getValidators', 'getAddress', 'getToken']) + for (const m of ['getValidators', 'getAddress', 'getToken', 'getDelegations', 'getStatus']) expect(sdk.explorer[m], 'sdk.explorer.' + m).to.be.a('function') for (const m of ['getBalances', 'session']) expect(sdk[m], 'sdk.' + m).to.be.a('function') diff --git a/test/unit/validator_stake_service.test/unstake_validator.test.js b/test/unit/validator_stake_service.test/unstake_validator.test.js index 21fc02ed..97aed054 100644 --- a/test/unit/validator_stake_service.test/unstake_validator.test.js +++ b/test/unit/validator_stake_service.test/unstake_validator.test.js @@ -13,10 +13,13 @@ const { expect } = require('chai') const { unstakeValidator } = require('../../../src/services/validator_stake_service') +const { classifyUnstakeRows } = require('../../../src/services/validator_stake_service/unstake_operations') +const { COIN_NETWORKS } = require('../../../src/services/validator_service') const sinon = require('sinon') const PUBKEY = 'ab'.repeat(32) const ADDRESS = 'mStakeAddress' +const TIP = 150400 function makeExplorer(chain) { return { @@ -33,8 +36,14 @@ function makeExplorer(chain) { ? sinon.stub().rejects(new Error(chain.validatorsThrow)) : sinon.stub().resolves({ data: [ { status: 'valid', signing_pubkey: 'ff'.repeat(32), amount: '25000', action_index: '1', activation_block: '1' }, - ...(chain.existing ? [chain.existing] : []) - ] }) + ...(chain.existing ? [chain.existing] : []), + ...(chain.rows || []) + ] }), + // The explorer's indexed tip, under every stake coin so each network's run can read it. + getStatus: chain.statusThrow + ? sinon.stub().rejects(new Error(chain.statusThrow)) + : sinon.stub().resolves({ last_block: Object.fromEntries( + Object.values(COIN_NETWORKS).map(c => [c.stake, chain.tip ?? TIP])) }) } } @@ -245,3 +254,99 @@ describe('ValidatorStakeService', function () { }) }) }) + +// The explorer keeps status='valid' on a row after UNSTAKE or eviction; only +// deactivation_block moves. Each case below is one the indexer rejects, so it +// must reach no broadcast even with --broadcast. +describe('ValidatorStakeService', function () { + + describe('unstakeValidator() stake lifecycle', function () { + + const DEACTIVATED = { ...STAKED, deactivation_block: '150350' } + + it('reports an unstake already in progress and sends nothing', async function () { + const { result, unstakeCalls, logged } = await runUnstake({ broadcast: true }, { existing: DEACTIVATED }) + expect(unstakeCalls).to.have.length(0) + expect(result.alreadyUnstaking).to.be.true + expect(result.deactivationBlock).to.equal(150350) + // UNSTAKE landed at 150350 - 6; cooldown runs 1000 blocks from there. + expect(result.cooldownEndBlock).to.equal(151344) + const out = logged.join('\n') + expect(out).to.include('already withdrawn') + expect(out).to.include('cooldown ends at block 151344') + expect(out).to.not.include('active stake :') + }) + + it('sends nothing for a stake unstaked and cooled down long ago', async function () { + const old = { ...STAKED, activation_block: '1000', deactivation_block: '1100' } + const { result, unstakeCalls } = await runUnstake({ broadcast: true }, { existing: old }) + expect(unstakeCalls).to.have.length(0) + expect(result.alreadyUnstaking).to.be.true + }) + + it('refuses a stake still inside its activation delay', async function () { + const pending = { ...STAKED, activation_block: String(TIP + 5) } + let err = null + let sent = null + try { + const r = await runUnstake({ broadcast: true }, { existing: pending }) + sent = r.unstakeCalls + } catch (e) { err = e } + expect(sent).to.equal(null) + expect(err).to.exist + expect(err.message).to.match(/not active yet/) + expect(err.message).to.include('block ' + (TIP + 5)) + expect(err.message).to.include('Nothing was sent') + }) + + it('refuses the partial-unstake shape: old rows deactivated, residual still pending', async function () { + const residual = { ...STAKED, action_index: '60', amount: '5000', activation_block: String(TIP + 3) } + let err = null + try { await runUnstake({ broadcast: true }, { existing: DEACTIVATED, rows: [residual] }) } catch (e) { err = e } + expect(err).to.exist + expect(err.message).to.match(/not active yet/) + }) + }) +}) + +describe('ValidatorStakeService', function () { + + describe('unstakeValidator() stake lifecycle', function () { + + it('refuses when the chain tip cannot be read', async function () { + let err = null + try { await runUnstake({ broadcast: true }, { existing: STAKED, statusThrow: 'explorer 503' }) } catch (e) { err = e } + expect(err).to.exist + expect(err.message).to.match(/could not read the chain tip/) + }) + + it('withdraws only the admissible rows and reports their sum', async function () { + const second = { ...STAKED, action_index: '61', amount: '5000', activation_block: '150320' } + const { result, unstakeCalls, logged } = await runUnstake({ broadcast: true }, + { existing: STAKED, rows: [second, { ...STAKED, action_index: '12', deactivation_block: '140000' }] }) + expect(unstakeCalls).to.have.length(1) + expect(result.unstaked).to.be.true + expect(logged.join('\n')).to.include('active stake : 30000 XCHAIN (action 44, 61') + }) + + it('reports that sum exactly where a float would round it away', async function () { + const big = { ...STAKED, amount: '900000000.00000001' } + const dust = { ...STAKED, action_index: '61', amount: '0.00000001', activation_block: '150320' } + const { logged } = await runUnstake({}, { existing: big, rows: [dust] }) + expect(logged.join('\n')).to.include('active stake : 900000000.00000002 XCHAIN') + }) + + it('classifyUnstakeRows sorts every lifecycle state', function () { + const rows = [ + { activation_block: '10' }, + { activation_block: '10', deactivation_block: '20' }, + { activation_block: '101' }, + { activation_block: '102' } + ] + const c = classifyUnstakeRows(rows, 101) + expect(c.admissible.map(r => r.activation_block)).to.deep.equal(['10', '101']) + expect(c.pending.map(r => r.activation_block)).to.deep.equal(['102']) + expect(c.deactivated).to.have.length(1) + }) + }) +}) diff --git a/test/unit/validator_stake_service.test/upstream_reads.test.js b/test/unit/validator_stake_service.test/upstream_reads.test.js new file mode 100644 index 00000000..82a928f0 --- /dev/null +++ b/test/unit/validator_stake_service.test/upstream_reads.test.js @@ -0,0 +1,88 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +// How the stake CLI reads the explorer's address and token answers: an unknown +// balance stays unknown, and the one-element array envelope is unwrapped. + +const { expect } = require('chai') + +const { readChainState, planMints } = require('../../../src/services/validator_stake_service') +const { stakeBlockers, logStakeBalances } = require('../../../src/services/validator_stake_service/stake_operations') + +const PUBKEY = 'ab'.repeat(32) +const ADDRESS = 'mStakeAddress' +const COINS = { stake: 'bitcoin-testnet', stakeCoin: 'TBTC' } +const KEY_CTX = { network: 'testnet', coins: COINS, cooldownBlocks: 1000 } + +// An SDK that answers the address and token reads with the given bodies, and an +// empty validator set and delegation list so no key holder is found. +function fakeSdk(addressBody, tokenBody) { + return { + explorer: { + getAddress: async () => addressBody, + getToken: async () => tokenBody, + getValidators: async () => ({ data: [] }), + getDelegations: async () => ({ total: 0, data: [] }), + getStatus: async () => ({ last_block: { 'bitcoin-testnet': 200000 } }) + }, + getBalances: async () => ({ data: [{ tick: 'XCHAIN', amount: '0' }] }) + } +} + +const TOKEN = { info: { tick: 'XCHAIN' }, mints: { max: '10000', address_max: '50000' } } + +describe('validator stake upstream reads', function () { + + it('reads a tracker outage as an unknown balance, not as zero', async function () { + const outage = { balances: { confirmed: null, pending: null, received: null }, tracker_available: false } + const state = await readChainState(fakeSdk(outage, TOKEN), ADDRESS, PUBKEY, KEY_CTX) + expect(state.coinBal).to.equal(null) + expect(state.coinPending).to.equal(null) + }) + + it('blocks on an unknown balance as an outage, never as an address to fund', function () { + const plan = { short: 0, mints: [], reason: null } + const why = stakeBlockers({ coinBal: null }, plan, COINS, ADDRESS).join(' ') + expect(why).to.include('TBTC balance at ' + ADDRESS + ' is unavailable') + expect(why).to.not.include('fund it first') + expect(stakeBlockers({ coinBal: 0 }, plan, COINS, ADDRESS).join(' ')).to.include('fund it first') + }) + + it('prints an unknown balance as unavailable rather than "null confirmed"', function () { + const logged = [] + const plan = { short: 0, mints: [], reason: null } + logStakeBalances(m => logged.push(m), 'testnet', COINS, PUBKEY, ADDRESS, 25000, + { coinBal: null, coinPending: null, tokenBal: 25000 }, plan, 'XCHAIN') + expect(logged.join('\n')).to.include('unavailable (the explorer could not read it') + expect(logged.join('\n')).to.not.include('null confirmed') + }) + + it('unwraps a one-element array envelope on the address and token reads', async function () { + const addr = [{ balances: { confirmed: '0.002', pending: '0.001' }, tracker_available: true }] + const state = await readChainState(fakeSdk(addr, [TOKEN]), ADDRESS, PUBKEY, KEY_CTX) + expect(state.coinBal).to.equal(0.002) + expect(state.coinPending).to.equal(0.001) + expect(state.mintMax).to.equal(10000) + expect(state.mintAddressMax).to.equal(50000) + expect(state.mintUnreadable).to.equal(null) + }) + + it('names a token with no mints group instead of reporting MAX_MINT 0', async function () { + const addr = { balances: { confirmed: '0.002', pending: '0' } } + const state = await readChainState(fakeSdk(addr, { info: { tick: 'XCHAIN' } }), ADDRESS, PUBKEY, KEY_CTX) + expect(state.mintUnreadable).to.include('unexpected shape (no mints group)') + const plan = planMints('testnet', 0, 25000, state.mintMax, state.mintAddressMax, state.mintUnreadable) + expect(plan.mints).to.have.length(0) + expect(plan.reason).to.include('unexpected shape') + expect(plan.reason).to.not.include('MAX_MINT is 0') + }) +}) diff --git a/test/unit/validator_stake_service.test/validator_set_paging.test.js b/test/unit/validator_stake_service.test/validator_set_paging.test.js new file mode 100644 index 00000000..d7ab8cc4 --- /dev/null +++ b/test/unit/validator_stake_service.test/validator_set_paging.test.js @@ -0,0 +1,165 @@ +'use strict' + +// Copyright © 2025–2026 Dankest, LLC +// Based on XChain Platform by Dankest, LLC – https://dankest.llc +// +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This file is part of XChain Platform. Licensed under the GNU Affero +// General Public License v3.0 or later; see LICENSE.md. A commercial +// license (without AGPL source-disclosure terms) is available - +// contact legal@dankest.llc. + +const { expect } = require('chai') +const sinon = require('sinon') + +const { stakeValidator, unstakeValidator } = require('../../../src/services/validator_stake_service') +const { readValidatorSet } = require('../../../src/services/validator_stake_service/validator_set_read') +const { COIN_NETWORKS } = require('../../../src/services/validator_service') + +const PUBKEY = 'ab'.repeat(32) +const ADDRESS = 'mStakeAddress' +const TIP = 150400 + +// One stakes row per action, the shape /validators returns. +function stakeRow(actionIndex, pubkey) { + return { status: 'valid', signing_pubkey: pubkey, amount: '25000', + action_index: String(actionIndex), activation_block: '1' } +} + +// The target validator staked first, then 150 others, so it is the OLDEST row +// and sits off the newest-first first page the explorer serves by default. +function busySet() { + const rows = [stakeRow(5, PUBKEY)] + for (let i = 0; i < 150; i++) rows.push(stakeRow(100 + i, 'ff'.repeat(31) + String(10 + (i % 90)).padStart(2, '0'))) + return rows +} + +// A /validators endpoint that pages the way the explorer does: `page` from 1, +// `limit` capped at 100, newest first unless sortorder=ASC, `total` the whole set. +function pagedValidators(rows, fault = {}) { + return sinon.stub().callsFake(async (opts = {}) => { + const limit = Math.max(1, Math.min(Number(opts.limit) || 100, 100)) + const page = Math.max(1, Number(opts.page) || 1) + if (fault.rejectPage === page) throw new Error('explorer 503') + const asc = String(opts.sortorder || 'DESC').toUpperCase() === 'ASC' + const ordered = asc ? rows.slice() : rows.slice().reverse() + const data = fault.emptyPage === page ? [] : ordered.slice((page - 1) * limit, page * limit) + return { total: rows.length, data } + }) +} + +function makeSdk(getValidators) { + const calls = { submit: [] } + const sdk = { + explorer: { + getAddress: sinon.stub().resolves({ balances: { confirmed: '0.001', pending: '0' } }), + getToken: sinon.stub().resolves({ mints: { max: 10000, address_max: 50000 } }), + getValidators, + getStatus: sinon.stub().resolves({ last_block: Object.fromEntries( + Object.values(COIN_NETWORKS).map(c => [c.stake, TIP])) }) + }, + getBalances: async () => ({ data: [{ tick: 'XCHAIN', amount: '25000' }] }), + session: () => ({ + address: ADDRESS, + submit: async (a) => { calls.submit.push(a); return { txid: 'tx' + calls.submit.length, spentInputs: [] } } + }), + requireEncoder: () => ({ getUTXOs: async () => ({ utxos: [] }) }) + } + return { sdk, calls } +} + +function deps(sdk, logged) { + return { + settings: { enabled: true, pubkey: PUBKEY, network: 'testnet', P2P_PORT: 10002 }, + wallets: { NETWORK: 'testnet', STAKE_ADDRESS: ADDRESS, STAKE_WIF_SECRET: 'cFakeWif' }, + makeSdk: () => sdk, + sdk: {}, + log: m => logged.push(String(m)) + } +} + +describe('ValidatorStakeService', function () { + + describe('validator set paging', function () { + + it('stake finds a stake that sits past the first page and sends nothing', async function () { + const { sdk, calls } = makeSdk(pagedValidators(busySet())) + const logged = [] + const result = await stakeValidator({ broadcast: true }, deps(sdk, logged)) + expect(result.staked).to.be.false + expect(result.existing.action_index).to.equal('5') + expect(calls.submit).to.have.length(0) + expect(logged.join('\n')).to.include('already carries a valid STAKE') + }) + + it('stake reports an unreadable page as unknown, never as "not staked"', async function () { + const { sdk } = makeSdk(pagedValidators(busySet(), { rejectPage: 2 })) + const logged = [] + await stakeValidator({}, deps(sdk, logged)) + expect(logged.join('\n')).to.include('could not read the validator set') + }) + + it('stake reports a page that comes back short of the total as unknown', async function () { + const { sdk } = makeSdk(pagedValidators(busySet(), { emptyPage: 2 })) + const logged = [] + await stakeValidator({}, deps(sdk, logged)) + expect(logged.join('\n')).to.include('could not read the validator set (validator set read incomplete') + }) + }) +}) + +describe('ValidatorStakeService', function () { + + describe('validator set paging', function () { + + it('unstake finds a stake that sits past the first page', async function () { + const { sdk, calls } = makeSdk(pagedValidators(busySet())) + const logged = [] + const result = await unstakeValidator({}, deps(sdk, logged)) + expect(result.dryRun).to.be.true + expect(result.nothingStaked).to.not.equal(true) + expect(calls.submit).to.have.length(0) + }) + + it('unstake refuses and sends nothing when a page cannot be read', async function () { + const { sdk, calls } = makeSdk(pagedValidators(busySet(), { rejectPage: 2 })) + let err = null + try { await unstakeValidator({ broadcast: true }, deps(sdk, [])) } catch (e) { err = e } + expect(err).to.exist + expect(err.message).to.match(/could not read the validator set/) + expect(calls.submit).to.have.length(0) + }) + }) +}) + +describe('ValidatorStakeService', function () { + + describe('validator set paging', function () { + + it('pages oldest first from page 1 and stops at the total', async function () { + const getValidators = pagedValidators(busySet()) + const set = await readValidatorSet({ explorer: { getValidators } }) + expect(getValidators.firstCall.args[0]).to.deep.equal({ page: 1, limit: 100, sortorder: 'ASC' }) + expect(getValidators.callCount).to.equal(2) + expect(set.data).to.have.length(151) + expect(set.data[0].action_index, 'oldest first, in the order it was paged').to.equal('5') + expect(set.data[150].action_index).to.equal('249') + }) + + it('treats a short page with no total as the whole set', async function () { + const getValidators = sinon.stub().resolves({ data: [stakeRow(1, PUBKEY)] }) + const set = await readValidatorSet({ explorer: { getValidators } }) + expect(getValidators.callCount).to.equal(1) + expect(set.data).to.have.length(1) + }) + + it('refuses a response with no data array rather than reading it as empty', async function () { + const getValidators = sinon.stub().resolves({ error: 'unexpected' }) + let err = null + try { await readValidatorSet({ explorer: { getValidators } }) } catch (e) { err = e } + expect(err).to.exist + expect(err.message).to.match(/validator set read incomplete/) + }) + }) +}) From 3b8a9c3e24cf387130c34e9b54d0ca6feee240b5 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Tue, 29 Sep 2026 11:01:11 -0700 Subject: [PATCH 38/50] chore(release): v0.21.0 --- CHANGELOG.md | 13 +++++++++++++ README.md | 2 +- package-lock.json | 4 ++-- package.json | 2 +- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a68f507..20dc8c2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.21.0] - 2026-09-29 + +### Added +- Published the hub peer port in validator mode. + +### Changed +- Validated the running indexer's migration status before service use. +- Passed the anchor fold arming variable to the regtest indexer and hub. + +### Fixed +- Used regtest namespaces and the correct encoder URL in validator links. +- Started module containers under an init process and bounded health-check requests. + ## [0.20.1] - 2026-09-23 ### Fixed diff --git a/README.md b/README.md index eeeea22e..c6ec5cef 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ # XChain Platform Node

- Version + Version Tests Node License diff --git a/package-lock.json b/package-lock.json index 49036a27..500da099 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "xchain-node", - "version": "0.20.1", + "version": "0.21.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "xchain-node", - "version": "0.20.1", + "version": "0.21.0", "license": "AGPL-3.0-or-later", "dependencies": { "@dankest-llc/xchain-sdk": "^0.18.0", diff --git a/package.json b/package.json index 53f5b969..4ef54b17 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "xchain-node", - "version": "0.20.1", + "version": "0.21.0", "description": "xchain-node allows users to install, configure and run XChain platform nodes.", "license": "AGPL-3.0-or-later", "repository": { From 4443e32e8a902b7aa9bb9deaf59d44488399babb Mon Sep 17 00:00:00 2001 From: J-Dog Date: Tue, 29 Sep 2026 12:22:27 -0700 Subject: [PATCH 39/50] Pin CI Node to 22.22.3, the fleet's consensus runtime --- .github/workflows/ci.yml | 10 +++++----- .github/workflows/nightly-e2e.yml | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05433556..39e7fe8c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,9 +20,9 @@ on: jobs: ci: uses: XChain-Platform/.github/.github/workflows/ci-reusable.yml@fd9160695d677f0538e959460ba5d8dfbb7f3c8e # pin: XChain-Platform/.github @ master 2026-08-31; bump deliberately - # Override the Node version for a repo if ever needed: - # with: - # node-version: "20" + with: + # The fleet's consensus runtime; a floating "22" now resolves 22.23.3, which the VM runtime gate refuses. + node-version: "22.22.3" # Cross-repo drift guards: this repo vendors the canonical # coin registry from xchain-hub, and per-repo CI cannot see sibling repos, so @@ -49,7 +49,7 @@ jobs: - name: Use Node.js 22 uses: actions/setup-node@v4 with: - node-version: "22" + node-version: "22.22.3" - name: Coin-registry byte-identity vs canonical run: xchain-hub/bin/sync-coins.sh --check --only xchain-node @@ -97,7 +97,7 @@ jobs: - name: Use Node.js 22 uses: actions/setup-node@v4 with: - node-version: "22" + node-version: "22.22.3" cache: npm - name: Install dependencies diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml index bd9d819c..f8975321 100644 --- a/.github/workflows/nightly-e2e.yml +++ b/.github/workflows/nightly-e2e.yml @@ -190,7 +190,7 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: '22' + node-version: "22.22.3" # WHY THE COIN LIST IS EVENT-DEPENDENT. A scheduled run carries NO inputs, so # without this it would silently test bitcoin alone and the nightly would be @@ -357,7 +357,7 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: '22' + node-version: "22.22.3" - name: Reclaim runner disk (coin daemon + per-service images are large) run: | @@ -773,7 +773,7 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: '22' + node-version: "22.22.3" # A shard that never uploaded simply has no artifact here; the merge names # it as missing, so a failed download is not allowed to stop the job first. From 688ba64d3f1a5eb7498f2b1dde87c17261cb3598 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Tue, 29 Sep 2026 12:37:25 -0700 Subject: [PATCH 40/50] Give the matrix stack boot the job token for GitHub API reads --- .github/workflows/nightly-e2e.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml index f8975321..1e871776 100644 --- a/.github/workflows/nightly-e2e.yml +++ b/.github/workflows/nightly-e2e.yml @@ -581,6 +581,10 @@ jobs: # on the non-BTC legs (see the ports step above for why). `install` is # idempotent for the shared hub and explorer, so the second call adds the # bitcoin coin stack and pushes both coins' config to the one hub. + # The installer's api.github.com reads share a 60 per hour anonymous quota per + # runner IP, which a busy day exhausts; the job token raises it to 5000. + env: + GITHUB_TOKEN: ${{ github.token }} run: | mkdir -p "$XCHAIN_NODE_DATA_DIR" echo "Installing the stack at ref: $STACK_REF" From 8b4711644c0b1b38477bdb95577d23da0295fb54 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Tue, 29 Sep 2026 12:43:03 -0700 Subject: [PATCH 41/50] Give every matrix shard step the job token for GitHub API reads --- .github/workflows/nightly-e2e.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml index 1e871776..fdd2a5f3 100644 --- a/.github/workflows/nightly-e2e.yml +++ b/.github/workflows/nightly-e2e.yml @@ -245,6 +245,10 @@ jobs: COIN: ${{ matrix.coin }} SHARD: ${{ matrix.shard }} PLAN_JSON: ${{ needs.plan.outputs.plan }} + # Every node src/index.js call below (validator init, install, e2etest) reads + # api.github.com; anonymous reads share 60 per hour per runner IP, which a busy + # day exhausts, and the job token raises that to 5000. + GITHUB_TOKEN: ${{ github.token }} # The anchor-reward attestation barrier holds each block until the hub-mirror # stream watermark is 120s past that block's own timestamp. On a shared ledger # that is free, because blocks are ten minutes apart and the watermark is long @@ -581,10 +585,6 @@ jobs: # on the non-BTC legs (see the ports step above for why). `install` is # idempotent for the shared hub and explorer, so the second call adds the # bitcoin coin stack and pushes both coins' config to the one hub. - # The installer's api.github.com reads share a 60 per hour anonymous quota per - # runner IP, which a busy day exhausts; the job token raises it to 5000. - env: - GITHUB_TOKEN: ${{ github.token }} run: | mkdir -p "$XCHAIN_NODE_DATA_DIR" echo "Installing the stack at ref: $STACK_REF" From 666aa239897935e0ef98ea895bd3cb6d7d8e13f2 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Tue, 29 Sep 2026 12:22:27 -0700 Subject: [PATCH 42/50] Pin CI Node to 22.22.3, the fleet's consensus runtime --- .github/workflows/ci.yml | 10 +++++----- .github/workflows/nightly-e2e.yml | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05433556..39e7fe8c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,9 +20,9 @@ on: jobs: ci: uses: XChain-Platform/.github/.github/workflows/ci-reusable.yml@fd9160695d677f0538e959460ba5d8dfbb7f3c8e # pin: XChain-Platform/.github @ master 2026-08-31; bump deliberately - # Override the Node version for a repo if ever needed: - # with: - # node-version: "20" + with: + # The fleet's consensus runtime; a floating "22" now resolves 22.23.3, which the VM runtime gate refuses. + node-version: "22.22.3" # Cross-repo drift guards: this repo vendors the canonical # coin registry from xchain-hub, and per-repo CI cannot see sibling repos, so @@ -49,7 +49,7 @@ jobs: - name: Use Node.js 22 uses: actions/setup-node@v4 with: - node-version: "22" + node-version: "22.22.3" - name: Coin-registry byte-identity vs canonical run: xchain-hub/bin/sync-coins.sh --check --only xchain-node @@ -97,7 +97,7 @@ jobs: - name: Use Node.js 22 uses: actions/setup-node@v4 with: - node-version: "22" + node-version: "22.22.3" cache: npm - name: Install dependencies diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml index bd9d819c..f8975321 100644 --- a/.github/workflows/nightly-e2e.yml +++ b/.github/workflows/nightly-e2e.yml @@ -190,7 +190,7 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: '22' + node-version: "22.22.3" # WHY THE COIN LIST IS EVENT-DEPENDENT. A scheduled run carries NO inputs, so # without this it would silently test bitcoin alone and the nightly would be @@ -357,7 +357,7 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: '22' + node-version: "22.22.3" - name: Reclaim runner disk (coin daemon + per-service images are large) run: | @@ -773,7 +773,7 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: '22' + node-version: "22.22.3" # A shard that never uploaded simply has no artifact here; the merge names # it as missing, so a failed download is not allowed to stop the job first. From daed528b85b8fc67bd131ae1eb360c4412a032b0 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Tue, 29 Sep 2026 12:37:25 -0700 Subject: [PATCH 43/50] Give the matrix stack boot the job token for GitHub API reads --- .github/workflows/nightly-e2e.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml index f8975321..1e871776 100644 --- a/.github/workflows/nightly-e2e.yml +++ b/.github/workflows/nightly-e2e.yml @@ -581,6 +581,10 @@ jobs: # on the non-BTC legs (see the ports step above for why). `install` is # idempotent for the shared hub and explorer, so the second call adds the # bitcoin coin stack and pushes both coins' config to the one hub. + # The installer's api.github.com reads share a 60 per hour anonymous quota per + # runner IP, which a busy day exhausts; the job token raises it to 5000. + env: + GITHUB_TOKEN: ${{ github.token }} run: | mkdir -p "$XCHAIN_NODE_DATA_DIR" echo "Installing the stack at ref: $STACK_REF" From 4a74ba29fea56b89fa12087e8084c4ea47c1ea4b Mon Sep 17 00:00:00 2001 From: J-Dog Date: Tue, 29 Sep 2026 12:43:03 -0700 Subject: [PATCH 44/50] Give every matrix shard step the job token for GitHub API reads --- .github/workflows/nightly-e2e.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nightly-e2e.yml b/.github/workflows/nightly-e2e.yml index 1e871776..fdd2a5f3 100644 --- a/.github/workflows/nightly-e2e.yml +++ b/.github/workflows/nightly-e2e.yml @@ -245,6 +245,10 @@ jobs: COIN: ${{ matrix.coin }} SHARD: ${{ matrix.shard }} PLAN_JSON: ${{ needs.plan.outputs.plan }} + # Every node src/index.js call below (validator init, install, e2etest) reads + # api.github.com; anonymous reads share 60 per hour per runner IP, which a busy + # day exhausts, and the job token raises that to 5000. + GITHUB_TOKEN: ${{ github.token }} # The anchor-reward attestation barrier holds each block until the hub-mirror # stream watermark is 120s past that block's own timestamp. On a shared ledger # that is free, because blocks are ten minutes apart and the watermark is long @@ -581,10 +585,6 @@ jobs: # on the non-BTC legs (see the ports step above for why). `install` is # idempotent for the shared hub and explorer, so the second call adds the # bitcoin coin stack and pushes both coins' config to the one hub. - # The installer's api.github.com reads share a 60 per hour anonymous quota per - # runner IP, which a busy day exhausts; the job token raises it to 5000. - env: - GITHUB_TOKEN: ${{ github.token }} run: | mkdir -p "$XCHAIN_NODE_DATA_DIR" echo "Installing the stack at ref: $STACK_REF" From 6de78f07c4642991e86ba8d89a77ba574d696baf Mon Sep 17 00:00:00 2001 From: J-Dog Date: Tue, 29 Sep 2026 13:57:05 -0700 Subject: [PATCH 45/50] Run nightly attest-mirror legs on hosted runners Copy the hosted rail workflows and helper stack. Schedule only proven attest-mirror legs while keeping bridge execution manual. --- .github/attest-mirror-run.json | 1 + .github/attest-mirror/compose.yml | 639 +++++++++++++++++++ .github/attest-mirror/run-leg.sh | 59 ++ .github/attest-mirror/seed-stack-config.js | 26 + .github/attest-mirror/setup-stack.sh | 68 ++ .github/attest-mirror/wait-indexer-schema.js | 23 + .github/rail-legs.json | 146 +++++ .github/workflows/attest-mirror-leg.yml | 231 +++++++ .github/workflows/rail-leg.yml | 329 ++++++++++ 9 files changed, 1522 insertions(+) create mode 100644 .github/attest-mirror-run.json create mode 100644 .github/attest-mirror/compose.yml create mode 100755 .github/attest-mirror/run-leg.sh create mode 100644 .github/attest-mirror/seed-stack-config.js create mode 100755 .github/attest-mirror/setup-stack.sh create mode 100644 .github/attest-mirror/wait-indexer-schema.js create mode 100644 .github/rail-legs.json create mode 100644 .github/workflows/attest-mirror-leg.yml create mode 100644 .github/workflows/rail-leg.yml diff --git a/.github/attest-mirror-run.json b/.github/attest-mirror-run.json new file mode 100644 index 00000000..a2e96b42 --- /dev/null +++ b/.github/attest-mirror-run.json @@ -0,0 +1 @@ +{"e2e_ref": "gh-rail/l004-at2b-fix", "legs": ["test/attestMirror/at2b-forward-margin.test.js"]} diff --git a/.github/attest-mirror/compose.yml b/.github/attest-mirror/compose.yml new file mode 100644 index 00000000..64f093f6 --- /dev/null +++ b/.github/attest-mirror/compose.yml @@ -0,0 +1,639 @@ +services: + db: + image: xchain-node-database + command: ["--max-connections=1000", "--innodb-flush-log-at-trx-commit=2", "--innodb-doublewrite=0", "--innodb-buffer-pool-size=1G"] + environment: + MARIADB_ROOT_PASSWORD: ${DB_PASSWORD} + MARIADB_ROOT_HOST: "%" + ports: + - "127.0.0.1:${DB_HOST_PORT}:3306" + volumes: + - db-data:/var/lib/mysql + healthcheck: + test: ["CMD-SHELL", "healthcheck.sh --connect --innodb_initialized"] + interval: 5s + timeout: 5s + retries: 30 + start_period: 20s + restart: unless-stopped + + db-init: + image: xchain-node-database + depends_on: + db: + condition: service_healthy + environment: + MARIADB_ROOT_PASSWORD: ${DB_PASSWORD} + entrypoint: ["/bin/sh", "-ec"] + command: + - | + mariadb -h db -uroot -p"$${MARIADB_ROOT_PASSWORD}" -e "CREATE DATABASE IF NOT EXISTS XChain_BTC_Regtest_Decoder; CREATE DATABASE IF NOT EXISTS XChain_BTC_Regtest_Indexer; CREATE DATABASE IF NOT EXISTS XChain_Hub; CREATE DATABASE IF NOT EXISTS XChain_LTC_Regtest_Decoder; CREATE DATABASE IF NOT EXISTS XChain_LTC_Regtest_Indexer; CREATE DATABASE IF NOT EXISTS XChain_DOGE_Regtest_Decoder; CREATE DATABASE IF NOT EXISTS XChain_DOGE_Regtest_Indexer;" + restart: "no" + + node: + image: xchain-node-bitcoin-regtest-node + environment: + NODE_USER: ${RPC_USER} + NODE_PASSWORD: ${RPC_PASSWORD} + entrypoint: ["/bin/sh", "-ec"] + command: + - | + umask 077 + sed -e "s|^rpcuser=.*|rpcuser=$${NODE_USER}|" -e "s|^rpcpassword=.*|rpcpassword=$${NODE_PASSWORD}|" /etc/bitcoin/bitcoin.conf > /run/bitcoin.conf + exec bitcoind -conf=/run/bitcoin.conf -datadir=/root/.bitcoin/ + tmpfs: + - /run + ports: + - "127.0.0.1:${NODE_HOST_PORT}:18444" + volumes: + - node-data:/root/.bitcoin + healthcheck: + test: ["CMD-SHELL", "bitcoin-cli -conf=/run/bitcoin.conf -datadir=/root/.bitcoin/ getblockchaininfo >/dev/null 2>&1"] + interval: 5s + timeout: 5s + retries: 60 + start_period: 20s + restart: unless-stopped + + miner: + image: xchain-node-bitcoin-regtest-xchain-regtest-miner + depends_on: + node: + condition: service_healthy + environment: + NETWORK: bitcoin-regtest + NODE_URL: node + NODE_PORT: 18444 + NODE_USER: ${RPC_USER} + NODE_PASSWORD: ${RPC_PASSWORD} + REGTEST_MINER_API_PORT: 3005 + ports: + - "127.0.0.1:${MINER_HOST_PORT}:3005" + healthcheck: + test: ["CMD-SHELL", "node -e \"const h=require('http');const q=h.request({host:'127.0.0.1',port:3005,method:'POST',headers:{'content-type':'application/json'}},r=>process.exit(r.statusCode===200?0:1));q.on('error',()=>process.exit(1));q.end(JSON.stringify({jsonrpc:'2.0',method:'health',id:1}))\""] + interval: 10s + timeout: 5s + retries: 60 + start_period: 90s + restart: unless-stopped + + tracker: + image: xchain-node-bitcoin-regtest-xchain-utxo-tracker + depends_on: + node: + condition: service_healthy + environment: + COIN: bitcoin + NETWORK: bitcoin-regtest + NODE_URL: node + NODE_PORT: 18444 + NODE_USER: ${RPC_USER} + NODE_PASSWORD: ${RPC_PASSWORD} + UTXO_TRACKER_API_PORT: 3001 + CORS_ORIGIN: "*" + ports: + - "127.0.0.1:${TRACKER_HOST_PORT}:3001" + volumes: + - tracker-data:/data/xchain-utxo-tracker + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3001/status >/dev/null 2>&1"] + interval: 10s + timeout: 5s + retries: 60 + start_period: 90s + restart: unless-stopped + + decoder: + image: xchain-node-bitcoin-regtest-xchain-decoder + depends_on: + db-init: + condition: service_completed_successfully + node: + condition: service_healthy + environment: + COIN: bitcoin + NETWORK: bitcoin-regtest + NODE_URL: node + NODE_PORT: 18444 + NODE_USER: ${RPC_USER} + NODE_PASSWORD: ${RPC_PASSWORD} + DECODER_DB_HOST: db + DECODER_DB_PORT: 3306 + DECODER_DB_NAME: XChain_BTC_Regtest_Decoder + DECODER_DB_USER: root + DECODER_DB_PASS: ${DB_PASSWORD} + DECODER_API_PORT: 3002 + ports: + - "127.0.0.1:${DECODER_HOST_PORT}:3002" + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3002/live >/dev/null 2>&1"] + interval: 10s + timeout: 5s + retries: 60 + start_period: 90s + restart: unless-stopped + + encoder: + image: xchain-node-bitcoin-regtest-xchain-encoder + depends_on: + node: + condition: service_healthy + tracker: + condition: service_healthy + environment: + NETWORK: bitcoin-regtest + NODE_URL: node + NODE_PORT: 18444 + NODE_USER: ${RPC_USER} + NODE_PASSWORD: ${RPC_PASSWORD} + UTXO_TRACKER_URL: tracker + UTXO_TRACKER_API_PORT: 3001 + ENCODER_API_PORT: 3003 + ENCODER_RATE_LIMIT_RPM: 99999 + CORS_ORIGIN: "*" + ports: + - "127.0.0.1:${ENCODER_HOST_PORT}:3003" + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3003/status >/dev/null 2>&1"] + interval: 10s + timeout: 5s + retries: 60 + start_period: 90s + restart: unless-stopped + + hub: + image: xchain-node-xchain-hub + depends_on: + db-init: + condition: service_completed_successfully + environment: + HUB_HOST: 0.0.0.0 + HUB_PORT: 10000 + HUB_DB_HOST: db + HUB_DB_PORT: 3306 + HUB_DB_NAME: XChain_Hub + HUB_DB_USER: root + HUB_DB_SECRET: ${DB_PASSWORD} + HUB_ALLOW_UNAUTHENTICATED: "true" + HUB_NETWORK: regtest + BTC_INDEXER_API_URL: http://indexer:3004 + ports: + - "127.0.0.1:${HUB_HOST_PORT}:10000" + healthcheck: + test: ["CMD-SHELL", "wget -qO- --post-data='{\"jsonrpc\":\"2.0\",\"method\":\"health\",\"id\":1}' --header='Content-Type: application/json' http://127.0.0.1:10000/ >/dev/null 2>&1"] + interval: 10s + timeout: 5s + retries: 60 + start_period: 90s + restart: unless-stopped + + indexer: + image: xchain-node-bitcoin-regtest-xchain-indexer + depends_on: + db-init: + condition: service_completed_successfully + decoder: + condition: service_healthy + tracker: + condition: service_healthy + hub: + condition: service_healthy + environment: + INDEXER_COIN: BTC + INDEXER_NETWORK: regtest + INDEXER_API_PORT: 3004 + INDEXER_DB_HOST: db + INDEXER_DB_PORT: 3306 + INDEXER_DB_NAME: XChain_BTC_Regtest_Indexer + INDEXER_DB_USER: root + INDEXER_DB_PASS: ${DB_PASSWORD} + DECODER_DB_HOST: db + DECODER_DB_PORT: 3306 + DECODER_DB_NAME: XChain_BTC_Regtest_Decoder + DECODER_DB_USER: root + DECODER_DB_PASS: ${DB_PASSWORD} + UTXO_TRACKER_URL: tracker + UTXO_TRACKER_API_PORT: 3001 + HUB_API_URL: http://hub:10000 + HUB_DB_HOST: db + HUB_DB_PORT: 3306 + HUB_DB_NAME: XChain_BTC_Regtest_Indexer + HUB_DB_USER: root + HUB_DB_PASS: ${DB_PASSWORD} + HUB_DB_SYNC_ENABLED: "true" + INDEXER_ALLOW_UNAUTHENTICATED: "true" + HUB_SYNC_PRICE_GRACE_S: 0 + HUB_SYNC_ORACLE_GRACE_S: 0 + HUB_SYNC_ATTEST_RESPONSE_GRACE_S: 0 + HUB_SYNC_MATCH_GRACE_S: 0 + HUB_SYNC_CALL_GRACE_S: 0 + HUB_SYNC_ANCHOR_ATTEST_GRACE_S: 0 + ports: + - "127.0.0.1:${INDEXER_HOST_PORT}:3004" + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3004/status >/dev/null 2>&1"] + interval: 10s + timeout: 5s + retries: 90 + start_period: 120s + restart: unless-stopped + + explorer: + image: xchain-node-xchain-explorer + depends_on: + hub: + condition: service_healthy + indexer: + condition: service_healthy + environment: + NETWORK: regtest + HUB_API_HOST: hub + HUB_PORT: 10000 + EXPLORER_API_PORT_HTTP: 8080 + EXPLORER_API_PORT_HTTPS: 8081 + ALLOW_NO_COLOCATED_HUB_DB: 1 + ports: + - "127.0.0.1:${EXPLORER_HTTP_HOST_PORT}:8080" + - "127.0.0.1:${EXPLORER_HTTPS_HOST_PORT}:8081" + healthcheck: + test: ["CMD-SHELL", "wget -qO- --post-data='{\"jsonrpc\":\"2.0\",\"method\":\"ping\",\"id\":1}' --header='Content-Type: application/json' http://127.0.0.1:8080/ >/dev/null 2>&1"] + interval: 10s + timeout: 5s + retries: 90 + start_period: 120s + restart: unless-stopped + + ltc-node: + image: xchain-node-litecoin-regtest-node + environment: + NODE_USER: ${RPC_USER} + NODE_PASSWORD: ${RPC_PASSWORD} + entrypoint: ["/bin/sh", "-ec"] + command: + - | + umask 077 + sed -e "s|^rpcuser=.*|rpcuser=$${NODE_USER}|" -e "s|^rpcpassword=.*|rpcpassword=$${NODE_PASSWORD}|" /etc/litecoin/litecoin.conf > /run/litecoin.conf + exec litecoind -conf=/run/litecoin.conf -datadir=/root/.litecoin/ + tmpfs: + - /run + ports: + - "127.0.0.1:${LTC_NODE_HOST_PORT}:18444" + volumes: + - ltc-node-data:/root/.litecoin + healthcheck: + test: ["CMD-SHELL", "litecoin-cli -conf=/run/litecoin.conf -datadir=/root/.litecoin/ getblockchaininfo >/dev/null 2>&1"] + interval: 5s + timeout: 5s + retries: 60 + start_period: 20s + restart: unless-stopped + + ltc-miner: + image: xchain-node-litecoin-regtest-xchain-regtest-miner + depends_on: + ltc-node: + condition: service_healthy + environment: + NETWORK: litecoin-regtest + NODE_URL: ltc-node + NODE_PORT: 18444 + NODE_USER: ${RPC_USER} + NODE_PASSWORD: ${RPC_PASSWORD} + REGTEST_MINER_API_PORT: 3005 + ports: + - "127.0.0.1:${LTC_MINER_HOST_PORT}:3005" + healthcheck: + test: ["CMD-SHELL", "node -e \"const h=require('http');const q=h.request({host:'127.0.0.1',port:3005,method:'POST',headers:{'content-type':'application/json'}},r=>process.exit(r.statusCode===200?0:1));q.on('error',()=>process.exit(1));q.end(JSON.stringify({jsonrpc:'2.0',method:'health',id:1}))\""] + interval: 10s + timeout: 5s + retries: 60 + start_period: 90s + restart: unless-stopped + + ltc-tracker: + image: xchain-node-litecoin-regtest-xchain-utxo-tracker + depends_on: + ltc-node: + condition: service_healthy + environment: + COIN: litecoin + NETWORK: litecoin-regtest + NODE_URL: ltc-node + NODE_PORT: 18444 + NODE_USER: ${RPC_USER} + NODE_PASSWORD: ${RPC_PASSWORD} + UTXO_TRACKER_API_PORT: 3001 + CORS_ORIGIN: "*" + ports: + - "127.0.0.1:${LTC_TRACKER_HOST_PORT}:3001" + volumes: + - ltc-tracker-data:/data/xchain-utxo-tracker + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3001/status >/dev/null 2>&1"] + interval: 10s + timeout: 5s + retries: 60 + start_period: 90s + restart: unless-stopped + + ltc-decoder: + image: xchain-node-litecoin-regtest-xchain-decoder + depends_on: + db: + condition: service_healthy + ltc-node: + condition: service_healthy + environment: + COIN: litecoin + NETWORK: litecoin-regtest + NODE_URL: ltc-node + NODE_PORT: 18444 + NODE_USER: ${RPC_USER} + NODE_PASSWORD: ${RPC_PASSWORD} + DECODER_DB_HOST: db + DECODER_DB_PORT: 3306 + DECODER_DB_NAME: XChain_LTC_Regtest_Decoder + DECODER_DB_USER: root + DECODER_DB_PASS: ${DB_PASSWORD} + DECODER_API_PORT: 3002 + ports: + - "127.0.0.1:${LTC_DECODER_HOST_PORT}:3002" + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3002/live >/dev/null 2>&1"] + interval: 10s + timeout: 5s + retries: 60 + start_period: 90s + restart: unless-stopped + + ltc-encoder: + image: xchain-node-litecoin-regtest-xchain-encoder + depends_on: + ltc-node: + condition: service_healthy + ltc-tracker: + condition: service_healthy + environment: + NETWORK: litecoin-regtest + NODE_URL: ltc-node + NODE_PORT: 18444 + NODE_USER: ${RPC_USER} + NODE_PASSWORD: ${RPC_PASSWORD} + UTXO_TRACKER_URL: ltc-tracker + UTXO_TRACKER_API_PORT: 3001 + ENCODER_API_PORT: 3003 + ENCODER_RATE_LIMIT_RPM: 99999 + CORS_ORIGIN: "*" + ports: + - "127.0.0.1:${LTC_ENCODER_HOST_PORT}:3003" + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3003/status >/dev/null 2>&1"] + interval: 10s + timeout: 5s + retries: 60 + start_period: 90s + restart: unless-stopped + + ltc-indexer: + image: xchain-node-litecoin-regtest-xchain-indexer + depends_on: + db: + condition: service_healthy + ltc-decoder: + condition: service_healthy + ltc-tracker: + condition: service_healthy + hub: + condition: service_healthy + environment: + INDEXER_COIN: LTC + INDEXER_NETWORK: regtest + INDEXER_API_PORT: 3004 + INDEXER_DB_HOST: db + INDEXER_DB_PORT: 3306 + INDEXER_DB_NAME: XChain_LTC_Regtest_Indexer + INDEXER_DB_USER: root + INDEXER_DB_PASS: ${DB_PASSWORD} + DECODER_DB_HOST: db + DECODER_DB_PORT: 3306 + DECODER_DB_NAME: XChain_LTC_Regtest_Decoder + DECODER_DB_USER: root + DECODER_DB_PASS: ${DB_PASSWORD} + UTXO_TRACKER_URL: ltc-tracker + UTXO_TRACKER_API_PORT: 3001 + HUB_API_URL: http://hub:10000 + HUB_DB_HOST: db + HUB_DB_PORT: 3306 + HUB_DB_NAME: XChain_LTC_Regtest_Indexer + HUB_DB_USER: root + HUB_DB_PASS: ${DB_PASSWORD} + HUB_DB_SYNC_ENABLED: "true" + INDEXER_ALLOW_UNAUTHENTICATED: "true" + HUB_SYNC_PRICE_GRACE_S: 0 + HUB_SYNC_ORACLE_GRACE_S: 0 + HUB_SYNC_ATTEST_RESPONSE_GRACE_S: 0 + HUB_SYNC_MATCH_GRACE_S: 0 + HUB_SYNC_CALL_GRACE_S: 0 + HUB_SYNC_ANCHOR_ATTEST_GRACE_S: 0 + ports: + - "127.0.0.1:${LTC_INDEXER_HOST_PORT}:3004" + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3004/status >/dev/null 2>&1"] + interval: 10s + timeout: 5s + retries: 90 + start_period: 120s + restart: unless-stopped + + + doge-node: + image: xchain-node-dogecoin-regtest-node + environment: + NODE_USER: ${RPC_USER} + NODE_PASSWORD: ${RPC_PASSWORD} + entrypoint: ["/bin/sh", "-ec"] + command: + - | + umask 077 + sed -e "s|^rpcuser=.*|rpcuser=$${NODE_USER}|" -e "s|^rpcpassword=.*|rpcpassword=$${NODE_PASSWORD}|" /etc/dogecoin/dogecoin.conf > /run/dogecoin.conf + exec dogecoind -conf=/run/dogecoin.conf -datadir=/root/.dogecoin/ + tmpfs: + - /run + ports: + - "127.0.0.1:${DOGE_NODE_HOST_PORT}:18444" + volumes: + - doge-node-data:/root/.dogecoin + healthcheck: + test: ["CMD-SHELL", "dogecoin-cli -conf=/run/dogecoin.conf -datadir=/root/.dogecoin/ getblockchaininfo >/dev/null 2>&1"] + interval: 5s + timeout: 5s + retries: 60 + start_period: 20s + restart: unless-stopped + + doge-miner: + image: xchain-node-dogecoin-regtest-xchain-regtest-miner + depends_on: + doge-node: + condition: service_healthy + environment: + NETWORK: dogecoin-regtest + NODE_URL: doge-node + NODE_PORT: 18444 + NODE_USER: ${RPC_USER} + NODE_PASSWORD: ${RPC_PASSWORD} + REGTEST_MINER_API_PORT: 3005 + ports: + - "127.0.0.1:${DOGE_MINER_HOST_PORT}:3005" + healthcheck: + test: ["CMD-SHELL", "node -e \"const h=require('http');const q=h.request({host:'127.0.0.1',port:3005,method:'POST',headers:{'content-type':'application/json'}},r=>process.exit(r.statusCode===200?0:1));q.on('error',()=>process.exit(1));q.end(JSON.stringify({jsonrpc:'2.0',method:'health',id:1}))\""] + interval: 10s + timeout: 5s + retries: 60 + start_period: 90s + restart: unless-stopped + + doge-tracker: + image: xchain-node-dogecoin-regtest-xchain-utxo-tracker + depends_on: + doge-node: + condition: service_healthy + environment: + COIN: dogecoin + NETWORK: dogecoin-regtest + NODE_URL: doge-node + NODE_PORT: 18444 + NODE_USER: ${RPC_USER} + NODE_PASSWORD: ${RPC_PASSWORD} + UTXO_TRACKER_API_PORT: 3001 + CORS_ORIGIN: "*" + ports: + - "127.0.0.1:${DOGE_TRACKER_HOST_PORT}:3001" + volumes: + - doge-tracker-data:/data/xchain-utxo-tracker + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3001/status >/dev/null 2>&1"] + interval: 10s + timeout: 5s + retries: 60 + start_period: 90s + restart: unless-stopped + + doge-decoder: + image: xchain-node-dogecoin-regtest-xchain-decoder + depends_on: + db: + condition: service_healthy + doge-node: + condition: service_healthy + environment: + COIN: dogecoin + NETWORK: dogecoin-regtest + NODE_URL: doge-node + NODE_PORT: 18444 + NODE_USER: ${RPC_USER} + NODE_PASSWORD: ${RPC_PASSWORD} + DECODER_DB_HOST: db + DECODER_DB_PORT: 3306 + DECODER_DB_NAME: XChain_DOGE_Regtest_Decoder + DECODER_DB_USER: root + DECODER_DB_PASS: ${DB_PASSWORD} + DECODER_API_PORT: 3002 + ports: + - "127.0.0.1:${DOGE_DECODER_HOST_PORT}:3002" + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3002/live >/dev/null 2>&1"] + interval: 10s + timeout: 5s + retries: 60 + start_period: 90s + restart: unless-stopped + + doge-encoder: + image: xchain-node-dogecoin-regtest-xchain-encoder + depends_on: + doge-node: + condition: service_healthy + doge-tracker: + condition: service_healthy + environment: + NETWORK: dogecoin-regtest + NODE_URL: doge-node + NODE_PORT: 18444 + NODE_USER: ${RPC_USER} + NODE_PASSWORD: ${RPC_PASSWORD} + UTXO_TRACKER_URL: doge-tracker + UTXO_TRACKER_API_PORT: 3001 + ENCODER_API_PORT: 3003 + ENCODER_RATE_LIMIT_RPM: 99999 + CORS_ORIGIN: "*" + ports: + - "127.0.0.1:${DOGE_ENCODER_HOST_PORT}:3003" + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3003/status >/dev/null 2>&1"] + interval: 10s + timeout: 5s + retries: 60 + start_period: 90s + restart: unless-stopped + + doge-indexer: + image: xchain-node-dogecoin-regtest-xchain-indexer + depends_on: + db: + condition: service_healthy + doge-decoder: + condition: service_healthy + doge-tracker: + condition: service_healthy + hub: + condition: service_healthy + environment: + INDEXER_COIN: DOGE + INDEXER_NETWORK: regtest + INDEXER_API_PORT: 3004 + INDEXER_DB_HOST: db + INDEXER_DB_PORT: 3306 + INDEXER_DB_NAME: XChain_DOGE_Regtest_Indexer + INDEXER_DB_USER: root + INDEXER_DB_PASS: ${DB_PASSWORD} + DECODER_DB_HOST: db + DECODER_DB_PORT: 3306 + DECODER_DB_NAME: XChain_DOGE_Regtest_Decoder + DECODER_DB_USER: root + DECODER_DB_PASS: ${DB_PASSWORD} + UTXO_TRACKER_URL: doge-tracker + UTXO_TRACKER_API_PORT: 3001 + HUB_API_URL: http://hub:10000 + HUB_DB_HOST: db + HUB_DB_PORT: 3306 + HUB_DB_NAME: XChain_DOGE_Regtest_Indexer + HUB_DB_USER: root + HUB_DB_PASS: ${DB_PASSWORD} + HUB_DB_SYNC_ENABLED: "true" + INDEXER_ALLOW_UNAUTHENTICATED: "true" + HUB_SYNC_PRICE_GRACE_S: 0 + HUB_SYNC_ORACLE_GRACE_S: 0 + HUB_SYNC_ATTEST_RESPONSE_GRACE_S: 0 + HUB_SYNC_MATCH_GRACE_S: 0 + HUB_SYNC_CALL_GRACE_S: 0 + HUB_SYNC_ANCHOR_ATTEST_GRACE_S: 0 + ports: + - "127.0.0.1:${DOGE_INDEXER_HOST_PORT}:3004" + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3004/status >/dev/null 2>&1"] + interval: 10s + timeout: 5s + retries: 90 + start_period: 120s + restart: unless-stopped + + +volumes: + db-data: + node-data: + tracker-data: + ltc-node-data: + ltc-tracker-data: + doge-node-data: + doge-tracker-data: diff --git a/.github/attest-mirror/run-leg.sh b/.github/attest-mirror/run-leg.sh new file mode 100755 index 00000000..0a12018c --- /dev/null +++ b/.github/attest-mirror/run-leg.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Runs one attest-mirror leg file with the npm script's own mocha arguments on one xca7 stack; lines UTC-stamped. +set -u +tag=$1 +stack=$2 +venue_base=$3 +leg=$4 +shift 4 +here=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +root=${L004_VENUE_ROOT:?L004_VENUE_ROOT is required} +fed_env=${L004_FED_ENV:-$here/at7.env} +stack_dir=$here/xca7${stack} +log=${L004_LOG_DIR:?L004_LOG_DIR is required}/${tag}.log +venue_logs=$L004_LOG_DIR/venue/${tag} +mkdir -p "$venue_logs" +exec >"$log" 2>&1 +: node from PATH +set -a +. "$stack_dir/.env" +set +a +export COIN=bitcoin NETWORK=regtest +export NODE_URL=127.0.0.1 NODE_PORT=$NODE_HOST_PORT NODE_USER=$RPC_USER NODE_PASSWORD=$RPC_PASSWORD +export UTXO_TRACKER_URL=127.0.0.1 UTXO_TRACKER_API_PORT=$TRACKER_HOST_PORT +export ENCODER_URL=127.0.0.1 ENCODER_API_PORT=$ENCODER_HOST_PORT +export DECODER_URL=127.0.0.1 DECODER_API_PORT=$DECODER_HOST_PORT +export DECODER_DB_HOST=127.0.0.1 DECODER_DB_PORT=$DB_HOST_PORT DECODER_DB_NAME=XChain_BTC_Regtest_Decoder DECODER_DB_USER=root DECODER_DB_PASS=$DB_PASSWORD +export INDEXER_URL=127.0.0.1 INDEXER_API_PORT=$INDEXER_HOST_PORT INDEXER_DB_NAME=XChain_BTC_Regtest_Indexer INDEXER_DB_USER=root INDEXER_DB_PASS=$DB_PASSWORD +export DATABASE_URL=127.0.0.1 DATABASE_PORT=$DB_HOST_PORT +export REGTEST_MINER_URL=127.0.0.1 REGTEST_MINER_API_PORT=$MINER_HOST_PORT +export EXPLORER_URL=127.0.0.1 EXPLORER_API_PORT=$EXPLORER_HTTP_HOST_PORT +export HUB_URL=127.0.0.1 HUB_API_HOST=127.0.0.1 HUB_HOST=127.0.0.1 HUB_PORT=$HUB_HOST_PORT +export HUB_DB_HOST=127.0.0.1 HUB_DB_PORT=$DB_HOST_PORT HUB_DB_USER=root HUB_DB_PASS=$DB_PASSWORD +export BTC_SERVICE_HOST=127.0.0.1 BTC_NODE_PORT=$NODE_HOST_PORT BTC_NODE_USER=$RPC_USER BTC_NODE_PASSWORD=$RPC_PASSWORD +export BTC_UTXO_TRACKER_API_PORT=$TRACKER_HOST_PORT BTC_DECODER_API_PORT=$DECODER_HOST_PORT BTC_ENCODER_API_PORT=$ENCODER_HOST_PORT +export BTC_INDEXER_API_PORT=$INDEXER_HOST_PORT BTC_REGTEST_MINER_API_PORT=$MINER_HOST_PORT +export BTC_INDEXER_DB_NAME=XChain_BTC_Regtest_Indexer BTC_INDEXER_DB_USER=root BTC_INDEXER_DB_PASS=$DB_PASSWORD +for C in LTC DOGE; do + eval "export ${C}_SERVICE_HOST=127.0.0.1" + eval "export ${C}_NODE_PORT=\$${C}_NODE_HOST_PORT ${C}_UTXO_TRACKER_API_PORT=\$${C}_TRACKER_HOST_PORT" + eval "export ${C}_DECODER_API_PORT=\$${C}_DECODER_HOST_PORT ${C}_ENCODER_API_PORT=\$${C}_ENCODER_HOST_PORT" + eval "export ${C}_INDEXER_API_PORT=\$${C}_INDEXER_HOST_PORT ${C}_REGTEST_MINER_API_PORT=\$${C}_MINER_HOST_PORT" + eval "export ${C}_NODE_USER=\$RPC_USER ${C}_NODE_PASSWORD=\$RPC_PASSWORD" + eval "export ${C}_INDEXER_DB_USER=root ${C}_INDEXER_DB_PASS=\$DB_PASSWORD" +done +export LTC_INDEXER_DB_NAME=XChain_LTC_Regtest_Indexer DOGE_INDEXER_DB_NAME=XChain_DOGE_Regtest_Indexer +export XCHAIN_VENUE_REPO_ROOT=$root ATTEST_VENUE_LOG_DIR=$venue_logs AB_VENUE_BASE_PORT=$venue_base +[ -f "$fed_env" ] && { set -a; . "$fed_env"; set +a; } +for p in "$BTC_NODE_PORT" "$LTC_NODE_PORT" "$DOGE_NODE_PORT"; do + if ! curl -fsS --user "$RPC_USER:$RPC_PASSWORD" --data-binary '{"jsonrpc":"1.0","id":"runner","method":"getblockcount","params":[]}' \ + -H 'content-type:text/plain' "http://127.0.0.1:$p/" >/dev/null; then + printf "runner RPC check failed on port %s for xca7%s\n" "$p" "$stack"; exit 90 + fi +done +cd "${ATTEST_MIRROR_E2E_DIR:-$root/xchain-e2e-test}" +printf "=== leg %s start %s node %s stack xca7%s venue-base %s file %s extra-env [%s]\n" "$tag" "$(date -u +%FT%TZ)" "$(node -v)" "$stack" "$venue_base" "$leg" "$*" +env "$@" npx mocha --timeout 0 --exit --require ./test/initialCheck.test.js "$leg" 2>&1 | node -e 'const rl=require("readline").createInterface({input:process.stdin});rl.on("line",l=>process.stdout.write(new Date().toISOString().slice(11,19)+" "+l+"\n"))' +rc=${PIPESTATUS[0]} +printf "=== leg %s exit=%s end %s\n" "$tag" "$rc" "$(date -u +%FT%TZ)" +exit "$rc" diff --git a/.github/attest-mirror/seed-stack-config.js b/.github/attest-mirror/seed-stack-config.js new file mode 100644 index 00000000..5c028ac7 --- /dev/null +++ b/.github/attest-mirror/seed-stack-config.js @@ -0,0 +1,26 @@ +'use strict'; +// Seeds the stack hub's chain config for BTC, LTC and DOGE so chainRail.createRail can resolve each coin. +const port = Number(process.argv[2]); +const pass = process.env.DB_PASSWORD; +if (!port || !pass) process.exit(2); +const ixHost = { BTC: 'indexer', LTC: 'ltc-indexer', DOGE: 'doge-indexer' }; +function services(code) { + return { + 'xchain-decoder': { host: 'db', port: '3306', name: `XChain_${code}_Regtest_Decoder`, user: 'root', pass }, + 'xchain-indexer': { host: ixHost[code], port: '3004', name: `XChain_${code}_Regtest_Indexer`, user: 'root', pass } + }; +} +const config = { + bitcoin: { regtest: services('BTC') }, + litecoin: { regtest: services('LTC') }, + dogecoin: { regtest: services('DOGE') } +}; +fetch(`http://127.0.0.1:${port}/`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'updateconfig', params: { config } }) +}).then(async (response) => { + const body = await response.json(); + if (!response.ok || !body.result || body.result.status !== 'success') process.exit(3); + process.stdout.write('stack config seeded (bitcoin, litecoin, dogecoin)\n'); +}).catch(() => process.exit(4)); diff --git a/.github/attest-mirror/setup-stack.sh b/.github/attest-mirror/setup-stack.sh new file mode 100755 index 00000000..597d93c5 --- /dev/null +++ b/.github/attest-mirror/setup-stack.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Bring up one xca7 isolated stack (BTC, LTC and DOGE regtest services) and seed the stack hub's chain config. +set -eu +stack=$1 +base=$2 +root=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +dir=$root/xca7${stack} +# Host ports must sit above the kernel ephemeral range (32768 to 60999): a 57xxx base collided with a client socket on 2026-09-17. +case $base in 57730) base=64200 ;; 57760) base=64100 ;; esac +mkdir -p "$dir" +cp "$root/compose.yml" "$dir/compose.yml" +cd "$dir" +umask 077 +rpc_secret=$(openssl rand -hex 16) +db_secret=$(openssl rand -hex 16) +printf "%s\n" \ + "RPC_USER=xca7${stack}" \ + "RPC_PASSWORD=$rpc_secret" \ + "DB_PASSWORD=$db_secret" \ + "DB_HOST_PORT=$base" \ + "NODE_HOST_PORT=$((base + 1))" \ + "TRACKER_HOST_PORT=$((base + 2))" \ + "DECODER_HOST_PORT=$((base + 3))" \ + "ENCODER_HOST_PORT=$((base + 4))" \ + "INDEXER_HOST_PORT=$((base + 5))" \ + "MINER_HOST_PORT=$((base + 6))" \ + "EXPLORER_HTTP_HOST_PORT=$((base + 7))" \ + "EXPLORER_HTTPS_HOST_PORT=$((base + 8))" \ + "HUB_HOST_PORT=$((base + 9))" \ + "LTC_NODE_HOST_PORT=$((base + 10))" \ + "LTC_TRACKER_HOST_PORT=$((base + 11))" \ + "LTC_DECODER_HOST_PORT=$((base + 12))" \ + "LTC_ENCODER_HOST_PORT=$((base + 13))" \ + "LTC_INDEXER_HOST_PORT=$((base + 14))" \ + "LTC_MINER_HOST_PORT=$((base + 15))" \ + "DOGE_NODE_HOST_PORT=$((base + 16))" \ + "DOGE_TRACKER_HOST_PORT=$((base + 17))" \ + "DOGE_DECODER_HOST_PORT=$((base + 18))" \ + "DOGE_ENCODER_HOST_PORT=$((base + 19))" \ + "DOGE_INDEXER_HOST_PORT=$((base + 20))" \ + "DOGE_MINER_HOST_PORT=$((base + 21))" > .env +chmod 600 .env +date -u +docker compose -p "xca7${stack}" up -d +# Health is read from docker ps status text, never docker inspect (inspect can print env). +health() { + docker ps --filter "name=^xca7${stack}-$1-1\$" --format "{{.Status}}" +} +for _ in $(seq 1 180); do + ok=1 + for svc in hub indexer ltc-indexer doge-indexer miner tracker decoder encoder ltc-miner doge-miner; do + case "$(health $svc)" in *"(healthy)"*) ;; *) ok=0 ;; esac + done + if [ "$ok" = 1 ]; then + set -a + . ./.env + set +a + node "$root/seed-stack-config.js" "$HUB_HOST_PORT" + node "$root/wait-indexer-schema.js" + date -u + exit 0 + fi + sleep 2 +done +date -u +for svc in hub indexer ltc-indexer doge-indexer; do printf "%s=%s\n" "$svc" "$(health $svc)"; done +printf "health timeout for xca7%s\n" "$stack" +exit 1 diff --git a/.github/attest-mirror/wait-indexer-schema.js b/.github/attest-mirror/wait-indexer-schema.js new file mode 100644 index 00000000..3b55483a --- /dev/null +++ b/.github/attest-mirror/wait-indexer-schema.js @@ -0,0 +1,23 @@ +'use strict'; +// Venue warm-up: waits until the stack indexer's schema exists (issues table) and its /status answers, so a leg's +// initialCheck ISSUE never races the indexer's first migration. Credentials come from the stack .env via env. +const mariadb = require('module').createRequire(require('path').join(process.env.ATTEST_MIRROR_E2E_DIR, 'package.json'))('mariadb'); +const waitMs = Number(process.env.ATTEST_SCHEMA_WAIT_MS || 1200000); +const deadline = Date.now() + waitMs; +(async () => { + for (;;) { + let ok = false; + try { + const options = { host: '127.0.0.1', port: Number(process.env.DB_HOST_PORT), user: 'root' }; + options['pass' + 'word'] = process.env['DB_' + 'PASSWORD']; + const c = await mariadb.createConnection(options); + const rows = await c.query("SELECT COUNT(*) AS n FROM information_schema.tables WHERE table_schema = 'XChain_BTC_Regtest_Indexer' AND table_name = 'issues'"); + await c.end(); + const res = await fetch(`http://127.0.0.1:${process.env.INDEXER_HOST_PORT}/status`).catch(() => null); + ok = Number(rows[0].n) === 1 && !!res && res.ok; + } catch (e) { ok = false; } + if (ok) { console.log('indexer schema ready'); return; } + if (Date.now() > deadline) { console.log('indexer schema NOT ready after ' + Math.round(waitMs / 1000) + ' s'); process.exit(5); } + await new Promise((r) => setTimeout(r, 3000)); + } +})(); diff --git a/.github/rail-legs.json b/.github/rail-legs.json new file mode 100644 index 00000000..98d2b12e --- /dev/null +++ b/.github/rail-legs.json @@ -0,0 +1,146 @@ +[ + { + "row": "L004-ACC", + "drive": ".github/attest-mirror/run-leg.sh", + "leg": "test/attestMirror/barrier_family/ab4_below_activation.test.js", + "e2e_ref": "develop" + }, + { + "row": "L004-ACC", + "drive": ".github/attest-mirror/run-leg.sh", + "leg": "test/attestMirror/venue_levers_smoke.test.js", + "e2e_ref": "develop" + }, + { + "row": "L004-ACC", + "drive": ".github/attest-mirror/run-leg.sh", + "leg": "test/attestMirror/at2-dissemination-determinism.test.js", + "e2e_ref": "develop" + }, + { + "row": "L004-ACC", + "drive": ".github/attest-mirror/run-leg.sh", + "leg": "test/attestMirror/barrier_family/bf6_producer_follower_parity.test.js", + "e2e_ref": "develop" + }, + { + "row": "L004-ACC", + "drive": ".github/attest-mirror/run-leg.sh", + "leg": "test/attestMirror/at1_mirror_finalize.test.js", + "e2e_ref": "develop" + }, + { + "row": "L004-ACC", + "drive": ".github/attest-mirror/run-leg.sh", + "leg": "test/attestMirror/zc4_fallthrough.test.js", + "e2e_ref": "develop" + }, + { + "row": "L004-ACC", + "drive": ".github/attest-mirror/run-leg.sh", + "leg": "test/attestMirror/at5_batch_replay.test.js", + "e2e_ref": "develop" + }, + { + "row": "L004-ACC", + "drive": ".github/attest-mirror/run-leg.sh", + "leg": "test/attestMirror/barrier_family/bf8_second_coin.test.js", + "e2e_ref": "develop" + }, + { + "row": "L004-ACC", + "drive": ".github/attest-mirror/run-leg.sh", + "leg": "test/attestMirror/barrier_family/bf2_armed_fix.test.js", + "e2e_ref": "develop" + }, + { + "row": "L004-ACC", + "drive": ".github/attest-mirror/run-leg.sh", + "leg": "test/attestMirror/barrier_family/bf4_boundary.test.js", + "e2e_ref": "develop" + }, + { + "row": "L004-ACC", + "drive": ".github/attest-mirror/run-leg.sh", + "leg": "test/attestMirror/barrier_family/bf5_flag_day.test.js", + "e2e_ref": "develop" + }, + { + "row": "L004-ACC", + "drive": ".github/attest-mirror/run-leg.sh", + "leg": "test/attestMirror/at6_flag_day.test.js", + "e2e_ref": "develop" + }, + { + "row": "L004-ACC", + "drive": ".github/attest-mirror/run-leg.sh", + "leg": "test/attestMirror/zc1_zero_conf.test.js", + "e2e_ref": "develop" + }, + { + "row": "L004-ACC", + "drive": ".github/attest-mirror/run-leg.sh", + "leg": "test/attestMirror/barrier_family/ab3_safety_preserved.test.js", + "e2e_ref": "develop" + }, + { + "row": "L004-ACC", + "drive": ".github/attest-mirror/run-leg.sh", + "leg": "test/attestMirror/zc3_headroom.test.js", + "e2e_ref": "develop" + }, + { + "row": "L004-ACC", + "drive": ".github/attest-mirror/run-leg.sh", + "leg": "test/attestMirror/at4_reorg.test.js", + "e2e_ref": "develop" + }, + { + "row": "L004-ACC", + "drive": ".github/attest-mirror/run-leg.sh", + "leg": "test/attestMirror/barrier_family/ab5_mixed_fleet.test.js", + "e2e_ref": "develop" + }, + { + "row": "L004-ACC", + "drive": ".github/attest-mirror/run-leg.sh", + "leg": "test/attestMirror/barrier_family/at4_signed_admission_corpus.test.js", + "e2e_ref": "develop" + }, + { + "row": "L004-ACC", + "drive": ".github/attest-mirror/run-leg.sh", + "leg": "test/attestMirror/at0-anti-wedge.test.js", + "e2e_ref": "develop" + }, + { + "row": "L004-ACC", + "drive": ".github/attest-mirror/run-leg.sh", + "leg": "test/attestMirror/zc2_remine.test.js", + "e2e_ref": "develop" + }, + { + "row": "L004-ACC", + "drive": ".github/attest-mirror/run-leg.sh", + "leg": "test/attestMirror/venue.smoke.test.js", + "e2e_ref": "develop" + }, + { + "row": "L004-ACC", + "drive": ".github/attest-mirror/run-leg.sh", + "leg": "test/attestMirror/zc5_flag_day.test.js", + "e2e_ref": "develop" + }, + { + "row": "L004-ACC", + "drive": ".github/attest-mirror/run-leg.sh", + "leg": "test/attestMirror/barrier_family/ab2_armed_fix.test.js", + "e2e_ref": "develop" + }, + { + "row": "L004-ACC", + "drive": ".github/attest-mirror/run-leg.sh", + "leg": "test/attestMirror/at3-deadline.test.js", + "e2e_ref": "develop" + } +] diff --git a/.github/workflows/attest-mirror-leg.yml b/.github/workflows/attest-mirror-leg.yml new file mode 100644 index 00000000..445a7c29 --- /dev/null +++ b/.github/workflows/attest-mirror-leg.yml @@ -0,0 +1,231 @@ +name: Attest-mirror leg (regtest) + +run-name: >- + Attest-mirror legs @ e2e ${{ inputs.e2e_ref || 'develop' }} ${{ inputs.legs || '(all)' }} + +# Runs L004-ACC's attest-mirror legs (`BF_STAMP_AHEAD_S=900 npm run test:attest-mirror` +# in xchain-e2e-test) on GitHub-hosted runners, ONE leg per job, all in parallel, +# through the same runner the self-hosted venues use (scripts/run-attest-mirror.js +# with scripts/attest-mirror-stack-driver.sh), instead of three legs at a time on +# one venue disk. +# +# The stack root in .github/attest-mirror/ is test02's pb-v020-at7 root (compose.yml, +# setup-stack.sh, run-leg.sh, seed-stack-config.js, wait-indexer-schema.js) with +# only its host paths made relative: the driver boots an isolated BTC+LTC+DOGE +# compose stack per leg from the xchain-node images, which this job builds first by +# installing the three regtest stacks. The federation mnemonic the venue keeps in +# at7.env is generated per job and masked. +# +# EVIDENCE: artifact am- holds the runner output, the leg log and the +# venue logs; the leg's verdict is its exit code, as on the venue. + +on: + schedule: + - cron: '30 6 * * *' + workflow_dispatch: + inputs: + e2e_ref: + description: 'xchain-e2e-test ref the legs run from' + type: string + default: develop + legs: + description: 'Comma-separated leg files (test/attestMirror/...); empty runs all' + type: string + default: '' + stack_ref: + description: 'Ref the service images and source roots are built at' + type: string + default: develop + +permissions: + contents: read + +jobs: + plan: + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + legs: ${{ steps.legs.outputs.legs }} + e2e_ref: ${{ steps.legs.outputs.e2e_ref }} + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: .github + - name: List the legs + id: legs + env: + E2E_REF: ${{ inputs.e2e_ref || 'develop' }} + LEGS_INPUT: ${{ inputs.legs }} + EVENT: ${{ github.event_name }} + run: | + if [ "$EVENT" = schedule ]; then + E2E_REF=develop + LEGS_INPUT=$(node -e 'const l=require("./.github/rail-legs.json").filter(x=>x.drive===".github/attest-mirror/run-leg.sh").map(x=>x.leg);if(!l.length)throw new Error("no scheduled attest-mirror legs");process.stdout.write(l.join(","))') + fi + echo "e2e_ref=$E2E_REF" >> "$GITHUB_OUTPUT" + git clone -q https://github.com/XChain-Platform/xchain-e2e-test.git e2e + git -C e2e fetch -q origin "$E2E_REF" && git -C e2e checkout -q FETCH_HEAD + if [ -n "$LEGS_INPUT" ]; then + LEGS=$(node -e 'console.log(JSON.stringify(process.argv[1].split(",").map(s=>s.trim()).filter(Boolean)))' "$LEGS_INPUT") + else + LEGS=$(cd e2e && ls test/attestMirror/*.test.js test/attestMirror/barrier_family/*.test.js | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>console.log(JSON.stringify(s.split("\n").filter(Boolean))))') + fi + echo "$LEGS" + echo "legs=$LEGS" >> "$GITHUB_OUTPUT" + + leg: + needs: plan + name: ${{ matrix.leg }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + leg: ${{ fromJSON(needs.plan.outputs.legs) }} + timeout-minutes: 240 + env: + GITHUB_TOKEN: ${{ github.token }} + STACK_REF: ${{ github.event_name == 'schedule' && 'develop' || inputs.stack_ref || 'develop' }} + E2E_REF: ${{ needs.plan.outputs.e2e_ref }} + XCHAIN_NODE_EXTERNAL_DB: '1' + XCHAIN_NODE_EXTERNAL_DB_HOST: '172.17.0.1' + XCHAIN_NODE_EXTERNAL_DB_PORT: '3306' + XCHAIN_NODE_EXTERNAL_DB_ROOT_USER: 'root' + XCHAIN_NODE_DATA_DIR: ${{ github.workspace }}/.xchain-data + steps: + - uses: actions/checkout@v4 + + - name: Reclaim runner disk + run: | + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android "$AGENT_TOOLSDIRECTORY" || true + docker system prune -af || true + df -h /; nproc; free -g + + # AFTER the reclaim, which deletes the tool cache setup-node installs into; + # before it, later steps silently fell back to the image's own Node (v22.23.2 + # on some images, v22.23.3 on others, the latter failing the consensus pin). + - uses: actions/setup-node@v4 + with: + node-version: '22.22.3' + + - name: Start headless MariaDB (for the image-building install) + run: | + XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD="$(openssl rand -hex 24)" + echo "::add-mask::$XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD" + echo "XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD=$XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD" >> "$GITHUB_ENV" + docker run -d --name xchain-ci-db \ + -e MARIADB_ROOT_PASSWORD="$XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD" \ + -p 3306:3306 mariadb:11 --max-connections=1000 + for i in $(seq 1 30); do + if docker exec xchain-ci-db mariadb-admin -uroot -p"$XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD" ping 2>/dev/null | grep -q "is alive"; then + echo "MariaDB ready"; break + fi + sleep 3 + done + + - name: Build the service images (install the three regtest stacks) + # Authenticated API reads: the anonymous limit (60 an hour per runner IP) is + # shared by every job on the runner pool. + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + npm ci + git config --global url."https://github.com/".insteadOf "git@github.com:" + write_ports() { + local file="$1" base="$2"; shift 2 + { + echo "NODE_EXPOSED_PORT=$base" + echo "UTXO_TRACKER_PORT=$((base + 1))" + echo "DECODER_PORT=$((base + 2))" + echo "ENCODER_PORT=$((base + 3))" + echo "INDEXER_PORT=$((base + 4))" + echo "REGTEST_MINER_PORT=$((base + 5))" + for line in "$@"; do echo "$line"; done + } > "$file" + } + mkdir -p config "$XCHAIN_NODE_DATA_DIR" + write_ports config/bitcoin-regtest 3020 + write_ports config/dogecoin-regtest 3120 "BTC_SERVICE_HOST=$XCHAIN_NODE_EXTERNAL_DB_HOST" \ + "BTC_INDEXER_API_URL=http://xchain-node-bitcoin-regtest-xchain-indexer:3004" + write_ports config/litecoin-regtest 3220 "BTC_SERVICE_HOST=$XCHAIN_NODE_EXTERNAL_DB_HOST" \ + "BTC_INDEXER_API_URL=http://xchain-node-bitcoin-regtest-xchain-indexer:3004" + node src/index.js install "$STACK_REF" all bitcoin regtest + node src/index.js install "$STACK_REF" all litecoin regtest + node src/index.js install "$STACK_REF" all dogecoin regtest + # The standing stacks stay up, as on test02: legs read the standing + # explorer (18080) and the standing LTC/DOGE indexers (at0-anti-wedge reads 3224). + docker pull -q mariadb:10.11 && docker tag mariadb:10.11 xchain-node-database + docker images --format '{{.Repository}}' | grep '^xchain-node-' | sort + docker ps --format '{{.Names}} {{.Status}}' + + - name: Build the source root (e2e at ${{ needs.plan.outputs.e2e_ref }}) + run: | + ROOT="$RUNNER_TEMP/rail" + mkdir -p "$ROOT" + echo "RAIL_ROOT=$ROOT" >> "$GITHUB_ENV" + ln -sfn "$GITHUB_WORKSPACE" "$ROOT/xchain-node" + git clone -q https://github.com/XChain-Platform/xchain-e2e-test.git "$ROOT/xchain-e2e-test" + git -C "$ROOT/xchain-e2e-test" fetch -q origin "$E2E_REF" + git -C "$ROOT/xchain-e2e-test" checkout -q FETCH_HEAD + echo "e2e at $(git -C "$ROOT/xchain-e2e-test" rev-parse HEAD) ($E2E_REF)" + REPOS="xchain-hub xchain-indexer xchain-vm xchain-sdk xchain-decoder xchain-utxo-tracker xchain-encoder xchain-explorer xchain-documentation xchain-contracts xchain-regtest-miner xchain-sync" + for r in $REPOS; do git clone -q --depth 1 --branch "$STACK_REF" "https://github.com/XChain-Platform/$r.git" "$ROOT/$r" & done + wait + for r in $REPOS; do echo "$r at $(git -C "$ROOT/$r" rev-parse --short HEAD 2>/dev/null)"; done + ln -sfn ../xchain-vm "$ROOT/xchain-indexer/xchain-vm" + for s in xchain-hub xchain-sdk xchain-vm; do ln -sfn "../$s" "$ROOT/xchain-e2e-test/$s"; done + for r in xchain-vm xchain-hub xchain-sdk xchain-decoder xchain-utxo-tracker xchain-encoder; do + (cd "$ROOT/$r" && npm ci --no-audit --no-fund --silent > "$RUNNER_TEMP/npm-$r.log" 2>&1 || { echo "npm ci failed in $r"; tail -30 "$RUNNER_TEMP/npm-$r.log"; exit 1; }) & + done + wait + (cd "$ROOT/xchain-indexer" && npm ci --no-audit --no-fund --silent) + (cd "$ROOT/xchain-e2e-test" && npm ci --no-audit --no-fund --silent) + # The in-process venue indexers refuse any runtime but the fleet's pinned + # one (xchain-vm/src/consensus-runtime.js: v8, icu, unicode, cldr, ABI 127), + # so check the job's Node against that pin before spending a drive on it. + node -e 'const c=require(process.argv[1]+"/xchain-vm/src/consensus-runtime.js"); const r=c.checkConsensusRuntime(); console.log("node "+process.version+" consensus runtime: "+c.describeMismatch(r)); process.exit(r.ok?0:1)' "$ROOT" + + - name: Stage the stack root + run: | + S="$RUNNER_TEMP/am" + cp -r .github/attest-mirror "$S" + chmod +x "$S"/*.sh + umask 077 + M=$(node -e 'console.log(require(process.argv[1] + "/node_modules/bip39").generateMnemonic())' "$RAIL_ROOT/xchain-e2e-test") + echo "::add-mask::$M" + printf 'XC_ROLLCALL_FEDERATION_MNEMONIC="%s"\n' "$M" > "$S/at7.env" + echo "AM_ROOT=$S" >> "$GITHUB_ENV" + echo "L004_LOG_DIR=$RUNNER_TEMP/am-logs" >> "$GITHUB_ENV" + mkdir -p "$RUNNER_TEMP/am-logs" + + - name: Run ${{ matrix.leg }} + working-directory: ${{ env.RAIL_ROOT }}/xchain-e2e-test + env: + LEG: ${{ matrix.leg }} + run: | + set +e + env L004_VENUE_ROOT="$RAIL_ROOT" ATTEST_MIRROR_CONCURRENCY=1 ATTEST_MIRROR_READY_TIMEOUT_MS=1200000 \ + ATTEST_MIRROR_STACK_ROOT="$AM_ROOT" ATTEST_MIRROR_STACK_PREFIX=xca7 \ + ATTEST_MIRROR_PORT_BASE=62100 ATTEST_MIRROR_VENUE_PORT_BASE=63400 \ + ATTEST_MIRROR_LEGS="$LEG" \ + ATTEST_MIRROR_STACK_DRIVER="$RAIL_ROOT/xchain-e2e-test/scripts/attest-mirror-stack-driver.sh" \ + ATTEST_MIRROR_E2E_DIR="$RAIL_ROOT/xchain-e2e-test" ATTEST_MIRROR_EXPLORER_PORT=18080 \ + BF_STAMP_AHEAD_S=900 node scripts/run-attest-mirror.js 2>&1 | tee "$L004_LOG_DIR/runner.log" + rc=${PIPESTATUS[0]} + echo "runner exit $rc" + for f in "$L004_LOG_DIR"/*.log; do [ "$f" = "$L004_LOG_DIR/runner.log" ] && continue; echo "=== $f"; tail -60 "$f"; done + exit "$rc" + + - name: Name the artifact + if: always() + env: + LEG: ${{ matrix.leg }} + run: echo "AM_ARTIFACT=am-$(basename "$LEG" .test.js)" >> "$GITHUB_ENV" + + - name: Upload the leg logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ env.AM_ARTIFACT }} + path: ${{ runner.temp }}/am-logs/ + if-no-files-found: warn + retention-days: 30 diff --git a/.github/workflows/rail-leg.yml b/.github/workflows/rail-leg.yml new file mode 100644 index 00000000..c25af55c --- /dev/null +++ b/.github/workflows/rail-leg.yml @@ -0,0 +1,329 @@ +name: Bridge rail leg (regtest) + +run-name: >- + Rail legs ${{ inputs.legs }} @ stack ${{ inputs.stack_ref || 'develop' }} + +# Runs xchain-e2e-test bridge rail acceptance legs (scripts/rail_leg_drive.js) on +# GitHub-hosted runners, one leg per job, many in parallel, beside the three +# self-hosted rails (DankServer, test01, test02) whose single drive slot per venue +# is the release push's bottleneck. +# +# WHY A HOSTED RUNNER CAN CARRY A LEG. A leg needs a BTC and a DOGE regtest stack, +# a hub and indexers; the nightly's dogecoin shard already boots exactly that +# two-stack shape (DOGE on host ports 3120-3125, BTC on 3020-3025, hub 10000) in +# about five minutes. The drive then runs on the runner HOST, like on the rails: +# it spawns its own in-process venue hubs and indexers from sibling source trees +# under BRIDGE_RAIL_REPO_ROOT and reaches the standing services through their +# published ports, via a host .env that scripts/rail_leg_env.js derives from the +# running containers. A fresh chain needs no operator secret: the bridge +# capability set is near empty and XCHAIN MINT is an open faucet far below +# MAX_SUPPLY, so the harness stakes its own quorum. +# +# EVIDENCE. Each job uploads the leg's journal dir (case-journal.jsonl plus the +# venue logs) as artifact rail--, so a row verify reads it with +# `gh run download -R XChain-Platform/xchain-node -n rail-- -D `. +# +on: + workflow_dispatch: + inputs: + legs: + description: 'JSON array of {"row","drive","leg","e2e_ref"}' + type: string + required: true + stack_ref: + description: 'Ref every service repo is installed and cloned at' + type: string + default: develop + limit_minutes: + description: 'Per-leg bound passed to rail_leg_drive.js --limit-minutes' + type: string + default: '200' + +permissions: + contents: read + +jobs: + plan: + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + legs: ${{ steps.legs.outputs.legs }} + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: .github + - name: Resolve the leg list + id: legs + env: + LEGS_INPUT: ${{ inputs.legs }} + run: | + test -n "$LEGS_INPUT" + printf '%s' "$LEGS_INPUT" > "$RUNNER_TEMP/legs.json" + LEGS=$(node -e 'const l=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8")); if(!Array.isArray(l)||!l.length) throw new Error("empty leg list"); for (const x of l) for (const k of ["row","drive","leg","e2e_ref"]) if(!x[k]) throw new Error("leg missing "+k+": "+JSON.stringify(x)); console.log(JSON.stringify(l))' "$RUNNER_TEMP/legs.json") + echo "$LEGS" + echo "legs=$LEGS" >> "$GITHUB_OUTPUT" + + leg: + needs: plan + name: ${{ matrix.row }} ${{ matrix.drive }} ${{ matrix.leg }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + max-parallel: 20 + matrix: + include: ${{ fromJSON(needs.plan.outputs.legs) }} + timeout-minutes: 350 + env: + GITHUB_TOKEN: ${{ github.token }} + COIN: dogecoin + STACK_REF: ${{ inputs.stack_ref || 'develop' }} + LIMIT_MINUTES: ${{ inputs.limit_minutes || '200' }} + XCHAIN_NODE_EXTERNAL_DB: '1' + XCHAIN_NODE_EXTERNAL_DB_HOST: '172.17.0.1' + XCHAIN_NODE_EXTERNAL_DB_PORT: '3306' + XCHAIN_NODE_EXTERNAL_DB_ROOT_USER: 'root' + XCHAIN_NODE_DATA_DIR: ${{ github.workspace }}/.xchain-data + steps: + - uses: actions/checkout@v4 + + - name: Reclaim runner disk + run: | + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android "$AGENT_TOOLSDIRECTORY" || true + docker system prune -af || true + df -h /; nproc; free -g + + # AFTER the reclaim, which deletes the tool cache setup-node installs into; + # before it, later steps silently fell back to the image's own Node (v22.23.2 + # on some images, v22.23.3 on others, the latter failing the consensus pin). + - uses: actions/setup-node@v4 + with: + node-version: '22.22.3' + + - name: Start headless MariaDB + run: | + XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD="$(openssl rand -hex 24)" + echo "::add-mask::$XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD" + echo "XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD=$XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD" >> "$GITHUB_ENV" + docker run -d --name xchain-ci-db \ + -e MARIADB_ROOT_PASSWORD="$XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD" \ + -p 3306:3306 mariadb:11 --max-connections=2000 + for i in $(seq 1 30); do + if docker exec xchain-ci-db mariadb-admin -uroot -p"$XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD" ping 2>/dev/null | grep -q "is alive"; then + echo "MariaDB ready"; break + fi + sleep 3 + done + + - name: Install xchain-node deps + # Authenticated API reads: the anonymous limit (60 an hour per runner IP) is + # shared by every job on the runner pool. + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + npm ci + git config --global url."https://github.com/".insteadOf "git@github.com:" + + # Same shape as the nightly's dogecoin shard: a local validator hub, DOGE + # stack on 3120-3125 with the bitcoin stack beside it on 3020-3025. The stack + # settings go to a file the install step sources, NOT to GITHUB_ENV: they + # carry docker hostnames (xchain-node-bitcoin-regtest-xchain-indexer), and the + # drive's in-process venue hubs inherit the job env, so a leaked + # BTC_INDEXER_API_URL left every venue hub unable to read its capability set. + - name: Initialize the validator identity + run: | + node src/index.js validator init \ + --network regtest \ + --oracle-epoch-start "$(date -u +%s)000" \ + --capabilities price,cross_chain,oracle_publish,attestation + { + echo "HUB_NETWORK=regtest" + echo "HUB_SYNC_ANCHOR_ATTEST_GRACE_S=2" + echo "HUB_SYNC_BRIDGE_GRACE_S=2" + echo "HUB_SYNC_POLICY_GRACE_S=2" + echo "HUB_PRICE_SYNC_TIMEOUT_MS=10000" + echo "ORACLE_MIN_SUBMISSIONS=1" + echo "XDEX_SEED_LOCAL_VALIDATOR=1" + echo "BTC_INDEXER_API_URL=http://xchain-node-bitcoin-regtest-xchain-indexer:3004" + echo "BTC_INDEXER_URL=http://xchain-node-bitcoin-regtest-xchain-indexer:3004" + echo "DOGE_INDEXER_URL=http://xchain-node-dogecoin-regtest-xchain-indexer:3004" + echo "XCHAIN_CONFIRMATIONS_BTC=1" + echo "XCHAIN_CONFIRMATIONS_LTC=1" + echo "XCHAIN_CONFIRMATIONS_DOGE=1" + } > "$RUNNER_TEMP/stack.env" + + - name: Publish the two-stack host ports + run: | + write_ports() { + local file="$1" base="$2"; shift 2 + { + echo "NODE_EXPOSED_PORT=$base" + echo "UTXO_TRACKER_PORT=$((base + 1))" + echo "DECODER_PORT=$((base + 2))" + echo "ENCODER_PORT=$((base + 3))" + echo "INDEXER_PORT=$((base + 4))" + echo "REGTEST_MINER_PORT=$((base + 5))" + for line in "$@"; do echo "$line"; done + } > "$file" + } + mkdir -p config + write_ports config/dogecoin-regtest 3120 \ + "BTC_SERVICE_HOST=$XCHAIN_NODE_EXTERNAL_DB_HOST" \ + "BTC_INDEXER_API_URL=http://xchain-node-bitcoin-regtest-xchain-indexer:3004" + write_ports config/bitcoin-regtest 3020 + + - name: Boot the two regtest stacks at ${{ inputs.stack_ref || 'develop' }} + # Authenticated API reads: the anonymous limit (60 an hour per runner IP) is + # shared by every job on the runner pool. + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + mkdir -p "$XCHAIN_NODE_DATA_DIR" + set -a; . "$RUNNER_TEMP/stack.env"; set +a + node src/index.js install "$STACK_REF" all dogecoin regtest + node src/index.js install "$STACK_REF" all bitcoin regtest + docker ps --format '{{.Names}} {{.Status}} {{.Ports}}' + + - name: Build the rail root (e2e at ${{ matrix.e2e_ref }}, services at the stack ref) + env: + E2E_REF: ${{ matrix.e2e_ref }} + run: | + ROOT="$RUNNER_TEMP/rail" + mkdir -p "$ROOT" + echo "RAIL_ROOT=$ROOT" >> "$GITHUB_ENV" + ln -sfn "$GITHUB_WORKSPACE" "$ROOT/xchain-node" + git clone -q https://github.com/XChain-Platform/xchain-e2e-test.git "$ROOT/xchain-e2e-test" + git -C "$ROOT/xchain-e2e-test" fetch -q origin "$E2E_REF" + git -C "$ROOT/xchain-e2e-test" checkout -q FETCH_HEAD + echo "e2e at $(git -C "$ROOT/xchain-e2e-test" rev-parse HEAD) ($E2E_REF)" + for r in xchain-hub xchain-indexer xchain-vm xchain-sdk xchain-decoder xchain-utxo-tracker xchain-encoder xchain-explorer xchain-documentation xchain-contracts; do + git clone -q --depth 1 --branch "$STACK_REF" "https://github.com/XChain-Platform/$r.git" "$ROOT/$r" & + done + wait + for r in xchain-hub xchain-indexer xchain-vm xchain-sdk xchain-decoder xchain-utxo-tracker xchain-encoder xchain-explorer xchain-documentation xchain-contracts; do + echo "$r at $(git -C "$ROOT/$r" rev-parse --short HEAD)" + done + ln -sfn ../xchain-vm "$ROOT/xchain-indexer/xchain-vm" + for s in xchain-hub xchain-sdk xchain-vm; do ln -sfn "../$s" "$ROOT/xchain-e2e-test/$s"; done + for r in xchain-vm xchain-hub xchain-sdk xchain-decoder xchain-utxo-tracker xchain-encoder; do + (cd "$ROOT/$r" && npm ci --no-audit --no-fund --silent > "$RUNNER_TEMP/npm-$r.log" 2>&1 || { echo "npm ci failed in $r"; tail -30 "$RUNNER_TEMP/npm-$r.log"; exit 1; }) & + done + wait + (cd "$ROOT/xchain-indexer" && npm ci --no-audit --no-fund --silent) + (cd "$ROOT/xchain-e2e-test" && npm ci --no-audit --no-fund --silent) + # The in-process venue indexers refuse any runtime but the fleet's pinned + # one (xchain-vm/src/consensus-runtime.js: v8, icu, unicode, cldr, ABI 127), + # so check the job's Node against that pin before spending a drive on it. + node -e 'const c=require(process.argv[1]+"/xchain-vm/src/consensus-runtime.js"); const r=c.checkConsensusRuntime(); console.log("node "+process.version+" consensus runtime: "+c.describeMismatch(r)); process.exit(r.ok?0:1)' "$ROOT" + node -e 'require(process.argv[1] + "/xchain-indexer/node_modules/xchain-vm"); console.log("xchain-indexer loads xchain-vm")' "$ROOT" + + - name: Write the host .env for the drive + run: | + node scripts/rail_leg_env.js "$RAIL_ROOT/xchain-e2e-test/.env" bitcoin + mkdir -p "$RAIL_ROOT/xchain-e2e-test/drill-keys" + chmod 700 "$RAIL_ROOT/xchain-e2e-test/drill-keys" + + # The self-hosted rails were seeded once with the four-source ROLLCALL + # federation (test/tools/rollcallSeedFederation.test.js), whose keys the + # harness can sign for; the bridge drives reuse that seated set. A fresh + # chain has an EMPTY capability set, and the policy bootstrap only places + # new signers in gaps between seated keys, so with nothing seated it stakes + # nothing. Seeding here gives the runner the same shape the rails have. The + # mnemonic is generated per job, masked, and dies with the runner: it + # guards nothing but this ephemeral chain. E2E_STAKE_TEARDOWN=off keeps the + # seeded stakes after the tool exits, as on the rails (the default teardown + # unstakes them, leaving the drive an empty set again). + - name: Seed the rail federation (ephemeral mnemonic) + working-directory: ${{ env.RAIL_ROOT }}/xchain-e2e-test + run: | + M=$(node -e 'console.log(require("bip39").generateMnemonic())') + echo "::add-mask::$M" + echo "XC_ROLLCALL_FEDERATION_MNEMONIC=$M" >> "$GITHUB_ENV" + env XC_ROLLCALL_FEDERATION_MNEMONIC="$M" COIN=bitcoin NETWORK=regtest NODE_PATH="$RAIL_ROOT" E2E_STAKE_TEARDOWN=off \ + ./node_modules/.bin/mocha --no-config --timeout 0 --exit \ + --require ./test/initialCheck.test.js test/tools/rollcallSeedFederation.test.js + + # Fail fast when the seeded federation is not a member of the sets the drive + # resolves its quorum from, instead of 25 s into the drive. Reads the BTC + # indexer the way the seed tool does: getstakeweightsbycapability at the tip. + - name: Check the seated capability sets + run: | + node - <<'JS' + const rpc = async (method, params) => { + const r = await fetch('http://127.0.0.1:3024/', { method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ jsonrpc: '2.0', id: 1, method, params }) }) + const j = await r.json() + if (j.error) throw new Error(method + ': ' + JSON.stringify(j.error)) + return j.result + } + ;(async () => { + const tip = Number((await rpc('getblockhashes', {})).block_index) + let crossChain = null + for (const capability of ['oracle_publish', 'cross_chain', 'price', 'attestation']) { + const res = await rpc('getstakeweightsbycapability', { capability, block_index: tip }) + if (!res || !Array.isArray(res.validators)) { console.log(capability + ': unexpected shape ' + JSON.stringify(res).slice(0, 300)); continue } + const v = res.validators + const weight = v.reduce((sum, x) => sum + Number(x.weight || 0), 0) + console.log(capability + ' at block ' + tip + ': ' + v.length + ' member(s), weight ' + weight) + if (capability === 'cross_chain') crossChain = v.length + } + if (crossChain === 0) { console.log('::error::cross_chain is empty after seeding; the drive cannot resolve a quorum'); process.exit(1) } + })().catch((e) => { console.log('::warning::capability read failed, not gating: ' + e.message) }) + JS + + - name: Drive ${{ matrix.drive }} ${{ matrix.leg }} + id: drive + working-directory: ${{ env.RAIL_ROOT }}/xchain-e2e-test + env: + ROW: ${{ matrix.row }} + DRIVE: ${{ matrix.drive }} + LEG: ${{ matrix.leg }} + run: | + J="$RUNNER_TEMP/journal/$ROW/$LEG" + mkdir -p "$J" + echo "JOURNAL_DIR=$J" >> "$GITHUB_ENV" + set +e + # Shape watchdog: a wiring fault (a hostname the host cannot resolve, a + # runtime the VM refuses, a missing module) otherwise surfaces only as a + # 30-minute wait inside a case. The drive runs in its own process group; + # once a fault signature repeats in the drive or venue logs, that group + # alone is ended and the job fails naming it. + setsid env DOGE_INDEXER_API_PORT=3124 \ + NODE_PATH="$RAIL_ROOT" BRIDGE_RAIL_REPO_ROOT="$RAIL_ROOT" \ + BRIDGE_RAIL_MINER_PAUSE_FILE="$RUNNER_TEMP/btc-loop.pause" \ + BRIDGE_RAIL_DOGE_MINER_PAUSE_FILE="$RUNNER_TEMP/doge-loop.pause" \ + node scripts/rail_leg_drive.js "$DRIVE" "$LEG" --journal-dir "$J" --limit-minutes "$LIMIT_MINUTES" > "$J/drive.log" 2>&1 & + pgid=$! + SHAPE='EAI_AGAIN|ENOTFOUND|CONSENSUS RUNTIME MISMATCH|Cannot find module' + while kill -0 "$pgid" 2>/dev/null; do + sleep 60 + hits=$(cat "$J"/*.log 2>/dev/null | grep -cE "$SHAPE") + if [ "$hits" -ge 5 ]; then + echo "::error::venue shape fault ($hits hits): $(cat "$J"/*.log | grep -m1 -oE ".{0,160}($SHAPE).{0,120}")" + kill -TERM -- "-$pgid" 2>/dev/null; sleep 30; kill -KILL -- "-$pgid" 2>/dev/null + break + fi + done + wait "$pgid"; rc=$? + [ "${hits:-0}" -ge 5 ] && rc=97 + cat "$J/drive.log" + echo "drive exit $rc" + test -s "$J/case-journal.jsonl" && node scripts/rail_journal_triage.js "$J/case-journal.jsonl" 2>/dev/null | tail -5 || true + exit "$rc" + + - name: Capture stack diagnostics + if: always() + run: | + OUT="${JOURNAL_DIR:-$RUNNER_TEMP/journal}/stack" + mkdir -p "$OUT" + docker ps -a > "$OUT/docker-ps.txt" 2>&1 || true + for c in $(docker ps -a --format '{{.Names}}'); do docker logs --tail 3000 "$c" > "$OUT/log-$c.txt" 2>&1 || true; done + df -h / > "$OUT/df.txt"; free -m > "$OUT/free.txt" + + - name: Upload the leg journal + if: always() + uses: actions/upload-artifact@v4 + with: + name: rail-${{ matrix.row }}-${{ matrix.leg }} + path: ${{ runner.temp }}/journal/ + if-no-files-found: warn + retention-days: 30 From 35b2bcab061022bde7aa3995c22d554cdce76b89 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Tue, 29 Sep 2026 14:04:37 -0700 Subject: [PATCH 46/50] Generalize workflow venue comments Remove environment-specific names from hosted runner documentation while preserving workflow behavior. --- .github/workflows/attest-mirror-leg.yml | 8 ++++---- .github/workflows/rail-leg.yml | 5 ++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/attest-mirror-leg.yml b/.github/workflows/attest-mirror-leg.yml index 445a7c29..b7acc674 100644 --- a/.github/workflows/attest-mirror-leg.yml +++ b/.github/workflows/attest-mirror-leg.yml @@ -9,9 +9,9 @@ run-name: >- # with scripts/attest-mirror-stack-driver.sh), instead of three legs at a time on # one venue disk. # -# The stack root in .github/attest-mirror/ is test02's pb-v020-at7 root (compose.yml, -# setup-stack.sh, run-leg.sh, seed-stack-config.js, wait-indexer-schema.js) with -# only its host paths made relative: the driver boots an isolated BTC+LTC+DOGE +# The stack root in .github/attest-mirror/ contains compose.yml, setup-stack.sh, +# run-leg.sh, seed-stack-config.js and wait-indexer-schema.js. Its host paths are +# relative, so the driver boots an isolated BTC+LTC+DOGE # compose stack per leg from the xchain-node images, which this job builds first by # installing the three regtest stacks. The federation mnemonic the venue keeps in # at7.env is generated per job and masked. @@ -151,7 +151,7 @@ jobs: node src/index.js install "$STACK_REF" all bitcoin regtest node src/index.js install "$STACK_REF" all litecoin regtest node src/index.js install "$STACK_REF" all dogecoin regtest - # The standing stacks stay up, as on test02: legs read the standing + # The standing stacks stay up: legs read the standing # explorer (18080) and the standing LTC/DOGE indexers (at0-anti-wedge reads 3224). docker pull -q mariadb:10.11 && docker tag mariadb:10.11 xchain-node-database docker images --format '{{.Repository}}' | grep '^xchain-node-' | sort diff --git a/.github/workflows/rail-leg.yml b/.github/workflows/rail-leg.yml index c25af55c..c87401a2 100644 --- a/.github/workflows/rail-leg.yml +++ b/.github/workflows/rail-leg.yml @@ -4,9 +4,8 @@ run-name: >- Rail legs ${{ inputs.legs }} @ stack ${{ inputs.stack_ref || 'develop' }} # Runs xchain-e2e-test bridge rail acceptance legs (scripts/rail_leg_drive.js) on -# GitHub-hosted runners, one leg per job, many in parallel, beside the three -# self-hosted rails (DankServer, test01, test02) whose single drive slot per venue -# is the release push's bottleneck. +# GitHub-hosted runners, one leg per job, many in parallel, beside the +# self-hosted rails whose single drive slot per venue is the release bottleneck. # # WHY A HOSTED RUNNER CAN CARRY A LEG. A leg needs a BTC and a DOGE regtest stack, # a hub and indexers; the nightly's dogecoin shard already boots exactly that From 1df13732e0db712a47864a65f0e6b1504463bc43 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Tue, 29 Sep 2026 14:11:13 -0700 Subject: [PATCH 47/50] Add selector replay analysis Replay first-parent develop history against current and narrowed consensus prefixes. Report selection shares and verify required source-to-test mappings. --- bin/ci_fast_select.js | 164 ++++++++++++++++++++++++++++++-- bin/test/ci_fast_select.test.js | 76 +++++++++++++++ 2 files changed, 233 insertions(+), 7 deletions(-) diff --git a/bin/ci_fast_select.js b/bin/ci_fast_select.js index 16885379..15ebbd0a 100644 --- a/bin/ci_fast_select.js +++ b/bin/ci_fast_select.js @@ -115,16 +115,16 @@ function collectSourceData(changedSources, findRequirers) { })) } -function consensusReasons(changedFiles, sourceData) { +function consensusReasons(changedFiles, sourceData, consensusPrefixes) { const reasons = [] for (const file of changedFiles) { - if (file === 'package.json' || startsWithAny(file, CONSENSUS) || startsWithAny(file, WIDEN)) { + if (file === 'package.json' || startsWithAny(file, consensusPrefixes) || startsWithAny(file, WIDEN)) { reasons.push(`consensus: ${file}`) } } for (const { file, requirers } of sourceData) { for (const importer of requirers) { - if (startsWithAny(importer, CONSENSUS) && requiresFile(importer, file)) { + if (startsWithAny(importer, consensusPrefixes) && requiresFile(importer, file)) { reasons.push(`consensus importer: ${importer} requires ${file}`) } } @@ -145,12 +145,16 @@ function selectedTestFiles(changedFiles, tests, sourceData, reasons) { return selected } -function selectFastTests(changedFiles, { listTests, findRequirers }) { +function selectFastTests( + changedFiles, + { listTests, findRequirers }, + { consensusPrefixes = CONSENSUS } = {}, +) { const changed = splitLines(changedFiles) const tests = splitLines(listTests()).filter((file) => groupFor(file) && fs.existsSync(file)) const sources = changed.filter((file) => file.startsWith('src/') && file.endsWith('.js')) const sourceData = collectSourceData(sources, findRequirers) - const reasons = consensusReasons(changed, sourceData) + const reasons = consensusReasons(changed, sourceData, consensusPrefixes) const selected = selectedTestFiles(changed, tests, sourceData, reasons) const consensus = reasons.some((reason) => reason.startsWith('consensus')) const planned = consensus ? [] : [...selected] @@ -177,6 +181,150 @@ function findRequirers(basename) { return splitLines(result.stdout) } +function selectionDependencies({ indexed = false } = {}) { + const files = listTrackedFiles() + if (indexed) { + const sources = files + .filter((file) => file.endsWith('.js') && fs.existsSync(file)) + .map((file) => [file, sourceText(file)]) + const cache = new Map() + return { + listTests: () => files, + findRequirers: (basename) => { + if (!cache.has(basename)) { + cache.set(basename, sources + .filter(([, source]) => source.includes(basename)) + .map(([file]) => file)) + } + return cache.get(basename) + }, + } + } + return { listTests: () => files, findRequirers } +} + +function withoutConsensusPrefixes(prefixes) { + const removed = new Set(prefixes.flatMap((prefix) => { + const trimmed = prefix.trim() + if (!trimmed) return [] + return [trimmed, trimmed.endsWith('/') ? trimmed.slice(0, -1) : `${trimmed}/`] + })) + return CONSENSUS.filter((prefix) => !removed.has(prefix)) +} + +function changedFilesForCommit(commit) { + const revision = splitLines(git(['rev-list', '--parents', '-n', '1', commit]))[0] + const [, parent] = revision.split(' ') + if (parent) return splitLines(git(['diff', '--name-only', `${parent}..${commit}`])) + return splitLines(git(['diff-tree', '--root', '--no-commit-id', '--name-only', '-r', commit])) +} + +function emptyReplayCounts() { + return { wholeUnit: 0, changedTests: 0, testOnly: 0, noTests: 0 } +} + +function countReplayPlan(counts, changed, plan) { + if (plan.consensus) { + counts.wholeUnit++ + } else if (plan.tests.length && changed.every((file) => file.startsWith('test/'))) { + counts.testOnly++ + } else if (plan.tests.length) { + counts.changedTests++ + } else { + counts.noTests++ + } +} + +function replayPlans(limit, narrowPrefixes) { + const commits = splitLines(git([ + 'log', '--first-parent', '-n', String(limit), '--format=%H', 'origin/develop', + ])) + const current = emptyReplayCounts() + const narrowed = emptyReplayCounts() + const consensusPrefixes = withoutConsensusPrefixes(narrowPrefixes) + const dependencies = selectionDependencies({ indexed: true }) + for (const commit of commits) { + const changed = changedFilesForCommit(commit) + countReplayPlan(current, changed, selectFastTests(changed, dependencies)) + countReplayPlan(narrowed, changed, selectFastTests(changed, dependencies, { + consensusPrefixes, + })) + } + return { commits, current, narrowed, consensusPrefixes, dependencies } +} + +function fraction(value, total) { + return `${value}/${total}` +} + +function printReplayRow(name, total, counts) { + console.log([ + name, + total, + fraction(counts.wholeUnit, total), + fraction(counts.changedTests, total), + fraction(counts.testOnly, total), + fraction(counts.noTests, total), + ].join(' ')) +} + +function parseList(value) { + return value.split(',').map((item) => item.trim()).filter(Boolean) +} + +function parseMustSelect(value) { + return parseList(value).map((pair) => { + const separator = pair.indexOf(':') + if (separator <= 0 || separator === pair.length - 1) { + throw new Error(`invalid --must-select pair: ${pair}`) + } + return { source: pair.slice(0, separator), test: pair.slice(separator + 1) } + }) +} + +function replayOptions(args) { + const limit = Number(args[0]) + if (!Number.isSafeInteger(limit) || limit < 1) { + throw new Error('--replay requires a positive integer') + } + const options = { limit, narrowPrefixes: [], mustSelect: [] } + for (let index = 1; index < args.length; index += 2) { + const flag = args[index] + const value = args[index + 1] + if (!value || (flag !== '--narrow' && flag !== '--must-select')) { + throw new Error(`invalid replay option: ${flag || ''}`.trim()) + } + if (flag === '--narrow') options.narrowPrefixes.push(...parseList(value)) + else options.mustSelect.push(...parseMustSelect(value)) + } + return options +} + +function runReplay(args) { + try { + const options = replayOptions(args) + const result = replayPlans(options.limit, options.narrowPrefixes) + console.log('plan commits consensus-1 changed-tests test-only no-tests') + printReplayRow('current', result.commits.length, result.current) + if (options.narrowPrefixes.length) { + printReplayRow('narrowed', result.commits.length, result.narrowed) + } + let failed = false + for (const pair of options.mustSelect) { + const plan = selectFastTests([pair.source], result.dependencies, { + consensusPrefixes: result.consensusPrefixes, + }) + const selected = plan.tests.some((test) => test.file === pair.test) + console.log(`must-select ${selected ? 'PASS' : 'FAIL'} ${pair.source}:${pair.test}`) + if (!selected) failed = true + } + return failed ? 1 : 0 + } catch (error) { + console.error(`replay-error ${error.message}`) + return 2 + } +} + function resolvePlan() { const base = resolveBase({ env: process.env, git }) if (!base) return { error: 'git could not resolve a usable push base' } @@ -209,8 +357,10 @@ function runPlan(plan) { function main() { const mode = process.argv[2] + if (mode === '--replay') return runReplay(process.argv.slice(3)) if (mode !== '--plan' && mode !== '--run') { - console.error('usage: node bin/ci_fast_select.js --plan|--run') + console.error('usage: node bin/ci_fast_select.js --plan|--run|--replay N ' + + '[--narrow prefix,...] [--must-select file:testfile,...]') return 2 } let result @@ -228,6 +378,6 @@ function main() { return mode === '--run' ? runPlan(result.plan) : 0 } -module.exports = { resolveBase, selectFastTests } +module.exports = { replayPlans, resolveBase, selectFastTests } if (require.main === module) process.exitCode = main() diff --git a/bin/test/ci_fast_select.test.js b/bin/test/ci_fast_select.test.js index 6d2f2310..b43f0382 100644 --- a/bin/test/ci_fast_select.test.js +++ b/bin/test/ci_fast_select.test.js @@ -2,6 +2,7 @@ const assert = require('node:assert') const fs = require('node:fs') +const os = require('node:os') const path = require('node:path') const { execFileSync, spawnSync } = require('node:child_process') @@ -41,6 +42,25 @@ function files(plan) { return plan.tests.map((test) => test.file) } +function scratchGit(cwd, args) { + return execFileSync('git', args, { cwd, encoding: 'utf8' }).trim() +} + +function writeScratchFile(cwd, file, source) { + const target = path.join(cwd, file) + fs.mkdirSync(path.dirname(target), { recursive: true }) + fs.writeFileSync(target, source) +} + +function scratchCommit(cwd, message, files) { + scratchGit(cwd, ['add', '--', ...files]) + scratchGit(cwd, [ + '-c', 'user.name=Selector Test', + '-c', 'user.email=selector-test', + 'commit', '-m', message, + ]) +} + describe('bin/ci_fast_select.js', () => { it('maps command_lock to its unit test without widening the suite', () => { const plan = select(['src/utils/command_lock.js']) @@ -131,4 +151,60 @@ describe('bin/ci_fast_select.js', () => { assert.match(script, /CI_TIER:-full[^\n]+fast[\s\S]+ci_fast_select\.js --plan/) assert.ok(script.includes('run_tier "ci" env XCHAIN_REQUIRE_SIBLINGS=1 npm run ci')) }) + + it('replays develop history, compares narrowing, and checks required selections', () => { + const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'ci-fast-select-replay-')) + try { + scratchGit(cwd, ['init', '--initial-branch=develop']) + const initial = { + 'src/coins/rule.js': "module.exports = 'rule'\n", + 'src/feature/plain.js': "module.exports = 'plain'\n", + 'test/unit/coins/rule.test.js': "require('../../../src/coins/rule')\n", + 'test/unit/feature/plain.test.js': "require('../../../src/feature/plain')\n", + 'test/unit/only.test.js': "module.exports = 'only'\n", + 'test/unit/other/unrelated.test.js': "module.exports = 'unrelated'\n", + } + for (const [file, source] of Object.entries(initial)) { + writeScratchFile(cwd, file, source) + } + scratchCommit(cwd, 'initial files', Object.keys(initial)) + + const consensusFile = 'src/coins/rule.js' + fs.appendFileSync(path.join(cwd, consensusFile), "module.exports += ' changed'\n") + scratchCommit(cwd, 'consensus change', [consensusFile]) + + const plainFile = 'src/feature/plain.js' + fs.appendFileSync(path.join(cwd, plainFile), "module.exports += ' changed'\n") + scratchCommit(cwd, 'plain source change', [plainFile]) + + const testFile = 'test/unit/only.test.js' + fs.appendFileSync(path.join(cwd, testFile), "module.exports += ' changed'\n") + scratchCommit(cwd, 'test only change', [testFile]) + scratchGit(cwd, ['update-ref', 'refs/remotes/origin/develop', 'HEAD']) + + const selector = path.resolve(__dirname, '../ci_fast_select.js') + const mustSelect = [ + 'src/coins/rule.js:test/unit/coins/rule.test.js', + 'src/feature/plain.js:test/unit/other/unrelated.test.js', + ].join(',') + const result = spawnSync(process.execPath, [ + selector, + '--replay', '3', + '--narrow', 'src/coins/', + '--must-select', mustSelect, + ], { cwd, encoding: 'utf8' }) + + assert.strictEqual(result.status, 1, result.stderr) + const lines = result.stdout.trim().split(/\r?\n/) + assert.ok(lines.includes('plan commits consensus-1 changed-tests test-only no-tests')) + assert.ok(lines.includes('current 3 1/3 1/3 1/3 0/3')) + assert.ok(lines.includes('narrowed 3 0/3 2/3 1/3 0/3')) + assert.ok(lines.includes( + 'must-select PASS src/coins/rule.js:test/unit/coins/rule.test.js')) + assert.ok(lines.includes( + 'must-select FAIL src/feature/plain.js:test/unit/other/unrelated.test.js')) + } finally { + fs.rmSync(cwd, { recursive: true, force: true }) + } + }) }) From 15794d09877766f19a0f98a76285e1db3c002405 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Tue, 29 Sep 2026 14:11:47 -0700 Subject: [PATCH 48/50] Validate the attest-mirror drive script Fail the plan job before matrix creation when the local driver is missing or the scheduled manifest names an unsupported driver. --- .github/workflows/attest-mirror-leg.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/attest-mirror-leg.yml b/.github/workflows/attest-mirror-leg.yml index b7acc674..d3fd72fa 100644 --- a/.github/workflows/attest-mirror-leg.yml +++ b/.github/workflows/attest-mirror-leg.yml @@ -58,9 +58,11 @@ jobs: LEGS_INPUT: ${{ inputs.legs }} EVENT: ${{ github.event_name }} run: | + DRIVE=.github/attest-mirror/run-leg.sh + test -f "$DRIVE" || { echo "::error::missing drive script: $DRIVE"; exit 1; } if [ "$EVENT" = schedule ]; then E2E_REF=develop - LEGS_INPUT=$(node -e 'const l=require("./.github/rail-legs.json").filter(x=>x.drive===".github/attest-mirror/run-leg.sh").map(x=>x.leg);if(!l.length)throw new Error("no scheduled attest-mirror legs");process.stdout.write(l.join(","))') + LEGS_INPUT=$(node -e 'const fs=require("fs");const drive=process.argv[1];const l=require("./.github/rail-legs.json");if(!l.length)throw new Error("no scheduled attest-mirror legs");for(const x of l){if(x.drive!==drive)throw new Error("unsupported scheduled drive: "+x.drive);if(!fs.existsSync(x.drive)||!fs.statSync(x.drive).isFile())throw new Error("missing scheduled drive script: "+x.drive)}process.stdout.write(l.map(x=>x.leg).join(","))' "$DRIVE") fi echo "e2e_ref=$E2E_REF" >> "$GITHUB_OUTPUT" git clone -q https://github.com/XChain-Platform/xchain-e2e-test.git e2e From eaaa84a53673d3144b2b918b3d6e5ca8ec75d800 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Tue, 29 Sep 2026 14:32:16 -0700 Subject: [PATCH 49/50] Revert "Merge develop into release/v0.21.0" This reverts commit a403bfa95d30456c87cf2292476c9f72a355c116, reversing changes made to 765ba6826b8776cda58f8598c43523f34ea0fe85. --- .github/attest-mirror-run.json | 1 - .github/attest-mirror/compose.yml | 639 ------------------- .github/attest-mirror/run-leg.sh | 59 -- .github/attest-mirror/seed-stack-config.js | 26 - .github/attest-mirror/setup-stack.sh | 68 -- .github/attest-mirror/wait-indexer-schema.js | 23 - .github/rail-legs.json | 146 ----- .github/workflows/attest-mirror-leg.yml | 233 ------- .github/workflows/rail-leg.yml | 328 ---------- bin/ci_fast_select.js | 164 +---- bin/test/ci_fast_select.test.js | 76 --- 11 files changed, 7 insertions(+), 1756 deletions(-) delete mode 100644 .github/attest-mirror-run.json delete mode 100644 .github/attest-mirror/compose.yml delete mode 100755 .github/attest-mirror/run-leg.sh delete mode 100644 .github/attest-mirror/seed-stack-config.js delete mode 100755 .github/attest-mirror/setup-stack.sh delete mode 100644 .github/attest-mirror/wait-indexer-schema.js delete mode 100644 .github/rail-legs.json delete mode 100644 .github/workflows/attest-mirror-leg.yml delete mode 100644 .github/workflows/rail-leg.yml diff --git a/.github/attest-mirror-run.json b/.github/attest-mirror-run.json deleted file mode 100644 index a2e96b42..00000000 --- a/.github/attest-mirror-run.json +++ /dev/null @@ -1 +0,0 @@ -{"e2e_ref": "gh-rail/l004-at2b-fix", "legs": ["test/attestMirror/at2b-forward-margin.test.js"]} diff --git a/.github/attest-mirror/compose.yml b/.github/attest-mirror/compose.yml deleted file mode 100644 index 64f093f6..00000000 --- a/.github/attest-mirror/compose.yml +++ /dev/null @@ -1,639 +0,0 @@ -services: - db: - image: xchain-node-database - command: ["--max-connections=1000", "--innodb-flush-log-at-trx-commit=2", "--innodb-doublewrite=0", "--innodb-buffer-pool-size=1G"] - environment: - MARIADB_ROOT_PASSWORD: ${DB_PASSWORD} - MARIADB_ROOT_HOST: "%" - ports: - - "127.0.0.1:${DB_HOST_PORT}:3306" - volumes: - - db-data:/var/lib/mysql - healthcheck: - test: ["CMD-SHELL", "healthcheck.sh --connect --innodb_initialized"] - interval: 5s - timeout: 5s - retries: 30 - start_period: 20s - restart: unless-stopped - - db-init: - image: xchain-node-database - depends_on: - db: - condition: service_healthy - environment: - MARIADB_ROOT_PASSWORD: ${DB_PASSWORD} - entrypoint: ["/bin/sh", "-ec"] - command: - - | - mariadb -h db -uroot -p"$${MARIADB_ROOT_PASSWORD}" -e "CREATE DATABASE IF NOT EXISTS XChain_BTC_Regtest_Decoder; CREATE DATABASE IF NOT EXISTS XChain_BTC_Regtest_Indexer; CREATE DATABASE IF NOT EXISTS XChain_Hub; CREATE DATABASE IF NOT EXISTS XChain_LTC_Regtest_Decoder; CREATE DATABASE IF NOT EXISTS XChain_LTC_Regtest_Indexer; CREATE DATABASE IF NOT EXISTS XChain_DOGE_Regtest_Decoder; CREATE DATABASE IF NOT EXISTS XChain_DOGE_Regtest_Indexer;" - restart: "no" - - node: - image: xchain-node-bitcoin-regtest-node - environment: - NODE_USER: ${RPC_USER} - NODE_PASSWORD: ${RPC_PASSWORD} - entrypoint: ["/bin/sh", "-ec"] - command: - - | - umask 077 - sed -e "s|^rpcuser=.*|rpcuser=$${NODE_USER}|" -e "s|^rpcpassword=.*|rpcpassword=$${NODE_PASSWORD}|" /etc/bitcoin/bitcoin.conf > /run/bitcoin.conf - exec bitcoind -conf=/run/bitcoin.conf -datadir=/root/.bitcoin/ - tmpfs: - - /run - ports: - - "127.0.0.1:${NODE_HOST_PORT}:18444" - volumes: - - node-data:/root/.bitcoin - healthcheck: - test: ["CMD-SHELL", "bitcoin-cli -conf=/run/bitcoin.conf -datadir=/root/.bitcoin/ getblockchaininfo >/dev/null 2>&1"] - interval: 5s - timeout: 5s - retries: 60 - start_period: 20s - restart: unless-stopped - - miner: - image: xchain-node-bitcoin-regtest-xchain-regtest-miner - depends_on: - node: - condition: service_healthy - environment: - NETWORK: bitcoin-regtest - NODE_URL: node - NODE_PORT: 18444 - NODE_USER: ${RPC_USER} - NODE_PASSWORD: ${RPC_PASSWORD} - REGTEST_MINER_API_PORT: 3005 - ports: - - "127.0.0.1:${MINER_HOST_PORT}:3005" - healthcheck: - test: ["CMD-SHELL", "node -e \"const h=require('http');const q=h.request({host:'127.0.0.1',port:3005,method:'POST',headers:{'content-type':'application/json'}},r=>process.exit(r.statusCode===200?0:1));q.on('error',()=>process.exit(1));q.end(JSON.stringify({jsonrpc:'2.0',method:'health',id:1}))\""] - interval: 10s - timeout: 5s - retries: 60 - start_period: 90s - restart: unless-stopped - - tracker: - image: xchain-node-bitcoin-regtest-xchain-utxo-tracker - depends_on: - node: - condition: service_healthy - environment: - COIN: bitcoin - NETWORK: bitcoin-regtest - NODE_URL: node - NODE_PORT: 18444 - NODE_USER: ${RPC_USER} - NODE_PASSWORD: ${RPC_PASSWORD} - UTXO_TRACKER_API_PORT: 3001 - CORS_ORIGIN: "*" - ports: - - "127.0.0.1:${TRACKER_HOST_PORT}:3001" - volumes: - - tracker-data:/data/xchain-utxo-tracker - healthcheck: - test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3001/status >/dev/null 2>&1"] - interval: 10s - timeout: 5s - retries: 60 - start_period: 90s - restart: unless-stopped - - decoder: - image: xchain-node-bitcoin-regtest-xchain-decoder - depends_on: - db-init: - condition: service_completed_successfully - node: - condition: service_healthy - environment: - COIN: bitcoin - NETWORK: bitcoin-regtest - NODE_URL: node - NODE_PORT: 18444 - NODE_USER: ${RPC_USER} - NODE_PASSWORD: ${RPC_PASSWORD} - DECODER_DB_HOST: db - DECODER_DB_PORT: 3306 - DECODER_DB_NAME: XChain_BTC_Regtest_Decoder - DECODER_DB_USER: root - DECODER_DB_PASS: ${DB_PASSWORD} - DECODER_API_PORT: 3002 - ports: - - "127.0.0.1:${DECODER_HOST_PORT}:3002" - healthcheck: - test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3002/live >/dev/null 2>&1"] - interval: 10s - timeout: 5s - retries: 60 - start_period: 90s - restart: unless-stopped - - encoder: - image: xchain-node-bitcoin-regtest-xchain-encoder - depends_on: - node: - condition: service_healthy - tracker: - condition: service_healthy - environment: - NETWORK: bitcoin-regtest - NODE_URL: node - NODE_PORT: 18444 - NODE_USER: ${RPC_USER} - NODE_PASSWORD: ${RPC_PASSWORD} - UTXO_TRACKER_URL: tracker - UTXO_TRACKER_API_PORT: 3001 - ENCODER_API_PORT: 3003 - ENCODER_RATE_LIMIT_RPM: 99999 - CORS_ORIGIN: "*" - ports: - - "127.0.0.1:${ENCODER_HOST_PORT}:3003" - healthcheck: - test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3003/status >/dev/null 2>&1"] - interval: 10s - timeout: 5s - retries: 60 - start_period: 90s - restart: unless-stopped - - hub: - image: xchain-node-xchain-hub - depends_on: - db-init: - condition: service_completed_successfully - environment: - HUB_HOST: 0.0.0.0 - HUB_PORT: 10000 - HUB_DB_HOST: db - HUB_DB_PORT: 3306 - HUB_DB_NAME: XChain_Hub - HUB_DB_USER: root - HUB_DB_SECRET: ${DB_PASSWORD} - HUB_ALLOW_UNAUTHENTICATED: "true" - HUB_NETWORK: regtest - BTC_INDEXER_API_URL: http://indexer:3004 - ports: - - "127.0.0.1:${HUB_HOST_PORT}:10000" - healthcheck: - test: ["CMD-SHELL", "wget -qO- --post-data='{\"jsonrpc\":\"2.0\",\"method\":\"health\",\"id\":1}' --header='Content-Type: application/json' http://127.0.0.1:10000/ >/dev/null 2>&1"] - interval: 10s - timeout: 5s - retries: 60 - start_period: 90s - restart: unless-stopped - - indexer: - image: xchain-node-bitcoin-regtest-xchain-indexer - depends_on: - db-init: - condition: service_completed_successfully - decoder: - condition: service_healthy - tracker: - condition: service_healthy - hub: - condition: service_healthy - environment: - INDEXER_COIN: BTC - INDEXER_NETWORK: regtest - INDEXER_API_PORT: 3004 - INDEXER_DB_HOST: db - INDEXER_DB_PORT: 3306 - INDEXER_DB_NAME: XChain_BTC_Regtest_Indexer - INDEXER_DB_USER: root - INDEXER_DB_PASS: ${DB_PASSWORD} - DECODER_DB_HOST: db - DECODER_DB_PORT: 3306 - DECODER_DB_NAME: XChain_BTC_Regtest_Decoder - DECODER_DB_USER: root - DECODER_DB_PASS: ${DB_PASSWORD} - UTXO_TRACKER_URL: tracker - UTXO_TRACKER_API_PORT: 3001 - HUB_API_URL: http://hub:10000 - HUB_DB_HOST: db - HUB_DB_PORT: 3306 - HUB_DB_NAME: XChain_BTC_Regtest_Indexer - HUB_DB_USER: root - HUB_DB_PASS: ${DB_PASSWORD} - HUB_DB_SYNC_ENABLED: "true" - INDEXER_ALLOW_UNAUTHENTICATED: "true" - HUB_SYNC_PRICE_GRACE_S: 0 - HUB_SYNC_ORACLE_GRACE_S: 0 - HUB_SYNC_ATTEST_RESPONSE_GRACE_S: 0 - HUB_SYNC_MATCH_GRACE_S: 0 - HUB_SYNC_CALL_GRACE_S: 0 - HUB_SYNC_ANCHOR_ATTEST_GRACE_S: 0 - ports: - - "127.0.0.1:${INDEXER_HOST_PORT}:3004" - healthcheck: - test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3004/status >/dev/null 2>&1"] - interval: 10s - timeout: 5s - retries: 90 - start_period: 120s - restart: unless-stopped - - explorer: - image: xchain-node-xchain-explorer - depends_on: - hub: - condition: service_healthy - indexer: - condition: service_healthy - environment: - NETWORK: regtest - HUB_API_HOST: hub - HUB_PORT: 10000 - EXPLORER_API_PORT_HTTP: 8080 - EXPLORER_API_PORT_HTTPS: 8081 - ALLOW_NO_COLOCATED_HUB_DB: 1 - ports: - - "127.0.0.1:${EXPLORER_HTTP_HOST_PORT}:8080" - - "127.0.0.1:${EXPLORER_HTTPS_HOST_PORT}:8081" - healthcheck: - test: ["CMD-SHELL", "wget -qO- --post-data='{\"jsonrpc\":\"2.0\",\"method\":\"ping\",\"id\":1}' --header='Content-Type: application/json' http://127.0.0.1:8080/ >/dev/null 2>&1"] - interval: 10s - timeout: 5s - retries: 90 - start_period: 120s - restart: unless-stopped - - ltc-node: - image: xchain-node-litecoin-regtest-node - environment: - NODE_USER: ${RPC_USER} - NODE_PASSWORD: ${RPC_PASSWORD} - entrypoint: ["/bin/sh", "-ec"] - command: - - | - umask 077 - sed -e "s|^rpcuser=.*|rpcuser=$${NODE_USER}|" -e "s|^rpcpassword=.*|rpcpassword=$${NODE_PASSWORD}|" /etc/litecoin/litecoin.conf > /run/litecoin.conf - exec litecoind -conf=/run/litecoin.conf -datadir=/root/.litecoin/ - tmpfs: - - /run - ports: - - "127.0.0.1:${LTC_NODE_HOST_PORT}:18444" - volumes: - - ltc-node-data:/root/.litecoin - healthcheck: - test: ["CMD-SHELL", "litecoin-cli -conf=/run/litecoin.conf -datadir=/root/.litecoin/ getblockchaininfo >/dev/null 2>&1"] - interval: 5s - timeout: 5s - retries: 60 - start_period: 20s - restart: unless-stopped - - ltc-miner: - image: xchain-node-litecoin-regtest-xchain-regtest-miner - depends_on: - ltc-node: - condition: service_healthy - environment: - NETWORK: litecoin-regtest - NODE_URL: ltc-node - NODE_PORT: 18444 - NODE_USER: ${RPC_USER} - NODE_PASSWORD: ${RPC_PASSWORD} - REGTEST_MINER_API_PORT: 3005 - ports: - - "127.0.0.1:${LTC_MINER_HOST_PORT}:3005" - healthcheck: - test: ["CMD-SHELL", "node -e \"const h=require('http');const q=h.request({host:'127.0.0.1',port:3005,method:'POST',headers:{'content-type':'application/json'}},r=>process.exit(r.statusCode===200?0:1));q.on('error',()=>process.exit(1));q.end(JSON.stringify({jsonrpc:'2.0',method:'health',id:1}))\""] - interval: 10s - timeout: 5s - retries: 60 - start_period: 90s - restart: unless-stopped - - ltc-tracker: - image: xchain-node-litecoin-regtest-xchain-utxo-tracker - depends_on: - ltc-node: - condition: service_healthy - environment: - COIN: litecoin - NETWORK: litecoin-regtest - NODE_URL: ltc-node - NODE_PORT: 18444 - NODE_USER: ${RPC_USER} - NODE_PASSWORD: ${RPC_PASSWORD} - UTXO_TRACKER_API_PORT: 3001 - CORS_ORIGIN: "*" - ports: - - "127.0.0.1:${LTC_TRACKER_HOST_PORT}:3001" - volumes: - - ltc-tracker-data:/data/xchain-utxo-tracker - healthcheck: - test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3001/status >/dev/null 2>&1"] - interval: 10s - timeout: 5s - retries: 60 - start_period: 90s - restart: unless-stopped - - ltc-decoder: - image: xchain-node-litecoin-regtest-xchain-decoder - depends_on: - db: - condition: service_healthy - ltc-node: - condition: service_healthy - environment: - COIN: litecoin - NETWORK: litecoin-regtest - NODE_URL: ltc-node - NODE_PORT: 18444 - NODE_USER: ${RPC_USER} - NODE_PASSWORD: ${RPC_PASSWORD} - DECODER_DB_HOST: db - DECODER_DB_PORT: 3306 - DECODER_DB_NAME: XChain_LTC_Regtest_Decoder - DECODER_DB_USER: root - DECODER_DB_PASS: ${DB_PASSWORD} - DECODER_API_PORT: 3002 - ports: - - "127.0.0.1:${LTC_DECODER_HOST_PORT}:3002" - healthcheck: - test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3002/live >/dev/null 2>&1"] - interval: 10s - timeout: 5s - retries: 60 - start_period: 90s - restart: unless-stopped - - ltc-encoder: - image: xchain-node-litecoin-regtest-xchain-encoder - depends_on: - ltc-node: - condition: service_healthy - ltc-tracker: - condition: service_healthy - environment: - NETWORK: litecoin-regtest - NODE_URL: ltc-node - NODE_PORT: 18444 - NODE_USER: ${RPC_USER} - NODE_PASSWORD: ${RPC_PASSWORD} - UTXO_TRACKER_URL: ltc-tracker - UTXO_TRACKER_API_PORT: 3001 - ENCODER_API_PORT: 3003 - ENCODER_RATE_LIMIT_RPM: 99999 - CORS_ORIGIN: "*" - ports: - - "127.0.0.1:${LTC_ENCODER_HOST_PORT}:3003" - healthcheck: - test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3003/status >/dev/null 2>&1"] - interval: 10s - timeout: 5s - retries: 60 - start_period: 90s - restart: unless-stopped - - ltc-indexer: - image: xchain-node-litecoin-regtest-xchain-indexer - depends_on: - db: - condition: service_healthy - ltc-decoder: - condition: service_healthy - ltc-tracker: - condition: service_healthy - hub: - condition: service_healthy - environment: - INDEXER_COIN: LTC - INDEXER_NETWORK: regtest - INDEXER_API_PORT: 3004 - INDEXER_DB_HOST: db - INDEXER_DB_PORT: 3306 - INDEXER_DB_NAME: XChain_LTC_Regtest_Indexer - INDEXER_DB_USER: root - INDEXER_DB_PASS: ${DB_PASSWORD} - DECODER_DB_HOST: db - DECODER_DB_PORT: 3306 - DECODER_DB_NAME: XChain_LTC_Regtest_Decoder - DECODER_DB_USER: root - DECODER_DB_PASS: ${DB_PASSWORD} - UTXO_TRACKER_URL: ltc-tracker - UTXO_TRACKER_API_PORT: 3001 - HUB_API_URL: http://hub:10000 - HUB_DB_HOST: db - HUB_DB_PORT: 3306 - HUB_DB_NAME: XChain_LTC_Regtest_Indexer - HUB_DB_USER: root - HUB_DB_PASS: ${DB_PASSWORD} - HUB_DB_SYNC_ENABLED: "true" - INDEXER_ALLOW_UNAUTHENTICATED: "true" - HUB_SYNC_PRICE_GRACE_S: 0 - HUB_SYNC_ORACLE_GRACE_S: 0 - HUB_SYNC_ATTEST_RESPONSE_GRACE_S: 0 - HUB_SYNC_MATCH_GRACE_S: 0 - HUB_SYNC_CALL_GRACE_S: 0 - HUB_SYNC_ANCHOR_ATTEST_GRACE_S: 0 - ports: - - "127.0.0.1:${LTC_INDEXER_HOST_PORT}:3004" - healthcheck: - test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3004/status >/dev/null 2>&1"] - interval: 10s - timeout: 5s - retries: 90 - start_period: 120s - restart: unless-stopped - - - doge-node: - image: xchain-node-dogecoin-regtest-node - environment: - NODE_USER: ${RPC_USER} - NODE_PASSWORD: ${RPC_PASSWORD} - entrypoint: ["/bin/sh", "-ec"] - command: - - | - umask 077 - sed -e "s|^rpcuser=.*|rpcuser=$${NODE_USER}|" -e "s|^rpcpassword=.*|rpcpassword=$${NODE_PASSWORD}|" /etc/dogecoin/dogecoin.conf > /run/dogecoin.conf - exec dogecoind -conf=/run/dogecoin.conf -datadir=/root/.dogecoin/ - tmpfs: - - /run - ports: - - "127.0.0.1:${DOGE_NODE_HOST_PORT}:18444" - volumes: - - doge-node-data:/root/.dogecoin - healthcheck: - test: ["CMD-SHELL", "dogecoin-cli -conf=/run/dogecoin.conf -datadir=/root/.dogecoin/ getblockchaininfo >/dev/null 2>&1"] - interval: 5s - timeout: 5s - retries: 60 - start_period: 20s - restart: unless-stopped - - doge-miner: - image: xchain-node-dogecoin-regtest-xchain-regtest-miner - depends_on: - doge-node: - condition: service_healthy - environment: - NETWORK: dogecoin-regtest - NODE_URL: doge-node - NODE_PORT: 18444 - NODE_USER: ${RPC_USER} - NODE_PASSWORD: ${RPC_PASSWORD} - REGTEST_MINER_API_PORT: 3005 - ports: - - "127.0.0.1:${DOGE_MINER_HOST_PORT}:3005" - healthcheck: - test: ["CMD-SHELL", "node -e \"const h=require('http');const q=h.request({host:'127.0.0.1',port:3005,method:'POST',headers:{'content-type':'application/json'}},r=>process.exit(r.statusCode===200?0:1));q.on('error',()=>process.exit(1));q.end(JSON.stringify({jsonrpc:'2.0',method:'health',id:1}))\""] - interval: 10s - timeout: 5s - retries: 60 - start_period: 90s - restart: unless-stopped - - doge-tracker: - image: xchain-node-dogecoin-regtest-xchain-utxo-tracker - depends_on: - doge-node: - condition: service_healthy - environment: - COIN: dogecoin - NETWORK: dogecoin-regtest - NODE_URL: doge-node - NODE_PORT: 18444 - NODE_USER: ${RPC_USER} - NODE_PASSWORD: ${RPC_PASSWORD} - UTXO_TRACKER_API_PORT: 3001 - CORS_ORIGIN: "*" - ports: - - "127.0.0.1:${DOGE_TRACKER_HOST_PORT}:3001" - volumes: - - doge-tracker-data:/data/xchain-utxo-tracker - healthcheck: - test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3001/status >/dev/null 2>&1"] - interval: 10s - timeout: 5s - retries: 60 - start_period: 90s - restart: unless-stopped - - doge-decoder: - image: xchain-node-dogecoin-regtest-xchain-decoder - depends_on: - db: - condition: service_healthy - doge-node: - condition: service_healthy - environment: - COIN: dogecoin - NETWORK: dogecoin-regtest - NODE_URL: doge-node - NODE_PORT: 18444 - NODE_USER: ${RPC_USER} - NODE_PASSWORD: ${RPC_PASSWORD} - DECODER_DB_HOST: db - DECODER_DB_PORT: 3306 - DECODER_DB_NAME: XChain_DOGE_Regtest_Decoder - DECODER_DB_USER: root - DECODER_DB_PASS: ${DB_PASSWORD} - DECODER_API_PORT: 3002 - ports: - - "127.0.0.1:${DOGE_DECODER_HOST_PORT}:3002" - healthcheck: - test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3002/live >/dev/null 2>&1"] - interval: 10s - timeout: 5s - retries: 60 - start_period: 90s - restart: unless-stopped - - doge-encoder: - image: xchain-node-dogecoin-regtest-xchain-encoder - depends_on: - doge-node: - condition: service_healthy - doge-tracker: - condition: service_healthy - environment: - NETWORK: dogecoin-regtest - NODE_URL: doge-node - NODE_PORT: 18444 - NODE_USER: ${RPC_USER} - NODE_PASSWORD: ${RPC_PASSWORD} - UTXO_TRACKER_URL: doge-tracker - UTXO_TRACKER_API_PORT: 3001 - ENCODER_API_PORT: 3003 - ENCODER_RATE_LIMIT_RPM: 99999 - CORS_ORIGIN: "*" - ports: - - "127.0.0.1:${DOGE_ENCODER_HOST_PORT}:3003" - healthcheck: - test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3003/status >/dev/null 2>&1"] - interval: 10s - timeout: 5s - retries: 60 - start_period: 90s - restart: unless-stopped - - doge-indexer: - image: xchain-node-dogecoin-regtest-xchain-indexer - depends_on: - db: - condition: service_healthy - doge-decoder: - condition: service_healthy - doge-tracker: - condition: service_healthy - hub: - condition: service_healthy - environment: - INDEXER_COIN: DOGE - INDEXER_NETWORK: regtest - INDEXER_API_PORT: 3004 - INDEXER_DB_HOST: db - INDEXER_DB_PORT: 3306 - INDEXER_DB_NAME: XChain_DOGE_Regtest_Indexer - INDEXER_DB_USER: root - INDEXER_DB_PASS: ${DB_PASSWORD} - DECODER_DB_HOST: db - DECODER_DB_PORT: 3306 - DECODER_DB_NAME: XChain_DOGE_Regtest_Decoder - DECODER_DB_USER: root - DECODER_DB_PASS: ${DB_PASSWORD} - UTXO_TRACKER_URL: doge-tracker - UTXO_TRACKER_API_PORT: 3001 - HUB_API_URL: http://hub:10000 - HUB_DB_HOST: db - HUB_DB_PORT: 3306 - HUB_DB_NAME: XChain_DOGE_Regtest_Indexer - HUB_DB_USER: root - HUB_DB_PASS: ${DB_PASSWORD} - HUB_DB_SYNC_ENABLED: "true" - INDEXER_ALLOW_UNAUTHENTICATED: "true" - HUB_SYNC_PRICE_GRACE_S: 0 - HUB_SYNC_ORACLE_GRACE_S: 0 - HUB_SYNC_ATTEST_RESPONSE_GRACE_S: 0 - HUB_SYNC_MATCH_GRACE_S: 0 - HUB_SYNC_CALL_GRACE_S: 0 - HUB_SYNC_ANCHOR_ATTEST_GRACE_S: 0 - ports: - - "127.0.0.1:${DOGE_INDEXER_HOST_PORT}:3004" - healthcheck: - test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:3004/status >/dev/null 2>&1"] - interval: 10s - timeout: 5s - retries: 90 - start_period: 120s - restart: unless-stopped - - -volumes: - db-data: - node-data: - tracker-data: - ltc-node-data: - ltc-tracker-data: - doge-node-data: - doge-tracker-data: diff --git a/.github/attest-mirror/run-leg.sh b/.github/attest-mirror/run-leg.sh deleted file mode 100755 index 0a12018c..00000000 --- a/.github/attest-mirror/run-leg.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env bash -# Runs one attest-mirror leg file with the npm script's own mocha arguments on one xca7 stack; lines UTC-stamped. -set -u -tag=$1 -stack=$2 -venue_base=$3 -leg=$4 -shift 4 -here=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -root=${L004_VENUE_ROOT:?L004_VENUE_ROOT is required} -fed_env=${L004_FED_ENV:-$here/at7.env} -stack_dir=$here/xca7${stack} -log=${L004_LOG_DIR:?L004_LOG_DIR is required}/${tag}.log -venue_logs=$L004_LOG_DIR/venue/${tag} -mkdir -p "$venue_logs" -exec >"$log" 2>&1 -: node from PATH -set -a -. "$stack_dir/.env" -set +a -export COIN=bitcoin NETWORK=regtest -export NODE_URL=127.0.0.1 NODE_PORT=$NODE_HOST_PORT NODE_USER=$RPC_USER NODE_PASSWORD=$RPC_PASSWORD -export UTXO_TRACKER_URL=127.0.0.1 UTXO_TRACKER_API_PORT=$TRACKER_HOST_PORT -export ENCODER_URL=127.0.0.1 ENCODER_API_PORT=$ENCODER_HOST_PORT -export DECODER_URL=127.0.0.1 DECODER_API_PORT=$DECODER_HOST_PORT -export DECODER_DB_HOST=127.0.0.1 DECODER_DB_PORT=$DB_HOST_PORT DECODER_DB_NAME=XChain_BTC_Regtest_Decoder DECODER_DB_USER=root DECODER_DB_PASS=$DB_PASSWORD -export INDEXER_URL=127.0.0.1 INDEXER_API_PORT=$INDEXER_HOST_PORT INDEXER_DB_NAME=XChain_BTC_Regtest_Indexer INDEXER_DB_USER=root INDEXER_DB_PASS=$DB_PASSWORD -export DATABASE_URL=127.0.0.1 DATABASE_PORT=$DB_HOST_PORT -export REGTEST_MINER_URL=127.0.0.1 REGTEST_MINER_API_PORT=$MINER_HOST_PORT -export EXPLORER_URL=127.0.0.1 EXPLORER_API_PORT=$EXPLORER_HTTP_HOST_PORT -export HUB_URL=127.0.0.1 HUB_API_HOST=127.0.0.1 HUB_HOST=127.0.0.1 HUB_PORT=$HUB_HOST_PORT -export HUB_DB_HOST=127.0.0.1 HUB_DB_PORT=$DB_HOST_PORT HUB_DB_USER=root HUB_DB_PASS=$DB_PASSWORD -export BTC_SERVICE_HOST=127.0.0.1 BTC_NODE_PORT=$NODE_HOST_PORT BTC_NODE_USER=$RPC_USER BTC_NODE_PASSWORD=$RPC_PASSWORD -export BTC_UTXO_TRACKER_API_PORT=$TRACKER_HOST_PORT BTC_DECODER_API_PORT=$DECODER_HOST_PORT BTC_ENCODER_API_PORT=$ENCODER_HOST_PORT -export BTC_INDEXER_API_PORT=$INDEXER_HOST_PORT BTC_REGTEST_MINER_API_PORT=$MINER_HOST_PORT -export BTC_INDEXER_DB_NAME=XChain_BTC_Regtest_Indexer BTC_INDEXER_DB_USER=root BTC_INDEXER_DB_PASS=$DB_PASSWORD -for C in LTC DOGE; do - eval "export ${C}_SERVICE_HOST=127.0.0.1" - eval "export ${C}_NODE_PORT=\$${C}_NODE_HOST_PORT ${C}_UTXO_TRACKER_API_PORT=\$${C}_TRACKER_HOST_PORT" - eval "export ${C}_DECODER_API_PORT=\$${C}_DECODER_HOST_PORT ${C}_ENCODER_API_PORT=\$${C}_ENCODER_HOST_PORT" - eval "export ${C}_INDEXER_API_PORT=\$${C}_INDEXER_HOST_PORT ${C}_REGTEST_MINER_API_PORT=\$${C}_MINER_HOST_PORT" - eval "export ${C}_NODE_USER=\$RPC_USER ${C}_NODE_PASSWORD=\$RPC_PASSWORD" - eval "export ${C}_INDEXER_DB_USER=root ${C}_INDEXER_DB_PASS=\$DB_PASSWORD" -done -export LTC_INDEXER_DB_NAME=XChain_LTC_Regtest_Indexer DOGE_INDEXER_DB_NAME=XChain_DOGE_Regtest_Indexer -export XCHAIN_VENUE_REPO_ROOT=$root ATTEST_VENUE_LOG_DIR=$venue_logs AB_VENUE_BASE_PORT=$venue_base -[ -f "$fed_env" ] && { set -a; . "$fed_env"; set +a; } -for p in "$BTC_NODE_PORT" "$LTC_NODE_PORT" "$DOGE_NODE_PORT"; do - if ! curl -fsS --user "$RPC_USER:$RPC_PASSWORD" --data-binary '{"jsonrpc":"1.0","id":"runner","method":"getblockcount","params":[]}' \ - -H 'content-type:text/plain' "http://127.0.0.1:$p/" >/dev/null; then - printf "runner RPC check failed on port %s for xca7%s\n" "$p" "$stack"; exit 90 - fi -done -cd "${ATTEST_MIRROR_E2E_DIR:-$root/xchain-e2e-test}" -printf "=== leg %s start %s node %s stack xca7%s venue-base %s file %s extra-env [%s]\n" "$tag" "$(date -u +%FT%TZ)" "$(node -v)" "$stack" "$venue_base" "$leg" "$*" -env "$@" npx mocha --timeout 0 --exit --require ./test/initialCheck.test.js "$leg" 2>&1 | node -e 'const rl=require("readline").createInterface({input:process.stdin});rl.on("line",l=>process.stdout.write(new Date().toISOString().slice(11,19)+" "+l+"\n"))' -rc=${PIPESTATUS[0]} -printf "=== leg %s exit=%s end %s\n" "$tag" "$rc" "$(date -u +%FT%TZ)" -exit "$rc" diff --git a/.github/attest-mirror/seed-stack-config.js b/.github/attest-mirror/seed-stack-config.js deleted file mode 100644 index 5c028ac7..00000000 --- a/.github/attest-mirror/seed-stack-config.js +++ /dev/null @@ -1,26 +0,0 @@ -'use strict'; -// Seeds the stack hub's chain config for BTC, LTC and DOGE so chainRail.createRail can resolve each coin. -const port = Number(process.argv[2]); -const pass = process.env.DB_PASSWORD; -if (!port || !pass) process.exit(2); -const ixHost = { BTC: 'indexer', LTC: 'ltc-indexer', DOGE: 'doge-indexer' }; -function services(code) { - return { - 'xchain-decoder': { host: 'db', port: '3306', name: `XChain_${code}_Regtest_Decoder`, user: 'root', pass }, - 'xchain-indexer': { host: ixHost[code], port: '3004', name: `XChain_${code}_Regtest_Indexer`, user: 'root', pass } - }; -} -const config = { - bitcoin: { regtest: services('BTC') }, - litecoin: { regtest: services('LTC') }, - dogecoin: { regtest: services('DOGE') } -}; -fetch(`http://127.0.0.1:${port}/`, { - method: 'POST', - headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'updateconfig', params: { config } }) -}).then(async (response) => { - const body = await response.json(); - if (!response.ok || !body.result || body.result.status !== 'success') process.exit(3); - process.stdout.write('stack config seeded (bitcoin, litecoin, dogecoin)\n'); -}).catch(() => process.exit(4)); diff --git a/.github/attest-mirror/setup-stack.sh b/.github/attest-mirror/setup-stack.sh deleted file mode 100755 index 597d93c5..00000000 --- a/.github/attest-mirror/setup-stack.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -# Bring up one xca7 isolated stack (BTC, LTC and DOGE regtest services) and seed the stack hub's chain config. -set -eu -stack=$1 -base=$2 -root=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -dir=$root/xca7${stack} -# Host ports must sit above the kernel ephemeral range (32768 to 60999): a 57xxx base collided with a client socket on 2026-09-17. -case $base in 57730) base=64200 ;; 57760) base=64100 ;; esac -mkdir -p "$dir" -cp "$root/compose.yml" "$dir/compose.yml" -cd "$dir" -umask 077 -rpc_secret=$(openssl rand -hex 16) -db_secret=$(openssl rand -hex 16) -printf "%s\n" \ - "RPC_USER=xca7${stack}" \ - "RPC_PASSWORD=$rpc_secret" \ - "DB_PASSWORD=$db_secret" \ - "DB_HOST_PORT=$base" \ - "NODE_HOST_PORT=$((base + 1))" \ - "TRACKER_HOST_PORT=$((base + 2))" \ - "DECODER_HOST_PORT=$((base + 3))" \ - "ENCODER_HOST_PORT=$((base + 4))" \ - "INDEXER_HOST_PORT=$((base + 5))" \ - "MINER_HOST_PORT=$((base + 6))" \ - "EXPLORER_HTTP_HOST_PORT=$((base + 7))" \ - "EXPLORER_HTTPS_HOST_PORT=$((base + 8))" \ - "HUB_HOST_PORT=$((base + 9))" \ - "LTC_NODE_HOST_PORT=$((base + 10))" \ - "LTC_TRACKER_HOST_PORT=$((base + 11))" \ - "LTC_DECODER_HOST_PORT=$((base + 12))" \ - "LTC_ENCODER_HOST_PORT=$((base + 13))" \ - "LTC_INDEXER_HOST_PORT=$((base + 14))" \ - "LTC_MINER_HOST_PORT=$((base + 15))" \ - "DOGE_NODE_HOST_PORT=$((base + 16))" \ - "DOGE_TRACKER_HOST_PORT=$((base + 17))" \ - "DOGE_DECODER_HOST_PORT=$((base + 18))" \ - "DOGE_ENCODER_HOST_PORT=$((base + 19))" \ - "DOGE_INDEXER_HOST_PORT=$((base + 20))" \ - "DOGE_MINER_HOST_PORT=$((base + 21))" > .env -chmod 600 .env -date -u -docker compose -p "xca7${stack}" up -d -# Health is read from docker ps status text, never docker inspect (inspect can print env). -health() { - docker ps --filter "name=^xca7${stack}-$1-1\$" --format "{{.Status}}" -} -for _ in $(seq 1 180); do - ok=1 - for svc in hub indexer ltc-indexer doge-indexer miner tracker decoder encoder ltc-miner doge-miner; do - case "$(health $svc)" in *"(healthy)"*) ;; *) ok=0 ;; esac - done - if [ "$ok" = 1 ]; then - set -a - . ./.env - set +a - node "$root/seed-stack-config.js" "$HUB_HOST_PORT" - node "$root/wait-indexer-schema.js" - date -u - exit 0 - fi - sleep 2 -done -date -u -for svc in hub indexer ltc-indexer doge-indexer; do printf "%s=%s\n" "$svc" "$(health $svc)"; done -printf "health timeout for xca7%s\n" "$stack" -exit 1 diff --git a/.github/attest-mirror/wait-indexer-schema.js b/.github/attest-mirror/wait-indexer-schema.js deleted file mode 100644 index 3b55483a..00000000 --- a/.github/attest-mirror/wait-indexer-schema.js +++ /dev/null @@ -1,23 +0,0 @@ -'use strict'; -// Venue warm-up: waits until the stack indexer's schema exists (issues table) and its /status answers, so a leg's -// initialCheck ISSUE never races the indexer's first migration. Credentials come from the stack .env via env. -const mariadb = require('module').createRequire(require('path').join(process.env.ATTEST_MIRROR_E2E_DIR, 'package.json'))('mariadb'); -const waitMs = Number(process.env.ATTEST_SCHEMA_WAIT_MS || 1200000); -const deadline = Date.now() + waitMs; -(async () => { - for (;;) { - let ok = false; - try { - const options = { host: '127.0.0.1', port: Number(process.env.DB_HOST_PORT), user: 'root' }; - options['pass' + 'word'] = process.env['DB_' + 'PASSWORD']; - const c = await mariadb.createConnection(options); - const rows = await c.query("SELECT COUNT(*) AS n FROM information_schema.tables WHERE table_schema = 'XChain_BTC_Regtest_Indexer' AND table_name = 'issues'"); - await c.end(); - const res = await fetch(`http://127.0.0.1:${process.env.INDEXER_HOST_PORT}/status`).catch(() => null); - ok = Number(rows[0].n) === 1 && !!res && res.ok; - } catch (e) { ok = false; } - if (ok) { console.log('indexer schema ready'); return; } - if (Date.now() > deadline) { console.log('indexer schema NOT ready after ' + Math.round(waitMs / 1000) + ' s'); process.exit(5); } - await new Promise((r) => setTimeout(r, 3000)); - } -})(); diff --git a/.github/rail-legs.json b/.github/rail-legs.json deleted file mode 100644 index 98d2b12e..00000000 --- a/.github/rail-legs.json +++ /dev/null @@ -1,146 +0,0 @@ -[ - { - "row": "L004-ACC", - "drive": ".github/attest-mirror/run-leg.sh", - "leg": "test/attestMirror/barrier_family/ab4_below_activation.test.js", - "e2e_ref": "develop" - }, - { - "row": "L004-ACC", - "drive": ".github/attest-mirror/run-leg.sh", - "leg": "test/attestMirror/venue_levers_smoke.test.js", - "e2e_ref": "develop" - }, - { - "row": "L004-ACC", - "drive": ".github/attest-mirror/run-leg.sh", - "leg": "test/attestMirror/at2-dissemination-determinism.test.js", - "e2e_ref": "develop" - }, - { - "row": "L004-ACC", - "drive": ".github/attest-mirror/run-leg.sh", - "leg": "test/attestMirror/barrier_family/bf6_producer_follower_parity.test.js", - "e2e_ref": "develop" - }, - { - "row": "L004-ACC", - "drive": ".github/attest-mirror/run-leg.sh", - "leg": "test/attestMirror/at1_mirror_finalize.test.js", - "e2e_ref": "develop" - }, - { - "row": "L004-ACC", - "drive": ".github/attest-mirror/run-leg.sh", - "leg": "test/attestMirror/zc4_fallthrough.test.js", - "e2e_ref": "develop" - }, - { - "row": "L004-ACC", - "drive": ".github/attest-mirror/run-leg.sh", - "leg": "test/attestMirror/at5_batch_replay.test.js", - "e2e_ref": "develop" - }, - { - "row": "L004-ACC", - "drive": ".github/attest-mirror/run-leg.sh", - "leg": "test/attestMirror/barrier_family/bf8_second_coin.test.js", - "e2e_ref": "develop" - }, - { - "row": "L004-ACC", - "drive": ".github/attest-mirror/run-leg.sh", - "leg": "test/attestMirror/barrier_family/bf2_armed_fix.test.js", - "e2e_ref": "develop" - }, - { - "row": "L004-ACC", - "drive": ".github/attest-mirror/run-leg.sh", - "leg": "test/attestMirror/barrier_family/bf4_boundary.test.js", - "e2e_ref": "develop" - }, - { - "row": "L004-ACC", - "drive": ".github/attest-mirror/run-leg.sh", - "leg": "test/attestMirror/barrier_family/bf5_flag_day.test.js", - "e2e_ref": "develop" - }, - { - "row": "L004-ACC", - "drive": ".github/attest-mirror/run-leg.sh", - "leg": "test/attestMirror/at6_flag_day.test.js", - "e2e_ref": "develop" - }, - { - "row": "L004-ACC", - "drive": ".github/attest-mirror/run-leg.sh", - "leg": "test/attestMirror/zc1_zero_conf.test.js", - "e2e_ref": "develop" - }, - { - "row": "L004-ACC", - "drive": ".github/attest-mirror/run-leg.sh", - "leg": "test/attestMirror/barrier_family/ab3_safety_preserved.test.js", - "e2e_ref": "develop" - }, - { - "row": "L004-ACC", - "drive": ".github/attest-mirror/run-leg.sh", - "leg": "test/attestMirror/zc3_headroom.test.js", - "e2e_ref": "develop" - }, - { - "row": "L004-ACC", - "drive": ".github/attest-mirror/run-leg.sh", - "leg": "test/attestMirror/at4_reorg.test.js", - "e2e_ref": "develop" - }, - { - "row": "L004-ACC", - "drive": ".github/attest-mirror/run-leg.sh", - "leg": "test/attestMirror/barrier_family/ab5_mixed_fleet.test.js", - "e2e_ref": "develop" - }, - { - "row": "L004-ACC", - "drive": ".github/attest-mirror/run-leg.sh", - "leg": "test/attestMirror/barrier_family/at4_signed_admission_corpus.test.js", - "e2e_ref": "develop" - }, - { - "row": "L004-ACC", - "drive": ".github/attest-mirror/run-leg.sh", - "leg": "test/attestMirror/at0-anti-wedge.test.js", - "e2e_ref": "develop" - }, - { - "row": "L004-ACC", - "drive": ".github/attest-mirror/run-leg.sh", - "leg": "test/attestMirror/zc2_remine.test.js", - "e2e_ref": "develop" - }, - { - "row": "L004-ACC", - "drive": ".github/attest-mirror/run-leg.sh", - "leg": "test/attestMirror/venue.smoke.test.js", - "e2e_ref": "develop" - }, - { - "row": "L004-ACC", - "drive": ".github/attest-mirror/run-leg.sh", - "leg": "test/attestMirror/zc5_flag_day.test.js", - "e2e_ref": "develop" - }, - { - "row": "L004-ACC", - "drive": ".github/attest-mirror/run-leg.sh", - "leg": "test/attestMirror/barrier_family/ab2_armed_fix.test.js", - "e2e_ref": "develop" - }, - { - "row": "L004-ACC", - "drive": ".github/attest-mirror/run-leg.sh", - "leg": "test/attestMirror/at3-deadline.test.js", - "e2e_ref": "develop" - } -] diff --git a/.github/workflows/attest-mirror-leg.yml b/.github/workflows/attest-mirror-leg.yml deleted file mode 100644 index d3fd72fa..00000000 --- a/.github/workflows/attest-mirror-leg.yml +++ /dev/null @@ -1,233 +0,0 @@ -name: Attest-mirror leg (regtest) - -run-name: >- - Attest-mirror legs @ e2e ${{ inputs.e2e_ref || 'develop' }} ${{ inputs.legs || '(all)' }} - -# Runs L004-ACC's attest-mirror legs (`BF_STAMP_AHEAD_S=900 npm run test:attest-mirror` -# in xchain-e2e-test) on GitHub-hosted runners, ONE leg per job, all in parallel, -# through the same runner the self-hosted venues use (scripts/run-attest-mirror.js -# with scripts/attest-mirror-stack-driver.sh), instead of three legs at a time on -# one venue disk. -# -# The stack root in .github/attest-mirror/ contains compose.yml, setup-stack.sh, -# run-leg.sh, seed-stack-config.js and wait-indexer-schema.js. Its host paths are -# relative, so the driver boots an isolated BTC+LTC+DOGE -# compose stack per leg from the xchain-node images, which this job builds first by -# installing the three regtest stacks. The federation mnemonic the venue keeps in -# at7.env is generated per job and masked. -# -# EVIDENCE: artifact am- holds the runner output, the leg log and the -# venue logs; the leg's verdict is its exit code, as on the venue. - -on: - schedule: - - cron: '30 6 * * *' - workflow_dispatch: - inputs: - e2e_ref: - description: 'xchain-e2e-test ref the legs run from' - type: string - default: develop - legs: - description: 'Comma-separated leg files (test/attestMirror/...); empty runs all' - type: string - default: '' - stack_ref: - description: 'Ref the service images and source roots are built at' - type: string - default: develop - -permissions: - contents: read - -jobs: - plan: - runs-on: ubuntu-latest - timeout-minutes: 5 - outputs: - legs: ${{ steps.legs.outputs.legs }} - e2e_ref: ${{ steps.legs.outputs.e2e_ref }} - steps: - - uses: actions/checkout@v4 - with: - sparse-checkout: .github - - name: List the legs - id: legs - env: - E2E_REF: ${{ inputs.e2e_ref || 'develop' }} - LEGS_INPUT: ${{ inputs.legs }} - EVENT: ${{ github.event_name }} - run: | - DRIVE=.github/attest-mirror/run-leg.sh - test -f "$DRIVE" || { echo "::error::missing drive script: $DRIVE"; exit 1; } - if [ "$EVENT" = schedule ]; then - E2E_REF=develop - LEGS_INPUT=$(node -e 'const fs=require("fs");const drive=process.argv[1];const l=require("./.github/rail-legs.json");if(!l.length)throw new Error("no scheduled attest-mirror legs");for(const x of l){if(x.drive!==drive)throw new Error("unsupported scheduled drive: "+x.drive);if(!fs.existsSync(x.drive)||!fs.statSync(x.drive).isFile())throw new Error("missing scheduled drive script: "+x.drive)}process.stdout.write(l.map(x=>x.leg).join(","))' "$DRIVE") - fi - echo "e2e_ref=$E2E_REF" >> "$GITHUB_OUTPUT" - git clone -q https://github.com/XChain-Platform/xchain-e2e-test.git e2e - git -C e2e fetch -q origin "$E2E_REF" && git -C e2e checkout -q FETCH_HEAD - if [ -n "$LEGS_INPUT" ]; then - LEGS=$(node -e 'console.log(JSON.stringify(process.argv[1].split(",").map(s=>s.trim()).filter(Boolean)))' "$LEGS_INPUT") - else - LEGS=$(cd e2e && ls test/attestMirror/*.test.js test/attestMirror/barrier_family/*.test.js | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>console.log(JSON.stringify(s.split("\n").filter(Boolean))))') - fi - echo "$LEGS" - echo "legs=$LEGS" >> "$GITHUB_OUTPUT" - - leg: - needs: plan - name: ${{ matrix.leg }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - leg: ${{ fromJSON(needs.plan.outputs.legs) }} - timeout-minutes: 240 - env: - GITHUB_TOKEN: ${{ github.token }} - STACK_REF: ${{ github.event_name == 'schedule' && 'develop' || inputs.stack_ref || 'develop' }} - E2E_REF: ${{ needs.plan.outputs.e2e_ref }} - XCHAIN_NODE_EXTERNAL_DB: '1' - XCHAIN_NODE_EXTERNAL_DB_HOST: '172.17.0.1' - XCHAIN_NODE_EXTERNAL_DB_PORT: '3306' - XCHAIN_NODE_EXTERNAL_DB_ROOT_USER: 'root' - XCHAIN_NODE_DATA_DIR: ${{ github.workspace }}/.xchain-data - steps: - - uses: actions/checkout@v4 - - - name: Reclaim runner disk - run: | - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android "$AGENT_TOOLSDIRECTORY" || true - docker system prune -af || true - df -h /; nproc; free -g - - # AFTER the reclaim, which deletes the tool cache setup-node installs into; - # before it, later steps silently fell back to the image's own Node (v22.23.2 - # on some images, v22.23.3 on others, the latter failing the consensus pin). - - uses: actions/setup-node@v4 - with: - node-version: '22.22.3' - - - name: Start headless MariaDB (for the image-building install) - run: | - XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD="$(openssl rand -hex 24)" - echo "::add-mask::$XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD" - echo "XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD=$XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD" >> "$GITHUB_ENV" - docker run -d --name xchain-ci-db \ - -e MARIADB_ROOT_PASSWORD="$XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD" \ - -p 3306:3306 mariadb:11 --max-connections=1000 - for i in $(seq 1 30); do - if docker exec xchain-ci-db mariadb-admin -uroot -p"$XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD" ping 2>/dev/null | grep -q "is alive"; then - echo "MariaDB ready"; break - fi - sleep 3 - done - - - name: Build the service images (install the three regtest stacks) - # Authenticated API reads: the anonymous limit (60 an hour per runner IP) is - # shared by every job on the runner pool. - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - npm ci - git config --global url."https://github.com/".insteadOf "git@github.com:" - write_ports() { - local file="$1" base="$2"; shift 2 - { - echo "NODE_EXPOSED_PORT=$base" - echo "UTXO_TRACKER_PORT=$((base + 1))" - echo "DECODER_PORT=$((base + 2))" - echo "ENCODER_PORT=$((base + 3))" - echo "INDEXER_PORT=$((base + 4))" - echo "REGTEST_MINER_PORT=$((base + 5))" - for line in "$@"; do echo "$line"; done - } > "$file" - } - mkdir -p config "$XCHAIN_NODE_DATA_DIR" - write_ports config/bitcoin-regtest 3020 - write_ports config/dogecoin-regtest 3120 "BTC_SERVICE_HOST=$XCHAIN_NODE_EXTERNAL_DB_HOST" \ - "BTC_INDEXER_API_URL=http://xchain-node-bitcoin-regtest-xchain-indexer:3004" - write_ports config/litecoin-regtest 3220 "BTC_SERVICE_HOST=$XCHAIN_NODE_EXTERNAL_DB_HOST" \ - "BTC_INDEXER_API_URL=http://xchain-node-bitcoin-regtest-xchain-indexer:3004" - node src/index.js install "$STACK_REF" all bitcoin regtest - node src/index.js install "$STACK_REF" all litecoin regtest - node src/index.js install "$STACK_REF" all dogecoin regtest - # The standing stacks stay up: legs read the standing - # explorer (18080) and the standing LTC/DOGE indexers (at0-anti-wedge reads 3224). - docker pull -q mariadb:10.11 && docker tag mariadb:10.11 xchain-node-database - docker images --format '{{.Repository}}' | grep '^xchain-node-' | sort - docker ps --format '{{.Names}} {{.Status}}' - - - name: Build the source root (e2e at ${{ needs.plan.outputs.e2e_ref }}) - run: | - ROOT="$RUNNER_TEMP/rail" - mkdir -p "$ROOT" - echo "RAIL_ROOT=$ROOT" >> "$GITHUB_ENV" - ln -sfn "$GITHUB_WORKSPACE" "$ROOT/xchain-node" - git clone -q https://github.com/XChain-Platform/xchain-e2e-test.git "$ROOT/xchain-e2e-test" - git -C "$ROOT/xchain-e2e-test" fetch -q origin "$E2E_REF" - git -C "$ROOT/xchain-e2e-test" checkout -q FETCH_HEAD - echo "e2e at $(git -C "$ROOT/xchain-e2e-test" rev-parse HEAD) ($E2E_REF)" - REPOS="xchain-hub xchain-indexer xchain-vm xchain-sdk xchain-decoder xchain-utxo-tracker xchain-encoder xchain-explorer xchain-documentation xchain-contracts xchain-regtest-miner xchain-sync" - for r in $REPOS; do git clone -q --depth 1 --branch "$STACK_REF" "https://github.com/XChain-Platform/$r.git" "$ROOT/$r" & done - wait - for r in $REPOS; do echo "$r at $(git -C "$ROOT/$r" rev-parse --short HEAD 2>/dev/null)"; done - ln -sfn ../xchain-vm "$ROOT/xchain-indexer/xchain-vm" - for s in xchain-hub xchain-sdk xchain-vm; do ln -sfn "../$s" "$ROOT/xchain-e2e-test/$s"; done - for r in xchain-vm xchain-hub xchain-sdk xchain-decoder xchain-utxo-tracker xchain-encoder; do - (cd "$ROOT/$r" && npm ci --no-audit --no-fund --silent > "$RUNNER_TEMP/npm-$r.log" 2>&1 || { echo "npm ci failed in $r"; tail -30 "$RUNNER_TEMP/npm-$r.log"; exit 1; }) & - done - wait - (cd "$ROOT/xchain-indexer" && npm ci --no-audit --no-fund --silent) - (cd "$ROOT/xchain-e2e-test" && npm ci --no-audit --no-fund --silent) - # The in-process venue indexers refuse any runtime but the fleet's pinned - # one (xchain-vm/src/consensus-runtime.js: v8, icu, unicode, cldr, ABI 127), - # so check the job's Node against that pin before spending a drive on it. - node -e 'const c=require(process.argv[1]+"/xchain-vm/src/consensus-runtime.js"); const r=c.checkConsensusRuntime(); console.log("node "+process.version+" consensus runtime: "+c.describeMismatch(r)); process.exit(r.ok?0:1)' "$ROOT" - - - name: Stage the stack root - run: | - S="$RUNNER_TEMP/am" - cp -r .github/attest-mirror "$S" - chmod +x "$S"/*.sh - umask 077 - M=$(node -e 'console.log(require(process.argv[1] + "/node_modules/bip39").generateMnemonic())' "$RAIL_ROOT/xchain-e2e-test") - echo "::add-mask::$M" - printf 'XC_ROLLCALL_FEDERATION_MNEMONIC="%s"\n' "$M" > "$S/at7.env" - echo "AM_ROOT=$S" >> "$GITHUB_ENV" - echo "L004_LOG_DIR=$RUNNER_TEMP/am-logs" >> "$GITHUB_ENV" - mkdir -p "$RUNNER_TEMP/am-logs" - - - name: Run ${{ matrix.leg }} - working-directory: ${{ env.RAIL_ROOT }}/xchain-e2e-test - env: - LEG: ${{ matrix.leg }} - run: | - set +e - env L004_VENUE_ROOT="$RAIL_ROOT" ATTEST_MIRROR_CONCURRENCY=1 ATTEST_MIRROR_READY_TIMEOUT_MS=1200000 \ - ATTEST_MIRROR_STACK_ROOT="$AM_ROOT" ATTEST_MIRROR_STACK_PREFIX=xca7 \ - ATTEST_MIRROR_PORT_BASE=62100 ATTEST_MIRROR_VENUE_PORT_BASE=63400 \ - ATTEST_MIRROR_LEGS="$LEG" \ - ATTEST_MIRROR_STACK_DRIVER="$RAIL_ROOT/xchain-e2e-test/scripts/attest-mirror-stack-driver.sh" \ - ATTEST_MIRROR_E2E_DIR="$RAIL_ROOT/xchain-e2e-test" ATTEST_MIRROR_EXPLORER_PORT=18080 \ - BF_STAMP_AHEAD_S=900 node scripts/run-attest-mirror.js 2>&1 | tee "$L004_LOG_DIR/runner.log" - rc=${PIPESTATUS[0]} - echo "runner exit $rc" - for f in "$L004_LOG_DIR"/*.log; do [ "$f" = "$L004_LOG_DIR/runner.log" ] && continue; echo "=== $f"; tail -60 "$f"; done - exit "$rc" - - - name: Name the artifact - if: always() - env: - LEG: ${{ matrix.leg }} - run: echo "AM_ARTIFACT=am-$(basename "$LEG" .test.js)" >> "$GITHUB_ENV" - - - name: Upload the leg logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: ${{ env.AM_ARTIFACT }} - path: ${{ runner.temp }}/am-logs/ - if-no-files-found: warn - retention-days: 30 diff --git a/.github/workflows/rail-leg.yml b/.github/workflows/rail-leg.yml deleted file mode 100644 index c87401a2..00000000 --- a/.github/workflows/rail-leg.yml +++ /dev/null @@ -1,328 +0,0 @@ -name: Bridge rail leg (regtest) - -run-name: >- - Rail legs ${{ inputs.legs }} @ stack ${{ inputs.stack_ref || 'develop' }} - -# Runs xchain-e2e-test bridge rail acceptance legs (scripts/rail_leg_drive.js) on -# GitHub-hosted runners, one leg per job, many in parallel, beside the -# self-hosted rails whose single drive slot per venue is the release bottleneck. -# -# WHY A HOSTED RUNNER CAN CARRY A LEG. A leg needs a BTC and a DOGE regtest stack, -# a hub and indexers; the nightly's dogecoin shard already boots exactly that -# two-stack shape (DOGE on host ports 3120-3125, BTC on 3020-3025, hub 10000) in -# about five minutes. The drive then runs on the runner HOST, like on the rails: -# it spawns its own in-process venue hubs and indexers from sibling source trees -# under BRIDGE_RAIL_REPO_ROOT and reaches the standing services through their -# published ports, via a host .env that scripts/rail_leg_env.js derives from the -# running containers. A fresh chain needs no operator secret: the bridge -# capability set is near empty and XCHAIN MINT is an open faucet far below -# MAX_SUPPLY, so the harness stakes its own quorum. -# -# EVIDENCE. Each job uploads the leg's journal dir (case-journal.jsonl plus the -# venue logs) as artifact rail--, so a row verify reads it with -# `gh run download -R XChain-Platform/xchain-node -n rail-- -D `. -# -on: - workflow_dispatch: - inputs: - legs: - description: 'JSON array of {"row","drive","leg","e2e_ref"}' - type: string - required: true - stack_ref: - description: 'Ref every service repo is installed and cloned at' - type: string - default: develop - limit_minutes: - description: 'Per-leg bound passed to rail_leg_drive.js --limit-minutes' - type: string - default: '200' - -permissions: - contents: read - -jobs: - plan: - runs-on: ubuntu-latest - timeout-minutes: 5 - outputs: - legs: ${{ steps.legs.outputs.legs }} - steps: - - uses: actions/checkout@v4 - with: - sparse-checkout: .github - - name: Resolve the leg list - id: legs - env: - LEGS_INPUT: ${{ inputs.legs }} - run: | - test -n "$LEGS_INPUT" - printf '%s' "$LEGS_INPUT" > "$RUNNER_TEMP/legs.json" - LEGS=$(node -e 'const l=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8")); if(!Array.isArray(l)||!l.length) throw new Error("empty leg list"); for (const x of l) for (const k of ["row","drive","leg","e2e_ref"]) if(!x[k]) throw new Error("leg missing "+k+": "+JSON.stringify(x)); console.log(JSON.stringify(l))' "$RUNNER_TEMP/legs.json") - echo "$LEGS" - echo "legs=$LEGS" >> "$GITHUB_OUTPUT" - - leg: - needs: plan - name: ${{ matrix.row }} ${{ matrix.drive }} ${{ matrix.leg }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - max-parallel: 20 - matrix: - include: ${{ fromJSON(needs.plan.outputs.legs) }} - timeout-minutes: 350 - env: - GITHUB_TOKEN: ${{ github.token }} - COIN: dogecoin - STACK_REF: ${{ inputs.stack_ref || 'develop' }} - LIMIT_MINUTES: ${{ inputs.limit_minutes || '200' }} - XCHAIN_NODE_EXTERNAL_DB: '1' - XCHAIN_NODE_EXTERNAL_DB_HOST: '172.17.0.1' - XCHAIN_NODE_EXTERNAL_DB_PORT: '3306' - XCHAIN_NODE_EXTERNAL_DB_ROOT_USER: 'root' - XCHAIN_NODE_DATA_DIR: ${{ github.workspace }}/.xchain-data - steps: - - uses: actions/checkout@v4 - - - name: Reclaim runner disk - run: | - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android "$AGENT_TOOLSDIRECTORY" || true - docker system prune -af || true - df -h /; nproc; free -g - - # AFTER the reclaim, which deletes the tool cache setup-node installs into; - # before it, later steps silently fell back to the image's own Node (v22.23.2 - # on some images, v22.23.3 on others, the latter failing the consensus pin). - - uses: actions/setup-node@v4 - with: - node-version: '22.22.3' - - - name: Start headless MariaDB - run: | - XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD="$(openssl rand -hex 24)" - echo "::add-mask::$XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD" - echo "XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD=$XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD" >> "$GITHUB_ENV" - docker run -d --name xchain-ci-db \ - -e MARIADB_ROOT_PASSWORD="$XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD" \ - -p 3306:3306 mariadb:11 --max-connections=2000 - for i in $(seq 1 30); do - if docker exec xchain-ci-db mariadb-admin -uroot -p"$XCHAIN_NODE_EXTERNAL_DB_ROOT_PASSWORD" ping 2>/dev/null | grep -q "is alive"; then - echo "MariaDB ready"; break - fi - sleep 3 - done - - - name: Install xchain-node deps - # Authenticated API reads: the anonymous limit (60 an hour per runner IP) is - # shared by every job on the runner pool. - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - npm ci - git config --global url."https://github.com/".insteadOf "git@github.com:" - - # Same shape as the nightly's dogecoin shard: a local validator hub, DOGE - # stack on 3120-3125 with the bitcoin stack beside it on 3020-3025. The stack - # settings go to a file the install step sources, NOT to GITHUB_ENV: they - # carry docker hostnames (xchain-node-bitcoin-regtest-xchain-indexer), and the - # drive's in-process venue hubs inherit the job env, so a leaked - # BTC_INDEXER_API_URL left every venue hub unable to read its capability set. - - name: Initialize the validator identity - run: | - node src/index.js validator init \ - --network regtest \ - --oracle-epoch-start "$(date -u +%s)000" \ - --capabilities price,cross_chain,oracle_publish,attestation - { - echo "HUB_NETWORK=regtest" - echo "HUB_SYNC_ANCHOR_ATTEST_GRACE_S=2" - echo "HUB_SYNC_BRIDGE_GRACE_S=2" - echo "HUB_SYNC_POLICY_GRACE_S=2" - echo "HUB_PRICE_SYNC_TIMEOUT_MS=10000" - echo "ORACLE_MIN_SUBMISSIONS=1" - echo "XDEX_SEED_LOCAL_VALIDATOR=1" - echo "BTC_INDEXER_API_URL=http://xchain-node-bitcoin-regtest-xchain-indexer:3004" - echo "BTC_INDEXER_URL=http://xchain-node-bitcoin-regtest-xchain-indexer:3004" - echo "DOGE_INDEXER_URL=http://xchain-node-dogecoin-regtest-xchain-indexer:3004" - echo "XCHAIN_CONFIRMATIONS_BTC=1" - echo "XCHAIN_CONFIRMATIONS_LTC=1" - echo "XCHAIN_CONFIRMATIONS_DOGE=1" - } > "$RUNNER_TEMP/stack.env" - - - name: Publish the two-stack host ports - run: | - write_ports() { - local file="$1" base="$2"; shift 2 - { - echo "NODE_EXPOSED_PORT=$base" - echo "UTXO_TRACKER_PORT=$((base + 1))" - echo "DECODER_PORT=$((base + 2))" - echo "ENCODER_PORT=$((base + 3))" - echo "INDEXER_PORT=$((base + 4))" - echo "REGTEST_MINER_PORT=$((base + 5))" - for line in "$@"; do echo "$line"; done - } > "$file" - } - mkdir -p config - write_ports config/dogecoin-regtest 3120 \ - "BTC_SERVICE_HOST=$XCHAIN_NODE_EXTERNAL_DB_HOST" \ - "BTC_INDEXER_API_URL=http://xchain-node-bitcoin-regtest-xchain-indexer:3004" - write_ports config/bitcoin-regtest 3020 - - - name: Boot the two regtest stacks at ${{ inputs.stack_ref || 'develop' }} - # Authenticated API reads: the anonymous limit (60 an hour per runner IP) is - # shared by every job on the runner pool. - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - mkdir -p "$XCHAIN_NODE_DATA_DIR" - set -a; . "$RUNNER_TEMP/stack.env"; set +a - node src/index.js install "$STACK_REF" all dogecoin regtest - node src/index.js install "$STACK_REF" all bitcoin regtest - docker ps --format '{{.Names}} {{.Status}} {{.Ports}}' - - - name: Build the rail root (e2e at ${{ matrix.e2e_ref }}, services at the stack ref) - env: - E2E_REF: ${{ matrix.e2e_ref }} - run: | - ROOT="$RUNNER_TEMP/rail" - mkdir -p "$ROOT" - echo "RAIL_ROOT=$ROOT" >> "$GITHUB_ENV" - ln -sfn "$GITHUB_WORKSPACE" "$ROOT/xchain-node" - git clone -q https://github.com/XChain-Platform/xchain-e2e-test.git "$ROOT/xchain-e2e-test" - git -C "$ROOT/xchain-e2e-test" fetch -q origin "$E2E_REF" - git -C "$ROOT/xchain-e2e-test" checkout -q FETCH_HEAD - echo "e2e at $(git -C "$ROOT/xchain-e2e-test" rev-parse HEAD) ($E2E_REF)" - for r in xchain-hub xchain-indexer xchain-vm xchain-sdk xchain-decoder xchain-utxo-tracker xchain-encoder xchain-explorer xchain-documentation xchain-contracts; do - git clone -q --depth 1 --branch "$STACK_REF" "https://github.com/XChain-Platform/$r.git" "$ROOT/$r" & - done - wait - for r in xchain-hub xchain-indexer xchain-vm xchain-sdk xchain-decoder xchain-utxo-tracker xchain-encoder xchain-explorer xchain-documentation xchain-contracts; do - echo "$r at $(git -C "$ROOT/$r" rev-parse --short HEAD)" - done - ln -sfn ../xchain-vm "$ROOT/xchain-indexer/xchain-vm" - for s in xchain-hub xchain-sdk xchain-vm; do ln -sfn "../$s" "$ROOT/xchain-e2e-test/$s"; done - for r in xchain-vm xchain-hub xchain-sdk xchain-decoder xchain-utxo-tracker xchain-encoder; do - (cd "$ROOT/$r" && npm ci --no-audit --no-fund --silent > "$RUNNER_TEMP/npm-$r.log" 2>&1 || { echo "npm ci failed in $r"; tail -30 "$RUNNER_TEMP/npm-$r.log"; exit 1; }) & - done - wait - (cd "$ROOT/xchain-indexer" && npm ci --no-audit --no-fund --silent) - (cd "$ROOT/xchain-e2e-test" && npm ci --no-audit --no-fund --silent) - # The in-process venue indexers refuse any runtime but the fleet's pinned - # one (xchain-vm/src/consensus-runtime.js: v8, icu, unicode, cldr, ABI 127), - # so check the job's Node against that pin before spending a drive on it. - node -e 'const c=require(process.argv[1]+"/xchain-vm/src/consensus-runtime.js"); const r=c.checkConsensusRuntime(); console.log("node "+process.version+" consensus runtime: "+c.describeMismatch(r)); process.exit(r.ok?0:1)' "$ROOT" - node -e 'require(process.argv[1] + "/xchain-indexer/node_modules/xchain-vm"); console.log("xchain-indexer loads xchain-vm")' "$ROOT" - - - name: Write the host .env for the drive - run: | - node scripts/rail_leg_env.js "$RAIL_ROOT/xchain-e2e-test/.env" bitcoin - mkdir -p "$RAIL_ROOT/xchain-e2e-test/drill-keys" - chmod 700 "$RAIL_ROOT/xchain-e2e-test/drill-keys" - - # The self-hosted rails were seeded once with the four-source ROLLCALL - # federation (test/tools/rollcallSeedFederation.test.js), whose keys the - # harness can sign for; the bridge drives reuse that seated set. A fresh - # chain has an EMPTY capability set, and the policy bootstrap only places - # new signers in gaps between seated keys, so with nothing seated it stakes - # nothing. Seeding here gives the runner the same shape the rails have. The - # mnemonic is generated per job, masked, and dies with the runner: it - # guards nothing but this ephemeral chain. E2E_STAKE_TEARDOWN=off keeps the - # seeded stakes after the tool exits, as on the rails (the default teardown - # unstakes them, leaving the drive an empty set again). - - name: Seed the rail federation (ephemeral mnemonic) - working-directory: ${{ env.RAIL_ROOT }}/xchain-e2e-test - run: | - M=$(node -e 'console.log(require("bip39").generateMnemonic())') - echo "::add-mask::$M" - echo "XC_ROLLCALL_FEDERATION_MNEMONIC=$M" >> "$GITHUB_ENV" - env XC_ROLLCALL_FEDERATION_MNEMONIC="$M" COIN=bitcoin NETWORK=regtest NODE_PATH="$RAIL_ROOT" E2E_STAKE_TEARDOWN=off \ - ./node_modules/.bin/mocha --no-config --timeout 0 --exit \ - --require ./test/initialCheck.test.js test/tools/rollcallSeedFederation.test.js - - # Fail fast when the seeded federation is not a member of the sets the drive - # resolves its quorum from, instead of 25 s into the drive. Reads the BTC - # indexer the way the seed tool does: getstakeweightsbycapability at the tip. - - name: Check the seated capability sets - run: | - node - <<'JS' - const rpc = async (method, params) => { - const r = await fetch('http://127.0.0.1:3024/', { method: 'POST', headers: { 'content-type': 'application/json' }, - body: JSON.stringify({ jsonrpc: '2.0', id: 1, method, params }) }) - const j = await r.json() - if (j.error) throw new Error(method + ': ' + JSON.stringify(j.error)) - return j.result - } - ;(async () => { - const tip = Number((await rpc('getblockhashes', {})).block_index) - let crossChain = null - for (const capability of ['oracle_publish', 'cross_chain', 'price', 'attestation']) { - const res = await rpc('getstakeweightsbycapability', { capability, block_index: tip }) - if (!res || !Array.isArray(res.validators)) { console.log(capability + ': unexpected shape ' + JSON.stringify(res).slice(0, 300)); continue } - const v = res.validators - const weight = v.reduce((sum, x) => sum + Number(x.weight || 0), 0) - console.log(capability + ' at block ' + tip + ': ' + v.length + ' member(s), weight ' + weight) - if (capability === 'cross_chain') crossChain = v.length - } - if (crossChain === 0) { console.log('::error::cross_chain is empty after seeding; the drive cannot resolve a quorum'); process.exit(1) } - })().catch((e) => { console.log('::warning::capability read failed, not gating: ' + e.message) }) - JS - - - name: Drive ${{ matrix.drive }} ${{ matrix.leg }} - id: drive - working-directory: ${{ env.RAIL_ROOT }}/xchain-e2e-test - env: - ROW: ${{ matrix.row }} - DRIVE: ${{ matrix.drive }} - LEG: ${{ matrix.leg }} - run: | - J="$RUNNER_TEMP/journal/$ROW/$LEG" - mkdir -p "$J" - echo "JOURNAL_DIR=$J" >> "$GITHUB_ENV" - set +e - # Shape watchdog: a wiring fault (a hostname the host cannot resolve, a - # runtime the VM refuses, a missing module) otherwise surfaces only as a - # 30-minute wait inside a case. The drive runs in its own process group; - # once a fault signature repeats in the drive or venue logs, that group - # alone is ended and the job fails naming it. - setsid env DOGE_INDEXER_API_PORT=3124 \ - NODE_PATH="$RAIL_ROOT" BRIDGE_RAIL_REPO_ROOT="$RAIL_ROOT" \ - BRIDGE_RAIL_MINER_PAUSE_FILE="$RUNNER_TEMP/btc-loop.pause" \ - BRIDGE_RAIL_DOGE_MINER_PAUSE_FILE="$RUNNER_TEMP/doge-loop.pause" \ - node scripts/rail_leg_drive.js "$DRIVE" "$LEG" --journal-dir "$J" --limit-minutes "$LIMIT_MINUTES" > "$J/drive.log" 2>&1 & - pgid=$! - SHAPE='EAI_AGAIN|ENOTFOUND|CONSENSUS RUNTIME MISMATCH|Cannot find module' - while kill -0 "$pgid" 2>/dev/null; do - sleep 60 - hits=$(cat "$J"/*.log 2>/dev/null | grep -cE "$SHAPE") - if [ "$hits" -ge 5 ]; then - echo "::error::venue shape fault ($hits hits): $(cat "$J"/*.log | grep -m1 -oE ".{0,160}($SHAPE).{0,120}")" - kill -TERM -- "-$pgid" 2>/dev/null; sleep 30; kill -KILL -- "-$pgid" 2>/dev/null - break - fi - done - wait "$pgid"; rc=$? - [ "${hits:-0}" -ge 5 ] && rc=97 - cat "$J/drive.log" - echo "drive exit $rc" - test -s "$J/case-journal.jsonl" && node scripts/rail_journal_triage.js "$J/case-journal.jsonl" 2>/dev/null | tail -5 || true - exit "$rc" - - - name: Capture stack diagnostics - if: always() - run: | - OUT="${JOURNAL_DIR:-$RUNNER_TEMP/journal}/stack" - mkdir -p "$OUT" - docker ps -a > "$OUT/docker-ps.txt" 2>&1 || true - for c in $(docker ps -a --format '{{.Names}}'); do docker logs --tail 3000 "$c" > "$OUT/log-$c.txt" 2>&1 || true; done - df -h / > "$OUT/df.txt"; free -m > "$OUT/free.txt" - - - name: Upload the leg journal - if: always() - uses: actions/upload-artifact@v4 - with: - name: rail-${{ matrix.row }}-${{ matrix.leg }} - path: ${{ runner.temp }}/journal/ - if-no-files-found: warn - retention-days: 30 diff --git a/bin/ci_fast_select.js b/bin/ci_fast_select.js index 15ebbd0a..16885379 100644 --- a/bin/ci_fast_select.js +++ b/bin/ci_fast_select.js @@ -115,16 +115,16 @@ function collectSourceData(changedSources, findRequirers) { })) } -function consensusReasons(changedFiles, sourceData, consensusPrefixes) { +function consensusReasons(changedFiles, sourceData) { const reasons = [] for (const file of changedFiles) { - if (file === 'package.json' || startsWithAny(file, consensusPrefixes) || startsWithAny(file, WIDEN)) { + if (file === 'package.json' || startsWithAny(file, CONSENSUS) || startsWithAny(file, WIDEN)) { reasons.push(`consensus: ${file}`) } } for (const { file, requirers } of sourceData) { for (const importer of requirers) { - if (startsWithAny(importer, consensusPrefixes) && requiresFile(importer, file)) { + if (startsWithAny(importer, CONSENSUS) && requiresFile(importer, file)) { reasons.push(`consensus importer: ${importer} requires ${file}`) } } @@ -145,16 +145,12 @@ function selectedTestFiles(changedFiles, tests, sourceData, reasons) { return selected } -function selectFastTests( - changedFiles, - { listTests, findRequirers }, - { consensusPrefixes = CONSENSUS } = {}, -) { +function selectFastTests(changedFiles, { listTests, findRequirers }) { const changed = splitLines(changedFiles) const tests = splitLines(listTests()).filter((file) => groupFor(file) && fs.existsSync(file)) const sources = changed.filter((file) => file.startsWith('src/') && file.endsWith('.js')) const sourceData = collectSourceData(sources, findRequirers) - const reasons = consensusReasons(changed, sourceData, consensusPrefixes) + const reasons = consensusReasons(changed, sourceData) const selected = selectedTestFiles(changed, tests, sourceData, reasons) const consensus = reasons.some((reason) => reason.startsWith('consensus')) const planned = consensus ? [] : [...selected] @@ -181,150 +177,6 @@ function findRequirers(basename) { return splitLines(result.stdout) } -function selectionDependencies({ indexed = false } = {}) { - const files = listTrackedFiles() - if (indexed) { - const sources = files - .filter((file) => file.endsWith('.js') && fs.existsSync(file)) - .map((file) => [file, sourceText(file)]) - const cache = new Map() - return { - listTests: () => files, - findRequirers: (basename) => { - if (!cache.has(basename)) { - cache.set(basename, sources - .filter(([, source]) => source.includes(basename)) - .map(([file]) => file)) - } - return cache.get(basename) - }, - } - } - return { listTests: () => files, findRequirers } -} - -function withoutConsensusPrefixes(prefixes) { - const removed = new Set(prefixes.flatMap((prefix) => { - const trimmed = prefix.trim() - if (!trimmed) return [] - return [trimmed, trimmed.endsWith('/') ? trimmed.slice(0, -1) : `${trimmed}/`] - })) - return CONSENSUS.filter((prefix) => !removed.has(prefix)) -} - -function changedFilesForCommit(commit) { - const revision = splitLines(git(['rev-list', '--parents', '-n', '1', commit]))[0] - const [, parent] = revision.split(' ') - if (parent) return splitLines(git(['diff', '--name-only', `${parent}..${commit}`])) - return splitLines(git(['diff-tree', '--root', '--no-commit-id', '--name-only', '-r', commit])) -} - -function emptyReplayCounts() { - return { wholeUnit: 0, changedTests: 0, testOnly: 0, noTests: 0 } -} - -function countReplayPlan(counts, changed, plan) { - if (plan.consensus) { - counts.wholeUnit++ - } else if (plan.tests.length && changed.every((file) => file.startsWith('test/'))) { - counts.testOnly++ - } else if (plan.tests.length) { - counts.changedTests++ - } else { - counts.noTests++ - } -} - -function replayPlans(limit, narrowPrefixes) { - const commits = splitLines(git([ - 'log', '--first-parent', '-n', String(limit), '--format=%H', 'origin/develop', - ])) - const current = emptyReplayCounts() - const narrowed = emptyReplayCounts() - const consensusPrefixes = withoutConsensusPrefixes(narrowPrefixes) - const dependencies = selectionDependencies({ indexed: true }) - for (const commit of commits) { - const changed = changedFilesForCommit(commit) - countReplayPlan(current, changed, selectFastTests(changed, dependencies)) - countReplayPlan(narrowed, changed, selectFastTests(changed, dependencies, { - consensusPrefixes, - })) - } - return { commits, current, narrowed, consensusPrefixes, dependencies } -} - -function fraction(value, total) { - return `${value}/${total}` -} - -function printReplayRow(name, total, counts) { - console.log([ - name, - total, - fraction(counts.wholeUnit, total), - fraction(counts.changedTests, total), - fraction(counts.testOnly, total), - fraction(counts.noTests, total), - ].join(' ')) -} - -function parseList(value) { - return value.split(',').map((item) => item.trim()).filter(Boolean) -} - -function parseMustSelect(value) { - return parseList(value).map((pair) => { - const separator = pair.indexOf(':') - if (separator <= 0 || separator === pair.length - 1) { - throw new Error(`invalid --must-select pair: ${pair}`) - } - return { source: pair.slice(0, separator), test: pair.slice(separator + 1) } - }) -} - -function replayOptions(args) { - const limit = Number(args[0]) - if (!Number.isSafeInteger(limit) || limit < 1) { - throw new Error('--replay requires a positive integer') - } - const options = { limit, narrowPrefixes: [], mustSelect: [] } - for (let index = 1; index < args.length; index += 2) { - const flag = args[index] - const value = args[index + 1] - if (!value || (flag !== '--narrow' && flag !== '--must-select')) { - throw new Error(`invalid replay option: ${flag || ''}`.trim()) - } - if (flag === '--narrow') options.narrowPrefixes.push(...parseList(value)) - else options.mustSelect.push(...parseMustSelect(value)) - } - return options -} - -function runReplay(args) { - try { - const options = replayOptions(args) - const result = replayPlans(options.limit, options.narrowPrefixes) - console.log('plan commits consensus-1 changed-tests test-only no-tests') - printReplayRow('current', result.commits.length, result.current) - if (options.narrowPrefixes.length) { - printReplayRow('narrowed', result.commits.length, result.narrowed) - } - let failed = false - for (const pair of options.mustSelect) { - const plan = selectFastTests([pair.source], result.dependencies, { - consensusPrefixes: result.consensusPrefixes, - }) - const selected = plan.tests.some((test) => test.file === pair.test) - console.log(`must-select ${selected ? 'PASS' : 'FAIL'} ${pair.source}:${pair.test}`) - if (!selected) failed = true - } - return failed ? 1 : 0 - } catch (error) { - console.error(`replay-error ${error.message}`) - return 2 - } -} - function resolvePlan() { const base = resolveBase({ env: process.env, git }) if (!base) return { error: 'git could not resolve a usable push base' } @@ -357,10 +209,8 @@ function runPlan(plan) { function main() { const mode = process.argv[2] - if (mode === '--replay') return runReplay(process.argv.slice(3)) if (mode !== '--plan' && mode !== '--run') { - console.error('usage: node bin/ci_fast_select.js --plan|--run|--replay N ' + - '[--narrow prefix,...] [--must-select file:testfile,...]') + console.error('usage: node bin/ci_fast_select.js --plan|--run') return 2 } let result @@ -378,6 +228,6 @@ function main() { return mode === '--run' ? runPlan(result.plan) : 0 } -module.exports = { replayPlans, resolveBase, selectFastTests } +module.exports = { resolveBase, selectFastTests } if (require.main === module) process.exitCode = main() diff --git a/bin/test/ci_fast_select.test.js b/bin/test/ci_fast_select.test.js index b43f0382..6d2f2310 100644 --- a/bin/test/ci_fast_select.test.js +++ b/bin/test/ci_fast_select.test.js @@ -2,7 +2,6 @@ const assert = require('node:assert') const fs = require('node:fs') -const os = require('node:os') const path = require('node:path') const { execFileSync, spawnSync } = require('node:child_process') @@ -42,25 +41,6 @@ function files(plan) { return plan.tests.map((test) => test.file) } -function scratchGit(cwd, args) { - return execFileSync('git', args, { cwd, encoding: 'utf8' }).trim() -} - -function writeScratchFile(cwd, file, source) { - const target = path.join(cwd, file) - fs.mkdirSync(path.dirname(target), { recursive: true }) - fs.writeFileSync(target, source) -} - -function scratchCommit(cwd, message, files) { - scratchGit(cwd, ['add', '--', ...files]) - scratchGit(cwd, [ - '-c', 'user.name=Selector Test', - '-c', 'user.email=selector-test', - 'commit', '-m', message, - ]) -} - describe('bin/ci_fast_select.js', () => { it('maps command_lock to its unit test without widening the suite', () => { const plan = select(['src/utils/command_lock.js']) @@ -151,60 +131,4 @@ describe('bin/ci_fast_select.js', () => { assert.match(script, /CI_TIER:-full[^\n]+fast[\s\S]+ci_fast_select\.js --plan/) assert.ok(script.includes('run_tier "ci" env XCHAIN_REQUIRE_SIBLINGS=1 npm run ci')) }) - - it('replays develop history, compares narrowing, and checks required selections', () => { - const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'ci-fast-select-replay-')) - try { - scratchGit(cwd, ['init', '--initial-branch=develop']) - const initial = { - 'src/coins/rule.js': "module.exports = 'rule'\n", - 'src/feature/plain.js': "module.exports = 'plain'\n", - 'test/unit/coins/rule.test.js': "require('../../../src/coins/rule')\n", - 'test/unit/feature/plain.test.js': "require('../../../src/feature/plain')\n", - 'test/unit/only.test.js': "module.exports = 'only'\n", - 'test/unit/other/unrelated.test.js': "module.exports = 'unrelated'\n", - } - for (const [file, source] of Object.entries(initial)) { - writeScratchFile(cwd, file, source) - } - scratchCommit(cwd, 'initial files', Object.keys(initial)) - - const consensusFile = 'src/coins/rule.js' - fs.appendFileSync(path.join(cwd, consensusFile), "module.exports += ' changed'\n") - scratchCommit(cwd, 'consensus change', [consensusFile]) - - const plainFile = 'src/feature/plain.js' - fs.appendFileSync(path.join(cwd, plainFile), "module.exports += ' changed'\n") - scratchCommit(cwd, 'plain source change', [plainFile]) - - const testFile = 'test/unit/only.test.js' - fs.appendFileSync(path.join(cwd, testFile), "module.exports += ' changed'\n") - scratchCommit(cwd, 'test only change', [testFile]) - scratchGit(cwd, ['update-ref', 'refs/remotes/origin/develop', 'HEAD']) - - const selector = path.resolve(__dirname, '../ci_fast_select.js') - const mustSelect = [ - 'src/coins/rule.js:test/unit/coins/rule.test.js', - 'src/feature/plain.js:test/unit/other/unrelated.test.js', - ].join(',') - const result = spawnSync(process.execPath, [ - selector, - '--replay', '3', - '--narrow', 'src/coins/', - '--must-select', mustSelect, - ], { cwd, encoding: 'utf8' }) - - assert.strictEqual(result.status, 1, result.stderr) - const lines = result.stdout.trim().split(/\r?\n/) - assert.ok(lines.includes('plan commits consensus-1 changed-tests test-only no-tests')) - assert.ok(lines.includes('current 3 1/3 1/3 1/3 0/3')) - assert.ok(lines.includes('narrowed 3 0/3 2/3 1/3 0/3')) - assert.ok(lines.includes( - 'must-select PASS src/coins/rule.js:test/unit/coins/rule.test.js')) - assert.ok(lines.includes( - 'must-select FAIL src/feature/plain.js:test/unit/other/unrelated.test.js')) - } finally { - fs.rmSync(cwd, { recursive: true, force: true }) - } - }) }) From 71255cb6ab4c299404ef1c161420978e97b5bef5 Mon Sep 17 00:00:00 2001 From: J-Dog Date: Tue, 29 Sep 2026 18:50:37 -0700 Subject: [PATCH 50/50] Release v0.21.0 manifest --- src/release-manifest.json | 58 +++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/src/release-manifest.json b/src/release-manifest.json index 7ca8d0f1..b3031953 100644 --- a/src/release-manifest.json +++ b/src/release-manifest.json @@ -1,6 +1,6 @@ { "_comment": [ - "Pinned component set for XChain Platform v0.20.1.", + "Pinned component set for XChain Platform v0.21.0.", "Generated by bin/write-release-manifest.js from the ACTUAL tagged master merge commits.", "xchain-node is the carrier and is not listed: checking out its tag IS this manifest.", "A train tags only the repos it touches. This one moves xchain-vm, xchain-decoder, xchain-indexer, xchain-hub, xchain-sync, xchain-encoder, xchain-utxo-tracker, xchain-explorer, xchain-sdk, xchain-e2e-test and xchain-regtest-miner.", @@ -12,70 +12,70 @@ "repo's origin/master, and is a GPG-signed annotated tag verifying against the", "platform release key (fingerprint 1DA7C4896F56EA22CF491EDF4361611A82F90B70)." ], - "platform_version": "0.20.1", - "released": "2026-09-24", + "platform_version": "0.21.0", + "released": "2026-09-30", "components": { "xchain-vm": { - "tag": "v0.20.1", - "commit": "e7dbc3e3a00a26ebdfb22941bac4e284cca180d7" + "tag": "v0.21.0", + "commit": "d97442aaa61bf3709bf2288bf3cab1ac4c5d8d31" }, "xchain-decoder": { - "tag": "v0.20.1", - "commit": "6d49ed2c583b444ad127e8bb5800cc589631590f" + "tag": "v0.21.0", + "commit": "cc2c0802f1674e5aee397b0b9c4c4fce5dc2e551" }, "xchain-indexer": { - "tag": "v0.20.1", - "commit": "10ae153b0bb197bc05903cd7e504cb559104c20d" + "tag": "v0.21.0", + "commit": "919adc2af6b2ef0fcb8fdaf65d02f63567d9ae34" }, "xchain-hub": { - "tag": "v0.20.1", - "commit": "933fba792e1ab71d22219301c3b43bc1c85f3433" + "tag": "v0.21.0", + "commit": "aa48261994e9c98c1ba41d48064dc0eee568d769" }, "xchain-sync": { - "tag": "v0.20.1", - "commit": "d2f37e168aee2fa122ad3ae950965dee17901105" + "tag": "v0.21.0", + "commit": "e33920fedbcc648029d8b2aa5c49b0d4b421570c" }, "xchain-encoder": { - "tag": "v0.20.1", - "commit": "0f5b08f4ceb2f8c37e2f3f13c7cc30dd98b82d82" + "tag": "v0.21.0", + "commit": "3083725d7a9624bf13bdc5e59eac7fd79b35dd32" }, "xchain-utxo-tracker": { - "tag": "v0.20.1", - "commit": "dee5717e4b03176bb9b432391a39c762b50c3c18" + "tag": "v0.21.0", + "commit": "8b2092686b6d5eb26117df5625a6899839d3caf4" }, "xchain-explorer": { - "tag": "v0.20.1", - "commit": "1f1eb1b50af6815c8a795326f88910f892da359f" + "tag": "v0.21.0", + "commit": "e274491ce5838703dde00e87045f37047f1add6f" }, "xchain-sdk": { - "tag": "v0.20.1", - "commit": "fc192852f0b86a489b45689eb940f3e710ae4e98" + "tag": "v0.21.0", + "commit": "6631e9ae7eac9c40e23cf24a7ef84ebfaf82131c" }, "xchain-e2e-test": { - "tag": "v0.20.1", - "commit": "2da7650c0098e34962f48cf663dd8ae491bc504d" + "tag": "v0.21.0", + "commit": "b0f70f96284483c3e70264c72b378027bd89d78a" }, "xchain-contracts": { "tag": "v0.17.0", "commit": "684a6311be7232542346c984bd9f2d62f3251810" }, "xchain-regtest-miner": { - "tag": "v0.20.1", - "commit": "8e818394d854d796951f8959c840405c4d5ccaf6" + "tag": "v0.21.0", + "commit": "a4282f8a53d546320b1568c3db851dd7b5fc6b43" } }, "trainActivation": { - "ruleSetVersion": "0.20.0", + "ruleSetVersion": "0.21.0", "classification": "major", "heights": { "mainnet": 9999999999, "regtest": 0, - "testnet": 154074 + "testnet": 154566 }, "computedFromBtcTip": { - "mainnet": 967625, + "mainnet": 969236, "regtest": 0, - "testnet": 153698 + "testnet": 154472 } } }