diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e8bc3253c05..438a31c30b2 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,18 +1,50 @@ -name: Docker build on tag +name: Container images env: DOCKER_CLI_EXPERIMENTAL: enabled - TAG_FMT: "^refs/tags/(((.?[0-9]+){3,4}))$" DOCKER_BUILDKIT: 1 # Enable BuildKit for better performance COMPOSE_DOCKER_CLI_BUILD: 0 +# Two routes, never mixed: +# - A pull request to develop or main that touches the container templates or +# the runtime configuration they render gets a build-only gate: the images +# are built and started locally on the runner and nothing is pushed. +# - Publication to the registry happens only from an explicit dispatch that +# says so. There is no tag trigger and no label shortcut: nobody publishes +# an image by pushing a ref or adding a label to a pull request. on: - push: - tags: - - v[0-9]+.[0-9]+.[0-9]+ - - v[0-9]+.[0-9]+.[0-9]+-* pull_request: - branches: [master] - types: [opened, synchronize, reopened, labeled] + branches: [develop, main] + types: [opened, synchronize, reopened] + paths: + - 'docker/**' + - 'production/mempool-config.*.json' + - 'production/mempool-frontend-config.*.json' + - 'backend/mempool-config.sample.json' + - 'frontend/mempool-frontend-config.sample.json' + - 'nginx.conf' + - 'nginx-mempool.conf' + - 'rust/gbt/rust-toolchain' + - 'backend/package.json' + - 'backend/package-lock.json' + - 'frontend/package.json' + - 'frontend/package-lock.json' + - '.github/workflows/docker.yml' + workflow_dispatch: + inputs: + tag: + description: 'Image tag to publish (for example universe-2026.09.17)' + required: true + type: string + publish: + description: 'Push the built images to the registry' + required: true + type: boolean + default: false + latest: + description: 'Also move the latest tag to this build (publish must be true)' + required: false + type: boolean + default: false permissions: contents: read @@ -21,7 +53,7 @@ jobs: test-images: env: COMPOSE_PROJECT_NAME: mempool-images-${{ github.run_id }}-${{ github.run_attempt }} - # Always run on tag pushes and all pull requests + # Build-only: runs for every pull request in scope and every dispatch. runs-on: [self-hosted, linux, x64, universe-super] timeout-minutes: 30 name: Test built Docker images @@ -38,10 +70,15 @@ jobs: fi echo "SHORT_SHA=${SHA:0:8}" >> $GITHUB_ENV - - name: Set TAG from pushed tag or package.json + - name: Set TAG from the dispatch input or package.json + env: + DISPATCH_TAG: ${{ github.event.inputs.tag }} run: | - if [ "${{ github.event_name }}" = "push" ]; then - TAG="${GITHUB_REF/refs\/tags\//}" + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + case "$DISPATCH_TAG" in + ''|*[!A-Za-z0-9._-]*) echo "Error: tag must be a plain image tag"; exit 1 ;; + esac + TAG="$DISPATCH_TAG" else FRONTEND_VERSION=$(jq -r '.version' frontend/package.json) BACKEND_VERSION=$(jq -r '.version' backend/package.json) @@ -226,11 +263,12 @@ jobs: build: needs: [test-images] - # Run on tag pushes OR on PRs with "docker-push" label (after test-images passes) + # Publication: only an explicit dispatch that asked for it, and only after + # the build-only gate passed on the same commit. if: | needs.test-images.result == 'success' && - (github.event_name == 'push' || - (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'docker-push'))) + github.event_name == 'workflow_dispatch' && + github.event.inputs.publish == 'true' strategy: matrix: service: @@ -242,26 +280,24 @@ jobs: outputs: image-digest-frontend: ${{ matrix.service == 'frontend' && steps.docker-build.outputs.digest || '' }} image-digest-backend: ${{ matrix.service == 'backend' && steps.docker-build.outputs.digest || '' }} - tag: ${{ matrix.service == 'frontend' && (steps.set-tag-push.outputs.tag || steps.set-tag-pr.outputs.tag) || '' }} + tag: ${{ matrix.service == 'frontend' && steps.set-tag.outputs.tag || '' }} steps: # Shared runner capacity is provisioned outside jobs. Never replace host # swap, hide Docker data, or restart the shared daemon during a build. - # Only for tag pushes: use the Git tag as TAG - - name: Set TAG from pushed tag - if: github.event_name == 'push' - id: set-tag-push + - name: Set TAG from the dispatch input + id: set-tag + env: + DISPATCH_TAG: ${{ github.event.inputs.tag }} run: | - TAG="${GITHUB_REF/refs\/tags\//}" - echo "TAG=${TAG}" >> $GITHUB_ENV - echo "tag=${TAG}" >> $GITHUB_OUTPUT + case "$DISPATCH_TAG" in + ''|*[!A-Za-z0-9._-]*) echo "Error: tag must be a plain image tag"; exit 1 ;; + esac + echo "TAG=${DISPATCH_TAG}" >> $GITHUB_ENV + echo "tag=${DISPATCH_TAG}" >> $GITHUB_OUTPUT - name: Add SHORT_SHA env property with commit short sha run: | - if [ "${{ github.event_name }}" = "pull_request" ]; then - SHA="${{ github.event.pull_request.head.sha }}" - else - SHA="${GITHUB_SHA}" - fi + SHA="${GITHUB_SHA}" echo "SHORT_SHA=${SHA:0:8}" >> $GITHUB_ENV @@ -274,20 +310,6 @@ jobs: - name: Checkout project uses: actions/checkout@v4 - # For PRs: use package.json version + short sha as TAG - - name: Set TAG from service package.json for pull requests - if: github.event_name == 'pull_request' - id: set-tag-pr - run: | - if [ "${{ matrix.service }}" = "frontend" ]; then - VERSION=$(jq -r '.version' frontend/package.json) - else - VERSION=$(jq -r '.version' backend/package.json) - fi - TAG="v${VERSION}-${SHORT_SHA}" - echo "TAG=${TAG}" >> $GITHUB_ENV - echo "tag=${TAG}" >> $GITHUB_OUTPUT - - name: Show set environment variables run: | printf " TAG: %s\n" "$TAG" @@ -338,8 +360,8 @@ jobs: tag-latest: needs: [build] - # Only for successful tag pushes (not PRs with docker-push label) and only for "plain" versions (no '-') - if: ${{ needs.build.result == 'success' && github.event_name == 'push' && !contains(github.ref_name, '-') }} + # Only when the same dispatch asked for it. + if: ${{ needs.build.result == 'success' && github.event_name == 'workflow_dispatch' && github.event.inputs.publish == 'true' && github.event.inputs.latest == 'true' }} runs-on: [self-hosted, linux, x64, universe-super] timeout-minutes: 30 name: Tag release build as latest @@ -350,7 +372,13 @@ jobs: - backend steps: - name: Set env variables - run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV + env: + DISPATCH_TAG: ${{ github.event.inputs.tag }} + run: | + case "$DISPATCH_TAG" in + ''|*[!A-Za-z0-9._-]*) echo "Error: tag must be a plain image tag"; exit 1 ;; + esac + echo "TAG=${DISPATCH_TAG}" >> $GITHUB_ENV - name: Set up QEMU uses: docker/setup-qemu-action@v3 diff --git a/.github/workflows/e2e_parameterized.yml b/.github/workflows/e2e_parameterized.yml index 5efdfcfd80d..86afe5efb58 100644 --- a/.github/workflows/e2e_parameterized.yml +++ b/.github/workflows/e2e_parameterized.yml @@ -6,7 +6,7 @@ on: mempool_hostname: description: 'Mempool Hostname' required: true - default: 'mempool.space' + default: 'explorer.bitcoinuniverse.io' type: string liquid_hostname: description: 'Liquid Hostname' diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index c8d811f86df..c1250f44668 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -26,8 +26,8 @@ "USER_AGENT": "mempool", "STDOUT_LOG_MIN_PRIORITY": "debug", "AUTOMATIC_POOLS_UPDATE": false, - "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json", - "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master", + "POOLS_JSON_URL": "", + "POOLS_JSON_TREE_URL": "", "POOLS_JSON_FILE": "tasks/pools/pools-v2.json", "POOLS_UPDATE_DELAY": 604800, "AUDIT": false, @@ -131,7 +131,7 @@ }, "EXTERNAL_DATA_SERVER": { "MEMPOOL_API": "", - "MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1", + "MEMPOOL_ONION": "", "LIQUID_API": "https://liquid.network/api/v1", "LIQUID_ONION": "http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1" }, diff --git a/backend/mempool-config.test.json b/backend/mempool-config.test.json index 4b69da9b0f6..cb72666b73c 100644 --- a/backend/mempool-config.test.json +++ b/backend/mempool-config.test.json @@ -130,7 +130,7 @@ }, "EXTERNAL_DATA_SERVER": { "MEMPOOL_API": "", - "MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1", + "MEMPOOL_ONION": "", "LIQUID_API": "https://liquid.network/api/v1", "LIQUID_ONION": "http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1" }, diff --git a/backend/src/__tests__/admin-adapter-deployment-control.test.ts b/backend/src/__tests__/admin-adapter-deployment-control.test.ts new file mode 100644 index 00000000000..de62afed011 --- /dev/null +++ b/backend/src/__tests__/admin-adapter-deployment-control.test.ts @@ -0,0 +1,286 @@ +import { createHmac } from 'crypto'; +import * as http from 'http'; +import { AddressInfo } from 'net'; + +jest.mock('../config', () => ({ DATABASE: { ENABLED: true }, MEMPOOL: { NETWORK: 'signet' }, CORE_RPC: { HOST: '127.0.0.1', PORT: 38332 } })); +jest.mock('../database', () => ({ query: jest.fn() })); +jest.mock('../api/backend-info', () => ({ getBackendInfo: () => ({}) })); +jest.mock('../api/bitcoin/bitcoin-client', () => ({})); +jest.mock('../api/bitcoin/address-index', () => ({ $probeAddressIndex: jest.fn() })); +jest.mock('../api/blocks', () => ({ getCurrentBlockHeight: () => 1 })); +jest.mock('../api/capabilities', () => ({ $report: jest.fn() })); +jest.mock('../indexer', () => ({ reindex: jest.fn() })); +jest.mock('../tasks/pools-updater', () => ({})); +jest.mock('../tasks/price-updater', () => ({})); +jest.mock('../api/redis-cache', () => ({})); +jest.mock('../api/admin-adapter/admin-adapter.identity', () => ({ explorerEnvironment: () => 'test', releaseShaOrNull: () => null })); +const transitions: Array<{ runId: string; state: string; patch: unknown }> = []; +jest.mock('../api/admin-adapter/admin-adapter.runs', () => ({ __esModule: true, default: { transition: jest.fn(async (runId: string, state: string, patch: unknown) => { transitions.push({ runId, state, patch }); return {}; }) } })); + +import { + DEPLOYMENT_CONTROL_REASON, + deploymentControlAvailability, + explorerOperationDefinitions, + findExplorerOperationDefinition, +} from '../api/admin-adapter/admin-adapter.catalog'; +import { + DEPLOYMENT_SIGNATURE_HEADER, + DeploymentControlClient, + DeploymentTransport, + deploymentControlConfig, + parseCapabilities, + rollbackTargetFromJournal, + signDeploymentRequest, + verifyDeploymentSignature, +} from '../api/admin-adapter/deployment-control.client'; +import { findExplorerOperation, useDeploymentControlClient } from '../api/admin-adapter/admin-adapter.operations'; + +const KEY = 'deployment-control-shared-key-with-enough-length'; +const OLD = 'a'.repeat(40); +const NEW = 'b'.repeat(40); +const TARGET = 'explorer/test-host'; + +/** A loopback adapter that speaks the contract, verifies signatures and can be told how to behave. */ +class FakeAdapterServer { + public readonly server: http.Server; + public requests: Array<{ method: string; path: string; body: unknown }> = []; + public journal = [{ release: OLD, verifiedAt: '2026-09-10T00:00:00.000Z', operation: 'cutover' }, { release: NEW, verifiedAt: '2026-09-15T00:00:00.000Z', operation: 'cutover' }]; + public current: string | null = NEW; + public supports = { restart: true, rollback: true }; + public jobs = new Map>(); + public idempotency = new Map(); + public pollsUntilDone = 1; + public hang = false; + private ids = 0; + + constructor() { + this.server = http.createServer((request, response) => { + const chunks: Buffer[] = []; + request.on('data', chunk => chunks.push(chunk)); + request.on('end', () => { + const body = Buffer.concat(chunks).toString('utf8'); + const path = request.url ?? '/'; + if (!verifyDeploymentSignature(Buffer.from(KEY), request.method ?? 'GET', path, body, request.headers[DEPLOYMENT_SIGNATURE_HEADER] as string | undefined)) { + response.writeHead(401, { 'content-type': 'application/json' }).end(JSON.stringify({ error: 'bad signature' })); + return; + } + if (this.hang) { return; } + const parsed = body ? JSON.parse(body) : null; + this.requests.push({ method: request.method ?? 'GET', path, body: parsed }); + const answer = this.route(request.method ?? 'GET', path, parsed); + response.writeHead(answer.status, { 'content-type': 'application/json' }).end(JSON.stringify(answer.body)); + }); + }); + } + + private route(method: string, path: string, body: any): { status: number; body: unknown } { + if (method === 'GET' && path === '/capabilities') { + return { status: 200, body: { application: 'explorer', target: TARGET, currentRelease: this.current, journal: this.journal, supports: this.supports, reasons: { restart: null, rollback: null }, adapterVersion: 'fake/1', observedAt: new Date().toISOString() } }; + } + if (method === 'POST' && (path === '/restart' || path === '/rollback')) { + if (body.target !== TARGET) { return { status: 400, body: { error: 'wrong target' } }; } + const replay = this.idempotency.get(body.idempotencyKey); + if (replay) { return { status: 202, body: { jobId: replay, state: this.jobs.get(replay)!.state, replayed: true } }; } + if (path === '/rollback' && body.release !== OLD) { return { status: 409, body: { error: 'not the journal target' } }; } + const jobId = `job-${++this.ids}`; + const after = path === '/rollback' ? OLD : this.current; + this.jobs.set(jobId, { jobId, operation: path.slice(1), state: 'running', target: TARGET, releaseBefore: this.current, releaseAfter: null, requestedRelease: path === '/rollback' ? OLD : null, startedAt: new Date().toISOString(), finishedAt: null, error: null, evidence: ['gate: ok'], polls: 0, after }); + this.idempotency.set(body.idempotencyKey, jobId); + return { status: 202, body: { jobId, state: 'running', replayed: false } }; + } + const match = /^\/jobs\/(.+)$/.exec(path); + if (method === 'GET' && match) { + const job = this.jobs.get(match[1]); + if (!job) { return { status: 404, body: { error: 'no such job' } }; } + job.polls = (job.polls as number) + 1; + if ((job.polls as number) >= this.pollsUntilDone && job.state === 'running') { + job.state = 'succeeded'; + job.releaseAfter = job.after; + job.finishedAt = new Date().toISOString(); + job.evidence = ['gate: ok', 'verify_live: gateway is answering', 'cutover complete']; + } + const { polls, after, ...view } = job; + void polls; void after; + return { status: 200, body: view }; + } + return { status: 404, body: { error: 'no such route' } }; + } + + public async start(): Promise { + await new Promise(resolve => this.server.listen(0, '127.0.0.1', resolve)); + return `http://127.0.0.1:${(this.server.address() as AddressInfo).port}`; + } + + public async stop(): Promise { + this.server.closeAllConnections(); + await new Promise(resolve => this.server.close(() => resolve())); + } +} + +function client(endpoint: string, transport?: DeploymentTransport): DeploymentControlClient { + return new DeploymentControlClient(() => ({ EXPLORER_DEPLOYMENT_CONTROL_ENDPOINT: endpoint, EXPLORER_DEPLOYMENT_CONTROL_KEY: KEY }), transport); +} + +describe('deployment control configuration and readiness', () => { + it('is unconfigured without the endpoint and key, and a flag alone enables nothing', () => { + expect(deploymentControlConfig({})).toBeNull(); + expect(deploymentControlConfig({ EXPLORER_DEPLOYMENT_CONTROL: 'enabled' })).toBeNull(); + expect(() => deploymentControlConfig({ EXPLORER_DEPLOYMENT_CONTROL_ENDPOINT: 'http://127.0.0.1:8790' })).toThrow(/must both be set/); + expect(() => deploymentControlConfig({ EXPLORER_DEPLOYMENT_CONTROL_ENDPOINT: 'http://127.0.0.1:8790', EXPLORER_DEPLOYMENT_CONTROL_KEY: 'short' })).toThrow(/at least 32/); + expect(() => deploymentControlConfig({ EXPLORER_DEPLOYMENT_CONTROL_ENDPOINT: 'http://10.0.0.5:8790', EXPLORER_DEPLOYMENT_CONTROL_KEY: KEY })).toThrow(/loopback/); + expect(() => deploymentControlConfig({ EXPLORER_DEPLOYMENT_CONTROL_ENDPOINT: 'https://127.0.0.1:8790', EXPLORER_DEPLOYMENT_CONTROL_KEY: KEY })).toThrow(/loopback/); + expect(deploymentControlConfig({ EXPLORER_DEPLOYMENT_CONTROL_ENDPOINT: 'http://127.0.0.1:8790', EXPLORER_DEPLOYMENT_CONTROL_KEY: KEY })?.endpoint).toEqual({ kind: 'http', host: '127.0.0.1', port: 8790 }); + expect(deploymentControlConfig({ EXPLORER_DEPLOYMENT_CONTROL_ENDPOINT: '/run/universe/deploy.sock', EXPLORER_DEPLOYMENT_CONTROL_KEY: KEY })?.endpoint).toEqual({ kind: 'socket', path: '/run/universe/deploy.sock' }); + + for (const id of ['explorer.service.restart', 'explorer.release.rollback']) { + const flagged = findExplorerOperationDefinition(id, { EXPLORER_DEPLOYMENT_CONTROL: 'enabled' }); + expect(flagged.availability).toBe('not_configured'); + expect(flagged.availabilityReason).toBe(DEPLOYMENT_CONTROL_REASON); + const configured = findExplorerOperationDefinition(id, { EXPLORER_DEPLOYMENT_CONTROL_ENDPOINT: 'http://127.0.0.1:8790', EXPLORER_DEPLOYMENT_CONTROL_KEY: KEY }); + expect(configured.availability).toBe('unavailable'); + expect(configured.availabilityReason).toMatch(/has not answered its capability route/); + } + }); + + it('enables an operation only from a capability document that supports it', () => { + const ready = { state: 'ready' as const, reason: null, probedAt: new Date().toISOString(), capabilities: parseCapabilities({ application: 'explorer', target: TARGET, currentRelease: NEW, journal: [{ release: NEW, verifiedAt: '2026-09-15T00:00:00.000Z', operation: 'cutover' }], supports: { restart: true, rollback: true } }) }; + expect(deploymentControlAvailability('explorer.service.restart', ready)).toEqual({ availability: 'enabled', availabilityReason: null }); + expect(deploymentControlAvailability('explorer.release.rollback', ready).availability).toBe('unavailable'); + expect(deploymentControlAvailability('explorer.release.rollback', ready).availabilityReason).toMatch(/no previous verified release/); + const unsupported = { ...ready, capabilities: { ...ready.capabilities, supports: { restart: false, rollback: false }, reasons: { restart: 'no current release', rollback: null } } }; + expect(deploymentControlAvailability('explorer.service.restart', unsupported)).toEqual({ availability: 'unavailable', availabilityReason: 'no current release' }); + const unreachable = { state: 'unreachable' as const, reason: 'refused', capabilities: null, probedAt: new Date().toISOString() }; + expect(deploymentControlAvailability('explorer.service.restart', unreachable)).toEqual({ availability: 'unavailable', availabilityReason: 'refused' }); + expect(explorerOperationDefinitions({}, ready).find(op => op.id === 'explorer.service.restart')?.availability).toBe('enabled'); + }); + + it('derives the rollback target from the journal and refuses malformed capability documents', () => { + const caps = parseCapabilities({ application: 'explorer', target: TARGET, currentRelease: NEW, journal: [{ release: OLD, verifiedAt: '2026-09-10T00:00:00.000Z', operation: 'cutover' }, { release: NEW, verifiedAt: '2026-09-15T00:00:00.000Z', operation: 'cutover' }], supports: { restart: true, rollback: true } }); + expect(rollbackTargetFromJournal(caps)?.release).toBe(OLD); + expect(rollbackTargetFromJournal({ ...caps, journal: [caps.journal[1]] })).toBeNull(); + expect(() => parseCapabilities({ application: 'other', target: TARGET, journal: [], supports: {} })).toThrow(/does not understand/); + expect(() => parseCapabilities({ application: 'explorer', target: TARGET, currentRelease: 'not a sha', journal: [], supports: {} })).toThrow(/not a commit sha/); + expect(() => parseCapabilities({ application: 'explorer', target: TARGET, currentRelease: null, journal: [{ release: 'x' }], supports: {} })).toThrow(/journal is malformed/); + }); + + it('signs method, path and body with the shared key', () => { + const signature = signDeploymentRequest(Buffer.from(KEY), 'POST', '/restart', '{"a":1}'); + expect(signature).toBe('sha256=' + createHmac('sha256', KEY).update('POST\n/restart\n{"a":1}').digest('hex')); + expect(verifyDeploymentSignature(Buffer.from(KEY), 'POST', '/restart', '{"a":1}', signature)).toBe(true); + expect(verifyDeploymentSignature(Buffer.from(KEY), 'POST', '/restart', '{"a":2}', signature)).toBe(false); + expect(verifyDeploymentSignature(Buffer.from(KEY), 'POST', '/rollback', '{"a":1}', signature)).toBe(false); + expect(verifyDeploymentSignature(Buffer.from(KEY), 'POST', '/restart', '{"a":1}', undefined)).toBe(false); + }); +}); + +describe('deployment control against a fake adapter server', () => { + let adapter: FakeAdapterServer; + let endpoint: string; + const context = (runId: string, confirmation: string, idempotencyKey: string | null = null) => ({ runId, correlationId: 'corr', actor: 'test', reason: null, idempotencyKey, input: { confirmation } }); + + beforeEach(async () => { + adapter = new FakeAdapterServer(); + endpoint = await adapter.start(); + transitions.length = 0; + useDeploymentControlClient(client(endpoint)); + }); + afterEach(async () => { + useDeploymentControlClient(null); + await adapter.stop(); + }); + + it('previews from the adapter\'s capability document', async () => { + const restart = await findExplorerOperation('explorer.service.restart').buildPreview({}); + expect(restart.available).toBe(true); + expect(restart.preconditions[0]).toMatchObject({ id: 'deployment-control', satisfied: true }); + expect(restart.preconditions[0].detail).toContain(TARGET); + expect(restart.effects[0]).toContain(NEW.slice(0, 12)); + const rollback = await findExplorerOperation('explorer.release.rollback').buildPreview({}); + expect(rollback.available).toBe(true); + expect(rollback.effects[0]).toContain(`from release ${NEW.slice(0, 12)} to ${OLD.slice(0, 12)}`); + expect(adapter.requests.filter(r => r.path === '/capabilities')).toHaveLength(2); + }); + + it('preview is unavailable with the adapter\'s own reason when it does not support the action, and with the transport fault when it is down', async () => { + adapter.supports = { restart: false, rollback: false }; + const preview = await findExplorerOperation('explorer.service.restart').buildPreview({}); + expect(preview.available).toBe(false); + expect(preview.unavailableReason).toMatch(/does not support a restart/); + await adapter.stop(); + const down = await findExplorerOperation('explorer.service.restart').buildPreview({}); + expect(down.available).toBe(false); + expect(down.unavailableReason).toMatch(/did not answer its capability route/); + adapter = new FakeAdapterServer(); + endpoint = await adapter.start(); + }); + + it('executes a restart: signed request, job id recorded on the run, readback with release before and after', async () => { + const outcome = await findExplorerOperation('explorer.service.restart').execute(context('run-1', 'RESTART EXPLORER BACKEND', 'idem-1')); + expect(outcome.verification.verified).toBe(true); + expect(outcome.result).toMatchObject({ jobId: 'job-1', releaseBefore: NEW, releaseAfter: NEW, expectedRelease: NEW, jobState: 'succeeded', timedOut: false, adapterTarget: TARGET, replayedByAdapter: false }); + expect(outcome.verification.evidence).toEqual(expect.arrayContaining([expect.stringContaining('Adapter: cutover complete')])); + const submit = adapter.requests.find(r => r.path === '/restart'); + expect(submit?.body).toEqual({ operationId: 'run-1', idempotencyKey: 'idem-1', target: TARGET }); + expect(transitions).toEqual([{ runId: 'run-1', state: 'RUNNING', patch: { progressPercent: 30, result: { jobId: 'job-1', adapterTarget: TARGET, releaseBefore: NEW, requestedRelease: null, replayedByAdapter: false } } }]); + }); + + it('refuses without the typed confirmation and never contacts the adapter', async () => { + await expect(findExplorerOperation('explorer.service.restart').execute(context('run-x', 'restart please'))).rejects.toThrow(/typed confirmation/); + await expect(findExplorerOperation('explorer.release.rollback').execute(context('run-y', ''))).rejects.toThrow(/typed confirmation/); + expect(adapter.requests).toEqual([]); + }); + + it('replays a duplicate idempotent request as the same adapter job', async () => { + await findExplorerOperation('explorer.service.restart').execute(context('run-1', 'RESTART EXPLORER BACKEND', 'idem-dup')); + const again = await findExplorerOperation('explorer.service.restart').execute(context('run-2', 'RESTART EXPLORER BACKEND', 'idem-dup')); + expect(again.result.jobId).toBe('job-1'); + expect(again.result.replayedByAdapter).toBe(true); + expect(adapter.jobs.size).toBe(1); + }); + + it('rolls back to the journal target and verifies the serving release against it', async () => { + const outcome = await findExplorerOperation('explorer.release.rollback').execute(context('run-3', 'ROLL BACK EXPLORER RELEASE', 'idem-3')); + expect(outcome.verification.verified).toBe(true); + expect(outcome.result).toMatchObject({ requestedRelease: OLD, expectedRelease: OLD, releaseBefore: NEW, releaseAfter: OLD }); + expect(adapter.requests.find(r => r.path === '/rollback')?.body).toMatchObject({ release: OLD, target: TARGET }); + }); + + it('refuses a rollback when the journal has no previous verified release', async () => { + adapter.journal = [adapter.journal[1]]; + const preview = await findExplorerOperation('explorer.release.rollback').buildPreview({}); + expect(preview.available).toBe(false); + expect(preview.unavailableReason).toMatch(/no previous verified release/); + await expect(findExplorerOperation('explorer.release.rollback').execute(context('run-4', 'ROLL BACK EXPLORER RELEASE'))).rejects.toThrow(/no previous verified release/); + expect(adapter.requests.find(r => r.path === '/rollback')).toBeUndefined(); + }); + + it('reports a job that outlives the wait as unverified with its id, never as done', async () => { + adapter.pollsUntilDone = 1_000; + let now = Date.now(); + const slow = new DeploymentControlClient(() => ({ EXPLORER_DEPLOYMENT_CONTROL_ENDPOINT: endpoint, EXPLORER_DEPLOYMENT_CONTROL_KEY: KEY }), undefined, () => now); + const originalWait = slow.waitForJob.bind(slow); + slow.waitForJob = (jobId, deadlineMs) => originalWait(jobId, deadlineMs, async () => { now += 400_000; }); + useDeploymentControlClient(slow); + const outcome = await findExplorerOperation('explorer.service.restart').execute(context('run-5', 'RESTART EXPLORER BACKEND')); + expect(outcome.verification.verified).toBe(false); + expect(outcome.result).toMatchObject({ jobId: 'job-1', jobState: 'running', timedOut: true, releaseAfter: null }); + expect(outcome.summary).toMatch(/still running/); + }); + + it('fails the run when the adapter times out on the request itself', async () => { + adapter.hang = true; + const hanging = client(endpoint, (config, method, path, body) => new Promise((_, reject) => setTimeout(() => reject(Object.assign(new Error(`The deployment adapter did not answer ${method} ${path} within 5000 ms.`), { code: 'timeout' })), 10))); + useDeploymentControlClient(hanging); + const preview = await findExplorerOperation('explorer.service.restart').buildPreview({}); + expect(preview.available).toBe(false); + expect(preview.unavailableReason).toMatch(/did not answer/); + await expect(findExplorerOperation('explorer.service.restart').execute(context('run-6', 'RESTART EXPLORER BACKEND'))).rejects.toThrow(/did not answer/); + }); + + it('fails the run when the adapter rejects the signature', async () => { + const wrongKey = new DeploymentControlClient(() => ({ EXPLORER_DEPLOYMENT_CONTROL_ENDPOINT: endpoint, EXPLORER_DEPLOYMENT_CONTROL_KEY: 'x'.repeat(48) })); + useDeploymentControlClient(wrongKey); + const preview = await findExplorerOperation('explorer.service.restart').buildPreview({}); + expect(preview.unavailableReason).toMatch(/HTTP 401/); + }); +}); diff --git a/backend/src/__tests__/admin-adapter-execute-route.test.ts b/backend/src/__tests__/admin-adapter-execute-route.test.ts new file mode 100644 index 00000000000..c90bcaa1d75 --- /dev/null +++ b/backend/src/__tests__/admin-adapter-execute-route.test.ts @@ -0,0 +1,131 @@ +/** + * The execute route's handling of the run lifecycle, with the run store + * replaced by a recorder: which transitions it asks for, with which owner + * token, and what it answers when the lock is held by another run. + */ +const store = { + reconcileAbandonedRuns: jest.fn(), + create: jest.fn(), + acquireLock: jest.fn(), + transition: jest.fn(), + withHeartbeat: jest.fn(), + get: jest.fn(), + list: jest.fn(), + auditEntries: jest.fn(), + requestCancel: jest.fn(), +}; +const operation = { + id: 'explorer.capabilities.refresh', + version: '1', + name: 'Refresh capability probes', + risk: 'SAFE', + availability: 'enabled', + availabilityReason: null, + cancellable: false, + rollbackSupported: false, + lock: 'explorer:explorer.capabilities.refresh', + buildPreview: jest.fn(), + execute: jest.fn(), +}; + +jest.mock('../config', () => ({ __esModule: true, default: { DATABASE: { ENABLED: true }, REDIS: { ENABLED: false } } })); +jest.mock('../api/backend-info', () => ({ __esModule: true, default: { getBackendInfo: () => ({}) } })); +jest.mock('../api/blocks', () => ({ __esModule: true, default: { getCurrentBlockHeight: () => 0 } })); +jest.mock('../api/capabilities', () => ({ __esModule: true, default: {} })); +jest.mock('../api/mempool', () => ({ __esModule: true, default: { getMempool: () => ({}) } })); +jest.mock('../api/admin-adapter/admin-adapter.snapshot', () => ({ buildExplorerSnapshot: jest.fn(), capabilityLabel: () => '', capabilityState: () => 'unknown' })); +jest.mock('../api/admin-adapter/admin-adapter.operations', () => ({ + findExplorerOperation: () => operation, + listExplorerOperations: () => [operation], +})); +jest.mock('../api/admin-adapter/admin-adapter.replay', () => ({ createAdminReplayStore: () => ({ kind: 'memory', claim: async () => 'claimed' }) })); +jest.mock('../api/admin-adapter/admin-adapter.runs', () => { + class AdminRunNotFound extends Error {} + class AdminRunConflict extends Error {} + return { __esModule: true, default: store, AdminRunNotFound, AdminRunConflict }; +}); + +import adminAdapterRoutes from '../api/admin-adapter/admin-adapter.routes'; +import { AdminRunConflict } from '../api/admin-adapter/admin-adapter.runs'; + +const RUN_ID = '11111111-1111-4111-8111-111111111111'; +const TOKEN = '22222222-2222-4222-8222-222222222222'; + +function mountExecute(): (request: any, response: any) => Promise { + const routes: Record = {}; + const app = { + use: jest.fn(), + get: (path: string, handler: any) => { routes[`GET ${path}`] = handler; }, + post: (path: string, handler: any) => { routes[`POST ${path}`] = handler; }, + }; + adminAdapterRoutes.initRoutes(app as any); + return routes['POST /internal/admin/v1/operations/:operationId/execute']; +} + +function respond(): { response: any; done: Promise<{ status: number; body: any }> } { + const response: any = { locals: {}, status: jest.fn(), json: jest.fn() }; + let status = 200; + response.status.mockImplementation((code: number) => { status = code; return response; }); + const done = new Promise<{ status: number; body: any }>((resolve) => { + response.json.mockImplementation((body: any) => { resolve({ status, body }); return response; }); + }); + return { response, done }; +} + +function run(state: string, extra: Record = {}) { + return { runId: RUN_ID, state, reason: null, idempotencyKey: null, ...extra }; +} + +describe('admin execute route lifecycle', () => { + let execute: (request: any, response: any) => Promise; + + beforeEach(() => { + jest.clearAllMocks(); + execute = mountExecute(); + operation.buildPreview.mockResolvedValue({ available: true, target: 'explorer/test/capabilities' }); + store.reconcileAbandonedRuns.mockResolvedValue({ reconciled: 0, remaining: 0, verified: true }); + store.create.mockResolvedValue({ run: run('QUEUED'), replayed: false, ownerToken: TOKEN }); + store.transition.mockImplementation(async (_runId: string, state: string) => run(state)); + store.withHeartbeat.mockImplementation(async (_runId: string, _token: string, work: () => Promise) => work()); + }); + + it('records a lock conflict as a FAILED run and answers 409 naming it', /** @asyncUnsafe Jest owns the test promise. */ async () => { + store.acquireLock.mockRejectedValue(new AdminRunConflict('Another operation (other) is already working on this target.')); + const { response, done } = respond(); + await execute({ params: { operationId: operation.id }, headers: {}, body: { input: {} } }, response); + const answer = await done; + expect(answer.status).toBe(409); + expect(answer.body.code).toBe('RUN_CONFLICT'); + expect(answer.body.message).toContain(RUN_ID); + expect(store.transition).toHaveBeenCalledTimes(1); + expect(store.transition).toHaveBeenCalledWith(RUN_ID, 'FAILED', expect.objectContaining({ + error: expect.objectContaining({ class: 'lock_conflict', retryable: true }), + }), TOKEN); + expect(operation.execute).not.toHaveBeenCalled(); + }); + + it('runs the operation under a heartbeat and finishes it with the owner token', /** @asyncUnsafe Jest owns the test promise. */ async () => { + store.acquireLock.mockResolvedValue(undefined); + operation.execute.mockResolvedValue({ summary: 'ok', result: {}, verification: { verified: true, evidence: ['x'] } }); + const { response, done } = respond(); + await execute({ params: { operationId: operation.id }, headers: {}, body: { input: {} } }, response); + const answer = await done; + expect(answer.status).toBe(200); + expect(answer.body.run.state).toBe('SUCCEEDED'); + expect(store.withHeartbeat).toHaveBeenCalledWith(RUN_ID, TOKEN, expect.any(Function)); + const states = store.transition.mock.calls.map((call) => [call[1], call[3]]); + expect(states).toEqual([['PRECHECK', TOKEN], ['RUNNING', TOKEN], ['VERIFYING', TOKEN], ['SUCCEEDED', TOKEN]]); + }); + + it('marks a failed operation FAILED with its error under the owner token', /** @asyncUnsafe Jest owns the test promise. */ async () => { + store.acquireLock.mockResolvedValue(undefined); + operation.execute.mockRejectedValue(new Error('The blocksPrices task failed: isolated task failure')); + const { response, done } = respond(); + await execute({ params: { operationId: operation.id }, headers: {}, body: { input: {} } }, response); + const answer = await done; + expect(answer.body.run.state).toBe('FAILED'); + expect(store.transition).toHaveBeenLastCalledWith(RUN_ID, 'FAILED', expect.objectContaining({ + error: expect.objectContaining({ class: 'operation_failed', message: expect.stringContaining('isolated task failure') }), + }), TOKEN); + }); +}); diff --git a/backend/src/__tests__/admin-adapter-indexer-operations.test.ts b/backend/src/__tests__/admin-adapter-indexer-operations.test.ts new file mode 100644 index 00000000000..39995f689b4 --- /dev/null +++ b/backend/src/__tests__/admin-adapter-indexer-operations.test.ts @@ -0,0 +1,120 @@ +/** + * The indexer and reconcile operation handlers consume typed outcomes. Only + * a completed task whose persisted rows read back complete may be verified. + */ +const indexerState = { runSingleTask: jest.fn(), reindex: jest.fn() }; +const dbState = { query: jest.fn() }; +const runStoreState = { reconcileAbandonedRuns: jest.fn() }; + +jest.mock('../config', () => ({ + __esModule: true, + default: { DATABASE: { ENABLED: true }, REDIS: { ENABLED: false }, FIAT_PRICE: { ENABLED: true }, MEMPOOL: { INDEXING_BLOCKS_AMOUNT: -1, NETWORK: 'signet' } }, +})); +jest.mock('../database', () => ({ __esModule: true, default: { query: (...args: unknown[]) => dbState.query(...args) } })); +jest.mock('../api/backend-info', () => ({ __esModule: true, default: { getBackendInfo: () => ({}) } })); +jest.mock('../api/bitcoin/bitcoin-client', () => ({ __esModule: true, default: {} })); +jest.mock('../api/bitcoin/address-index', () => ({ $probeAddressIndex: jest.fn() })); +jest.mock('../api/blocks', () => ({ __esModule: true, default: {} })); +jest.mock('../api/capabilities', () => ({ __esModule: true, default: {} })); +jest.mock('../indexer', () => ({ __esModule: true, default: indexerState })); +jest.mock('../api/mempool', () => ({ __esModule: true, default: {} })); +jest.mock('../tasks/pools-updater', () => ({ __esModule: true, default: {} })); +jest.mock('../tasks/price-updater', () => ({ __esModule: true, default: {} })); +jest.mock('../api/redis-cache', () => ({ __esModule: true, default: {} })); +jest.mock('../api/admin-adapter/admin-adapter.runs', () => ({ __esModule: true, default: runStoreState })); + +import { findExplorerOperation } from '../api/admin-adapter/admin-adapter.operations'; + +const context = (input: Record) => ({ + runId: 'run', correlationId: 'c', actor: 'a', reason: null, idempotencyKey: null, input, +}); + +describe('explorer.indexer.task.run', () => { + const operation = findExplorerOperation('explorer.indexer.task.run'); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it.each(['disabled', 'already-running', 'deferred', 'network-inapplicable'] as const)( + 'never verifies a task that answered %s', + /** @asyncUnsafe Jest owns the test promise. */ async (status) => { + indexerState.runSingleTask.mockResolvedValue({ task: 'coinStatsIndex', status, reason: `fixture ${status}` }); + const outcome = await operation.execute(context({ task: 'coinStatsIndex' })); + expect(outcome.verification.verified).toBe(false); + expect(outcome.result.status).toBe(status); + expect(outcome.result.reason).toBe(`fixture ${status}`); + expect(outcome.verification.evidence.join(' ')).toContain(status); + expect(dbState.query).not.toHaveBeenCalled(); + }, + ); + + it('turns a failed task into a run failure carrying the error', /** @asyncUnsafe Jest owns the test promise. */ async () => { + indexerState.runSingleTask.mockResolvedValue({ task: 'coinStatsIndex', status: 'failed', error: 'isolated coinstats failure' }); + await expect(operation.execute(context({ task: 'coinStatsIndex' }))).rejects.toThrow('isolated coinstats failure'); + }); + + it('verifies a completed task only when the persisted rows read back complete', /** @asyncUnsafe Jest owns the test promise. */ async () => { + indexerState.runSingleTask.mockResolvedValue({ + task: 'coinStatsIndex', status: 'completed', checkpoint: { startedAt: '2026-09-17T12:00:00.000Z', finishedAt: '2026-09-17T12:00:05.000Z' }, + }); + dbState.query.mockResolvedValueOnce([[{ remaining: 0 }]]).mockResolvedValueOnce([[{ tip: 250_000 }]]); + const verified = await operation.execute(context({ task: 'coinStatsIndex' })); + expect(verified.verification.verified).toBe(true); + expect(verified.result).toMatchObject({ status: 'completed', remaining: 0, indexedTip: 250_000, startedAt: '2026-09-17T12:00:00.000Z', finishedAt: '2026-09-17T12:00:05.000Z' }); + + dbState.query.mockResolvedValueOnce([[{ remaining: 12 }]]).mockResolvedValueOnce([[{ tip: 249_000 }]]); + const incomplete = await operation.execute(context({ task: 'coinStatsIndex' })); + expect(incomplete.verification.verified).toBe(false); + expect(incomplete.result.remaining).toBe(12); + + dbState.query.mockRejectedValueOnce(new Error('isolated unavailable database fixture')); + const unreadable = await operation.execute(context({ task: 'coinStatsIndex' })); + expect(unreadable.verification.verified).toBe(false); + expect(unreadable.result.checkpointError).toBe('isolated unavailable database fixture'); + }); + + it('reads the blocks_prices checkpoint for the blocksPrices task', /** @asyncUnsafe Jest owns the test promise. */ async () => { + indexerState.runSingleTask.mockResolvedValue({ task: 'blocksPrices', status: 'completed', checkpoint: { startedAt: 'a', finishedAt: 'b' } }); + dbState.query.mockResolvedValueOnce([[{ remaining: 0 }]]).mockResolvedValueOnce([[{ tip: 100 }]]); + const outcome = await operation.execute(context({ task: 'blocksPrices' })); + expect(outcome.verification.verified).toBe(true); + expect(String(dbState.query.mock.calls[0][0])).toContain('blocks_prices'); + }); +}); + +describe('explorer.indexer.reindex', () => { + const operation = findExplorerOperation('explorer.indexer.reindex'); + + it('reports scheduled only when the indexer accepted, and never completion', /** @asyncUnsafe Jest owns the test promise. */ async () => { + indexerState.reindex.mockReturnValue(true); + const accepted = await operation.execute(context({ confirmation: 'REINDEX EXPLORER BLOCKS' })); + expect(accepted.result).toEqual({ scheduled: true, indexingCompleted: false }); + expect(accepted.summary).toContain('not complete'); + + indexerState.reindex.mockReturnValue(false); + await expect(operation.execute(context({ confirmation: 'REINDEX EXPLORER BLOCKS' }))).rejects.toThrow(/refused/); + }); +}); + +describe('explorer.runs.reconcile', () => { + const operation = findExplorerOperation('explorer.runs.reconcile'); + + it('verifies only a clean pass with nothing remaining', /** @asyncUnsafe Jest owns the test promise. */ async () => { + runStoreState.reconcileAbandonedRuns.mockResolvedValue({ reconciled: 3, remaining: 0, verified: true }); + const clean = await operation.execute(context({})); + expect(clean.verification.verified).toBe(true); + expect(clean.result).toEqual({ reconciled: 3, remaining: 0, storageError: null }); + + runStoreState.reconcileAbandonedRuns.mockResolvedValue({ reconciled: 200, remaining: 7, verified: false }); + const partial = await operation.execute(context({})); + expect(partial.verification.verified).toBe(false); + expect(partial.result.remaining).toBe(7); + + runStoreState.reconcileAbandonedRuns.mockResolvedValue({ reconciled: 0, remaining: null, verified: false, error: 'isolated unavailable database fixture' }); + const failed = await operation.execute(context({})); + expect(failed.verification.verified).toBe(false); + expect(failed.result.storageError).toBe('isolated unavailable database fixture'); + expect(failed.summary).toContain('storage error'); + }); +}); diff --git a/backend/src/__tests__/admin-adapter-replay.test.ts b/backend/src/__tests__/admin-adapter-replay.test.ts new file mode 100644 index 00000000000..ae1e170d1fb --- /dev/null +++ b/backend/src/__tests__/admin-adapter-replay.test.ts @@ -0,0 +1,134 @@ +/** + * The shared replay stores behind the admin guard, against a stand-in for the + * admin_nonces table (a primary key that refuses duplicates, an UPDATE that + * reports touched rows) and a stand-in for Redis SET NX PX. + */ +const configState = { DATABASE: { ENABLED: true }, REDIS: { ENABLED: false } }; + +class DuplicateKey extends Error { code = 'ER_DUP_ENTRY'; errno = 1062; } + +const table = { + rows: new Map(), + failing: null as string | null, + /** @asyncUnsafe test double */ + async query(sql: string, params: any[] = []): Promise { + if (table.failing !== null) throw new Error(table.failing); + const s = sql.replace(/\s+/g, ' ').trim(); + if (s.startsWith('DELETE FROM admin_nonces WHERE expires_at <= ?')) { + for (const [key, expiresAt] of table.rows) if (expiresAt.getTime() <= params[0].getTime()) table.rows.delete(key); + return [{ affectedRows: 0 }]; + } + if (s.startsWith('INSERT INTO admin_nonces')) { + const key = `${params[0]}\n${params[1]}`; + if (table.rows.has(key)) throw new DuplicateKey('dup'); + table.rows.set(key, params[2]); + return [{ affectedRows: 1 }]; + } + if (s.startsWith('UPDATE admin_nonces SET expires_at = ? WHERE scope = ? AND nonce = ? AND expires_at <= ?')) { + const key = `${params[1]}\n${params[2]}`; + const current = table.rows.get(key); + if (!current || current.getTime() > params[3].getTime()) return [{ affectedRows: 0 }]; + table.rows.set(key, params[0]); + return [{ affectedRows: 1 }]; + } + throw new Error('fixture does not understand: ' + s); + }, +}; + +const redis = { + keys: new Map(), + failing: null as string | null, + /** @asyncUnsafe test double */ + async $claimOnce(key: string, ttlMs: number): Promise { + if (redis.failing !== null) throw new Error(redis.failing); + if (redis.keys.has(key)) return false; + redis.keys.set(key, ttlMs); + return true; + }, +}; + +jest.mock('../config', () => ({ __esModule: true, default: configState })); +jest.mock('../database', () => ({ __esModule: true, default: { query: (sql: string, params?: any[]) => table.query(sql, params) } })); +jest.mock('../api/redis-cache', () => ({ __esModule: true, default: redis })); + +import { + ADMIN_REPLAY_STORE_ENVIRONMENT_VARIABLE, + MysqlAdminReplayStore, + RedisAdminReplayStore, + createAdminReplayStore, +} from '../api/admin-adapter/admin-adapter.replay'; +import { ADMIN_NONCE_RETENTION_MS } from '../api/admin-adapter/admin-adapter.security'; + +const NOW = Date.UTC(2026, 8, 17, 12, 0, 0); + +describe('database replay store', () => { + beforeEach(() => { + table.rows.clear(); + table.failing = null; + }); + + it('lets two workers sharing the table accept a nonce exactly once', /** @asyncUnsafe Jest owns the test promise. */ async () => { + const workerA = new MysqlAdminReplayStore(); + const workerB = new MysqlAdminReplayStore(); + expect(await workerA.claim('key@1.0.0', 'isolated-nonce-1234', NOW)).toBe('claimed'); + expect(await workerB.claim('key@1.0.0', 'isolated-nonce-1234', NOW + 1)).toBe('replayed'); + expect(await workerA.claim('key@1.0.0', 'isolated-nonce-1234', NOW + 2)).toBe('replayed'); + // A different key scope is a different claim. + expect(await workerB.claim('other@1.0.0', 'isolated-nonce-1234', NOW + 3)).toBe('claimed'); + }); + + it('never releases a live nonce under pressure and reuses it only after retention', /** @asyncUnsafe Jest owns the test promise. */ async () => { + const store = new MysqlAdminReplayStore(); + expect(await store.claim('key@1.0.0', 'nonce-a', NOW)).toBe('claimed'); + for (let i = 0; i < 25_000; i++) { + table.rows.set(`key@1.0.0\nfiller-${i}`, new Date(NOW + ADMIN_NONCE_RETENTION_MS)); + } + expect(await store.claim('key@1.0.0', 'nonce-a', NOW + ADMIN_NONCE_RETENTION_MS - 1)).toBe('replayed'); + expect(await store.claim('key@1.0.0', 'nonce-a', NOW + ADMIN_NONCE_RETENTION_MS)).toBe('claimed'); + }); + + it('answers unavailable when the table cannot be reached, never claimed', /** @asyncUnsafe Jest owns the test promise. */ async () => { + const store = new MysqlAdminReplayStore(); + table.failing = 'isolated unavailable database fixture'; + expect(await store.claim('key@1.0.0', 'nonce-a', NOW)).toBe('unavailable'); + }); +}); + +describe('redis replay store', () => { + beforeEach(() => { + redis.keys.clear(); + redis.failing = null; + }); + + it('claims with SET NX and refuses the second claim', /** @asyncUnsafe Jest owns the test promise. */ async () => { + const workerA = new RedisAdminReplayStore(); + const workerB = new RedisAdminReplayStore(); + expect(await workerA.claim('key@1.0.0', 'nonce-a', NOW)).toBe('claimed'); + expect(await workerB.claim('key@1.0.0', 'nonce-a', NOW)).toBe('replayed'); + expect(redis.keys.get('admin-nonce:key@1.0.0:nonce-a')).toBe(ADMIN_NONCE_RETENTION_MS); + }); + + it('answers unavailable when Redis is not connected', /** @asyncUnsafe Jest owns the test promise. */ async () => { + redis.failing = 'Redis is not connected'; + expect(await new RedisAdminReplayStore().claim('key@1.0.0', 'nonce-a', NOW)).toBe('unavailable'); + }); +}); + +describe('replay store selection', () => { + it('prefers the database, then Redis, and otherwise fails closed', () => { + expect(createAdminReplayStore({}, { database: true, redis: true }).kind).toBe('database'); + expect(createAdminReplayStore({}, { database: false, redis: true }).kind).toBe('redis'); + expect(createAdminReplayStore({}, { database: false, redis: false }).kind).toBe('unavailable'); + }); + + it('honours the explicit override and refuses one that cannot be met', () => { + expect(createAdminReplayStore({ [ADMIN_REPLAY_STORE_ENVIRONMENT_VARIABLE]: 'redis' }, { database: true, redis: true }).kind).toBe('redis'); + expect(createAdminReplayStore({ [ADMIN_REPLAY_STORE_ENVIRONMENT_VARIABLE]: 'redis' }, { database: true, redis: false }).kind).toBe('unavailable'); + expect(createAdminReplayStore({ [ADMIN_REPLAY_STORE_ENVIRONMENT_VARIABLE]: 'bogus' }, { database: true, redis: true }).kind).toBe('unavailable'); + }); + + it('uses memory only on the explicit single-process opt-in', () => { + expect(createAdminReplayStore({ [ADMIN_REPLAY_STORE_ENVIRONMENT_VARIABLE]: 'memory' }, { database: false, redis: false }).kind).toBe('memory'); + expect(createAdminReplayStore({}, { database: false, redis: false }).kind).not.toBe('memory'); + }); +}); diff --git a/backend/src/__tests__/admin-adapter-runs.test.ts b/backend/src/__tests__/admin-adapter-runs.test.ts new file mode 100644 index 00000000000..59da551b7e1 --- /dev/null +++ b/backend/src/__tests__/admin-adapter-runs.test.ts @@ -0,0 +1,319 @@ +/** + * The durable run lifecycle against a small in-memory stand-in for the two + * admin tables. The stand-in answers exactly the statements the store issues + * and reproduces the one database behaviour the store relies on: a duplicate + * primary key is an error, and an UPDATE reports how many rows it touched. + */ +const configState = { DATABASE: { ENABLED: true }, REDIS: { ENABLED: false } }; +const clock = { now: Date.UTC(2026, 8, 17, 12, 0, 0) }; + +interface RunRow { [key: string]: any } +interface LockRow { lock_key: string; run_id: string; acquired_at: Date; expires_at: Date } + +class DuplicateKey extends Error { code = 'ER_DUP_ENTRY'; errno = 1062; } + +class FakeDb { + runs = new Map(); + locks = new Map(); + failing: string | null = null; + refuseReconcileUpdates = false; + statements: string[] = []; + + private abandoned(row: RunRow, now: Date, cutoff: Date): boolean { + const open = !['SUCCEEDED', 'FAILED', 'CANCELLED', 'NEEDS_REVIEW', 'ROLLED_BACK', 'ROLLBACK_FAILED'].includes(row.state); + if (!open) return false; + if (row.lease_expires_at !== null) return row.lease_expires_at.getTime() <= now.getTime(); + return row.state === 'QUEUED' && row.queued_at.getTime() <= cutoff.getTime(); + } + + /** @asyncUnsafe test double */ + async query(sql: string, params: any[] = []): Promise { + this.statements.push(sql); + if (this.failing !== null) { + throw new Error(this.failing); + } + const s = sql.replace(/\s+/g, ' ').trim(); + if (s.startsWith('INSERT INTO admin_adapter_runs')) { + const columns = s.slice(s.indexOf('(') + 1, s.indexOf(')')).split(',').map((c) => c.trim()); + const row: RunRow = { + run_id: null, operation_id: null, operation_version: null, state: null, target: null, correlation_id: null, + idempotency_key: null, actor: null, reason: null, queued_at: null, started_at: null, updated_at: null, + finished_at: null, heartbeat_at: null, lease_expires_at: null, owner_token: null, progress_percent: null, + cancel_requested: 0, document: null, + }; + let index = 0; + for (const column of columns) { + if (column === 'cancel_requested') { row.cancel_requested = 0; continue; } + row[column] = params[index++]; + } + if (this.runs.has(row.run_id)) throw new DuplicateKey('dup'); + this.runs.set(row.run_id, row); + return [{ affectedRows: 1 }]; + } + if (s.startsWith('SELECT * FROM admin_adapter_runs WHERE run_id = ?')) { + const row = this.runs.get(params[0]); + return [row ? [{ ...row }] : []]; + } + if (s.startsWith('SELECT * FROM admin_adapter_runs WHERE operation_id = ? AND idempotency_key = ?')) { + const found = [...this.runs.values()].find((r) => r.operation_id === params[0] && r.idempotency_key === params[1]); + return [found ? [{ ...found }] : []]; + } + if (s.startsWith('SELECT document FROM admin_adapter_runs WHERE run_id = ?')) { + const row = this.runs.get(params[0]); + return [row ? [{ document: row.document }] : []]; + } + if (s.startsWith('SELECT cancel_requested FROM admin_adapter_runs')) { + const row = this.runs.get(params[0]); + return [row ? [{ cancel_requested: row.cancel_requested }] : []]; + } + if (s.startsWith('UPDATE admin_adapter_runs SET cancel_requested = 1')) { + const row = this.runs.get(params[1]); + if (row) { row.cancel_requested = 1; row.updated_at = params[0]; } + return [{ affectedRows: row ? 1 : 0 }]; + } + if (s.startsWith('UPDATE admin_adapter_runs SET state = ?, updated_at = ?, started_at = COALESCE')) { + const [state, updatedAt, startedAt, finishedAt, heartbeatAt, lease, progress, document, runId, expectedState, ownerToken] = params; + const row = this.runs.get(runId); + const guarded = s.includes('AND state = ? AND owner_token = ?'); + if (!row || (guarded && (row.state !== expectedState || row.owner_token !== ownerToken))) { + return [{ affectedRows: 0 }]; + } + Object.assign(row, { + state, updated_at: updatedAt, started_at: row.started_at ?? startedAt, finished_at: finishedAt, + heartbeat_at: heartbeatAt, lease_expires_at: lease, progress_percent: progress, document, + }); + return [{ affectedRows: 1 }]; + } + if (s.startsWith('UPDATE admin_adapter_runs SET heartbeat_at = ?')) { + const [heartbeatAt, updatedAt, lease, runId, ownerToken] = params; + const row = this.runs.get(runId); + if (!row || row.state !== 'RUNNING' || row.owner_token !== ownerToken) return [{ affectedRows: 0 }]; + Object.assign(row, { heartbeat_at: heartbeatAt, updated_at: updatedAt, lease_expires_at: lease }); + return [{ affectedRows: 1 }]; + } + if (s.startsWith('SELECT run_id, state FROM admin_adapter_runs WHERE')) { + const [now, cutoff] = params; + const limit = Number(/LIMIT (\d+)/.exec(s)?.[1] ?? 200); + const rows = [...this.runs.values()].filter((r) => this.abandoned(r, now, cutoff)).slice(0, limit); + return [rows.map((r) => ({ run_id: r.run_id, state: r.state }))]; + } + if (s.startsWith('SELECT COUNT(*) AS remaining FROM admin_adapter_runs WHERE')) { + const [now, cutoff] = params; + return [[{ remaining: [...this.runs.values()].filter((r) => this.abandoned(r, now, cutoff)).length }]]; + } + if (s.startsWith('UPDATE admin_adapter_runs SET state = ?, updated_at = ?, finished_at = ?, lease_expires_at = NULL, document = JSON_SET')) { + const [state, updatedAt, finishedAt, errorClass, message, retryable, runId, now, cutoff] = params; + const row = this.runs.get(runId); + if (!row || this.refuseReconcileUpdates || !this.abandoned(row, now, cutoff)) return [{ affectedRows: 0 }]; + const document = JSON.parse(row.document); + document.error = { class: errorClass, message, retryable }; + Object.assign(row, { state, updated_at: updatedAt, finished_at: finishedAt, lease_expires_at: null, document: JSON.stringify(document) }); + return [{ affectedRows: 1 }]; + } + if (s.startsWith('DELETE FROM admin_adapter_locks WHERE expires_at <= ?')) { + for (const [key, lock] of this.locks) if (lock.expires_at.getTime() <= params[0].getTime()) this.locks.delete(key); + return [{ affectedRows: 0 }]; + } + if (s.startsWith('DELETE FROM admin_adapter_locks WHERE run_id = ?')) { + for (const [key, lock] of this.locks) if (lock.run_id === params[0]) this.locks.delete(key); + return [{ affectedRows: 0 }]; + } + if (s.startsWith('INSERT INTO admin_adapter_locks')) { + const [lockKey, runId, acquiredAt, expiresAt] = params; + if (this.locks.has(lockKey)) throw new DuplicateKey('dup'); + this.locks.set(lockKey, { lock_key: lockKey, run_id: runId, acquired_at: acquiredAt, expires_at: expiresAt }); + return [{ affectedRows: 1 }]; + } + if (s.startsWith('SELECT run_id FROM admin_adapter_locks WHERE lock_key = ?')) { + const lock = this.locks.get(params[0]); + return [lock ? [{ run_id: lock.run_id }] : []]; + } + if (s.startsWith('UPDATE admin_adapter_locks SET expires_at = ? WHERE run_id = ?')) { + let touched = 0; + for (const lock of this.locks.values()) if (lock.run_id === params[1]) { lock.expires_at = params[0]; touched++; } + return [{ affectedRows: touched }]; + } + if (s.startsWith('SELECT * FROM admin_adapter_runs WHERE LOCATE') || s.startsWith('SELECT * FROM admin_adapter_runs ORDER BY queued_at')) { + return [[...this.runs.values()].map((r) => ({ ...r }))]; + } + throw new Error('FakeDb does not understand: ' + s); + } +} + +const db = new FakeDb(); + +jest.mock('../config', () => ({ __esModule: true, default: configState })); +jest.mock('../database', () => ({ __esModule: true, default: { query: (sql: string, params?: any[]) => db.query(sql, params) } })); +jest.mock('../api/backend-info', () => ({ __esModule: true, default: { getBackendInfo: () => ({}) } })); + +import runStore, { AdminRunConflict, HEARTBEAT_MS, LEASE_MS, toRun } from '../api/admin-adapter/admin-adapter.runs'; + +function creation(overrides: Record = {}) { + return { + operationId: 'explorer.capabilities.refresh', + operationVersion: '1', + target: 'explorer/test/capabilities', + actor: 'test', + reason: null, + correlationId: 'corr', + idempotencyKey: null, + cancellable: false, + rollbackSupported: false, + redactedInput: {}, + ...overrides, + }; +} + +describe('admin adapter durable runs', () => { + beforeEach(() => { + db.runs.clear(); + db.locks.clear(); + db.failing = null; + db.refuseReconcileUpdates = false; + db.statements = []; + jest.useFakeTimers({ now: clock.now }); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it('persists a non-cancellable operation as non-cancellable and refuses to cancel it', /** @asyncUnsafe Jest owns the test promise. */ async () => { + const { run, ownerToken } = await runStore.create(creation()); + expect(run.cancellable).toBe(false); + expect(ownerToken).toMatch(/^[0-9a-f-]{36}$/); + expect((await runStore.get(run.runId)).cancellable).toBe(false); + await runStore.transition(run.runId, 'PRECHECK', {}, ownerToken as string); + const running = await runStore.transition(run.runId, 'RUNNING', {}, ownerToken as string); + expect(running.cancellable).toBe(false); + await expect(runStore.requestCancel(run.runId)).rejects.toThrow(/does not support cancellation/); + const after = await runStore.get(run.runId); + expect(after.state).toBe('RUNNING'); + expect(db.runs.get(run.runId)?.cancel_requested).toBe(0); + expect(await runStore.cancelRequested(run.runId)).toBe(false); + // The lists read the same persisted truth. + expect((await runStore.list())[0].cancellable).toBe(false); + expect((await runStore.auditEntries())[0].cancellable).toBe(false); + }); + + it('reads an older row without the persisted flag as not cancellable', () => { + const row = { + run_id: 'legacy', operation_id: 'explorer.capabilities.refresh', operation_version: '1', state: 'RUNNING', + target: 'explorer/test/capabilities', correlation_id: 'c', actor: 'a', cancel_requested: 0, + queued_at: new Date(clock.now), updated_at: new Date(clock.now), progress_percent: 10, document: '{}', + }; + expect(toRun(row).cancellable).toBe(false); + expect(toRun({ ...row, document: JSON.stringify({ cancellable: true }) }).cancellable).toBe(true); + }); + + it('gives a new run an initial lease and recovers it as FAILED when that lease expires', /** @asyncUnsafe Jest owns the test promise. */ async () => { + const { run, ownerToken } = await runStore.create(creation()); + await runStore.acquireLock('explorer:test', run.runId); + const row = db.runs.get(run.runId) as RunRow; + expect(row.lease_expires_at.getTime()).toBe(clock.now + LEASE_MS); + expect(row.owner_token).toBe(ownerToken); + + // Nothing to do while the lease is live. + expect(await runStore.reconcileAbandonedRuns()).toEqual({ reconciled: 0, remaining: 0, verified: true }); + expect(db.locks.size).toBe(1); + + jest.setSystemTime(clock.now + LEASE_MS + 1); + expect(await runStore.reconcileAbandonedRuns()).toEqual({ reconciled: 1, remaining: 0, verified: true }); + const recovered = await runStore.get(run.runId); + expect(recovered.state).toBe('FAILED'); + expect(recovered.error).toEqual({ class: 'abandoned_before_start', message: expect.stringContaining('nothing was executed'), retryable: true }); + expect(db.locks.size).toBe(0); + }); + + it('recovers a QUEUED row written before initial leases existed', /** @asyncUnsafe Jest owns the test promise. */ async () => { + const { run } = await runStore.create(creation()); + const row = db.runs.get(run.runId) as RunRow; + row.lease_expires_at = null; + row.owner_token = null; + row.queued_at = new Date(clock.now - LEASE_MS - 1); + expect(await runStore.reconcileAbandonedRuns()).toEqual({ reconciled: 1, remaining: 0, verified: true }); + expect((await runStore.get(run.runId)).state).toBe('FAILED'); + }); + + it('records a lock conflict as a terminal, inspectable failure', /** @asyncUnsafe Jest owns the test promise. */ async () => { + const first = await runStore.create(creation()); + await runStore.acquireLock('explorer:test:target', first.run.runId); + const second = await runStore.create(creation()); + await expect(runStore.acquireLock('explorer:test:target', second.run.runId)).rejects.toThrow(AdminRunConflict); + const failed = await runStore.transition(second.run.runId, 'FAILED', { + error: { class: 'lock_conflict', message: 'held by ' + first.run.runId, retryable: true }, + }, second.ownerToken as string); + expect(failed.state).toBe('FAILED'); + expect(failed.error?.class).toBe('lock_conflict'); + expect(failed.finishedAt).not.toBeNull(); + // The holder keeps its lock. + expect(db.locks.get('explorer:test:target')?.run_id).toBe(first.run.runId); + }); + + it('refuses a transition from an executor whose run was taken over', /** @asyncUnsafe Jest owns the test promise. */ async () => { + const { run, ownerToken } = await runStore.create(creation()); + await runStore.transition(run.runId, 'PRECHECK', {}, ownerToken as string); + await runStore.transition(run.runId, 'RUNNING', {}, ownerToken as string); + (db.runs.get(run.runId) as RunRow).owner_token = 'someone-else'; + await expect(runStore.transition(run.runId, 'VERIFYING', {}, ownerToken as string)).rejects.toThrow(/changed underneath/); + expect((await runStore.get(run.runId)).state).toBe('RUNNING'); + }); + + it('keeps a long run RUNNING and locked while its heartbeat continues', /** @asyncUnsafe Jest owns the test promise. */ async () => { + const { run, ownerToken } = await runStore.create(creation()); + await runStore.acquireLock('explorer:long', run.runId); + await runStore.transition(run.runId, 'PRECHECK', {}, ownerToken as string); + await runStore.transition(run.runId, 'RUNNING', {}, ownerToken as string); + + let finish: () => void = () => undefined; + const work = new Promise((resolve) => { finish = () => resolve('done'); }); + const running = runStore.withHeartbeat(run.runId, ownerToken as string, () => work); + + const durationMs = 150_000; + for (let elapsed = 0; elapsed < durationMs; elapsed += HEARTBEAT_MS) { + jest.advanceTimersByTime(HEARTBEAT_MS); + for (let i = 0; i < 10; i++) await Promise.resolve(); + // A reconciler passing at any point sees a live lease and leaves the run alone. + expect(await runStore.reconcileAbandonedRuns()).toEqual({ reconciled: 0, remaining: 0, verified: true }); + const row = db.runs.get(run.runId) as RunRow; + expect(row.state).toBe('RUNNING'); + expect(row.lease_expires_at.getTime()).toBeGreaterThan(Date.now()); + expect(db.locks.get('explorer:long')?.run_id).toBe(run.runId); + } + finish(); + expect(await running).toBe('done'); + expect(jest.getTimerCount()).toBe(0); + expect((db.runs.get(run.runId) as RunRow).heartbeat_at.getTime()).toBeGreaterThan(clock.now + durationMs - HEARTBEAT_MS - 1); + + // Once the executor stops heartbeating, the lease runs out and recovery moves it to review. + jest.setSystemTime(Date.now() + LEASE_MS + 1); + expect(await runStore.reconcileAbandonedRuns()).toEqual({ reconciled: 1, remaining: 0, verified: true }); + const reviewed = await runStore.get(run.runId); + expect(reviewed.state).toBe('NEEDS_REVIEW'); + expect(reviewed.error?.class).toBe('abandoned_in_flight'); + expect(db.locks.size).toBe(0); + }); + + it('drains a backlog larger than one batch and reports remaining truthfully', /** @asyncUnsafe Jest owns the test promise. */ async () => { + for (let i = 0; i < 201; i++) { + await runStore.create(creation({ correlationId: `c${i}` })); + } + jest.setSystemTime(clock.now + LEASE_MS + 1); + expect(await runStore.reconcileAbandonedRuns()).toEqual({ reconciled: 201, remaining: 0, verified: true }); + + for (let i = 0; i < 3; i++) { + await runStore.create(creation({ correlationId: `late${i}` })); + } + jest.setSystemTime(Date.now() + LEASE_MS + 1); + db.refuseReconcileUpdates = true; + expect(await runStore.reconcileAbandonedRuns()).toEqual({ reconciled: 0, remaining: 3, verified: false }); + }); + + it('surfaces a storage failure instead of claiming everything was reconciled', /** @asyncUnsafe Jest owns the test promise. */ async () => { + await runStore.create(creation()); + jest.setSystemTime(clock.now + LEASE_MS + 1); + db.failing = 'isolated unavailable database fixture'; + const outcome = await runStore.reconcileAbandonedRuns(); + expect(outcome).toEqual({ reconciled: 0, remaining: null, verified: false, error: 'isolated unavailable database fixture' }); + }); +}); diff --git a/backend/src/__tests__/admin-adapter.test.ts b/backend/src/__tests__/admin-adapter.test.ts index 224e146cf19..e19344647a0 100644 --- a/backend/src/__tests__/admin-adapter.test.ts +++ b/backend/src/__tests__/admin-adapter.test.ts @@ -5,13 +5,16 @@ import { join } from 'path'; import { Readable } from 'stream'; import adminControl from '@bitcoinuniverse/ecosystem-contracts/admin-control'; import { + ADMIN_NONCE_RETENTION_MS, AdminAdapterNonceStore, adminAdapterGuard, adminAdapterJsonParser, hasSignedAdminElevation, isPrivateRemoteAddress, parseAdminAdapterKeys, + UnavailableAdminReplayStore, verifyAdminAdapterRequest, + verifyAdminAdapterRequestWithStore, } from '../api/admin-adapter/admin-adapter.security'; import { ALLOWED_INDEXER_TASKS, @@ -107,7 +110,8 @@ describe('Admin parser and signed elevation through actual middleware', () => { else process.env.EXPLORER_ADMIN_ADAPTER_KEYS = previousKeys; }); - function invoke(signedRequest: ReturnType, captureBeforeGeneralParser = true): Promise<{ request: any; response: any; reached: boolean }> { + const guardStore = new AdminAdapterNonceStore(); + function invoke(signedRequest: ReturnType, captureBeforeGeneralParser = true, store: AdminAdapterNonceStore | UnavailableAdminReplayStore = guardStore): Promise<{ request: any; response: any; reached: boolean }> { return new Promise((resolve, reject) => { const request = Object.assign(Readable.from([signedRequest.rawBody]), { method: signedRequest.method, originalUrl: signedRequest.originalUrl, url: signedRequest.originalUrl, @@ -120,7 +124,7 @@ describe('Admin parser and signed elevation through actual middleware', () => { const middleware: RequestHandler[] = [ ...(captureBeforeGeneralParser ? [adminAdapterJsonParser()] : []), express.urlencoded({ extended: true, limit: '10mb' }), express.text({ type: ['text/plain', 'application/base64'], limit: '10mb' }), - express.json({ limit: '10mb' }), adminAdapterJsonParser(), adminAdapterGuard(), + express.json({ limit: '10mb' }), adminAdapterJsonParser(), adminAdapterGuard(store), ]; const next = (index: number): void => { if (index === middleware.length) { resolve({ request, response, reached: true }); return; } @@ -170,6 +174,58 @@ describe('Admin parser and signed elevation through actual middleware', () => { const result = await invoke(signed({ rawBody: Buffer.from(JSON.stringify({ input: {}, adminAuthorization: { elevated: 'true' } })) })); expect(result.reached).toBe(true); expect(hasSignedAdminElevation(result.response)).toBe(false); }); + it('refuses a replay through the guard store shared by every request', /** @asyncUnsafe Jest owns the test promise. */ async () => { + const request = signed(); + expect((await invoke(request)).reached).toBe(true); + const replay = await invoke(request); + expect(replay.reached).toBe(false); expect(replay.response.status).toHaveBeenCalledWith(401); + }); + it('fails closed when no shared replay store is available', /** @asyncUnsafe Jest owns the test promise. */ async () => { + const result = await invoke(signed(), true, new UnavailableAdminReplayStore('isolated fixture')); + expect(result.reached).toBe(false); expect(result.response.status).toHaveBeenCalledWith(503); + expect(result.response.json).toHaveBeenCalledWith(expect.objectContaining({ code: 'REPLAY_STORE_UNAVAILABLE' })); + }); + it('fails closed by default when the guard is mounted without a store', /** @asyncUnsafe Jest owns the test promise. */ async () => { + const request = signed(); + const response: any = { locals: {}, setHeader: jest.fn(), removeHeader: jest.fn(), status: jest.fn() }; + response.status.mockReturnValue(response); + const refused = new Promise((resolve) => { response.json = jest.fn(() => { resolve(); return response; }); }); + adminAdapterGuard()({ ...request, socket: { remoteAddress: '127.0.0.1' }, rawBody: request.rawBody } as any, response, () => { throw new Error('must not reach the handler'); }); + await refused; + expect(response.status).toHaveBeenCalledWith(503); + }); +}); + +describe('AdminAdapterNonceStore', () => { + it('accepts a nonce exactly once and never evicts a live one under capacity pressure', () => { + const store = new AdminAdapterNonceStore(2); + expect(store.accept('nonce-a', 100_000)).toBe(true); + expect(store.accept('nonce-b', 100_001)).toBe(true); + // Capacity is reached: a new nonce is refused rather than forgetting an old one. + expect(store.accept('nonce-c', 100_002)).toBe(false); + expect(store.accept('nonce-a', 100_003)).toBe(false); + expect(store.accept('nonce-b', 100_003)).toBe(false); + // Only once retention has passed may the slots be reused. + expect(store.accept('nonce-c', 100_003 + ADMIN_NONCE_RETENTION_MS)).toBe(true); + }); + + it('keeps nonces apart per key scope', () => { + const store = new AdminAdapterNonceStore(); + expect(store.accept('shared', 1, 'key-a@1.0.0')).toBe(true); + expect(store.accept('shared', 1, 'key-b@1.0.0')).toBe(true); + expect(store.accept('shared', 1, 'key-a@1.0.0')).toBe(false); + }); + + it('verifies with the shared-store path exactly as with the synchronous one', /** @asyncUnsafe Jest owns the test promise. */ async () => { + const store = new AdminAdapterNonceStore(); + const request = signed(); + expect((await verifyAdminAdapterRequestWithStore(request, store)).ok).toBe(true); + expect((await verifyAdminAdapterRequestWithStore(request, store)).ok).toBe(false); + const unavailable = await verifyAdminAdapterRequestWithStore(signed(), new UnavailableAdminReplayStore('fixture')); + expect(unavailable.ok).toBe(false); + expect(unavailable.ok === false && unavailable.status).toBe(503); + expect(unavailable.ok === false && unavailable.code).toBe('REPLAY_STORE_UNAVAILABLE'); + }); }); describe('isPrivateRemoteAddress', () => { @@ -332,15 +388,18 @@ describe('Explorer operation catalog', () => { } }); - it('enables the deployment-controlled operations only on the exact configured value', () => { + it('treats the adapter configuration as configured, never as enabled', () => { expect(deploymentControlConfigured({})).toBe(false); - expect(deploymentControlConfigured({ EXPLORER_DEPLOYMENT_CONTROL: 'true' })).toBe(false); - expect(deploymentControlConfigured({ EXPLORER_DEPLOYMENT_CONTROL: 'enabled' })).toBe(true); - const enabled = findExplorerOperationDefinition('explorer.service.restart', { - EXPLORER_DEPLOYMENT_CONTROL: 'enabled', - }); - expect(enabled.availability).toBe('enabled'); - expect(enabled.availabilityReason).toBeNull(); + expect(deploymentControlConfigured({ EXPLORER_DEPLOYMENT_CONTROL: 'enabled' })).toBe(false); + const configured = { + EXPLORER_DEPLOYMENT_CONTROL_ENDPOINT: 'http://127.0.0.1:8790', + EXPLORER_DEPLOYMENT_CONTROL_KEY: 'deployment-control-shared-key-with-enough-length', + }; + expect(deploymentControlConfigured(configured)).toBe(true); + // Configured is not ready: only the adapter's capability document enables an operation. + const restart = findExplorerOperationDefinition('explorer.service.restart', configured); + expect(restart.availability).toBe('unavailable'); + expect(restart.availabilityReason).toMatch(/capability route/); }); it('covers the Explorer subsystems the Control Center has to reach', () => { diff --git a/backend/src/__tests__/admin-resource-stream-boundaries.test.ts b/backend/src/__tests__/admin-resource-stream-boundaries.test.ts index f6afdd96b92..81962a6075f 100644 --- a/backend/src/__tests__/admin-resource-stream-boundaries.test.ts +++ b/backend/src/__tests__/admin-resource-stream-boundaries.test.ts @@ -2,7 +2,7 @@ import fs from 'fs'; import path from 'path'; import vm from 'vm'; import ts fro function load(relative:string,deps:any,extra:any={}){const exports:any={};vm.runInNewContext(ts.transpileModule(fs.readFileSync(path.join(__dirname,'..',relative),'utf8'),{compilerOptions:{module:ts.ModuleKind.CommonJS,target:ts.ScriptTarget.ES2020,esModuleInterop:true}}).outputText,{exports,require:(id:string)=>deps[id]||{},setInterval,clearInterval,Error,...extra});return exports.default;} class Missing extends Error{} const identity={adminEnvelope:()=>({}),adminTimestamp:()=> '2026-09-15T00:00:00Z',explorerEnvironment:()=> 'test',explorerNetwork:()=> 'regtest',explorerRelease:()=>({backendSha:'abc',version:'1'})}; -function setup(store:any={},extra:any={}){const handlers:any={};const app:any={use:()=>app,get:(url:string,fn:any)=>{handlers[url]=fn;return app;},post:()=>app};const source=load('api/admin-adapter/admin-adapter.routes.ts',{'crypto':{randomUUID:()=> 'id'},'@bitcoinuniverse/ecosystem-contracts/admin-control':{ADMIN_RESOURCE_KINDS:['run','release'],ADMIN_CONTROL_SUPPORTED_VERSIONS:[]},'../../config':{DATABASE:{ENABLED:true}},'../../logger':{info:()=>{},err:()=>{}},'./admin-adapter.security':{adminAdapterJsonParser:()=>{},adminAdapterGuard:()=>{}},'./admin-adapter.identity':identity,'./admin-adapter.runs':{__esModule:true,default:store,AdminRunNotFound:Missing},'../backend-info':{getBackendInfo:()=>({})},'../blocks':{getCurrentBlockHeight:()=>1},'../mempool':{getMempool:()=>({})},'./admin-adapter.snapshot':{buildExplorerSnapshot:async()=>({components:[],dependencies:[],indexers:[]})}},extra);source.initRoutes(app);return handlers;} +function setup(store:any={},extra:any={}){const handlers:any={};const app:any={use:()=>app,get:(url:string,fn:any)=>{handlers[url]=fn;return app;},post:()=>app};const source=load('api/admin-adapter/admin-adapter.routes.ts',{'crypto':{randomUUID:()=> 'id'},'@bitcoinuniverse/ecosystem-contracts/admin-control':{ADMIN_RESOURCE_KINDS:['run','release'],ADMIN_CONTROL_SUPPORTED_VERSIONS:[]},'../../config':{DATABASE:{ENABLED:true}},'../../logger':{info:()=>{},err:()=>{}},'./admin-adapter.security':{adminAdapterJsonParser:()=>{},adminAdapterGuard:()=>{}},'./admin-adapter.replay':{createAdminReplayStore:()=>({kind:'memory',claim:async()=>'claimed'})},'./admin-adapter.identity':identity,'./admin-adapter.runs':{__esModule:true,default:store,AdminRunNotFound:Missing},'../backend-info':{getBackendInfo:()=>({})},'../blocks':{getCurrentBlockHeight:()=>1},'../mempool':{getMempool:()=>({})},'./admin-adapter.snapshot':{buildExplorerSnapshot:async()=>({components:[],dependencies:[],indexers:[]})}},extra);source.initRoutes(app);return handlers;} function response(){const r:any=new EventEmitter();r.statusCode=200;r.status=(n:number)=>{r.statusCode=n;return r;};r.json=(b:any)=>{r.body=b;return r;};r.setHeader=()=>{};r.flushHeaders=()=>{};r.end=jest.fn();r.write=jest.fn(()=>true);return r;} const run={runId:'old',operationId:'verify',state:'SUCCEEDED',target:'node'}; test('resource lookup reads an exact run even outside latest200',async()=>{const store={get:jest.fn(async()=>run),list:jest.fn(async()=>[])};const r=response();await setup(store)['/internal/admin/v1/resources/:kind/:id']({params:{kind:'run',id:'old'}},r);expect(r.statusCode).toBe(200);expect(r.body.resource.id).toBe('old');expect(store.get).toHaveBeenCalledWith('old');expect(store.list).not.toHaveBeenCalled();}); diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index d9ed56321a2..e88b738f163 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -124,7 +124,7 @@ describe('Mempool Backend Config', () => { expect(config.EXTERNAL_DATA_SERVER).toStrictEqual({ MEMPOOL_API: '', - MEMPOOL_ONION: 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1', + MEMPOOL_ONION: '', LIQUID_API: 'https://liquid.network/api/v1', LIQUID_ONION: 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1' }); diff --git a/backend/src/__tests__/deployment-control-resume.test.ts b/backend/src/__tests__/deployment-control-resume.test.ts new file mode 100644 index 00000000000..4e46c84332f --- /dev/null +++ b/backend/src/__tests__/deployment-control-resume.test.ts @@ -0,0 +1,73 @@ +import { resumeDeploymentRuns } from '../api/admin-adapter/deployment-control.resume'; + +const queries: unknown[][] = []; +const transitions: Array<{ runId: string; state: string; patch: any }> = []; +const jobs = new Map(); + +jest.mock('../config', () => { const actual = jest.requireActual('../config').default; return { __esModule: true, default: { ...actual, DATABASE: { ...actual.DATABASE, ENABLED: true } } }; }); +jest.mock('../logger', () => ({ __esModule: true, default: { info: jest.fn(), warn: jest.fn(), err: jest.fn(), debug: jest.fn() } })); +jest.mock('../database', () => ({ + __esModule: true, + default: { + query: jest.fn(async (sql: string, params?: unknown[]) => { + queries.push([sql, params]); + return [Array.from(jobs.values()).filter(j => j.row).map(j => j.row)]; + }), + }, +})); +jest.mock('../api/admin-adapter/admin-adapter.runs', () => ({ + __esModule: true, + AdminRunConflict: class AdminRunConflict extends Error {}, + default: { transition: jest.fn(async (runId: string, state: string, patch: any) => { transitions.push({ runId, state, patch }); return { runId, state }; }) }, +})); +jest.mock('../api/admin-adapter/deployment-control.client', () => ({ + __esModule: true, + DEPLOYMENT_CONTROL_LIMITS: { pollIntervalMs: 1 }, + deploymentControlClient: { + waitForJob: jest.fn(async (jobId: string) => jobs.get(jobId)?.answer), + }, +})); + +function seed(runId: string, operationId: string, jobId: string, result: any, answer: any): void { + jobs.set(jobId, { row: { run_id: runId, operation_id: operationId, document: JSON.stringify({ result: { jobId, ...result } }) }, answer }); +} + +describe('deployment runs interrupted by their own restart', () => { + beforeEach(() => { queries.length = 0; transitions.length = 0; jobs.clear(); }); + + it('closes a succeeded restart as SUCCEEDED with the adapter readback', async () => { + seed('run-1', 'explorer.service.restart', 'job-1', { releaseBefore: 'abc1234', requestedRelease: null }, + { job: { jobId: 'job-1', operation: 'restart', state: 'succeeded', releaseAfter: 'abc1234', evidence: ['verify_live ok'], startedAt: 't0', finishedAt: 't1', error: null }, timedOut: false }); + const outcome = await resumeDeploymentRuns(); + expect(outcome).toEqual({ resumed: 1, skipped: 0 }); + expect(transitions.map(t => t.state)).toEqual(['VERIFYING', 'SUCCEEDED']); + expect(transitions[1].patch.verification.verified).toBe(true); + expect(transitions[1].patch.result.resumedAfterRestart).toBe(true); + }); + + it('a rollback whose serving release differs from the journal target needs review', async () => { + seed('run-2', 'explorer.release.rollback', 'job-2', { releaseBefore: 'abc1234', requestedRelease: 'def5678' }, + { job: { jobId: 'job-2', operation: 'rollback', state: 'succeeded', releaseAfter: 'abc1234', evidence: [], startedAt: 't0', finishedAt: 't1', error: null }, timedOut: false }); + await resumeDeploymentRuns(); + expect(transitions[1]).toMatchObject({ runId: 'run-2', state: 'NEEDS_REVIEW' }); + expect(transitions[1].patch.verification.verified).toBe(false); + }); + + it('a failed adapter job fails the run and a still-running job is left for review, never marked done', async () => { + seed('run-3', 'explorer.service.restart', 'job-3', { releaseBefore: 'abc1234' }, + { job: { jobId: 'job-3', operation: 'restart', state: 'failed', releaseAfter: null, evidence: [], startedAt: 't0', finishedAt: 't1', error: 'cutover gate failed' }, timedOut: false }); + seed('run-4', 'explorer.service.restart', 'job-4', { releaseBefore: 'abc1234' }, + { job: { jobId: 'job-4', operation: 'restart', state: 'running', releaseAfter: null, evidence: [], startedAt: 't0', finishedAt: null, error: null }, timedOut: true }); + await resumeDeploymentRuns(); + const byRun = Object.fromEntries(transitions.filter(t => t.state !== 'VERIFYING').map(t => [t.runId, t])); + expect(byRun['run-3'].state).toBe('FAILED'); + expect(byRun['run-4'].state).toBe('NEEDS_REVIEW'); + expect(byRun['run-4'].patch.verification.verified).toBe(false); + }); + + it('runs without a recorded job id are left to the reconciler', async () => { + jobs.set('none', { row: { run_id: 'run-5', operation_id: 'explorer.service.restart', document: JSON.stringify({ result: {} }) }, answer: null }); + expect(await resumeDeploymentRuns()).toEqual({ resumed: 0, skipped: 1 }); + expect(transitions).toHaveLength(0); + }); +}); diff --git a/backend/src/__tests__/indexer-single-task.test.ts b/backend/src/__tests__/indexer-single-task.test.ts new file mode 100644 index 00000000000..c472127a898 --- /dev/null +++ b/backend/src/__tests__/indexer-single-task.test.ts @@ -0,0 +1,181 @@ +const configState = { + MEMPOOL: { NETWORK: 'mainnet', INDEXING_BLOCKS_AMOUNT: -1, ENABLED: true }, + DATABASE: { ENABLED: true }, + FIAT_PRICE: { ENABLED: true }, +}; +const commonState = { indexingEnabled: true }; +const miningState = { + indexBlockPrices: jest.fn(async () => undefined), + indexCoinStatsIndex: jest.fn(async () => undefined), +}; +const pricesState = { latestPriceId: jest.fn(async (): Promise => 7) }; +const priceUpdaterState = { historyInserted: true }; + +jest.mock('../config', () => ({ __esModule: true, default: configState })); +jest.mock('../api/common', () => ({ Common: { indexingEnabled: () => commonState.indexingEnabled } })); +jest.mock('../api/mining/mining', () => ({ + __esModule: true, + default: { + $indexBlockPrices: () => miningState.indexBlockPrices(), + $indexCoinStatsIndex: () => miningState.indexCoinStatsIndex(), + }, +})); +jest.mock('../repositories/PricesRepository', () => ({ + __esModule: true, + default: { $getLatestPriceId: () => pricesState.latestPriceId() }, +})); +jest.mock('../tasks/price-updater', () => ({ __esModule: true, default: priceUpdaterState })); +jest.mock('../api/blocks', () => ({ __esModule: true, default: {} })); +jest.mock('../api/bitcoin/bitcoin-client', () => ({ __esModule: true, default: {} })); +jest.mock('../replication/AuditReplication', () => ({ __esModule: true, default: {} })); +jest.mock('../replication/StatisticsReplication', () => ({ __esModule: true, default: {} })); +jest.mock('../repositories/AccelerationRepository', () => ({ __esModule: true, default: {} })); +jest.mock('../repositories/BlocksAuditsRepository', () => ({ __esModule: true, default: {} })); +jest.mock('../repositories/BlocksRepository', () => ({ __esModule: true, default: {} })); + +import indexer from '../indexer'; + +/** Lets every promise chain queued so far settle while timers are faked. */ +async function flush(): Promise { + for (let i = 0; i < 10; i++) { + await Promise.resolve(); + } +} + +describe('indexer single tasks', () => { + beforeEach(() => { + jest.useFakeTimers(); + configState.MEMPOOL.NETWORK = 'mainnet'; + configState.FIAT_PRICE.ENABLED = true; + commonState.indexingEnabled = true; + priceUpdaterState.historyInserted = true; + miningState.indexBlockPrices.mockReset().mockResolvedValue(undefined); + miningState.indexCoinStatsIndex.mockReset().mockResolvedValue(undefined); + pricesState.latestPriceId.mockReset().mockResolvedValue(7); + }); + + afterEach(() => { + jest.clearAllTimers(); + jest.useRealTimers(); + }); + + it('executes a first and a second scheduled call', /** @asyncUnsafe Jest owns the test promise. */ async () => { + indexer.scheduleSingleTask('coinStatsIndex', 1000); + jest.advanceTimersByTime(1000); + await flush(); + expect(miningState.indexCoinStatsIndex).toHaveBeenCalledTimes(1); + + indexer.scheduleSingleTask('coinStatsIndex', 1000); + jest.advanceTimersByTime(1000); + await flush(); + expect(miningState.indexCoinStatsIndex).toHaveBeenCalledTimes(2); + }); + + it('lets a task reschedule itself while it runs', /** @asyncUnsafe Jest owns the test promise. */ async () => { + // A blocksPrices run without price history defers itself with a new timer. + priceUpdaterState.historyInserted = false; + indexer.scheduleSingleTask('blocksPrices', 1000); + jest.advanceTimersByTime(1000); + await flush(); + expect(miningState.indexBlockPrices).not.toHaveBeenCalled(); + expect(jest.getTimerCount()).toBe(1); + + priceUpdaterState.historyInserted = true; + jest.advanceTimersByTime(10_000); + await flush(); + expect(miningState.indexBlockPrices).toHaveBeenCalledTimes(1); + }); + + it('does not let a replaced timer cancel its replacement', /** @asyncUnsafe Jest owns the test promise. */ async () => { + indexer.scheduleSingleTask('coinStatsIndex', 1000); + indexer.scheduleSingleTask('coinStatsIndex', 5000, true); + jest.advanceTimersByTime(1000); + await flush(); + expect(miningState.indexCoinStatsIndex).not.toHaveBeenCalled(); + jest.advanceTimersByTime(4000); + await flush(); + expect(miningState.indexCoinStatsIndex).toHaveBeenCalledTimes(1); + // The replacement fired; a further throttled call must still be accepted. + indexer.scheduleSingleTask('coinStatsIndex', 1000); + jest.advanceTimersByTime(1000); + await flush(); + expect(miningState.indexCoinStatsIndex).toHaveBeenCalledTimes(2); + }); + + it('throttles a second call while one is pending', /** @asyncUnsafe Jest owns the test promise. */ async () => { + indexer.scheduleSingleTask('coinStatsIndex', 1000); + indexer.scheduleSingleTask('coinStatsIndex', 1000); + expect(jest.getTimerCount()).toBe(1); + jest.advanceTimersByTime(1000); + await flush(); + expect(miningState.indexCoinStatsIndex).toHaveBeenCalledTimes(1); + }); + + it('reports a rejected job as failed and permits a later retry', /** @asyncUnsafe Jest owns the test promise. */ async () => { + miningState.indexBlockPrices.mockRejectedValueOnce(new Error('isolated task failure')); + const failed = await indexer.runSingleTask('blocksPrices'); + expect(failed).toEqual({ task: 'blocksPrices', status: 'failed', error: 'isolated task failure' }); + + const retried = await indexer.runSingleTask('blocksPrices'); + expect(retried.status).toBe('completed'); + expect(retried.checkpoint?.startedAt).toBeDefined(); + expect(retried.checkpoint?.finishedAt).toBeDefined(); + expect(miningState.indexBlockPrices).toHaveBeenCalledTimes(2); + }); + + it('does not run a duplicate concurrent task twice', /** @asyncUnsafe Jest owns the test promise. */ async () => { + let release: () => void = () => undefined; + miningState.indexCoinStatsIndex.mockImplementationOnce(() => new Promise((resolve) => { release = () => resolve(undefined); })); + const first = indexer.runSingleTask('coinStatsIndex'); + await flush(); + const second = await indexer.runSingleTask('coinStatsIndex'); + expect(second.status).toBe('already-running'); + release(); + expect((await first).status).toBe('completed'); + expect(miningState.indexCoinStatsIndex).toHaveBeenCalledTimes(1); + }); + + it('reports disabled indexing without touching the job', /** @asyncUnsafe Jest owns the test promise. */ async () => { + commonState.indexingEnabled = false; + const outcome = await indexer.runSingleTask('coinStatsIndex'); + expect(outcome.status).toBe('disabled'); + expect(miningState.indexCoinStatsIndex).not.toHaveBeenCalled(); + }); + + it('reports disabled fiat prices for the blocksPrices task', /** @asyncUnsafe Jest owns the test promise. */ async () => { + configState.FIAT_PRICE.ENABLED = false; + const outcome = await indexer.runSingleTask('blocksPrices'); + expect(outcome.status).toBe('disabled'); + expect(miningState.indexBlockPrices).not.toHaveBeenCalled(); + }); + + it('reports network-inapplicable for blocksPrices on signet', /** @asyncUnsafe Jest owns the test promise. */ async () => { + configState.MEMPOOL.NETWORK = 'signet'; + const outcome = await indexer.runSingleTask('blocksPrices'); + expect(outcome.status).toBe('network-inapplicable'); + expect(miningState.indexBlockPrices).not.toHaveBeenCalled(); + }); + + it('reports deferred when price history is unavailable and schedules a real retry', /** @asyncUnsafe Jest owns the test promise. */ async () => { + pricesState.latestPriceId.mockResolvedValueOnce(null); + const outcome = await indexer.runSingleTask('blocksPrices'); + expect(outcome.status).toBe('deferred'); + expect(miningState.indexBlockPrices).not.toHaveBeenCalled(); + expect(jest.getTimerCount()).toBe(1); + jest.advanceTimersByTime(10_000); + await flush(); + expect(miningState.indexBlockPrices).toHaveBeenCalledTimes(1); + }); + + it('does not hide a coinStatsIndex failure from the caller', /** @asyncUnsafe Jest owns the test promise. */ async () => { + miningState.indexCoinStatsIndex.mockRejectedValueOnce(new Error('isolated coinstats failure')); + const outcome = await indexer.runSingleTask('coinStatsIndex'); + expect(outcome).toEqual({ task: 'coinStatsIndex', status: 'failed', error: 'isolated coinstats failure' }); + }); + + it('accepts a reindex only when indexing is enabled', () => { + expect(indexer.reindex()).toBe(true); + commonState.indexingEnabled = false; + expect(indexer.reindex()).toBe(false); + }); +}); diff --git a/backend/src/__tests__/price-updater-stale-rates.test.ts b/backend/src/__tests__/price-updater-stale-rates.test.ts new file mode 100644 index 00000000000..c5aae57e69e --- /dev/null +++ b/backend/src/__tests__/price-updater-stale-rates.test.ts @@ -0,0 +1,118 @@ +/** + * A stored fiat observation is never advertised as the current price unless + * the feed is enabled for this network and the observation is fresh; the + * stored row is exposed with its timestamp, source and an explicit state. + */ +const configState = { + MEMPOOL: { NETWORK: 'mainnet', PRICE_UPDATES_PER_HOUR: 1 }, + DATABASE: { ENABLED: true }, + FIAT_PRICE: { ENABLED: false, API_KEY: '' }, +}; +const repositoryState = { stored: null as Record | null }; + +jest.mock('../config', () => ({ __esModule: true, default: configState })); +jest.mock('../repositories/PricesRepository', () => ({ + __esModule: true, + MAX_PRICES: {}, + default: { + $getLatestConversionRates: /** @asyncUnsafe test double */ async () => repositoryState.stored ?? emptyPrices(), + }, +})); +jest.mock('../tasks/price-feeds/bitfinex-api', () => ({ __esModule: true, default: class { currencies = []; } })); +jest.mock('../tasks/price-feeds/bitflyer-api', () => ({ __esModule: true, default: class { currencies = []; } })); +jest.mock('../tasks/price-feeds/coinbase-api', () => ({ __esModule: true, default: class { currencies = []; } })); +jest.mock('../tasks/price-feeds/gemini-api', () => ({ __esModule: true, default: class { currencies = []; } })); +jest.mock('../tasks/price-feeds/kraken-api', () => ({ __esModule: true, default: class { currencies = []; } })); +jest.mock('../tasks/price-feeds/free-currency-api', () => ({ __esModule: true, default: class {} })); + +import priceUpdater, { PRICE_MAX_AGE_SECONDS } from '../tasks/price-updater'; + +function emptyPrices(): Record { + const prices: Record = { time: 0 }; + for (const currency of ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY', 'BGN', 'BRL', 'CNY', 'CZK', 'DKK', 'HKD', 'HRK', 'HUF', 'IDR', 'ILS', 'INR', 'ISK', 'KRW', 'MXN', 'MYR', 'NOK', 'NZD', 'PHP', 'PLN', 'RON', 'RUB', 'SEK', 'SGD', 'THB', 'TRY', 'ZAR']) { + prices[currency] = -1; + } + return prices; +} + +const NOW_MS = Date.UTC(2026, 8, 17, 12, 0, 0); +const STALE_TIME = Math.round(NOW_MS / 1000) - 30 * 24 * 3600; +const FRESH_TIME = Math.round(NOW_MS / 1000) - 600; + +function storedFixture(time: number): Record { + return { ...emptyPrices(), time, USD: 61_234, EUR: 56_789 }; +} + +function reset(): void { + const subject = priceUpdater as unknown as Record; + subject.latestPrices = priceUpdater.getEmptyPricesObj(); + subject.latestGoodPrices = priceUpdater.getEmptyPricesObj(); + subject.storedPrices = null; + subject.liveSource = null; +} + +describe('stored fiat rates with the feed disabled', () => { + beforeEach(() => { + reset(); + configState.MEMPOOL.NETWORK = 'mainnet'; + configState.FIAT_PRICE.ENABLED = false; + repositoryState.stored = storedFixture(STALE_TIME); + }); + + it('does not advertise the stored observation as the current price', /** @asyncUnsafe Jest owns the test promise. */ async () => { + await priceUpdater.$initializeLatestPriceWithDb(NOW_MS); + expect(priceUpdater.getLatestPrices().USD).toBe(-1); + const advertised = priceUpdater.getAdvertisedPrices(NOW_MS); + expect(advertised.USD).toBe(-1); + expect(advertised).toMatchObject({ + state: 'disabled', + disabled: true, + stale: true, + source: 'database', + observedAt: STALE_TIME, + ageSeconds: 30 * 24 * 3600, + }); + expect(advertised.storedObservation).toMatchObject({ time: STALE_TIME, USD: 61_234, EUR: 56_789 }); + }); + + it('reports network-inapplicable on signet even with a stored row', /** @asyncUnsafe Jest owns the test promise. */ async () => { + configState.MEMPOOL.NETWORK = 'signet'; + configState.FIAT_PRICE.ENABLED = true; + repositoryState.stored = storedFixture(FRESH_TIME); + await priceUpdater.$initializeLatestPriceWithDb(NOW_MS); + expect(priceUpdater.getLatestPrices().USD).toBe(-1); + expect(priceUpdater.getAdvertisedPrices(NOW_MS)).toMatchObject({ state: 'network-inapplicable', disabled: true, USD: -1 }); + }); +}); + +describe('stored fiat rates with the feed enabled', () => { + beforeEach(() => { + reset(); + configState.MEMPOOL.NETWORK = 'mainnet'; + configState.FIAT_PRICE.ENABLED = true; + }); + + it('serves a fresh stored observation as live from the database', /** @asyncUnsafe Jest owns the test promise. */ async () => { + repositoryState.stored = storedFixture(FRESH_TIME); + await priceUpdater.$initializeLatestPriceWithDb(NOW_MS); + expect(priceUpdater.getLatestPrices().USD).toBe(61_234); + expect(priceUpdater.getAdvertisedPrices(NOW_MS)).toMatchObject({ + state: 'live', stale: false, disabled: false, source: 'database', observedAt: FRESH_TIME, USD: 61_234, storedObservation: null, + }); + }); + + it('keeps a stale stored observation out of the live price', /** @asyncUnsafe Jest owns the test promise. */ async () => { + repositoryState.stored = storedFixture(Math.round(NOW_MS / 1000) - PRICE_MAX_AGE_SECONDS - 1); + await priceUpdater.$initializeLatestPriceWithDb(NOW_MS); + expect(priceUpdater.getLatestPrices().USD).toBe(-1); + const advertised = priceUpdater.getAdvertisedPrices(NOW_MS); + expect(advertised).toMatchObject({ state: 'stale', stale: true, disabled: false, source: 'database', USD: -1 }); + expect(advertised.storedObservation?.USD).toBe(61_234); + }); + + it('reports unavailable when nothing is stored', /** @asyncUnsafe Jest owns the test promise. */ async () => { + repositoryState.stored = null; + await priceUpdater.$initializeLatestPriceWithDb(NOW_MS); + expect(priceUpdater.getAdvertisedPrices(NOW_MS)).toMatchObject({ state: 'unavailable', stale: false, source: null, observedAt: null, storedObservation: null }); + }); +}); diff --git a/backend/src/__tests__/tx-selection-sigops.test.ts b/backend/src/__tests__/tx-selection-sigops.test.ts new file mode 100644 index 00000000000..8b2b3285448 --- /dev/null +++ b/backend/src/__tests__/tx-selection-sigops.test.ts @@ -0,0 +1,75 @@ +jest.mock('../config', () => ({ + __esModule: true, + default: { MEMPOOL: { BLOCK_WEIGHT_UNITS: 4_000_000 } }, +})); +jest.mock('worker_threads', () => ({ parentPort: null })); + +import { makeBlockTemplates } from '../api/tx-selection-worker'; +import { CompactThreadTransaction } from '../mempool.interfaces'; + +const MAX_BLOCK_SIGOPS = 80_000; + +function transaction(uid: number, overrides: Partial = {}): CompactThreadTransaction { + return { + uid, + fee: 200_000, + weight: 100_000, + feePerVsize: 8, + effectiveFeePerVsize: 8, + sigops: 16_000, + inputs: [], + ...overrides, + }; +} + +function sigopsOf(block: number[], mempool: Map): number { + return block.reduce((sum, uid) => sum + (mempool.get(uid)?.sigops ?? 0), 0); +} + +describe('JavaScript block template sigop accounting', () => { + it('keeps every bounded projected block within 80000 sigops for independent transactions', () => { + const mempool = new Map(); + for (let uid = 1; uid <= 6; uid++) { + mempool.set(uid, transaction(uid)); + } + const { blocks } = makeBlockTemplates(mempool); + // Six transactions of 16000 sigops each total 96000, which is more than one + // block allows and far below the weight limit, so a second block is needed. + expect(blocks.length).toBe(2); + expect(blocks[0].length).toBe(5); + expect(blocks[1]).toEqual([6]); + for (const block of blocks.slice(0, 7)) { + expect(sigopsOf(block, mempool)).toBeLessThanOrEqual(MAX_BLOCK_SIGOPS); + } + expect(blocks.flat().sort((a, b) => a - b)).toEqual([1, 2, 3, 4, 5, 6]); + }); + + it('counts a parent once when it is placed as part of a child package', () => { + const mempool = new Map(); + // Parent 1 (low fee) is pulled in by child 2 (high fee); their package + // costs 32000 sigops. Three more independent 16000-sigop transactions + // bring the total to exactly the limit, so all fit into one block only if + // the parent is not counted twice. + mempool.set(1, transaction(1, { fee: 1_000, feePerVsize: 0.04, effectiveFeePerVsize: 0.04 })); + mempool.set(2, transaction(2, { fee: 400_000, feePerVsize: 16, effectiveFeePerVsize: 16, inputs: [1] })); + mempool.set(3, transaction(3)); + mempool.set(4, transaction(4)); + mempool.set(5, transaction(5)); + const { blocks } = makeBlockTemplates(mempool); + expect(blocks.length).toBe(1); + expect(sigopsOf(blocks[0], mempool)).toBe(MAX_BLOCK_SIGOPS); + expect(blocks[0].sort((a, b) => a - b)).toEqual([1, 2, 3, 4, 5]); + }); + + it('resets the sigop budget for each new bounded block', () => { + const mempool = new Map(); + for (let uid = 1; uid <= 15; uid++) { + mempool.set(uid, transaction(uid)); + } + const { blocks } = makeBlockTemplates(mempool); + expect(blocks.map((block) => block.length)).toEqual([5, 5, 5]); + for (const block of blocks) { + expect(sigopsOf(block, mempool)).toBeLessThanOrEqual(MAX_BLOCK_SIGOPS); + } + }); +}); diff --git a/backend/src/__tests__/websocket-track-txs.test.ts b/backend/src/__tests__/websocket-track-txs.test.ts new file mode 100644 index 00000000000..27ff8544628 --- /dev/null +++ b/backend/src/__tests__/websocket-track-txs.test.ts @@ -0,0 +1,136 @@ +const mempoolState: { txs: Record } = { txs: {} }; +const rbfState: { replaced: Record } = { replaced: {} }; +const nodeState = { getRawTransaction: jest.fn() }; + +jest.mock('../config', () => ({ + __esModule: true, + default: { + MEMPOOL: { NETWORK: 'signet', BACKEND: 'electrum', INITIAL_BLOCKS_AMOUNT: 8, MAX_TRACKED_ADDRESSES: 1, API_URL_PREFIX: '/api/v1/' }, + FIAT_PRICE: { ENABLED: false }, + DATABASE: { ENABLED: false }, + REDIS: { ENABLED: false }, + STATISTICS: { ENABLED: false }, + MEMPOOL_SERVICES: { ACCELERATIONS: false }, + }, +})); +jest.mock('../api/mempool', () => ({ + __esModule: true, + default: { getMempool: () => mempoolState.txs, getMempoolInfo: () => ({}), getVBytesPerSecond: () => 0, getLatestTransactions: () => [], getAccelerationPositions: () => undefined, isInSync: () => true }, +})); +jest.mock('../api/rbf-cache', () => ({ + __esModule: true, + default: { getReplacedBy: (txid: string) => rbfState.replaced[txid] }, +})); +jest.mock('../api/bitcoin/bitcoin-api-factory', () => ({ + __esModule: true, + default: { $getRawTransaction: (...args: unknown[]) => nodeState.getRawTransaction(...args), getHealthStatus: () => [] }, +})); +jest.mock('../api/blocks', () => ({ __esModule: true, default: { getBlocks: () => [], getCurrentBlockHeight: () => 0 } })); +jest.mock('../api/backend-info', () => ({ __esModule: true, default: { getBackendInfo: () => ({}) } })); +jest.mock('../api/mempool-blocks', () => ({ __esModule: true, default: { getMempoolBlocks: () => [] } })); +jest.mock('../api/loading-indicators', () => ({ __esModule: true, default: { getLoadingIndicators: () => ({}) } })); +jest.mock('../api/transaction-utils', () => ({ __esModule: true, default: {} })); +jest.mock('../api/difficulty-adjustment', () => ({ __esModule: true, default: { getDifficultyAdjustment: () => null } })); +jest.mock('../api/fee-api', () => ({ __esModule: true, default: { getPreciseRecommendedFee: () => ({}) } })); +jest.mock('../repositories/BlocksAuditsRepository', () => ({ __esModule: true, default: {} })); +jest.mock('../repositories/BlocksSummariesRepository', () => ({ __esModule: true, default: {} })); +jest.mock('../repositories/AccelerationRepository', () => ({ __esModule: true, default: {} })); +jest.mock('../api/audit', () => ({ __esModule: true, default: {} })); +jest.mock('../tasks/price-updater', () => ({ __esModule: true, default: { getLatestPrices: () => ({}), getAdvertisedPrices: () => ({}), getPriceObservation: () => ({}) } })); +jest.mock('../api/services/acceleration', () => ({ __esModule: true, default: {} })); +jest.mock('../api/statistics/statistics', () => ({ __esModule: true, default: {} })); +jest.mock('../api/services/wallets', () => ({ __esModule: true, default: {} })); +jest.mock('../api/bitcoin/bitcoin-second-client', () => ({ __esModule: true, default: {} })); +jest.mock('../api/cpfp', () => ({ calculateMempoolTxCpfp: () => undefined })); +jest.mock('../api/services/stratum', () => ({ __esModule: true, default: {} })); + +import websocketHandler from '../api/websocket-handler'; + +const TXID_MEMPOOL = '1'.repeat(64); +const TXID_REPLACED = '2'.repeat(64); +const TXID_CONFIRMED = '3'.repeat(64); +const TXID_UNKNOWN = '4'.repeat(64); +const TXID_DOWN = '5'.repeat(64); +const TXID_NODE_MEMPOOL = '6'.repeat(64); +const HASH = 'a'.repeat(64); + +function connect(): { send: (message: Record) => Promise[]> } { + const events: Record = {}; + const clientEvents: Record = {}; + const messages: Record[] = []; + const server = { on: (name: string, fn: any) => { events[name] = fn; } }; + (websocketHandler as any).webSocketServers = []; + websocketHandler.addWebsocketServer(server as any); + websocketHandler.setupConnectionHandling(); + const client = { on: (name: string, fn: any) => { clientEvents[name] = fn; }, send: (data: string) => messages.push(JSON.parse(data)), close: () => { throw new Error('unexpected close'); } }; + events.connection(client, { headers: {}, socket: { remoteAddress: '127.0.0.1' } }); + return { + send: /** @asyncUnsafe test helper */ async (message) => { + messages.length = 0; + await clientEvents.message(JSON.stringify(message)); + return messages; + }, + }; +} + +describe('websocket track-txs initial status', () => { + beforeEach(() => { + mempoolState.txs = { [TXID_MEMPOOL]: { txid: TXID_MEMPOOL, position: { block: 0, vsize: 100 } } }; + rbfState.replaced = { [TXID_REPLACED]: 'b'.repeat(64) }; + nodeState.getRawTransaction.mockReset().mockImplementation(async (txid: string) => { + if (txid === TXID_CONFIRMED) { + return { txid, status: { confirmed: true, block_height: 250_000, block_hash: HASH } }; + } + if (txid === TXID_NODE_MEMPOOL) { + return { txid, status: { confirmed: false } }; + } + if (txid === TXID_DOWN) { + throw new Error('ECONNREFUSED'); + } + throw new Error('No such mempool or blockchain transaction'); + }); + }); + + it('never reports a never-seen transaction as confirmed', /** @asyncUnsafe Jest owns the test promise. */ async () => { + const socket = connect(); + const [message] = await socket.send({ 'track-txs': [TXID_UNKNOWN] }); + expect(message['tracked-txs'][TXID_UNKNOWN]).toEqual({ confirmed: false, status: 'unknown' }); + expect(nodeState.getRawTransaction).toHaveBeenCalledWith(TXID_UNKNOWN, true, false); + }); + + it('distinguishes mempool, replaced, confirmed, node-mempool and unavailable', /** @asyncUnsafe Jest owns the test promise. */ async () => { + const socket = connect(); + const [message] = await socket.send({ + 'track-txs': [TXID_MEMPOOL, TXID_REPLACED, TXID_CONFIRMED, TXID_NODE_MEMPOOL, TXID_DOWN], + }); + const tracked = message['tracked-txs']; + expect(tracked[TXID_MEMPOOL]).toEqual({ confirmed: false, status: 'mempool', position: { block: 0, vsize: 100 } }); + expect(tracked[TXID_REPLACED]).toEqual({ confirmed: false, status: 'replaced', replacedBy: 'b'.repeat(64) }); + expect(tracked[TXID_CONFIRMED]).toEqual({ confirmed: true, status: 'confirmed', blockHeight: 250_000, blockHash: HASH }); + expect(tracked[TXID_NODE_MEMPOOL]).toEqual({ confirmed: false, status: 'mempool' }); + expect(tracked[TXID_DOWN]).toEqual({ status: 'unavailable' }); + expect(tracked[TXID_DOWN].confirmed).toBeUndefined(); + // Local and replaced transactions never reach the node. + expect(nodeState.getRawTransaction).toHaveBeenCalledTimes(3); + }); + + it('does not confirm on a node answer without a real block identity', /** @asyncUnsafe Jest owns the test promise. */ async () => { + nodeState.getRawTransaction.mockResolvedValueOnce({ txid: TXID_CONFIRMED, status: { confirmed: true } }); + const socket = connect(); + const [message] = await socket.send({ 'track-txs': [TXID_CONFIRMED] }); + expect(message['tracked-txs'][TXID_CONFIRMED].confirmed).toBe(false); + }); + + it('deduplicates ids and refuses an oversized request', /** @asyncUnsafe Jest owns the test promise. */ async () => { + const socket = connect(); + const [message] = await socket.send({ 'track-txs': [TXID_UNKNOWN, TXID_UNKNOWN, 'not-a-txid'] }); + expect(Object.keys(message['tracked-txs'])).toEqual([TXID_UNKNOWN]); + expect(nodeState.getRawTransaction).toHaveBeenCalledTimes(1); + + const many = Array.from({ length: 101 }, (_, i) => i.toString(16).padStart(64, '0')); + const [refused] = await socket.send({ 'track-txs': many }); + expect(refused['track-txs-error']).toContain('maximum of 100'); + expect(refused['tracked-txs']).toBeUndefined(); + expect(nodeState.getRawTransaction).toHaveBeenCalledTimes(1); + }); +}); diff --git a/backend/src/api/admin-adapter/admin-adapter.catalog.ts b/backend/src/api/admin-adapter/admin-adapter.catalog.ts index e690f08fd85..f6b879b5220 100644 --- a/backend/src/api/admin-adapter/admin-adapter.catalog.ts +++ b/backend/src/api/admin-adapter/admin-adapter.catalog.ts @@ -2,6 +2,14 @@ import type { AdminEnvironment, AdminOperationDefinition, } from '@bitcoinuniverse/ecosystem-contracts/admin-control'; +import { + DEPLOYMENT_CONTROL_ENDPOINT_VARIABLE, + DEPLOYMENT_CONTROL_KEY_VARIABLE, + deploymentControlClient, + deploymentControlConfig, + rollbackTargetFromJournal, + type DeploymentControlState, +} from './deployment-control.client'; /** * What the Control Center may ask this Explorer to do, described without @@ -23,10 +31,13 @@ const ALL_ENVIRONMENTS: AdminEnvironment[] = [ 'development', ]; -export const DEPLOYMENT_CONTROL_ENVIRONMENT_VARIABLE = 'EXPLORER_DEPLOYMENT_CONTROL'; +export const DEPLOYMENT_CONTROL_ENVIRONMENT_VARIABLES = [ + DEPLOYMENT_CONTROL_ENDPOINT_VARIABLE, + DEPLOYMENT_CONTROL_KEY_VARIABLE, +] as const; export const DEPLOYMENT_CONTROL_REASON = - 'Host-level deployment control is not enabled for this process. An operator has to configure the deployment adapter and set EXPLORER_DEPLOYMENT_CONTROL=enabled before the Control Center can restart or roll back this Explorer.'; + `Host-level deployment control is not configured for this process. An operator has to run the owned deployment adapter on this host and set ${DEPLOYMENT_CONTROL_ENDPOINT_VARIABLE} and ${DEPLOYMENT_CONTROL_KEY_VARIABLE} before the Control Center can restart or roll back this Explorer.`; /** Operations that cannot run without host-level deployment control. */ export const DEPLOYMENT_CONTROLLED_OPERATIONS = [ @@ -38,10 +49,49 @@ export const DEPLOYMENT_CONTROLLED_OPERATIONS = [ export const ALLOWED_INDEXER_TASKS = ['blocksPrices', 'coinStatsIndex'] as const; export type AllowedIndexerTask = (typeof ALLOWED_INDEXER_TASKS)[number]; +/** Whether the adapter is named at all. Configured is not ready: readiness is what the adapter answers. */ export function deploymentControlConfigured( environment: Record = process.env, ): boolean { - return String(environment[DEPLOYMENT_CONTROL_ENVIRONMENT_VARIABLE] ?? '').trim() === 'enabled'; + try { + return deploymentControlConfig(environment) !== null; + } catch { + return false; + } +} + +/** + * The availability of one deployment-controlled operation, from what the + * adapter has actually answered. A flag cannot make this 'enabled'; only a + * capability document that supports the action can, and for a rollback the + * journal has to hold a previous verified release as well. + */ +export function deploymentControlAvailability( + operationId: string, + control: DeploymentControlState, +): { availability: AdminOperationDefinition['availability']; availabilityReason: string | null } { + if (control.state === 'unconfigured') { + return { availability: 'not_configured', availabilityReason: control.reason }; + } + if (control.state === 'unprobed' || control.state === 'unreachable') { + return { availability: 'unavailable', availabilityReason: control.reason }; + } + const { capabilities } = control; + if (operationId === 'explorer.service.restart') { + return capabilities.supports.restart + ? { availability: 'enabled', availabilityReason: null } + : { availability: 'unavailable', availabilityReason: capabilities.reasons.restart ?? 'The deployment adapter does not support a restart on this host.' }; + } + if (operationId === 'explorer.release.rollback') { + if (!capabilities.supports.rollback) { + return { availability: 'unavailable', availabilityReason: capabilities.reasons.rollback ?? 'The deployment adapter does not support a rollback on this host.' }; + } + if (!rollbackTargetFromJournal(capabilities)) { + return { availability: 'unavailable', availabilityReason: 'The deployment adapter journal holds no previous verified release to roll back to.' }; + } + return { availability: 'enabled', availabilityReason: null }; + } + return { availability: 'enabled', availabilityReason: null }; } function definition( @@ -278,13 +328,14 @@ const DEFINITIONS: AdminOperationDefinition[] = [ pattern: null, }, ], - sideEffects: ['Stops and starts the Explorer backend process.'], - postconditions: ['The Explorer answers its capability route again from a new process.'], - timeoutSeconds: 120, + sideEffects: ['Runs the release tooling\'s health-checked cutover of the current release, which restarts the backend and overlay units and verifies the live origin.'], + postconditions: ['The deployment adapter reports the job succeeded with the same release serving from a new process.'], + timeoutSeconds: 600, retryPolicy: 'none', lock: 'explorer:service:restart', userImpact: 'Every open WebSocket stream is dropped and reconnects.', - publicServiceImpact: 'The Explorer is unreachable for the length of the restart.', + publicServiceImpact: + 'The gateway keeps the public origin up while the backend and overlay restart; API routes answer errors until the new process passes the live verification.', }), definition({ id: 'explorer.release.rollback', @@ -293,7 +344,7 @@ const DEFINITIONS: AdminOperationDefinition[] = [ action: 'rollback', name: 'Roll back to the previous verified Explorer release', description: - 'Asks the operator-controlled deployment adapter to put the previous verified release back in service. The target release comes from the recorded promotion journal, never from this request.', + 'Asks the operator-controlled deployment adapter to put the previous verified release back in service through the release tooling\'s health-checked rollback. The target release comes from the adapter\'s promotion journal, never from this request.', risk: 'IRREVERSIBLE', requiredPermission: 'operate.irreversible', inputFields: [ @@ -310,31 +361,33 @@ const DEFINITIONS: AdminOperationDefinition[] = [ ], sideEffects: ['Replaces the running Explorer release with the previous verified one.'], postconditions: ['The Explorer reports the previous verified release sha.'], - timeoutSeconds: 600, + timeoutSeconds: 900, retryPolicy: 'none', lock: 'explorer:release:rollback', - userImpact: 'The site briefly serves the previous build.', - publicServiceImpact: 'A short interruption while the switch happens.', + userImpact: 'The site serves the previous build; every open WebSocket stream reconnects.', + publicServiceImpact: + 'The gateway keeps the public origin up while the units restart on the previous release; API routes answer errors until it passes the live verification.', }), ]; /** * The catalog, with the availability of the deployment-controlled operations - * recomputed on every call. Host configuration can change between restarts, - * and an operation that is unavailable stays visible with the exact reason - * rather than disappearing from the panel. + * recomputed on every call from what the deployment adapter last answered. + * Host configuration can change between restarts, and an operation that is + * unavailable stays visible with the exact reason rather than disappearing + * from the panel. */ export function explorerOperationDefinitions( environment: Record = process.env, + control: DeploymentControlState = environment === process.env + ? deploymentControlClient.known() + : deploymentControlConfigured(environment) + ? { state: 'unprobed', reason: 'The deployment adapter is configured but has not answered its capability route yet.', capabilities: null, probedAt: null } + : { state: 'unconfigured', reason: DEPLOYMENT_CONTROL_REASON, capabilities: null, probedAt: null }, ): AdminOperationDefinition[] { - const enabled = deploymentControlConfigured(environment); return DEFINITIONS.map((operation) => DEPLOYMENT_CONTROLLED_OPERATIONS.includes(operation.id) - ? { - ...operation, - availability: enabled ? ('enabled' as const) : ('not_configured' as const), - availabilityReason: enabled ? null : DEPLOYMENT_CONTROL_REASON, - } + ? { ...operation, ...deploymentControlAvailability(operation.id, control) } : operation, ); } @@ -342,8 +395,9 @@ export function explorerOperationDefinitions( export function findExplorerOperationDefinition( operationId: string, environment: Record = process.env, + control?: DeploymentControlState, ): AdminOperationDefinition { - const found = explorerOperationDefinitions(environment).find( + const found = explorerOperationDefinitions(environment, control).find( (operation) => operation.id === operationId, ); if (!found) { diff --git a/backend/src/api/admin-adapter/admin-adapter.operations.ts b/backend/src/api/admin-adapter/admin-adapter.operations.ts index a4098f30c4c..2bbd6b9b947 100644 --- a/backend/src/api/admin-adapter/admin-adapter.operations.ts +++ b/backend/src/api/admin-adapter/admin-adapter.operations.ts @@ -18,13 +18,53 @@ import { ALLOWED_INDEXER_TASKS, DEPLOYMENT_CONTROL_REASON, type AllowedIndexerTask, + deploymentControlAvailability, deploymentControlConfigured, explorerOperationDefinitions, findExplorerOperationDefinition, } from './admin-adapter.catalog'; +import { + DEPLOYMENT_CONTROL_LIMITS, + deploymentControlClient, + rollbackTargetFromJournal, + type DeploymentCapabilities, + type DeploymentControlClient, + type DeploymentControlState, + type DeploymentRequest, +} from './deployment-control.client'; import { explorerEnvironment, releaseShaOrNull } from './admin-adapter.identity'; import runStore from './admin-adapter.runs'; +/** + * Reads back what an indexing task is supposed to have persisted, so a run + * is only called complete when the rows say so. Every query error is returned, + * never swallowed, because an unreadable checkpoint proves nothing. + * + * @asyncUnsafe The caller wraps the readback in try/catch. + */ +async function readIndexerCheckpoint(task: AllowedIndexerTask): Promise<{ remaining: number; indexedTip: number | null }> { + if (task === 'blocksPrices') { + const [remainingRows]: any[] = await DB.query( + 'SELECT COUNT(*) AS remaining FROM blocks LEFT JOIN blocks_prices ON blocks.height = blocks_prices.height LEFT JOIN prices ON blocks_prices.price_id = prices.id WHERE blocks_prices.height IS NULL OR prices.id IS NULL', + ); + const [tipRows]: any[] = await DB.query('SELECT MAX(height) AS tip FROM blocks_prices'); + return { + remaining: Number(remainingRows[0]?.remaining ?? 0), + indexedTip: tipRows[0]?.tip === null || tipRows[0]?.tip === undefined ? null : Number(tipRows[0].tip), + }; + } + const [remainingRows]: any[] = await DB.query( + 'SELECT COUNT(*) AS remaining FROM blocks WHERE (utxoset_size IS NULL OR total_input_amt IS NULL) AND stale = 0', + ); + const [tipRows]: any[] = await DB.query( + 'SELECT MAX(height) AS tip FROM blocks WHERE utxoset_size IS NOT NULL AND total_input_amt IS NOT NULL AND stale = 0', + ); + return { + remaining: Number(remainingRows[0]?.remaining ?? 0), + indexedTip: tipRows[0]?.tip === null || tipRows[0]?.tip === undefined ? null : Number(tipRows[0].tip), + }; +} + /** * The handlers behind the catalog. * @@ -127,14 +167,127 @@ function databasePrecondition() { ); } -function deploymentControlPrecondition() { - return deploymentControlConfigured() - ? satisfied('deployment-control', 'Host deployment control is enabled') - : unsatisfied( +/** Test seam: the adapter client the deployment operations talk to. */ +let controlClient: DeploymentControlClient = deploymentControlClient; +export function useDeploymentControlClient(client: DeploymentControlClient | null): void { + controlClient = client ?? deploymentControlClient; +} + +/** + * The adapter's own answer, fresh, is the precondition. It is not a flag: + * an operator who sets the endpoint without running the adapter gets the + * unreachable reason, and an adapter whose capability document does not + * support the action gets that adapter's reason. + */ +function deploymentControlPrecondition(operationId: string, control: DeploymentControlState) { + const { availability, availabilityReason } = deploymentControlAvailability(operationId, control); + return availability === 'enabled' + ? satisfied( 'deployment-control', - 'Host deployment control is enabled', - DEPLOYMENT_CONTROL_REASON, - ); + 'The deployment adapter answers and supports this operation', + control.state === 'ready' + ? `Adapter ${control.capabilities.adapterVersion} on target ${control.capabilities.target}, current release ${control.capabilities.currentRelease ?? 'unknown'}.` + : null, + ) + : unsatisfied('deployment-control', 'The deployment adapter answers and supports this operation', availabilityReason ?? DEPLOYMENT_CONTROL_REASON); +} + +/** @asyncSafe probe never rejects; every outcome is a state, and a thrown one is reported as unreachable. */ +async function probeControl(): Promise { + try { + return await controlClient.probe(); + } catch (e) { + return { state: 'unreachable', reason: 'The deployment adapter did not answer its capability route: ' + (e instanceof Error ? e.message : String(e)), capabilities: null, probedAt: new Date().toISOString() }; + } +} + +function releaseLabel(sha: string | null): string { + return sha ? sha.slice(0, 12) : 'unknown'; +} + +/** + * Submits one job to the adapter and drives it to a terminal state. + * + * The job id and the release serving before the request are written to the + * run before the job is awaited, because a restart replaces this very + * process: if the wait is cut short, the run still names the job that can + * be read back from the adapter. A wait that reaches the operation's bound + * reports the job as still running, never as done. + * @asyncUnsafe The route turns a rejection into a FAILED run. + */ +async function driveDeployment( + operation: 'restart' | 'rollback', + operationId: string, + context: ExplorerExecutionContext, + timeoutSeconds: number, +): Promise { + const control = await probeControl(); + const precondition = deploymentControlPrecondition(operationId, control); + if (!precondition.satisfied || control.state !== 'ready') { + throw new Error(precondition.detail ?? DEPLOYMENT_CONTROL_REASON); + } + const capabilities: DeploymentCapabilities = control.capabilities; + const request: DeploymentRequest = { + operationId: context.runId, + idempotencyKey: context.idempotencyKey ?? context.runId, + target: capabilities.target, + }; + let expectedRelease = capabilities.currentRelease; + if (operation === 'rollback') { + const journalTarget = rollbackTargetFromJournal(capabilities); + if (!journalTarget) { + throw new Error('The deployment adapter journal holds no previous verified release to roll back to.'); + } + request.release = journalTarget.release; + expectedRelease = journalTarget.release; + } + const startedAt = Date.now(); + const accepted = await controlClient.submit(operation, request); + const partial = { + jobId: accepted.jobId, + adapterTarget: capabilities.target, + releaseBefore: capabilities.currentRelease, + requestedRelease: request.release ?? null, + replayedByAdapter: accepted.replayed, + }; + await runStore.transition(context.runId, 'RUNNING', { progressPercent: 30, result: partial }); + + const budgetMs = Math.max(DEPLOYMENT_CONTROL_LIMITS.pollIntervalMs, timeoutSeconds * 1000 - (Date.now() - startedAt)); + const { job, timedOut } = await controlClient.waitForJob(accepted.jobId, budgetMs); + const evidence = [ + `Adapter job ${job.jobId} (${job.operation}) on ${job.target || capabilities.target} is ${job.state}.`, + `Release before: ${releaseLabel(job.releaseBefore ?? capabilities.currentRelease)}; after: ${releaseLabel(job.releaseAfter)}; expected: ${releaseLabel(expectedRelease)}.`, + ...job.evidence.map((line) => 'Adapter: ' + line), + ]; + const result = { + ...partial, + jobState: job.state, + releaseAfter: job.releaseAfter, + expectedRelease, + jobStartedAt: job.startedAt, + jobFinishedAt: job.finishedAt, + jobError: job.error, + waitedMs: Date.now() - startedAt, + timedOut, + }; + if (timedOut) { + return { + summary: `The adapter job ${job.jobId} was still ${job.state} after ${timeoutSeconds}s; read it back from the adapter before treating the ${operation} as done.`, + result, + verification: { verified: false, evidence: [...evidence, 'The wait reached the operation bound before the job reached a terminal state.'] }, + }; + } + if (job.state === 'failed') { + throw new Error(`The deployment adapter reported the ${operation} job ${job.jobId} failed: ${job.error ?? 'no reason given'}.`); + } + const verified = job.state === 'succeeded' && job.releaseAfter !== null && job.releaseAfter === expectedRelease; + return { + summary: verified + ? `The ${operation} completed: release ${releaseLabel(job.releaseAfter)} is serving after the adapter's health-checked cutover.` + : `The adapter reported the ${operation} job succeeded but the serving release (${releaseLabel(job.releaseAfter)}) does not match the expected ${releaseLabel(expectedRelease)}.`, + result, + verification: { verified, evidence }, + }; } const HANDLERS: Record = { @@ -375,16 +528,28 @@ const HANDLERS: Record = { }); }, async execute() { - const reconciled = await runStore.reconcileAbandonedRuns(); + const outcome = await runStore.reconcileAbandonedRuns(); + const verified = outcome.verified && outcome.error === undefined && outcome.remaining === 0; return { - summary: - reconciled === 0 - ? 'No abandoned runs were found.' - : `${reconciled} abandoned runs were moved to NEEDS_REVIEW.`, - result: { reconciled }, + summary: outcome.error !== undefined + ? `Reconciliation stopped on a storage error after ${outcome.reconciled} runs.` + : (outcome.remaining ?? 0) > 0 + ? `${outcome.reconciled} abandoned runs were moved to NEEDS_REVIEW; ${outcome.remaining} remain.` + : outcome.reconciled === 0 + ? 'No abandoned runs were found.' + : `${outcome.reconciled} abandoned runs were moved to NEEDS_REVIEW.`, + result: { + reconciled: outcome.reconciled, + remaining: outcome.remaining, + storageError: outcome.error ?? null, + }, verification: { - verified: true, - evidence: ['Every run with an expired lease now carries a terminal state.'], + verified, + evidence: verified + ? ['The expired-lease query answered and no run with an expired lease remains.'] + : outcome.error !== undefined + ? [`The run store answered with an error: ${outcome.error}`] + : [`${outcome.remaining} runs with an expired lease still remain.`], }, }; }, @@ -484,18 +649,62 @@ const HANDLERS: Record = { redactedInput: { task }, }); }, - async execute(context) { + async execute(context): Promise { const task = String(context.input.task ?? ''); if (!(ALLOWED_INDEXER_TASKS as readonly string[]).includes(task)) { throw new Error(`Only ${ALLOWED_INDEXER_TASKS.join(' and ')} can be run this way.`); } - await indexer.runSingleTask(task as AllowedIndexerTask); + const outcome = await indexer.runSingleTask(task as AllowedIndexerTask); + if (outcome.status === 'failed') { + throw new Error(`The ${task} task failed: ${outcome.error ?? 'unknown error'}`); + } + if (outcome.status !== 'completed') { + return { + summary: `The ${task} task did not run: ${outcome.reason ?? outcome.status}.`, + result: { task, status: outcome.status, reason: outcome.reason ?? null }, + verification: { + verified: false, + evidence: [`The indexer answered ${outcome.status}: ${outcome.reason ?? 'no reason given'}.`], + }, + }; + } + let checkpoint: Awaited>; + try { + checkpoint = await readIndexerCheckpoint(task as AllowedIndexerTask); + } catch (e) { + return { + summary: `The ${task} task returned, but its persisted checkpoint could not be read.`, + result: { + task, + status: outcome.status, + startedAt: outcome.checkpoint?.startedAt ?? null, + finishedAt: outcome.checkpoint?.finishedAt ?? null, + checkpointError: e instanceof Error ? e.message : String(e), + }, + verification: { + verified: false, + evidence: ['The checkpoint readback failed, so completion is unproven.'], + }, + }; + } + const verified = checkpoint.remaining === 0; return { - summary: `The ${task} task ran to completion.`, - result: { task }, + summary: verified + ? `The ${task} task completed and no block is left without its ${task} data.` + : `The ${task} task returned but ${checkpoint.remaining} blocks are still missing ${task} data.`, + result: { + task, + status: outcome.status, + startedAt: outcome.checkpoint?.startedAt ?? null, + finishedAt: outcome.checkpoint?.finishedAt ?? null, + remaining: checkpoint.remaining, + indexedTip: checkpoint.indexedTip, + }, verification: { - verified: true, - evidence: [`The indexer returned from ${task} without throwing.`], + verified, + evidence: verified + ? [`The database reports 0 blocks missing ${task} data; the newest indexed height is ${checkpoint.indexedTip ?? 'unknown'}.`] + : [`The database still reports ${checkpoint.remaining} blocks missing ${task} data.`], }, }; }, @@ -530,13 +739,16 @@ const HANDLERS: Record = { if (String(context.input.confirmation ?? '') !== 'REINDEX EXPLORER BLOCKS') { throw new Error('This operation requires the exact typed confirmation.'); } - indexer.reindex(); + const scheduled = indexer.reindex(); + if (!scheduled) { + throw new Error('The indexer refused the request because indexing is disabled in this deployment.'); + } return { - summary: 'The indexing loop was released to run again.', - result: { scheduled: true }, + summary: 'The indexing loop was released to run again. Indexing itself runs in the background and is not complete.', + result: { scheduled, indexingCompleted: false }, verification: { verified: true, - evidence: ['The indexer accepted the request to run again.'], + evidence: ['The indexer accepted the request to run again; this proves scheduling, not completion.'], }, }; }, @@ -544,39 +756,67 @@ const HANDLERS: Record = { 'explorer.service.restart': { async buildPreview() { + const control = await probeControl(); + const current = control.state === 'ready' ? control.capabilities.currentRelease : null; return preview({ target: target('service'), - preconditions: [deploymentControlPrecondition()], - effects: ['Stops and starts the Explorer backend process.'], - expectedPostconditions: ['The Explorer answers its capability route again from a new process.'], - warnings: ['Every open WebSocket stream is dropped and has to reconnect.'], + preconditions: [deploymentControlPrecondition('explorer.service.restart', control)], + effects: [ + `Asks the deployment adapter to run the release tooling's health-checked cutover of release ${releaseLabel(current)}, restarting the backend and overlay units.`, + 'The run records the adapter job id before the restart replaces this process; the job is then polled to its terminal state.', + ], + expectedPostconditions: [`The adapter job succeeds and reports release ${releaseLabel(current)} serving from a new process.`], + warnings: [ + 'Every open WebSocket stream is dropped and has to reconnect.', + 'This process is the one being restarted: if the run record shows the job still running, read the job back from the adapter after the restart.', + ], reversible: false, users: 'Every open WebSocket stream is dropped and reconnects.', - publicServices: 'The Explorer is unreachable for the length of the restart.', + publicServices: + 'The gateway keeps the public origin up while the backend and overlay restart; API routes answer errors until the new process passes the live verification.', + durationSeconds: 120, }); }, - async execute() { - // No shell, no unit name, no fallback. Until an operator wires the - // deployment adapter, this refuses with the exact unmet precondition. - throw new Error(DEPLOYMENT_CONTROL_REASON); + /** @asyncUnsafe The route turns a rejection into a FAILED run. */ + async execute(context) { + if (String(context.input.confirmation ?? '') !== 'RESTART EXPLORER BACKEND') { + throw new Error('This operation requires the exact typed confirmation.'); + } + // No shell, no unit name, no fallback: one signed request to the adapter. + return driveDeployment('restart', 'explorer.service.restart', context, 600); }, }, 'explorer.release.rollback': { async buildPreview() { + const control = await probeControl(); + const capabilities = control.state === 'ready' ? control.capabilities : null; + const journalTarget = capabilities ? rollbackTargetFromJournal(capabilities) : null; return preview({ target: target('release'), - preconditions: [deploymentControlPrecondition()], - effects: ['Replaces the running Explorer release with the previous verified one.'], - expectedPostconditions: ['The Explorer reports the previous verified release sha.'], + preconditions: [deploymentControlPrecondition('explorer.release.rollback', control)], + effects: [ + `Asks the deployment adapter to roll back from release ${releaseLabel(capabilities?.currentRelease ?? null)} to ${releaseLabel(journalTarget?.release ?? null)}, the previous release its journal verified${journalTarget ? ' at ' + journalTarget.verifiedAt : ''}.`, + 'The rollback runs through the release tooling and passes the live verification before it is reported done.', + ], + expectedPostconditions: [`The adapter job succeeds and reports release ${releaseLabel(journalTarget?.release ?? null)} serving.`], warnings: [ 'Rolling back healthy code because of a dependency incident makes things worse, not better.', + 'This process is replaced by the rollback: if the run record shows the job still running, read the job back from the adapter afterwards.', ], reversible: false, + users: 'The site serves the previous build; every open WebSocket stream reconnects.', + publicServices: + 'The gateway keeps the public origin up while the units restart on the previous release; API routes answer errors until it passes the live verification.', + durationSeconds: 300, }); }, - async execute() { - throw new Error(DEPLOYMENT_CONTROL_REASON); + /** @asyncUnsafe The route turns a rejection into a FAILED run. */ + async execute(context) { + if (String(context.input.confirmation ?? '') !== 'ROLL BACK EXPLORER RELEASE') { + throw new Error('This operation requires the exact typed confirmation.'); + } + return driveDeployment('rollback', 'explorer.release.rollback', context, 900); }, }, }; diff --git a/backend/src/api/admin-adapter/admin-adapter.replay.ts b/backend/src/api/admin-adapter/admin-adapter.replay.ts new file mode 100644 index 00000000000..c01baedf8f1 --- /dev/null +++ b/backend/src/api/admin-adapter/admin-adapter.replay.ts @@ -0,0 +1,127 @@ +import config from '../../config'; +import DB from '../../database'; +import logger from '../../logger'; +import redisCache from '../redis-cache'; +import { + ADMIN_NONCE_RETENTION_MS, + AdminAdapterNonceStore, + type AdminReplayClaim, + type AdminReplayStore, + UnavailableAdminReplayStore, +} from './admin-adapter.security'; + +/** + * The shared replay stores behind the admin guard. + * + * A nonce is claimed once, for the whole freshness window, in a store every + * backend worker shares. The MySQL store is preferred because the adapter + * already needs the database for durable runs; Redis is the shared store when + * the database is off; with neither the guard fails closed. The in-memory + * store is only honoured on an explicit single-process opt-in, and is logged + * as such, because it cannot stop a replay against a second process. + */ + +export const ADMIN_REPLAY_STORE_ENVIRONMENT_VARIABLE = 'EXPLORER_ADMIN_REPLAY_STORE'; + +/** How often at most the MySQL store sweeps expired rows. */ +const SWEEP_INTERVAL_MS = 60_000; + +function isDuplicateKey(e: unknown): boolean { + const error = e as { code?: string; errno?: number } | null; + return error?.code === 'ER_DUP_ENTRY' || error?.errno === 1062; +} + +/** One row per claimed nonce in the owned database. */ +export class MysqlAdminReplayStore implements AdminReplayStore { + readonly kind = 'database'; + private lastSweepMs = 0; + + /** @asyncUnsafe Every rejection is turned into an unavailable claim. */ + async claim(scope: string, nonce: string, nowMs: number): Promise { + const now = new Date(nowMs); + const expiresAt = new Date(nowMs + ADMIN_NONCE_RETENTION_MS); + try { + if (nowMs - this.lastSweepMs >= SWEEP_INTERVAL_MS) { + this.lastSweepMs = nowMs; + // Only rows past their retention go; a live nonce is never removed. + await DB.query('DELETE FROM admin_nonces WHERE expires_at <= ?', [now]); + } + try { + await DB.query('INSERT INTO admin_nonces (scope, nonce, expires_at) VALUES (?, ?, ?)', [ + scope, + nonce, + expiresAt, + ]); + return 'claimed'; + } catch (e) { + if (!isDuplicateKey(e)) { + throw e; + } + // The row exists. It may only be reused once its retention ended; + // the update is atomic on that condition. + const [result]: any[] = await DB.query( + 'UPDATE admin_nonces SET expires_at = ? WHERE scope = ? AND nonce = ? AND expires_at <= ?', + [expiresAt, scope, nonce, now], + ); + return Number(result?.affectedRows ?? 0) === 1 ? 'claimed' : 'replayed'; + } + } catch (e) { + logger.warn('[admin-adapter] The database replay store could not claim a nonce: ' + (e instanceof Error ? e.message : e)); + return 'unavailable'; + } + } +} + +/** SET NX PX in the shared Redis, keyed by scope and nonce. */ +export class RedisAdminReplayStore implements AdminReplayStore { + readonly kind = 'redis'; + + /** @asyncUnsafe Every rejection is turned into an unavailable claim. */ + async claim(scope: string, nonce: string, _nowMs: number): Promise { + try { + const claimed = await redisCache.$claimOnce(`admin-nonce:${scope}:${nonce}`, ADMIN_NONCE_RETENTION_MS); + return claimed ? 'claimed' : 'replayed'; + } catch (e) { + logger.warn('[admin-adapter] The Redis replay store could not claim a nonce: ' + (e instanceof Error ? e.message : e)); + return 'unavailable'; + } + } +} + +/** + * Picks the store for this process and logs the choice once. The environment + * variable is an override for operators; unset, the choice follows what is + * already enabled. `memory` is never chosen implicitly. + */ +export function createAdminReplayStore( + environment: Record = process.env, + enabled: { database: boolean; redis: boolean } = { + database: config.DATABASE.ENABLED === true, + redis: config.REDIS.ENABLED === true, + }, +): AdminReplayStore { + const requested = String(environment[ADMIN_REPLAY_STORE_ENVIRONMENT_VARIABLE] ?? '').trim().toLowerCase(); + let store: AdminReplayStore; + if (requested === 'memory') { + logger.warn( + `[admin-adapter] ${ADMIN_REPLAY_STORE_ENVIRONMENT_VARIABLE}=memory: admin replay protection is single-process only and cannot stop a replay against another worker.`, + ); + store = new AdminAdapterNonceStore(); + } else if (requested === 'database' || (requested === '' && enabled.database)) { + store = enabled.database + ? new MysqlAdminReplayStore() + : new UnavailableAdminReplayStore(`${ADMIN_REPLAY_STORE_ENVIRONMENT_VARIABLE}=database was requested but the database is disabled`); + } else if (requested === 'redis' || (requested === '' && enabled.redis)) { + store = enabled.redis + ? new RedisAdminReplayStore() + : new UnavailableAdminReplayStore(`${ADMIN_REPLAY_STORE_ENVIRONMENT_VARIABLE}=redis was requested but Redis is disabled`); + } else if (requested !== '') { + store = new UnavailableAdminReplayStore(`${ADMIN_REPLAY_STORE_ENVIRONMENT_VARIABLE}=${requested} is not a known replay store`); + } else { + store = new UnavailableAdminReplayStore( + `neither the database nor Redis is enabled, so no shared replay store exists (set ${ADMIN_REPLAY_STORE_ENVIRONMENT_VARIABLE}=memory only for a single-process deployment)`, + ); + } + logger.info(`[admin-adapter] Admin replay store: ${store.kind}.`); + return store; +} diff --git a/backend/src/api/admin-adapter/admin-adapter.routes.ts b/backend/src/api/admin-adapter/admin-adapter.routes.ts index 52d24cdac5d..a17bbf91b00 100644 --- a/backend/src/api/admin-adapter/admin-adapter.routes.ts +++ b/backend/src/api/admin-adapter/admin-adapter.routes.ts @@ -13,6 +13,7 @@ import blocks from '../blocks'; import capabilities from '../capabilities'; import memPool from '../mempool'; import { adminAdapterGuard, adminAdapterJsonParser, hasSignedAdminElevation } from './admin-adapter.security'; +import { createAdminReplayStore } from './admin-adapter.replay'; import { adminEnvelope, adminTimestamp, @@ -30,6 +31,7 @@ import { listExplorerOperations, } from './admin-adapter.operations'; import runStore, { AdminRunConflict, AdminRunNotFound } from './admin-adapter.runs'; +import { resumeDeploymentRuns } from './deployment-control.resume'; const { ADMIN_CONTROL_SUPPORTED_VERSIONS, @@ -214,7 +216,7 @@ class AdminAdapterRoutes { public initRoutes(app: Application): void { // Also support standalone registration; the full app captures these bytes // before its general parsers consume the request stream. - app.use(PREFIX, adminAdapterJsonParser(), adminAdapterGuard()); + app.use(PREFIX, adminAdapterJsonParser(), adminAdapterGuard(createAdminReplayStore())); app.get(`${PREFIX}/manifest`, async (_request: Request, response: Response) => { try { @@ -376,7 +378,7 @@ class AdminAdapterRoutes { const correlationId = String(request.headers['x-bu-admin-correlation-id'] ?? '').slice(0, 128) || randomUUID(); const actor = String(request.headers['x-bu-admin-actor'] ?? '').slice(0, 200) || 'control-center'; - const { run, replayed } = await runStore.create({ + const { run, replayed, ownerToken } = await runStore.create({ operationId: operation.id, operationVersion: operation.version, target: previewBody.target, @@ -388,29 +390,42 @@ class AdminAdapterRoutes { rollbackSupported: operation.rollbackSupported, redactedInput: input, }); - if (replayed) { + if (replayed || ownerToken === null) { response.json({ ...adminEnvelope(), run }); return; } if (operation.lock) { - await runStore.acquireLock(`${operation.lock}:${previewBody.target}`, run.runId); + try { + await runStore.acquireLock(`${operation.lock}:${previewBody.target}`, run.runId); + } catch (e) { + // The run was already inserted QUEUED. Leave a terminal, inspectable + // record rather than a row that waits forever for a lock it lost. + const message = e instanceof Error ? e.message : String(e); + await runStore.transition(run.runId, 'FAILED', { + error: { class: 'lock_conflict', message: message.slice(0, 700), retryable: true }, + logs: [{ at: adminTimestamp(), level: 'error', message: message.slice(0, 1000) }], + }, ownerToken); + throw e instanceof AdminRunConflict + ? new AdminRunConflict(`${message} Run ${run.runId} was recorded as FAILED.`) + : e; + } } await runStore.transition(run.runId, 'PRECHECK', { logs: [{ at: adminTimestamp(), level: 'info', message: 'Preconditions verified.' }], - }); - await runStore.transition(run.runId, 'RUNNING', { progressPercent: 10 }); + }, ownerToken); + await runStore.transition(run.runId, 'RUNNING', { progressPercent: 10 }, ownerToken); try { - const outcome = await operation.execute({ + const outcome = await runStore.withHeartbeat(run.runId, ownerToken, () => operation.execute({ runId: run.runId, correlationId, actor, reason: run.reason, idempotencyKey: run.idempotencyKey, input, - }); - await runStore.transition(run.runId, 'VERIFYING', { progressPercent: 90 }); + })); + await runStore.transition(run.runId, 'VERIFYING', { progressPercent: 90 }, ownerToken); const finished = await runStore.transition( run.runId, outcome.verification.verified ? 'SUCCEEDED' : 'NEEDS_REVIEW', @@ -432,6 +447,7 @@ class AdminAdapterRoutes { ]), ], }, + ownerToken, ); response.json({ ...adminEnvelope(), run: finished }); } catch (e) { @@ -439,7 +455,7 @@ class AdminAdapterRoutes { const failed = await runStore.transition(run.runId, 'FAILED', { error: { class: 'operation_failed', message: message.slice(0, 700), retryable: true }, logs: [{ at: adminTimestamp(), level: 'error', message: message.slice(0, 1000) }], - }); + }, ownerToken); response.json({ ...adminEnvelope(), run: failed }); } } catch (e) { @@ -585,6 +601,14 @@ class AdminAdapterRoutes { logger.info( `[admin-adapter] Explorer admin adapter mounted at ${PREFIX} (database ${config.DATABASE.ENABLED ? 'enabled' : 'disabled'}).`, ); + // A restart run is cut short by the restart it asked for; the process + // that comes up closes it from the adapter job it recorded. + // (The boundary tests load this module in a bare VM without timers.) + if (typeof setTimeout === 'function') { + setTimeout(() => { + resumeDeploymentRuns().catch((e) => logger.warn('[admin-adapter] deployment run resume failed: ' + (e instanceof Error ? e.message : String(e)))); + }, 5_000).unref(); + } } } diff --git a/backend/src/api/admin-adapter/admin-adapter.runs.ts b/backend/src/api/admin-adapter/admin-adapter.runs.ts index 7b55bf3fcce..ee84e1c25fc 100644 --- a/backend/src/api/admin-adapter/admin-adapter.runs.ts +++ b/backend/src/api/admin-adapter/admin-adapter.runs.ts @@ -9,10 +9,43 @@ import { adminTimestamp, optionalAdminTimestamp } from './admin-adapter.identity const { isAdminRunTerminal, isAdminRunTransitionAllowed, redactAdminPayload } = adminControl as typeof import('@bitcoinuniverse/ecosystem-contracts/admin-control'); -/** How long a running operation may go silent before it is abandoned. */ -const LEASE_MS = 120_000; -/** How long an advisory lock is held before another run may reclaim it. */ +/** + * How long a run may go silent before it is abandoned. + * + * The catalog allows operations of up to 900 seconds, so the lease is not + * sized to the operation. It is sized to the heartbeat: a live executor renews + * it every HEARTBEAT_MS (well under LEASE_MS) for as long as its work runs, so + * an abandoned run is noticed within LEASE_MS of the last heartbeat however + * long the operation was allowed to take. A run inserted QUEUED gets the same + * initial lease so a crash before the first transition is recovered too. + */ +export const LEASE_MS = 120_000; +/** How often a live executor renews its lease and lock. */ +export const HEARTBEAT_MS = 30_000; +/** How long an advisory lock is held before another run may reclaim it. Renewed by the heartbeat. */ const LOCK_MS = 15 * 60_000; +/** How many expired runs one reconciliation pass handles per query. */ +const RECONCILE_BATCH = 200; +/** How many batches one reconciliation pass may process before it reports what remains. */ +const RECONCILE_MAX_BATCHES = 25; +/** States that are not terminal. Kept in one place so every query agrees. */ +const OPEN_STATES_SQL = "state NOT IN ('SUCCEEDED','FAILED','CANCELLED','NEEDS_REVIEW','ROLLED_BACK','ROLLBACK_FAILED')"; +/** + * A run counts as abandoned when its lease expired, or, for rows written before + * the initial lease existed, when it has sat QUEUED for longer than a lease. + */ +const ABANDONED_SQL = `${OPEN_STATES_SQL} AND ((lease_expires_at IS NOT NULL AND lease_expires_at <= ?) OR (lease_expires_at IS NULL AND state = 'QUEUED' AND queued_at <= ?))`; + +export interface ReconcileResult { + /** Runs moved to a terminal state by this pass. */ + reconciled: number; + /** Runs with an expired lease still open after this pass; null when the count could not be read. */ + remaining: number | null; + /** True only when every query answered and no expired run remains. */ + verified: boolean; + /** The storage failure that stopped the pass, when one did. */ + error?: string; +} export class AdminRunNotFound extends Error {} export class AdminRunConflict extends Error {} @@ -25,6 +58,8 @@ interface RunDocument { error: AdminRun['error']; rollback: AdminRun['rollback']; redactedInput: Record; + /** Persisted at creation from the catalog. Absent on older rows, which reads as false. */ + cancellable?: boolean; } function emptyDocument(redactedInput: Record = {}): RunDocument { @@ -36,6 +71,7 @@ function emptyDocument(redactedInput: Record = {}): RunDocument error: null, rollback: { supported: false, state: null }, redactedInput, + cancellable: false, }; } @@ -53,7 +89,7 @@ function parseDocument(value: unknown): RunDocument { return emptyDocument(); } -function toRun(row: any): AdminRun { +export function toRun(row: any): AdminRun { const document = parseDocument(row.document); return { runId: row.run_id, @@ -71,7 +107,9 @@ function toRun(row: any): AdminRun { finishedAt: optionalAdminTimestamp(row.finished_at), heartbeatAt: optionalAdminTimestamp(row.heartbeat_at), progressPercent: row.progress_percent === null ? null : Number(row.progress_percent), - cancellable: Number(row.cancel_requested) === 0 && !isAdminRunTerminal(row.state), + // Only what the catalog declared at creation, never inferred from the + // absence of a cancel request. Older rows without the field read as false. + cancellable: document.cancellable === true && Number(row.cancel_requested) === 0 && !isAdminRunTerminal(row.state), replayed: false, steps: document.steps, logs: document.logs, @@ -118,22 +156,24 @@ class AdminAdapterRunStore { cancellable: boolean; rollbackSupported: boolean; redactedInput: Record; - }): Promise<{ run: AdminRun; replayed: boolean }> { + }): Promise<{ run: AdminRun; replayed: boolean; ownerToken: string | null }> { this.assertAvailable(); if (input.idempotencyKey) { const existing = await this.findByIdempotencyKey(input.operationId, input.idempotencyKey); if (existing) { - return { run: { ...existing, replayed: true }, replayed: true }; + return { run: { ...existing, replayed: true }, replayed: true, ownerToken: null }; } } const runId = randomUUID(); + const ownerToken = randomUUID(); const now = new Date(); const document = emptyDocument( redactAdminPayload(input.redactedInput) as Record, ); document.rollback = { supported: input.rollbackSupported, state: null }; + document.cancellable = input.cancellable === true; await DB.query( - 'INSERT INTO admin_adapter_runs (run_id, operation_id, operation_version, state, target, correlation_id, idempotency_key, actor, reason, queued_at, updated_at, cancel_requested, document) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?)', + 'INSERT INTO admin_adapter_runs (run_id, operation_id, operation_version, state, target, correlation_id, idempotency_key, actor, reason, queued_at, updated_at, lease_expires_at, owner_token, cancel_requested, document) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?)', [ runId, input.operationId, @@ -146,11 +186,13 @@ class AdminAdapterRunStore { input.reason, now, now, + new Date(now.getTime() + LEASE_MS), + ownerToken, JSON.stringify(document), ], ); const run = await this.get(runId); - return { run: { ...run, cancellable: input.cancellable }, replayed: false }; + return { run, replayed: false, ownerToken }; } /** @asyncUnsafe Callers turn a rejection into an exact HTTP answer. */ @@ -177,11 +219,19 @@ class AdminAdapterRunStore { return rows[0] ? toRun(rows[0]) : null; } - /** @asyncUnsafe Callers turn a rejection into an exact HTTP answer. */ + /** + * Moves a run to its next state. With an owner token the update is a + * compare-and-set on run id, the state just read and the token, so an + * executor whose run was reconciled away underneath it gets a conflict + * instead of overwriting the reviewer's record. + * + * @asyncUnsafe Callers turn a rejection into an exact HTTP answer. + */ async transition( runId: string, next: AdminRunState, patch: Partial & { progressPercent?: number | null } = {}, + ownerToken?: string, ): Promise { this.assertAvailable(); const current = await this.get(runId); @@ -201,8 +251,9 @@ class AdminAdapterRunStore { const now = new Date(); const terminal = isAdminRunTerminal(next); - await DB.query( - 'UPDATE admin_adapter_runs SET state = ?, updated_at = ?, started_at = COALESCE(started_at, ?), finished_at = ?, heartbeat_at = ?, lease_expires_at = ?, progress_percent = ?, document = ? WHERE run_id = ?', + const guard = ownerToken === undefined ? '' : ' AND state = ? AND owner_token = ?'; + const [result]: any[] = await DB.query( + `UPDATE admin_adapter_runs SET state = ?, updated_at = ?, started_at = COALESCE(started_at, ?), finished_at = ?, heartbeat_at = ?, lease_expires_at = ?, progress_percent = ?, document = ? WHERE run_id = ?${guard}`, [ next, now, @@ -213,14 +264,70 @@ class AdminAdapterRunStore { patch.progressPercent === undefined ? current.progressPercent : patch.progressPercent, JSON.stringify(document), runId, + ...(ownerToken === undefined ? [] : [current.state, ownerToken]), ], ); + if (ownerToken !== undefined && Number(result?.affectedRows ?? 0) !== 1) { + throw new AdminRunConflict( + `Run ${runId} changed underneath its executor (expected ${current.state}); its record was left as found.`, + ); + } if (terminal) { await DB.query('DELETE FROM admin_adapter_locks WHERE run_id = ?', [runId]); } return this.get(runId); } + /** + * Renews the lease and lock of a RUNNING run, but only for the executor that + * owns it. Returns false when the run is no longer RUNNING under this token, + * which means a reconciler or another actor already took it over. + * + * @asyncUnsafe The heartbeat loop reports a rejection and keeps going. + */ + async heartbeat(runId: string, ownerToken: string): Promise { + this.assertAvailable(); + const now = new Date(); + const [result]: any[] = await DB.query( + "UPDATE admin_adapter_runs SET heartbeat_at = ?, updated_at = ?, lease_expires_at = ? WHERE run_id = ? AND state = 'RUNNING' AND owner_token = ?", + [now, now, new Date(now.getTime() + LEASE_MS), runId, ownerToken], + ); + if (Number(result?.affectedRows ?? 0) !== 1) { + return false; + } + await DB.query('UPDATE admin_adapter_locks SET expires_at = ? WHERE run_id = ?', [ + new Date(now.getTime() + LOCK_MS), + runId, + ]); + return true; + } + + /** + * Runs the operation while renewing its lease every HEARTBEAT_MS. The + * rejection of the work itself is passed through untouched. + * + * @asyncUnsafe The caller turns a rejection of the work into a FAILED run. + */ + async withHeartbeat(runId: string, ownerToken: string, work: () => Promise): Promise { + const timer = setInterval(() => { + this.heartbeat(runId, ownerToken) + .then((alive) => { + if (!alive) { + logger.warn(`[admin-adapter] Run ${runId} lost its lease while its operation was still running.`); + } + }) + .catch((e) => { + logger.warn(`[admin-adapter] Run ${runId} could not renew its lease: ` + (e instanceof Error ? e.message : e)); + }); + }, HEARTBEAT_MS); + timer.unref?.(); + try { + return await work(); + } finally { + clearInterval(timer); + } + } + /** @asyncUnsafe Callers turn a rejection into an exact HTTP answer. */ async requestCancel(runId: string): Promise { const run = await this.get(runId); @@ -267,32 +374,83 @@ class AdminAdapterRunStore { } } - /** @asyncSafe */ - async reconcileAbandonedRuns(): Promise { + /** + * Moves runs whose lease expired to a terminal state, in bounded batches. + * + * A run that never started (QUEUED or PRECHECK) becomes FAILED, the only + * terminal state the contract allows from there; a run that was in flight + * becomes NEEDS_REVIEW because nobody can prove what it did. Each update + * re-checks the lease, so a run whose executor heartbeats between the read + * and the write is left alone and keeps its lock. + * + * @asyncSafe Every storage failure is returned in the result, never thrown. + */ + async reconcileAbandonedRuns(): Promise { if (!this.available()) { - return 0; + return { reconciled: 0, remaining: null, verified: false, error: 'The explorer database is switched off.' }; } + let reconciled = 0; try { - const [rows]: any[] = await DB.query( - "SELECT run_id FROM admin_adapter_runs WHERE lease_expires_at IS NOT NULL AND lease_expires_at <= ? AND state NOT IN ('SUCCEEDED','FAILED','CANCELLED','NEEDS_REVIEW','ROLLED_BACK','ROLLBACK_FAILED') LIMIT 200", - [new Date()], - ); - for (const row of rows) { - await DB.query( - "UPDATE admin_adapter_runs SET state = 'NEEDS_REVIEW', updated_at = ?, finished_at = ?, lease_expires_at = NULL WHERE run_id = ?", - [new Date(), new Date(), row.run_id], + for (let batch = 0; batch < RECONCILE_MAX_BATCHES; batch++) { + const now = new Date(); + const [rows]: any[] = await DB.query( + `SELECT run_id, state FROM admin_adapter_runs WHERE ${ABANDONED_SQL} LIMIT ${RECONCILE_BATCH}`, + [now, new Date(now.getTime() - LEASE_MS)], ); - await DB.query('DELETE FROM admin_adapter_locks WHERE run_id = ?', [row.run_id]); + if (!Array.isArray(rows) || rows.length === 0) { + break; + } + let movedInBatch = 0; + for (const row of rows) { + const at = new Date(); + const neverStarted = row.state === 'QUEUED' || row.state === 'PRECHECK'; + const message = neverStarted + ? 'The run was abandoned before its operation started; nothing was executed.' + : 'The run went silent while its operation was in flight; the outcome is unknown.'; + const [result]: any[] = await DB.query( + `UPDATE admin_adapter_runs SET state = ?, updated_at = ?, finished_at = ?, lease_expires_at = NULL, document = JSON_SET(document, '$.error', JSON_OBJECT('class', ?, 'message', ?, 'retryable', ?)) WHERE run_id = ? AND ${ABANDONED_SQL}`, + [ + neverStarted ? 'FAILED' : 'NEEDS_REVIEW', + at, + at, + neverStarted ? 'abandoned_before_start' : 'abandoned_in_flight', + message, + neverStarted, + row.run_id, + at, + new Date(at.getTime() - LEASE_MS), + ], + ); + if (Number(result?.affectedRows ?? 0) === 1) { + await DB.query('DELETE FROM admin_adapter_locks WHERE run_id = ?', [row.run_id]); + movedInBatch++; + } + } + reconciled += movedInBatch; + if (rows.length < RECONCILE_BATCH) { + break; + } + if (movedInBatch === 0) { + // Every row in this batch was renewed under us; stop rather than spin. + break; + } } - if (rows.length > 0) { + const now = new Date(); + const [countRows]: any[] = await DB.query( + `SELECT COUNT(*) AS remaining FROM admin_adapter_runs WHERE ${ABANDONED_SQL}`, + [now, new Date(now.getTime() - LEASE_MS)], + ); + const remaining = Number(countRows?.[0]?.remaining ?? 0); + if (reconciled > 0) { logger.warn( - `[admin-adapter] Moved ${rows.length} abandoned operation runs to NEEDS_REVIEW after their lease expired.`, + `[admin-adapter] Moved ${reconciled} abandoned operation runs to a terminal state after their lease expired${remaining > 0 ? `; ${remaining} remain` : ''}.`, ); } - return rows.length; + return { reconciled, remaining, verified: remaining === 0 }; } catch (e) { - logger.debug('[admin-adapter] Could not reconcile abandoned runs: ' + (e instanceof Error ? e.message : e)); - return 0; + const error = e instanceof Error ? e.message : String(e); + logger.warn(`[admin-adapter] Could not reconcile abandoned runs after ${reconciled}: ` + error); + return { reconciled, remaining: null, verified: false, error }; } } diff --git a/backend/src/api/admin-adapter/admin-adapter.security.ts b/backend/src/api/admin-adapter/admin-adapter.security.ts index 412a6b9df27..ed04a42b31a 100644 --- a/backend/src/api/admin-adapter/admin-adapter.security.ts +++ b/backend/src/api/admin-adapter/admin-adapter.security.ts @@ -77,41 +77,91 @@ export function isPrivateRemoteAddress(address: string): boolean { return /^f[cd][0-9a-f]{2}:/.test(value) || value.startsWith('fe80:'); } +/** How long a claimed nonce has to stay claimed: the whole freshness window. */ +export const ADMIN_NONCE_RETENTION_MS = ADMIN_SERVICE_NONCE_TTL_SECONDS * 1000; + +/** + * The answer of a replay store. `unavailable` is the fail-closed answer: the + * request is refused because nobody can say whether the nonce was seen. + */ +export type AdminReplayClaim = 'claimed' | 'replayed' | 'unavailable'; + +/** + * A shared, atomic claim on a nonce. Every worker that verifies admin requests + * must consult the same store, otherwise a signed request replays cleanly + * against the next process. + */ +export interface AdminReplayStore { + readonly kind: string; + claim(scope: string, nonce: string, nowMs: number): Promise; +} + /** - * Remembers nonces for exactly as long as a timestamp can stay fresh, so a - * replay inside the window is refused and memory stays bounded. + * Remembers nonces in this process for exactly as long as a timestamp can + * stay fresh. Unexpired nonces are never evicted: a store that forgot a live + * nonce to make room would reopen the replay it exists to close, so when the + * bound is reached new nonces are refused instead. + * + * Single-process only. Production selects a shared store; this one is for + * unit tests and for an explicit single-process opt-in. */ -export class AdminAdapterNonceStore { +export class AdminAdapterNonceStore implements AdminReplayStore { + readonly kind = 'memory'; private seen = new Map(); constructor(private maxEntries = 20_000) {} - accept(nonce: string, nowMs: number): boolean { + /** Synchronous acceptance for the pure verifier. Nonces are keyed by scope. */ + accept(nonce: string, nowMs: number, scope = ''): boolean { this.evict(nowMs); - if (this.seen.has(nonce)) { + const key = `${scope}\n${nonce}`; + if (this.seen.has(key)) { return false; } - this.seen.set(nonce, nowMs); - if (this.seen.size > this.maxEntries) { - const oldest = this.seen.keys().next(); - if (!oldest.done) { - this.seen.delete(oldest.value); - } + if (this.seen.size >= this.maxEntries) { + return false; } + this.seen.set(key, nowMs); return true; } + claim(scope: string, nonce: string, nowMs: number): Promise { + return Promise.resolve(this.accept(nonce, nowMs, scope) ? 'claimed' : 'replayed'); + } + + /** Only expired entries are removed, never a live one. */ private evict(nowMs: number): void { - const horizon = nowMs - ADMIN_SERVICE_NONCE_TTL_SECONDS * 1000; - for (const [nonce, at] of this.seen) { + const horizon = nowMs - ADMIN_NONCE_RETENTION_MS; + for (const [key, at] of this.seen) { if (at >= horizon) { break; } - this.seen.delete(nonce); + this.seen.delete(key); + } + } +} + +/** The fail-closed store: every claim is unavailable, with the reason logged once. */ +export class UnavailableAdminReplayStore implements AdminReplayStore { + readonly kind = 'unavailable'; + private logged = false; + + constructor(private reason: string) {} + + claim(): Promise { + if (!this.logged) { + this.logged = true; + logger.err(`[admin-adapter] Admin requests are refused: ${this.reason}`); } + return Promise.resolve('unavailable'); } } +/** The key/version pair a nonce is claimed under, so keys cannot collide. */ +export function adminNonceScope(keyId: string, contractVersion: string): string { + return `${keyId}@${contractVersion}`; +} + function header(request: Request, name: string): string { const value = request.headers[name]; if (Array.isArray(value)) { @@ -130,15 +180,12 @@ export type AdminAdapterVerdict = | { ok: true; authorization: { keyId: string; elevated: boolean } } | { ok: false; status: number; code: string; message: string; reason: string }; -const nonces = new AdminAdapterNonceStore(); +/** A verified signature whose nonce has not been claimed yet. */ +export type AdminAdapterSignatureVerdict = + | { ok: true; authorization: { keyId: string; elevated: boolean }; nonce: string; scope: string } + | { ok: false; status: number; code: string; message: string; reason: string }; -/** - * The whole verification, expressed as a pure-ish function so it can be tested - * without an HTTP server. `rawBody` is the exact bytes received, because - * re-serialising a parsed object would change whitespace and key order and - * break every signature. - */ -export function verifyAdminAdapterRequest(input: { +export interface AdminAdapterRequestInput { method: string; originalUrl: string; headers: Record; @@ -146,9 +193,64 @@ export function verifyAdminAdapterRequest(input: { remoteAddress: string; keys?: AdminAdapterKey[]; nowMs?: number; - nonceStore?: AdminAdapterNonceStore; +} + +/** + * The whole verification, expressed as a pure-ish function so it can be tested + * without an HTTP server. `rawBody` is the exact bytes received, because + * re-serialising a parsed object would change whitespace and key order and + * break every signature. + * + * This variant claims the nonce in the given single-process store and exists + * for unit tests and the isolated harness. The guard uses + * `verifyAdminAdapterRequestWithStore` against the shared store. + */ +export function verifyAdminAdapterRequest(input: AdminAdapterRequestInput & { + nonceStore: AdminAdapterNonceStore; }): AdminAdapterVerdict { - const rejected = (reason: string): AdminAdapterVerdict => ({ + const verdict = checkAdminAdapterSignature(input); + if (!verdict.ok) { + return verdict; + } + if (!input.nonceStore.accept(verdict.nonce, input.nowMs ?? Date.now(), verdict.scope)) { + return unauthorized('A signed service request replayed a nonce.'); + } + return { ok: true, authorization: verdict.authorization }; +} + +/** + * The verification the guard runs: every signature check, then one atomic + * claim in the shared replay store. An unavailable store refuses the request + * with its own code so the operator can tell it apart from a bad signature. + * + * @asyncUnsafe The guard catches a rejection and refuses the request. + */ +export async function verifyAdminAdapterRequestWithStore( + input: AdminAdapterRequestInput, + store: AdminReplayStore, +): Promise { + const verdict = checkAdminAdapterSignature(input); + if (!verdict.ok) { + return verdict; + } + const claim = await store.claim(verdict.scope, verdict.nonce, input.nowMs ?? Date.now()); + if (claim === 'replayed') { + return unauthorized('A signed service request replayed a nonce.'); + } + if (claim !== 'claimed') { + return { + ok: false, + status: 503, + code: 'REPLAY_STORE_UNAVAILABLE', + message: 'The Explorer admin adapter cannot verify request freshness right now.', + reason: `A signed service request was refused because the ${store.kind} replay store is unavailable.`, + }; + } + return { ok: true, authorization: verdict.authorization }; +} + +function unauthorized(reason: string): { ok: false; status: number; code: string; message: string; reason: string } { + return { ok: false, status: 401, code: 'UNAUTHORIZED', @@ -156,7 +258,12 @@ export function verifyAdminAdapterRequest(input: { // failed. The operator still gets the reason in the log. message: 'Admin adapter request verification failed.', reason, - }); + }; +} + +/** Every check except the nonce claim. */ +export function checkAdminAdapterSignature(input: AdminAdapterRequestInput): AdminAdapterSignatureVerdict { + const rejected = unauthorized; if (!isPrivateRemoteAddress(input.remoteAddress)) { return { @@ -242,11 +349,6 @@ export function verifyAdminAdapterRequest(input: { return rejected('A signed service request signature did not verify.'); } - const store = input.nonceStore ?? nonces; - if (!store.accept(nonce, input.nowMs ?? Date.now())) { - return rejected('A signed service request replayed a nonce.'); - } - // This declaration is covered by the existing signed body digest. An // unsigned forwarding header cannot upgrade a request's authority. let elevated = false; @@ -258,7 +360,7 @@ export function verifyAdminAdapterRequest(input: { return rejected('A signed admin request did not contain valid JSON.'); } } - return { ok: true, authorization: { keyId, elevated } }; + return { ok: true, authorization: { keyId, elevated }, nonce, scope: adminNonceScope(keyId, contractVersion) }; } /** Mount before any general body parser so signatures retain the exact bytes. */ @@ -291,22 +393,41 @@ export function adminAdapterResponseHeaders(response: Response): void { response.removeHeader('Access-Control-Allow-Headers'); } -export function adminAdapterGuard() { +/** + * The guard. The replay store is injected so the choice of store is made once, + * at mount time, and logged; without one every request fails closed. + */ +export function adminAdapterGuard( + store: AdminReplayStore = new UnavailableAdminReplayStore('no replay store was provided to the admin guard'), +) { return (request: Request, response: Response, next: NextFunction): void => { adminAdapterResponseHeaders(response); - const verdict = verifyAdminAdapterRequest({ + const refuse = (verdict: { status: number; code: string; message: string; reason: string }): void => { + logger.warn(`[admin-adapter] ${verdict.reason}`); + response.status(verdict.status).json({ code: verdict.code, message: verdict.message }); + }; + verifyAdminAdapterRequestWithStore({ method: request.method, originalUrl: request.originalUrl, headers: request.headers as Record, rawBody: (request as Request & { rawBody?: Buffer }).rawBody ?? Buffer.alloc(0), remoteAddress: request.socket.remoteAddress ?? '', - }); - if (verdict.ok) { - response.locals.adminAdapterAuthorization = verdict.authorization; - next(); - return; - } - logger.warn(`[admin-adapter] ${verdict.reason}`); - response.status(verdict.status).json({ code: verdict.code, message: verdict.message }); + }, store) + .then((verdict) => { + if (verdict.ok) { + response.locals.adminAdapterAuthorization = verdict.authorization; + next(); + return; + } + refuse(verdict); + }) + .catch((e) => { + refuse({ + status: 503, + code: 'REPLAY_STORE_UNAVAILABLE', + message: 'The Explorer admin adapter cannot verify request freshness right now.', + reason: 'The replay store threw while claiming a nonce: ' + (e instanceof Error ? e.message : String(e)), + }); + }); }; } diff --git a/backend/src/api/admin-adapter/deployment-control.client.ts b/backend/src/api/admin-adapter/deployment-control.client.ts new file mode 100644 index 00000000000..37f574e21d7 --- /dev/null +++ b/backend/src/api/admin-adapter/deployment-control.client.ts @@ -0,0 +1,349 @@ +import { createHmac, timingSafeEqual } from 'crypto'; +import * as http from 'http'; +import logger from '../../logger'; + +/** + * The owned deployment adapter, as this process sees it. + * + * Restart and rollback are not things a backend can do to itself: the process + * that would carry them out is the one being replaced. They belong to a + * small adapter on the host (scripts/universe/deployment-control-adapter.mjs) + * that wraps the release tooling's health-checked cutover. This client is + * the whole of what the Explorer knows about it. + * + * Contract (every response is JSON): + * + * GET /capabilities -> DeploymentCapabilities + * POST /restart -> DeploymentJobAccepted body: DeploymentRequest + * POST /rollback -> DeploymentJobAccepted body: DeploymentRequest + release + * GET /jobs/:jobId -> DeploymentJob + * + * Every request carries `X-Deployment-Signature: sha256=` where the hex + * is HMAC-SHA256 over `${method}\n${path}\n${body}` with the shared key from + * EXPLORER_DEPLOYMENT_CONTROL_KEY. The endpoint is EXPLORER_DEPLOYMENT_CONTROL_ENDPOINT: + * a loopback http:// URL or the absolute path of a unix socket. Nothing else + * is accepted, so the adapter can never be a remote host. + * + * A rollback target is never a request input. It is the previous verified + * release in the adapter's own journal, read here from the capability + * document and sent back to the adapter, which checks it against the same + * journal. + */ + +export const DEPLOYMENT_CONTROL_ENDPOINT_VARIABLE = 'EXPLORER_DEPLOYMENT_CONTROL_ENDPOINT'; +export const DEPLOYMENT_CONTROL_KEY_VARIABLE = 'EXPLORER_DEPLOYMENT_CONTROL_KEY'; +export const DEPLOYMENT_SIGNATURE_HEADER = 'x-deployment-signature'; + +export const DEPLOYMENT_CONTROL_LIMITS = { + requestTimeoutMs: 5_000, + capabilityCacheMs: 30_000, + pollIntervalMs: 2_000, + responseBytes: 256 * 1024, + keyMinLength: 32, +} as const; + +export interface DeploymentJournalEntry { + release: string; + verifiedAt: string; + operation: 'cutover' | 'rollback' | 'record'; +} + +export interface DeploymentCapabilities { + application: string; + target: string; + currentRelease: string | null; + journal: DeploymentJournalEntry[]; + supports: { restart: boolean; rollback: boolean }; + reasons: { restart: string | null; rollback: string | null }; + adapterVersion: string; + observedAt: string; +} + +export interface DeploymentRequest { + operationId: string; + idempotencyKey: string; + target: string; + release?: string; +} + +export interface DeploymentJobAccepted { + jobId: string; + state: 'queued' | 'running' | 'succeeded' | 'failed'; + replayed: boolean; +} + +export interface DeploymentJob { + jobId: string; + operation: 'restart' | 'rollback'; + state: 'queued' | 'running' | 'succeeded' | 'failed'; + target: string; + releaseBefore: string | null; + releaseAfter: string | null; + requestedRelease: string | null; + startedAt: string | null; + finishedAt: string | null; + error: string | null; + evidence: string[]; +} + +export type DeploymentControlState = + | { state: 'unconfigured'; reason: string; capabilities: null; probedAt: null } + | { state: 'unprobed'; reason: string; capabilities: null; probedAt: null } + | { state: 'unreachable'; reason: string; capabilities: null; probedAt: string } + | { state: 'ready'; reason: null; capabilities: DeploymentCapabilities; probedAt: string }; + +export class DeploymentControlError extends Error { + constructor(public readonly code: 'unconfigured' | 'unreachable' | 'rejected' | 'timeout' | 'protocol', message: string) { + super(message); + } +} + +export interface DeploymentControlConfig { + endpoint: { kind: 'http'; host: string; port: number } | { kind: 'socket'; path: string }; + key: Buffer; +} + +const LOOPBACK_HOSTS = new Set(['127.0.0.1', 'localhost', '[::1]', '::1']); + +/** null when the endpoint or key is absent; throws when one is present but unusable. */ +export function deploymentControlConfig(environment: Record = process.env): DeploymentControlConfig | null { + const endpoint = String(environment[DEPLOYMENT_CONTROL_ENDPOINT_VARIABLE] ?? '').trim(); + const key = String(environment[DEPLOYMENT_CONTROL_KEY_VARIABLE] ?? '').trim(); + if (!endpoint && !key) { return null; } + if (!endpoint || !key) { + throw new DeploymentControlError('unconfigured', `${DEPLOYMENT_CONTROL_ENDPOINT_VARIABLE} and ${DEPLOYMENT_CONTROL_KEY_VARIABLE} must both be set.`); + } + if (key.length < DEPLOYMENT_CONTROL_LIMITS.keyMinLength) { + throw new DeploymentControlError('unconfigured', `${DEPLOYMENT_CONTROL_KEY_VARIABLE} must be at least ${DEPLOYMENT_CONTROL_LIMITS.keyMinLength} characters.`); + } + if (endpoint.startsWith('/')) { + return { endpoint: { kind: 'socket', path: endpoint }, key: Buffer.from(key, 'utf8') }; + } + let url: URL; + try { + url = new URL(endpoint); + } catch { + throw new DeploymentControlError('unconfigured', `${DEPLOYMENT_CONTROL_ENDPOINT_VARIABLE} must be a loopback http:// URL or an absolute unix socket path.`); + } + if (url.protocol !== 'http:' || !LOOPBACK_HOSTS.has(url.hostname) || url.username || url.password || url.search || url.hash || (url.pathname !== '/' && url.pathname !== '')) { + throw new DeploymentControlError('unconfigured', `${DEPLOYMENT_CONTROL_ENDPOINT_VARIABLE} must be a plain loopback http:// origin such as http://127.0.0.1:8790.`); + } + return { endpoint: { kind: 'http', host: url.hostname.replace(/^\[|\]$/g, ''), port: url.port ? Number(url.port) : 80 }, key: Buffer.from(key, 'utf8') }; +} + +export function signDeploymentRequest(key: Buffer, method: string, path: string, body: string): string { + return 'sha256=' + createHmac('sha256', key).update(`${method.toUpperCase()}\n${path}\n${body}`).digest('hex'); +} + +export function verifyDeploymentSignature(key: Buffer, method: string, path: string, body: string, header: string | undefined): boolean { + const expected = Buffer.from(signDeploymentRequest(key, method, path, body)); + const presented = Buffer.from(String(header ?? '')); + return expected.length === presented.length && timingSafeEqual(expected, presented); +} + +const RELEASE_SHA = /^[0-9a-f]{7,64}$/; + +function isJournalEntry(value: unknown): value is DeploymentJournalEntry { + const entry = value as DeploymentJournalEntry; + return !!entry && typeof entry === 'object' && RELEASE_SHA.test(String(entry.release)) && typeof entry.verifiedAt === 'string' && !Number.isNaN(Date.parse(entry.verifiedAt)) && ['cutover', 'rollback', 'record'].includes(entry.operation); +} + +export function parseCapabilities(value: unknown): DeploymentCapabilities { + const doc = value as DeploymentCapabilities; + if (!doc || typeof doc !== 'object' || doc.application !== 'explorer' || typeof doc.target !== 'string' || !doc.target || !Array.isArray(doc.journal) || !doc.supports || typeof doc.supports !== 'object') { + throw new DeploymentControlError('protocol', 'The deployment adapter answered its capability route with a document this Explorer does not understand.'); + } + if (doc.currentRelease !== null && !RELEASE_SHA.test(String(doc.currentRelease))) { + throw new DeploymentControlError('protocol', 'The deployment adapter reported a current release that is not a commit sha.'); + } + if (!doc.journal.every(isJournalEntry) || doc.journal.length > 500) { + throw new DeploymentControlError('protocol', 'The deployment adapter journal is malformed.'); + } + return { + application: 'explorer', + target: doc.target.slice(0, 200), + currentRelease: doc.currentRelease, + journal: doc.journal.map(entry => ({ release: entry.release, verifiedAt: entry.verifiedAt, operation: entry.operation })), + supports: { restart: doc.supports.restart === true, rollback: doc.supports.rollback === true }, + reasons: { restart: typeof doc.reasons?.restart === 'string' ? doc.reasons.restart : null, rollback: typeof doc.reasons?.rollback === 'string' ? doc.reasons.rollback : null }, + adapterVersion: String(doc.adapterVersion ?? 'unknown').slice(0, 64), + observedAt: typeof doc.observedAt === 'string' ? doc.observedAt : new Date().toISOString(), + }; +} + +/** + * The previous verified release: the newest journal entry whose release is + * not the one serving now. Derived, never supplied. + */ +export function rollbackTargetFromJournal(capabilities: DeploymentCapabilities): DeploymentJournalEntry | null { + const ordered = [...capabilities.journal].sort((a, b) => Date.parse(b.verifiedAt) - Date.parse(a.verifiedAt)); + return ordered.find(entry => entry.release !== capabilities.currentRelease) ?? null; +} + +export function parseJob(value: unknown): DeploymentJob { + const job = value as DeploymentJob; + if (!job || typeof job !== 'object' || typeof job.jobId !== 'string' || !['restart', 'rollback'].includes(job.operation) || !['queued', 'running', 'succeeded', 'failed'].includes(job.state)) { + throw new DeploymentControlError('protocol', 'The deployment adapter answered a job read with a document this Explorer does not understand.'); + } + const sha = (v: unknown): string | null => (v === null || v === undefined ? null : RELEASE_SHA.test(String(v)) ? String(v) : null); + return { + jobId: job.jobId.slice(0, 128), + operation: job.operation, + state: job.state, + target: String(job.target ?? '').slice(0, 200), + releaseBefore: sha(job.releaseBefore), + releaseAfter: sha(job.releaseAfter), + requestedRelease: sha(job.requestedRelease), + startedAt: typeof job.startedAt === 'string' ? job.startedAt : null, + finishedAt: typeof job.finishedAt === 'string' ? job.finishedAt : null, + error: typeof job.error === 'string' ? job.error.slice(0, 1000) : null, + evidence: Array.isArray(job.evidence) ? job.evidence.filter((e): e is string => typeof e === 'string').slice(0, 50).map(e => e.slice(0, 500)) : [], + }; +} + +export interface DeploymentTransport { + (config: DeploymentControlConfig, method: 'GET' | 'POST', path: string, body: string, timeoutMs: number): Promise<{ status: number; body: string }>; +} + +/** The one place a socket to the adapter is opened. */ +export const httpDeploymentTransport: DeploymentTransport = (config, method, path, body, timeoutMs) => new Promise((resolve, reject) => { + const request = http.request({ + ...(config.endpoint.kind === 'socket' ? { socketPath: config.endpoint.path } : { host: config.endpoint.host, port: config.endpoint.port }), + method, + path, + headers: { + 'content-type': 'application/json', + 'content-length': Buffer.byteLength(body), + [DEPLOYMENT_SIGNATURE_HEADER]: signDeploymentRequest(config.key, method, path, body), + }, + timeout: timeoutMs, + }, response => { + const chunks: Buffer[] = []; + let size = 0; + response.on('data', (chunk: Buffer) => { + size += chunk.length; + if (size > DEPLOYMENT_CONTROL_LIMITS.responseBytes) { request.destroy(new DeploymentControlError('protocol', 'The deployment adapter response is too large.')); return; } + chunks.push(chunk); + }); + response.on('end', () => resolve({ status: response.statusCode ?? 0, body: Buffer.concat(chunks).toString('utf8') })); + response.on('error', reject); + }); + request.on('timeout', () => request.destroy(new DeploymentControlError('timeout', `The deployment adapter did not answer ${method} ${path} within ${timeoutMs} ms.`))); + request.on('error', error => reject(error instanceof DeploymentControlError ? error : new DeploymentControlError('unreachable', `The deployment adapter could not be reached: ${error.message}`))); + request.end(body); +}); + +export class DeploymentControlClient { + private snapshot: DeploymentControlState; + private inflight: Promise | null = null; + + constructor( + private readonly environment: () => Record = () => process.env, + private readonly transport: DeploymentTransport = httpDeploymentTransport, + private readonly now: () => number = Date.now, + ) { + this.snapshot = this.initialState(); + } + + private initialState(): DeploymentControlState { + try { + const config = deploymentControlConfig(this.environment()); + if (!config) { + return { state: 'unconfigured', reason: `Host-level deployment control is not configured for this process. An operator has to run the owned deployment adapter on this host and set ${DEPLOYMENT_CONTROL_ENDPOINT_VARIABLE} and ${DEPLOYMENT_CONTROL_KEY_VARIABLE} before the Control Center can restart or roll back this Explorer.`, capabilities: null, probedAt: null }; + } + return { state: 'unprobed', reason: 'The deployment adapter is configured but has not answered its capability route yet.', capabilities: null, probedAt: null }; + } catch (e) { + return { state: 'unconfigured', reason: e instanceof Error ? e.message : String(e), capabilities: null, probedAt: null }; + } + } + + /** What is known right now, without a network round trip. A stale or unprobed state starts a probe in the background. */ + public known(): DeploymentControlState { + const current = this.initialState(); + if (current.state === 'unconfigured') { + this.snapshot = current; + return current; + } + const stale = this.snapshot.probedAt === null || this.now() - Date.parse(this.snapshot.probedAt) > DEPLOYMENT_CONTROL_LIMITS.capabilityCacheMs; + if (stale) { + this.probe().catch(() => undefined); + } + return this.snapshot; + } + + /** @asyncUnsafe A rejection never escapes: every outcome becomes a state. */ + private async request(method: 'GET' | 'POST', path: string, body: unknown = ''): Promise { + const config = deploymentControlConfig(this.environment()); + if (!config) { throw new DeploymentControlError('unconfigured', this.initialState().reason ?? 'unconfigured'); } + const text = body === '' ? '' : JSON.stringify(body); + const response = await this.transport(config, method, path, text, DEPLOYMENT_CONTROL_LIMITS.requestTimeoutMs); + let parsed: unknown; + try { + parsed = response.body ? JSON.parse(response.body) : null; + } catch { + throw new DeploymentControlError('protocol', `The deployment adapter answered ${method} ${path} with a body that is not JSON.`); + } + if (response.status < 200 || response.status >= 300) { + const message = (parsed as { error?: unknown })?.error; + throw new DeploymentControlError('rejected', `The deployment adapter refused ${method} ${path} (HTTP ${response.status}): ${typeof message === 'string' ? message : 'no reason given'}.`); + } + return parsed; + } + + /** A fresh capability read; the result also becomes the cached state. @asyncUnsafe Errors become an unreachable state. */ + public async probe(): Promise { + if (this.inflight) { return this.inflight; } + this.inflight = (async (): Promise => { + const probedAt = new Date(this.now()).toISOString(); + try { + const capabilities = parseCapabilities(await this.request('GET', '/capabilities')); + this.snapshot = { state: 'ready', reason: null, capabilities, probedAt }; + } catch (e) { + const reason = e instanceof Error ? e.message : String(e); + if (e instanceof DeploymentControlError && e.code === 'unconfigured') { + this.snapshot = { state: 'unconfigured', reason, capabilities: null, probedAt: null }; + } else { + this.snapshot = { state: 'unreachable', reason: 'The deployment adapter did not answer its capability route: ' + reason, capabilities: null, probedAt }; + logger.debug('[admin-adapter] deployment adapter capability probe failed: ' + reason); + } + } finally { + this.inflight = null; + } + return this.snapshot; + })(); + return this.inflight; + } + + /** @asyncUnsafe The operation handler turns a rejection into a failed run. */ + public async submit(operation: 'restart' | 'rollback', request: DeploymentRequest): Promise { + const accepted = await this.request('POST', `/${operation}`, request) as DeploymentJobAccepted; + if (!accepted || typeof accepted.jobId !== 'string' || !accepted.jobId) { + throw new DeploymentControlError('protocol', `The deployment adapter accepted ${operation} without a job id.`); + } + return { jobId: accepted.jobId.slice(0, 128), state: accepted.state, replayed: accepted.replayed === true }; + } + + /** @asyncUnsafe The operation handler turns a rejection into a failed run. */ + public async job(jobId: string): Promise { + if (!/^[A-Za-z0-9_-]{1,128}$/.test(jobId)) { throw new DeploymentControlError('protocol', 'Invalid job id.'); } + return parseJob(await this.request('GET', `/jobs/${jobId}`)); + } + + /** + * Polls a job until it is terminal or the deadline passes. The deadline + * is a bound on waiting, not on the adapter: a job still running when it + * passes is reported as such, with its id, so it can be read again later. + * @asyncUnsafe The operation handler turns a rejection into a failed run. + */ + public async waitForJob(jobId: string, deadlineMs: number, sleep: (ms: number) => Promise = ms => new Promise(resolve => setTimeout(resolve, ms))): Promise<{ job: DeploymentJob; timedOut: boolean }> { + const deadline = this.now() + deadlineMs; + for (;;) { + const job = await this.job(jobId); + if (job.state === 'succeeded' || job.state === 'failed') { return { job, timedOut: false }; } + if (this.now() >= deadline) { return { job, timedOut: true }; } + await sleep(Math.min(DEPLOYMENT_CONTROL_LIMITS.pollIntervalMs, Math.max(0, deadline - this.now()))); + } + } +} + +export const deploymentControlClient = new DeploymentControlClient(); diff --git a/backend/src/api/admin-adapter/deployment-control.resume.ts b/backend/src/api/admin-adapter/deployment-control.resume.ts new file mode 100644 index 00000000000..11947d634c2 --- /dev/null +++ b/backend/src/api/admin-adapter/deployment-control.resume.ts @@ -0,0 +1,68 @@ +import DB from '../../database'; +import config from '../../config'; +import logger from '../../logger'; +import runStore, { AdminRunConflict } from './admin-adapter.runs'; +import { adminTimestamp } from './admin-adapter.identity'; +import { deploymentControlClient as controlClient, DEPLOYMENT_CONTROL_LIMITS } from './deployment-control.client'; + +const DEPLOYMENT_OPERATIONS = new Set(['explorer.service.restart', 'explorer.release.rollback']); + +/** + * Finishes the deployment runs the previous process could not. + * + * A restart or rollback replaces the very backend that is driving it. The + * run already names the adapter job before the wait starts, so the process + * that comes up afterwards can read that job back and close the run with + * the same evidence the original executor would have written. Without this + * the run would sit RUNNING until the reconciler marked it abandoned, which + * would make every successful restart read as an unknown outcome. + * + * Only runs of the two deployment operations that are RUNNING and carry a + * job id are touched; anything else belongs to the reconciler. + * @asyncUnsafe Startup work: a failure is logged and the runs stay for the reconciler. + */ +export async function resumeDeploymentRuns(): Promise<{ resumed: number; skipped: number }> { + if (!config.DATABASE.ENABLED) { return { resumed: 0, skipped: 0 }; } + const [rows]: any[] = await DB.query( + "SELECT run_id, operation_id, document FROM admin_adapter_runs WHERE state = 'RUNNING' AND operation_id IN ('explorer.service.restart', 'explorer.release.rollback') ORDER BY started_at ASC LIMIT 20", + ); + let resumed = 0; + let skipped = 0; + for (const row of rows) { + let document: any = {}; + try { document = typeof row.document === 'string' ? JSON.parse(row.document) : (row.document ?? {}); } catch { document = {}; } + const jobId = document?.result?.jobId; + if (!DEPLOYMENT_OPERATIONS.has(row.operation_id) || typeof jobId !== 'string' || !jobId) { skipped += 1; continue; } + try { + const { job, timedOut } = await controlClient.waitForJob(jobId, DEPLOYMENT_CONTROL_LIMITS.pollIntervalMs * 10); + const expectedRelease = document.result?.requestedRelease ?? document.result?.releaseBefore ?? null; + const verified = !timedOut && job.state === 'succeeded' && job.releaseAfter !== null && (expectedRelease === null || job.releaseAfter === expectedRelease); + const result = { ...document.result, jobState: job.state, releaseAfter: job.releaseAfter, expectedRelease, jobStartedAt: job.startedAt, jobFinishedAt: job.finishedAt, jobError: job.error, resumedAfterRestart: true }; + const evidence = [ + `Resumed by the replacement process after the restart cut the original executor short.`, + `Adapter job ${job.jobId} (${job.operation}) is ${job.state}.`, + `Release before: ${document.result?.releaseBefore ?? 'unknown'}; after: ${job.releaseAfter ?? 'unknown'}; expected: ${expectedRelease ?? 'unknown'}.`, + ...job.evidence.map((line: string) => 'Adapter: ' + line), + ]; + // The executor path goes RUNNING -> VERIFYING -> terminal; the contract + // allows no direct jump from RUNNING to SUCCEEDED. + await runStore.transition(row.run_id, 'VERIFYING', { progressPercent: 90 }); + if (timedOut) { + await runStore.transition(row.run_id, 'NEEDS_REVIEW', { progressPercent: 100, result, verification: { verified: false, evidence: [...evidence, 'The adapter job had not reached a terminal state when the replacement process looked it up.'] }, logs: [{ at: adminTimestamp(), level: 'warn', message: `Adapter job ${jobId} still ${job.state} after the restart.` }] }); + } else if (job.state === 'failed') { + await runStore.transition(row.run_id, 'FAILED', { result, error: { class: 'operation_failed', message: (job.error ?? 'The adapter reported the job failed.').slice(0, 700), retryable: true }, logs: [{ at: adminTimestamp(), level: 'error', message: `Adapter job ${jobId} failed: ${job.error ?? 'no reason given'}` }] }); + } else { + await runStore.transition(row.run_id, verified ? 'SUCCEEDED' : 'NEEDS_REVIEW', { progressPercent: 100, result, verification: { verified, evidence }, logs: [{ at: adminTimestamp(), level: verified ? 'info' : 'warn', message: verified ? `Release ${job.releaseAfter} is serving after the adapter's health-checked cutover (resumed after restart).` : `The adapter job succeeded but the serving release ${job.releaseAfter ?? 'unknown'} does not match the expected ${expectedRelease ?? 'unknown'}.` }] }); + } + resumed += 1; + } catch (e) { + if (e instanceof AdminRunConflict) { skipped += 1; continue; } + logger.warn(`[admin-adapter] could not resume deployment run ${row.run_id}: ` + (e instanceof Error ? e.message : String(e))); + skipped += 1; + } + } + if (resumed || skipped) { + logger.info(`[admin-adapter] deployment runs resumed after restart: ${resumed} closed, ${skipped} left for the reconciler.`); + } + return { resumed, skipped }; +} diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 8628a094f50..08b8971eeef 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 109; + private static currentVersion = 112; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -1467,6 +1467,84 @@ class DatabaseMigration { ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4`); await this.updateToSchemaVersion(109); } + if (databaseSchemaVersion < 110) { + // Admin adapter: a token so only the executor that owns a run can renew + // or finish it, and one row per claimed request nonce shared by every + // backend worker so a signed request cannot replay against another + // process. Both additive; existing runs keep working with a NULL token. + await this.$executeQuery('ALTER TABLE admin_adapter_runs ADD owner_token CHAR(36) NULL'); + await this.$executeQuery(`CREATE TABLE IF NOT EXISTS admin_nonces ( + scope VARCHAR(160) NOT NULL, + nonce VARCHAR(128) NOT NULL, + expires_at DATETIME(3) NOT NULL, + PRIMARY KEY (scope, nonce), + INDEX admin_nonces_expiry (expires_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4`); + await this.updateToSchemaVersion(110); + } + if (databaseSchemaVersion < 111) { + // Private relay submissions: every raw transaction handed to an owned + // Tor or I2P endpoint, with the lease the worker holds while relaying, + // the attempt count and the hash of the owner token that authorizes + // readback and abort. The raw hex stays here because a retry needs it; + // the token itself is never stored. + await this.$executeQuery(`CREATE TABLE IF NOT EXISTS intelligence_private_relay_submissions ( + submission_id CHAR(36) NOT NULL, + network VARCHAR(16) NOT NULL, + txid CHAR(64) NOT NULL, + raw_tx MEDIUMTEXT NOT NULL, + method VARCHAR(32) NOT NULL, + state VARCHAR(16) NOT NULL, + relay_endpoint_id VARCHAR(64) NULL, + attempts INT UNSIGNED NOT NULL DEFAULT 0, + lease_until DATETIME(3) NULL, + lease_owner VARCHAR(64) NULL, + owner_token_hash CHAR(64) NOT NULL, + created_at DATETIME(3) NOT NULL, + updated_at DATETIME(3) NOT NULL, + relayed_at DATETIME(3) NULL, + confirmed_block_height INT UNSIGNED NULL, + last_error VARCHAR(512) NULL, + PRIMARY KEY (submission_id), + UNIQUE INDEX intelligence_private_relay_txid (network, txid), + INDEX intelligence_private_relay_claim (network, state, lease_until) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4`); + await this.updateToSchemaVersion(111); + } + if (databaseSchemaVersion < 112) { + // Bootstrap (AssumeUTXO) verification runs and operator jobs. Each row + // carries the backend's own network; the JSON document is the record. + await this.$executeQuery(`CREATE TABLE IF NOT EXISTS universe_bootstrap_verifications ( + verification_id CHAR(36) NOT NULL, + snapshot_id VARCHAR(128) NOT NULL, + network VARCHAR(16) NOT NULL, + state VARCHAR(16) NOT NULL, + created_at DATETIME(3) NOT NULL, + updated_at DATETIME(3) NOT NULL, + document JSON NOT NULL, + PRIMARY KEY (verification_id), + INDEX universe_bootstrap_verifications_snapshot (snapshot_id, network, created_at), + INDEX universe_bootstrap_verifications_state (network, state, updated_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4`); + await this.$executeQuery(`CREATE TABLE IF NOT EXISTS universe_bootstrap_jobs ( + job_id CHAR(36) NOT NULL, + network VARCHAR(16) NOT NULL, + node_id VARCHAR(64) NOT NULL, + job_type VARCHAR(32) NOT NULL, + idempotency_key VARCHAR(128) NOT NULL, + state VARCHAR(16) NOT NULL, + lease_owner VARCHAR(128) NULL, + lease_expires_at DATETIME(3) NULL, + created_at DATETIME(3) NOT NULL, + updated_at DATETIME(3) NOT NULL, + document JSON NOT NULL, + PRIMARY KEY (job_id), + UNIQUE INDEX universe_bootstrap_jobs_idempotency (network, idempotency_key), + INDEX universe_bootstrap_jobs_claim (network, state, lease_expires_at, created_at), + INDEX universe_bootstrap_jobs_node (network, node_id, state) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4`); + await this.updateToSchemaVersion(112); + } if (databaseSchemaVersion < 106 && config.MEMPOOL.NETWORK === 'liquid') { // In a specific setup it's possible that 3G6neksSBMp51kHJ2if8SeDUrzT8iVETWT and bc1qwnevjp8nsq7adu3hxlvdvslrf242q4vuavfg0y929jp2zntp3vgq7cq6z2 diff --git a/backend/src/api/intelligence/bootstrap/__fixtures__/README.md b/backend/src/api/intelligence/bootstrap/__fixtures__/README.md new file mode 100644 index 00000000000..cb32bc7f03a --- /dev/null +++ b/backend/src/api/intelligence/bootstrap/__fixtures__/README.md @@ -0,0 +1,13 @@ +# Bootstrap fixtures + +`regtest-105.snapshot.base64` is the exact `dumptxoutset` output of an offline +Bitcoin Core 28.0 regtest node (no peers, no network) at height 105 with 107 +coins covering P2WPKH, P2PKH, P2SH-P2WPKH, P2TR, compressed P2PK, uncompressed +P2PK, a raw OP_TRUE script and a three-output transaction. Core reported: + +- base_hash `0be30936f2332969e5dd82fe06d0c4d8d25d86351442ec813c2f70aa132c2acb`, base_height 105 +- coins_written 107 +- `gettxoutsetinfo hash_serialized_3` = `2fff9d5b75bf5539b20a99cbae5639819995d0b560d2cfda171aafed870418fc` +- file sha256 `facbe9af2dede0a3322ec7b9c46328f4f86cfc4764cab11b90b3821104cceb7e` (6349 bytes) + +The tests decode it with the streaming parser and must reproduce those values. diff --git a/backend/src/api/intelligence/bootstrap/__fixtures__/regtest-105.snapshot.base64 b/backend/src/api/intelligence/bootstrap/__fixtures__/regtest-105.snapshot.base64 new file mode 100644 index 00000000000..f192bc4e52e --- /dev/null +++ b/backend/src/api/intelligence/bootstrap/__fixtures__/regtest-105.snapshot.base64 @@ -0,0 +1 @@ +dXR4b/8CAPq/tdrLKiwTqnAvPIHsQhQ1hl3S2MTQBv6C3eVpKTPyNgnjC2sAAAAAAAAAAHXm5YFd622oqbWFIhY57IDbrBgwwsKnv7yKoTTmRy0BAH0yHAAU9sHlbyVoiZooiU5idBL7v62KADgAustdwKLTQDLQ874gLOEcrHD7Dpc0R/QKAr0Ej8gmkgEAbzIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOAQbFvLf6lEnqBdOtKOfBJkOcfyMGT7EsvxGiQs0mJmTAQAxMhwAFPbB5W8laImaKIlOYnQS+7+tigA4B2RScrkNGRJ57q3im2fvTh2IxUqcV/23tWqXsFCyFF0BAG0yHAAU9sHlbyVoiZooiU5idBL7v62KADgMNZ2XdyZ9QeNVuXm6RVCiXf+yV/m/EjrcDNBvGQak5wEAOzIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOBCIiOxNA7frL5knDH0PywhC93druLhfe0O8+7SJlK1WAQCAKzIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOBC39yCWqrpGEx9UIJEqiFAla2/OoXu93rhiIDx2IR6nAQCAPzIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOBSvcHD6yi82l9gryUuOVB51arsFkwxrib2tofLh5L1MAQANMhwAFPbB5W8laImaKIlOYnQS+7+tigA4Fu+QyxroM435qIl6Gx3s5V2NEZ8WqA+YN2eZw3YlGvcBAF0yHAAU9sHlbyVoiZooiU5idBL7v62KADgX8LbMydlUsKTnTNaFYMEh2Vpzv3CyJ8V1JqZ6dBX/3wEAgAEyHAAU9sHlbyVoiZooiU5idBL7v62KADgdfSLuGDK3W5Lzg4N2R7lQqIezCRi6g+Qcf9cvqGXthQEAETIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOCIDpzknwoDO1zcf0SgY/Kqn4s6oyEh0kC+Lde5+UqSRAQCAETIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOCM4VzvqhsVLkNxcLbHU4Ik7u+38jnoJAqdoWD+TWID8AQB3MhwAFPbB5W8laImaKIlOYnQS+7+tigA4JEm8RTrhMHslRPdRia5ee53rzZM6VJ7JrSlU78w8FjkBAIA7MhwAFPbB5W8laImaKIlOYnQS+7+tigA4JIv5+/EJ5n8JXTVs9Ds07/ez/R1AYD5smLMnBaNDq4oBAB8yHAAU9sHlbyVoiZooiU5idBL7v62KADgkr3nKApIDM2C/7upZBrN8A30rOM/1/h+QX+GUIQty9wEAgAsyHAAU9sHlbyVoiZooiU5idBL7v62KADglQcfc+CWKmMIQXF0PDi17p9GZOIRCfVkmzjR+/+HjjAEASTIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOCcHwqgeNFvQueO8kPm8lcmiZhogNT2wIa/2G+d1GQ0tAQCAPTIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOCwyzddQUC2KQSBCdwjPOwoTm5fri4DpO05ws/qhrlL9AQATMhwAFPbB5W8laImaKIlOYnQS+7+tigA4L2IHvaHZHOEIi6GLZInDUbtFfQfklGNG00lp4XhbEMoBAHMyHAAU9sHlbyVoiZooiU5idBL7v62KADg/1Jw6k+VJPyZ/Vygf4vtHGeVYYVgorryI1DgcOTSENAEAUTIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOEBcWiLvFqvs7m+b0IgI1kQfTBMQagXE9lr7qVTmsn73AQA3MhwAFPbB5W8laImaKIlOYnQS+7+tigA4QKIimiw6xdaLrP++9T+8q84NPOGElJK4mOSd1LYUW7UBAIAzMhwAFPbB5W8laImaKIlOYnQS+7+tigA4Q176yjCJGK9fIlbiSC5HGSXOXFyzCl8/TnmqO3DHCesBABkyHAAU9sHlbyVoiZooiU5idBL7v62KADhGB9kw0apT4lJFcNr0Xiac9gGsYaxYiMGVc5fn1Ys0WgEAIzIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOEbqezFAm0tEZBKNcNHoeErXj3aVbRnKTNev8VMKml9vAQAvMhwAFPbB5W8laImaKIlOYnQS+7+tigA4SHX3N16PQmuBZPu/TMsUSY8jT44F7LmRJODqsGcN3vMBAIBRMgdRSi+m3kLS7wEjHBjCnxOev34lu4sCpvKD6htu13MlAV8BAHsyHAAU9sHlbyVoiZooiU5idBL7v62KADhUAHVO4CTOeJfBiOiiCL3VcLBh6tyeWflPOqBK8HhoOwEAYzIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOFdts/LV1PyQwUXqaZ3AScTtkwm06VLGtUokJ99JhQRDAQBVMhwAFPbB5W8laImaKIlOYnQS+7+tigA4WT4Vcnv+DbXlQKELNNDGbdpUuiZ8OISS04Cgd9Jl8EUBAIAZMhwAFPbB5W8laImaKIlOYnQS+7+tigA4WnqMM2+8B38FvW25jWpHlxrAZlys+jOQpqLpipSQ5mQBAIAVMhwAFPbB5W8laImaKIlOYnQS+7+tigA4XO/DSXZ8eA5Rp30DUsYk7q0EmRoGPPePQLybENQI8K0BAGkyHAAU9sHlbyVoiZooiU5idBL7v62KADhfFxmCT3vxxPR8XcjTGF9FvEiitxPcR31QiLmNnSxm7AEAgDcyHAAU9sHlbyVoiZooiU5idBL7v62KADhfjKtpflVidrtInDiJp2SXs9oR0RQu92oq+I2m119OAwEAgEkyHAAU9sHlbyVoiZooiU5idBL7v62KADhhM/3sXYf4d/TTVwXcrqqIZnMcpflbhXgJup/N3pbItAQAgFKAor6zzlcoUSCmUvLoTIOzCEy5JvSCcU1zueioowUASj4/ArSVz9z5OQGAUoOQ5+o9AK04LR2/zouRwJBvVEyFBxc8WblwAoBSMihRIPTjmvgPf89QercQJ63ylT6+kqvXYk+QgT8LatX+Hz2bA4BSBgGo0wD45z/+WUc55akW2ORzoN92zGMJ8AJhCLRd6DCuDbI3xWxRUDhTgFwZaWvff019B2SSAQBFMhwAFPbB5W8laImaKIlOYnQS+7+tigA4Yz5t2XOyMa7pGVdQtJ/DANF5EdSS96P7gSGve5g2CBgBAB0yHAAU9sHlbyVoiZooiU5idBL7v62KADhjXZWpnYcMCVgX7cU/g5o/eC5mf1Wr/SVFWZUUiLNKBgEAazIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOGaKuKV2Mwl5r/bs7o/1+JmbemBA0i/BC/ykSS67/jchAQCARzIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOGmsa1leXuzfsVx8gkjoyeIWLte+/DJHAlheBQLfAhBWAQArMhwAFPbB5W8laImaKIlOYnQS+7+tigA4aiHOeBv+OlwcG2clKa6MqLYVcvAvSyMcw8RfdgfRUL0BAAUyHAAU9sHlbyVoiZooiU5idBL7v62KADhr0TdKF6wtY2wCJzXIHzaFbAbYkN/xz0MBd7MqUwQeAAEAgAMyHAAU9sHlbyVoiZooiU5idBL7v62KADhuGcWDpv3UOrU9kT0F1/MUqUUC54kjEPvatOBS8m2VqAEALTIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOG80WJRSNpFyTLfp9RhOEsz/Hj9eyFUBN+ZF8l/yak9GAQCAIzIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOHMBHCgTIIwi5oKSXPBwLoDXP2lE5sQlyBgtp9sHiRFJAQBLMhwAFPbB5W8laImaKIlOYnQS+7+tigA4eq75k/uumoBGfVlPcExATrhEFdY9q+Vqg9UeIamUYqkBAH8yHAAU9sHlbyVoiZooiU5idBL7v62KADh8+/pSr3mzueEfCWASv5U+d/W5NLAbazbs2Kk1zt1uVQEAgB8yHAAU9sHlbyVoiZooiU5idBL7v62KADh9ddhWCL/Xzeo5BhzDD9a5vfL80dhO0WAKKTUf/7Hl4AEAgAUyHAAU9sHlbyVoiZooiU5idBL7v62KADh+vI8u4D4ofKPUpn56zyxN4CTEumZoHOyjsWkpYwumCwEAJzIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOH9FMc/HQHA5xhEOnrx0IA6CWeRMZcAxDC8DFDse2VdvAQCADTIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOIP50TW3A7LuXpaWF+DU5JMtcjvmWl+XkXk8FOODlJ7/AQAXMhwAFPbB5W8laImaKIlOYnQS+7+tigA4hQY11faLiwIEerbYU94estMJNPJ3w3iZvglJpTD49mEBAIAXMhwAFPbB5W8laImaKIlOYnQS+7+tigA4hifoh7hARyC8d0yyerozak7rOn70BTssagIcwHfgNgsBAD0yHAAU9sHlbyVoiZooiU5idBL7v62KADiGhKXkobgVz4+sNyqsXAPfZYKgzLW9oWjun+p1GpGHjAEAgCkyHAAU9sHlbyVoiZooiU5idBL7v62KADiIah9OGe3pNMJmONFjxk3yUIlVev1T1QgTw9T9xcSFJAEAPzIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOInDNpr4KF6ZBbkuajaLOksYBlTpkw2Fa30pCSnxbfw3AQCATzIEeb5mfvncu6xVoGKVzocLBwKb/NstzijZWfKBWxb4F5iOW/6LmMpE4fJHCSWSLvZuaNPp9TGI1VcOycOwmh95QAEAgCcyHAAU9sHlbyVoiZooiU5idBL7v62KADiQLX2jGtDkIYd/fBwWp0lLxoKOSHYSrb7zvKybCHSXMAEAFTIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOJAzs2QSmOyr3ES10ngZTtn2YinA71fSoB0MH1JcCYBbAQBPMhwAFPbB5W8laImaKIlOYnQS+7+tigA4kLfS11HjDMtqspYqsG2v6nWVL6sVOSj5N8xnblSzeD8BAGUyHAAU9sHlbyVoiZooiU5idBL7v62KADiSd1DEUOzzTxBZhUBJlyTurtHHPEXXbVZL7TUvPdDjSAEAWzIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOJPw4uySptcjUnee2QRX1FZDGRypFNRiN+gYn91btS6eAQBZMhwAFPbB5W8laImaKIlOYnQS+7+tigA4ljH/aUHJ2xRW2GmTC712gC519p/RVJbOCNNxvUee8jkBAIATMhwAFPbB5W8laImaKIlOYnQS+7+tigA4lvjW49yRnACWRdNaH+aaQ3b+7dPP/f0wK2iPKT6nJ64BAIA1MhwAFPbB5W8laImaKIlOYnQS+7+tigA4mEEb01o3bdPYrG19CUkWdlzA8Bb2E6g5msiGTxyJ+NsBADMyHAAU9sHlbyVoiZooiU5idBL7v62KADieToYAwfSyTDbZeM5jGGBJrGzdm9/rQ0J0mlt48SVQygEAgC8yHAAU9sHlbyVoiZooiU5idBL7v62KADiexAlW9ttZT/2wDTRCcpq1tYN2rIM49QQQv55fAZz+kgEAXzIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOJ7sfzULuL1JyjX6oSJY9na6vTxYhjw7eJ+HXuIPCuqtAQBBMhwAFPbB5W8laImaKIlOYnQS+7+tigA4n35I/T4InhwgaH197MUfaWgT3KuThFp8BW1wXVyUQ9QBAHUyHAAU9sHlbyVoiZooiU5idBL7v62KADiiTqffO1EzzveJqFr8q7C5n/HA09o37JtHovLr9iv9xwEADzIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOKRHCW2KVirQ/Hpu4OB6I7XsX3x04yr6Ug4HDbCqNoxjAQCAOTIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOKUyZEYoQdFVtGbj/3ktfzOuu1bOnekM/foGQZiSQzrqAQBXMhwAFPbB5W8laImaKIlOYnQS+7+tigA4p4EhZD5tcQDKzcXEPnKNfdGNRsTzyVJkO8KaBxkn894BAHkyHAAU9sHlbyVoiZooiU5idBL7v62KADipXS+idvDdrRq+mOHXCl8le60b08STm9Rgt8wucmGGrQEACzIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOKnLGuyd8RWdWnqYQNfuPBC/u4W7JEKPjuUdLQXCDas/AQAlMhwAFPbB5W8laImaKIlOYnQS+7+tigA4sClqH+5tASzOPYVUwjVtM4CkkxsAJd88Sj2A2Dj3qGwBAAcyHAAU9sHlbyVoiZooiU5idBL7v62KADiyLZ8Ld8JAboV2Z0ZvfgMf4iyGjRraI55yaNQzoItEOAEAgBsyHAAU9sHlbyVoiZooiU5idBL7v62KADi4U1bdPijm4/QgiayMZbOUSilPcynxYsTI30mMMerdUQEAYTIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOLp4o+4rZJacf1xG7GsEgvsaPJKt+jtQdJq8h3nS26dOAQAhMhwAFPbB5W8laImaKIlOYnQS+7+tigA4vC+wqzsZl6ma3X8ek6jTNasZmHN/V+0KfT3Mi8vEUCcBAEcyHAAU9sHlbyVoiZooiU5idBL7v62KADjAV4fBuQVYzCcD+GxXgnjZSRqAC2+F6nyE/D4+o+rwGQEAgEMyHAAU9sHlbyVoiZooiU5idBL7v62KADjBSrx7ic4B6Hs8jwTfO31ZeXLYVPPJbTFHOwWTo+s3DgEAgDEyHAAU9sHlbyVoiZooiU5idBL7v62KADjB+APhb9vv4IpvkKgTLQSEYWGzkcJjI5Ai2zyg2tgSfgEAgAkyHAAU9sHlbyVoiZooiU5idBL7v62KADjEm3Q+EsumoQvStfo4hPDe5IeGh292epckDqvxHvjyswEAgEsyHAAU9sHlbyVoiZooiU5idBL7v62KADjF2vIdR9zYKP6TRc/e/lJzteRVMIAIREyFD4JGziZKkwEAGzIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOMhW2/i6IUptmyriA1ccuOwZi/p2AbsKpIcyd5XOhYZjAQApMhwAFPbB5W8laImaKIlOYnQS+7+tigA4yXDl3P0yR/10U/xvCFOrQcLfV/qyoUoWX7aOEkjhtO4BAHEyHAAU9sHlbyVoiZooiU5idBL7v62KADjKBdAPkJzMfbIryTl+c9zWhfkwxozUI44yCO5qp2T8MgEAgAcyHAAU9sHlbyVoiZooiU5idBL7v62KADjKP1lqgeO+iW3YeTtKsoUmkQb1R8/K8W2Svz6d1OrbmwEAgA8yHAAU9sHlbyVoiZooiU5idBL7v62KADjLf+2IWWvx6n6CqBPd0v07A7laGEgtLClmAObLAgrECgEAgC0yHAAU9sHlbyVoiZooiU5idBL7v62KADjPot85utqo8C6+QnCanHYyuLsB3uSwhqe/Cyh9r52aoQEAgB0yHAAU9sHlbyVoiZooiU5idBL7v62KADjSPYqnNlPRQAo3JHuJVW9yRMxafIBUg+IDxll2TsM0mgEAgE0yAnm+Zn753LusVaBilc6HCwcCm/zbLc4o2VnygVsW+BeY2ZKd/P721gVcWbuW09LAnYdXk9Jobk9zJDZXrgr+2YYBAGcyHAAU9sHlbyVoiZooiU5idBL7v62KADjdnQjyD/8FL/Uzy17Fd4Rbmqu1Elj9FTMgGGvCtFDZHQEAQzIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOOlTnj6SL3RKF+EI7WweJYCEWaTqImsuLP1G9+6CtzJwAQA1MhwAFPbB5W8laImaKIlOYnQS+7+tigA47kbtkioLGuxRoZSOC86phRM8caDbuqLkwuqttDOXro8BADkyHAAU9sHlbyVoiZooiU5idBL7v62KADjwPVJhD9uGaCHDcgiooARPe+qo9We7hBHC5pgx6hMQ9wEAgEUyHAAU9sHlbyVoiZooiU5idBL7v62KADjxFc+WicbKG+pS/9hdAy7+fk2Dh0QLjfNgoxPitydqoQEAgCEyHAAU9sHlbyVoiZooiU5idBL7v62KADj1hsiYResWD3RsKDHrewiOfYJqMRwwAvu3Fw28nMi27AEATTIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOPddwd4JKkpi6e9oYxVMhSHa4re+SOeH2iyMaOgkWnzMAQCAUzICeb5mfvncu6xVoGKVzocLBwKb/NstzijZWfKBWxb4F5j5g4v1F4jj4l96gohQtkCXHV/97fQDmciXIOvTu7hDhgEAgEEyHAAU9sHlbyVoiZooiU5idBL7v62KADj7Sjj6qyKLliVNBqsqM8urWD+aCrGC6ZRZn2ABCgwKggEAUzIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOP4pmIVY0gfU74sr1mX1zy17jUuDTWpGZqA3H05WK0Z7AQCAJTIcABT2weVvJWiJmiiJTmJ0Evu/rYoAOA== diff --git a/backend/src/api/intelligence/bootstrap/__fixtures__/support.ts b/backend/src/api/intelligence/bootstrap/__fixtures__/support.ts new file mode 100644 index 00000000000..23809b6f532 --- /dev/null +++ b/backend/src/api/intelligence/bootstrap/__fixtures__/support.ts @@ -0,0 +1,149 @@ +import { createHash, generateKeyPairSync, KeyObject, sign } from 'crypto'; +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { BootstrapEnvironment } from '../bootstrap-config'; +import { BootstrapCatalogue, BootstrapCatalogueSnapshot } from '../bootstrap.models'; +import { deterministicSnapshotJson } from '../snapshot-catalogue'; +import { SnapshotByteSource } from '../snapshot-verifier'; + +/** Test doubles shared by the bootstrap suites. Nothing here runs in production. */ + +export const FIXTURE_BYTES = Buffer.from(readFileSync(join(__dirname, 'regtest-105.snapshot.base64'), 'utf8'), 'base64'); +export const FIXTURE = { + baseHash: '0be30936f2332969e5dd82fe06d0c4d8d25d86351442ec813c2f70aa132c2acb', + height: 105, + coins: 107, + hashSerialized3: '2fff9d5b75bf5539b20a99cbae5639819995d0b560d2cfda171aafed870418fc', + sha256: createHash('sha256').update(FIXTURE_BYTES).digest('hex'), + genesis: '0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206', +}; + +export function producerKeys(): { privateKey: KeyObject; publicKeyHex: string } { + const { privateKey, publicKey } = generateKeyPairSync('ed25519'); + const jwk = publicKey.export({ format: 'jwk' }) as { x: string }; + return { privateKey, publicKeyHex: Buffer.from(jwk.x, 'base64url').toString('hex') }; +} + +export function signSnapshot(snapshot: BootstrapCatalogueSnapshot, privateKey: KeyObject): string { + return sign(null, deterministicSnapshotJson(snapshot), privateKey).toString('hex'); +} + +export function catalogueFor( + privateKey: KeyObject, + publicKeyHex: string, + overrides: Partial = {}, + pinned: Partial = {} +): BootstrapCatalogue { + const snapshot: BootstrapCatalogueSnapshot = { + id: 'regtest-105', + network: 'regtest', + coreVersion: '28.0.0', + height: FIXTURE.height, + blockHash: FIXTURE.baseHash, + sizeBytes: FIXTURE_BYTES.length, + sha256: FIXTURE.sha256, + source: '/allowed/regtest-105.dat', + manifest: { producerId: 'lab', signature: '' }, + ...overrides, + }; + snapshot.manifest = { producerId: snapshot.manifest.producerId, signature: snapshot.manifest.signature || signSnapshot(snapshot, privateKey) }; + return { + producers: [{ id: 'lab', algorithm: 'ed25519', publicKey: publicKeyHex }], + pinnedCommitments: [{ network: 'regtest', coreVersion: '28.0.0', height: FIXTURE.height, blockHash: FIXTURE.baseHash, utxoCommitment: FIXTURE.hashSerialized3, coinCount: FIXTURE.coins, ...pinned }], + snapshots: [snapshot], + }; +} + +export function testEnvironment(overrides: Partial = {}): BootstrapEnvironment { + return { + catalogueFile: undefined, + sourceAllowlist: ['/allowed'], + maxSnapshotBytes: 1 << 20, + verifyDeadlineMs: 5000, + snapshotDir: '/var/lib/bitcoin/snapshots', + datadir: '/var/lib/bitcoin', + capacityBytes: undefined, + capacityMeasuredAt: undefined, + capacityMaxAgeMs: 24 * 3600 * 1000, + jobTimeoutMs: 5000, + jobLeaseMs: 60000, + jobPollMs: 60000, + ...overrides, + }; +} + +/** Serves a buffer in chunks; `stall` never ends, for deadline tests. */ +export function byteSource(bytes: Buffer, options: { chunk?: number; stall?: boolean } = {}): SnapshotByteSource & { opened: string[] } { + const opened: string[] = []; + return { + opened, + async open(source): Promise & { destroy?: () => void }> { + opened.push(source); + let destroyed = false; + const chunk = options.chunk ?? 512; + /** @asyncUnsafe Test double; the consumer owns the iteration. */ + async function* generate(): AsyncGenerator { + for (let i = 0; i < bytes.length; i += chunk) { + if (destroyed) { + return; + } + yield bytes.subarray(i, Math.min(bytes.length, i + chunk)); + } + if (options.stall) { + while (!destroyed) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + } + } + const iterable = generate() as AsyncGenerator & { destroy?: () => void }; + iterable.destroy = () => { + destroyed = true; + }; + return iterable; + }, + }; +} + +export type CoreCall = (method: string, params: unknown[]) => Promise; + +/** A regtest Core 28.0 at the fixture height, fully validated, reachable through the reader RPCs. */ +export function fakeCore(overrides: { blocks?: number; headers?: number; ibd?: boolean; subversion?: string; help?: string; chainstates?: any[]; calls?: string[] } = {}): { network: string; call: CoreCall; calls: string[] } { + const calls = overrides.calls ?? []; + const blocks = overrides.blocks ?? FIXTURE.height; + const headers = overrides.headers ?? blocks; + const hashes: Record = { 0: FIXTURE.genesis, [FIXTURE.height]: FIXTURE.baseHash, 104: '0330fd86eb1429db0fd4dec8c659556aeb265f3c98ac9bd75d0f18c0204b10a3' }; + const hashAt = (h: number): string => hashes[h] ?? createHash('sha256').update('block' + h).digest('hex'); + const tip = hashAt(blocks); + const chainstates = overrides.chainstates ?? [{ blocks, bestblockhash: tip, verificationprogress: 1, validated: true, coins_db_cache_bytes: 8388608, coins_tip_cache_bytes: 461373440 }]; + return { + network: 'regtest', + calls, + async call(method, params) { + calls.push(method); + switch (method) { + case 'getblockhash': + return hashAt(Number(params[0])); + case 'getblockheader': { + const height = Object.keys(hashes).map(Number).find((h) => hashes[h] === params[0]); + const found = height ?? [...Array(headers + 1).keys()].find((h) => hashAt(h) === params[0]); + if (found === undefined) { + throw new Error('Block not found'); + } + return { hash: params[0], height: found, confirmations: blocks - found + 1 }; + } + case 'getblockchaininfo': + return { chain: 'regtest', blocks, headers, bestblockhash: tip, initialblockdownload: overrides.ibd ?? false, size_on_disk: 12345678 }; + case 'getnetworkinfo': + return { version: 280000, subversion: overrides.subversion ?? '/Satoshi:28.0.0/' }; + case 'getchainstates': + return { headers, chainstates }; + case 'help': + return overrides.help ?? 'dumptxoutset "path"\ngetchainstates\nloadtxoutset "path"\n'; + case 'getbestblockhash': + return tip; + default: + throw new Error('unexpected RPC ' + method); + } + }, + }; +} diff --git a/backend/src/api/intelligence/bootstrap/bootstrap-capacity.ts b/backend/src/api/intelligence/bootstrap/bootstrap-capacity.ts new file mode 100644 index 00000000000..a55cd470e24 --- /dev/null +++ b/backend/src/api/intelligence/bootstrap/bootstrap-capacity.ts @@ -0,0 +1,70 @@ +import { promises as fs } from 'fs'; +import { BootstrapEnvironment } from './bootstrap-config'; +import { BootstrapEvidenceError } from './bootstrap-errors'; + +export interface CapacityMeasurement { + method: 'statfs' | 'configured'; + measured_at: string; + free_bytes: number; + total_bytes: number | null; +} + +export type StatfsReader = (path: string) => Promise<{ bsize: number; bavail: number; blocks: number }>; + +/** + * The node's disk capacity as this backend can actually measure it: a statfs + * of the configured datadir when it is on this host, otherwise the operator's + * own measurement (UNIVERSE_BOOTSTRAP_CAPACITY_BYTES with the time it was + * taken) minus the block data Core reports. A caller's declared free disk is + * never a substitute for either. + */ +/** @asyncUnsafe Rejections are typed BootstrapEvidenceErrors for the caller. */ +export async function measureCapacity( + env: BootstrapEnvironment, + blocksOnDiskBytes: number, + now: number = Date.now(), + statfs: StatfsReader = (path) => fs.statfs(path) as Promise<{ bsize: number; bavail: number; blocks: number }> +): Promise { + if (env.datadir) { + try { + const stat = await statfs(env.datadir); + return { + method: 'statfs', + measured_at: new Date(now).toISOString(), + free_bytes: Number(stat.bavail) * Number(stat.bsize), + total_bytes: Number(stat.blocks) * Number(stat.bsize), + }; + } catch (e) { + throw new BootstrapEvidenceError( + 'capacity-not-measured', + 'The configured datadir could not be measured: ' + (e instanceof Error ? e.message : String(e)) + ); + } + } + if (env.capacityBytes === undefined) { + throw new BootstrapEvidenceError( + 'capacity-not-measured', + 'No owned capacity measurement is configured (UNIVERSE_BOOTSTRAP_DATADIR or UNIVERSE_BOOTSTRAP_CAPACITY_BYTES with UNIVERSE_BOOTSTRAP_CAPACITY_MEASURED_AT).' + ); + } + const measuredAt = Date.parse(env.capacityMeasuredAt ?? ''); + if (!Number.isFinite(measuredAt)) { + throw new BootstrapEvidenceError( + 'capacity-not-measured', + 'The configured capacity carries no measurement time (UNIVERSE_BOOTSTRAP_CAPACITY_MEASURED_AT).' + ); + } + if (now - measuredAt > env.capacityMaxAgeMs || measuredAt > now) { + throw new BootstrapEvidenceError( + 'stale-measurement', + `The configured capacity was measured at ${new Date(measuredAt).toISOString()}, outside the accepted age of ${env.capacityMaxAgeMs} ms.`, + 409 + ); + } + return { + method: 'configured', + measured_at: new Date(measuredAt).toISOString(), + free_bytes: Math.max(0, env.capacityBytes - blocksOnDiskBytes), + total_bytes: env.capacityBytes, + }; +} diff --git a/backend/src/api/intelligence/bootstrap/bootstrap-config.ts b/backend/src/api/intelligence/bootstrap/bootstrap-config.ts new file mode 100644 index 00000000000..541115c0e70 --- /dev/null +++ b/backend/src/api/intelligence/bootstrap/bootstrap-config.ts @@ -0,0 +1,55 @@ +/** + * Additive environment configuration for the bootstrap surface. Every key is + * optional; an absent key leaves the matching subfeature unavailable and the + * read-only node observation untouched. + */ +export interface BootstrapEnvironment { + /** Path to the trusted catalogue JSON (producers, pinnedCommitments, snapshots). */ + catalogueFile?: string; + /** Comma separated https origins and local directory prefixes a snapshot source may live under. */ + sourceAllowlist: string[]; + /** Upper bound on snapshot bytes a verification will stream. */ + maxSnapshotBytes: number; + /** Wall clock deadline for one verification run. */ + verifyDeadlineMs: number; + /** Directory on the Core host that generated snapshots are written to. */ + snapshotDir?: string; + /** Core datadir path for a statfs capacity measurement. */ + datadir?: string; + /** Operator measured volume capacity in bytes, with the time it was measured. */ + capacityBytes?: number; + capacityMeasuredAt?: string; + capacityMaxAgeMs: number; + /** Deadline for one operator RPC (dumptxoutset or loadtxoutset). */ + jobTimeoutMs: number; + /** Lease a worker holds on a running job; a crashed worker's job is reclaimed after it lapses. */ + jobLeaseMs: number; + /** How often the worker looks for queued or lapsed jobs. */ + jobPollMs: number; +} + +const integer = (value: string | undefined, fallback: number): number => { + const n = Number(value); + return value !== undefined && Number.isSafeInteger(n) && n > 0 ? n : fallback; +}; + +export function readBootstrapEnvironment(env: NodeJS.ProcessEnv = process.env): BootstrapEnvironment { + const capacity = Number(env.UNIVERSE_BOOTSTRAP_CAPACITY_BYTES); + return { + catalogueFile: env.UNIVERSE_BOOTSTRAP_SNAPSHOT_CATALOGUE || undefined, + sourceAllowlist: String(env.UNIVERSE_BOOTSTRAP_SOURCE_ALLOWLIST || '') + .split(',') + .map((entry) => entry.trim()) + .filter(Boolean), + maxSnapshotBytes: integer(env.UNIVERSE_BOOTSTRAP_MAX_SNAPSHOT_BYTES, 32 * 1024 ** 3), + verifyDeadlineMs: integer(env.UNIVERSE_BOOTSTRAP_VERIFY_DEADLINE_MS, 60 * 60 * 1000), + snapshotDir: env.UNIVERSE_BOOTSTRAP_SNAPSHOT_DIR || undefined, + datadir: env.UNIVERSE_BOOTSTRAP_DATADIR || undefined, + capacityBytes: Number.isSafeInteger(capacity) && capacity > 0 ? capacity : undefined, + capacityMeasuredAt: env.UNIVERSE_BOOTSTRAP_CAPACITY_MEASURED_AT || undefined, + capacityMaxAgeMs: integer(env.UNIVERSE_BOOTSTRAP_CAPACITY_MAX_AGE_MS, 24 * 60 * 60 * 1000), + jobTimeoutMs: integer(env.UNIVERSE_BOOTSTRAP_JOB_TIMEOUT_MS, 6 * 60 * 60 * 1000), + jobLeaseMs: integer(env.UNIVERSE_BOOTSTRAP_JOB_LEASE_MS, 5 * 60 * 1000), + jobPollMs: integer(env.UNIVERSE_BOOTSTRAP_JOB_POLL_MS, 15 * 1000), + }; +} diff --git a/backend/src/api/intelligence/bootstrap/bootstrap-errors.ts b/backend/src/api/intelligence/bootstrap/bootstrap-errors.ts new file mode 100644 index 00000000000..7ae987eb584 --- /dev/null +++ b/backend/src/api/intelligence/bootstrap/bootstrap-errors.ts @@ -0,0 +1,16 @@ +export class BootstrapEvidenceError extends Error { + constructor( + public readonly code: string, + message: string, + public readonly status = 503 + ) { + super(message); + } +} + +export const unavailable = (code: string, prerequisite: string): never => { + throw new BootstrapEvidenceError( + code, + 'Bootstrap evidence is unavailable. ' + prerequisite + ); +}; diff --git a/backend/src/api/intelligence/bootstrap/bootstrap-executor.test.ts b/backend/src/api/intelligence/bootstrap/bootstrap-executor.test.ts new file mode 100644 index 00000000000..b131c3e4437 --- /dev/null +++ b/backend/src/api/intelligence/bootstrap/bootstrap-executor.test.ts @@ -0,0 +1,220 @@ +import { BootstrapJobExecutor } from './bootstrap-executor'; +import { MemoryBootstrapStore } from './bootstrap-store'; +import { BootstrapService } from './bootstrap.service'; +import { BootstrapNodeReader } from './node-reader'; +import { NodeBootstrapJob } from './bootstrap.models'; +import { byteSource, catalogueFor, fakeCore, FIXTURE, FIXTURE_BYTES, producerKeys, testEnvironment } from './__fixtures__/support'; + +const keys = producerKeys(); +const catalogue = catalogueFor(keys.privateKey, keys.publicKeyHex); +const statfsPlenty = async (): Promise<{ bsize: number; bavail: number; blocks: number }> => ({ bsize: 4096, bavail: 1 << 20, blocks: 1 << 21 }); +const auth = { keyId: 'control-plane', elevated: true }; +const key = (suffix: string): string => 'idem-' + suffix + '-' + Math.random().toString(16).slice(2, 10); + +/** A Core whose chainstates can be swapped after loadtxoutset, as the real node does. */ +function switchableCore(): { current: ReturnType; reader: BootstrapNodeReader; calls: string[]; swap: (next: ReturnType) => void } { + const calls: string[] = []; + const holder = { current: fakeCore({ blocks: 10, headers: 200, ibd: true, calls }) }; + const proxy = { network: 'regtest', call: (m: string, p: unknown[]) => holder.current.call(m, p) }; + return { get current() { return holder.current; }, reader: new BootstrapNodeReader(proxy), calls, swap: (next) => { holder.current = next; } }; +} + +function operatorCore(handler: (method: string, params: unknown[]) => Promise): { calls: Array<{ method: string; params: unknown[] }>; call: (method: 'dumptxoutset' | 'loadtxoutset', params: unknown[]) => Promise } { + const calls: Array<{ method: string; params: unknown[] }> = []; + return { calls, call: (method, params) => { calls.push({ method, params }); return handler(method, params); } }; +} + +function build(options: { core?: ReturnType; store?: MemoryBootstrapStore | null; env?: Parameters[0]; handler?: (method: string, params: unknown[]) => Promise } = {}): { + service: BootstrapService; store: MemoryBootstrapStore; core: ReturnType; operator: ReturnType; env: ReturnType; +} { + const core = options.core ?? switchableCore(); + const store = options.store === undefined ? new MemoryBootstrapStore() : (options.store as MemoryBootstrapStore); + const env = testEnvironment(options.env); + const operator = operatorCore(options.handler ?? (async () => { throw new Error('no handler'); })); + const service = new BootstrapService({ + nodes: core.reader, core: { network: 'regtest', call: (m, p) => core.current.call(m, p) }, env, store, bytes: byteSource(FIXTURE_BYTES), network: 'regtest', + statfs: statfsPlenty, catalogue: () => catalogue, operatorCore: operator, workerOwner: 'worker-a', + }); + return { service, store, core, operator, env }; +} + +async function verifyFixture(service: BootstrapService): Promise { + const pending = await service.verifySnapshot({ snapshot_id: 'regtest-105' }); + const done = await service.awaitVerification(pending.verification_id); + expect(done?.state).toBe('valid'); +} + +function executor(built: ReturnType, overrides: Partial[0]> = {}): BootstrapJobExecutor { + return new BootstrapJobExecutor({ + env: built.env, network: 'regtest', store: built.store, nodes: built.core.reader, core: { network: 'regtest', call: (m, p) => built.core.current.call(m, p) }, + operatorCore: built.operator, catalogue: () => catalogue, owner: 'worker-a', sleep: async () => undefined, statfs: statfsPlenty, loadObservationAttempts: 3, ...overrides, + }); +} + +const loadedCore = (): ReturnType => fakeCore({ + blocks: FIXTURE.height, headers: 200, ibd: true, + chainstates: [ + { blocks: 10, bestblockhash: require('crypto').createHash('sha256').update('block10').digest('hex'), verificationprogress: 0.1, validated: true, coins_db_cache_bytes: 1, coins_tip_cache_bytes: 1 }, + { blocks: FIXTURE.height, bestblockhash: FIXTURE.baseHash, verificationprogress: 0.5, validated: false, coins_db_cache_bytes: 1, coins_tip_cache_bytes: 1, snapshot_blockhash: FIXTURE.baseHash }, + ], +}); + +describe('Bootstrap operator jobs', () => { + it('rejects unauthorized and unelevated requests before observing the node or touching the store', async () => { + const built = build(); + await expect(built.service.createOperatorJob({ job_type: 'generate_snapshot', idempotency_key: key('a') }, undefined)).rejects.toThrow(expect.objectContaining({ code: 'UNAUTHORIZED', status: 401 })); + await expect(built.service.createOperatorJob({ job_type: 'load_snapshot', snapshot_id: 'regtest-105', idempotency_key: key('b'), confirm: 'load_snapshot' }, { keyId: 'k', elevated: false })).rejects.toThrow(expect.objectContaining({ code: 'ELEVATION_REQUIRED', status: 403 })); + await expect(built.service.createOperatorJob({ job_type: 'load_snapshot', snapshot_id: 'regtest-105', idempotency_key: key('c') }, auth)).rejects.toThrow(expect.objectContaining({ code: 'invalid-input', status: 400 })); + await expect(built.service.createOperatorJob({ job_type: 'generate_snapshot' }, auth)).rejects.toThrow(expect.objectContaining({ code: 'invalid-input' })); + await expect(built.service.createOperatorJob({ job_type: 'reindex' as any, idempotency_key: key('d') }, auth)).rejects.toThrow(expect.objectContaining({ code: 'invalid-input' })); + expect(built.core.calls).toEqual([]); + expect(built.store.jobs.size).toBe(0); + expect(built.operator.calls).toEqual([]); + }); + + it('needs the durable store and an allowlisted output directory', async () => { + const noStore = build({ store: null }); + await expect(noStore.service.createOperatorJob({ job_type: 'generate_snapshot', idempotency_key: key('e') }, auth)).rejects.toThrow(expect.objectContaining({ code: 'durable-store-unavailable' })); + await expect(noStore.service.getJob('x')).rejects.toThrow(expect.objectContaining({ code: 'durable-store-unavailable' })); + const noDir = build({ env: { snapshotDir: undefined } }); + await expect(noDir.service.createOperatorJob({ job_type: 'generate_snapshot', idempotency_key: key('f') }, auth)).rejects.toThrow(expect.objectContaining({ code: 'unavailable-operator' })); + expect(noDir.service.startWorker()).toBeUndefined(); + expect(await build().service.getJob('missing')).toBeUndefined(); + }); + + it('generates a snapshot through dumptxoutset at a configured path and records what Core answered', async () => { + const core = switchableCore(); + core.swap(fakeCore({ calls: core.calls })); + const built = build({ + core, + handler: async (method, params) => ({ coins_written: 107, base_hash: FIXTURE.baseHash, base_height: FIXTURE.height, txoutset_hash: FIXTURE.hashSerialized3, path: params[0], nchaintx: 107 }), + }); + const idem = key('gen'); + const job = await built.service.createOperatorJob({ job_type: 'generate_snapshot', node_id: 'owned-core-regtest', idempotency_key: idem }, auth); + expect(job).toMatchObject({ state: 'queued', status: 'queued', job_type: 'generate_snapshot', node_id: 'owned-core-regtest', requested_by: 'control-plane', attempts: 0, rpc: null }); + expect(built.operator.calls).toEqual([]); + built.service.stopWorker(); + + const worker = executor(built); + const ran = await worker.runOnce(); + expect(ran?.job_id).toBe(job.job_id); + const stored = await built.service.getJob(job.job_id); + expect(stored).toMatchObject({ state: 'completed', attempts: 1, progress_pct: 100, lease: null, rpc: { method: 'dumptxoutset' } }); + expect(stored?.evidence).toMatchObject({ coins_written: 107, base_hash: FIXTURE.baseHash, base_height: FIXTURE.height, txoutset_hash: FIXTURE.hashSerialized3, core_block_hash_at_base_height: FIXTURE.baseHash }); + expect(stored?.preconditions).toMatchObject({ core_version: '28.0.0', phase: 'fully_validated', capacity_method: 'statfs' }); + expect(built.operator.calls).toHaveLength(1); + expect(built.operator.calls[0].method).toBe('dumptxoutset'); + expect(built.operator.calls[0].params).toEqual([`/var/lib/bitcoin/snapshots/utxo-regtest-${FIXTURE.height}-${job.job_id}.dat`]); + expect(stored?.checkpoints.map((c) => c.stage)).toEqual(['queued', 'claimed', 'rpc-issued', 'rpc-returned', 'completed']); + expect(await worker.runOnce()).toBeNull(); + }); + + it('refuses to generate from a node that is not fully validated', async () => { + const built = build(); + await expect(built.service.createOperatorJob({ job_type: 'generate_snapshot', idempotency_key: key('ibd') }, auth)).rejects.toThrow(expect.objectContaining({ code: 'chainstate-not-eligible', status: 409 })); + }); + + it('loads a verified snapshot through loadtxoutset and completes only after the node shows the snapshot chainstate', async () => { + const core = switchableCore(); + const built = build({ + core, + handler: async (method) => { + if (method === 'loadtxoutset') { + core.swap(loadedCore()); + return { coins_loaded: 107, tip_hash: FIXTURE.baseHash, base_height: FIXTURE.height, path: '/allowed/regtest-105.dat' }; + } + throw new Error('unexpected ' + method); + }, + }); + await expect(built.service.createOperatorJob({ job_type: 'load_snapshot', snapshot_id: 'regtest-105', idempotency_key: key('l0'), confirm: 'load_snapshot' }, auth)).rejects.toThrow(expect.objectContaining({ code: 'snapshot-not-verified' })); + await verifyFixture(built.service); + await expect(built.service.createOperatorJob({ job_type: 'load_snapshot', snapshot_id: 'unknown', idempotency_key: key('l1'), confirm: 'load_snapshot' }, auth)).rejects.toThrow(expect.objectContaining({ code: 'snapshot-not-in-catalogue', status: 404 })); + const job = await built.service.createOperatorJob({ job_type: 'load_snapshot', snapshot_id: 'regtest-105', idempotency_key: key('l2'), confirm: 'load_snapshot' }, auth); + built.service.stopWorker(); + expect(job.state).toBe('queued'); + await expect(built.service.createOperatorJob({ job_type: 'load_snapshot', snapshot_id: 'regtest-105', idempotency_key: key('l3'), confirm: 'load_snapshot' }, auth)).rejects.toThrow(expect.objectContaining({ code: 'job-already-active', status: 409 })); + + await executor(built).runOnce(); + const stored = await built.service.getJob(job.job_id) as NodeBootstrapJob; + expect(stored.state).toBe('completed'); + expect(stored.evidence).toMatchObject({ coins_loaded: 107, tip_hash: FIXTURE.baseHash, base_height: FIXTURE.height, observed_phase: 'background_validation', active_chainstate_type: 'snapshot', snapshot_chainstate_height: FIXTURE.height, background_ibd_height: 10 }); + expect(stored.preconditions).toMatchObject({ pinned_commitment: FIXTURE.hashSerialized3, snapshot_height: FIXTURE.height, phase: 'traditional_ibd' }); + expect(built.operator.calls).toEqual([{ method: 'loadtxoutset', params: ['/allowed/regtest-105.dat'] }]); + expect(stored.checkpoints.map((c) => c.stage)).toContain('chainstate-observation'); + }); + + it('leaves a load in needs-review when Core answers but no snapshot chainstate appears', async () => { + const built = build({ handler: async () => ({ coins_loaded: 107, tip_hash: FIXTURE.baseHash, base_height: FIXTURE.height }) }); + await verifyFixture(built.service); + const job = await built.service.createOperatorJob({ job_type: 'load_snapshot', snapshot_id: 'regtest-105', idempotency_key: key('nr'), confirm: 'load_snapshot' }, auth); + built.service.stopWorker(); + await executor(built).runOnce(); + expect(await built.service.getJob(job.job_id)).toMatchObject({ state: 'needs-review', reason: 'chainstate-not-observed' }); + }); + + it('returns the same job for a duplicate idempotency key and never runs Core twice', async () => { + const core = switchableCore(); + core.swap(fakeCore({ calls: core.calls })); + const built = build({ core, handler: async (_m, params) => ({ coins_written: 1, base_hash: FIXTURE.baseHash, base_height: FIXTURE.height, txoutset_hash: FIXTURE.hashSerialized3, path: params[0] }) }); + const idem = key('dup'); + const first = await built.service.createOperatorJob({ job_type: 'generate_snapshot', idempotency_key: idem }, auth); + const second = await built.service.createOperatorJob({ job_type: 'generate_snapshot', idempotency_key: idem }, auth); + built.service.stopWorker(); + expect(second.job_id).toBe(first.job_id); + expect(built.store.jobs.size).toBe(1); + const worker = executor(built); + await worker.runOnce(); + await worker.runOnce(); + expect(built.operator.calls).toHaveLength(1); + const again = await built.service.createOperatorJob({ job_type: 'generate_snapshot', idempotency_key: idem }, auth); + built.service.stopWorker(); + expect(again).toMatchObject({ job_id: first.job_id, state: 'completed' }); + }); + + it('reclaims a job whose worker died before issuing the RPC, and reconciles one that died after', async () => { + const core = switchableCore(); + core.swap(fakeCore({ calls: core.calls })); + const built = build({ core, handler: async (_m, params) => ({ coins_written: 1, base_hash: FIXTURE.baseHash, base_height: FIXTURE.height, txoutset_hash: FIXTURE.hashSerialized3, path: params[0] }) }); + const crashedBefore = await built.service.createOperatorJob({ job_type: 'generate_snapshot', idempotency_key: key('cb') }, auth); + built.service.stopWorker(); + const past = new Date(Date.now() - 1000).toISOString(); + const stale = built.store.jobs.get(crashedBefore.job_id) as NodeBootstrapJob; + Object.assign(stale, { state: 'running', status: 'running', attempts: 1, lease: { owner: 'worker-dead', expires_at: past } }); + const other = executor(built, { owner: 'worker-b', clock: () => Date.now() }); + expect((await other.runOnce())?.job_id).toBe(crashedBefore.job_id); + expect(await built.service.getJob(crashedBefore.job_id)).toMatchObject({ state: 'completed', attempts: 2 }); + expect(built.operator.calls).toHaveLength(1); + + const crashedAfter: NodeBootstrapJob = { ...(await built.service.getJob(crashedBefore.job_id) as NodeBootstrapJob), job_id: 'crashed-after', idempotency_key: key('ca'), state: 'running', status: 'running', attempts: 1, evidence: {}, checkpoints: [], finished_at: undefined, lease: { owner: 'worker-dead', expires_at: past }, rpc: { method: 'dumptxoutset', started_at: past } }; + built.store.jobs.set(crashedAfter.job_id, crashedAfter); + expect((await other.runOnce())?.job_id).toBe('crashed-after'); + expect(await built.service.getJob('crashed-after')).toMatchObject({ state: 'needs-review', reason: 'rpc-outcome-unknown-after-restart', attempts: 2 }); + expect(built.operator.calls).toHaveLength(1); + const held = built.store.jobs.get(crashedAfter.job_id) as NodeBootstrapJob; + Object.assign(held, { state: 'running', lease: { owner: 'worker-live', expires_at: new Date(Date.now() + 60000).toISOString() } }); + expect(await other.runOnce()).toBeNull(); + }); + + it('marks a job needs-review when Core does not answer within the deadline', async () => { + const core = switchableCore(); + core.swap(fakeCore({ calls: core.calls })); + const built = build({ core, env: { jobTimeoutMs: 20 }, handler: () => new Promise(() => undefined) }); + const job = await built.service.createOperatorJob({ job_type: 'generate_snapshot', idempotency_key: key('to') }, auth); + built.service.stopWorker(); + await executor(built).runOnce(); + const stored = await built.service.getJob(job.job_id); + expect(stored).toMatchObject({ state: 'needs-review', reason: 'executor-timeout', lease: null }); + expect(stored?.rpc?.finished_at).toBeUndefined(); + expect(stored?.message).toMatch(/Core may still be running it/); + }); + + it('records a Core RPC error as a failed job', async () => { + const core = switchableCore(); + core.swap(fakeCore({ calls: core.calls })); + const built = build({ core, handler: async () => { throw new Error('already exists. If you are sure this is what you want, move it out of the way first'); } }); + const job = await built.service.createOperatorJob({ job_type: 'generate_snapshot', idempotency_key: key('err') }, auth); + built.service.stopWorker(); + await executor(built).runOnce(); + expect(await built.service.getJob(job.job_id)).toMatchObject({ state: 'failed', reason: 'core-rpc-error' }); + }); +}); diff --git a/backend/src/api/intelligence/bootstrap/bootstrap-executor.ts b/backend/src/api/intelligence/bootstrap/bootstrap-executor.ts new file mode 100644 index 00000000000..fec14849678 --- /dev/null +++ b/backend/src/api/intelligence/bootstrap/bootstrap-executor.ts @@ -0,0 +1,397 @@ +import logger from '../../../logger'; +import { WorkbenchCoreReader } from '../workbench/workbench-core'; +import { BootstrapEnvironment } from './bootstrap-config'; +import { BootstrapEvidenceError } from './bootstrap-errors'; +import { evaluateLoadEligibility, NodeObservation, requireSupportedNode } from './bootstrap-preconditions'; +import { BootstrapStore } from './bootstrap-store'; +import { BootstrapCatalogue, NodeBootstrapJob, NodeBootstrapJobState } from './bootstrap.models'; +import { BootstrapNodeReader } from './node-reader'; +import { measureCapacity, StatfsReader } from './bootstrap-capacity'; +import { classifySource } from './snapshot-catalogue'; + +/** The owned Core, reached with a timeout long enough for a snapshot RPC. */ +export interface OperatorCore { + call(method: 'dumptxoutset' | 'loadtxoutset', params: unknown[]): Promise; +} + +export interface ExecutorDependencies { + env: BootstrapEnvironment; + network: string; + store: BootstrapStore; + nodes: BootstrapNodeReader; + core: WorkbenchCoreReader; + operatorCore: OperatorCore; + catalogue: () => BootstrapCatalogue; + owner: string; + clock?: () => number; + sleep?: (ms: number) => Promise; + statfs?: StatfsReader; + /** How many chainstate observations to take after loadtxoutset returns. */ + loadObservationAttempts?: number; +} + +const hash = (s: unknown): s is string => typeof s === 'string' && /^[0-9a-f]{64}$/.test(s); +const integer = (n: unknown): n is number => Number.isSafeInteger(n) && Number(n) >= 0; + +/** + * Bounded worker for operator jobs. It claims one job at a time under a + * lease, re-checks every precondition against the live node, issues exactly + * one Core RPC with a fixed path derived from configuration, and records what + * Core answered. A lapsed lease (worker crash) lets another worker reclaim + * the job; a job whose RPC was already issued is reconciled from the node, + * never re-issued. + */ +export class BootstrapJobExecutor { + private timer?: NodeJS.Timeout; + private busy = false; + private readonly clock: () => number; + private readonly sleep: (ms: number) => Promise; + + constructor(private readonly deps: ExecutorDependencies) { + this.clock = deps.clock ?? (() => Date.now()); + this.sleep = deps.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))); + } + + public start(): void { + if (this.timer) { + return; + } + this.timer = setInterval(() => { + this.runOnce().catch((e) => logger.warn('[bootstrap-executor] poll failed: ' + (e instanceof Error ? e.message : e))); + }, this.deps.env.jobPollMs); + this.timer.unref(); + } + + public stop(): void { + if (this.timer) { + clearInterval(this.timer); + this.timer = undefined; + } + } + + /** Claims and executes at most one job; returns it, or null when nothing was claimable. */ + /** @asyncUnsafe The caller logs a rejection; the job itself is written before rethrowing. */ + public async runOnce(): Promise { + if (this.busy) { + return null; + } + this.busy = true; + try { + const job = await this.deps.store.claimJob(this.deps.network, this.deps.owner, this.iso(), this.deps.env.jobLeaseMs); + if (!job) { + return null; + } + await this.execute(job); + return job; + } finally { + this.busy = false; + } + } + + private iso(): string { + return new Date(this.clock()).toISOString(); + } + + /** @asyncUnsafe Every branch ends by writing the job; a lost lease stops writes. */ + private async execute(job: NodeBootstrapJob): Promise { + const owner = this.deps.owner; + let leaseHeld = true; + /** @asyncUnsafe A failed write rejects into execute(), which records it. */ + const write = async (): Promise => { + if (!leaseHeld) { + return; + } + job.updated_at = this.iso(); + job.status = job.state; + leaseHeld = await this.deps.store.updateJob(job, owner); + }; + const checkpoint = (stage: string, detail?: string): void => { + job.checkpoints.push({ at: this.iso(), stage, ...(detail ? { detail } : {}) }); + }; + /** @asyncUnsafe Rejections are typed for execute(). */ + const finish = async (state: Exclude, message: string, reason?: string): Promise => { + job.state = state; + job.message = message; + job.reason = reason; + job.finished_at = this.iso(); + job.lease = null; + checkpoint(state, message); + await write(); + }; + const heartbeat = setInterval(() => { + if (!leaseHeld || !job.lease) { + return; + } + job.lease = { owner, expires_at: new Date(this.clock() + this.deps.env.jobLeaseMs).toISOString() }; + write().catch((e) => logger.warn('[bootstrap-executor] lease renewal failed: ' + (e instanceof Error ? e.message : e))); + }, Math.max(1000, Math.floor(this.deps.env.jobLeaseMs / 3))); + heartbeat.unref(); + + try { + job.attempts += 1; + job.started_at = job.started_at ?? this.iso(); + checkpoint('claimed', `${owner} attempt ${job.attempts}`); + await write(); + + if (job.rpc?.started_at && !job.rpc.finished_at) { + await this.reconcile(job, finish, checkpoint); + return; + } + + let observation: NodeObservation; + try { + observation = await this.deps.nodes.read(); + } catch (e) { + await finish('failed', 'The owned node could not be observed before execution: ' + (e instanceof Error ? e.message : String(e)), 'unavailable-node-source'); + return; + } + if (observation.capability.node_id !== job.node_id) { + await finish('failed', `The observed node is ${observation.capability.node_id}, not ${job.node_id}.`, 'node-not-observed'); + return; + } + + if (job.job_type === 'generate_snapshot') { + await this.generate(job, observation, finish, checkpoint, write); + } else { + await this.load(job, observation, finish, checkpoint, write); + } + } catch (e) { + if (e instanceof BootstrapEvidenceError) { + await finish('failed', e.message, e.code); + } else { + await finish('failed', 'The executor failed: ' + (e instanceof Error ? e.message : String(e)), 'executor-error'); + } + } finally { + clearInterval(heartbeat); + } + } + + /** @asyncUnsafe Rejections are typed for execute(). */ + private async generate( + job: NodeBootstrapJob, + observation: NodeObservation, + finish: (state: 'completed' | 'failed' | 'needs-review', message: string, reason?: string) => Promise, + checkpoint: (stage: string, detail?: string) => void, + write: () => Promise + ): Promise { + const version = requireSupportedNode(observation, this.deps.network); + if (!observation.capability.supports_dumptxoutset) { + throw new BootstrapEvidenceError('node-capability-missing', 'The owned node does not expose dumptxoutset.', 409); + } + const chain = observation.observation; + if (chain.current_phase !== 'fully_validated') { + throw new BootstrapEvidenceError('chainstate-not-eligible', `A snapshot is only generated from a fully validated chainstate; the node is in ${chain.current_phase}.`, 409); + } + if (!this.deps.env.snapshotDir) { + throw new BootstrapEvidenceError('unavailable-operator', 'No allowlisted snapshot output directory is configured (UNIVERSE_BOOTSTRAP_SNAPSHOT_DIR).'); + } + const capacity = await measureCapacity(this.deps.env, Math.round(chain.disk_used_gb * 1073741824), this.clock(), this.deps.statfs); + const path = this.deps.env.snapshotDir.replace(/[\\/]+$/, '') + `/utxo-${this.deps.network}-${chain.tip_height}-${job.job_id}.dat`; + job.preconditions = { + core_version: version.version, + phase: chain.current_phase, + tip_height: chain.tip_height, + tip_hash: chain.active_chainstate.best_block_hash, + capacity_method: capacity.method, + capacity_free_bytes: capacity.free_bytes, + output_path: path, + }; + const params = version.major >= 29 ? [path, 'latest'] : [path]; + const result = await this.issue(job, 'dumptxoutset', params, checkpoint, write); + if (result === undefined) { + return; + } + if (!integer(result?.coins_written) || !hash(result?.base_hash) || !integer(result?.base_height) || !hash(result?.txoutset_hash)) { + await finish('needs-review', 'dumptxoutset returned without the expected fields; inspect the node.', 'core-result-malformed'); + return; + } + const hashAtHeight = await this.deps.core.call('getblockhash', [result.base_height]); + job.evidence = { + coins_written: result.coins_written, + base_hash: result.base_hash, + base_height: result.base_height, + txoutset_hash: result.txoutset_hash, + path: typeof result.path === 'string' ? result.path : path, + core_block_hash_at_base_height: typeof hashAtHeight === 'string' ? hashAtHeight : null, + }; + if (hashAtHeight !== result.base_hash) { + await finish('needs-review', 'The snapshot base hash is not on the owned node active chain at its height.', 'base-hash-not-on-chain'); + return; + } + job.progress_pct = 100; + await finish('completed', `Core wrote ${result.coins_written} coins at height ${result.base_height} (${result.txoutset_hash}). The file is not catalogued or verified by this job.`); + } + + /** @asyncUnsafe Rejections are typed for execute(). */ + private async load( + job: NodeBootstrapJob, + observation: NodeObservation, + finish: (state: 'completed' | 'failed' | 'needs-review', message: string, reason?: string) => Promise, + checkpoint: (stage: string, detail?: string) => void, + write: () => Promise + ): Promise { + const catalogue = this.deps.catalogue(); + const snapshot = catalogue.snapshots.find((s) => s.id === job.snapshot_id); + if (!snapshot) { + throw new BootstrapEvidenceError('snapshot-not-in-catalogue', `Snapshot ${job.snapshot_id} is not in the trusted catalogue.`, 409); + } + const eligibility = await evaluateLoadEligibility({ + observation, network: this.deps.network, snapshot, catalogue, store: this.deps.store, env: this.deps.env, now: this.clock(), statfs: this.deps.statfs, + }); + const source = classifySource(snapshot.source, this.deps.env.sourceAllowlist); + if (source.kind !== 'file') { + throw new BootstrapEvidenceError('snapshot-source-not-local', 'loadtxoutset reads a path on the Core host; the catalogue source for this snapshot is not a local path.', 409); + } + job.preconditions = { + core_version: eligibility.version.version, + phase: observation.observation.current_phase, + tip_height: observation.observation.tip_height, + snapshot_height: snapshot.height, + snapshot_block_hash: snapshot.blockHash, + pinned_commitment: eligibility.pinned.utxoCommitment, + verification_id: eligibility.verification.verification_id, + capacity_method: eligibility.capacity.method, + capacity_free_bytes: eligibility.capacity.free_bytes, + required_bytes: eligibility.required.required_bytes, + }; + const result = await this.issue(job, 'loadtxoutset', [snapshot.source], checkpoint, write); + if (result === undefined) { + return; + } + if (!integer(result?.coins_loaded) || !hash(result?.tip_hash) || !integer(result?.base_height)) { + await finish('needs-review', 'loadtxoutset returned without the expected fields; inspect the node.', 'core-result-malformed'); + return; + } + job.evidence = { coins_loaded: result.coins_loaded, tip_hash: result.tip_hash, base_height: result.base_height }; + if (result.tip_hash !== snapshot.blockHash || result.base_height !== snapshot.height) { + await finish('needs-review', 'Core activated a snapshot whose base is not the catalogue block.', 'base-mismatch'); + return; + } + await this.observeLoadedChainstate(job, snapshot.height, finish, checkpoint, write); + } + + /** Issues the single RPC under the job deadline; undefined means the job was already finished. */ + /** @asyncUnsafe A Core rejection is recorded as the job outcome. */ + private async issue( + job: NodeBootstrapJob, + method: 'dumptxoutset' | 'loadtxoutset', + params: unknown[], + checkpoint: (stage: string, detail?: string) => void, + write: () => Promise + ): Promise { + const rpc: NonNullable = { method, started_at: this.iso() }; + job.rpc = rpc; + checkpoint('rpc-issued', method); + await write(); + let timer: NodeJS.Timeout | undefined; + const timeout = new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new ExecutorTimeout()), this.deps.env.jobTimeoutMs); + timer.unref(); + }); + try { + const result = await Promise.race([this.deps.operatorCore.call(method, params), timeout]); + rpc.finished_at = this.iso(); + checkpoint('rpc-returned', method); + return result; + } catch (e) { + if (e instanceof ExecutorTimeout) { + job.state = 'needs-review'; + job.message = `${method} did not return within ${this.deps.env.jobTimeoutMs} ms; Core may still be running it. Inspect the node before retrying.`; + job.reason = 'executor-timeout'; + } else { + rpc.finished_at = this.iso(); + job.state = 'failed'; + job.message = `${method} failed: ` + (e instanceof Error ? e.message : String(e)); + job.reason = 'core-rpc-error'; + } + job.finished_at = this.iso(); + job.lease = null; + checkpoint(job.state, job.message); + await write(); + return undefined; + } finally { + if (timer) { + clearTimeout(timer); + } + } + } + + /** @asyncUnsafe Rejections are typed for execute(). */ + private async observeLoadedChainstate( + job: NodeBootstrapJob, + height: number, + finish: (state: 'completed' | 'failed' | 'needs-review', message: string, reason?: string) => Promise, + checkpoint: (stage: string, detail?: string) => void, + write: () => Promise + ): Promise { + const attempts = this.deps.loadObservationAttempts ?? 20; + for (let i = 0; i < attempts; i++) { + let observed: NodeObservation | undefined; + try { + observed = await this.deps.nodes.read(); + } catch (e) { + checkpoint('chainstate-observation', 'unavailable: ' + (e instanceof Error ? e.message : String(e))); + } + if (observed) { + const chain = observed.observation; + job.evidence = { + ...job.evidence, + observed_phase: chain.current_phase, + active_chainstate_type: chain.active_chainstate.type, + snapshot_chainstate_height: chain.snapshot_chainstate_height, + background_ibd_height: chain.background_ibd_height, + sync_percent: chain.sync_percent, + observed_at: chain.observed_at, + }; + checkpoint('chainstate-observation', `${chain.current_phase} at ${chain.tip_height}`); + if (chain.active_chainstate.type === 'snapshot' && chain.snapshot_chainstate_height !== null && chain.snapshot_chainstate_height >= height) { + job.progress_pct = 100; + await finish('completed', `Core activated the snapshot chainstate; background validation continues (${chain.current_phase}).`); + return; + } + await write(); + } + await this.sleep(this.deps.env.jobPollMs); + } + await finish('needs-review', 'loadtxoutset returned but no snapshot chainstate was observed on the node afterwards.', 'chainstate-not-observed'); + } + + /** After a crash mid-RPC the outcome is read from the node, never re-issued. */ + /** @asyncUnsafe Rejections are typed for execute(). */ + private async reconcile( + job: NodeBootstrapJob, + finish: (state: 'completed' | 'failed' | 'needs-review', message: string, reason?: string) => Promise, + checkpoint: (stage: string, detail?: string) => void + ): Promise { + checkpoint('reconcile', `${job.rpc?.method} was issued by a previous attempt`); + if (job.job_type === 'generate_snapshot') { + await finish('needs-review', 'dumptxoutset was issued before this worker restarted; whether Core finished writing the file is unknown. Inspect the node and the output path before retrying.', 'rpc-outcome-unknown-after-restart'); + return; + } + let observed: NodeObservation; + try { + observed = await this.deps.nodes.read(); + } catch (e) { + await finish('needs-review', 'loadtxoutset was issued before this worker restarted and the node cannot be observed: ' + (e instanceof Error ? e.message : String(e)), 'rpc-outcome-unknown-after-restart'); + return; + } + const chain = observed.observation; + const height = typeof job.preconditions.snapshot_height === 'number' ? job.preconditions.snapshot_height : -1; + job.evidence = { + ...job.evidence, + observed_phase: chain.current_phase, + active_chainstate_type: chain.active_chainstate.type, + snapshot_chainstate_height: chain.snapshot_chainstate_height, + background_ibd_height: chain.background_ibd_height, + observed_at: chain.observed_at, + }; + if (chain.active_chainstate.type === 'snapshot' && chain.snapshot_chainstate_height !== null && chain.snapshot_chainstate_height >= height && height >= 0) { + job.progress_pct = 100; + await finish('completed', 'The node shows the snapshot chainstate active after a worker restart; background validation continues.'); + return; + } + await finish('needs-review', 'loadtxoutset was issued before this worker restarted and the node shows no snapshot chainstate. Inspect the node before retrying.', 'rpc-outcome-unknown-after-restart'); + } +} + +class ExecutorTimeout extends Error {} diff --git a/backend/src/api/intelligence/bootstrap/bootstrap-preconditions.ts b/backend/src/api/intelligence/bootstrap/bootstrap-preconditions.ts new file mode 100644 index 00000000000..92e4cb00124 --- /dev/null +++ b/backend/src/api/intelligence/bootstrap/bootstrap-preconditions.ts @@ -0,0 +1,132 @@ +import { BootstrapNodeReader } from './node-reader'; +import { BootstrapEnvironment } from './bootstrap-config'; +import { BootstrapEvidenceError } from './bootstrap-errors'; +import { BootstrapStore } from './bootstrap-store'; +import { CapacityMeasurement, measureCapacity, StatfsReader } from './bootstrap-capacity'; +import { + BootstrapCatalogue, + BootstrapCatalogueSnapshot, + BootstrapPinnedCommitment, + NodeBootstrapVerification, +} from './bootstrap.models'; +import { pinnedCommitmentFor } from './snapshot-catalogue'; +import { NETWORK_MAGIC } from './snapshot-format'; + +export type NodeObservation = Awaited>; + +/** assumeutxo shipped in 26.0; the versioned snapshot file format in 28.0. */ +export const MIN_SUPPORTED_CORE_MAJOR = 26; +export const VERSIONED_SNAPSHOT_MAJOR = 28; +/** Chainstate on disk is assumed to need as much as the snapshot file; headroom is a tenth on top. */ +export const CHAINSTATE_ESTIMATE_FACTOR = 1; +export const HEADROOM_FACTOR = 0.1; + +export function parseCoreVersion(subversion: string): { version: string; major: number } | null { + const match = /^\/Satoshi:(\d+)\.(\d+)\.(\d+)/.exec(subversion); + return match ? { version: `${match[1]}.${match[2]}.${match[3]}`, major: Number(match[1]) } : null; +} + +export function requireSupportedNode(observation: NodeObservation, network: string): { version: string; major: number } { + const { capability } = observation; + if (!NETWORK_MAGIC[capability.network] || capability.network !== network) { + throw new BootstrapEvidenceError('unsupported-network', `The owned node serves ${capability.network}, not ${network}.`, 409); + } + const parsed = parseCoreVersion(capability.exact_version); + if (!parsed || parsed.major < MIN_SUPPORTED_CORE_MAJOR) { + throw new BootstrapEvidenceError('unsupported-version', `The owned node reports ${capability.exact_version}; AssumeUTXO needs Bitcoin Core ${MIN_SUPPORTED_CORE_MAJOR}.0 or later.`, 409); + } + return parsed; +} + +export function requireFreshObservation(observation: NodeObservation, now: number, maxAgeMs: number): void { + const age = now - Date.parse(observation.observation.observed_at); + if (!Number.isFinite(age) || age > maxAgeMs || age < 0) { + throw new BootstrapEvidenceError('stale-measurement', `The node observation is ${age} ms old, outside the accepted ${maxAgeMs} ms.`, 409); + } +} + +export function requiredBytesForLoad(snapshotBytes: number): { chainstate_estimate_bytes: number; headroom_bytes: number; required_bytes: number } { + const chainstate = Math.ceil(snapshotBytes * CHAINSTATE_ESTIMATE_FACTOR); + const headroom = Math.ceil((snapshotBytes + chainstate) * HEADROOM_FACTOR); + return { chainstate_estimate_bytes: chainstate, headroom_bytes: headroom, required_bytes: snapshotBytes + chainstate + headroom }; +} + +export interface LoadEligibility { + version: { version: string; major: number }; + pinned: BootstrapPinnedCommitment; + verification: NodeBootstrapVerification; + capacity: CapacityMeasurement; + required: ReturnType; +} + +/** A snapshot the observed node can load: same format family and a commitment pinned for its exact release. */ +export function compatibleSnapshots( + catalogue: BootstrapCatalogue, + network: string, + version: { version: string; major: number } +): Array<{ snapshot: BootstrapCatalogueSnapshot; pinned: BootstrapPinnedCommitment }> { + const versioned = version.major >= VERSIONED_SNAPSHOT_MAJOR; + const out: Array<{ snapshot: BootstrapCatalogueSnapshot; pinned: BootstrapPinnedCommitment }> = []; + for (const snapshot of catalogue.snapshots) { + const producedBy = parseCoreVersion('/Satoshi:' + snapshot.coreVersion + '/'); + if (snapshot.network !== network || !producedBy || (producedBy.major >= VERSIONED_SNAPSHOT_MAJOR) !== versioned) { + continue; + } + const pinned = pinnedCommitmentFor(catalogue, { ...snapshot, coreVersion: version.version }); + if (pinned) { + out.push({ snapshot, pinned }); + } + } + return out.sort((a, b) => b.snapshot.height - a.snapshot.height); +} + +/** + * Everything a load needs, checked against measurements rather than the + * request: node version and network, chainstate phase, a commitment pinned + * for the node's exact release, a verification run that reached valid over + * the bytes, and measured capacity for file, chainstate and headroom. + */ +/** @asyncUnsafe Rejections are typed BootstrapEvidenceErrors for the caller. */ +export async function evaluateLoadEligibility(input: { + observation: NodeObservation; + network: string; + snapshot: BootstrapCatalogueSnapshot; + catalogue: BootstrapCatalogue; + store: BootstrapStore; + env: BootstrapEnvironment; + now: number; + statfs?: StatfsReader; +}): Promise { + const { observation, snapshot, catalogue } = input; + const version = requireSupportedNode(observation, input.network); + if (!observation.capability.supports_loadtxoutset) { + throw new BootstrapEvidenceError('node-capability-missing', 'The owned node does not expose loadtxoutset.', 409); + } + if (snapshot.network !== input.network) { + throw new BootstrapEvidenceError('unsupported-network', `Snapshot ${snapshot.id} is for ${snapshot.network}.`, 409); + } + const match = compatibleSnapshots(catalogue, input.network, version).find((entry) => entry.snapshot.id === snapshot.id); + if (!match) { + throw new BootstrapEvidenceError('no-compatible-snapshot', `No commitment is pinned for snapshot ${snapshot.id} on Bitcoin Core ${version.version}, or its file format predates that release.`, 409); + } + const verification = await input.store.latestVerification(snapshot.id, input.network); + if (!verification || verification.state !== 'valid') { + throw new BootstrapEvidenceError('snapshot-not-verified', `Snapshot ${snapshot.id} has no verification run that reached valid (latest: ${verification?.state ?? 'none'}).`, 409); + } + const chain = observation.observation; + if ( + chain.current_phase !== 'traditional_ibd' || + chain.dual_chainstate_active || + chain.active_chainstate.type !== 'ibd' || + chain.tip_height >= snapshot.height || + observation.observation.estimated_remaining_blocks + chain.tip_height < snapshot.height + ) { + throw new BootstrapEvidenceError('chainstate-not-eligible', `The node is in phase ${chain.current_phase} at height ${chain.tip_height}; loading needs a plain IBD chainstate below the snapshot height with headers past it.`, 409); + } + const capacity = await measureCapacity(input.env, Math.round(chain.disk_used_gb * 1073741824), input.now, input.statfs); + const required = requiredBytesForLoad(snapshot.sizeBytes); + if (capacity.free_bytes < required.required_bytes) { + throw new BootstrapEvidenceError('insufficient-capacity', `Measured free space ${capacity.free_bytes} bytes is below the ${required.required_bytes} bytes the load needs.`, 409); + } + return { version, pinned: match.pinned, verification, capacity, required }; +} diff --git a/backend/src/api/intelligence/bootstrap/bootstrap-store.test.ts b/backend/src/api/intelligence/bootstrap/bootstrap-store.test.ts new file mode 100644 index 00000000000..e82a979838c --- /dev/null +++ b/backend/src/api/intelligence/bootstrap/bootstrap-store.test.ts @@ -0,0 +1,70 @@ +jest.mock('../../../database', () => ({ __esModule: true, default: { query: jest.fn() } })); + +import DB from '../../../database'; +import { JOBS_TABLE, MysqlBootstrapStore, VERIFICATIONS_TABLE } from './bootstrap-store'; +import { NodeBootstrapJob } from './bootstrap.models'; + +const query = DB.query as jest.Mock; + +function job(overrides: Partial = {}): NodeBootstrapJob { + const now = '2026-09-17T00:00:00.000Z'; + return { + job_id: 'job-1', job_type: 'generate_snapshot', network: 'signet', node_id: 'owned-core-signet', idempotency_key: 'idem-1', state: 'queued', status: 'queued', + progress_pct: 0, message: '', requested_by: 'ops', created_at: now, updated_at: now, attempts: 0, lease: null, rpc: null, preconditions: {}, evidence: {}, checkpoints: [], ...overrides, + }; +} + +describe('MysqlBootstrapStore', () => { + const store = new MysqlBootstrapStore(); + beforeEach(() => query.mockReset()); + + it('scopes verification reads by network and tells a missing row apart from a row', async () => { + query.mockResolvedValueOnce([[]]); + expect(await store.getVerification('v1', 'signet')).toBeNull(); + expect(query.mock.calls[0][0]).toContain(`FROM ${VERIFICATIONS_TABLE} WHERE verification_id = ? AND network = ?`); + expect(query.mock.calls[0][1]).toEqual(['v1', 'signet']); + query.mockResolvedValueOnce([[{ document: JSON.stringify({ verification_id: 'v1', state: 'valid' }) }]]); + expect(await store.getVerification('v1', 'signet')).toEqual({ verification_id: 'v1', state: 'valid' }); + query.mockRejectedValueOnce(new Error('gone')); + await expect(store.getVerification('v1', 'signet')).rejects.toThrow('gone'); + }); + + it('returns the stored job when the idempotency key is already taken', async () => { + query.mockResolvedValueOnce([{ affectedRows: 1 }]); + expect(await store.insertJob(job())).toEqual({ job: job(), created: true }); + expect(query.mock.calls[0][0]).toContain(`INSERT IGNORE INTO ${JOBS_TABLE}`); + query.mockResolvedValueOnce([{ affectedRows: 0 }]); + query.mockResolvedValueOnce([[{ document: JSON.stringify(job({ job_id: 'job-0', state: 'completed' })) }]]); + const duplicate = await store.insertJob(job({ job_id: 'job-9' })); + expect(duplicate).toEqual({ job: job({ job_id: 'job-0', state: 'completed' }), created: false }); + expect(query.mock.calls[2][0]).toContain('WHERE network = ? AND idempotency_key = ?'); + expect(query.mock.calls[2][1]).toEqual(['signet', 'idem-1']); + }); + + it('claims a queued or lapsed job only through a conditional lease update', async () => { + query.mockResolvedValueOnce([[{ job_id: 'job-1' }]]); + query.mockResolvedValueOnce([{ affectedRows: 0 }]); + expect(await store.claimJob('signet', 'worker-a', '2026-09-17T00:00:00.000Z', 60000)).toBeNull(); + expect(query.mock.calls[0][0]).toContain(`state = 'queued' OR (state = 'running' AND lease_expires_at < ?)`); + expect(query.mock.calls[1][0]).toContain(`SET state = 'running', lease_owner = ?, lease_expires_at = ?`); + expect(query.mock.calls[1][0]).toContain(`WHERE job_id = ? AND network = ? AND (state = 'queued' OR (state = 'running' AND lease_expires_at < ?))`); + query.mockReset(); + query.mockResolvedValueOnce([[{ job_id: 'job-1' }]]); + query.mockResolvedValueOnce([{ affectedRows: 1 }]); + query.mockResolvedValueOnce([[{ document: JSON.stringify(job()) }]]); + const claimed = await store.claimJob('signet', 'worker-a', '2026-09-17T00:00:00.000Z', 60000); + expect(claimed).toMatchObject({ job_id: 'job-1', state: 'running', lease: { owner: 'worker-a', expires_at: '2026-09-17T00:01:00.000Z' } }); + query.mockResolvedValueOnce([[]]); + expect(await store.claimJob('signet', 'worker-a', '2026-09-17T00:00:00.000Z', 60000)).toBeNull(); + }); + + it('writes a job only while the same owner still holds the lease', async () => { + query.mockResolvedValueOnce([{ affectedRows: 0 }]); + expect(await store.updateJob(job({ state: 'running', lease: { owner: 'worker-a', expires_at: '2026-09-17T00:01:00.000Z' } }), 'worker-a')).toBe(false); + expect(query.mock.calls[0][0]).toContain('WHERE job_id = ? AND network = ? AND lease_owner = ?'); + expect(query.mock.calls[0][1].slice(-3)).toEqual(['job-1', 'signet', 'worker-a']); + query.mockResolvedValueOnce([{ affectedRows: 1 }]); + expect(await store.updateJob(job({ state: 'completed', lease: null }), 'worker-a')).toBe(true); + expect(query.mock.calls[1][1][1]).toBeNull(); + }); +}); diff --git a/backend/src/api/intelligence/bootstrap/bootstrap-store.ts b/backend/src/api/intelligence/bootstrap/bootstrap-store.ts new file mode 100644 index 00000000000..9df06eb9a30 --- /dev/null +++ b/backend/src/api/intelligence/bootstrap/bootstrap-store.ts @@ -0,0 +1,236 @@ +import DB from '../../../database'; +import { NodeBootstrapJob, NodeBootstrapVerification } from './bootstrap.models'; + +/** + * Durable, network-scoped records of verification runs and operator jobs. + * Rows carry the backend's own network; a caller never chooses it. The JSON + * document is the record; the columns exist for the reads and the claims. + */ +export interface BootstrapStore { + readonly kind: 'mysql' | 'memory'; + insertVerification(record: NodeBootstrapVerification): Promise; + updateVerification(record: NodeBootstrapVerification): Promise; + getVerification(verificationId: string, network: string): Promise; + /** Newest run for a snapshot, whatever its state. */ + latestVerification(snapshotId: string, network: string): Promise; + /** A run that is still pending or verifying, so a duplicate request reuses it. */ + activeVerification(snapshotId: string, network: string): Promise; + /** Inserts, or returns the job already stored under the same idempotency key. */ + insertJob(job: NodeBootstrapJob): Promise<{ job: NodeBootstrapJob; created: boolean }>; + getJob(jobId: string, network: string): Promise; + /** A job still queued or running for this node, so two disruptive jobs never overlap. */ + activeJob(network: string, nodeId: string): Promise; + /** Atomically leases the oldest queued job, or a running one whose lease lapsed. */ + claimJob(network: string, owner: string, nowIso: string, leaseMs: number): Promise; + /** Writes the document only while this owner still holds the lease. */ + updateJob(job: NodeBootstrapJob, owner: string): Promise; +} + +export const VERIFICATIONS_TABLE = 'universe_bootstrap_verifications'; +export const JOBS_TABLE = 'universe_bootstrap_jobs'; + +function toDate(iso: string): Date { + return new Date(iso); +} +function parseDocument(document: unknown): T { + if (typeof document === 'string') { + return JSON.parse(document); + } + if (Buffer.isBuffer(document)) { + return JSON.parse(document.toString('utf8')); + } + return document as T; +} + +export class MysqlBootstrapStore implements BootstrapStore { + public readonly kind = 'mysql' as const; + + /** @asyncUnsafe Callers turn a rejection into an exact HTTP answer. */ + public async insertVerification(record: NodeBootstrapVerification): Promise { + await DB.query( + `INSERT INTO ${VERIFICATIONS_TABLE} (verification_id, snapshot_id, network, state, created_at, updated_at, document) VALUES (?, ?, ?, ?, ?, ?, ?)`, + [record.verification_id, record.snapshot_id, record.network, record.state, toDate(record.requested_at), toDate(record.requested_at), JSON.stringify(record)] + ); + } + + /** @asyncUnsafe Callers turn a rejection into an exact HTTP answer. */ + public async updateVerification(record: NodeBootstrapVerification): Promise { + await DB.query( + `UPDATE ${VERIFICATIONS_TABLE} SET state = ?, updated_at = ?, document = ? WHERE verification_id = ? AND network = ?`, + [record.state, new Date(), JSON.stringify(record), record.verification_id, record.network] + ); + } + + /** @asyncUnsafe Callers turn a rejection into an exact HTTP answer. */ + public async getVerification(verificationId: string, network: string): Promise { + const [rows]: any[] = await DB.query( + `SELECT document FROM ${VERIFICATIONS_TABLE} WHERE verification_id = ? AND network = ? LIMIT 1`, + [verificationId, network] + ); + return rows?.length ? parseDocument(rows[0].document) : null; + } + + /** @asyncUnsafe Callers turn a rejection into an exact HTTP answer. */ + public async latestVerification(snapshotId: string, network: string): Promise { + const [rows]: any[] = await DB.query( + `SELECT document FROM ${VERIFICATIONS_TABLE} WHERE snapshot_id = ? AND network = ? ORDER BY created_at DESC LIMIT 1`, + [snapshotId, network] + ); + return rows?.length ? parseDocument(rows[0].document) : null; + } + + /** @asyncUnsafe Callers turn a rejection into an exact HTTP answer. */ + public async activeVerification(snapshotId: string, network: string): Promise { + const [rows]: any[] = await DB.query( + `SELECT document FROM ${VERIFICATIONS_TABLE} WHERE snapshot_id = ? AND network = ? AND state IN ('pending', 'verifying') ORDER BY created_at DESC LIMIT 1`, + [snapshotId, network] + ); + return rows?.length ? parseDocument(rows[0].document) : null; + } + + /** @asyncUnsafe Callers turn a rejection into an exact HTTP answer. */ + public async insertJob(job: NodeBootstrapJob): Promise<{ job: NodeBootstrapJob; created: boolean }> { + const [result]: any[] = await DB.query( + `INSERT IGNORE INTO ${JOBS_TABLE} (job_id, network, node_id, job_type, idempotency_key, state, lease_owner, lease_expires_at, created_at, updated_at, document) VALUES (?, ?, ?, ?, ?, ?, NULL, NULL, ?, ?, ?)`, + [job.job_id, job.network, job.node_id, job.job_type, job.idempotency_key, job.state, toDate(job.created_at), toDate(job.updated_at), JSON.stringify(job)] + ); + if (Number(result?.affectedRows ?? 0) > 0) { + return { job, created: true }; + } + const [rows]: any[] = await DB.query( + `SELECT document FROM ${JOBS_TABLE} WHERE network = ? AND idempotency_key = ? LIMIT 1`, + [job.network, job.idempotency_key] + ); + if (!rows?.length) { + throw new Error('The job store neither inserted nor found the idempotent job.'); + } + return { job: parseDocument(rows[0].document), created: false }; + } + + /** @asyncUnsafe Callers turn a rejection into an exact HTTP answer. */ + public async getJob(jobId: string, network: string): Promise { + const [rows]: any[] = await DB.query( + `SELECT document FROM ${JOBS_TABLE} WHERE job_id = ? AND network = ? LIMIT 1`, + [jobId, network] + ); + return rows?.length ? parseDocument(rows[0].document) : null; + } + + /** @asyncUnsafe Callers turn a rejection into an exact HTTP answer. */ + public async activeJob(network: string, nodeId: string): Promise { + const [rows]: any[] = await DB.query( + `SELECT document FROM ${JOBS_TABLE} WHERE network = ? AND node_id = ? AND state IN ('queued', 'running') ORDER BY created_at ASC LIMIT 1`, + [network, nodeId] + ); + return rows?.length ? parseDocument(rows[0].document) : null; + } + + /** @asyncUnsafe The worker logs and retries on the next poll. */ + public async claimJob(network: string, owner: string, nowIso: string, leaseMs: number): Promise { + const now = toDate(nowIso); + const [rows]: any[] = await DB.query( + `SELECT job_id FROM ${JOBS_TABLE} WHERE network = ? AND (state = 'queued' OR (state = 'running' AND lease_expires_at < ?)) ORDER BY created_at ASC LIMIT 1`, + [network, now] + ); + if (!rows?.length) { + return null; + } + const jobId = rows[0].job_id; + const expires = new Date(now.getTime() + leaseMs); + const [result]: any[] = await DB.query( + `UPDATE ${JOBS_TABLE} SET state = 'running', lease_owner = ?, lease_expires_at = ?, updated_at = ? WHERE job_id = ? AND network = ? AND (state = 'queued' OR (state = 'running' AND lease_expires_at < ?))`, + [owner, expires, now, jobId, network, now] + ); + if (Number(result?.affectedRows ?? 0) === 0) { + return null; + } + const job = await this.getJob(jobId, network); + if (!job) { + return null; + } + job.state = 'running'; + job.status = 'running'; + job.lease = { owner, expires_at: expires.toISOString() }; + return job; + } + + /** @asyncUnsafe The worker logs and retries on the next poll. */ + public async updateJob(job: NodeBootstrapJob, owner: string): Promise { + const [result]: any[] = await DB.query( + `UPDATE ${JOBS_TABLE} SET state = ?, lease_owner = ?, lease_expires_at = ?, updated_at = ?, document = ? WHERE job_id = ? AND network = ? AND lease_owner = ?`, + [job.state, job.lease?.owner ?? null, job.lease ? toDate(job.lease.expires_at) : null, toDate(job.updated_at), JSON.stringify(job), job.job_id, job.network, owner] + ); + return Number(result?.affectedRows ?? 0) > 0; + } +} + +/** Test double with the same claim and lease semantics; never used in production. */ +export class MemoryBootstrapStore implements BootstrapStore { + public readonly kind = 'memory' as const; + public readonly verifications = new Map(); + public readonly jobs = new Map(); + private clone(value: T): T { + return JSON.parse(JSON.stringify(value)); + } + async insertVerification(record: NodeBootstrapVerification): Promise { + this.verifications.set(record.verification_id, this.clone(record)); + } + async updateVerification(record: NodeBootstrapVerification): Promise { + this.verifications.set(record.verification_id, this.clone(record)); + } + async getVerification(id: string, network: string): Promise { + const found = this.verifications.get(id); + return found && found.network === network ? this.clone(found) : null; + } + async latestVerification(snapshotId: string, network: string): Promise { + const all = [...this.verifications.values()].filter((v) => v.snapshot_id === snapshotId && v.network === network); + all.sort((a, b) => b.requested_at.localeCompare(a.requested_at)); + return all.length ? this.clone(all[0]) : null; + } + async activeVerification(snapshotId: string, network: string): Promise { + const found = [...this.verifications.values()].find( + (v) => v.snapshot_id === snapshotId && v.network === network && (v.state === 'pending' || v.state === 'verifying') + ); + return found ? this.clone(found) : null; + } + async insertJob(job: NodeBootstrapJob): Promise<{ job: NodeBootstrapJob; created: boolean }> { + const existing = [...this.jobs.values()].find((j) => j.network === job.network && j.idempotency_key === job.idempotency_key); + if (existing) { + return { job: this.clone(existing), created: false }; + } + this.jobs.set(job.job_id, this.clone(job)); + return { job, created: true }; + } + async getJob(jobId: string, network: string): Promise { + const found = this.jobs.get(jobId); + return found && found.network === network ? this.clone(found) : null; + } + async activeJob(network: string, nodeId: string): Promise { + const found = [...this.jobs.values()].find( + (j) => j.network === network && j.node_id === nodeId && (j.state === 'queued' || j.state === 'running') + ); + return found ? this.clone(found) : null; + } + async claimJob(network: string, owner: string, nowIso: string, leaseMs: number): Promise { + const candidates = [...this.jobs.values()] + .filter((j) => j.network === network && (j.state === 'queued' || (j.state === 'running' && !!j.lease && j.lease.expires_at < nowIso))) + .sort((a, b) => a.created_at.localeCompare(b.created_at)); + if (!candidates.length) { + return null; + } + const job = candidates[0]; + job.state = 'running'; + job.status = 'running'; + job.lease = { owner, expires_at: new Date(Date.parse(nowIso) + leaseMs).toISOString() }; + job.updated_at = nowIso; + return this.clone(job); + } + async updateJob(job: NodeBootstrapJob, owner: string): Promise { + const current = this.jobs.get(job.job_id); + if (!current || current.lease?.owner !== owner) { + return false; + } + this.jobs.set(job.job_id, this.clone(job)); + return true; + } +} diff --git a/backend/src/api/intelligence/bootstrap/bootstrap.models.ts b/backend/src/api/intelligence/bootstrap/bootstrap.models.ts index 8f232de421b..a244747d00c 100644 --- a/backend/src/api/intelligence/bootstrap/bootstrap.models.ts +++ b/backend/src/api/intelligence/bootstrap/bootstrap.models.ts @@ -50,24 +50,131 @@ export interface NodeBootstrapSnapshotManifest { manifest_hash: string; } -export interface NodeBootstrapSnapshot { - snapshot_id: string; - name: string; +/** + * Trusted catalogue file named by UNIVERSE_BOOTSTRAP_SNAPSHOT_CATALOGUE. + * + * pinnedCommitments are typed in by the operator from Bitcoin Core's + * src/kernel/chainparams.cpp (m_assumeutxo_data: height, blockhash and + * hash_serialized for the exact release the node runs). They are the + * independent authority: a snapshot whose bytes hash to a commitment that is + * not pinned for the node's exact version cannot be loaded by that node, + * however well signed its manifest is. Catalogue metadata is never proof of + * the bytes; only a verification run over the bytes is. + */ +export interface BootstrapCatalogueProducer { + id: string; + algorithm: 'ed25519'; + /** 32 byte raw ed25519 public key, hex. */ + publicKey: string; +} + +export interface BootstrapPinnedCommitment { network: string; - base_height: number; - base_block_hash: string; - coins_count: number; - txoutset_hash: string; - size_gb: number; - download_url: string; - manifest: NodeBootstrapSnapshotManifest; - is_verified: boolean; - verification_status: 'verified' | 'unverified' | 'hash_mismatch'; + /** Exact Core release, for example 28.0.0 or 29.0.0. */ + coreVersion: string; + height: number; + blockHash: string; + /** Core hash_serialized_3, display byte order. */ + utxoCommitment: string; + coinCount: number | null; +} + +export interface BootstrapCatalogueSnapshot { + id: string; + network: string; + coreVersion: string; + height: number; + blockHash: string; + sizeBytes: number; + sha256: string; + /** Allowlisted https URL or local path; never taken from a request. */ + source: string; + manifest: { + producerId: string; + /** ed25519 signature, hex, over deterministicSnapshotJson(snapshot). */ + signature: string; + }; +} + +export interface BootstrapCatalogue { + producers: BootstrapCatalogueProducer[]; + pinnedCommitments: BootstrapPinnedCommitment[]; + snapshots: BootstrapCatalogueSnapshot[]; +} + +export type NodeBootstrapVerificationState = + | 'pending' + | 'verifying' + | 'valid' + | 'invalid' + | 'unavailable'; +export type NodeBootstrapCheckStatus = + | 'pending' + | 'valid' + | 'invalid' + | 'not-evaluated'; + +export interface NodeBootstrapCheck { + status: NodeBootstrapCheckStatus; + expected?: string | number | null; + observed?: string | number | null; + reason?: string; } +export interface NodeBootstrapCheckpoint { + at: string; + stage: string; + detail?: string; +} + +/** + * One verification run over the actual snapshot bytes. Every check is + * independent: sha256 of the bytes against the signed manifest, the ed25519 + * manifest signature against the producer key, network magic, base block hash + * and height against the catalogue and the owned Core node, and the streamed + * hash_serialized_3 against the operator-pinned Core commitment. Caller + * supplied checksums are compared and reported in caller_inputs; they never + * decide the state. + */ export interface NodeBootstrapVerification { verification_id: string; snapshot_id: string; + network: string; + state: NodeBootstrapVerificationState; + valid: boolean; + status: string; + requested_at: string; + started_at?: string; + finished_at?: string; + checks: { + file_size: NodeBootstrapCheck; + sha256: NodeBootstrapCheck; + manifest_signature: NodeBootstrapCheck; + network_magic: NodeBootstrapCheck; + base_block_hash: NodeBootstrapCheck; + base_height: NodeBootstrapCheck; + coins_count: NodeBootstrapCheck; + utxo_commitment: NodeBootstrapCheck; + }; + evidence: { + source_kind: 'file' | 'https' | null; + source_ref: string | null; + bytes_read: number; + header_format: 'versioned' | 'legacy' | null; + snapshot_version: number | null; + core_node_id: string | null; + core_block_hash_at_height: string | null; + pinned_commitment_source: string | null; + }; + checkpoints: NodeBootstrapCheckpoint[]; + caller_inputs: { + sha256: string | null; + utxo_hash: string | null; + height: number | null; + matched: boolean | null; + }; + reason?: string; + // Fields kept from the earlier response shape. file_size_valid: boolean; sha256_valid: boolean; manifest_hash_valid: boolean; @@ -78,6 +185,42 @@ export interface NodeBootstrapVerification { verified_at: string; } +export type NodeBootstrapSnapshotStatus = + | 'pinned_core' + | 'unverified' + | 'pending' + | 'verifying' + | 'invalid' + | 'unavailable'; + +/** A catalogue entry joined with its pinned commitment and latest verification. */ +export interface NodeBootstrapSnapshot { + snapshot_id: string; + network: string; + height: number; + block_hash: string; + coins_count: number | null; + base_utxo_hash: string | null; + sha256_checksum: string; + file_size_bytes: number; + release_version: string; + /** pinned_core only after a verification run over the bytes reached valid. */ + status: NodeBootstrapSnapshotStatus; + download_url?: string; + producer_id: string; + pinned_commitment: boolean; + latest_verification_id: string | null; + latest_verification_state: NodeBootstrapVerificationState | null; + // Fields kept from the earlier response shape. + name: string; + base_height: number; + base_block_hash: string; + txoutset_hash: string | null; + size_gb: number; + is_verified: boolean; + verification_status: 'verified' | 'unverified' | 'hash_mismatch'; +} + export interface NodeBootstrapChainstateObservation { node_id: string; active_chainstate: { @@ -100,39 +243,117 @@ export interface NodeBootstrapChainstateObservation { observed_at: string; } +export type NodeBootstrapPlanRejection = + | 'unsupported-network' + | 'unsupported-version' + | 'node-capability-missing' + | 'capacity-not-measured' + | 'stale-measurement' + | 'insufficient-capacity' + | 'no-compatible-snapshot' + | 'snapshot-not-verified' + | 'chainstate-not-eligible'; + +/** + * A feasibility statement. Nothing in it has been executed; every number is + * either measured (and says where it was measured) or a stated assumption. + */ export interface NodeBootstrapPlan { plan_id: string; - node_version: string; network: string; - traditional_ibd: { - estimated_download_gb: number; - estimated_disk_gb: number; - estimated_duration_hours_range: [number, number]; - requires_background_validation: false; + node_id: string; + node_version: string; + target_height: number; + measurement_status: 'measured'; + measured: { + node_observed_at: string; + current_phase: NodeBootstrapChainstatePhase; + tip_height: number; + headers: number; + blocks_on_disk_bytes: number; + capacity: { + method: 'statfs' | 'configured'; + measured_at: string; + free_bytes: number; + total_bytes: number | null; + }; + supports_loadtxoutset: boolean; + }; + selected_snapshot: { + snapshot_id: string; + height: number; + block_hash: string; + core_version: string; + size_bytes: number; + sha256: string; + utxo_commitment: string; + verification_id: string; + verified_at: string; }; - assumeutxo: { - snapshot_download_gb: number; - temporary_disk_extra_gb: number; - time_to_tip_minutes_range: [number, number]; - background_validation_duration_hours_range: [number, number]; - requires_background_validation: true; + requirements: { + snapshot_file_bytes: number; + chainstate_estimate_bytes: number; + headroom_bytes: number; + required_bytes: number; + free_bytes: number; + feasible: true; }; - selected_snapshot_id?: string; - index_compatibility_warning?: string; + assumptions: string[]; + caller_declared: { + available_disk_gb: number | null; + bandwidth_mbps: number | null; + matches_measurement: boolean | null; + }; + estimates: { + snapshot_download_hours: number | null; + ibd_download_hours: number | null; + basis: string; + }; + expected_transitions: NodeBootstrapChainstatePhase[]; + actions_not_executed: string[]; rollback_instructions: string[]; created_at: string; } +export type NodeBootstrapJobState = + | 'queued' + | 'running' + | 'completed' + | 'failed' + | 'needs-review'; + +/** + * A durable operator job. Queued means accepted, nothing more; only Core's own + * RPC result and the chainstate observation that follows count as completion + * evidence. + */ export interface NodeBootstrapJob { job_id: string; - job_type: 'generate_snapshot' | 'verify_snapshot' | 'load_snapshot'; + job_type: 'generate_snapshot' | 'load_snapshot'; + network: string; node_id: string; snapshot_id?: string; + idempotency_key: string; + state: NodeBootstrapJobState; + status: NodeBootstrapJobState; progress_pct: number; - status: 'pending' | 'running' | 'completed' | 'failed' | 'cancelled'; message: string; - started_at: string; + requested_by: string; + created_at: string; + updated_at: string; + started_at?: string; finished_at?: string; + attempts: number; + lease: { owner: string; expires_at: string } | null; + rpc: { + method: 'dumptxoutset' | 'loadtxoutset'; + started_at?: string; + finished_at?: string; + } | null; + preconditions: Record; + evidence: Record; + checkpoints: NodeBootstrapCheckpoint[]; + reason?: string; } export interface BootstrapOverviewResponse { diff --git a/backend/src/api/intelligence/bootstrap/bootstrap.routes.test.ts b/backend/src/api/intelligence/bootstrap/bootstrap.routes.test.ts new file mode 100644 index 00000000000..77eeb996c62 --- /dev/null +++ b/backend/src/api/intelligence/bootstrap/bootstrap.routes.test.ts @@ -0,0 +1,154 @@ +import { createHash, createHmac } from 'crypto'; +import adminControl from '@bitcoinuniverse/ecosystem-contracts/admin-control'; + +jest.mock('./bootstrap.service', () => { + class BootstrapEvidenceError extends Error { + constructor(public readonly code: string, message: string, public readonly status = 503) { + super(message); + } + } + return { + __esModule: true, + BootstrapEvidenceError, + default: { + startWorker: jest.fn(), + createOperatorJob: jest.fn(async (params: any, authorization: any) => ({ job_id: 'job-1', state: 'queued', params, authorization })), + getJob: jest.fn(async () => undefined), + }, + }; +}); + +import bootstrapRoutes, { OPERATOR_PREFIX } from './bootstrap.routes'; +import bootstrapService from './bootstrap.service'; + +type Handler = (req: any, res: any, next: (err?: unknown) => void) => unknown; + +/** Captures what the routes mount, so a request can be pushed through the operator prefix chain. */ +function mountedApp(): { uses: Array<{ path: string; handlers: Handler[] }>; posts: Record } { + const uses: Array<{ path: string; handlers: Handler[] }> = []; + const posts: Record = {}; + const app: any = { + use: (path: string, ...handlers: Handler[]) => { uses.push({ path, handlers }); return app; }, + get: () => app, + post: (path: string, ...handlers: Handler[]) => { posts[path] = handlers; return app; }, + }; + // The guard is mounted with the process replay store; this single-process + // test opts into the in-memory one explicitly, as the production selector + // never chooses it on its own. + process.env.EXPLORER_ADMIN_REPLAY_STORE = 'memory'; + bootstrapRoutes.initRoutes(app); + return { uses, posts }; +} + +function response(): { status: jest.Mock; json: jest.Mock; setHeader: jest.Mock; removeHeader: jest.Mock; locals: Record; sent: () => { status: number; body: any } } { + const res: any = { locals: {}, statusCode: 200 }; + res.status = jest.fn((code: number) => { res.statusCode = code; return res; }); + res.json = jest.fn((body: unknown) => { res.body = body; return res; }); + res.setHeader = jest.fn(); + res.removeHeader = jest.fn(); + res.sent = () => ({ status: res.statusCode, body: res.body }); + return res; +} + +// The guard verifies against the replay store asynchronously and answers or +// calls next() later, so each handler is awaited until it either advanced or +// sent a response. +async function runChain(handlers: Handler[], req: any, res: any): Promise { + for (const handler of handlers) { + const outcome = await new Promise<'next' | 'sent'>((resolve) => { + const originalJson = res.json; + res.json = jest.fn((body: unknown) => { const r = originalJson(body); resolve('sent'); return r; }); + Promise.resolve(handler(req, res, () => resolve('next'))).catch(() => resolve('sent')); + }); + if (outcome !== 'next') { + return; + } + } +} + +function request(remoteAddress: string, body: Record, headers: Record = {}): any { + const raw = Buffer.from(JSON.stringify(body), 'utf8'); + return { method: 'POST', originalUrl: `${OPERATOR_PREFIX}/loads`, headers: { 'content-type': 'application/json', ...headers }, rawBody: raw, body, socket: { remoteAddress } }; +} + +function signedHeaders(secret: Buffer, keyId: string, rawBody: Buffer, path: string): Record { + const { ADMIN_SERVICE_HEADERS, ADMIN_CONTROL_CONTRACT_VERSION, adminServiceSigningString } = adminControl as any; + const timestamp = new Date().toISOString(); + const nonce = 'nonce-' + Math.random().toString(16).slice(2, 14); + const bodyDigest = createHash('sha256').update(rawBody).digest('hex'); + const signature = createHmac('sha256', secret).update(adminServiceSigningString({ method: 'POST', path, query: '', keyId, timestamp, nonce, bodyDigest })).digest('hex'); + return { + [ADMIN_SERVICE_HEADERS.contractVersion]: ADMIN_CONTROL_CONTRACT_VERSION, + [ADMIN_SERVICE_HEADERS.keyId]: keyId, + [ADMIN_SERVICE_HEADERS.timestamp]: timestamp, + [ADMIN_SERVICE_HEADERS.nonce]: nonce, + [ADMIN_SERVICE_HEADERS.bodyDigest]: bodyDigest, + [ADMIN_SERVICE_HEADERS.signature]: signature, + }; +} + +describe('bootstrap operator routes', () => { + const secret = Buffer.alloc(32, 7); + const previousKeys = process.env.EXPLORER_ADMIN_ADAPTER_KEYS; + beforeEach(() => { + (bootstrapService.createOperatorJob as jest.Mock).mockClear(); + process.env.EXPLORER_ADMIN_ADAPTER_KEYS = 'ops:' + secret.toString('base64'); + }); + afterAll(() => { + if (previousKeys === undefined) { + delete process.env.EXPLORER_ADMIN_ADAPTER_KEYS; + } else { + process.env.EXPLORER_ADMIN_ADAPTER_KEYS = previousKeys; + } + }); + + it('mounts the admin adapter guard on the operator prefix before any handler', () => { + const { uses, posts } = mountedApp(); + expect(uses.map((u) => u.path)).toEqual([OPERATOR_PREFIX]); + expect(uses[0].handlers).toHaveLength(2); + expect(Object.keys(posts)).toEqual(expect.arrayContaining([`${OPERATOR_PREFIX}/snapshots`, `${OPERATOR_PREFIX}/loads`])); + expect(bootstrapService.startWorker).toHaveBeenCalled(); + }); + + it('answers a public origin with 404 and never reaches the service', async () => { + const { uses, posts } = mountedApp(); + const res = response(); + const req = request('203.0.113.9', { snapshot_id: 's', idempotency_key: 'idem-12345678', confirm: 'load_snapshot' }); + await runChain([uses[0].handlers[1], ...posts[`${OPERATOR_PREFIX}/loads`]], req, res); + expect(res.sent()).toEqual({ status: 404, body: { code: 'NOT_FOUND', message: 'Not found.' } }); + expect(bootstrapService.createOperatorJob).not.toHaveBeenCalled(); + }); + + it('answers an unsigned private request with a rejection and never reaches the service', async () => { + const { uses, posts } = mountedApp(); + const res = response(); + const req = request('127.0.0.1', { snapshot_id: 's', idempotency_key: 'idem-12345678', confirm: 'load_snapshot' }); + await runChain([uses[0].handlers[1], ...posts[`${OPERATOR_PREFIX}/loads`]], req, res); + expect([401, 403]).toContain(res.sent().status); + expect(bootstrapService.createOperatorJob).not.toHaveBeenCalled(); + delete process.env.EXPLORER_ADMIN_ADAPTER_KEYS; + const unconfigured = response(); + await runChain([uses[0].handlers[1], ...posts[`${OPERATOR_PREFIX}/loads`]], request('127.0.0.1', {}), unconfigured); + expect(unconfigured.sent().status).toBe(503); + expect(bootstrapService.createOperatorJob).not.toHaveBeenCalled(); + }); + + it('passes a correctly signed private request to the service with its verified authorization', async () => { + const { uses, posts } = mountedApp(); + const body = { snapshot_id: 'regtest-105', idempotency_key: 'idem-12345678', confirm: 'load_snapshot', adminAuthorization: { elevated: true } }; + const raw = Buffer.from(JSON.stringify(body), 'utf8'); + const req = request('10.0.0.5', body, signedHeaders(secret, 'ops', raw, `${OPERATOR_PREFIX}/loads`)); + const res = response(); + await runChain([uses[0].handlers[1], ...posts[`${OPERATOR_PREFIX}/loads`]], req, res); + expect(res.sent().status).toBe(202); + expect(bootstrapService.createOperatorJob).toHaveBeenCalledWith( + { job_type: 'load_snapshot', node_id: undefined, snapshot_id: 'regtest-105', idempotency_key: 'idem-12345678', confirm: 'load_snapshot' }, + { keyId: 'ops', elevated: true } + ); + const tampered = request('10.0.0.5', { ...body, snapshot_id: 'other' }, signedHeaders(secret, 'ops', raw, `${OPERATOR_PREFIX}/loads`)); + const rejected = response(); + await runChain([uses[0].handlers[1], ...posts[`${OPERATOR_PREFIX}/loads`]], tampered, rejected); + expect(rejected.sent().status).toBe(401); + expect(bootstrapService.createOperatorJob).toHaveBeenCalledTimes(1); + }); +}); diff --git a/backend/src/api/intelligence/bootstrap/bootstrap.routes.ts b/backend/src/api/intelligence/bootstrap/bootstrap.routes.ts index 8b5e466042c..284ba035a8f 100644 --- a/backend/src/api/intelligence/bootstrap/bootstrap.routes.ts +++ b/backend/src/api/intelligence/bootstrap/bootstrap.routes.ts @@ -1,4 +1,9 @@ import { Application, Request, Response } from 'express'; +import { + adminAdapterGuard, + adminAdapterJsonParser, +} from '../../admin-adapter/admin-adapter.security'; +import { createAdminReplayStore } from '../../admin-adapter/admin-adapter.replay'; import bootstrapService, { BootstrapEvidenceError } from './bootstrap.service'; function fail(res: Response, err: unknown): Response { @@ -7,8 +12,22 @@ function fail(res: Response, err: unknown): Response { return res.status(500).json({ error: 'Internal error' }); } +/** + * Operator routes reuse the control plane's own trust: a private-path origin + * and a signed request verified by the admin adapter guard, mounted here so + * it runs before any handler. The signature covers the exact request bytes; + * the adapter parser captures them, so this prefix is mounted before the + * general JSON parser in index.ts (see the /internal/admin/v1 line). + */ +export const OPERATOR_PREFIX = '/api/v1/intelligence/bootstrap/operator'; + class BootstrapRoutes { public initRoutes(app: Application): void { + // The same shared replay store as the admin adapter: a nonce accepted + // here is claimed for every backend worker, and no store means fail closed. + app.use(OPERATOR_PREFIX, adminAdapterJsonParser(), adminAdapterGuard(createAdminReplayStore())); + bootstrapService.startWorker(); + app.get( '/api/v1/intelligence/bootstrap/overview', async (req: Request, res: Response) => { @@ -70,7 +89,7 @@ class BootstrapRoutes { '/api/v1/intelligence/bootstrap/snapshots', async (req: Request, res: Response) => { try { - const snapshots = bootstrapService.listSnapshots(); + const snapshots = await bootstrapService.listSnapshots(); res.json(snapshots); } catch (err: any) { fail(res, err); @@ -82,7 +101,7 @@ class BootstrapRoutes { '/api/v1/intelligence/bootstrap/snapshots/:snapshotId', async (req: Request, res: Response) => { try { - const snapshot = bootstrapService.getSnapshot(req.params.snapshotId); + const snapshot = await bootstrapService.getSnapshot(req.params.snapshotId); if (!snapshot) { return res.status(404).json({ error: 'Snapshot not found' }); } @@ -97,11 +116,11 @@ class BootstrapRoutes { '/api/v1/intelligence/bootstrap/snapshots/:snapshotId/manifest', async (req: Request, res: Response) => { try { - const snapshot = bootstrapService.getSnapshot(req.params.snapshotId); - if (!snapshot || !snapshot.manifest) { + const manifest = bootstrapService.getSnapshotManifest(req.params.snapshotId); + if (!manifest) { return res.status(404).json({ error: 'Manifest not found' }); } - res.json(snapshot.manifest); + res.json(manifest); } catch (err: any) { fail(res, err); } @@ -112,8 +131,8 @@ class BootstrapRoutes { '/api/v1/intelligence/bootstrap/verifications', async (req: Request, res: Response) => { try { - const verification = bootstrapService.verifySnapshot(req.body); - res.json(verification); + const verification = await bootstrapService.verifySnapshot(req.body); + res.status(verification.state === 'pending' || verification.state === 'verifying' ? 202 : 200).json(verification); } catch (err: any) { fail(res, err); } @@ -124,7 +143,7 @@ class BootstrapRoutes { '/api/v1/intelligence/bootstrap/verifications/:verificationId', async (req: Request, res: Response) => { try { - const verification = bootstrapService.getVerification( + const verification = await bootstrapService.getVerification( req.params.verificationId ); if (!verification) { @@ -143,7 +162,7 @@ class BootstrapRoutes { '/api/v1/intelligence/bootstrap/plans', async (req: Request, res: Response) => { try { - const plan = bootstrapService.createBootstrapPlan(req.body); + const plan = await bootstrapService.createBootstrapPlan(req.body); res.json(plan); } catch (err: any) { fail(res, err); @@ -152,14 +171,18 @@ class BootstrapRoutes { ); app.post( - '/api/v1/intelligence/bootstrap/operator/snapshots', + `${OPERATOR_PREFIX}/snapshots`, async (req: Request, res: Response) => { try { - const job = bootstrapService.createOperatorJob({ - job_type: 'generate_snapshot', - node_id: req.body.node_id, - }); - res.json(job); + const job = await bootstrapService.createOperatorJob( + { + job_type: 'generate_snapshot', + node_id: req.body?.node_id, + idempotency_key: req.body?.idempotency_key, + }, + res.locals.adminAdapterAuthorization + ); + res.status(202).json(job); } catch (err: any) { fail(res, err); } @@ -167,15 +190,20 @@ class BootstrapRoutes { ); app.post( - '/api/v1/intelligence/bootstrap/operator/loads', + `${OPERATOR_PREFIX}/loads`, async (req: Request, res: Response) => { try { - const job = bootstrapService.createOperatorJob({ - job_type: 'load_snapshot', - node_id: req.body.node_id, - snapshot_id: req.body.snapshot_id, - }); - res.json(job); + const job = await bootstrapService.createOperatorJob( + { + job_type: 'load_snapshot', + node_id: req.body?.node_id, + snapshot_id: req.body?.snapshot_id, + idempotency_key: req.body?.idempotency_key, + confirm: req.body?.confirm, + }, + res.locals.adminAdapterAuthorization + ); + res.status(202).json(job); } catch (err: any) { fail(res, err); } @@ -186,11 +214,17 @@ class BootstrapRoutes { '/api/v1/intelligence/bootstrap/jobs/:jobId', async (req: Request, res: Response) => { try { - const job = bootstrapService.getJob(req.params.jobId); + const job = await bootstrapService.getJob(req.params.jobId); if (!job) { return res.status(404).json({ error: 'Job not found' }); } - res.json(job); + // The job id is unguessable, but the readback is public: the operator + // key id and the host path of the written snapshot stay private. + const { requested_by: _requestedBy, ...publicJob } = job as any; + const result = (publicJob as any).result && typeof (publicJob as any).result === 'object' + ? { ...(publicJob as any).result, output_path: undefined } + : (publicJob as any).result; + res.json({ ...publicJob, result }); } catch (err: any) { fail(res, err); } diff --git a/backend/src/api/intelligence/bootstrap/bootstrap.service.ts b/backend/src/api/intelligence/bootstrap/bootstrap.service.ts index 4b8c6406c08..0fda66572ea 100644 --- a/backend/src/api/intelligence/bootstrap/bootstrap.service.ts +++ b/backend/src/api/intelligence/bootstrap/bootstrap.service.ts @@ -1,36 +1,123 @@ +import { randomUUID } from 'crypto'; +import { hostname } from 'os'; +import config from '../../../config'; +import logger from '../../../logger'; +import { ownedWorkbenchCore, WorkbenchCoreReader } from '../workbench/workbench-core'; import { BootstrapNodeReader } from './node-reader'; +import { BootstrapEnvironment, readBootstrapEnvironment } from './bootstrap-config'; +import { BootstrapEvidenceError, unavailable } from './bootstrap-errors'; +import { BootstrapStore, MysqlBootstrapStore } from './bootstrap-store'; +import { BootstrapJobExecutor, OperatorCore } from './bootstrap-executor'; import { - NodeBootstrapCapability, + compatibleSnapshots, + evaluateLoadEligibility, + NodeObservation, + requireFreshObservation, + requireSupportedNode, +} from './bootstrap-preconditions'; +import { StatfsReader } from './bootstrap-capacity'; +import { + BootstrapCatalogue, + BootstrapCatalogueSnapshot, + NodeBootstrapChainstatePhase, + NodeBootstrapJob, + NodeBootstrapPlan, NodeBootstrapSnapshot, NodeBootstrapVerification, - NodeBootstrapChainstateObservation, - NodeBootstrapPlan, - NodeBootstrapJob, - BootstrapOverviewResponse, } from './bootstrap.models'; +import { classifySource, deterministicSnapshotJson, loadCatalogue, pinnedCommitmentFor, producerFor } from './snapshot-catalogue'; +import { newVerificationRecord, SnapshotByteSource, SnapshotVerifier } from './snapshot-verifier'; -export class BootstrapEvidenceError extends Error { - constructor( - public readonly code: string, - message: string, - public readonly status = 503 - ) { - super(message); - } +export { BootstrapEvidenceError } from './bootstrap-errors'; + +const hex64 = (value: unknown): value is string => typeof value === 'string' && /^[0-9a-f]{64}$/i.test(value); + +/** Signed control-plane identity the routes verified before calling the service. */ +export interface OperatorAuthorization { + keyId: string; + elevated: boolean; +} + +export interface BootstrapServiceDependencies { + nodes?: BootstrapNodeReader; + core?: WorkbenchCoreReader; + operatorCore?: OperatorCore; + env?: BootstrapEnvironment; + store?: BootstrapStore | null; + bytes?: SnapshotByteSource; + network?: string; + clock?: () => number; + statfs?: StatfsReader; + catalogue?: () => BootstrapCatalogue; + workerOwner?: string; } -const unavailable = (code: string, prerequisite: string): never => { - throw new BootstrapEvidenceError( - code, - 'Bootstrap evidence is unavailable. ' + prerequisite - ); -}; +function longTimeoutOperatorCore(env: BootstrapEnvironment): OperatorCore { + // The shared client keeps a short timeout that suits reads; a snapshot RPC + // runs for a long time against the same fixed node identity. + // eslint-disable-next-line @typescript-eslint/no-var-requires + const bitcoin = require('../../../rpc-api/index'); + const client = new bitcoin.Client({ + host: config.CORE_RPC.HOST, + port: config.CORE_RPC.PORT, + user: config.CORE_RPC.USERNAME, + pass: config.CORE_RPC.PASSWORD, + timeout: env.jobTimeoutMs, + cookie: config.CORE_RPC.COOKIE ? config.CORE_RPC.COOKIE_PATH : undefined, + }); + return { call: (method, params) => client.rpc.call(method, params) }; +} -// These routes remain offered. Seeded nodes, manifests and in-memory jobs are -// not observations or operator execution; require their actual authorities. +/** + * Read-only node observation is always real. The catalogue, verification, + * planning and operator subfeatures each report their own availability and + * never invent a snapshot, a verification or a job. + */ export class BootstrapService { - constructor(private nodes = new BootstrapNodeReader()) {} - private async observation() { + private readonly nodes: BootstrapNodeReader; + private readonly core: WorkbenchCoreReader; + private readonly env: BootstrapEnvironment; + private readonly store: BootstrapStore | null; + private readonly network: string; + private readonly clock: () => number; + private readonly verifier: SnapshotVerifier; + private readonly readCatalogue: () => BootstrapCatalogue; + private readonly statfs?: StatfsReader; + private operatorCore?: OperatorCore; + private readonly workerOwner: string; + private executor?: BootstrapJobExecutor; + private readonly inflight = new Map>(); + + constructor(deps: BootstrapServiceDependencies = {}) { + this.nodes = deps.nodes ?? new BootstrapNodeReader(); + this.core = deps.core ?? ownedWorkbenchCore; + this.env = deps.env ?? readBootstrapEnvironment(); + this.network = deps.network ?? config.MEMPOOL.NETWORK; + this.store = deps.store === undefined ? (config.DATABASE.ENABLED ? new MysqlBootstrapStore() : null) : deps.store; + this.clock = deps.clock ?? (() => Date.now()); + this.statfs = deps.statfs; + this.readCatalogue = deps.catalogue ?? ((): BootstrapCatalogue => loadCatalogue(this.env.catalogueFile)); + this.operatorCore = deps.operatorCore; + this.workerOwner = deps.workerOwner ?? `${hostname()}:${process.pid}:${randomUUID().slice(0, 8)}`; + this.verifier = new SnapshotVerifier(this.env, this.nodes, this.core, deps.bytes, () => new Date(this.clock()).toISOString()); + } + + private requireStore(): BootstrapStore { + if (!this.store) { + throw new BootstrapEvidenceError( + 'durable-store-unavailable', + 'Bootstrap evidence is unavailable. Verification runs and operator jobs need the durable MySQL store (DATABASE.ENABLED).' + ); + } + return this.store; + } + + private now(): string { + return new Date(this.clock()).toISOString(); + } + + /** @asyncUnsafe rejections propagate to the caller, which handles them. */ + private async observation(): Promise { try { return await this.nodes.read(); } catch { @@ -40,25 +127,54 @@ export class BootstrapService { ); } } + + private catalogueStatus(): { status: 'available' | 'unavailable'; reason: string | null; catalogue: BootstrapCatalogue | null } { + try { + return { status: 'available', reason: null, catalogue: this.readCatalogue() }; + } catch (e) { + return { status: 'unavailable', reason: e instanceof Error ? e.message : String(e), catalogue: null }; + } + } + /** @asyncUnsafe rejections propagate to the caller, which handles them. */ - public async getOverview() { + public async getOverview(): Promise> { const { capability, observation } = await this.observation(); + const catalogue = this.catalogueStatus(); + let snapshots: NodeBootstrapSnapshot[] = []; + let snapshotReason = catalogue.reason; + if (catalogue.catalogue) { + try { + snapshots = await this.joinSnapshots(catalogue.catalogue); + } catch (e) { + snapshotReason = e instanceof Error ? e.message : String(e); + } + } + const verified = snapshots.filter((s) => s.status === 'pinned_core'); return { total_nodes: 1, nodes: [capability], active_chainstates: [observation], - snapshots: [], - total_snapshots: null, + snapshots, + total_snapshots: catalogue.catalogue ? snapshots.length : null, configured_nodes_count: 1, dual_chainstate_nodes_count: observation.dual_chainstate_active ? 1 : 0, - recommended_snapshot_height: null, - featured_snapshots: [], + recommended_snapshot_height: verified.length ? Math.max(...verified.map((s) => s.height)) : null, + featured_snapshots: verified, observed_nodes: [observation], - snapshot_catalogue_status: 'unavailable', - snapshot_catalogue_reason: - 'A trusted signed snapshot catalogue and independently pinned Core commitments are not configured.', + snapshot_catalogue_status: catalogue.catalogue && !snapshotReason ? 'available' : 'unavailable', + snapshot_catalogue_reason: snapshotReason, + verification_store_status: this.store ? 'available' : 'unavailable', + verification_store_reason: this.store ? null : 'Verification runs and operator jobs need the durable MySQL store (DATABASE.ENABLED).', + planning_status: catalogue.catalogue && this.store ? 'available' : 'unavailable', + operator_status: this.store && this.env.snapshotDir ? 'available' : 'unavailable', + operator_reason: !this.store + ? 'The durable job store is not connected.' + : !this.env.snapshotDir + ? 'No allowlisted snapshot directory is configured (UNIVERSE_BOOTSTRAP_SNAPSHOT_DIR).' + : null, }; } + /** @asyncUnsafe rejections propagate to the caller, which handles them. */ public async listNodes() { return [(await this.observation()).capability]; @@ -73,21 +189,125 @@ export class BootstrapService { return observation.node_id === nodeId ? observation : undefined; } - public listSnapshots(): NodeBootstrapSnapshot[] { - return unavailable( - 'unavailable-manifest', - 'A trusted signed snapshot catalogue, producer keys and pinned Bitcoin Core commitments are required.' - ); + /** @asyncUnsafe rejections propagate to the caller, which handles them. */ + private async joinSnapshots(catalogue: BootstrapCatalogue): Promise { + const out: NodeBootstrapSnapshot[] = []; + for (const snapshot of catalogue.snapshots.filter((s) => s.network === this.network)) { + const pinned = pinnedCommitmentFor(catalogue, snapshot); + let latest: NodeBootstrapVerification | null = null; + let storeUnavailable = false; + if (this.store) { + try { + latest = await this.store.latestVerification(snapshot.id, this.network); + } catch { + storeUnavailable = true; + } + } else { + storeUnavailable = true; + } + const state = latest?.state ?? null; + const status: NodeBootstrapSnapshot['status'] = storeUnavailable + ? 'unavailable' + : state === 'valid' + ? 'pinned_core' + : state === 'invalid' + ? 'invalid' + : state === 'pending' || state === 'verifying' + ? state + : 'unverified'; + let downloadUrl: string | undefined; + try { + const source = classifySource(snapshot.source, this.env.sourceAllowlist); + downloadUrl = source.kind === 'https' ? snapshot.source : undefined; + } catch { + downloadUrl = undefined; + } + out.push({ + snapshot_id: snapshot.id, + network: snapshot.network, + height: snapshot.height, + block_hash: snapshot.blockHash, + coins_count: pinned?.coinCount ?? null, + base_utxo_hash: pinned?.utxoCommitment ?? null, + sha256_checksum: snapshot.sha256, + file_size_bytes: snapshot.sizeBytes, + release_version: snapshot.coreVersion, + status, + ...(downloadUrl ? { download_url: downloadUrl } : {}), + producer_id: snapshot.manifest.producerId, + pinned_commitment: !!pinned, + latest_verification_id: latest?.verification_id ?? null, + latest_verification_state: state, + name: `${snapshot.network} ${snapshot.height} (Bitcoin Core ${snapshot.coreVersion})`, + base_height: snapshot.height, + base_block_hash: snapshot.blockHash, + txoutset_hash: pinned?.utxoCommitment ?? null, + size_gb: snapshot.sizeBytes / 1073741824, + is_verified: status === 'pinned_core', + verification_status: status === 'pinned_core' ? 'verified' : status === 'invalid' ? 'hash_mismatch' : 'unverified', + }); + } + return out.sort((a, b) => b.height - a.height); + } + + /** @asyncUnsafe rejections propagate to the caller, which handles them. */ + public async listSnapshots(): Promise { + return this.joinSnapshots(this.readCatalogue()); } - public getSnapshot(_snapshotId: string): NodeBootstrapSnapshot | undefined { - return unavailable( - 'unavailable-manifest', - 'A trusted signed snapshot catalogue, producer keys and pinned Bitcoin Core commitments are required.' + /** Looks a snapshot up by id, height or block hash on this network. */ + private resolveSnapshot(catalogue: BootstrapCatalogue, reference: string): BootstrapCatalogueSnapshot | undefined { + const ref = String(reference ?? '').trim(); + return catalogue.snapshots.find( + (s) => s.network === this.network && (s.id === ref || String(s.height) === ref || s.blockHash === ref.toLowerCase()) ); } - public verifySnapshot(data: { + /** @asyncUnsafe rejections propagate to the caller, which handles them. */ + public async getSnapshot(reference: string): Promise { + const catalogue = this.readCatalogue(); + const snapshot = this.resolveSnapshot(catalogue, reference); + if (!snapshot) { + return undefined; + } + return (await this.joinSnapshots(catalogue)).find((s) => s.snapshot_id === snapshot.id); + } + + /** The signed manifest of a catalogue snapshot: who signed which fields, and what Core pins for it. */ + public getSnapshotManifest(reference: string): Record | undefined { + const catalogue = this.readCatalogue(); + const snapshot = this.resolveSnapshot(catalogue, reference); + if (!snapshot) { + return undefined; + } + const producer = producerFor(catalogue, snapshot); + const pinned = pinnedCommitmentFor(catalogue, snapshot); + return { + snapshot_id: snapshot.id, + network: snapshot.network, + core_version: snapshot.coreVersion, + height: snapshot.height, + block_hash: snapshot.blockHash, + size_bytes: snapshot.sizeBytes, + sha256: snapshot.sha256, + producer_id: snapshot.manifest.producerId, + algorithm: producer?.algorithm ?? null, + public_key: producer?.publicKey ?? null, + signature: snapshot.manifest.signature, + signed_fields: deterministicSnapshotJson(snapshot).toString('utf8'), + pinned_commitment: pinned ? { utxo_commitment: pinned.utxoCommitment, coin_count: pinned.coinCount, source: 'operator-pinned Bitcoin Core chainparams for ' + pinned.coreVersion } : null, + note: 'A signed manifest identifies bytes; it is not evidence the bytes are valid. See the verification runs.', + }; + } + + /** + * Starts (or reuses) a verification run over the actual bytes. The answer is + * the durable record, which is pending until the run reaches a final state; + * poll getVerification. Caller checksums are recorded and compared, never + * trusted. + */ + /** @asyncUnsafe rejections propagate to the caller, which handles them. */ + public async verifySnapshot(data: { snapshot_id?: string; file_sha256?: string; base_height?: number; @@ -95,66 +315,340 @@ export class BootstrapService { height?: number; sha256?: string; utxo_hash?: string; - }): NodeBootstrapVerification { + }): Promise { const height = data?.base_height ?? data?.height; const sha256 = data?.file_sha256 ?? data?.sha256; const utxoHash = data?.expected_txoutset_hash ?? data?.utxo_hash; if ( - !Number.isSafeInteger(height) || - Number(height) < 0 || - typeof sha256 !== 'string' || - !/^[0-9a-f]{64}$/i.test(sha256) || - typeof utxoHash !== 'string' || - !/^[0-9a-f]{64}$/i.test(utxoHash) + (height !== undefined && (!Number.isSafeInteger(height) || Number(height) < 0)) || + (sha256 !== undefined && !hex64(sha256)) || + (utxoHash !== undefined && !hex64(utxoHash)) || + (data?.snapshot_id !== undefined && typeof data.snapshot_id !== 'string') || + (data?.snapshot_id === undefined && height === undefined) ) { throw new BootstrapEvidenceError( 'invalid-input', - 'A nonnegative base height and two 32-byte hexadecimal checksums are required.', + 'A snapshot_id or a nonnegative base height is required; checksums, when given, are 32-byte hexadecimal.', 400 ); } - return unavailable( - 'unavailable-manifest', - 'No trusted manifest, producer-key verification, snapshot bytes or pinned Core UTXO commitment were checked. Caller-supplied checksums alone cannot verify a snapshot.' + const catalogue = this.readCatalogue(); + const store = this.requireStore(); + const snapshot = this.resolveSnapshot(catalogue, data.snapshot_id ?? String(height)); + if (!snapshot) { + throw new BootstrapEvidenceError('snapshot-not-in-catalogue', 'No trusted catalogue snapshot matches that reference on this network.', 404); + } + if (height !== undefined && height !== snapshot.height) { + throw new BootstrapEvidenceError('invalid-input', `Snapshot ${snapshot.id} is at height ${snapshot.height}, not ${height}.`, 400); + } + const active = await store.activeVerification(snapshot.id, this.network); + if (active) { + return active; + } + const record = newVerificationRecord( + randomUUID(), + snapshot, + { sha256: sha256 ? sha256.toLowerCase() : null, utxo_hash: utxoHash ? utxoHash.toLowerCase() : null, height: height ?? null }, + this.now() ); + await store.insertVerification(record); + const run = this.verifier + .run(JSON.parse(JSON.stringify(record)), catalogue, snapshot, store) + .catch(async (e) => { + logger.err('[bootstrap] verification run failed: ' + (e instanceof Error ? e.message : e)); + record.state = 'unavailable'; + record.status = 'unavailable'; + record.reason = 'run-failed'; + record.details = 'The verification run stopped before recording an outcome: ' + (e instanceof Error ? e.message : String(e)); + record.finished_at = this.now(); + try { + await store.updateVerification(record); + } catch (storeError) { + logger.err('[bootstrap] verification outcome could not be stored: ' + (storeError instanceof Error ? storeError.message : storeError)); + } + return record; + }) + .finally(() => this.inflight.delete(record.verification_id)); + this.inflight.set(record.verification_id, run); + return record; } - public getVerification( - _verificationId: string - ): NodeBootstrapVerification | undefined { - return unavailable( - 'unavailable-verification-store', - 'No durable verification authority is connected; absence of a run cannot be established.' - ); + /** Resolves once the named run has a final state; for tests and callers that can wait. */ + public awaitVerification(verificationId: string): Promise { + return this.inflight.get(verificationId) ?? Promise.resolve(undefined); } - public createBootstrapPlan(_params: { - node_version: string; - network: string; - available_disk_gb: number; - }): NodeBootstrapPlan { - return unavailable( - 'unavailable-node-source', - 'Planning requires measured node capabilities, disk capacity and a compatible trusted snapshot.' - ); + /** @asyncUnsafe rejections propagate to the caller, which handles them. */ + public async getVerification(verificationId: string): Promise { + const store = this.requireStore(); + try { + return (await store.getVerification(verificationId, this.network)) ?? undefined; + } catch (e) { + throw new BootstrapEvidenceError( + 'durable-store-unavailable', + 'The durable verification store did not answer; absence of the run cannot be established: ' + (e instanceof Error ? e.message : String(e)) + ); + } } - public createOperatorJob(_params: { - job_type: 'generate_snapshot' | 'verify_snapshot' | 'load_snapshot'; - node_id: string; + /** + * A feasibility statement from measurements: the observed node, the + * measured capacity and a verified, pinned, compatible snapshot. Nothing is + * executed. Caller values are recorded as declarations or assumptions. + */ + /** @asyncUnsafe rejections propagate to the caller, which handles them. */ + public async createBootstrapPlan(params: { + node_id?: string; + node_version?: string; + network?: string; + target_height?: number; snapshot_id?: string; - }): NodeBootstrapJob { - return unavailable( - 'unavailable-operator', - 'The authorized node executor and durable job store are not connected. No node operation was started.' - ); + available_disk_gb?: number; + bandwidth_mbps?: number; + }): Promise { + const p = params ?? {}; + if (p.network !== undefined && p.network !== this.network) { + throw new BootstrapEvidenceError('unsupported-network', `This backend plans for ${this.network}, not ${p.network}.`, 409); + } + const catalogue = this.readCatalogue(); + const store = this.requireStore(); + const observation = await this.observation(); + const nowMs = this.clock(); + requireFreshObservation(observation, nowMs, 60000); + const version = requireSupportedNode(observation, this.network); + if (p.node_id !== undefined && p.node_id !== observation.capability.node_id) { + throw new BootstrapEvidenceError('node-not-observed', `The owned node is ${observation.capability.node_id}, not ${p.node_id}.`, 404); + } + if (p.node_version !== undefined && String(p.node_version).replace(/^v/, '') !== version.version) { + throw new BootstrapEvidenceError('unsupported-version', `The owned node runs Bitcoin Core ${version.version}, not ${p.node_version}; plans are made for the measured version.`, 409); + } + const candidates = compatibleSnapshots(catalogue, this.network, version); + let chosen = candidates; + if (p.snapshot_id !== undefined) { + chosen = candidates.filter((c) => c.snapshot.id === p.snapshot_id); + } else if (p.target_height !== undefined) { + chosen = candidates.filter((c) => c.snapshot.height === p.target_height); + } + if (!chosen.length) { + throw new BootstrapEvidenceError('no-compatible-snapshot', `No catalogue snapshot on ${this.network} has a commitment pinned for Bitcoin Core ${version.version}` + (p.target_height !== undefined ? ` at height ${p.target_height}.` : '.'), 409); + } + let selected: Awaited> | undefined; + let snapshot: BootstrapCatalogueSnapshot | undefined; + let lastError: BootstrapEvidenceError | undefined; + for (const candidate of chosen) { + try { + selected = await evaluateLoadEligibility({ observation, network: this.network, snapshot: candidate.snapshot, catalogue, store, env: this.env, now: nowMs, statfs: this.statfs }); + snapshot = candidate.snapshot; + break; + } catch (e) { + if (!(e instanceof BootstrapEvidenceError)) { + throw e; + } + lastError = e; + // Capacity and chainstate rejections hold for every candidate; only an + // unverified snapshot lets the next newest one be considered. + if (e.code !== 'snapshot-not-verified') { + throw e; + } + } + } + if (!selected || !snapshot) { + throw lastError ?? new BootstrapEvidenceError('no-compatible-snapshot', 'No compatible snapshot could be selected.', 409); + } + const chain = observation.observation; + const blocksOnDisk = Math.round(chain.disk_used_gb * 1073741824); + const declaredDisk = typeof p.available_disk_gb === 'number' && Number.isFinite(p.available_disk_gb) ? p.available_disk_gb : null; + const bandwidth = typeof p.bandwidth_mbps === 'number' && Number.isFinite(p.bandwidth_mbps) && p.bandwidth_mbps > 0 ? p.bandwidth_mbps : null; + const hours = (bytes: number): number | null => (bandwidth ? Math.round(((bytes * 8) / (bandwidth * 1e6) / 3600) * 100) / 100 : null); + const assumptions = [ + 'The chainstate built from the snapshot is assumed to need as much disk as the snapshot file; a tenth of both is added as headroom.', + 'Core reports size_on_disk for block and undo data only; chainstate and indexes are not in the measured usage.', + bandwidth ? `Download estimates assume the caller-declared ${bandwidth} Mbps and nothing else.` : 'No bandwidth was declared, so no download time is estimated.', + 'Validation durations are not estimated: this backend has no measurement of the node CPU or disk throughput.', + ]; + return { + plan_id: randomUUID(), + network: this.network, + node_id: observation.capability.node_id, + node_version: version.version, + target_height: snapshot.height, + measurement_status: 'measured', + measured: { + node_observed_at: chain.observed_at, + current_phase: chain.current_phase as NodeBootstrapChainstatePhase, + tip_height: chain.tip_height, + headers: chain.tip_height + chain.estimated_remaining_blocks, + blocks_on_disk_bytes: blocksOnDisk, + capacity: selected.capacity, + supports_loadtxoutset: observation.capability.supports_loadtxoutset, + }, + selected_snapshot: { + snapshot_id: snapshot.id, + height: snapshot.height, + block_hash: snapshot.blockHash, + core_version: snapshot.coreVersion, + size_bytes: snapshot.sizeBytes, + sha256: snapshot.sha256, + utxo_commitment: selected.pinned.utxoCommitment, + verification_id: selected.verification.verification_id, + verified_at: selected.verification.verified_at, + }, + requirements: { + snapshot_file_bytes: snapshot.sizeBytes, + chainstate_estimate_bytes: selected.required.chainstate_estimate_bytes, + headroom_bytes: selected.required.headroom_bytes, + required_bytes: selected.required.required_bytes, + free_bytes: selected.capacity.free_bytes, + feasible: true, + }, + assumptions, + caller_declared: { + available_disk_gb: declaredDisk, + bandwidth_mbps: bandwidth, + matches_measurement: declaredDisk === null ? null : Math.abs(declaredDisk * 1073741824 - selected.capacity.free_bytes) <= selected.capacity.free_bytes * 0.1, + }, + estimates: { + snapshot_download_hours: hours(snapshot.sizeBytes), + ibd_download_hours: hours(blocksOnDisk), + basis: bandwidth ? 'bytes over the caller-declared bandwidth; no validation time is included' : 'not estimated', + }, + expected_transitions: ['snapshot_loading', 'snapshot_active_syncing_to_tip', 'background_validation', 'fully_validated'], + actions_not_executed: [ + `loadtxoutset ${snapshot.source.replace(/.*[\\/]/, '')} on ${observation.capability.node_id} through an authorized operator job`, + 'Core validates the pinned hash_serialized_3 itself before activating the snapshot chainstate.', + ], + rollback_instructions: [ + 'Stop the node and remove the chainstate_snapshot directory to return to the validated chainstate.', + 'A snapshot chainstate that finished background validation replaces the original and needs no rollback.', + ], + created_at: this.now(), + }; } - public getJob(_jobId: string): NodeBootstrapJob | undefined { - return unavailable( - 'unavailable-operator', - 'The durable operator job store is not connected.' - ); + /** + * Accepts a job for the owned node after the route verified the signed + * control-plane request. Queued is all this promises; the bounded worker + * runs it and records what Core answered. + */ + /** @asyncUnsafe rejections propagate to the caller, which handles them. */ + public async createOperatorJob( + params: { + job_type: 'generate_snapshot' | 'load_snapshot'; + node_id?: string; + snapshot_id?: string; + idempotency_key?: string; + confirm?: string; + }, + authorization: OperatorAuthorization | undefined + ): Promise { + if (!authorization || typeof authorization.keyId !== 'string' || !authorization.keyId) { + throw new BootstrapEvidenceError('UNAUTHORIZED', 'Operator jobs need a verified signed control-plane request.', 401); + } + if (params?.job_type !== 'generate_snapshot' && params?.job_type !== 'load_snapshot') { + throw new BootstrapEvidenceError('invalid-input', 'job_type must be generate_snapshot or load_snapshot.', 400); + } + if (params.job_type === 'load_snapshot' && !authorization.elevated) { + throw new BootstrapEvidenceError('ELEVATION_REQUIRED', 'Loading a snapshot replaces the active chainstate; it requires adminAuthorization.elevated=true in the signed body.', 403); + } + if (params.job_type === 'load_snapshot' && params.confirm !== 'load_snapshot') { + throw new BootstrapEvidenceError('invalid-input', 'A load needs the typed confirmation confirm="load_snapshot".', 400); + } + const idempotencyKey = params.idempotency_key; + if (typeof idempotencyKey !== 'string' || !/^[A-Za-z0-9._:-]{8,128}$/.test(idempotencyKey)) { + throw new BootstrapEvidenceError('invalid-input', 'idempotency_key (8 to 128 characters of [A-Za-z0-9._:-]) is required.', 400); + } + const store = this.requireStore(); + if (!this.env.snapshotDir) { + throw new BootstrapEvidenceError('unavailable-operator', 'Bootstrap evidence is unavailable. No allowlisted snapshot directory is configured (UNIVERSE_BOOTSTRAP_SNAPSHOT_DIR); no node operation was started.'); + } + const observation = await this.observation(); + const nodeId = observation.capability.node_id; + if (params.node_id !== undefined && params.node_id !== nodeId) { + throw new BootstrapEvidenceError('node-not-observed', `The owned node is ${nodeId}, not ${params.node_id}.`, 404); + } + const catalogue = this.readCatalogue(); + if (params.job_type === 'generate_snapshot') { + requireSupportedNode(observation, this.network); + if (!observation.capability.supports_dumptxoutset) { + throw new BootstrapEvidenceError('node-capability-missing', 'The owned node does not expose dumptxoutset.', 409); + } + if (observation.observation.current_phase !== 'fully_validated') { + throw new BootstrapEvidenceError('chainstate-not-eligible', `A snapshot is only generated from a fully validated chainstate; the node is in ${observation.observation.current_phase}.`, 409); + } + } else { + const snapshot = catalogue.snapshots.find((s) => s.id === params.snapshot_id && s.network === this.network); + if (!snapshot) { + throw new BootstrapEvidenceError('snapshot-not-in-catalogue', 'No trusted catalogue snapshot matches snapshot_id on this network.', 404); + } + await evaluateLoadEligibility({ observation, network: this.network, snapshot, catalogue, store, env: this.env, now: this.clock(), statfs: this.statfs }); + } + const existing = await store.activeJob(this.network, nodeId); + if (existing && existing.idempotency_key !== idempotencyKey) { + throw new BootstrapEvidenceError('job-already-active', `Job ${existing.job_id} (${existing.job_type}) is ${existing.state} on ${nodeId}; one node runs one operator job at a time.`, 409); + } + const now = this.now(); + const job: NodeBootstrapJob = { + job_id: randomUUID(), + job_type: params.job_type, + network: this.network, + node_id: nodeId, + ...(params.snapshot_id ? { snapshot_id: params.snapshot_id } : {}), + idempotency_key: idempotencyKey, + state: 'queued', + status: 'queued', + progress_pct: 0, + message: 'Queued. No node operation has started; the worker re-checks every precondition before issuing the RPC.', + requested_by: authorization.keyId, + created_at: now, + updated_at: now, + attempts: 0, + lease: null, + rpc: null, + preconditions: {}, + evidence: {}, + checkpoints: [{ at: now, stage: 'queued', detail: 'requested by ' + authorization.keyId }], + }; + const stored = await store.insertJob(job); + this.startWorker(); + return stored.job; + } + + /** @asyncUnsafe rejections propagate to the caller, which handles them. */ + public async getJob(jobId: string): Promise { + const store = this.requireStore(); + try { + return (await store.getJob(jobId, this.network)) ?? undefined; + } catch (e) { + throw new BootstrapEvidenceError('durable-store-unavailable', 'The durable job store did not answer: ' + (e instanceof Error ? e.message : String(e))); + } + } + + /** Starts the bounded worker once the durable store and output directory exist; idle otherwise. */ + public startWorker(): BootstrapJobExecutor | undefined { + if (!this.store || !this.env.snapshotDir) { + return undefined; + } + if (!this.executor) { + this.executor = new BootstrapJobExecutor({ + env: this.env, + network: this.network, + store: this.store, + nodes: this.nodes, + core: this.core, + operatorCore: this.operatorCore ?? longTimeoutOperatorCore(this.env), + catalogue: this.readCatalogue, + owner: this.workerOwner, + clock: this.clock, + statfs: this.statfs, + }); + this.executor.start(); + } + return this.executor; + } + + public stopWorker(): void { + this.executor?.stop(); + this.executor = undefined; } } diff --git a/backend/src/api/intelligence/bootstrap/bootstrap.test.ts b/backend/src/api/intelligence/bootstrap/bootstrap.test.ts index 25b294734a9..fbfd3a623d6 100644 --- a/backend/src/api/intelligence/bootstrap/bootstrap.test.ts +++ b/backend/src/api/intelligence/bootstrap/bootstrap.test.ts @@ -1,35 +1,302 @@ -import bootstrapService, { BootstrapEvidenceError } from './bootstrap.service'; +import { BootstrapService, BootstrapEvidenceError } from './bootstrap.service'; +import { BootstrapNodeReader } from './node-reader'; +import { MemoryBootstrapStore } from './bootstrap-store'; +import { BootstrapCatalogue, NodeBootstrapVerification } from './bootstrap.models'; +import { parseCatalogue, classifySource, verifyManifestSignature } from './snapshot-catalogue'; +import { NETWORK_MAGIC } from './snapshot-format'; +import { byteSource, catalogueFor, fakeCore, FIXTURE, FIXTURE_BYTES, producerKeys, testEnvironment } from './__fixtures__/support'; -describe('BootstrapService', () => { - it('does not invent nodes, chainstates, manifests or verification records', () => { - for (const read of [ - () => bootstrapService.listSnapshots(), () => bootstrapService.getSnapshot('snap-840000-mainnet'), - () => bootstrapService.getVerification('unknown'), - ]) expect(read).toThrow(BootstrapEvidenceError); +const keys = producerKeys(); +const statfsPlenty = async (): Promise<{ bsize: number; bavail: number; blocks: number }> => ({ bsize: 4096, bavail: 1 << 20, blocks: 1 << 21 }); + +function service(options: { + catalogue?: BootstrapCatalogue | (() => BootstrapCatalogue); + store?: MemoryBootstrapStore | null; + bytes?: Buffer; + core?: ReturnType; + env?: Parameters[0]; + byteOptions?: { chunk?: number; stall?: boolean }; + statfs?: typeof statfsPlenty; + clock?: () => number; +} = {}): { service: BootstrapService; store: MemoryBootstrapStore | null; core: ReturnType; source: ReturnType } { + const core = options.core ?? fakeCore(); + const store = options.store === undefined ? new MemoryBootstrapStore() : options.store; + const source = byteSource(options.bytes ?? FIXTURE_BYTES, options.byteOptions); + const catalogue = options.catalogue ?? catalogueFor(keys.privateKey, keys.publicKeyHex); + return { + service: new BootstrapService({ + nodes: new BootstrapNodeReader(core), + core, + env: testEnvironment(options.env), + store, + bytes: source, + network: 'regtest', + statfs: options.statfs ?? statfsPlenty, + clock: options.clock, + catalogue: typeof catalogue === 'function' ? catalogue : () => catalogue, + operatorCore: { call: async () => { throw new Error('operator core must not be reached by verification'); } }, + }), + store, + core, + source, + }; +} + +async function verified(s: BootstrapService, data: Record = { snapshot_id: 'regtest-105' }): Promise { + const pending = await s.verifySnapshot(data); + expect(pending.state).toBe('pending'); + expect(pending.valid).toBe(false); + const done = await s.awaitVerification(pending.verification_id); + expect(done).toBeDefined(); + return done as NodeBootstrapVerification; +} + +describe('BootstrapService catalogue', () => { + it('parses a signed catalogue and rejects malformed ones with a typed reason', () => { + const catalogue = catalogueFor(keys.privateKey, keys.publicKeyHex); + const parsed = parseCatalogue(JSON.stringify(catalogue)); + expect(parsed.snapshots[0].id).toBe('regtest-105'); + expect(verifyManifestSignature(parsed.snapshots[0], parsed.producers[0])).toBe(true); + expect(verifyManifestSignature({ ...parsed.snapshots[0], height: 104 }, parsed.producers[0])).toBe(false); + for (const broken of [ + 'not json', + '[]', + JSON.stringify({ ...catalogue, producers: [{ id: 'lab', algorithm: 'rsa', publicKey: keys.publicKeyHex }] }), + JSON.stringify({ ...catalogue, pinnedCommitments: [{ network: 'mainnet', coreVersion: '28', height: 1, blockHash: 'x', utxoCommitment: 'y' }] }), + JSON.stringify({ ...catalogue, snapshots: [{ ...catalogue.snapshots[0], manifest: { producerId: 'nobody', signature: 'ab'.repeat(64) } }] }), + JSON.stringify({ ...catalogue, snapshots: [catalogue.snapshots[0], catalogue.snapshots[0]] }), + ]) { + expect(() => parseCatalogue(broken)).toThrow(expect.objectContaining({ code: 'invalid-catalogue' })); + } + }); + + it('only accepts sources under the operator allowlist', () => { + expect(classifySource('/allowed/x.dat', ['/allowed'])).toEqual({ kind: 'file', ref: 'x.dat' }); + expect(classifySource('https://snapshots.example/regtest/x.dat', ['https://snapshots.example/regtest'])).toEqual({ kind: 'https', ref: 'https://snapshots.example/x.dat' }); + for (const [source, allow] of [['/allowed/../etc/passwd', ['/allowed']], ['/other/x.dat', ['/allowed']], ['https://evil.example/x.dat', ['https://snapshots.example']], ['http://snapshots.example/x.dat', ['http://snapshots.example']], ['ftp://x/y', ['ftp://x']]] as const) { + expect(() => classifySource(source, [...allow])).toThrow(expect.objectContaining({ code: 'source-not-allowlisted' })); + } + }); + + it('keeps node and chainstate reads working and reports subfeatures truthfully without a catalogue', async () => { + const { service: s } = service({ catalogue: () => { throw new BootstrapEvidenceError('unavailable-manifest', 'no catalogue'); }, store: null }); + expect((await s.listNodes())[0].node_id).toBe('owned-core-regtest'); + expect((await s.listNodeChainstates())[0].current_phase).toBe('fully_validated'); + expect((await s.getNodeChainstates('owned-core-regtest'))?.tip_height).toBe(FIXTURE.height); + const overview = await s.getOverview(); + expect(overview.snapshot_catalogue_status).toBe('unavailable'); + expect(overview.verification_store_status).toBe('unavailable'); + expect(overview.operator_status).toBe('unavailable'); + expect(overview.snapshots).toEqual([]); + await expect(s.listSnapshots()).rejects.toThrow(expect.objectContaining({ code: 'unavailable-manifest' })); + await expect(s.getSnapshot('105')).rejects.toThrow(BootstrapEvidenceError); + }); + + it('lists catalogue snapshots as unverified until a run over the bytes reaches valid', async () => { + const { service: s } = service(); + const before = await s.listSnapshots(); + expect(before).toHaveLength(1); + expect(before[0]).toMatchObject({ snapshot_id: 'regtest-105', network: 'regtest', height: 105, status: 'unverified', pinned_commitment: true, base_utxo_hash: FIXTURE.hashSerialized3, is_verified: false }); + expect(before[0].download_url).toBeUndefined(); + await verified(s); + const after = await s.getSnapshot(FIXTURE.baseHash); + expect(after?.status).toBe('pinned_core'); + expect(after?.is_verified).toBe(true); + expect((await s.getOverview()).recommended_snapshot_height).toBe(105); + expect(s.getSnapshotManifest('105')).toMatchObject({ producer_id: 'lab', algorithm: 'ed25519', pinned_commitment: { utxo_commitment: FIXTURE.hashSerialized3 } }); }); +}); - it('requires authority-backed snapshot verification for both advertised request shapes', () => { - expect(() => bootstrapService.verifySnapshot({ snapshot_id: 'unknown', base_height: 840000, - file_sha256: 'ab'.repeat(32), expected_txoutset_hash: 'cd'.repeat(32) })).toThrow(/No trusted manifest/); - expect(() => bootstrapService.verifySnapshot({ height: 840000, sha256: 'ab'.repeat(32), utxo_hash: 'cd'.repeat(32) })) - .toThrow(/No trusted manifest/); +describe('BootstrapService verification', () => { + it('needs the durable store and tells a missing record from a missing store', async () => { + const { service: s } = service({ store: null }); + await expect(s.verifySnapshot({ snapshot_id: 'regtest-105' })).rejects.toThrow(expect.objectContaining({ code: 'durable-store-unavailable', status: 503 })); + await expect(s.getVerification('missing')).rejects.toThrow(expect.objectContaining({ code: 'durable-store-unavailable', status: 503 })); + const { service: withStore, store } = service(); + expect(await withStore.getVerification('missing')).toBeUndefined(); + (store as MemoryBootstrapStore).getVerification = async () => { throw new Error('connection lost'); }; + await expect(withStore.getVerification('missing')).rejects.toThrow(expect.objectContaining({ code: 'durable-store-unavailable' })); }); - it('rejects malformed input before reporting an authority as missing', () => { - for (const data of [{}, { height: -1, sha256: 'ab'.repeat(32), utxo_hash: 'cd'.repeat(32) }, - { height: 840000, sha256: 'z'.repeat(64), utxo_hash: 'cd'.repeat(32) }]) { - expect(() => bootstrapService.verifySnapshot(data)).toThrow(expect.objectContaining({ code: 'invalid-input', status: 400 })); + it('rejects malformed input and unknown snapshots before touching the store', async () => { + const { service: s } = service(); + for (const data of [{}, { height: -1 }, { height: 105, sha256: 'z'.repeat(64) }, { snapshot_id: 5 }]) { + await expect(s.verifySnapshot(data as any)).rejects.toThrow(expect.objectContaining({ code: 'invalid-input', status: 400 })); } + await expect(s.verifySnapshot({ height: 840000 })).rejects.toThrow(expect.objectContaining({ code: 'snapshot-not-in-catalogue', status: 404 })); + await expect(s.verifySnapshot({ snapshot_id: 'regtest-105', height: 104 })).rejects.toThrow(expect.objectContaining({ code: 'invalid-input' })); + }); + + it('verifies real bytes against the signed manifest, the owned node and the pinned Core commitment', async () => { + const { service: s, source, core } = service(); + const record = await verified(s, { height: 105, sha256: FIXTURE.sha256.toUpperCase(), utxo_hash: FIXTURE.hashSerialized3 }); + expect(record.state).toBe('valid'); + expect(record.valid).toBe(true); + expect(Object.values(record.checks).map((c) => c.status)).toEqual(Array(8).fill('valid')); + expect(record.checks.utxo_commitment).toEqual({ status: 'valid', expected: FIXTURE.hashSerialized3, observed: FIXTURE.hashSerialized3 }); + expect(record.evidence).toMatchObject({ source_kind: 'file', source_ref: 'regtest-105.dat', bytes_read: FIXTURE_BYTES.length, header_format: 'versioned', snapshot_version: 2, core_node_id: 'owned-core-regtest', core_block_hash_at_height: FIXTURE.baseHash }); + expect(record.caller_inputs).toEqual({ sha256: FIXTURE.sha256, utxo_hash: FIXTURE.hashSerialized3, height: 105, matched: true }); + expect(record.checkpoints.map((c) => c.stage)).toEqual(['queued', 'started', 'manifest-signature', 'core-base-block', 'bytes', 'bytes', 'finished']); + expect(source.opened).toEqual(['/allowed/regtest-105.dat']); + expect(core.calls).toContain('getblockhash'); + expect(await s.getVerification(record.verification_id)).toEqual(record); + }); + + it('fails on altered bytes while the signature check still passes', async () => { + const altered = Buffer.from(FIXTURE_BYTES); + altered[altered.length - 1] ^= 0x01; + const record = await verified(service({ bytes: altered }).service); + expect(record.state).toBe('invalid'); + expect(record.checks.sha256.status).toBe('invalid'); + expect(record.checks.manifest_signature.status).toBe('valid'); + expect(record.checks.utxo_commitment.status).toBe('invalid'); + expect(record.checks.base_block_hash.status).toBe('valid'); + }); + + it('fails on a bad manifest signature while the bytes still verify', async () => { + const other = producerKeys(); + const catalogue = catalogueFor(other.privateKey, keys.publicKeyHex); + const record = await verified(service({ catalogue }).service); + expect(record.state).toBe('invalid'); + expect(record.checks.manifest_signature.status).toBe('invalid'); + expect(record.checks.sha256.status).toBe('valid'); + expect(record.checks.utxo_commitment.status).toBe('valid'); + }); + + it('fails on the wrong network magic independently of the file hash', async () => { + const signet = Buffer.from(FIXTURE_BYTES); + Buffer.from(NETWORK_MAGIC.signet, 'hex').copy(signet, 7); + const sha256 = require('crypto').createHash('sha256').update(signet).digest('hex'); + const catalogue = catalogueFor(keys.privateKey, keys.publicKeyHex, { sha256 }); + const record = await verified(service({ catalogue, bytes: signet }).service); + expect(record.state).toBe('invalid'); + expect(record.checks.network_magic.status).toBe('invalid'); + expect(record.checks.sha256.status).toBe('valid'); + expect(record.checks.manifest_signature.status).toBe('valid'); + }); + + it('fails when the owned node has another block at the catalogue height', async () => { + const catalogue = catalogueFor(keys.privateKey, keys.publicKeyHex, { height: 104 }, { height: 104 }); + const record = await verified(service({ catalogue }).service, { snapshot_id: 'regtest-105' }); + expect(record.state).toBe('invalid'); + expect(record.checks.base_height).toMatchObject({ status: 'invalid', expected: 104, observed: 104 }); + expect(record.checks.sha256.status).toBe('valid'); }); - it('cannot select a seeded snapshot or estimate an unobserved node in a plan', () => { - expect(() => bootstrapService.createBootstrapPlan({ node_version: '28.0.0', network: 'mainnet', available_disk_gb: 1000 })) - .toThrow(expect.objectContaining({ code: 'unavailable-node-source' })); + it('fails when the pinned Core commitment differs from the streamed hash_serialized_3', async () => { + const catalogue = catalogueFor(keys.privateKey, keys.publicKeyHex, {}, { utxoCommitment: 'ab'.repeat(32) }); + const record = await verified(service({ catalogue }).service); + expect(record.state).toBe('invalid'); + expect(record.checks.utxo_commitment).toMatchObject({ status: 'invalid', expected: 'ab'.repeat(32), observed: FIXTURE.hashSerialized3 }); + expect(record.checks.sha256.status).toBe('valid'); + }); + + it('is unavailable, never valid, without a pinned commitment for the release', async () => { + const catalogue = catalogueFor(keys.privateKey, keys.publicKeyHex, {}, { coreVersion: '29.0.0' }); + const record = await verified(service({ catalogue }).service); + expect(record.state).toBe('unavailable'); + expect(record.valid).toBe(false); + expect(record.reason).toBe('no-pinned-commitment'); + expect(record.checks.utxo_commitment.status).toBe('not-evaluated'); + expect(record.checks.sha256.status).toBe('valid'); + }); + + it('is unavailable when the source exceeds the bounded size or the deadline', async () => { + const big = await verified(service({ env: { maxSnapshotBytes: 1000 } }).service); + expect(big.state).toBe('unavailable'); + expect(big.reason).toBe('size-exceeded'); + expect(big.checks.sha256.status).toBe('not-evaluated'); + expect(big.evidence.bytes_read).toBeGreaterThan(1000); + const slow = await verified(service({ env: { verifyDeadlineMs: 30 }, byteOptions: { stall: true } }).service); + expect(slow.state).toBe('unavailable'); + expect(slow.reason).toBe('deadline-exceeded'); + expect(slow.evidence.bytes_read).toBe(FIXTURE_BYTES.length); + const denied = await verified(service({ env: { sourceAllowlist: ['/elsewhere'] } }).service); + expect(denied.state).toBe('unavailable'); + expect(denied.reason).toBe('source-not-allowlisted'); + }); + + it('reuses the running record for a duplicate request and persists both states', async () => { + const { service: s, store } = service({ byteOptions: { chunk: 64 } }); + const first = await s.verifySnapshot({ snapshot_id: 'regtest-105' }); + const second = await s.verifySnapshot({ height: 105 }); + expect(second.verification_id).toBe(first.verification_id); + expect(['pending', 'verifying']).toContain(second.state); + const done = await s.awaitVerification(first.verification_id); + expect(done?.state).toBe('valid'); + expect((store as MemoryBootstrapStore).verifications.size).toBe(1); + const third = await s.verifySnapshot({ snapshot_id: 'regtest-105' }); + expect(third.verification_id).not.toBe(first.verification_id); + await s.awaitVerification(third.verification_id); + }); +}); + +describe('BootstrapService planning', () => { + const ibdCore = (): ReturnType => fakeCore({ blocks: 10, headers: 200, ibd: true }); + + async function ready(options: Parameters[0] = {}): Promise> { + const built = service({ core: ibdCore(), ...options }); + await verified(built.service); + return built; + } + + it('emits a measured, feasible plan with nothing executed', async () => { + const { service: s } = await ready(); + const plan = await s.createBootstrapPlan({ network: 'regtest', target_height: 105, available_disk_gb: 4, bandwidth_mbps: 100 }); + expect(plan).toMatchObject({ + network: 'regtest', node_id: 'owned-core-regtest', node_version: '28.0.0', target_height: 105, measurement_status: 'measured', + measured: { current_phase: 'traditional_ibd', tip_height: 10, headers: 200, capacity: { method: 'statfs', free_bytes: 4096 * (1 << 20) }, supports_loadtxoutset: true }, + selected_snapshot: { snapshot_id: 'regtest-105', utxo_commitment: FIXTURE.hashSerialized3 }, + requirements: { snapshot_file_bytes: FIXTURE_BYTES.length, feasible: true }, + caller_declared: { available_disk_gb: 4, bandwidth_mbps: 100, matches_measurement: true }, + expected_transitions: ['snapshot_loading', 'snapshot_active_syncing_to_tip', 'background_validation', 'fully_validated'], + }); + expect(plan.selected_snapshot.verification_id).toHaveLength(36); + expect(plan.requirements.required_bytes).toBe(FIXTURE_BYTES.length * 2 + Math.ceil(FIXTURE_BYTES.length * 2 * 0.1)); + expect(plan.estimates.snapshot_download_hours).not.toBeNull(); + expect(plan.actions_not_executed[0]).toMatch(/loadtxoutset regtest-105.dat on owned-core-regtest/); + expect(plan.assumptions.length).toBeGreaterThan(2); + }); + + it('rejects every unmeasured or incompatible precondition with a typed reason', async () => { + const { service: s } = await ready(); + await expect(s.createBootstrapPlan({ network: 'signet' })).rejects.toThrow(expect.objectContaining({ code: 'unsupported-network', status: 409 })); + await expect(s.createBootstrapPlan({ node_version: '27.0.0' })).rejects.toThrow(expect.objectContaining({ code: 'unsupported-version' })); + await expect(s.createBootstrapPlan({ node_id: 'other' })).rejects.toThrow(expect.objectContaining({ code: 'node-not-observed', status: 404 })); + await expect(s.createBootstrapPlan({ target_height: 840000 })).rejects.toThrow(expect.objectContaining({ code: 'no-compatible-snapshot' })); + + const old = await ready({ core: fakeCore({ blocks: 10, headers: 200, ibd: true, subversion: '/Satoshi:25.1.0/' }) }); + await expect(old.service.createBootstrapPlan({})).rejects.toThrow(expect.objectContaining({ code: 'unsupported-version' })); + + const noLoad = await ready({ core: fakeCore({ blocks: 10, headers: 200, ibd: true, help: 'getchainstates\n' }) }); + await expect(noLoad.service.createBootstrapPlan({})).rejects.toThrow(expect.objectContaining({ code: 'node-capability-missing' })); + + const v29 = await ready({ core: fakeCore({ blocks: 10, headers: 200, ibd: true, subversion: '/Satoshi:29.0.0/' }) }); + await expect(v29.service.createBootstrapPlan({})).rejects.toThrow(expect.objectContaining({ code: 'no-compatible-snapshot' })); + + const unverified = service({ core: ibdCore() }).service; + await expect(unverified.createBootstrapPlan({})).rejects.toThrow(expect.objectContaining({ code: 'snapshot-not-verified' })); + + const noCapacity = await ready({ env: { datadir: undefined } }); + await expect(noCapacity.service.createBootstrapPlan({})).rejects.toThrow(expect.objectContaining({ code: 'capacity-not-measured' })); + + const stale = await ready({ env: { datadir: undefined, capacityBytes: 10 ** 12, capacityMeasuredAt: new Date(Date.now() - 48 * 3600 * 1000).toISOString() } }); + await expect(stale.service.createBootstrapPlan({})).rejects.toThrow(expect.objectContaining({ code: 'stale-measurement', status: 409 })); + + const fresh = await ready({ env: { datadir: undefined, capacityBytes: 10 ** 12, capacityMeasuredAt: new Date().toISOString() } }); + expect((await fresh.service.createBootstrapPlan({})).measured.capacity.method).toBe('configured'); + + const small = await ready({ statfs: async () => ({ bsize: 512, bavail: 4, blocks: 8 }) }); + await expect(small.service.createBootstrapPlan({ available_disk_gb: 1000 })).rejects.toThrow(expect.objectContaining({ code: 'insufficient-capacity' })); + + const validated = await ready({ core: fakeCore() }); + await expect(validated.service.createBootstrapPlan({})).rejects.toThrow(expect.objectContaining({ code: 'chainstate-not-eligible' })); }); - it.each(['generate_snapshot', 'verify_snapshot', 'load_snapshot'] as const)('cannot fabricate a %s job', job_type => { - expect(() => bootstrapService.createOperatorJob({ job_type, node_id: 'unknown', snapshot_id: 'unknown' })) - .toThrow(expect.objectContaining({ code: 'unavailable-operator' })); - expect(() => bootstrapService.getJob('unknown')).toThrow(/durable operator job store/); + it('needs the catalogue and the durable store', async () => { + const { service: noStore } = service({ core: ibdCore(), store: null }); + await expect(noStore.createBootstrapPlan({})).rejects.toThrow(expect.objectContaining({ code: 'durable-store-unavailable' })); + const { service: noCatalogue } = service({ core: ibdCore(), catalogue: () => { throw new BootstrapEvidenceError('unavailable-manifest', 'none'); } }); + await expect(noCatalogue.createBootstrapPlan({})).rejects.toThrow(expect.objectContaining({ code: 'unavailable-manifest' })); }); }); diff --git a/backend/src/api/intelligence/bootstrap/node-reader.test.ts b/backend/src/api/intelligence/bootstrap/node-reader.test.ts index 76c6ffda679..35db818160d 100644 --- a/backend/src/api/intelligence/bootstrap/node-reader.test.ts +++ b/backend/src/api/intelligence/bootstrap/node-reader.test.ts @@ -107,7 +107,7 @@ describe('Owned bootstrap chainstates', () => { expect( f.call.mock.calls.filter(([m]) => m === 'getchainstates') ).toHaveLength(1); - const service = new BootstrapService(f.reader), + const service = new BootstrapService({ nodes: f.reader, store: null }), overview = await service.getOverview(); expect(overview).toMatchObject({ configured_nodes_count: 1, diff --git a/backend/src/api/intelligence/bootstrap/snapshot-catalogue.ts b/backend/src/api/intelligence/bootstrap/snapshot-catalogue.ts new file mode 100644 index 00000000000..6c825e4e4ab --- /dev/null +++ b/backend/src/api/intelligence/bootstrap/snapshot-catalogue.ts @@ -0,0 +1,229 @@ +import { createPublicKey, verify as verifySignature } from 'crypto'; +import { readFileSync } from 'fs'; +import { BootstrapEvidenceError } from './bootstrap-errors'; +import { + BootstrapCatalogue, + BootstrapCatalogueProducer, + BootstrapCatalogueSnapshot, + BootstrapPinnedCommitment, +} from './bootstrap.models'; +import { NETWORK_MAGIC } from './snapshot-format'; + +const hex = (value: unknown, bytes: number): value is string => + typeof value === 'string' && new RegExp(`^[0-9a-f]{${bytes * 2}}$`).test(value); +const id = (value: unknown): value is string => + typeof value === 'string' && /^[A-Za-z0-9._-]{1,128}$/.test(value); +const version = (value: unknown): value is string => + typeof value === 'string' && /^\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(value); +const height = (value: unknown): value is number => + Number.isSafeInteger(value) && Number(value) >= 0; +const invalid = (detail: string): never => { + throw new BootstrapEvidenceError( + 'invalid-catalogue', + 'The trusted snapshot catalogue is malformed: ' + detail + ); +}; + +/** ed25519 SPKI DER prefix for a raw 32 byte public key. */ +const ED25519_SPKI_PREFIX = Buffer.from('302a300506032b6570032100', 'hex'); + +/** The bytes a producer signs: the identifying snapshot fields, sorted keys, no whitespace. */ +export function deterministicSnapshotJson(snapshot: BootstrapCatalogueSnapshot): Buffer { + const signed: Record = { + blockHash: snapshot.blockHash, + coreVersion: snapshot.coreVersion, + height: snapshot.height, + id: snapshot.id, + network: snapshot.network, + sha256: snapshot.sha256, + sizeBytes: snapshot.sizeBytes, + }; + return Buffer.from(JSON.stringify(signed, Object.keys(signed).sort()), 'utf8'); +} + +export function verifyManifestSignature( + snapshot: BootstrapCatalogueSnapshot, + producer: BootstrapCatalogueProducer +): boolean { + try { + const key = createPublicKey({ + key: Buffer.concat([ED25519_SPKI_PREFIX, Buffer.from(producer.publicKey, 'hex')]), + format: 'der', + type: 'spki', + }); + return verifySignature( + null, + deterministicSnapshotJson(snapshot), + key, + Buffer.from(snapshot.manifest.signature, 'hex') + ); + } catch { + return false; + } +} + +export function parseCatalogue(text: string): BootstrapCatalogue { + let raw: any; + try { + raw = JSON.parse(text); + } catch { + return invalid('not JSON.'); + } + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) { + return invalid('the document is not an object.'); + } + const producers: BootstrapCatalogueProducer[] = []; + if (!Array.isArray(raw.producers)) { + return invalid('producers must be an array.'); + } + for (const p of raw.producers) { + if (!id(p?.id) || p.algorithm !== 'ed25519' || !hex(p.publicKey, 32)) { + return invalid('a producer needs an id, algorithm ed25519 and a 32 byte hex publicKey.'); + } + if (producers.some((known) => known.id === p.id)) { + return invalid(`producer ${p.id} is listed twice.`); + } + producers.push({ id: p.id, algorithm: 'ed25519', publicKey: p.publicKey }); + } + const pinned: BootstrapPinnedCommitment[] = []; + if (!Array.isArray(raw.pinnedCommitments)) { + return invalid('pinnedCommitments must be an array.'); + } + for (const c of raw.pinnedCommitments) { + if ( + !NETWORK_MAGIC[c?.network] || + !version(c.coreVersion) || + !height(c.height) || + !hex(c.blockHash, 32) || + !hex(c.utxoCommitment, 32) || + (c.coinCount !== undefined && c.coinCount !== null && !height(c.coinCount)) + ) { + return invalid('a pinned commitment needs network, coreVersion, height, 32 byte blockHash and utxoCommitment.'); + } + pinned.push({ + network: c.network, + coreVersion: c.coreVersion, + height: c.height, + blockHash: c.blockHash, + utxoCommitment: c.utxoCommitment, + coinCount: c.coinCount ?? null, + }); + } + const snapshots: BootstrapCatalogueSnapshot[] = []; + if (!Array.isArray(raw.snapshots)) { + return invalid('snapshots must be an array.'); + } + for (const s of raw.snapshots) { + if ( + !id(s?.id) || + !NETWORK_MAGIC[s.network] || + !version(s.coreVersion) || + !height(s.height) || + !hex(s.blockHash, 32) || + !height(s.sizeBytes) || + s.sizeBytes < 1 || + !hex(s.sha256, 32) || + typeof s.source !== 'string' || + !s.source || + !id(s.manifest?.producerId) || + !hex(s.manifest.signature, 64) + ) { + return invalid(`snapshot ${typeof s?.id === 'string' ? s.id : '(unnamed)'} is missing a required field.`); + } + if (snapshots.some((known) => known.id === s.id)) { + return invalid(`snapshot ${s.id} is listed twice.`); + } + if (!producers.some((p) => p.id === s.manifest.producerId)) { + return invalid(`snapshot ${s.id} names unknown producer ${s.manifest.producerId}.`); + } + snapshots.push({ + id: s.id, + network: s.network, + coreVersion: s.coreVersion, + height: s.height, + blockHash: s.blockHash, + sizeBytes: s.sizeBytes, + sha256: s.sha256, + source: s.source, + manifest: { producerId: s.manifest.producerId, signature: s.manifest.signature }, + }); + } + return { producers, pinnedCommitments: pinned, snapshots }; +} + +export function loadCatalogue(file: string | undefined): BootstrapCatalogue { + if (!file) { + throw new BootstrapEvidenceError( + 'unavailable-manifest', + 'Bootstrap evidence is unavailable. A trusted signed snapshot catalogue, producer keys and pinned Bitcoin Core commitments are required (UNIVERSE_BOOTSTRAP_SNAPSHOT_CATALOGUE).' + ); + } + let text: string; + try { + text = readFileSync(file, 'utf8'); + } catch { + throw new BootstrapEvidenceError( + 'unavailable-manifest', + 'Bootstrap evidence is unavailable. The configured snapshot catalogue could not be read.' + ); + } + return parseCatalogue(text); +} + +export function producerFor( + catalogue: BootstrapCatalogue, + snapshot: BootstrapCatalogueSnapshot +): BootstrapCatalogueProducer | undefined { + return catalogue.producers.find((p) => p.id === snapshot.manifest.producerId); +} + +/** The operator-pinned Core commitment for this exact network, release, height and block. */ +export function pinnedCommitmentFor( + catalogue: BootstrapCatalogue, + snapshot: Pick +): BootstrapPinnedCommitment | undefined { + return catalogue.pinnedCommitments.find( + (c) => + c.network === snapshot.network && + c.coreVersion === snapshot.coreVersion && + c.height === snapshot.height && + c.blockHash === snapshot.blockHash + ); +} + +/** A source is usable only under an operator allowlisted https origin or local directory. */ +export function classifySource( + source: string, + allowlist: string[] +): { kind: 'https' | 'file'; ref: string } { + if (/^https:\/\//i.test(source)) { + let url: URL; + try { + url = new URL(source); + } catch { + throw new BootstrapEvidenceError('source-not-allowlisted', 'The snapshot source URL is malformed.'); + } + if (url.username || url.password || url.protocol !== 'https:') { + throw new BootstrapEvidenceError('source-not-allowlisted', 'The snapshot source URL is not a plain https URL.'); + } + if (!allowlist.some((entry) => /^https:\/\//i.test(entry) && source.startsWith(entry.endsWith('/') ? entry : entry + '/'))) { + throw new BootstrapEvidenceError('source-not-allowlisted', 'The snapshot source origin is not allowlisted (UNIVERSE_BOOTSTRAP_SOURCE_ALLOWLIST).'); + } + return { kind: 'https', ref: url.origin + url.pathname.replace(/.*\//, '/') }; + } + if (/^[a-z]+:\/\//i.test(source)) { + throw new BootstrapEvidenceError('source-not-allowlisted', 'Only https URLs and local paths are accepted as snapshot sources.'); + } + const normalised = source.replace(/\\/g, '/'); + const allowed = allowlist.some((entry) => { + if (/^https:\/\//i.test(entry)) { + return false; + } + const dir = entry.replace(/\\/g, '/').replace(/\/+$/, '') + '/'; + return normalised.startsWith(dir) && !normalised.slice(dir.length).split('/').includes('..'); + }); + if (!allowed) { + throw new BootstrapEvidenceError('source-not-allowlisted', 'The snapshot source path is not under an allowlisted directory (UNIVERSE_BOOTSTRAP_SOURCE_ALLOWLIST).'); + } + return { kind: 'file', ref: normalised.replace(/.*\//, '') }; +} diff --git a/backend/src/api/intelligence/bootstrap/snapshot-format.test.ts b/backend/src/api/intelligence/bootstrap/snapshot-format.test.ts new file mode 100644 index 00000000000..b5c6435a21a --- /dev/null +++ b/backend/src/api/intelligence/bootstrap/snapshot-format.test.ts @@ -0,0 +1,132 @@ +import { createHash } from 'crypto'; +import { readFileSync } from 'fs'; +import { join } from 'path'; +import { + decodeSnapshot, + decompressAmount, + decompressPubkey, + NETWORK_MAGIC, + SnapshotFormatError, + SnapshotStreamDecoder, +} from './snapshot-format'; + +/** Exact dumptxoutset bytes of an offline Bitcoin Core 28.0 regtest node; see __fixtures__/README.md. */ +export const REGTEST_FIXTURE = { + bytes: Buffer.from(readFileSync(join(__dirname, '__fixtures__', 'regtest-105.snapshot.base64'), 'utf8'), 'base64'), + baseHash: '0be30936f2332969e5dd82fe06d0c4d8d25d86351442ec813c2f70aa132c2acb', + height: 105, + coins: 107, + hashSerialized3: '2fff9d5b75bf5539b20a99cbae5639819995d0b560d2cfda171aafed870418fc', + sha256: 'facbe9af2dede0a3322ec7b9c46328f4f86cfc4764cab11b90b3821104cceb7e', +}; + +/** compressor.cpp CompressAmount, transcribed for round trips. */ +function compressAmount(n: bigint): bigint { + if (n === 0n) { + return 0n; + } + let e = 0n; + while (n % 10n === 0n && e < 9n) { + n /= 10n; + e++; + } + if (e < 9n) { + const d = n % 10n; + n /= 10n; + return 1n + (n * 9n + d - 1n) * 10n + e; + } + return 1n + (n - 1n) * 10n + 9n; +} + +function varint(n: bigint): Buffer { + const out: number[] = []; + let first = true; + for (;;) { + out.unshift(Number(n & 0x7fn) | (first ? 0 : 0x80)); + if (n <= 0x7fn) { + break; + } + n = (n >> 7n) - 1n; + first = false; + } + return Buffer.from(out); +} + +const sha256d = (b: Buffer): string => Buffer.from(createHash('sha256').update(createHash('sha256').update(b).digest()).digest()).reverse().toString('hex'); + +describe('snapshot-format', () => { + it('reproduces Bitcoin Core 28.0 metadata and hash_serialized_3 for a real dumptxoutset file', () => { + for (const chunk of [1, 7, 64, 1024, 1 << 20]) { + const result = decodeSnapshot(REGTEST_FIXTURE.bytes, 'regtest', chunk); + expect(result.header).toEqual({ format: 'versioned', version: 2, network_magic: NETWORK_MAGIC.regtest, base_block_hash: REGTEST_FIXTURE.baseHash, coins_count: 107, header_bytes: 51 }); + expect(result.coins_read).toBe(REGTEST_FIXTURE.coins); + expect(result.bytes_read).toBe(REGTEST_FIXTURE.bytes.length); + expect(result.max_coin_height).toBe(REGTEST_FIXTURE.height); + expect(result.hash_serialized_3).toBe(REGTEST_FIXTURE.hashSerialized3); + expect(result.hash_reason).toBeNull(); + } + expect(createHash('sha256').update(REGTEST_FIXTURE.bytes).digest('hex')).toBe(REGTEST_FIXTURE.sha256); + }); + + it('hashes a hand-built coin identically in the versioned and the pre-28 layout', () => { + const txid = Buffer.alloc(32, 0xab); + const script = Buffer.alloc(20, 0x11); + const coin = Buffer.concat([varint(2n * 100n + 1n), varint(compressAmount(5000000000n)), varint(0n), script]); + const versioned = Buffer.concat([ + Buffer.from('7574786fff0200', 'hex'), Buffer.from(NETWORK_MAGIC.signet, 'hex'), Buffer.alloc(32, 0x01), Buffer.from('0100000000000000', 'hex'), + txid, Buffer.from([1]), Buffer.from([0]), coin, + ]); + const legacy = Buffer.concat([Buffer.alloc(32, 0x01), Buffer.from('0100000000000000', 'hex'), txid, Buffer.from('00000000', 'hex'), coin]); + const expectedScript = Buffer.concat([Buffer.from('76a914', 'hex'), script, Buffer.from('88ac', 'hex')]); + const txOutSer = Buffer.concat([txid, Buffer.from('00000000', 'hex'), Buffer.from('c9000000', 'hex'), Buffer.from('00f2052a01000000', 'hex'), Buffer.from([expectedScript.length]), expectedScript]); + const a = decodeSnapshot(versioned, 'signet'); + const b = decodeSnapshot(legacy); + expect(a.hash_serialized_3).toBe(sha256d(txOutSer)); + expect(b.hash_serialized_3).toBe(a.hash_serialized_3); + expect(b.header.format).toBe('legacy'); + expect(b.header.base_block_hash).toBe('01'.repeat(32)); + expect(a.max_coin_height).toBe(100); + }); + + it('refuses an unsupported header version, another network, truncation and trailing bytes', () => { + const bytes = REGTEST_FIXTURE.bytes; + const v1 = Buffer.from(bytes); + v1[5] = 1; + expect(() => decodeSnapshot(v1, 'regtest')).toThrow(expect.objectContaining({ reason: 'unsupported-snapshot-version' })); + expect(() => decodeSnapshot(bytes, 'signet')).toThrow(expect.objectContaining({ reason: 'network-magic-mismatch' })); + expect(() => decodeSnapshot(bytes.subarray(0, bytes.length - 1), 'regtest')).toThrow(expect.objectContaining({ reason: 'truncated' })); + expect(() => decodeSnapshot(bytes.subarray(0, 40), 'regtest')).toThrow(expect.objectContaining({ reason: 'truncated' })); + expect(() => decodeSnapshot(Buffer.concat([bytes, Buffer.from([0])]), 'regtest')).toThrow(expect.objectContaining({ reason: 'trailing-data' })); + const fewer = Buffer.from(bytes); + fewer.writeBigUInt64LE(106n, 43); + expect(() => decodeSnapshot(fewer, 'regtest')).toThrow(SnapshotFormatError); + const decoder = new SnapshotStreamDecoder('regtest'); + decoder.feed(bytes); + expect(() => decoder.feed(Buffer.from([1]))).toThrow(expect.objectContaining({ reason: 'trailing-data' })); + }); + + it('reports coins outside Core cursor order instead of hashing them as if sorted', () => { + const txid = (fill: number): Buffer => Buffer.alloc(32, fill); + const coin = Buffer.concat([varint(3n), varint(compressAmount(1n)), varint(0n), Buffer.alloc(20, 0)]); + const header = Buffer.concat([Buffer.from('7574786fff0200', 'hex'), Buffer.from(NETWORK_MAGIC.regtest, 'hex'), Buffer.alloc(32, 0x01), Buffer.from('0200000000000000', 'hex')]); + const ordered = Buffer.concat([header, txid(1), Buffer.from([1, 0]), coin, txid(2), Buffer.from([1, 0]), coin]); + const reversed = Buffer.concat([header, txid(2), Buffer.from([1, 0]), coin, txid(1), Buffer.from([1, 0]), coin]); + expect(decodeSnapshot(ordered, 'regtest').hash_serialized_3).toHaveLength(64); + const out = decodeSnapshot(reversed, 'regtest'); + expect(out.hash_serialized_3).toBeNull(); + expect(out.hash_reason).toBe('coins-not-in-cursor-order'); + expect(out.coins_read).toBe(2); + }); + + it('decompresses amounts and public keys as compressor.cpp and CPubKey do', () => { + for (const n of [0n, 1n, 9n, 10n, 12345n, 5000000000n, 2099999997690000n, 2100000000000000n]) { + expect(decompressAmount(compressAmount(n))).toBe(n); + } + expect(decompressAmount(50n)).toBe(5000000000n); + const gx = Buffer.from('79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798', 'hex'); + const gy = '483ada7726a3c4655da4fbfc0e1108a8fd17b448a68554199c47d08ffb10d4b8'; + expect(decompressPubkey(2, gx)?.toString('hex')).toBe('04' + gx.toString('hex') + gy); + expect(decompressPubkey(3, gx)?.toString('hex')).not.toBe('04' + gx.toString('hex') + gy); + expect(decompressPubkey(2, Buffer.alloc(32, 0xff))).toBeNull(); + }); +}); diff --git a/backend/src/api/intelligence/bootstrap/snapshot-format.ts b/backend/src/api/intelligence/bootstrap/snapshot-format.ts new file mode 100644 index 00000000000..1bed91c02ba --- /dev/null +++ b/backend/src/api/intelligence/bootstrap/snapshot-format.ts @@ -0,0 +1,457 @@ +import { createHash, Hash } from 'crypto'; + +/** + * Streaming decoder for Bitcoin Core `dumptxoutset` files and the + * `hash_serialized_3` commitment Core computes over the coins they carry. + * + * Layout (Core 28.x/29.x, src/node/utxo_snapshot.h SnapshotMetadata v2): + * 'utxo' 0xff | u16 LE version (2) | 4 byte network magic | + * 32 byte base block hash (internal byte order) | u64 LE coins count | + * then, per transaction: 32 byte txid, CompactSize coin count, and per coin + * CompactSize vout, VARINT((height << 1) | coinbase), VARINT(compressed + * amount), compressed script (src/compressor.h). + * Version 1 of the versioned header never shipped in a release and is refused. + * + * Layout (Core 26.x/27.x, no magic): 32 byte base block hash | u64 LE coins + * count | per coin: 32 byte txid, u32 LE vout, the same coin encoding. + * + * The commitment is SHA256d over, for every coin in cursor order, the + * outpoint (txid, u32 LE vout), u32 LE ((height << 1) | coinbase) and the + * standard CTxOut (i64 LE amount, CompactSize script length, script), which is + * kernel/coinstats.cpp TxOutSer. Core hashes its coins database in key order + * (txid bytes, then vout); the file is written from that same cursor, so the + * streamed hash equals Core's only when the file is in that order. An + * out-of-order file is reported instead of being hashed as if it were sorted. + */ + +export const NETWORK_MAGIC: Record = { + mainnet: 'f9beb4d9', + testnet: '0b110907', + testnet4: '1c163f28', + signet: '0a03cf40', + regtest: 'fabfb5da', +}; + +export const SNAPSHOT_MAGIC = Buffer.from('7574786fff', 'hex'); +export const SUPPORTED_SNAPSHOT_VERSIONS = [2]; +const MAX_SCRIPT_SIZE = 10000; +const MAX_COMPACT_SIZE = 0x02000000n; +const MAX_MONEY = 21000000n * 100000000n; +const SECP_P = (1n << 256n) - (1n << 32n) - 977n; + +export interface SnapshotHeader { + format: 'versioned' | 'legacy'; + version: number | null; + network_magic: string | null; + /** Display byte order, as RPCs print it. */ + base_block_hash: string; + coins_count: number; + header_bytes: number; +} + +export interface SnapshotDecodeResult { + header: SnapshotHeader; + coins_read: number; + bytes_read: number; + /** Display byte order, comparable with `gettxoutsetinfo hash_serialized_3`. */ + hash_serialized_3: string | null; + hash_reason: string | null; + max_coin_height: number; +} + +export class SnapshotFormatError extends Error { + constructor(public readonly reason: string, message: string) { + super(message); + } +} + +class NeedMore extends Error {} + +/** Cursor over a buffer; throws NeedMore when a record is incomplete. */ +class Cursor { + public pos = 0; + constructor(public buf: Buffer) {} + need(n: number): void { + if (this.pos + n > this.buf.length) { + throw new NeedMore(); + } + } + byte(): number { + this.need(1); + return this.buf[this.pos++]; + } + bytes(n: number): Buffer { + this.need(n); + const out = this.buf.subarray(this.pos, this.pos + n); + this.pos += n; + return out; + } + u16(): number { + this.need(2); + const v = this.buf.readUInt16LE(this.pos); + this.pos += 2; + return v; + } + u32(): number { + this.need(4); + const v = this.buf.readUInt32LE(this.pos); + this.pos += 4; + return v; + } + u64(): bigint { + this.need(8); + const v = this.buf.readBigUInt64LE(this.pos); + this.pos += 8; + return v; + } + /** serialize.h ReadCompactSize with range checks and minimal encoding. */ + compactSize(): bigint { + const first = this.byte(); + let value: bigint; + if (first < 253) { + value = BigInt(first); + } else if (first === 253) { + value = BigInt(this.u16()); + if (value < 253n) { + throw new SnapshotFormatError('malformed-compactsize', 'CompactSize is not minimally encoded.'); + } + } else if (first === 254) { + value = BigInt(this.u32()); + if (value < 0x10000n) { + throw new SnapshotFormatError('malformed-compactsize', 'CompactSize is not minimally encoded.'); + } + } else { + value = this.u64(); + if (value < 0x100000000n) { + throw new SnapshotFormatError('malformed-compactsize', 'CompactSize is not minimally encoded.'); + } + } + if (value > MAX_COMPACT_SIZE) { + throw new SnapshotFormatError('malformed-compactsize', 'CompactSize exceeds MAX_SIZE.'); + } + return value; + } + /** serialize.h ReadVarInt: MSB base-128 with the +1 continuation trick. */ + varint(): bigint { + let n = 0n; + for (let i = 0; i < 10; i++) { + const b = this.byte(); + n = (n << 7n) | BigInt(b & 0x7f); + if (b & 0x80) { + n += 1n; + } else { + return n; + } + } + throw new SnapshotFormatError('malformed-varint', 'VARINT is longer than 10 bytes.'); + } +} + +/** compressor.cpp DecompressAmount. */ +export function decompressAmount(x: bigint): bigint { + if (x === 0n) { + return 0n; + } + x -= 1n; + const e = x % 10n; + x /= 10n; + let n: bigint; + if (e < 9n) { + const d = (x % 9n) + 1n; + x /= 9n; + n = x * 10n + d; + } else { + n = x + 1n; + } + return n * 10n ** e; +} + +function modPow(base: bigint, exp: bigint, mod: bigint): bigint { + let result = 1n; + base %= mod; + while (exp > 0n) { + if (exp & 1n) { + result = (result * base) % mod; + } + base = (base * base) % mod; + exp >>= 1n; + } + return result; +} + +/** secp256k1 point decompression as CPubKey::Decompress; null when x is not on the curve. */ +export function decompressPubkey(prefix: number, x: Buffer): Buffer | null { + const xi = BigInt('0x' + x.toString('hex')); + if (xi >= SECP_P) { + return null; + } + const y2 = (modPow(xi, 3n, SECP_P) + 7n) % SECP_P; + let y = modPow(y2, (SECP_P + 1n) / 4n, SECP_P); + if ((y * y) % SECP_P !== y2) { + return null; + } + if ((y & 1n) !== BigInt(prefix & 1)) { + y = SECP_P - y; + } + return Buffer.concat([Buffer.from([0x04]), x, Buffer.from(y.toString(16).padStart(64, '0'), 'hex')]); +} + +/** compressor.cpp DecompressScript; an undecodable key leaves the script empty, as Core does. */ +export function decompressScript(kind: number, data: Buffer): Buffer { + switch (kind) { + case 0: + return Buffer.concat([Buffer.from([0x76, 0xa9, 0x14]), data, Buffer.from([0x88, 0xac])]); + case 1: + return Buffer.concat([Buffer.from([0xa9, 0x14]), data, Buffer.from([0x87])]); + case 2: + case 3: + return Buffer.concat([Buffer.from([0x21, kind]), data, Buffer.from([0xac])]); + case 4: + case 5: { + const key = decompressPubkey(kind - 2, data); + return key ? Buffer.concat([Buffer.from([0x41]), key, Buffer.from([0xac])]) : Buffer.alloc(0); + } + } + return Buffer.alloc(0); +} + +function readCoin(c: Cursor): { height: number; coinbase: boolean; amount: bigint; script: Buffer } { + const code = c.varint(); + if (code > 0xffffffffn) { + throw new SnapshotFormatError('malformed-coin', 'Coin height code exceeds 32 bits.'); + } + const amount = decompressAmount(c.varint()); + const nSize = c.varint(); + let script: Buffer; + if (nSize < 6n) { + script = decompressScript(Number(nSize), c.bytes(nSize < 2n ? 20 : 32)); + } else { + const len = nSize - 6n; + if (len > MAX_COMPACT_SIZE) { + throw new SnapshotFormatError('malformed-coin', 'Script length is not plausible.'); + } + if (len > BigInt(MAX_SCRIPT_SIZE)) { + // Core replaces an overly long script with OP_RETURN and skips the bytes. + c.bytes(Number(len)); + script = Buffer.from([0x6a]); + } else { + script = Buffer.from(c.bytes(Number(len))); + } + } + return { height: Number(code >> 1n), coinbase: (code & 1n) === 1n, amount, script }; +} + +function compactSizeBytes(n: number): Buffer { + if (n < 253) { + return Buffer.from([n]); + } + if (n <= 0xffff) { + const b = Buffer.alloc(3); + b[0] = 253; + b.writeUInt16LE(n, 1); + return b; + } + const b = Buffer.alloc(5); + b[0] = 254; + b.writeUInt32LE(n, 1); + return b; +} + +function displayHash(internal: Buffer): string { + return Buffer.from(internal).reverse().toString('hex'); +} + +export class SnapshotStreamDecoder { + private pending: Buffer = Buffer.alloc(0); + private header?: SnapshotHeader; + private hash: Hash | null = createHash('sha256'); + private hashReason: string | null = null; + private coinsRead = 0; + private bytesRead = 0; + private maxHeight = 0; + private done = false; + // versioned body state + private txid?: Buffer; + private coinsLeftInTx = 0; + private lastVout = -1; + private lastTxid?: Buffer; + + constructor(private readonly expectedNetwork?: string) {} + + public feed(chunk: Buffer): void { + if (this.done) { + throw new SnapshotFormatError('trailing-data', 'Bytes follow the last coin of the snapshot.'); + } + this.bytesRead += chunk.length; + this.pending = this.pending.length ? Buffer.concat([this.pending, chunk]) : chunk; + const c = new Cursor(this.pending); + try { + this.parse(c); + } catch (e) { + if (!(e instanceof NeedMore)) { + throw e; + } + } + this.pending = c.pos === this.pending.length ? Buffer.alloc(0) : Buffer.from(this.pending.subarray(c.pos)); + } + + public finish(): SnapshotDecodeResult { + if (!this.header) { + throw new SnapshotFormatError('truncated', 'The snapshot ended before its header was complete.'); + } + if (this.coinsRead !== this.header.coins_count || this.pending.length) { + throw new SnapshotFormatError( + 'truncated', + `The snapshot ended after ${this.coinsRead} of ${this.header.coins_count} coins with ${this.pending.length} undecoded bytes.` + ); + } + const digest = this.hash ? createHash('sha256').update(this.hash.digest()).digest() : null; + return { + header: this.header, + coins_read: this.coinsRead, + bytes_read: this.bytesRead, + hash_serialized_3: digest ? displayHash(digest) : null, + hash_reason: this.hashReason, + max_coin_height: this.maxHeight, + }; + } + + /** Each record is consumed atomically: an incomplete one rewinds the cursor to its start. */ + private parse(c: Cursor): void { + while (!this.done) { + const mark = c.pos; + try { + if (!this.header) { + this.header = this.readHeader(c); + if (this.header.coins_count === 0) { + this.done = true; + } + } else if (this.header.format === 'legacy') { + this.readLegacyCoin(c); + } else { + this.readVersionedRecord(c); + } + } catch (e) { + if (e instanceof NeedMore) { + c.pos = mark; + } + throw e; + } + } + if (c.pos < c.buf.length) { + throw new SnapshotFormatError('trailing-data', 'Bytes follow the last coin of the snapshot.'); + } + } + + private readHeader(c: Cursor): SnapshotHeader { + c.need(5); + const versioned = c.buf.subarray(0, 5).equals(SNAPSHOT_MAGIC); + if (versioned) { + c.bytes(5); + const version = c.u16(); + if (!SUPPORTED_SNAPSHOT_VERSIONS.includes(version)) { + throw new SnapshotFormatError('unsupported-snapshot-version', `Snapshot version ${version} is not supported.`); + } + const magic = c.bytes(4).toString('hex'); + if (this.expectedNetwork && NETWORK_MAGIC[this.expectedNetwork] !== magic) { + throw new SnapshotFormatError('network-magic-mismatch', `Snapshot network magic ${magic} is not ${this.expectedNetwork}.`); + } + const base = displayHash(c.bytes(32)); + const coins = c.u64(); + if (coins > BigInt(Number.MAX_SAFE_INTEGER)) { + throw new SnapshotFormatError('malformed-header', 'Coins count is not representable.'); + } + return { format: 'versioned', version, network_magic: magic, base_block_hash: base, coins_count: Number(coins), header_bytes: c.pos }; + } + const base = displayHash(c.bytes(32)); + const coins = c.u64(); + if (coins > BigInt(Number.MAX_SAFE_INTEGER)) { + throw new SnapshotFormatError('malformed-header', 'Coins count is not representable.'); + } + return { format: 'legacy', version: null, network_magic: null, base_block_hash: base, coins_count: Number(coins), header_bytes: c.pos }; + } + + /** Reads one coin; state is only committed once every byte of the record is present. */ + private readVersionedRecord(c: Cursor): void { + let txid = this.txid; + let coinsLeft = this.coinsLeftInTx; + let lastVout = this.lastVout; + let newTx = false; + if (!txid) { + txid = Buffer.from(c.bytes(32)); + const count = c.compactSize(); + if (count === 0n || count > BigInt(this.header!.coins_count - this.coinsRead)) { + throw new SnapshotFormatError('malformed-coin', 'Per-transaction coin count is zero or exceeds the coins left.'); + } + coinsLeft = Number(count); + lastVout = -1; + newTx = true; + } + const vout = c.compactSize(); + if (vout >= 0xffffffffn) { + throw new SnapshotFormatError('malformed-coin', 'Output index is out of range.'); + } + const coin = readCoin(c); + if (newTx) { + if (this.lastTxid && Buffer.compare(this.lastTxid, txid) >= 0) { + this.disableHash('coins-not-in-cursor-order'); + } + this.lastTxid = txid; + } + if (Number(vout) <= lastVout) { + this.disableHash('coins-not-in-cursor-order'); + } + this.lastVout = Number(vout); + this.applyCoin(txid, Number(vout), coin); + coinsLeft--; + this.txid = coinsLeft === 0 ? undefined : txid; + this.coinsLeftInTx = coinsLeft; + } + + private readLegacyCoin(c: Cursor): void { + const txid = Buffer.from(c.bytes(32)); + const vout = c.u32(); + const coin = readCoin(c); + if (this.lastTxid) { + const order = Buffer.compare(this.lastTxid, txid); + if (order > 0 || (order === 0 && vout <= this.lastVout)) { + this.disableHash('coins-not-in-cursor-order'); + } + } + this.lastTxid = txid; + this.lastVout = vout; + this.applyCoin(txid, vout, coin); + } + + private applyCoin(txid: Buffer, vout: number, coin: ReturnType): void { + if (coin.amount > MAX_MONEY) { + throw new SnapshotFormatError('malformed-coin', 'Coin amount is outside the money range.'); + } + this.coinsRead++; + this.maxHeight = Math.max(this.maxHeight, coin.height); + if (this.hash) { + const fixed = Buffer.alloc(16); + fixed.writeUInt32LE(vout, 0); + fixed.writeUInt32LE(((coin.height << 1) + (coin.coinbase ? 1 : 0)) >>> 0, 4); + fixed.writeBigInt64LE(coin.amount, 8); + this.hash.update(txid).update(fixed).update(compactSizeBytes(coin.script.length)).update(coin.script); + } + if (this.coinsRead === this.header!.coins_count) { + this.done = true; + } + } + + private disableHash(reason: string): void { + this.hash = null; + this.hashReason = reason; + } +} + +/** Convenience for tests and small files: decode a complete buffer. */ +export function decodeSnapshot(bytes: Buffer, expectedNetwork?: string, chunkSize = 1024): SnapshotDecodeResult { + const decoder = new SnapshotStreamDecoder(expectedNetwork); + for (let i = 0; i < bytes.length; i += chunkSize) { + decoder.feed(bytes.subarray(i, Math.min(bytes.length, i + chunkSize))); + } + return decoder.finish(); +} diff --git a/backend/src/api/intelligence/bootstrap/snapshot-verifier.ts b/backend/src/api/intelligence/bootstrap/snapshot-verifier.ts new file mode 100644 index 00000000000..a53627af017 --- /dev/null +++ b/backend/src/api/intelligence/bootstrap/snapshot-verifier.ts @@ -0,0 +1,344 @@ +import { createHash } from 'crypto'; +import { createReadStream } from 'fs'; +import * as https from 'https'; +import { WorkbenchCoreReader } from '../workbench/workbench-core'; +import { BootstrapEnvironment } from './bootstrap-config'; +import { BootstrapEvidenceError } from './bootstrap-errors'; +import { BootstrapStore } from './bootstrap-store'; +import { + BootstrapCatalogue, + BootstrapCatalogueSnapshot, + NodeBootstrapCheck, + NodeBootstrapVerification, +} from './bootstrap.models'; +import { BootstrapNodeReader } from './node-reader'; +import { + classifySource, + pinnedCommitmentFor, + producerFor, + verifyManifestSignature, +} from './snapshot-catalogue'; +import { NETWORK_MAGIC, SnapshotFormatError, SnapshotStreamDecoder } from './snapshot-format'; + +/** Streams the bytes of an allowlisted source; tests substitute a fake. */ +export interface SnapshotByteSource { + open(source: string, kind: 'file' | 'https'): Promise & { destroy?: (err?: Error) => void }>; +} + +export const nodeByteSource: SnapshotByteSource = { + open(source, kind) { + if (kind === 'file') { + return Promise.resolve(createReadStream(source, { highWaterMark: 4 * 1024 * 1024 })); + } + return new Promise((resolve, reject) => { + const request = https.get(source, { headers: { 'user-agent': 'universe-bootstrap-verifier' } }, (response) => { + if (response.statusCode !== 200) { + response.resume(); + reject(new Error(`The snapshot source answered HTTP ${response.statusCode}.`)); + return; + } + resolve(response); + }); + request.on('error', reject); + }); + }, +}; + +const pending = (): NodeBootstrapCheck => ({ status: 'pending' }); +const valid = (expected: string | number | null, observed: string | number | null): NodeBootstrapCheck => ({ status: 'valid', expected, observed }); +const invalid = (expected: string | number | null, observed: string | number | null, reason: string): NodeBootstrapCheck => ({ status: 'invalid', expected, observed, reason }); +const skipped = (reason: string, expected: string | number | null = null): NodeBootstrapCheck => ({ status: 'not-evaluated', expected, reason }); +const compare = (expected: string | number | null, observed: string | number | null, reason: string): NodeBootstrapCheck => + expected === observed ? valid(expected, observed) : invalid(expected, observed, reason); + +export function newVerificationRecord( + verificationId: string, + snapshot: BootstrapCatalogueSnapshot, + callerInputs: { sha256: string | null; utxo_hash: string | null; height: number | null }, + now: string +): NodeBootstrapVerification { + return { + verification_id: verificationId, + snapshot_id: snapshot.id, + network: snapshot.network, + state: 'pending', + valid: false, + status: 'pending', + requested_at: now, + checks: { + file_size: pending(), + sha256: pending(), + manifest_signature: pending(), + network_magic: pending(), + base_block_hash: pending(), + base_height: pending(), + coins_count: pending(), + utxo_commitment: pending(), + }, + evidence: { + source_kind: null, + source_ref: null, + bytes_read: 0, + header_format: null, + snapshot_version: null, + core_node_id: null, + core_block_hash_at_height: null, + pinned_commitment_source: null, + }, + checkpoints: [{ at: now, stage: 'queued' }], + caller_inputs: { ...callerInputs, matched: null }, + file_size_valid: false, + sha256_valid: false, + manifest_hash_valid: false, + signature_valid: false, + expected_metadata_match: false, + overall_verified: false, + details: 'Verification has not run yet. A queued run proves nothing about the bytes.', + verified_at: now, + }; +} + +function summarise(record: NodeBootstrapVerification): void { + const c = record.checks; + record.file_size_valid = c.file_size.status === 'valid'; + record.sha256_valid = c.sha256.status === 'valid'; + record.signature_valid = c.manifest_signature.status === 'valid'; + record.manifest_hash_valid = record.signature_valid; + record.expected_metadata_match = [c.network_magic, c.base_block_hash, c.base_height, c.coins_count].every((check) => check.status === 'valid'); + record.overall_verified = record.state === 'valid'; + record.valid = record.state === 'valid'; + record.status = record.state; + const observedSha = typeof c.sha256.observed === 'string' ? c.sha256.observed : null; + const observedUtxo = typeof c.utxo_commitment.observed === 'string' ? c.utxo_commitment.observed : null; + const inputs = record.caller_inputs; + if (inputs.sha256 || inputs.utxo_hash || inputs.height !== null) { + inputs.matched = + (inputs.sha256 === null || (observedSha !== null && inputs.sha256 === observedSha)) && + (inputs.utxo_hash === null || (observedUtxo !== null && inputs.utxo_hash === observedUtxo)) && + (inputs.height === null || inputs.height === c.base_height.expected); + } +} + +export class SnapshotVerifier { + constructor( + private readonly env: BootstrapEnvironment, + private readonly nodes: BootstrapNodeReader, + private readonly core: WorkbenchCoreReader, + private readonly bytes: SnapshotByteSource = nodeByteSource, + private readonly clock: () => string = () => new Date().toISOString() + ) {} + + /** + * Runs every check independently and persists the outcome. The record is + * valid only when every check is valid; a commitment that could not be + * evaluated leaves the record unavailable, never valid. + */ + /** @asyncUnsafe Every step writes its own outcome; a store failure is logged by the caller. */ + public async run( + record: NodeBootstrapVerification, + catalogue: BootstrapCatalogue, + snapshot: BootstrapCatalogueSnapshot, + store: BootstrapStore + ): Promise { + const checkpoint = (stage: string, detail?: string): void => { + record.checkpoints.push({ at: this.clock(), stage, ...(detail ? { detail } : {}) }); + }; + record.state = 'verifying'; + record.started_at = this.clock(); + checkpoint('started'); + summarise(record); + await store.updateVerification(record); + + const producer = producerFor(catalogue, snapshot); + record.checks.manifest_signature = producer && verifyManifestSignature(snapshot, producer) + ? valid(producer.id, producer.id) + : invalid(snapshot.manifest.producerId, null, 'The manifest signature does not verify against the producer key.'); + checkpoint('manifest-signature', record.checks.manifest_signature.status); + + const pinned = pinnedCommitmentFor(catalogue, snapshot); + record.evidence.pinned_commitment_source = pinned ? 'operator-pinned Bitcoin Core chainparams for ' + pinned.coreVersion : null; + + let unavailableReason: string | undefined; + try { + const observed = await this.nodes.read(); + record.evidence.core_node_id = observed.capability.node_id; + if (observed.capability.network !== snapshot.network) { + record.checks.base_block_hash = invalid(snapshot.blockHash, null, 'The owned node serves a different network than the snapshot.'); + record.checks.base_height = invalid(snapshot.height, null, 'The owned node serves a different network than the snapshot.'); + } else { + const hashAtHeight = await this.core.call('getblockhash', [snapshot.height]); + const header = typeof hashAtHeight === 'string' ? await this.core.call('getblockheader', [hashAtHeight, true]) : null; + record.evidence.core_block_hash_at_height = typeof hashAtHeight === 'string' ? hashAtHeight : null; + if (typeof hashAtHeight !== 'string' || header?.hash !== hashAtHeight || header?.height !== snapshot.height) { + throw new Error('The owned node did not return a consistent header for the snapshot height.'); + } + record.checks.base_height = hashAtHeight === snapshot.blockHash + ? valid(snapshot.height, header.height) + : invalid(snapshot.height, header.height, 'The owned node has a different block at the catalogue height.'); + } + } catch (e) { + unavailableReason = 'The owned Core node could not confirm the snapshot base block: ' + (e instanceof Error ? e.message : String(e)); + record.checks.base_height = skipped(unavailableReason, snapshot.height); + } + checkpoint('core-base-block', record.checks.base_height.status); + + let sha256: string | null = null; + let decoded: ReturnType | null = null; + let formatError: SnapshotFormatError | null = null; + try { + const source = classifySource(snapshot.source, this.env.sourceAllowlist); + record.evidence.source_kind = source.kind; + record.evidence.source_ref = source.ref; + const outcome = await this.stream(snapshot, source.kind, checkpoint); + sha256 = outcome.sha256; + decoded = outcome.decoded; + formatError = outcome.formatError; + record.evidence.bytes_read = outcome.bytesRead; + } catch (e) { + record.evidence.bytes_read = e instanceof StreamAbort ? e.bytesRead : record.evidence.bytes_read; + const message = e instanceof Error ? e.message : String(e); + unavailableReason = unavailableReason ?? message; + record.checks.file_size = skipped(message, snapshot.sizeBytes); + record.checks.sha256 = skipped(message, snapshot.sha256); + record.checks.network_magic = skipped(message, NETWORK_MAGIC[snapshot.network]); + record.checks.base_block_hash = skipped(message, snapshot.blockHash); + record.checks.coins_count = skipped(message, pinned?.coinCount ?? null); + record.checks.utxo_commitment = skipped(message, pinned?.utxoCommitment ?? null); + record.reason = e instanceof BootstrapEvidenceError ? e.code : e instanceof StreamAbort ? e.reason : 'source-read-failed'; + checkpoint('bytes', 'aborted: ' + (e instanceof Error ? e.message : String(e))); + } + + if (sha256 !== null) { + record.checks.file_size = compare(snapshot.sizeBytes, record.evidence.bytes_read, 'The source did not have the catalogue byte length.'); + record.checks.sha256 = compare(snapshot.sha256, sha256, 'The bytes do not hash to the signed manifest sha256.'); + } + if (formatError) { + // The bytes were still hashed in full; only the structural checks fall out. + const e = formatError; + record.checks.network_magic = e.reason === 'network-magic-mismatch' + ? invalid(NETWORK_MAGIC[snapshot.network], null, e.message) + : skipped(e.message, NETWORK_MAGIC[snapshot.network]); + record.checks.base_block_hash = invalid(snapshot.blockHash, null, e.message); + record.checks.coins_count = skipped(e.message, pinned?.coinCount ?? null); + record.checks.utxo_commitment = skipped(e.message, pinned?.utxoCommitment ?? null); + if (e.reason !== 'network-magic-mismatch') { + record.reason = 'snapshot-format:' + e.reason; + } + checkpoint('bytes', 'format: ' + e.message); + } + if (decoded !== null) { + record.evidence.header_format = decoded.header.format; + record.evidence.snapshot_version = decoded.header.version; + record.checks.network_magic = decoded.header.format === 'legacy' + ? skipped('A pre-28.0 snapshot header carries no network magic.', NETWORK_MAGIC[snapshot.network]) + : compare(NETWORK_MAGIC[snapshot.network], decoded.header.network_magic, 'The snapshot header names another network.'); + record.checks.base_block_hash = compare(snapshot.blockHash, decoded.header.base_block_hash, 'The snapshot header base block is not the catalogue block.'); + if (decoded.max_coin_height > snapshot.height) { + record.checks.base_block_hash = invalid(snapshot.blockHash, decoded.header.base_block_hash, `A coin is at height ${decoded.max_coin_height}, above the base height.`); + } + record.checks.coins_count = pinned?.coinCount !== null && pinned?.coinCount !== undefined + ? compare(pinned.coinCount, decoded.header.coins_count, 'The coins count differs from the pinned commitment.') + : valid(null, decoded.header.coins_count); + if (!pinned) { + record.checks.utxo_commitment = skipped('No commitment is pinned for this network, Core version, height and block; a node running that release cannot load it.', null); + record.reason = record.reason ?? 'no-pinned-commitment'; + } else if (decoded.hash_serialized_3 === null) { + record.checks.utxo_commitment = skipped('hash_serialized_3 was not computed: ' + decoded.hash_reason + '. The operator load job leaves the definitive check to Core.', pinned.utxoCommitment); + record.reason = record.reason ?? 'utxo-commitment-not-evaluated'; + } else { + record.checks.utxo_commitment = compare(pinned.utxoCommitment, decoded.hash_serialized_3, 'The streamed hash_serialized_3 is not the pinned Core commitment.'); + } + checkpoint('bytes', `${decoded.bytes_read} bytes, ${decoded.coins_read} coins`); + } + + const checks = Object.values(record.checks); + if (checks.some((check) => check.status === 'invalid')) { + record.state = 'invalid'; + record.details = 'At least one independent check failed; see checks.'; + } else if (checks.every((check) => check.status === 'valid' || (check === record.checks.network_magic && check.status === 'not-evaluated' && decoded?.header.format === 'legacy'))) { + record.state = 'valid'; + record.details = 'Bytes, manifest signature, base block, coins count and hash_serialized_3 all match the signed catalogue, the owned node and the pinned Core commitment.'; + record.reason = undefined; + } else { + record.state = 'unavailable'; + record.details = unavailableReason ?? 'A required check could not be evaluated; the snapshot is not verified.'; + record.reason = record.reason ?? 'check-not-evaluated'; + } + record.finished_at = this.clock(); + record.verified_at = record.finished_at; + checkpoint('finished', record.state); + summarise(record); + await store.updateVerification(record); + return record; + } + + /** @asyncUnsafe The caller records a rejection as the run outcome. */ + private async stream( + snapshot: BootstrapCatalogueSnapshot, + kind: 'file' | 'https', + checkpoint: (stage: string, detail?: string) => void + ): Promise<{ sha256: string; bytesRead: number; decoded: ReturnType | null; formatError: SnapshotFormatError | null }> { + const limit = Math.min(this.env.maxSnapshotBytes, snapshot.sizeBytes); + const hash = createHash('sha256'); + const decoder = new SnapshotStreamDecoder(snapshot.network); + let formatError: SnapshotFormatError | null = null; + let bytesRead = 0; + let abort: StreamAbort | undefined; + const stream = await this.bytes.open(snapshot.source, kind); + const deadline = setTimeout(() => { + abort = new StreamAbort('deadline-exceeded', `The verification deadline of ${this.env.verifyDeadlineMs} ms passed after ${bytesRead} bytes.`, bytesRead); + stream.destroy?.(abort); + }, this.env.verifyDeadlineMs); + checkpoint('bytes', 'streaming from ' + kind); + try { + for await (const chunk of stream) { + if (abort) { + throw abort; + } + bytesRead += chunk.length; + if (bytesRead > limit) { + abort = new StreamAbort('size-exceeded', `The source exceeded the bounded size of ${limit} bytes.`, bytesRead); + stream.destroy?.(abort); + throw abort; + } + hash.update(chunk); + if (!formatError) { + try { + decoder.feed(chunk); + } catch (e) { + if (!(e instanceof SnapshotFormatError)) { + throw e; + } + formatError = e; + } + } + } + if (abort) { + throw abort; + } + } catch (e) { + stream.destroy?.(); + throw e; + } finally { + clearTimeout(deadline); + } + let decoded: ReturnType | null = null; + if (!formatError) { + try { + decoded = decoder.finish(); + } catch (e) { + if (!(e instanceof SnapshotFormatError)) { + throw e; + } + formatError = e; + } + } + return { sha256: hash.digest('hex'), bytesRead, decoded, formatError }; + } +} + +export class StreamAbort extends Error { + constructor(public readonly reason: string, message: string, public readonly bytesRead: number) { + super(message); + } +} diff --git a/backend/src/api/intelligence/identity/owner-auth.ts b/backend/src/api/intelligence/identity/owner-auth.ts index 8982308da78..6cbb0f97e5a 100644 --- a/backend/src/api/intelligence/identity/owner-auth.ts +++ b/backend/src/api/intelligence/identity/owner-auth.ts @@ -1,5 +1,6 @@ import { NextFunction, Request, Response } from 'express'; import { AuthenticatedOwner, developerIdentity, IdentityError } from './developer-identity'; +import { ownerUsageLedger } from './owner-usage-ledger'; /** * Owner authentication for the intelligence surfaces. @@ -42,6 +43,7 @@ export function requireOwner(scope?: string) { return; } res.locals.owner = owner; + recordOwnerUsage(res, owner); next(); } catch (error) { res.status(503).json({ error: 'Owner authentication is unavailable: ' + (error instanceof Error ? error.message : String(error)), code: 'auth_unavailable' }); @@ -49,6 +51,12 @@ export function requireOwner(scope?: string) { }; } +/** Counts the authenticated request in the owned usage ledger once its response has finished. */ +export function recordOwnerUsage(res: Response, owner: AuthenticatedOwner, startedAt = Date.now()): void { + if (typeof res.once !== 'function') { return; } + res.once('finish', () => { ownerUsageLedger.record(owner.owner_id, owner.key_id, res.statusCode, Date.now() - startedAt); }); +} + /** Maps an IdentityError to its status and a plain body; anything else is a 500. */ export function sendIdentityError(res: Response, error: unknown, fallback: string): void { if (error instanceof IdentityError) { diff --git a/backend/src/api/intelligence/identity/owner-usage-ledger.ts b/backend/src/api/intelligence/identity/owner-usage-ledger.ts new file mode 100644 index 00000000000..2c8dcdf50eb --- /dev/null +++ b/backend/src/api/intelligence/identity/owner-usage-ledger.ts @@ -0,0 +1,145 @@ +/** + * Owned usage accounting for authenticated developer requests. + * + * Every request that authenticates an API key through `requireOwner` is + * recorded here when its response finishes: the owner, the key, the status + * class and the wall-clock latency. The ledger lives in this process's + * memory, so what it can answer is exactly what this backend instance has + * observed since it started, and it says so. It is bounded per owner and in + * the number of owners it tracks; an owner it has never seen has no + * observations, which is a distinct answer from zero requests. + */ + +export const USAGE_LEDGER_LIMITS = { + owners: 5_000, + latencySamples: 512, + keysPerOwner: 64, +} as const; + +interface KeyCounters { + key_id: string; + requests: number; + first_observed_at: string; + last_observed_at: string; +} + +interface OwnerCounters { + owner_id: string; + requests_total: number; + responses_2xx: number; + responses_4xx: number; + responses_5xx: number; + rate_limited: number; + first_observed_at: string; + last_observed_at: string; + latencies_ms: number[]; + keys: Map; +} + +export interface OwnerUsageObservation { + state: 'observed'; + owner_id: string; + requests_total: number; + responses_2xx: number; + responses_4xx: number; + responses_5xx: number; + rate_limited: number; + latency_ms: { p50: number; p95: number; max: number; samples: number }; + keys: Array<{ key_id: string; requests: number; first_observed_at: string; last_observed_at: string }>; + first_observed_at: string; + last_observed_at: string; +} + +export interface OwnerUsageAbsent { + state: 'no-observations'; + owner_id: string; +} + +export interface UsageLedgerCoverage { + observer_id: string; + persistence: 'process-memory-only'; + observed_since: string; + observed_at: string; + scope: string; +} + +function percentile(sorted: number[], fraction: number): number { + if (!sorted.length) { return 0; } + const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil(sorted.length * fraction) - 1)); + return sorted[index]; +} + +export class OwnerUsageLedger { + private owners = new Map(); + private readonly startedAt: string; + + constructor(private readonly now: () => number = Date.now, public readonly observerId = 'explorer-backend-' + process.pid) { + this.startedAt = new Date(this.now()).toISOString(); + } + + public record(ownerId: string, keyId: string, status: number, latencyMs: number): void { + if (!ownerId) { return; } + const at = new Date(this.now()).toISOString(); + let owner = this.owners.get(ownerId); + if (!owner) { + if (this.owners.size >= USAGE_LEDGER_LIMITS.owners) { + // Bounded: drop the owner that has been quiet the longest. + const oldest = this.owners.keys().next().value; + if (oldest !== undefined) { this.owners.delete(oldest); } + } + owner = { owner_id: ownerId, requests_total: 0, responses_2xx: 0, responses_4xx: 0, responses_5xx: 0, rate_limited: 0, first_observed_at: at, last_observed_at: at, latencies_ms: [], keys: new Map() }; + } + this.owners.delete(ownerId); + this.owners.set(ownerId, owner); + owner.requests_total++; + if (status >= 500) { owner.responses_5xx++; } else if (status >= 400) { owner.responses_4xx++; } else if (status >= 200 && status < 300) { owner.responses_2xx++; } + if (status === 429) { owner.rate_limited++; } + owner.last_observed_at = at; + owner.latencies_ms.push(Math.max(0, Math.round(latencyMs))); + if (owner.latencies_ms.length > USAGE_LEDGER_LIMITS.latencySamples) { owner.latencies_ms.shift(); } + const key = owner.keys.get(keyId); + if (key) { + key.requests++; + key.last_observed_at = at; + } else if (owner.keys.size < USAGE_LEDGER_LIMITS.keysPerOwner) { + owner.keys.set(keyId, { key_id: keyId, requests: 1, first_observed_at: at, last_observed_at: at }); + } + } + + /** The owner's own counters and nothing else; a foreign owner id reads as absent. */ + public read(ownerId: string): OwnerUsageObservation | OwnerUsageAbsent { + const owner = this.owners.get(ownerId); + if (!owner) { return { state: 'no-observations', owner_id: ownerId }; } + const sorted = [...owner.latencies_ms].sort((a, b) => a - b); + return { + state: 'observed', + owner_id: owner.owner_id, + requests_total: owner.requests_total, + responses_2xx: owner.responses_2xx, + responses_4xx: owner.responses_4xx, + responses_5xx: owner.responses_5xx, + rate_limited: owner.rate_limited, + latency_ms: { p50: percentile(sorted, 0.5), p95: percentile(sorted, 0.95), max: sorted[sorted.length - 1] ?? 0, samples: sorted.length }, + keys: [...owner.keys.values()].map(key => ({ ...key })), + first_observed_at: owner.first_observed_at, + last_observed_at: owner.last_observed_at, + }; + } + + public coverage(): UsageLedgerCoverage { + return { + observer_id: this.observerId, + persistence: 'process-memory-only', + observed_since: this.startedAt, + observed_at: new Date(this.now()).toISOString(), + scope: 'Requests authenticated with an API key by this backend instance since it started. Other instances, restarts and unauthenticated requests are not counted.', + }; + } + + /** Test seam. */ + public resetForTests(): void { + this.owners.clear(); + } +} + +export const ownerUsageLedger = new OwnerUsageLedger(); diff --git a/backend/src/api/intelligence/private-submission/accelerator-directory.ts b/backend/src/api/intelligence/private-submission/accelerator-directory.ts new file mode 100644 index 00000000000..f4c665e8750 --- /dev/null +++ b/backend/src/api/intelligence/private-submission/accelerator-directory.ts @@ -0,0 +1,310 @@ +import { createHash, createPublicKey, verify as verifyEd25519 } from 'crypto'; +import { readFileSync, statSync } from 'fs'; +import { verifySchnorr } from 'tiny-secp256k1'; +import config from '../../../config'; +import { AcceleratorProvider, AcceleratorProviderKey, AcceleratorReceipt, ReceiptVerificationResult } from './private-submission.models'; + +/** + * The owned accelerator provider directory and the receipt verification + * that depends on it. + * + * The directory is a JSON file named by UNIVERSE_ACCELERATOR_PROVIDER_DIRECTORY + * that an operator maintains: + * + * { + * "schema": "universe-accelerator-directory-v1", + * "revision": "2026-09-17.1", + * "providers": [{ + * "id": "provider-id", "name": "Display name", "networks": ["signet"], + * "issuer": "who vouches for these keys", "protocolVersion": "universe-accelerator-receipt-v1", + * "keys": [{ "kid": "k1", "algorithm": "ed25519" | "secp256k1-schnorr", "publicKey": "", + * "validFrom": "", "validUntil": "" | null }], + * "terms": { "minimum_fee_sats": 0, "maximum_tx_vsize": 0, "payment_methods": [], "partner_mining_claims": [] } + * }] + * } + * + * Trust comes from the file and nothing else: a receipt names a provider + * and a key id, and only a key the directory holds for that provider, valid + * at the receipt's issue time, can verify it. The signed payload encoding is + * documented on AcceleratorReceipt in the models file. + */ + +export const ACCELERATOR_DIRECTORY_VARIABLE = 'UNIVERSE_ACCELERATOR_PROVIDER_DIRECTORY'; +export const ACCELERATOR_DIRECTORY_SCHEMA = 'universe-accelerator-directory-v1'; +export const ACCELERATOR_RECEIPT_SCHEMA = 'universe-accelerator-receipt-v1'; +export const ACCELERATOR_LIMITS = { fileBytes: 1_048_576, providers: 200, keysPerProvider: 16, replayRecords: 50_000 } as const; + +export interface DirectoryProvider { + id: string; + name: string; + networks: string[]; + issuer: string; + protocolVersion: string; + keys: AcceleratorProviderKey[]; + terms: { minimum_fee_sats: number; maximum_tx_vsize: number; payment_methods: string[]; partner_mining_claims: string[] }; +} + +export interface AcceleratorDirectory { + source: string; + revision: string; + loaded_at_utc: string; + providers: DirectoryProvider[]; +} + +export class AcceleratorDirectoryError extends Error { + constructor(public readonly reason: 'unconfigured' | 'unreadable' | 'malformed', message: string) { + super(message); + } +} + +const ISO = (value: unknown): value is string => typeof value === 'string' && !Number.isNaN(Date.parse(value)); +const HEX = /^[0-9a-f]+$/i; +const ID = /^[A-Za-z0-9._:-]{1,128}$/; + +function parseProvider(raw: any, index: number): DirectoryProvider { + const where = `providers[${index}]`; + if (!raw || typeof raw !== 'object' || !ID.test(String(raw.id)) || typeof raw.name !== 'string' || !raw.name.trim() || raw.name.length > 128) { + throw new AcceleratorDirectoryError('malformed', `${where}: id and name are required.`); + } + if (!Array.isArray(raw.networks) || !raw.networks.length || raw.networks.some((n: unknown) => typeof n !== 'string' || !n)) { + throw new AcceleratorDirectoryError('malformed', `${where}: networks must list at least one network.`); + } + if (typeof raw.issuer !== 'string' || !raw.issuer.trim() || typeof raw.protocolVersion !== 'string' || !raw.protocolVersion) { + throw new AcceleratorDirectoryError('malformed', `${where}: issuer and protocolVersion are required.`); + } + if (!Array.isArray(raw.keys) || !raw.keys.length || raw.keys.length > ACCELERATOR_LIMITS.keysPerProvider) { + throw new AcceleratorDirectoryError('malformed', `${where}: keys must hold 1 to ${ACCELERATOR_LIMITS.keysPerProvider} entries.`); + } + const keys: AcceleratorProviderKey[] = raw.keys.map((key: any, k: number) => { + const keyWhere = `${where}.keys[${k}]`; + if (!key || !ID.test(String(key.kid)) || !['ed25519', 'secp256k1-schnorr'].includes(key.algorithm)) { + throw new AcceleratorDirectoryError('malformed', `${keyWhere}: kid and a supported algorithm are required.`); + } + if (typeof key.publicKey !== 'string' || !HEX.test(key.publicKey) || key.publicKey.length !== 64) { + throw new AcceleratorDirectoryError('malformed', `${keyWhere}: publicKey must be 32 bytes of hex (ed25519 key or x-only secp256k1 point).`); + } + if (!ISO(key.validFrom) || (key.validUntil !== null && key.validUntil !== undefined && !ISO(key.validUntil))) { + throw new AcceleratorDirectoryError('malformed', `${keyWhere}: validFrom must be a timestamp and validUntil a timestamp or null.`); + } + return { kid: key.kid, algorithm: key.algorithm, publicKey: key.publicKey.toLowerCase(), validFrom: key.validFrom, validUntil: key.validUntil ?? null }; + }); + if (new Set(keys.map(key => key.kid)).size !== keys.length) { + throw new AcceleratorDirectoryError('malformed', `${where}: key ids must be unique.`); + } + const terms = raw.terms ?? {}; + const list = (v: unknown): string[] => (Array.isArray(v) ? v.filter((x): x is string => typeof x === 'string').slice(0, 32) : []); + return { + id: raw.id, name: raw.name.trim(), networks: raw.networks, issuer: raw.issuer.trim(), protocolVersion: raw.protocolVersion, keys, + terms: { minimum_fee_sats: Number.isFinite(terms.minimum_fee_sats) ? terms.minimum_fee_sats : 0, maximum_tx_vsize: Number.isFinite(terms.maximum_tx_vsize) ? terms.maximum_tx_vsize : 0, payment_methods: list(terms.payment_methods), partner_mining_claims: list(terms.partner_mining_claims) }, + }; +} + +export function parseAcceleratorDirectory(text: string, source: string, loadedAt = Date.now()): AcceleratorDirectory { + let raw: any; + try { + raw = JSON.parse(text); + } catch { + throw new AcceleratorDirectoryError('malformed', 'The provider directory is not JSON.'); + } + if (!raw || raw.schema !== ACCELERATOR_DIRECTORY_SCHEMA || typeof raw.revision !== 'string' || !raw.revision || !Array.isArray(raw.providers)) { + throw new AcceleratorDirectoryError('malformed', `The provider directory must declare schema ${ACCELERATOR_DIRECTORY_SCHEMA}, a revision and a providers list.`); + } + if (raw.providers.length > ACCELERATOR_LIMITS.providers) { + throw new AcceleratorDirectoryError('malformed', `The provider directory lists more than ${ACCELERATOR_LIMITS.providers} providers.`); + } + const providers = raw.providers.map(parseProvider); + if (new Set(providers.map((p: DirectoryProvider) => p.id)).size !== providers.length) { + throw new AcceleratorDirectoryError('malformed', 'Provider ids must be unique.'); + } + return { source, revision: raw.revision, loaded_at_utc: new Date(loadedAt).toISOString(), providers }; +} + +let cache: { path: string; mtimeMs: number; directory: AcceleratorDirectory } | null = null; +let override: AcceleratorDirectory | null | undefined; + +/** The directory named by the environment; re-read when the file changes. */ +export function acceleratorDirectory(environment: Record = process.env): AcceleratorDirectory { + if (override !== undefined) { + if (override === null) { throw new AcceleratorDirectoryError('unconfigured', unconfiguredMessage()); } + return override; + } + const path = String(environment[ACCELERATOR_DIRECTORY_VARIABLE] ?? '').trim(); + if (!path) { throw new AcceleratorDirectoryError('unconfigured', unconfiguredMessage()); } + let mtimeMs: number; + let size: number; + try { + const stat = statSync(path); + mtimeMs = stat.mtimeMs; + size = stat.size; + } catch (e) { + throw new AcceleratorDirectoryError('unreadable', `The provider directory at ${ACCELERATOR_DIRECTORY_VARIABLE} could not be read: ${e instanceof Error ? e.message : String(e)}`); + } + if (size > ACCELERATOR_LIMITS.fileBytes) { throw new AcceleratorDirectoryError('malformed', 'The provider directory file is too large.'); } + if (cache && cache.path === path && cache.mtimeMs === mtimeMs) { return cache.directory; } + const directory = parseAcceleratorDirectory(readFileSync(path, 'utf8'), path); + cache = { path, mtimeMs, directory }; + return directory; +} + +function unconfiguredMessage(): string { + return `The accelerator provider directory is unavailable. Provider identities and receipt trust require the owned signed provider directory named by ${ACCELERATOR_DIRECTORY_VARIABLE}, which is not configured on this deployment.`; +} + +/** Test seam. */ +export function useAcceleratorDirectory(directory: AcceleratorDirectory | null | undefined): void { + override = directory; + replays.clear(); +} + +export function providerView(provider: DirectoryProvider, directory: AcceleratorDirectory, now = Date.now()): AcceleratorProvider { + const current = provider.keys.find(key => keyValidAt(key, now)) ?? provider.keys[0]; + const from = provider.keys.map(key => Date.parse(key.validFrom)).sort((a, b) => a - b)[0]; + const until = provider.keys.some(key => key.validUntil === null) ? null : provider.keys.map(key => Date.parse(key.validUntil as string)).sort((a, b) => b - a)[0]; + return { + provider_id: provider.id, + identity_key: current.publicKey, + name: provider.name, + supported_networks: provider.networks, + submission_modes: ['configured_accelerator'], + minimum_fee_sats: provider.terms.minimum_fee_sats, + maximum_tx_vsize: provider.terms.maximum_tx_vsize, + payment_methods: provider.terms.payment_methods, + partner_mining_claims: provider.terms.partner_mining_claims, + status_endpoint: null, + health_status: 'unmeasured', + effective_from: new Date(from).toISOString(), + expires_at: until === null ? null : new Date(until).toISOString(), + provider_signature: null, + issuer: provider.issuer, + protocol_version: provider.protocolVersion, + keys: provider.keys.map(key => ({ ...key })), + directory: { source: directory.source, revision: directory.revision, loaded_at_utc: directory.loaded_at_utc }, + }; +} + +export function keyValidAt(key: AcceleratorProviderKey, at: number): boolean { + return Date.parse(key.validFrom) <= at && (key.validUntil === null || Date.parse(key.validUntil) > at); +} + +/** Deterministic JSON: keys sorted at every level, no whitespace. */ +export function deterministicJson(value: unknown): string { + if (value === null || typeof value !== 'object') { return JSON.stringify(value); } + if (Array.isArray(value)) { return '[' + value.map(deterministicJson).join(',') + ']'; } + const keys = Object.keys(value as Record).filter(key => (value as Record)[key] !== undefined).sort(); + return '{' + keys.map(key => JSON.stringify(key) + ':' + deterministicJson((value as Record)[key])).join(',') + '}'; +} + +/** The bytes a provider signs for a receipt. */ +export function receiptSigningMessage(receipt: Omit): Buffer { + return Buffer.from(ACCELERATOR_RECEIPT_SCHEMA + '\n' + deterministicJson(receipt), 'utf8'); +} + +const ED25519_SPKI_PREFIX = Buffer.from('302a300506032b6570032100', 'hex'); + +export function verifyReceiptSignature(key: AcceleratorProviderKey, message: Buffer, signatureHex: string): boolean { + if (!HEX.test(signatureHex) || signatureHex.length !== 128) { return false; } + const signature = Buffer.from(signatureHex, 'hex'); + try { + if (key.algorithm === 'ed25519') { + const publicKey = createPublicKey({ key: Buffer.concat([ED25519_SPKI_PREFIX, Buffer.from(key.publicKey, 'hex')]), format: 'der', type: 'spki' }); + return verifyEd25519(null, message, publicKey, signature); + } + return verifySchnorr(createHash('sha256').update(message).digest(), Buffer.from(key.publicKey, 'hex'), signature); + } catch { + return false; + } +} + +const replays = new Map(); + +/** Structural checks on the caller's payload; these need no directory. */ +export function receiptStructuralErrors(receipt: Partial): string[] { + const errors: string[] = []; + for (const [key, max] of [['provider_id', 256], ['receipt_id', 256], ['provider_signature', 8192], ['key_id', 128], ['schema_version', 64], ['network', 32]] as const) { + if (typeof receipt[key] === 'string' && (receipt[key] as string).length > max) { errors.push(key + ' is too long'); } + } + if (typeof receipt.provider_id !== 'string' || !receipt.provider_id.trim()) { errors.push('provider_id is required'); } + if (typeof receipt.receipt_id !== 'string' || !receipt.receipt_id.trim()) { errors.push('receipt_id is required'); } + if (typeof receipt.provider_signature !== 'string' || !receipt.provider_signature.trim()) { errors.push('provider_signature is required'); } + if (typeof receipt.txid !== 'string' || !/^[0-9a-f]{64}$/i.test(receipt.txid)) { errors.push('Valid 32-byte txid is required'); } + return errors; +} + +function completeReceiptErrors(receipt: Partial): string[] { + const errors: string[] = []; + if (typeof receipt.key_id !== 'string' || !receipt.key_id.trim()) { errors.push('key_id is required'); } + if (typeof receipt.network !== 'string' || !receipt.network) { errors.push('network is required'); } + if (!ISO(receipt.submitted_at_utc)) { errors.push('submitted_at_utc must be a timestamp'); } + if (!ISO(receipt.expires_at_utc)) { errors.push('expires_at_utc must be a timestamp'); } + for (const field of ['target_feerate_sats_vb', 'provider_fee_sats', 'claimed_mining_coverage_pct'] as const) { + if (typeof receipt[field] !== 'number' || !Number.isFinite(receipt[field])) { errors.push(field + ' must be a number'); } + } + if (!Array.isArray(receipt.claimed_partner_pools) || receipt.claimed_partner_pools.some(p => typeof p !== 'string')) { errors.push('claimed_partner_pools must be a list of strings'); } + if (!['active', 'included', 'expired', 'refunded'].includes(String(receipt.status))) { errors.push('status must be active, included, expired or refunded'); } + if (receipt.wtxid !== undefined && (typeof receipt.wtxid !== 'string' || !/^[0-9a-f]{64}$/i.test(receipt.wtxid))) { errors.push('wtxid must be 32 bytes of hex when present'); } + return errors; +} + +/** + * Verifies a receipt against the directory. Every failure names its stage; + * a verified receipt is recorded so a replay of the same provider and + * receipt id is reported as a duplicate. + */ +export function verifyAcceleratorReceipt(receipt: Partial, options: { now?: number; network?: string; directory?: () => AcceleratorDirectory } = {}): ReceiptVerificationResult { + const now = options.now ?? Date.now(); + const network = options.network ?? config.MEMPOOL.NETWORK; + if (!receipt || typeof receipt !== 'object' || Array.isArray(receipt) || JSON.stringify(receipt).length > 16384) { + return { verified: false, stage: 'invalid', errors: ['Receipt must be a JSON object no larger than16KiB.'] }; + } + const structural = receiptStructuralErrors(receipt); + if (structural.length) { return { verified: false, stage: 'invalid', errors: structural }; } + const identity = { provider_id: receipt.provider_id as string, receipt_id: receipt.receipt_id as string }; + if (receipt.schema_version !== ACCELERATOR_RECEIPT_SCHEMA) { + return { verified: false, stage: 'unsupported', errors: [`schema_version must be ${ACCELERATOR_RECEIPT_SCHEMA}; no other receipt encoding is verifiable here.`], ...identity }; + } + const complete = completeReceiptErrors(receipt); + if (complete.length) { return { verified: false, stage: 'invalid', errors: complete, ...identity }; } + const full = receipt as AcceleratorReceipt; + + let directory: AcceleratorDirectory; + try { + directory = (options.directory ?? acceleratorDirectory)(); + } catch (e) { + return { verified: false, stage: 'unavailable-trust', errors: [e instanceof Error ? e.message : String(e)], ...identity, key_id: full.key_id }; + } + const info = { ...identity, key_id: full.key_id, directory: { source: directory.source, revision: directory.revision, loaded_at_utc: directory.loaded_at_utc } }; + const provider = directory.providers.find(p => p.id === full.provider_id); + if (!provider) { return { verified: false, stage: 'unavailable-trust', errors: [`Provider ${full.provider_id} is not in the owned directory (revision ${directory.revision}); its receipts cannot be trusted.`], ...info }; } + if (provider.protocolVersion !== ACCELERATOR_RECEIPT_SCHEMA) { return { verified: false, stage: 'unsupported', errors: [`Provider ${provider.id} issues ${provider.protocolVersion} receipts, which this deployment cannot verify.`], ...info }; } + const key = provider.keys.find(k => k.kid === full.key_id); + if (!key) { return { verified: false, stage: 'unavailable-trust', errors: [`Key ${full.key_id} is not a directory key of provider ${provider.id}.`], ...info }; } + const issuedAt = Date.parse(full.submitted_at_utc); + if (!keyValidAt(key, issuedAt)) { return { verified: false, stage: 'unavailable-trust', errors: [`Key ${key.kid} was not valid at ${full.submitted_at_utc}.`], ...info, algorithm: key.algorithm }; } + if (full.network !== network || !provider.networks.includes(full.network)) { + return { verified: false, stage: 'wrong-network', errors: [`Receipt is for ${full.network}; this deployment serves ${network} and the provider is listed for ${provider.networks.join(', ')}.`], ...info, algorithm: key.algorithm }; + } + const { provider_signature, key_id, ...payload } = full; + void key_id; + if (!verifyReceiptSignature(key, receiptSigningMessage(payload), provider_signature)) { + return { verified: false, stage: 'invalid', errors: ['provider_signature does not verify over the receipt payload with the directory key; a field or the signature was altered.'], ...info, algorithm: key.algorithm }; + } + if (Date.parse(full.expires_at_utc) <= now) { + return { verified: false, stage: 'expired', errors: [`Receipt expired at ${full.expires_at_utc}.`], ...info, algorithm: key.algorithm }; + } + if (Date.parse(full.expires_at_utc) <= issuedAt) { + return { verified: false, stage: 'invalid', errors: ['expires_at_utc precedes submitted_at_utc.'], ...info, algorithm: key.algorithm }; + } + const replayKey = `${provider.id}\u0000${full.receipt_id}`; + const seen = replays.get(replayKey); + const at = new Date(now).toISOString(); + if (seen) { + seen.seen_count++; + return { verified: false, stage: 'duplicate', errors: [`Receipt ${full.receipt_id} from ${provider.id} was already verified at ${seen.first_verified_at_utc}; this presentation is a replay.`], ...info, algorithm: key.algorithm, replay: { ...seen } }; + } + if (replays.size >= ACCELERATOR_LIMITS.replayRecords) { replays.delete(replays.keys().next().value as string); } + replays.set(replayKey, { first_verified_at_utc: at, seen_count: 1 }); + return { + verified: true, stage: 'verified', errors: [], ...info, algorithm: key.algorithm, verified_at_utc: at, replay: { first_verified_at_utc: at, seen_count: 1 }, + scope: 'The signature binds the provider, receipt, transaction, network, times and terms to a directory key. It proves the provider issued the receipt, not that the acceleration was delivered; replay records are kept in this process only.', + }; +} diff --git a/backend/src/api/intelligence/private-submission/ordering-evidence.service.ts b/backend/src/api/intelligence/private-submission/ordering-evidence.service.ts new file mode 100644 index 00000000000..5094e29baa3 --- /dev/null +++ b/backend/src/api/intelligence/private-submission/ordering-evidence.service.ts @@ -0,0 +1,250 @@ +import config from '../../../config'; +import { BlockExtended, TransactionExtended } from '../../../mempool.interfaces'; +import { RelayCollectorService, relayCollectorService, RelayEvidenceError } from '../relay/relay-collector.service'; +import { CandidateTemplate, TemplateCollectorService, templateCollectorService } from '../templates/template-collector.service'; +import { OrderingEvidenceState, TransactionOrderingEvidence } from './private-submission.models'; + +/** + * Ordering evidence: how a mined block's transaction order relates to the + * candidate templates this backend recorded for that height and to the + * relay collector's first-seen observations. + * + * Blocks reach this service through the block observation hub, the same + * path the template collector uses. For each mined transaction the + * evidence names what was observed: seen in the mempool before inclusion, + * present in a recorded template, both, or neither. A height with no + * recorded template is 'no-template-observed', which is a statement about + * this backend's coverage and not about the block. + */ + +export const ORDERING_LIMITS = { blocks: 50, transactionsPerBlock: 10_000, findingsPerBlock: 200 } as const; + +export interface RecordedBlockTransaction { + txid: string; + position: number; + fee: number; + vsize: number; + effective_feerate_sats_vb: number | null; + /** Inputs that spend outputs of transactions earlier in the same block. */ + dependency_txids: string[]; +} + +export interface RecordedBlock { + block_hash: string; + height: number; + previous_block_hash: string; + mined_timestamp_utc: string; + observed_at_utc: string; + transactions: RecordedBlockTransaction[]; + /** The templates recorded for this height and parent, snapshotted at block receipt. */ + templates: Array<{ template_id: string; source_id: string; observed_at_utc: string; positions: Map }>; + truncated: boolean; +} + +export type TemplateCoverage = 'template-observed' | 'no-template-observed'; + +export interface BlockOrderingEvidence { + block_hash: string; + height: number; + mined_timestamp_utc: string; + network: string; + template_coverage: TemplateCoverage; + templates_compared: string[]; + sensor: { observer_id: string | null; observers: 1 }; + transactions: TransactionOrderingEvidence[]; + truncated: boolean; + scope: string; +} + +export interface OrderingFindings { + network: string; + state: 'observed' | 'no-blocks-observed'; + findings: TransactionOrderingEvidence[]; + coverage: Array<{ block_hash: string; height: number; template_coverage: TemplateCoverage; transactions: number; findings: number }>; + retention: { blocks: number; retained_blocks: number }; + scope: string; +} + +export interface OrderingReaders { + templatesForHeight: (height: number) => CandidateTemplate[]; + firstSeen: (txid: string) => { first_observed_utc: string; source_id: string } | null; +} + +export class OrderingEvidenceError extends Error { + constructor(public readonly code: 'block-not-observed' | 'transaction-not-observed', message: string, public readonly status = 404) { + super(message); + } +} + +const SCOPE = 'One owned observer: this backend\'s mempool poll (first-seen) and the templates its owned Core node and projection produced for the height. Other nodes\' views, miner intent and private submission channels are not observed; findings describe recorded differences, not causes.'; + +function ownedReaders(relay: Pick, templates: Pick): OrderingReaders { + return { + templatesForHeight: height => templates.getTemplatesForHeight(height), + firstSeen: txid => { + try { + const record = relay.getPropagationForTx(txid); + return { first_observed_utc: record.first_observed_utc, source_id: record.source_id }; + } catch (e) { + if (e instanceof RelayEvidenceError) { return null; } + throw e; + } + }, + }; +} + +export class OrderingEvidenceService { + private blocks = new Map(); + private byTxid = new Map(); + + constructor(private readonly readers: OrderingReaders = ownedReaders(relayCollectorService, templateCollectorService), private readonly network: string = config.MEMPOOL.NETWORK) {} + + /** Called from the block hub with the mined block and its transactions in block order. */ + public observeBlock(block: BlockExtended, transactions: TransactionExtended[], now = Date.now()): RecordedBlock { + const inBlock = new Set(transactions.map(tx => tx.txid)); + const ordered = transactions.filter(tx => !tx.vin?.some(vin => vin.is_coinbase)); + const truncated = ordered.length > ORDERING_LIMITS.transactionsPerBlock; + const recorded: RecordedBlockTransaction[] = ordered.slice(0, ORDERING_LIMITS.transactionsPerBlock).map((tx, index) => ({ + txid: tx.txid, + position: index + 1, + fee: tx.fee ?? 0, + vsize: tx.vsize ?? (tx.weight ? Math.ceil(tx.weight / 4) : 0), + effective_feerate_sats_vb: typeof tx.effectiveFeePerVsize === 'number' && Number.isFinite(tx.effectiveFeePerVsize) ? tx.effectiveFeePerVsize : null, + dependency_txids: [...new Set((tx.vin ?? []).map(vin => vin.txid).filter((id): id is string => typeof id === 'string' && inBlock.has(id)))], + })); + const templates = this.readers.templatesForHeight(block.height) + .filter(template => template.prev_block_hash === block.previousblockhash && Date.parse(template.observed_at_utc) <= now) + .map(template => ({ template_id: template.template_id, source_id: template.source_id, observed_at_utc: template.observed_at_utc, positions: new Map(template.txids.map((txid, index) => [txid, index + 1])) })); + const record: RecordedBlock = { + block_hash: block.id, + height: block.height, + previous_block_hash: block.previousblockhash, + mined_timestamp_utc: new Date(block.timestamp * 1000).toISOString(), + observed_at_utc: new Date(now).toISOString(), + transactions: recorded, + templates, + truncated, + }; + this.blocks.delete(block.id); + this.blocks.set(block.id, record); + for (const tx of recorded) { this.byTxid.set(tx.txid, block.id); } + while (this.blocks.size > ORDERING_LIMITS.blocks) { + const oldest = this.blocks.keys().next().value as string; + for (const tx of this.blocks.get(oldest)?.transactions ?? []) { if (this.byTxid.get(tx.txid) === oldest) { this.byTxid.delete(tx.txid); } } + this.blocks.delete(oldest); + } + return record; + } + + private evidenceFor(block: RecordedBlock, tx: RecordedBlockTransaction): TransactionOrderingEvidence { + const seen = this.readers.firstSeen(tx.txid); + const minedAt = Date.parse(block.mined_timestamp_utc); + const seenBefore = seen !== null && Date.parse(seen.first_observed_utc) <= minedAt; + const inTemplates = block.templates.filter(template => template.positions.has(tx.txid)); + const earliestTemplate = inTemplates.length ? inTemplates.reduce((a, b) => (Date.parse(b.observed_at_utc) < Date.parse(a.observed_at_utc) ? b : a)) : null; + const hasTemplates = block.templates.length > 0; + let reordered = false; + if (earliestTemplate) { + // Relative order against the template: the block placed this transaction ahead of + // one the template placed before it (both in both), so the block moved it earlier. + const templatePosition = earliestTemplate.positions.get(tx.txid) as number; + reordered = block.transactions.some(other => other.position > tx.position && (earliestTemplate.positions.get(other.txid) ?? Infinity) < templatePosition); + } + let state: OrderingEvidenceState; + if (!hasTemplates) { + state = 'insufficient_coverage'; + } else if (tx.dependency_txids.length) { + state = 'dependency_required_order'; + } else if (reordered) { + state = 'ordering_changed_between_template_and_block'; + } else if (seenBefore && inTemplates.length) { + state = 'publicly_observed_before_inclusion'; + } else if (!seenBefore && inTemplates.length) { + state = 'observed_only_in_template'; + } else if (seenBefore) { + state = 'publicly_observed_before_inclusion'; + } else { + state = 'included_without_public_observation'; + } + const feerate = tx.vsize > 0 ? Math.round((tx.fee / tx.vsize) * 100) / 100 : 0; + return { + txid: tx.txid, + block_hash: block.block_hash, + block_height: block.height, + block_position: tx.position, + first_sensor_seen_utc: seen?.first_observed_utc, + first_template_seen_utc: earliestTemplate?.observed_at_utc, + private_receipt_timestamp_utc: undefined, + mined_timestamp_utc: block.mined_timestamp_utc, + evidence_state: state, + fee_sats_vb: feerate, + package_feerate_sats_vb: tx.effective_feerate_sats_vb ?? feerate, + dependency_txids: tx.dependency_txids, + is_ordering_sensitive: tx.dependency_txids.length > 0 || reordered, + protocol_impact_description: reordered ? 'The block placed this transaction earlier than the recorded template did relative to at least one other transaction both contained.' : undefined, + confidence_rating: hasTemplates && seen ? 'high' : hasTemplates || seen ? 'medium' : 'low', + template_coverage: hasTemplates ? 'template-observed' : 'no-template-observed', + template_ids: inTemplates.map(template => template.template_id), + template_position: earliestTemplate ? earliestTemplate.positions.get(tx.txid) ?? null : null, + sensor_observer_id: seen?.source_id ?? null, + }; + } + + public getTransactionOrdering(txid: string): TransactionOrderingEvidence { + const blockHash = this.byTxid.get(txid.toLowerCase()); + const block = blockHash ? this.blocks.get(blockHash) : undefined; + const tx = block?.transactions.find(entry => entry.txid === txid.toLowerCase()); + if (!block || !tx) { + throw new OrderingEvidenceError('transaction-not-observed', `No block retained by this backend (last ${ORDERING_LIMITS.blocks}) contains that transaction; ordering evidence exists only for mined transactions this observer recorded.`); + } + return this.evidenceFor(block, tx); + } + + public getBlockOrdering(blockHash: string): BlockOrderingEvidence { + const block = this.blocks.get(blockHash.toLowerCase()); + if (!block) { + throw new OrderingEvidenceError('block-not-observed', `This backend has not recorded that block (it retains the last ${ORDERING_LIMITS.blocks} it processed).`); + } + const transactions = block.transactions.map(tx => this.evidenceFor(block, tx)); + return { + block_hash: block.block_hash, + height: block.height, + mined_timestamp_utc: block.mined_timestamp_utc, + network: this.network, + template_coverage: block.templates.length ? 'template-observed' : 'no-template-observed', + templates_compared: block.templates.map(template => template.template_id), + sensor: { observer_id: transactions.find(tx => tx.sensor_observer_id)?.sensor_observer_id ?? null, observers: 1 }, + transactions, + truncated: block.truncated, + scope: SCOPE, + }; + } + + /** Notable differences across retained blocks, with per-block coverage so absence of findings is never mistaken for absence of data. */ + public listOrderingFindings(): OrderingFindings { + const notable: OrderingEvidenceState[] = ['ordering_changed_between_template_and_block', 'observed_only_in_template', 'included_without_public_observation', 'dependency_required_order']; + const findings: TransactionOrderingEvidence[] = []; + const coverage: OrderingFindings['coverage'] = []; + for (const block of [...this.blocks.values()].reverse()) { + const evidence = block.transactions.map(tx => this.evidenceFor(block, tx)).filter(entry => notable.includes(entry.evidence_state)).slice(0, ORDERING_LIMITS.findingsPerBlock); + findings.push(...evidence); + coverage.push({ block_hash: block.block_hash, height: block.height, template_coverage: block.templates.length ? 'template-observed' : 'no-template-observed', transactions: block.transactions.length, findings: evidence.length }); + } + return { + network: this.network, + state: this.blocks.size ? 'observed' : 'no-blocks-observed', + findings, + coverage, + retention: { blocks: ORDERING_LIMITS.blocks, retained_blocks: this.blocks.size }, + scope: SCOPE, + }; + } + + /** Test seam. */ + public resetForTests(): void { + this.blocks.clear(); + this.byTxid.clear(); + } +} + +export const orderingEvidenceService = new OrderingEvidenceService(); diff --git a/backend/src/api/intelligence/private-submission/private-relay.config.ts b/backend/src/api/intelligence/private-submission/private-relay.config.ts new file mode 100644 index 00000000000..14fd38683ca --- /dev/null +++ b/backend/src/api/intelligence/private-submission/private-relay.config.ts @@ -0,0 +1,120 @@ +import { PrivateRelayEndpoint, PrivateRelayEndpointIssue, PrivateRelayEndpointSet } from './private-relay.types'; + +/** + * Reads the owned relay endpoints from UNIVERSE_PRIVATE_RELAY_ENDPOINTS. + * + * The value is a JSON array of + * { id, transport: 'tor' | 'i2p', proxy: 'socks5h://host:port', + * submitUrl: 'http://.onion/api/tx', network, timeoutMs? } + * + * Every entry is checked against the backend's own network, and the submit + * host must be an onion or i2p name so that a misconfiguration cannot turn + * the private path into a clearnet post to somebody else's broadcaster. An + * entry that fails is reported by id with its reason and never used. + */ +export const PRIVATE_RELAY_ENDPOINTS_ENV = 'UNIVERSE_PRIVATE_RELAY_ENDPOINTS'; + +const DEFAULT_TIMEOUT_MS = 30_000; +const MIN_TIMEOUT_MS = 1_000; +const MAX_TIMEOUT_MS = 120_000; +const MAX_ENDPOINTS = 16; + +export function parsePrivateRelayEndpoints(value: string | undefined, network: string): PrivateRelayEndpointSet { + const trimmed = (value ?? '').trim(); + if (!trimmed) { + return { endpoints: [], issues: [], unconfigured: true }; + } + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch { + return { endpoints: [], issues: [{ id: '*', reason: 'not-json' }], unconfigured: false }; + } + if (!Array.isArray(parsed)) { + return { endpoints: [], issues: [{ id: '*', reason: 'not-an-array' }], unconfigured: false }; + } + if (parsed.length > MAX_ENDPOINTS) { + return { endpoints: [], issues: [{ id: '*', reason: 'too-many-endpoints' }], unconfigured: false }; + } + const endpoints: PrivateRelayEndpoint[] = []; + const issues: PrivateRelayEndpointIssue[] = []; + const seen = new Set(); + parsed.forEach((entry, index) => { + const result = parseEndpoint(entry, network); + const id = typeof entry?.id === 'string' && entry.id ? String(entry.id).slice(0, 64) : `#${index}`; + if ('reason' in result) { + issues.push({ id, reason: result.reason }); + return; + } + if (seen.has(result.endpoint.id)) { + issues.push({ id, reason: 'duplicate-id' }); + return; + } + seen.add(result.endpoint.id); + endpoints.push(result.endpoint); + }); + return { endpoints, issues, unconfigured: false }; +} + +function parseEndpoint(entry: any, network: string): { endpoint: PrivateRelayEndpoint } | { reason: string } { + if (!entry || typeof entry !== 'object' || Array.isArray(entry)) return { reason: 'not-an-object' }; + if (typeof entry.id !== 'string' || !/^[a-z0-9][a-z0-9-]{0,63}$/.test(entry.id)) return { reason: 'invalid-id' }; + if (entry.transport !== 'tor' && entry.transport !== 'i2p') return { reason: 'invalid-transport' }; + if (typeof entry.network !== 'string' || entry.network !== network) return { reason: 'network-mismatch' }; + + let proxy: URL; + try { + proxy = new URL(String(entry.proxy)); + } catch { + return { reason: 'invalid-proxy' }; + } + if (!['socks5h:', 'socks5:', 'socks:'].includes(proxy.protocol) || !proxy.hostname || !proxy.port || (proxy.pathname !== '' && proxy.pathname !== '/') || proxy.search || proxy.hash) { + return { reason: 'invalid-proxy' }; + } + + let submitUrl: URL; + try { + submitUrl = new URL(String(entry.submitUrl)); + } catch { + return { reason: 'invalid-submit-url' }; + } + if (!['http:', 'https:'].includes(submitUrl.protocol) || submitUrl.username || submitUrl.password || submitUrl.hash) { + return { reason: 'invalid-submit-url' }; + } + const host = submitUrl.hostname.toLowerCase(); + const expectedSuffix = entry.transport === 'tor' ? '.onion' : '.i2p'; + if (!host.endsWith(expectedSuffix)) return { reason: 'submit-host-not-' + entry.transport }; + + let timeoutMs = DEFAULT_TIMEOUT_MS; + if (entry.timeoutMs !== undefined) { + if (typeof entry.timeoutMs !== 'number' || !Number.isInteger(entry.timeoutMs) || entry.timeoutMs < MIN_TIMEOUT_MS || entry.timeoutMs > MAX_TIMEOUT_MS) { + return { reason: 'invalid-timeout' }; + } + timeoutMs = entry.timeoutMs; + } + + return { + endpoint: { + id: entry.id, + transport: entry.transport, + proxy: proxy.toString(), + submitUrl: submitUrl.toString(), + network: entry.network, + timeoutMs, + }, + }; +} + +/** Which transport a submission method asks for; null when the method is not a private relay method. */ +export function transportForMethod(method: string): PrivateRelayEndpoint['transport'] | 'any' | null { + switch (method) { + case 'privatebroadcast_tor': return 'tor'; + case 'privatebroadcast_i2p': return 'i2p'; + case 'configured_private_relay': return 'any'; + default: return null; + } +} + +export function endpointsForTransport(endpoints: PrivateRelayEndpoint[], transport: PrivateRelayEndpoint['transport'] | 'any'): PrivateRelayEndpoint[] { + return transport === 'any' ? endpoints : endpoints.filter(endpoint => endpoint.transport === transport); +} diff --git a/backend/src/api/intelligence/private-submission/private-relay.runtime.ts b/backend/src/api/intelligence/private-submission/private-relay.runtime.ts new file mode 100644 index 00000000000..d14061d945f --- /dev/null +++ b/backend/src/api/intelligence/private-submission/private-relay.runtime.ts @@ -0,0 +1,84 @@ +import config from '../../../config'; +import logger from '../../../logger'; +import { PRIVATE_RELAY_ENDPOINTS_ENV, parsePrivateRelayEndpoints } from './private-relay.config'; +import { MysqlPrivateRelayStore } from './private-relay.store'; +import { SocksPrivateRelayTransport } from './private-relay.transport'; +import { PrivateRelayEndpointSet, PrivateRelayStore } from './private-relay.types'; +import { PrivateRelayWorker } from './private-relay.worker'; + +/** + * The process-wide private relay wiring: the parsed endpoint set, the durable + * store (only when the database is enabled), and the worker. The service reads + * it lazily so tests can install a runtime with fakes; index.ts starts the + * worker once the HTTP surface is up. + */ +export interface PrivateRelayRuntime { + network: string; + endpoints: PrivateRelayEndpointSet; + /** null when config.DATABASE.ENABLED is false: the feature is unavailable, not in-memory. */ + store: PrivateRelayStore | null; + worker: PrivateRelayWorker | null; + coreVersion: () => string; +} + +let runtime: PrivateRelayRuntime | null = null; + +export function getPrivateRelayRuntime(): PrivateRelayRuntime { + if (!runtime) runtime = buildRuntime(); + return runtime; +} + +/** Tests install a runtime built on fakes; null restores the lazy default. */ +export function setPrivateRelayRuntime(override: PrivateRelayRuntime | null): void { + runtime = override; +} + +/** Called from index.ts once the backend is serving. Safe to call when nothing is configured. */ +export function startPrivateRelayWorker(): void { + const current = getPrivateRelayRuntime(); + if (!current.worker) { + const why = !current.store ? 'the database is disabled' : current.endpoints.unconfigured ? PRIVATE_RELAY_ENDPOINTS_ENV + ' is not set' : 'no declared endpoint is usable'; + logger.info('private relay worker not started: ' + why); + return; + } + current.worker.start(); + logger.info(`private relay worker started for ${current.network} with ${current.endpoints.endpoints.length} endpoint(s)`); +} + +function buildRuntime(): PrivateRelayRuntime { + const network = config.MEMPOOL.NETWORK; + const endpoints = parsePrivateRelayEndpoints(process.env[PRIVATE_RELAY_ENDPOINTS_ENV], network); + for (const issue of endpoints.issues) { + logger.warn(`private relay endpoint ${issue.id} ignored: ${issue.reason}`); + } + const store = config.DATABASE.ENABLED === true ? new MysqlPrivateRelayStore() : null; + const worker = store && endpoints.endpoints.length + ? new PrivateRelayWorker({ + store, + transport: new SocksPrivateRelayTransport(), + endpoints: endpoints.endpoints, + network, + confirmation: lookupConfirmation, + }) + : null; + return { network, endpoints, store, worker, coreVersion: readCoreVersion }; +} + +/** @asyncUnsafe The worker treats a rejection as "not observed yet". */ +async function lookupConfirmation(txid: string): Promise<{ confirmed: boolean; block_height?: number } | null> { + const bitcoinApi = (await import('../../bitcoin/bitcoin-api-factory')).default; + const tx = await bitcoinApi.$getRawTransaction(txid); + if (!tx?.status) return null; + return { confirmed: tx.status.confirmed === true, block_height: tx.status.block_height }; +} + +function readCoreVersion(): string { + try { + // Loaded lazily: backend-info opens the Core RPC client on import. + // eslint-disable-next-line @typescript-eslint/no-var-requires + const backendInfo = require('../../backend-info').default; + return String(backendInfo.getBackendInfo()?.coreVersion ?? '?'); + } catch { + return '?'; + } +} diff --git a/backend/src/api/intelligence/private-submission/private-relay.store.test.ts b/backend/src/api/intelligence/private-submission/private-relay.store.test.ts new file mode 100644 index 00000000000..cfe4012d316 --- /dev/null +++ b/backend/src/api/intelligence/private-submission/private-relay.store.test.ts @@ -0,0 +1,87 @@ +import DB from '../../../database'; +import config from '../../../config'; +import { MysqlPrivateRelayStore, PRIVATE_RELAY_TABLE } from './private-relay.store'; +import { PrivateRelaySubmissionRow } from './private-relay.types'; + +jest.mock('../../../database', () => ({ __esModule: true, default: { query: jest.fn() } })); + +/** + * The MySQL store against a mocked driver: what matters is that every + * competing transition is one UPDATE whose WHERE names the state (and holder), + * and that the store believes affectedRows rather than its own intent. + */ +const query = DB.query as jest.Mock; +const NOW = '2026-09-17T12:00:00.000Z'; + +const row: PrivateRelaySubmissionRow = { + submission_id: '11111111-1111-4111-8111-111111111111', network: 'signet', txid: 'ab'.repeat(32), raw_tx: '02'.repeat(80), + method: 'privatebroadcast_tor', state: 'queued', relay_endpoint_id: null, attempts: 0, lease_until: null, lease_owner: null, + owner_token_hash: 'cd'.repeat(32), created_at: NOW, updated_at: NOW, relayed_at: null, confirmed_block_height: null, last_error: null, +}; + +describe('MysqlPrivateRelayStore', () => { + const enabled = config.DATABASE.ENABLED; + const store = new MysqlPrivateRelayStore(); + beforeEach(() => { config.DATABASE.ENABLED = true; query.mockReset(); }); + afterAll(() => { config.DATABASE.ENABLED = enabled; }); + + it('inserts a new row and returns the existing one on the (network, txid) unique key', async () => { + query.mockResolvedValueOnce([{ affectedRows: 1 }]); + expect(await store.insertOrExisting(row)).toEqual({ row, created: true }); + expect(query.mock.calls[0][0]).toContain(`INSERT INTO ${PRIVATE_RELAY_TABLE}`); + + query.mockRejectedValueOnce(Object.assign(new Error('dup'), { code: 'ER_DUP_ENTRY' })); + query.mockResolvedValueOnce([[{ ...row, submission_id: '22222222-2222-4222-8222-222222222222', state: 'submitted', created_at: new Date(NOW), updated_at: new Date(NOW), lease_until: null, relayed_at: new Date(NOW) }]]); + const existing = await store.insertOrExisting(row); + expect(existing.created).toBe(false); + expect(existing.row).toMatchObject({ submission_id: '22222222-2222-4222-8222-222222222222', state: 'submitted', relayed_at: NOW, created_at: NOW }); + expect(query.mock.calls[2][0]).toContain('WHERE network = ? AND txid = ?'); + + query.mockRejectedValueOnce(new Error('connection lost')); + await expect(store.insertOrExisting(row)).rejects.toThrow('connection lost'); + }); + + it('cancels only a queued row and reports the loss when the worker already claimed it', async () => { + query.mockResolvedValueOnce([{ affectedRows: 1 }]); + expect(await store.cancelIfQueued(row.submission_id, NOW)).toBe(true); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/SET state = 'cancelled'/); + expect(sql).toMatch(/WHERE submission_id = \? AND state = 'queued'/); + expect(params[1]).toBe(row.submission_id); + + query.mockResolvedValueOnce([{ affectedRows: 0 }]); + expect(await store.cancelIfQueued(row.submission_id, NOW)).toBe(false); + }); + + it('claims with a compare-and-set on state, updated_at and lease, and yields to a competing claim', async () => { + const candidate = { ...row, created_at: new Date(NOW), updated_at: new Date(NOW) }; + query.mockResolvedValueOnce([[candidate]]).mockResolvedValueOnce([{ affectedRows: 1 }]); + const claimed = await store.claimNext({ network: 'signet', states: ['queued', 'relaying'], workerId: 'w1', nowIso: NOW, leaseUntilIso: '2026-09-17T12:02:00.000Z', maxAttempts: 5 }); + expect(claimed).toMatchObject({ submission_id: row.submission_id, state: 'relaying', lease_owner: 'w1', attempts: 1 }); + const [selectSql] = query.mock.calls[0]; + expect(selectSql).toMatch(/state IN \(\?, \?\) AND attempts <= \?/); + expect(selectSql).toMatch(/lease_until IS NULL OR lease_until <= \?/); + const [updateSql, updateParams] = query.mock.calls[1]; + expect(updateSql).toMatch(/SET state = 'relaying', lease_owner = \?, lease_until = \?, attempts = attempts \+ 1/); + expect(updateSql).toMatch(/WHERE submission_id = \? AND state = \? AND updated_at = \? AND \(lease_until IS NULL OR lease_until <= \?\)/); + expect(updateParams[4]).toBe('queued'); + + query.mockResolvedValueOnce([[candidate]]).mockResolvedValueOnce([{ affectedRows: 0 }]); + expect(await store.claimNext({ network: 'signet', states: ['queued'], workerId: 'w2', nowIso: NOW, leaseUntilIso: NOW, maxAttempts: 5 })).toBeNull(); + }); + + it('settles only for the holder in the expected state', async () => { + query.mockResolvedValueOnce([{ affectedRows: 0 }]); + expect(await store.settle({ submissionId: row.submission_id, workerId: 'w1', fromState: 'relaying', toState: 'submitted', nowIso: NOW, leaseUntilIso: null, relayEndpointId: 'tor-a', relayedAtIso: NOW, lastError: null })).toBe(false); + const [sql, params] = query.mock.calls[0]; + expect(sql).toMatch(/WHERE submission_id = \? AND state = \? AND lease_owner = \?$/); + expect(params.slice(-3)).toEqual([row.submission_id, 'relaying', 'w1']); + expect(sql).toMatch(/lease_owner = NULL/); + }); + + it('counts the queue by state for the backend network only', async () => { + query.mockResolvedValueOnce([[{ state: 'queued', n: '2' }, { state: 'confirmed', n: 3 }, { state: 'bogus', n: 9 }]]); + expect(await store.queueDepth('signet')).toEqual({ queued: 2, relaying: 0, submitted: 0, confirmed: 3, rejected: 0, cancelled: 0 }); + expect(query.mock.calls[0][1]).toEqual(['signet']); + }); +}); diff --git a/backend/src/api/intelligence/private-submission/private-relay.store.ts b/backend/src/api/intelligence/private-submission/private-relay.store.ts new file mode 100644 index 00000000000..1f252b6600c --- /dev/null +++ b/backend/src/api/intelligence/private-submission/private-relay.store.ts @@ -0,0 +1,214 @@ +import DB from '../../../database'; +import { + PrivateRelayQueueDepth, + PrivateRelayState, + PrivateRelayStore, + PrivateRelaySubmissionRow, +} from './private-relay.types'; + +/** + * The durable submission queue in MySQL. + * + * Rows are keyed by (network, txid) so a second submission of the same + * transaction finds the first. Every state change that competes with another + * actor is a single UPDATE whose WHERE clause names the state (and, for a held + * row, the worker holding it), and the caller reads affectedRows to learn + * whether it won. There is no in-memory variant: without a database the + * feature reports itself unavailable. + */ +export const PRIVATE_RELAY_TABLE = 'intelligence_private_relay_submissions'; + +const COLUMNS = [ + 'submission_id', 'network', 'txid', 'raw_tx', 'method', 'state', 'relay_endpoint_id', 'attempts', + 'lease_until', 'lease_owner', 'owner_token_hash', 'created_at', 'updated_at', 'relayed_at', + 'confirmed_block_height', 'last_error', +].join(', '); + +export class MysqlPrivateRelayStore implements PrivateRelayStore { + /** @asyncUnsafe Callers turn a rejection into an exact HTTP answer. */ + public async insertOrExisting(row: PrivateRelaySubmissionRow): Promise<{ row: PrivateRelaySubmissionRow; created: boolean }> { + try { + await DB.query( + `INSERT INTO ${PRIVATE_RELAY_TABLE} (${COLUMNS}) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + [ + row.submission_id, row.network, row.txid, row.raw_tx, row.method, row.state, row.relay_endpoint_id, row.attempts, + toDate(row.lease_until), row.lease_owner, row.owner_token_hash, toDate(row.created_at), toDate(row.updated_at), + toDate(row.relayed_at), row.confirmed_block_height, row.last_error, + ], + 'silent', + ); + return { row, created: true }; + } catch (error) { + if ((error as { code?: string })?.code !== 'ER_DUP_ENTRY') throw error; + const [rows]: any[] = await DB.query( + `SELECT ${COLUMNS} FROM ${PRIVATE_RELAY_TABLE} WHERE network = ? AND txid = ? LIMIT 1`, + [row.network, row.txid], + ); + if (!rows?.length) throw error; + return { row: fromRow(rows[0]), created: false }; + } + } + + /** @asyncUnsafe Callers turn a rejection into an exact HTTP answer. */ + public async getById(submissionId: string): Promise { + const [rows]: any[] = await DB.query( + `SELECT ${COLUMNS} FROM ${PRIVATE_RELAY_TABLE} WHERE submission_id = ? LIMIT 1`, + [submissionId], + ); + return rows?.length ? fromRow(rows[0]) : null; + } + + /** @asyncUnsafe The worker logs a rejection and tries again on its next tick. */ + public async claimNext(args: { + network: string; + states: PrivateRelayState[]; + workerId: string; + nowIso: string; + leaseUntilIso: string; + maxAttempts: number; + }): Promise { + const placeholders = args.states.map(() => '?').join(', '); + const [candidates]: any[] = await DB.query( + `SELECT ${COLUMNS} FROM ${PRIVATE_RELAY_TABLE} + WHERE network = ? AND state IN (${placeholders}) AND attempts <= ? + AND (lease_until IS NULL OR lease_until <= ?) + ORDER BY created_at ASC LIMIT 5`, + [args.network, ...args.states, args.maxAttempts, toDate(args.nowIso)], + ); + for (const candidate of candidates ?? []) { + const row = fromRow(candidate); + const [result]: any[] = await DB.query( + `UPDATE ${PRIVATE_RELAY_TABLE} + SET state = 'relaying', lease_owner = ?, lease_until = ?, attempts = attempts + 1, updated_at = ? + WHERE submission_id = ? AND state = ? AND updated_at = ? AND (lease_until IS NULL OR lease_until <= ?)`, + [args.workerId, toDate(args.leaseUntilIso), toDate(args.nowIso), row.submission_id, row.state, toDate(row.updated_at), toDate(args.nowIso)], + ); + if (Number(result?.affectedRows ?? 0) === 1) { + return { + ...row, + state: 'relaying', + lease_owner: args.workerId, + lease_until: args.leaseUntilIso, + attempts: row.attempts + 1, + updated_at: args.nowIso, + }; + } + } + return null; + } + + /** @asyncUnsafe The worker logs a rejection and tries again on its next tick. */ + public async settle(args: { + submissionId: string; + workerId: string; + fromState: PrivateRelayState; + toState: PrivateRelayState; + nowIso: string; + leaseUntilIso: string | null; + relayEndpointId?: string | null; + relayedAtIso?: string | null; + confirmedBlockHeight?: number | null; + lastError?: string | null; + }): Promise { + const sets: string[] = ['state = ?', 'lease_until = ?', 'lease_owner = NULL', 'updated_at = ?']; + const params: unknown[] = [args.toState, toDate(args.leaseUntilIso), toDate(args.nowIso)]; + if (args.relayEndpointId !== undefined) { sets.push('relay_endpoint_id = ?'); params.push(args.relayEndpointId); } + if (args.relayedAtIso !== undefined) { sets.push('relayed_at = ?'); params.push(toDate(args.relayedAtIso)); } + if (args.confirmedBlockHeight !== undefined) { sets.push('confirmed_block_height = ?'); params.push(args.confirmedBlockHeight); } + if (args.lastError !== undefined) { sets.push('last_error = ?'); params.push(args.lastError); } + params.push(args.submissionId, args.fromState, args.workerId); + const [result]: any[] = await DB.query( + `UPDATE ${PRIVATE_RELAY_TABLE} SET ${sets.join(', ')} WHERE submission_id = ? AND state = ? AND lease_owner = ?`, + params, + ); + return Number(result?.affectedRows ?? 0) === 1; + } + + /** @asyncUnsafe The worker logs a rejection and tries again on its next tick. */ + public async advanceTracking(args: { + submissionId: string; + expectedUpdatedAtIso: string; + toState: 'submitted' | 'confirmed'; + nowIso: string; + leaseUntilIso: string | null; + confirmedBlockHeight?: number | null; + lastError?: string | null; + }): Promise { + const sets: string[] = ['state = ?', 'lease_until = ?', 'updated_at = ?']; + const params: unknown[] = [args.toState, toDate(args.leaseUntilIso), toDate(args.nowIso)]; + if (args.confirmedBlockHeight !== undefined) { sets.push('confirmed_block_height = ?'); params.push(args.confirmedBlockHeight); } + if (args.lastError !== undefined) { sets.push('last_error = ?'); params.push(args.lastError); } + params.push(args.submissionId, toDate(args.expectedUpdatedAtIso)); + const [result]: any[] = await DB.query( + `UPDATE ${PRIVATE_RELAY_TABLE} SET ${sets.join(', ')} WHERE submission_id = ? AND state = 'submitted' AND updated_at = ?`, + params, + ); + return Number(result?.affectedRows ?? 0) === 1; + } + + /** @asyncUnsafe Callers turn a rejection into an exact HTTP answer. */ + public async cancelIfQueued(submissionId: string, nowIso: string): Promise { + const [result]: any[] = await DB.query( + `UPDATE ${PRIVATE_RELAY_TABLE} SET state = 'cancelled', lease_until = NULL, lease_owner = NULL, updated_at = ? + WHERE submission_id = ? AND state = 'queued'`, + [toDate(nowIso), submissionId], + ); + return Number(result?.affectedRows ?? 0) === 1; + } + + /** @asyncUnsafe The worker logs a rejection and tries again on its next tick. */ + public async dueForTracking(network: string, nowIso: string, limit: number): Promise { + const [rows]: any[] = await DB.query( + `SELECT ${COLUMNS} FROM ${PRIVATE_RELAY_TABLE} + WHERE network = ? AND state = 'submitted' AND lease_until IS NOT NULL AND lease_until <= ? + ORDER BY lease_until ASC LIMIT ?`, + [network, toDate(nowIso), limit], + ); + return (rows ?? []).map(fromRow); + } + + /** @asyncUnsafe Callers turn a rejection into an exact HTTP answer. */ + public async queueDepth(network: string): Promise { + const [rows]: any[] = await DB.query( + `SELECT state, COUNT(*) AS n FROM ${PRIVATE_RELAY_TABLE} WHERE network = ? GROUP BY state`, + [network], + ); + const depth: PrivateRelayQueueDepth = { queued: 0, relaying: 0, submitted: 0, confirmed: 0, rejected: 0, cancelled: 0 }; + for (const row of rows ?? []) { + if (row.state in depth) depth[row.state as keyof PrivateRelayQueueDepth] = Number(row.n ?? 0); + } + return depth; + } +} + +function toDate(iso: string | null | undefined): Date | null { + return iso ? new Date(iso) : null; +} + +function toIso(value: unknown): string | null { + if (value === null || value === undefined) return null; + if (value instanceof Date) return value.toISOString(); + const date = new Date(String(value)); + return Number.isNaN(date.getTime()) ? null : date.toISOString(); +} + +function fromRow(row: any): PrivateRelaySubmissionRow { + return { + submission_id: String(row.submission_id), + network: String(row.network), + txid: String(row.txid), + raw_tx: String(row.raw_tx), + method: String(row.method), + state: String(row.state) as PrivateRelayState, + relay_endpoint_id: row.relay_endpoint_id === null || row.relay_endpoint_id === undefined ? null : String(row.relay_endpoint_id), + attempts: Number(row.attempts ?? 0), + lease_until: toIso(row.lease_until), + lease_owner: row.lease_owner === null || row.lease_owner === undefined ? null : String(row.lease_owner), + owner_token_hash: String(row.owner_token_hash), + created_at: toIso(row.created_at) ?? new Date(0).toISOString(), + updated_at: toIso(row.updated_at) ?? new Date(0).toISOString(), + relayed_at: toIso(row.relayed_at), + confirmed_block_height: row.confirmed_block_height === null || row.confirmed_block_height === undefined ? null : Number(row.confirmed_block_height), + last_error: row.last_error === null || row.last_error === undefined ? null : String(row.last_error), + }; +} diff --git a/backend/src/api/intelligence/private-submission/private-relay.submissions.ts b/backend/src/api/intelligence/private-submission/private-relay.submissions.ts new file mode 100644 index 00000000000..23bfc4ed421 --- /dev/null +++ b/backend/src/api/intelligence/private-submission/private-relay.submissions.ts @@ -0,0 +1,208 @@ +import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'crypto'; +import { PRIVATE_RELAY_ENDPOINTS_ENV, endpointsForTransport, transportForMethod } from './private-relay.config'; +import { PrivateRelayRuntime, getPrivateRelayRuntime } from './private-relay.runtime'; +import { + PrivateRelayStore, + PrivateRelaySubmissionRow, + PrivateSubmissionAuthorizationError, + PrivateSubmissionNotFoundError, +} from './private-relay.types'; +import { validatePrivateRelayTransaction } from './private-relay.validation'; +import { PrivateBroadcastRecord, PrivateRelayOverview, SubmissionCapabilities } from './private-submission.models'; +import { SubmissionInputError } from './submission-input'; + +/** + * The service-facing operations behind the five private submission routes. + * + * Each one reads the runtime (store, endpoints, worker) and either answers + * from durable state or throws a typed error the routes map to a status: + * `durable-store-unavailable` when there is no database, `unavailable-relay` + * when no owned endpoint serves the requested transport, `unauthorized` when + * the owner token does not match, `not-found` for an unknown id. + */ +export class PrivateRelayUnavailableError extends Error { + constructor(public readonly code: 'durable-store-unavailable' | 'unavailable-relay', message: string) { + super(message); + } +} + +const QUEUE_LIMIT = 1000; + +function requireStore(runtime: PrivateRelayRuntime): PrivateRelayStore { + if (!runtime.store) { + throw new PrivateRelayUnavailableError( + 'durable-store-unavailable', + 'Private submission needs the durable submission store, and config.DATABASE.ENABLED is false on this deployment.', + ); + } + return runtime.store; +} + +function relayReason(runtime: PrivateRelayRuntime): string | null { + if (runtime.endpoints.unconfigured) return PRIVATE_RELAY_ENDPOINTS_ENV + ' is not set'; + if (!runtime.endpoints.endpoints.length) return 'every declared endpoint was rejected'; + return null; +} + +/** @asyncUnsafe The route turns a rejection into an exact HTTP answer. */ +export async function privateRelayOverview(runtime: PrivateRelayRuntime = getPrivateRelayRuntime()): Promise { + const store = requireStore(runtime); + const worker = runtime.worker?.status() ?? null; + const depth = await store.queueDepth(runtime.network); + return { + network: runtime.network, + relay: { + configured: runtime.endpoints.endpoints.length > 0, + reason: relayReason(runtime), + endpoints: runtime.endpoints.endpoints.map(endpoint => ({ + id: endpoint.id, + transport: endpoint.transport, + submit_host: new URL(endpoint.submitUrl).hostname, + proxy_reachable: worker?.proxy_reachable[endpoint.id] ?? null, + })), + rejected_endpoints: runtime.endpoints.issues.map(issue => ({ id: issue.id, reason: issue.reason })), + }, + queue: depth, + worker: { + running: worker?.running ?? false, + last_tick_at: worker?.last_tick_at ?? null, + last_error: worker?.last_error ?? null, + last_relay: worker?.last_relay ?? null, + }, + }; +} + +/** @asyncUnsafe The route turns a rejection into an exact HTTP answer. */ +export async function privateRelayCapabilities(runtime: PrivateRelayRuntime = getPrivateRelayRuntime()): Promise { + const store = requireStore(runtime); + const worker = runtime.worker?.status() ?? null; + const depth = await store.queueDepth(runtime.network); + const tor = endpointsForTransport(runtime.endpoints.endpoints, 'tor'); + const i2p = endpointsForTransport(runtime.endpoints.endpoints, 'i2p'); + const anyReachable = (ids: string[]): boolean => ids.some(id => worker?.proxy_reachable[id] === true); + return { + public_p2p_enabled: false, + privatebroadcast_tor_enabled: tor.length > 0, + privatebroadcast_i2p_enabled: i2p.length > 0, + core_version: runtime.coreVersion(), + tor_active: (worker?.running ?? false) && anyReachable(tor.map(endpoint => endpoint.id)), + i2p_active: (worker?.running ?? false) && anyReachable(i2p.map(endpoint => endpoint.id)), + queue_limit: QUEUE_LIMIT, + current_queue_count: depth.queued + depth.relaying, + }; +} + +/** @asyncUnsafe The route turns a rejection into an exact HTTP answer. */ +export async function submitPrivateRelay( + submission: { raw_tx: string; method: string }, + runtime: PrivateRelayRuntime = getPrivateRelayRuntime(), +): Promise { + const store = requireStore(runtime); + const transport = transportForMethod(submission.method); + if (!transport) { + throw new SubmissionInputError('This route serves privatebroadcast_tor, privatebroadcast_i2p and configured_private_relay only.'); + } + const reason = relayReason(runtime); + if (reason) { + throw new PrivateRelayUnavailableError('unavailable-relay', `Private broadcast is unavailable: ${reason}.`); + } + if (!endpointsForTransport(runtime.endpoints.endpoints, transport).length) { + throw new PrivateRelayUnavailableError('unavailable-relay', `Private broadcast over ${transport} is unavailable: no owned ${transport} endpoint is configured.`); + } + const validated = validatePrivateRelayTransaction(submission.raw_tx); + + const depth = await store.queueDepth(runtime.network); + if (depth.queued + depth.relaying >= QUEUE_LIMIT) { + throw new PrivateRelayUnavailableError('unavailable-relay', `The private relay queue is full (${QUEUE_LIMIT}).`); + } + + const ownerToken = randomBytes(32).toString('hex'); + const now = new Date().toISOString(); + const { row, created } = await store.insertOrExisting({ + submission_id: randomUUID(), + network: runtime.network, + txid: validated.txid, + raw_tx: validated.raw_tx, + method: submission.method, + state: 'queued', + relay_endpoint_id: null, + attempts: 0, + lease_until: null, + lease_owner: null, + owner_token_hash: hashToken(ownerToken), + created_at: now, + updated_at: now, + relayed_at: null, + confirmed_block_height: null, + last_error: null, + }); + const record = toRecord(row); + if (created) record.owner_token = ownerToken; + else record.duplicate = true; + return record; +} + +/** @asyncUnsafe The route turns a rejection into an exact HTTP answer. */ +export async function readPrivateRelaySubmission( + submissionId: string, + ownerToken: string | undefined, + runtime: PrivateRelayRuntime = getPrivateRelayRuntime(), +): Promise { + const store = requireStore(runtime); + const row = await authorizedRow(store, submissionId, ownerToken); + return toRecord(row); +} + +/** @asyncUnsafe The route turns a rejection into an exact HTTP answer. */ +export async function abortPrivateRelaySubmission( + submissionId: string, + ownerToken: string | undefined, + runtime: PrivateRelayRuntime = getPrivateRelayRuntime(), +): Promise<{ success: boolean; status: string }> { + const store = requireStore(runtime); + const row = await authorizedRow(store, submissionId, ownerToken); + if (row.state === 'cancelled') return { success: true, status: 'cancelled' }; + if (row.state === 'queued') { + const cancelled = await store.cancelIfQueued(row.submission_id, new Date().toISOString()); + if (cancelled) return { success: true, status: 'cancelled' }; + } + // The relay has started or finished: nothing here can take that back, and + // the answer must not suggest otherwise. + return { success: false, status: 'abort-too-late' }; +} + +/** @asyncUnsafe Callers turn a rejection into an exact HTTP answer. */ +async function authorizedRow(store: PrivateRelayStore, submissionId: string, ownerToken: string | undefined): Promise { + if (!/^[0-9a-f-]{36}$/i.test(submissionId)) throw new PrivateSubmissionNotFoundError(); + const row = await store.getById(submissionId); + if (!row) throw new PrivateSubmissionNotFoundError(); + if (typeof ownerToken !== 'string' || !/^[0-9a-f]{64}$/i.test(ownerToken)) throw new PrivateSubmissionAuthorizationError(); + const presented = Buffer.from(hashToken(ownerToken), 'hex'); + const stored = Buffer.from(row.owner_token_hash, 'hex'); + if (presented.length !== stored.length || !timingSafeEqual(presented, stored)) throw new PrivateSubmissionAuthorizationError(); + return row; +} + +export function hashToken(token: string): string { + return createHash('sha256').update(token.toLowerCase(), 'utf8').digest('hex'); +} + +/** Readback never carries the raw transaction, the token hash or the lease. */ +function toRecord(row: PrivateRelaySubmissionRow): PrivateBroadcastRecord { + const record: PrivateBroadcastRecord = { + submission_token: row.submission_id, + txid: row.txid, + method: row.method as PrivateBroadcastRecord['method'], + network: row.network, + queued_at_utc: row.created_at, + updated_at_utc: row.updated_at, + status: row.state, + retry_count: row.attempts, + can_abort: row.state === 'queued', + }; + if (row.last_error) record.last_error = row.last_error; + if (row.relay_endpoint_id) record.relay_endpoint_id = row.relay_endpoint_id; + if (row.relayed_at) record.relayed_at_utc = row.relayed_at; + if (row.confirmed_block_height !== null) record.confirmed_block_height = row.confirmed_block_height; + return record; +} diff --git a/backend/src/api/intelligence/private-submission/private-relay.test.ts b/backend/src/api/intelligence/private-submission/private-relay.test.ts new file mode 100644 index 00000000000..8c65ea40ff9 --- /dev/null +++ b/backend/src/api/intelligence/private-submission/private-relay.test.ts @@ -0,0 +1,555 @@ +import express from 'express'; +import { AddressInfo } from 'net'; +import { Transaction } from 'bitcoinjs-lib'; +import routes from './private-submission.routes'; +import { PRIVATE_SUBMISSION_OWNER_TOKEN_HEADER } from './private-submission.models'; +import { parsePrivateRelayEndpoints } from './private-relay.config'; +import { PrivateRelayRuntime, setPrivateRelayRuntime } from './private-relay.runtime'; +import { + PrivateRelayUnavailableError, + abortPrivateRelaySubmission, + hashToken, + privateRelayCapabilities, + privateRelayOverview, + readPrivateRelaySubmission, + submitPrivateRelay, +} from './private-relay.submissions'; +import { + PrivateRelayEndpoint, + PrivateRelayOutcome, + PrivateRelayQueueDepth, + PrivateRelayState, + PrivateRelayStore, + PrivateRelaySubmissionRow, + PrivateRelayTransportAdapter, + PrivateSubmissionAuthorizationError, + PrivateSubmissionNotFoundError, +} from './private-relay.types'; +import { validatePrivateRelayTransaction } from './private-relay.validation'; +import { PrivateRelayWorker } from './private-relay.worker'; +import privateSubmissionService from './private-submission.service'; +import { SubmissionInputError } from './submission-input'; + +/** + * The relay path with a fake durable store and a fake transport. The store + * keeps the compare-and-set semantics of the MySQL one (claim by state and + * lease, settle by holder, cancel by state) so the races the worker and the + * abort route can run into are exercised here rather than only in production. + */ +const NETWORK = 'signet'; +const ENDPOINTS_JSON = JSON.stringify([ + { id: 'tor-a', transport: 'tor', proxy: 'socks5h://127.0.0.1:9050', submitUrl: 'http://explorerabcdefghijklmnopqrstuvwxyz234567abcdefghijklmnopqrstu.onion/api/tx', network: NETWORK, timeoutMs: 5000 }, + { id: 'i2p-a', transport: 'i2p', proxy: 'socks5h://127.0.0.1:4447', submitUrl: 'http://explorer.b32.i2p/api/tx', network: NETWORK }, +]); + +function signedLookingTx(nonce = 1): string { + const tx = new Transaction(); + tx.addInput(Buffer.alloc(32, nonce), 0); + tx.addOutput(Buffer.from('0014' + '11'.repeat(20), 'hex'), 1000); + return tx.toHex(); +} + +function coinbaseTx(): string { + const tx = new Transaction(); + tx.addInput(Buffer.alloc(32, 0), 0xffffffff, 0xffffffff, Buffer.from('0102', 'hex')); + tx.addOutput(Buffer.from('0014' + '22'.repeat(20), 'hex'), 5000); + return tx.toHex(); +} + +class FakeStore implements PrivateRelayStore { + public rows = new Map(); + public failNext: Error | null = null; + + private guard(): void { + if (this.failNext) { const error = this.failNext; this.failNext = null; throw error; } + } + public async insertOrExisting(row: PrivateRelaySubmissionRow): Promise<{ row: PrivateRelaySubmissionRow; created: boolean }> { + this.guard(); + for (const existing of this.rows.values()) { + if (existing.network === row.network && existing.txid === row.txid) return { row: { ...existing }, created: false }; + } + this.rows.set(row.submission_id, { ...row }); + return { row: { ...row }, created: true }; + } + public async getById(submissionId: string): Promise { + this.guard(); + const row = this.rows.get(submissionId); + return row ? { ...row } : null; + } + public async claimNext(args: { network: string; states: PrivateRelayState[]; workerId: string; nowIso: string; leaseUntilIso: string; maxAttempts: number }): Promise { + this.guard(); + const candidates = [...this.rows.values()] + .filter(row => row.network === args.network && args.states.includes(row.state) && row.attempts <= args.maxAttempts && (row.lease_until === null || row.lease_until <= args.nowIso)) + .sort((a, b) => a.created_at.localeCompare(b.created_at)); + const row = candidates[0]; + if (!row) return null; + Object.assign(row, { state: 'relaying', lease_owner: args.workerId, lease_until: args.leaseUntilIso, attempts: row.attempts + 1, updated_at: args.nowIso }); + return { ...row }; + } + public async settle(args: { submissionId: string; workerId: string; fromState: PrivateRelayState; toState: PrivateRelayState; nowIso: string; leaseUntilIso: string | null; relayEndpointId?: string | null; relayedAtIso?: string | null; confirmedBlockHeight?: number | null; lastError?: string | null }): Promise { + this.guard(); + const row = this.rows.get(args.submissionId); + if (!row || row.state !== args.fromState || row.lease_owner !== args.workerId) return false; + row.state = args.toState; row.lease_until = args.leaseUntilIso; row.lease_owner = null; row.updated_at = args.nowIso; + if (args.relayEndpointId !== undefined) row.relay_endpoint_id = args.relayEndpointId; + if (args.relayedAtIso !== undefined) row.relayed_at = args.relayedAtIso; + if (args.confirmedBlockHeight !== undefined) row.confirmed_block_height = args.confirmedBlockHeight; + if (args.lastError !== undefined) row.last_error = args.lastError; + return true; + } + public async advanceTracking(args: { submissionId: string; expectedUpdatedAtIso: string; toState: 'submitted' | 'confirmed'; nowIso: string; leaseUntilIso: string | null; confirmedBlockHeight?: number | null; lastError?: string | null }): Promise { + this.guard(); + const row = this.rows.get(args.submissionId); + if (!row || row.state !== 'submitted' || row.updated_at !== args.expectedUpdatedAtIso) return false; + row.state = args.toState; row.lease_until = args.leaseUntilIso; row.updated_at = args.nowIso; + if (args.confirmedBlockHeight !== undefined) row.confirmed_block_height = args.confirmedBlockHeight; + if (args.lastError !== undefined) row.last_error = args.lastError; + return true; + } + public async cancelIfQueued(submissionId: string, nowIso: string): Promise { + this.guard(); + const row = this.rows.get(submissionId); + if (!row || row.state !== 'queued') return false; + row.state = 'cancelled'; row.lease_until = null; row.lease_owner = null; row.updated_at = nowIso; + return true; + } + public async dueForTracking(network: string, nowIso: string, limit: number): Promise { + this.guard(); + return [...this.rows.values()].filter(row => row.network === network && row.state === 'submitted' && row.lease_until !== null && row.lease_until <= nowIso).slice(0, limit).map(row => ({ ...row })); + } + public async queueDepth(network: string): Promise { + this.guard(); + const depth: PrivateRelayQueueDepth = { queued: 0, relaying: 0, submitted: 0, confirmed: 0, rejected: 0, cancelled: 0 }; + for (const row of this.rows.values()) if (row.network === network && row.state in depth) depth[row.state as keyof PrivateRelayQueueDepth]++; + return depth; + } +} + +class FakeTransport implements PrivateRelayTransportAdapter { + public calls: { endpoint: string; rawTx: string; txid: string }[] = []; + public outcomes: PrivateRelayOutcome[] = []; + public reachable = true; + public async submit(endpoint: PrivateRelayEndpoint, rawTx: string, expectedTxid: string): Promise { + this.calls.push({ endpoint: endpoint.id, rawTx, txid: expectedTxid }); + return this.outcomes.shift() ?? { kind: 'submitted', txid: expectedTxid, httpStatus: 200 }; + } + public async probeProxy(): Promise { return this.reachable; } +} + +interface Harness { + runtime: PrivateRelayRuntime; + store: FakeStore; + transport: FakeTransport; + worker: PrivateRelayWorker; + clock: { now: number }; + confirmations: Map; +} + +function harness(options: { endpointsJson?: string; database?: boolean; workerId?: string; store?: FakeStore; maxAttempts?: number } = {}): Harness { + const store = options.store ?? new FakeStore(); + const transport = new FakeTransport(); + const clock = { now: Date.parse('2026-09-17T12:00:00.000Z') }; + const confirmations = new Map(); + const endpoints = parsePrivateRelayEndpoints(options.endpointsJson ?? ENDPOINTS_JSON, NETWORK); + const worker = new PrivateRelayWorker({ + store, transport, endpoints: endpoints.endpoints, network: NETWORK, + confirmation: async (txid) => confirmations.get(txid) ?? null, + now: () => clock.now, workerId: options.workerId ?? 'worker-1', maxAttempts: options.maxAttempts ?? 3, + leaseMs: 60_000, retryBackoffMs: 30_000, trackingIntervalMs: 10_000, trackingWindowMs: 3600_000, probeIntervalMs: 0, + }); + const runtime: PrivateRelayRuntime = { + network: NETWORK, endpoints, + store: options.database === false ? null : store, + worker: options.database === false || !endpoints.endpoints.length ? null : worker, + coreVersion: () => 'v29.0-test', + }; + return { runtime, store, transport, worker, clock, confirmations }; +} + +describe('private relay transaction validation', () => { + it('rejects non-hex, oversized, undersized, coinbase and trailing-byte transactions', () => { + expect(() => validatePrivateRelayTransaction('zz')).toThrow(SubmissionInputError); + expect(() => validatePrivateRelayTransaction('ab'.repeat(400_001))).toThrow(/exceeds/); + expect(() => validatePrivateRelayTransaction('ab'.repeat(10))).toThrow(/smaller/); + expect(() => validatePrivateRelayTransaction(coinbaseTx())).toThrow(/coinbase/); + expect(() => validatePrivateRelayTransaction(signedLookingTx() + '00')).toThrow(/malformed|trailing/); + expect(() => validatePrivateRelayTransaction('ab'.repeat(80))).toThrow(SubmissionInputError); + }); + + it('yields the txid bitcoinjs computes for a well-formed transaction', () => { + const hex = signedLookingTx(); + const validated = validatePrivateRelayTransaction(hex.toUpperCase()); + expect(validated.txid).toBe(Transaction.fromHex(hex).getId()); + expect(validated.raw_tx).toBe(hex); + }); +}); + +describe('private relay endpoint configuration', () => { + it('reports unconfigured when the variable is absent', () => { + expect(parsePrivateRelayEndpoints(undefined, NETWORK)).toEqual({ endpoints: [], issues: [], unconfigured: true }); + expect(parsePrivateRelayEndpoints(' ', NETWORK).unconfigured).toBe(true); + }); + + it('rejects endpoints on another network, clearnet submit hosts, bad proxies and duplicates by id', () => { + const parsed = parsePrivateRelayEndpoints(JSON.stringify([ + { id: 'mainnet-tor', transport: 'tor', proxy: 'socks5h://127.0.0.1:9050', submitUrl: 'http://abc.onion/api/tx', network: 'mainnet' }, + { id: 'clearnet', transport: 'tor', proxy: 'socks5h://127.0.0.1:9050', submitUrl: 'https://example.com/api/tx', network: NETWORK }, // a clearnet submit host (not .onion) must be refused + { id: 'http-proxy', transport: 'tor', proxy: 'http://127.0.0.1:8118', submitUrl: 'http://abc.onion/api/tx', network: NETWORK }, + { id: 'ok', transport: 'i2p', proxy: 'socks5h://127.0.0.1:4447', submitUrl: 'http://abc.b32.i2p/api/tx', network: NETWORK }, + { id: 'ok', transport: 'i2p', proxy: 'socks5h://127.0.0.1:4447', submitUrl: 'http://def.b32.i2p/api/tx', network: NETWORK }, + { id: 'slow', transport: 'tor', proxy: 'socks5h://127.0.0.1:9050', submitUrl: 'http://abc.onion/api/tx', network: NETWORK, timeoutMs: 999_999 }, + ]), NETWORK); + expect(parsed.endpoints.map(endpoint => endpoint.id)).toEqual(['ok']); + expect(parsed.issues).toEqual([ + { id: 'mainnet-tor', reason: 'network-mismatch' }, + { id: 'clearnet', reason: 'submit-host-not-tor' }, + { id: 'http-proxy', reason: 'invalid-proxy' }, + { id: 'ok', reason: 'duplicate-id' }, + { id: 'slow', reason: 'invalid-timeout' }, + ]); + expect(parsePrivateRelayEndpoints('{not json', NETWORK).issues).toEqual([{ id: '*', reason: 'not-json' }]); + }); +}); + +describe('private relay submission', () => { + it('reports the durable store unavailable when the database is disabled', async () => { + const { runtime } = harness({ database: false }); + await expect(submitPrivateRelay({ raw_tx: signedLookingTx(), method: 'privatebroadcast_tor' }, runtime)).rejects.toMatchObject({ code: 'durable-store-unavailable' }); + await expect(privateRelayOverview(runtime)).rejects.toMatchObject({ code: 'durable-store-unavailable' }); + await expect(privateRelayCapabilities(runtime)).rejects.toMatchObject({ code: 'durable-store-unavailable' }); + await expect(readPrivateRelaySubmission('any', 'ab'.repeat(32), runtime)).rejects.toBeInstanceOf(PrivateRelayUnavailableError); + }); + + it('reports the relay unavailable when no endpoint is configured, while the overview still answers with the reason', async () => { + const { runtime, store } = harness({ endpointsJson: '' }); + await expect(submitPrivateRelay({ raw_tx: signedLookingTx(), method: 'privatebroadcast_tor' }, runtime)).rejects.toMatchObject({ code: 'unavailable-relay' }); + expect(store.rows.size).toBe(0); + const overview = await privateRelayOverview(runtime); + expect(overview.relay).toMatchObject({ configured: false, reason: 'UNIVERSE_PRIVATE_RELAY_ENDPOINTS is not set', endpoints: [] }); + expect(overview.worker.running).toBe(false); + const capabilities = await privateRelayCapabilities(runtime); + expect(capabilities).toMatchObject({ privatebroadcast_tor_enabled: false, privatebroadcast_i2p_enabled: false, tor_active: false, i2p_active: false, current_queue_count: 0 }); + }); + + it('refuses a transport that has no endpoint and a method the private relay does not serve', async () => { + const { runtime, store } = harness({ endpointsJson: JSON.stringify([JSON.parse(ENDPOINTS_JSON)[0]]) }); + await expect(submitPrivateRelay({ raw_tx: signedLookingTx(), method: 'privatebroadcast_i2p' }, runtime)).rejects.toMatchObject({ code: 'unavailable-relay' }); + await expect(submitPrivateRelay({ raw_tx: signedLookingTx(), method: 'public_p2p' }, runtime)).rejects.toBeInstanceOf(SubmissionInputError); + expect(store.rows.size).toBe(0); + }); + + it('rejects an invalid transaction before anything is queued', async () => { + const { runtime, store } = harness(); + await expect(submitPrivateRelay({ raw_tx: coinbaseTx(), method: 'privatebroadcast_tor' }, runtime)).rejects.toBeInstanceOf(SubmissionInputError); + expect(store.rows.size).toBe(0); + }); + + it('queues a durable record and returns the owner token exactly once, without the raw transaction', async () => { + const { runtime, store } = harness(); + const hex = signedLookingTx(); + const record = await submitPrivateRelay({ raw_tx: hex, method: 'privatebroadcast_tor' }, runtime); + expect(record.owner_token).toMatch(/^[0-9a-f]{64}$/); + expect(record).toMatchObject({ status: 'queued', can_abort: true, retry_count: 0, network: NETWORK, txid: Transaction.fromHex(hex).getId() }); + expect(JSON.stringify(record)).not.toContain(hex); + const stored = store.rows.get(record.submission_token)!; + expect(stored.raw_tx).toBe(hex); + expect(stored.owner_token_hash).toBe(hashToken(record.owner_token!)); + expect(stored.owner_token_hash).not.toBe(record.owner_token); + }); + + it('answers a duplicate submission with the existing record and no new token', async () => { + const { runtime, store } = harness(); + const hex = signedLookingTx(); + const first = await submitPrivateRelay({ raw_tx: hex, method: 'privatebroadcast_tor' }, runtime); + const second = await submitPrivateRelay({ raw_tx: hex, method: 'privatebroadcast_i2p' }, runtime); + expect(second.submission_token).toBe(first.submission_token); + expect(second.duplicate).toBe(true); + expect(second.owner_token).toBeUndefined(); + expect(second.method).toBe('privatebroadcast_tor'); + expect(store.rows.size).toBe(1); + }); + + it('requires the owner token for readback and abort, and never discloses the record to a wrong or missing token', async () => { + const { runtime } = harness(); + const record = await submitPrivateRelay({ raw_tx: signedLookingTx(), method: 'privatebroadcast_tor' }, runtime); + await expect(readPrivateRelaySubmission(record.submission_token, undefined, runtime)).rejects.toBeInstanceOf(PrivateSubmissionAuthorizationError); + await expect(readPrivateRelaySubmission(record.submission_token, 'ff'.repeat(32), runtime)).rejects.toBeInstanceOf(PrivateSubmissionAuthorizationError); + await expect(readPrivateRelaySubmission(record.submission_token, record.submission_token, runtime)).rejects.toBeInstanceOf(PrivateSubmissionAuthorizationError); + await expect(abortPrivateRelaySubmission(record.submission_token, 'ff'.repeat(32), runtime)).rejects.toBeInstanceOf(PrivateSubmissionAuthorizationError); + await expect(readPrivateRelaySubmission('00000000-0000-4000-8000-000000000000', record.owner_token, runtime)).rejects.toBeInstanceOf(PrivateSubmissionNotFoundError); + const readback = await readPrivateRelaySubmission(record.submission_token, record.owner_token, runtime); + expect(readback.status).toBe('queued'); + expect(readback.owner_token).toBeUndefined(); + expect(readback).not.toHaveProperty('raw_tx'); + expect(readback).not.toHaveProperty('owner_token_hash'); + const stillQueued = await readPrivateRelaySubmission(record.submission_token, record.owner_token, runtime); + expect(stillQueued.status).toBe('queued'); + }); +}); + +describe('private relay worker', () => { + it('claims the queued row, relays it through the endpoint for its transport, and tracks it to confirmation', async () => { + const { runtime, store, transport, worker, clock, confirmations } = harness(); + const hex = signedLookingTx(); + const record = await submitPrivateRelay({ raw_tx: hex, method: 'privatebroadcast_i2p' }, runtime); + + await worker.tick(); + expect(transport.calls).toEqual([{ endpoint: 'i2p-a', rawTx: hex, txid: record.txid }]); + let row = store.rows.get(record.submission_token)!; + expect(row).toMatchObject({ state: 'submitted', attempts: 1, relay_endpoint_id: 'i2p-a', lease_owner: null }); + expect(row.relayed_at).not.toBeNull(); + expect(worker.status().last_relay).toMatchObject({ endpoint_id: 'i2p-a', transport: 'i2p', outcome: 'submitted' }); + + // Not due yet: no confirmation poll, no state change. + await worker.tick(); + expect(transport.calls).toHaveLength(1); + expect(store.rows.get(record.submission_token)!.state).toBe('submitted'); + + clock.now += 11_000; + await worker.tick(); + expect(store.rows.get(record.submission_token)!.state).toBe('submitted'); + + confirmations.set(record.txid, { confirmed: true, block_height: 260_000 }); + clock.now += 11_000; + await worker.tick(); + row = store.rows.get(record.submission_token)!; + expect(row).toMatchObject({ state: 'confirmed', confirmed_block_height: 260_000, lease_until: null }); + + const readback = await readPrivateRelaySubmission(record.submission_token, record.owner_token, runtime); + expect(readback).toMatchObject({ status: 'confirmed', confirmed_block_height: 260_000, relay_endpoint_id: 'i2p-a', can_abort: false }); + expect(transport.calls).toHaveLength(1); + }); + + it('returns an unreachable relay to the queue with a backoff, and rejects after the bounded attempts', async () => { + const { runtime, store, transport, worker, clock } = harness({ maxAttempts: 2 }); + const record = await submitPrivateRelay({ raw_tx: signedLookingTx(), method: 'privatebroadcast_tor' }, runtime); + transport.outcomes.push({ kind: 'unreachable', reason: 'econnrefused' }, { kind: 'unreachable', reason: 'etimedout' }); + + await worker.tick(); + let row = store.rows.get(record.submission_token)!; + expect(row).toMatchObject({ state: 'queued', attempts: 1, last_error: 'econnrefused' }); + expect(row.lease_until! > new Date(clock.now).toISOString()).toBe(true); + + await worker.tick(); + expect(transport.calls).toHaveLength(1); + + clock.now += 31_000; + await worker.tick(); + row = store.rows.get(record.submission_token)!; + expect(row).toMatchObject({ state: 'rejected', attempts: 2 }); + expect(row.last_error).toContain('relay-attempts-exhausted'); + expect(transport.calls).toHaveLength(2); + }); + + it('records a node rejection without retrying', async () => { + const { runtime, store, transport, worker } = harness(); + const record = await submitPrivateRelay({ raw_tx: signedLookingTx(), method: 'privatebroadcast_tor' }, runtime); + transport.outcomes.push({ kind: 'rejected', reason: 'sendrawtransaction RPC error: bad-txns-inputs-missingorspent', httpStatus: 400 }); + await worker.tick(); + expect(store.rows.get(record.submission_token)).toMatchObject({ state: 'rejected', attempts: 1, last_error: expect.stringContaining('missingorspent') }); + const readback = await readPrivateRelaySubmission(record.submission_token, record.owner_token, runtime); + expect(readback).toMatchObject({ status: 'rejected', can_abort: false }); + }); + + it('reclaims a row whose lease a crashed worker left behind and relays it once more', async () => { + const store = new FakeStore(); + const crashed = harness({ store, workerId: 'worker-crashed' }); + const record = await submitPrivateRelay({ raw_tx: signedLookingTx(), method: 'privatebroadcast_tor' }, crashed.runtime); + // The crashed worker claimed the row and died before the transport answered. + const claimed = await store.claimNext({ network: NETWORK, states: ['queued'], workerId: 'worker-crashed', nowIso: new Date(crashed.clock.now).toISOString(), leaseUntilIso: new Date(crashed.clock.now + 60_000).toISOString(), maxAttempts: 3 }); + expect(claimed?.state).toBe('relaying'); + + const restarted = harness({ store, workerId: 'worker-restarted' }); + await restarted.worker.tick(); + expect(restarted.transport.calls).toHaveLength(0); + expect(store.rows.get(record.submission_token)!.state).toBe('relaying'); + + restarted.clock.now += 61_000; + await restarted.worker.tick(); + expect(restarted.transport.calls).toHaveLength(1); + expect(store.rows.get(record.submission_token)).toMatchObject({ state: 'submitted', attempts: 2, lease_owner: null }); + }); + + it('keeps the transport away from a row that failed to settle under a stolen lease', async () => { + const { runtime, store, worker } = harness(); + const record = await submitPrivateRelay({ raw_tx: signedLookingTx(), method: 'privatebroadcast_tor' }, runtime); + store.failNext = new Error('connection lost'); + await expect(worker.tick()).rejects.toThrow('connection lost'); + expect(worker.status().last_error).toBe('connection lost'); + expect(store.rows.get(record.submission_token)!.state).toBe('queued'); + await worker.tick(); + expect(worker.status().last_error).toBeNull(); + expect(store.rows.get(record.submission_token)!.state).toBe('submitted'); + }); + + it('closes the confirmation tracking window without inventing a confirmation', async () => { + const { runtime, store, worker, clock } = harness(); + const record = await submitPrivateRelay({ raw_tx: signedLookingTx(), method: 'privatebroadcast_tor' }, runtime); + await worker.tick(); + clock.now += 3600_000 + 11_000; + await worker.tick(); + expect(store.rows.get(record.submission_token)).toMatchObject({ state: 'submitted', lease_until: null, last_error: 'confirmation-tracking-window-closed' }); + }); +}); + +describe('private relay abort', () => { + it('cancels a queued submission before the relay starts, so the transport never sees it', async () => { + const { runtime, store, transport, worker } = harness(); + const record = await submitPrivateRelay({ raw_tx: signedLookingTx(), method: 'privatebroadcast_tor' }, runtime); + expect(await abortPrivateRelaySubmission(record.submission_token, record.owner_token, runtime)).toEqual({ success: true, status: 'cancelled' }); + await worker.tick(); + expect(transport.calls).toHaveLength(0); + expect(store.rows.get(record.submission_token)!.state).toBe('cancelled'); + expect(await readPrivateRelaySubmission(record.submission_token, record.owner_token, runtime)).toMatchObject({ status: 'cancelled', can_abort: false }); + expect(await abortPrivateRelaySubmission(record.submission_token, record.owner_token, runtime)).toEqual({ success: true, status: 'cancelled' }); + }); + + it('answers abort-too-late once relay has started or completed, and does not claim a reversal', async () => { + const { runtime, store, worker, clock } = harness(); + const record = await submitPrivateRelay({ raw_tx: signedLookingTx(), method: 'privatebroadcast_tor' }, runtime); + await store.claimNext({ network: NETWORK, states: ['queued'], workerId: 'worker-1', nowIso: new Date(clock.now).toISOString(), leaseUntilIso: new Date(clock.now + 60_000).toISOString(), maxAttempts: 3 }); + expect(await abortPrivateRelaySubmission(record.submission_token, record.owner_token, runtime)).toEqual({ success: false, status: 'abort-too-late' }); + expect(store.rows.get(record.submission_token)!.state).toBe('relaying'); + + const later = await submitPrivateRelay({ raw_tx: signedLookingTx(2), method: 'privatebroadcast_tor' }, runtime); + await worker.tick(); + await worker.tick(); + expect(store.rows.get(later.submission_token)!.state).toBe('submitted'); + expect(await abortPrivateRelaySubmission(later.submission_token, later.owner_token, runtime)).toEqual({ success: false, status: 'abort-too-late' }); + expect(store.rows.get(later.submission_token)!.state).toBe('submitted'); + }); + + it('loses the race to a claim that landed between the read and the cancel', async () => { + const { runtime, store, clock } = harness(); + const record = await submitPrivateRelay({ raw_tx: signedLookingTx(), method: 'privatebroadcast_tor' }, runtime); + const getById = store.getById.bind(store); + store.getById = async (id) => { + const row = await getById(id); + await store.claimNext({ network: NETWORK, states: ['queued'], workerId: 'worker-1', nowIso: new Date(clock.now).toISOString(), leaseUntilIso: new Date(clock.now + 60_000).toISOString(), maxAttempts: 3 }); + return row; + }; + expect(await abortPrivateRelaySubmission(record.submission_token, record.owner_token, runtime)).toEqual({ success: false, status: 'abort-too-late' }); + expect(store.rows.get(record.submission_token)!.state).toBe('relaying'); + }); +}); + +describe('private relay overview and capabilities', () => { + it('reports configured endpoints by transport, queue depth by state, worker liveness and the last relay outcome', async () => { + const { runtime, worker, transport } = harness(); + await submitPrivateRelay({ raw_tx: signedLookingTx(1), method: 'privatebroadcast_tor' }, runtime); + await submitPrivateRelay({ raw_tx: signedLookingTx(2), method: 'privatebroadcast_tor' }, runtime); + await worker.tick(); + worker.start(60_000); + try { + const overview = await privateRelayOverview(runtime); + expect(overview.relay.configured).toBe(true); + expect(overview.relay.endpoints).toEqual([ + { id: 'tor-a', transport: 'tor', submit_host: 'explorerabcdefghijklmnopqrstuvwxyz234567abcdefghijklmnopqrstu.onion', proxy_reachable: true }, + { id: 'i2p-a', transport: 'i2p', submit_host: 'explorer.b32.i2p', proxy_reachable: true }, + ]); + expect(overview.queue).toEqual({ queued: 1, relaying: 0, submitted: 1, confirmed: 0, rejected: 0, cancelled: 0 }); + expect(overview.worker).toMatchObject({ running: true, last_error: null, last_relay: { endpoint_id: 'tor-a', outcome: 'submitted' } }); + expect(JSON.stringify(overview)).not.toMatch(/[0-9a-f]{120,}/); + + const capabilities = await privateRelayCapabilities(runtime); + expect(capabilities).toEqual({ + public_p2p_enabled: false, privatebroadcast_tor_enabled: true, privatebroadcast_i2p_enabled: true, core_version: 'v29.0-test', + tor_active: true, i2p_active: true, queue_limit: 1000, current_queue_count: 1, + }); + + transport.reachable = false; + await worker.probeProxies(true); + expect((await privateRelayCapabilities(runtime)).tor_active).toBe(false); + expect((await privateRelayOverview(runtime)).relay.endpoints[0].proxy_reachable).toBe(false); + } finally { + worker.stop(); + } + }); +}); + +describe('PrivateSubmissionService private relay methods', () => { + afterEach(() => setPrivateRelayRuntime(null)); + + it('delegates to the installed runtime and returns its typed errors', async () => { + const { runtime } = harness(); + setPrivateRelayRuntime(runtime); + const record = await privateSubmissionService.submitPrivate({ raw_tx: signedLookingTx(), method: 'privatebroadcast_tor' }); + expect(record.owner_token).toBeDefined(); + expect((await privateSubmissionService.getPrivateSubmission(record.submission_token, record.owner_token)).status).toBe('queued'); + await expect(privateSubmissionService.getPrivateSubmission(record.submission_token, 'ee'.repeat(32))).rejects.toBeInstanceOf(PrivateSubmissionAuthorizationError); + expect(await privateSubmissionService.abortPrivateSubmission(record.submission_token, record.owner_token)).toEqual({ success: true, status: 'cancelled' }); + expect((await privateSubmissionService.getOverview()).queue.cancelled).toBe(1); + expect((await privateSubmissionService.getCapabilities()).privatebroadcast_tor_enabled).toBe(true); + }); + + it('reports the durable store unavailable through the service when the database is disabled', async () => { + setPrivateRelayRuntime(harness({ database: false }).runtime); + await expect(privateSubmissionService.submitPrivate({ raw_tx: signedLookingTx(), method: 'privatebroadcast_tor' })).rejects.toMatchObject({ code: 'durable-store-unavailable' }); + }); +}); + +describe('private relay HTTP contract', () => { + afterEach(() => setPrivateRelayRuntime(null)); + + it('returns the token once on 201, authenticates readback by header and abort by header or body, and answers 403/404 otherwise', async () => { + const { runtime } = harness(); + setPrivateRelayRuntime(runtime); + const app = express(); + app.use(express.json()); + routes.initRoutes(app); + const server = app.listen(0, '127.0.0.1'); + await new Promise((r) => server.once('listening', r)); + const base = 'http://127.0.0.1:' + (server.address() as AddressInfo).port + '/api/v1/intelligence/submission'; + const call = (method: string, path: string, body?: unknown, headers: Record = {}): Promise => + fetch(base + path, { method, headers: { 'Content-Type': 'application/json', ...headers }, ...(body ? { body: JSON.stringify(body) } : {}) }); + try { + const hex = signedLookingTx(7); + const created = await call('POST', '/private', { raw_tx: hex, method: 'privatebroadcast_tor' }); + expect(created.status).toBe(201); + const record: any = await created.json(); + expect(record.owner_token).toMatch(/^[0-9a-f]{64}$/); + expect(JSON.stringify(record)).not.toContain(hex); + + const again = await call('POST', '/private', { raw_tx: hex, method: 'privatebroadcast_tor' }); + expect(again.status).toBe(200); + expect(await again.json()).toMatchObject({ submission_token: record.submission_token, duplicate: true }); + + expect((await call('GET', '/private/' + record.submission_token)).status).toBe(403); + expect((await call('GET', '/private/' + record.submission_token, undefined, { [PRIVATE_SUBMISSION_OWNER_TOKEN_HEADER]: 'ff'.repeat(32) })).status).toBe(403); + expect((await call('GET', '/private/00000000-0000-4000-8000-000000000000', undefined, { [PRIVATE_SUBMISSION_OWNER_TOKEN_HEADER]: record.owner_token })).status).toBe(404); + const read = await call('GET', '/private/' + record.submission_token, undefined, { [PRIVATE_SUBMISSION_OWNER_TOKEN_HEADER]: record.owner_token }); + expect(read.status).toBe(200); + expect(await read.json()).toMatchObject({ status: 'queued', can_abort: true }); + + expect((await call('POST', '/private/' + record.submission_token + '/abort', { owner_token: 'ff'.repeat(32) })).status).toBe(403); + const aborted = await call('POST', '/private/' + record.submission_token + '/abort', { owner_token: record.owner_token }); + expect(aborted.status).toBe(200); + expect(await aborted.json()).toEqual({ success: true, status: 'cancelled' }); + + const overview = await call('GET', '/overview'); + expect(overview.status).toBe(200); + expect(((await overview.json()) as any).queue.cancelled).toBe(1); + } finally { + await new Promise((r) => server.close(() => r())); + } + }); + + it('answers 503 durable-store-unavailable on every private relay route without a database', async () => { + setPrivateRelayRuntime(harness({ database: false }).runtime); + const app = express(); + app.use(express.json()); + routes.initRoutes(app); + const server = app.listen(0, '127.0.0.1'); + await new Promise((r) => server.once('listening', r)); + const base = 'http://127.0.0.1:' + (server.address() as AddressInfo).port + '/api/v1/intelligence/submission'; + try { + for (const [method, path, body] of [['GET', '/overview'], ['GET', '/capabilities'], ['POST', '/private', { raw_tx: signedLookingTx(), method: 'privatebroadcast_tor' }], ['GET', '/private/x'], ['POST', '/private/x/abort', {}]] as [string, string, unknown?][]) { + const res = await fetch(base + path, { method, headers: { 'Content-Type': 'application/json' }, ...(body ? { body: JSON.stringify(body) } : {}) }); + expect(res.status).toBe(503); + expect(((await res.json()) as any).stage).toBe('durable-store-unavailable'); + } + } finally { + await new Promise((r) => server.close(() => r())); + } + }); +}); diff --git a/backend/src/api/intelligence/private-submission/private-relay.transport.test.ts b/backend/src/api/intelligence/private-submission/private-relay.transport.test.ts new file mode 100644 index 00000000000..0b8dc721bbe --- /dev/null +++ b/backend/src/api/intelligence/private-submission/private-relay.transport.test.ts @@ -0,0 +1,148 @@ +import * as http from 'http'; +import * as net from 'net'; +import { AddressInfo } from 'net'; +import { SocksPrivateRelayTransport, classifyResponse, describeTransportError } from './private-relay.transport'; +import { PrivateRelayEndpoint } from './private-relay.types'; + +/** + * The real SOCKS transport against a fake SOCKS5 server on localhost. The + * fake accepts the no-auth handshake, records the hostname the client asked + * for (which is how we know the onion name went to the proxy rather than to + * DNS), and pipes the connection to a local HTTP server standing in for the + * owned endpoint. + */ +const TXID = 'ab'.repeat(32); +const RAW_TX = '0200000001' + '00'.repeat(60); + +interface FakeSocks { + port: number; + requestedHosts: string[]; + close(): Promise; +} + +function startFakeSocks(targetPort: number): Promise { + const requestedHosts: string[] = []; + const server = net.createServer((client) => { + let stage: 'greeting' | 'request' | 'piped' = 'greeting'; + client.on('data', (chunk) => { + if (stage === 'greeting') { + if (chunk[0] !== 0x05) { client.destroy(); return; } + client.write(Buffer.from([0x05, 0x00])); + stage = 'request'; + return; + } + if (stage === 'request') { + if (chunk[0] !== 0x05 || chunk[1] !== 0x01 || chunk[3] !== 0x03) { client.destroy(); return; } + const length = chunk[4]; + requestedHosts.push(chunk.subarray(5, 5 + length).toString('utf8')); + const upstream = net.connect(targetPort, '127.0.0.1', () => { + client.write(Buffer.from([0x05, 0x00, 0x00, 0x01, 0, 0, 0, 0, 0, 0])); + client.pipe(upstream).pipe(client); + }); + upstream.on('error', () => client.destroy()); + stage = 'piped'; + } + }); + client.on('error', () => undefined); + }); + return new Promise((resolve) => { + server.listen(0, '127.0.0.1', () => { + resolve({ + port: (server.address() as AddressInfo).port, + requestedHosts, + close: () => new Promise((done) => server.close(() => done())), + }); + }); + }); +} + +function startTarget(handler: (body: string, res: http.ServerResponse, req: http.IncomingMessage) => void): Promise<{ port: number; close(): Promise }> { + const server = http.createServer((req, res) => { + let body = ''; + req.on('data', (chunk) => { body += chunk; }); + req.on('end', () => handler(body, res, req)); + }); + return new Promise((resolve) => { + server.listen(0, '127.0.0.1', () => resolve({ + port: (server.address() as AddressInfo).port, + close: () => new Promise((done) => server.close(() => done())), + })); + }); +} + +function endpoint(proxyPort: number, overrides: Partial = {}): PrivateRelayEndpoint { + return { + id: 'tor-test', transport: 'tor', proxy: `socks5h://127.0.0.1:${proxyPort}`, + submitUrl: 'http://ownedexplorertestonionname.onion/api/tx', network: 'signet', timeoutMs: 5_000, ...overrides, + }; +} + +describe('SocksPrivateRelayTransport', () => { + const transport = new SocksPrivateRelayTransport(); + + it('posts the raw hex through the SOCKS proxy to the onion name and reads the txid back', async () => { + const seen: { method?: string; path?: string; contentType?: string; body?: string } = {}; + const target = await startTarget((body, res, req) => { + seen.method = req.method; seen.path = req.url; seen.contentType = req.headers['content-type']; seen.body = body; + res.writeHead(200, { 'Content-Type': 'text/plain' }); + res.end(TXID); + }); + const socks = await startFakeSocks(target.port); + try { + const outcome = await transport.submit(endpoint(socks.port), RAW_TX, TXID); + expect(outcome).toEqual({ kind: 'submitted', txid: TXID, httpStatus: 200 }); + expect(socks.requestedHosts).toEqual(['ownedexplorertestonionname.onion']); + expect(seen).toEqual({ method: 'POST', path: '/api/tx', contentType: 'text/plain', body: RAW_TX }); + expect(await transport.probeProxy(endpoint(socks.port))).toBe(true); + } finally { + await socks.close(); + await target.close(); + } + }); + + it('maps the endpoint rejection text to a rejected outcome and does not retry through the proxy', async () => { + let hits = 0; + const target = await startTarget((_body, res) => { + hits++; + res.writeHead(400, { 'Content-Type': 'text/plain' }); + res.end('sendrawtransaction RPC error: {"code":-25,"message":"bad-txns-inputs-missingorspent"}'); + }); + const socks = await startFakeSocks(target.port); + try { + const outcome = await transport.submit(endpoint(socks.port), RAW_TX, TXID); + expect(outcome).toMatchObject({ kind: 'rejected', httpStatus: 400, reason: expect.stringContaining('missingorspent') }); + expect(hits).toBe(1); + } finally { + await socks.close(); + await target.close(); + } + }); + + it('reports a proxy that is not listening as unreachable, without a raw transaction in the reason', async () => { + const probe = net.createServer(); + await new Promise((resolve) => probe.listen(0, '127.0.0.1', () => resolve())); + const freePort = (probe.address() as AddressInfo).port; + await new Promise((resolve) => probe.close(() => resolve())); + + const outcome = await transport.submit(endpoint(freePort, { timeoutMs: 2_000 }), RAW_TX, TXID); + expect(outcome.kind).toBe('unreachable'); + expect(JSON.stringify(outcome)).not.toContain(RAW_TX); + expect(await transport.probeProxy(endpoint(freePort, { timeoutMs: 2_000 }))).toBe(false); + }); +}); + +describe('private relay response classification', () => { + it('treats a different txid as a rejection, a 5xx as unreachable, and a bare 2xx as submitted', () => { + expect(classifyResponse(200, 'cd'.repeat(32), TXID)).toEqual({ kind: 'rejected', reason: 'endpoint-returned-different-txid', httpStatus: 200 }); + expect(classifyResponse(200, TXID.toUpperCase() + '\n', TXID)).toEqual({ kind: 'submitted', txid: TXID, httpStatus: 200 }); + expect(classifyResponse(202, '', TXID)).toEqual({ kind: 'submitted', txid: TXID, httpStatus: 202 }); + expect(classifyResponse(502, 'bad gateway', TXID)).toEqual({ kind: 'unreachable', reason: 'http-502' }); + expect(classifyResponse(422, 'x'.repeat(1000), TXID)).toMatchObject({ kind: 'rejected', reason: 'x'.repeat(200) }); + }); + + it('describes transport errors by code and strips long hex from messages', () => { + expect(describeTransportError({ code: 'ECONNREFUSED' })).toBe('econnrefused'); + expect(describeTransportError(new Error('failed ' + RAW_TX))).toBe('failed '); + expect(describeTransportError('?')).toBe('transport-error'); + }); +}); diff --git a/backend/src/api/intelligence/private-submission/private-relay.transport.ts b/backend/src/api/intelligence/private-submission/private-relay.transport.ts new file mode 100644 index 00000000000..b3b430865c0 --- /dev/null +++ b/backend/src/api/intelligence/private-submission/private-relay.transport.ts @@ -0,0 +1,89 @@ +import axios, { AxiosError } from 'axios'; +import * as net from 'net'; +import { SocksProxyAgent } from 'socks-proxy-agent'; +import { PrivateRelayEndpoint, PrivateRelayOutcome, PrivateRelayTransportAdapter } from './private-relay.types'; + +/** + * Posts a raw transaction to an owned onion or i2p endpoint through the owned + * SOCKS proxy. The contract is the one the explorer's own /api/tx and every + * esplora/electrs instance accept: the hex as a text/plain body, the txid back + * as text on 200, the node's rejection reason as text on 400. + * + * Nothing about the transaction is logged. The outcome carries only a bounded + * reason string derived from the status or the error code. + */ +const MAX_REASON = 200; + +export class SocksPrivateRelayTransport implements PrivateRelayTransportAdapter { + /** @asyncUnsafe Every failure is folded into the returned outcome. */ + public async submit(endpoint: PrivateRelayEndpoint, rawTx: string, expectedTxid: string): Promise { + const agent = new SocksProxyAgent(endpoint.proxy, { timeout: endpoint.timeoutMs }); + try { + const response = await axios.post(endpoint.submitUrl, rawTx, { + headers: { 'Content-Type': 'text/plain', 'User-Agent': 'universe-private-relay' }, + httpAgent: agent, + httpsAgent: agent, + proxy: false, + timeout: endpoint.timeoutMs, + maxRedirects: 0, + maxContentLength: 4096, + maxBodyLength: rawTx.length + 64, + responseType: 'text', + transformResponse: [(data) => data], + validateStatus: () => true, + }); + return classifyResponse(response.status, response.data, expectedTxid); + } catch (error) { + return { kind: 'unreachable', reason: describeTransportError(error) }; + } + } + + /** @asyncUnsafe Every failure is folded into the returned boolean. */ + public async probeProxy(endpoint: PrivateRelayEndpoint): Promise { + let proxy: URL; + try { + proxy = new URL(endpoint.proxy); + } catch { + return false; + } + return new Promise((resolve) => { + const socket = net.connect({ host: proxy.hostname, port: Number(proxy.port) }); + const done = (ok: boolean): void => { socket.destroy(); resolve(ok); }; + socket.setTimeout(Math.min(endpoint.timeoutMs, 5_000)); + socket.once('connect', () => done(true)); + socket.once('timeout', () => done(false)); + socket.once('error', () => done(false)); + }); + } +} + +export function classifyResponse(status: number, body: unknown, expectedTxid: string): PrivateRelayOutcome { + const text = typeof body === 'string' ? body.trim() : ''; + if (status >= 200 && status < 300) { + const returned = text.toLowerCase(); + if (/^[0-9a-f]{64}$/.test(returned)) { + if (returned !== expectedTxid) { + return { kind: 'rejected', reason: 'endpoint-returned-different-txid', httpStatus: status }; + } + return { kind: 'submitted', txid: returned, httpStatus: status }; + } + // An owned endpoint that answers 2xx without a txid did accept the body; + // the confirmation tracker decides what became of it. + return { kind: 'submitted', txid: expectedTxid, httpStatus: status }; + } + if (status === 400 || status === 422) { + return { kind: 'rejected', reason: boundedReason(text || `http-${status}`), httpStatus: status }; + } + return { kind: 'unreachable', reason: `http-${status}` }; +} + +export function describeTransportError(error: unknown): string { + const axiosError = error as AxiosError; + if (axiosError?.code) return boundedReason(String(axiosError.code).toLowerCase()); + if (error instanceof Error && error.message) return boundedReason(error.message.replace(/[0-9a-f]{64,}/gi, '')); + return 'transport-error'; +} + +function boundedReason(reason: string): string { + return reason.replace(/\s+/g, ' ').slice(0, MAX_REASON); +} diff --git a/backend/src/api/intelligence/private-submission/private-relay.types.ts b/backend/src/api/intelligence/private-submission/private-relay.types.ts new file mode 100644 index 00000000000..bec01b0756f --- /dev/null +++ b/backend/src/api/intelligence/private-submission/private-relay.types.ts @@ -0,0 +1,178 @@ +/** + * Types shared by the private relay store, transport, worker and service. + * + * A private submission is a raw transaction the caller signed elsewhere and + * asked this deployment to hand to an owned Tor or I2P relay endpoint. Nothing + * here signs, nothing here falls back to a third-party broadcaster, and the + * only place a raw transaction is kept is the durable row the worker retries + * from. + */ +export type PrivateRelayTransport = 'tor' | 'i2p'; + +export type PrivateRelayState = + | 'queued' + | 'relaying' + | 'submitted' + | 'confirmed' + | 'rejected' + | 'cancelled' + | 'abort-too-late'; + +/** One owned relay endpoint, as declared in UNIVERSE_PRIVATE_RELAY_ENDPOINTS. */ +export interface PrivateRelayEndpoint { + id: string; + transport: PrivateRelayTransport; + /** The owned SOCKS proxy in front of the Tor or I2P daemon, e.g. socks5h://127.0.0.1:9050. */ + proxy: string; + /** The owned onion or i2p HTTP endpoint that accepts a raw hex transaction POST. */ + submitUrl: string; + network: string; + timeoutMs: number; +} + +/** An endpoint entry that was declared but cannot be used, with the reason. */ +export interface PrivateRelayEndpointIssue { + id: string; + reason: string; +} + +export interface PrivateRelayEndpointSet { + endpoints: PrivateRelayEndpoint[]; + issues: PrivateRelayEndpointIssue[]; + /** Set when the environment variable is absent or empty. */ + unconfigured: boolean; +} + +/** A durable submission row. raw_tx never leaves the store except to the transport. */ +export interface PrivateRelaySubmissionRow { + submission_id: string; + network: string; + txid: string; + raw_tx: string; + method: string; + state: PrivateRelayState; + relay_endpoint_id: string | null; + attempts: number; + lease_until: string | null; + lease_owner: string | null; + owner_token_hash: string; + created_at: string; + updated_at: string; + relayed_at: string | null; + confirmed_block_height: number | null; + last_error: string | null; +} + +export interface PrivateRelayQueueDepth { + queued: number; + relaying: number; + submitted: number; + confirmed: number; + rejected: number; + cancelled: number; +} + +/** + * The durable store. Every mutation that decides between two outcomes (claim, + * cancel, settle) is a compare-and-set on the row so two workers, or a worker + * and an abort, cannot both win. + */ +export interface PrivateRelayStore { + /** Inserts the row, or returns the existing row for the same network and txid. */ + insertOrExisting(row: PrivateRelaySubmissionRow): Promise<{ row: PrivateRelaySubmissionRow; created: boolean }>; + getById(submissionId: string): Promise; + /** + * Claims one row whose lease has expired (or was never taken) in the given + * states. Returns null when nothing is claimable. + */ + claimNext(args: { + network: string; + states: PrivateRelayState[]; + workerId: string; + nowIso: string; + leaseUntilIso: string; + maxAttempts: number; + }): Promise; + /** + * Moves a row the worker holds to its next state. Returns false when the + * row is no longer held by this worker in the expected state. + */ + settle(args: { + submissionId: string; + workerId: string; + fromState: PrivateRelayState; + toState: PrivateRelayState; + nowIso: string; + leaseUntilIso: string | null; + relayEndpointId?: string | null; + relayedAtIso?: string | null; + confirmedBlockHeight?: number | null; + lastError?: string | null; + }): Promise; + /** + * Advances a `submitted` row after a confirmation check. The compare is on + * updated_at so two trackers cannot both apply a result to the same read. + */ + advanceTracking(args: { + submissionId: string; + expectedUpdatedAtIso: string; + toState: 'submitted' | 'confirmed'; + nowIso: string; + leaseUntilIso: string | null; + confirmedBlockHeight?: number | null; + lastError?: string | null; + }): Promise; + /** queued -> cancelled, only while nothing has started relaying it. */ + cancelIfQueued(submissionId: string, nowIso: string): Promise; + /** Rows in `submitted` whose next confirmation check is due. */ + dueForTracking(network: string, nowIso: string, limit: number): Promise; + queueDepth(network: string): Promise; +} + +export type PrivateRelayOutcome = + | { kind: 'submitted'; txid: string; httpStatus: number } + | { kind: 'rejected'; reason: string; httpStatus: number } + | { kind: 'unreachable'; reason: string }; + +/** Sends a raw transaction through one endpoint. Never throws for relay outcomes. */ +export interface PrivateRelayTransportAdapter { + submit(endpoint: PrivateRelayEndpoint, rawTx: string, expectedTxid: string): Promise; + /** True when the endpoint's SOCKS proxy accepts a TCP connection. Does not open a circuit. */ + probeProxy(endpoint: PrivateRelayEndpoint): Promise; +} + +export interface PrivateRelayLastOutcome { + endpoint_id: string; + transport: PrivateRelayTransport; + outcome: PrivateRelayOutcome['kind']; + at: string; +} + +export interface PrivateRelayWorkerStatus { + running: boolean; + worker_id: string; + last_tick_at: string | null; + last_error: string | null; + last_relay: PrivateRelayLastOutcome | null; + proxy_reachable: Record; +} + +export interface ConfirmationLookup { + (txid: string): Promise<{ confirmed: boolean; block_height?: number } | null>; +} + +/** The presented owner token does not match the record. The routes answer 403. */ +export class PrivateSubmissionAuthorizationError extends Error { + public readonly code = 'unauthorized'; + constructor(message = 'The owner token does not authorize this submission.') { + super(message); + } +} + +/** No record carries this submission id. The routes answer 404. */ +export class PrivateSubmissionNotFoundError extends Error { + public readonly code = 'not-found'; + constructor(message = 'No private submission carries this id.') { + super(message); + } +} diff --git a/backend/src/api/intelligence/private-submission/private-relay.validation.ts b/backend/src/api/intelligence/private-submission/private-relay.validation.ts new file mode 100644 index 00000000000..59ddb4c8fc4 --- /dev/null +++ b/backend/src/api/intelligence/private-submission/private-relay.validation.ts @@ -0,0 +1,46 @@ +import { Transaction } from 'bitcoinjs-lib'; +import { SubmissionInputError } from './submission-input'; + +/** + * Bounded checks on a raw transaction before it is allowed into the durable + * queue. The route's submissionInput already proved the hex parses; this pass + * adds the bounds a relay endpoint would enforce anyway (consensus weight, + * no coinbase) and yields the txid the record is keyed on. + */ +export const MAX_PRIVATE_RELAY_TX_BYTES = 400_000; +export const MIN_PRIVATE_RELAY_TX_BYTES = 60; + +export interface ValidatedPrivateRelayTransaction { + raw_tx: string; + txid: string; + size_bytes: number; +} + +export function validatePrivateRelayTransaction(rawTx: unknown): ValidatedPrivateRelayTransaction { + if (typeof rawTx !== 'string' || !/^(?:[0-9a-f]{2})+$/i.test(rawTx)) { + throw new SubmissionInputError('Provide complete transaction hex.'); + } + const sizeBytes = rawTx.length / 2; + if (sizeBytes > MAX_PRIVATE_RELAY_TX_BYTES) { + throw new SubmissionInputError(`Transaction exceeds ${MAX_PRIVATE_RELAY_TX_BYTES} bytes.`); + } + if (sizeBytes < MIN_PRIVATE_RELAY_TX_BYTES) { + throw new SubmissionInputError(`Transaction is smaller than ${MIN_PRIVATE_RELAY_TX_BYTES} bytes.`); + } + let tx: Transaction; + try { + tx = Transaction.fromHex(rawTx); + } catch { + throw new SubmissionInputError('Transaction serialization is malformed.'); + } + if (!tx.ins.length || !tx.outs.length) { + throw new SubmissionInputError('Transaction needs at least one input and one output.'); + } + if (tx.isCoinbase()) { + throw new SubmissionInputError('A coinbase transaction cannot be relayed.'); + } + if (tx.toHex().length !== rawTx.length) { + throw new SubmissionInputError('Transaction hex carries trailing bytes.'); + } + return { raw_tx: rawTx.toLowerCase(), txid: tx.getId(), size_bytes: sizeBytes }; +} diff --git a/backend/src/api/intelligence/private-submission/private-relay.worker.ts b/backend/src/api/intelligence/private-submission/private-relay.worker.ts new file mode 100644 index 00000000000..d86624db61d --- /dev/null +++ b/backend/src/api/intelligence/private-submission/private-relay.worker.ts @@ -0,0 +1,242 @@ +import { randomUUID } from 'crypto'; +import logger from '../../../logger'; +import { endpointsForTransport, transportForMethod } from './private-relay.config'; +import { + ConfirmationLookup, + PrivateRelayEndpoint, + PrivateRelayLastOutcome, + PrivateRelayStore, + PrivateRelaySubmissionRow, + PrivateRelayTransportAdapter, + PrivateRelayWorkerStatus, +} from './private-relay.types'; + +/** + * The leased relay worker. + * + * Each tick claims at most one queued row (or one `relaying` row whose lease a + * crashed worker left behind), hands it to the transport for one endpoint of + * the requested kind, and settles the row from the outcome. A retryable + * failure returns the row to `queued` with a backoff lease so the next claim + * waits; a node rejection or exhausted attempts ends in `rejected`. Submitted + * rows are then polled against the owned node on a bounded cadence until they + * confirm or the tracking window closes. + * + * Log lines carry submission ids and outcome kinds only. Raw transactions and + * owner tokens never reach the logger. + */ +export interface PrivateRelayWorkerOptions { + store: PrivateRelayStore; + transport: PrivateRelayTransportAdapter; + endpoints: PrivateRelayEndpoint[]; + network: string; + confirmation: ConfirmationLookup; + now?: () => number; + workerId?: string; + maxAttempts?: number; + leaseMs?: number; + retryBackoffMs?: number; + trackingIntervalMs?: number; + trackingWindowMs?: number; + probeIntervalMs?: number; +} + +export class PrivateRelayWorker { + public readonly workerId: string; + private readonly store: PrivateRelayStore; + private readonly transport: PrivateRelayTransportAdapter; + private readonly endpoints: PrivateRelayEndpoint[]; + private readonly network: string; + private readonly confirmation: ConfirmationLookup; + private readonly now: () => number; + private readonly maxAttempts: number; + private readonly leaseMs: number; + private readonly retryBackoffMs: number; + private readonly trackingIntervalMs: number; + private readonly trackingWindowMs: number; + private readonly probeIntervalMs: number; + + private timer: NodeJS.Timeout | null = null; + private ticking = false; + private lastTickAt: string | null = null; + private lastError: string | null = null; + private lastRelay: PrivateRelayLastOutcome | null = null; + private proxyReachable: Record = {}; + private lastProbeAt = 0; + private roundRobin = 0; + + constructor(options: PrivateRelayWorkerOptions) { + this.store = options.store; + this.transport = options.transport; + this.endpoints = options.endpoints; + this.network = options.network; + this.confirmation = options.confirmation; + this.now = options.now ?? (() => Date.now()); + this.workerId = options.workerId ?? randomUUID(); + this.maxAttempts = options.maxAttempts ?? 5; + this.leaseMs = options.leaseMs ?? 120_000; + this.retryBackoffMs = options.retryBackoffMs ?? 60_000; + this.trackingIntervalMs = options.trackingIntervalMs ?? 60_000; + this.trackingWindowMs = options.trackingWindowMs ?? 24 * 3600_000; + this.probeIntervalMs = options.probeIntervalMs ?? 5 * 60_000; + for (const endpoint of this.endpoints) this.proxyReachable[endpoint.id] = null; + } + + public start(intervalMs = 2_000): void { + if (this.timer) return; + this.timer = setInterval(() => { + this.tick().catch(error => logger.warn('private relay worker: ' + describe(error))); + }, intervalMs); + this.timer.unref?.(); + } + + public stop(): void { + if (this.timer) { clearInterval(this.timer); this.timer = null; } + } + + public status(): PrivateRelayWorkerStatus { + return { + running: this.timer !== null, + worker_id: this.workerId, + last_tick_at: this.lastTickAt, + last_error: this.lastError, + last_relay: this.lastRelay ? { ...this.lastRelay } : null, + proxy_reachable: { ...this.proxyReachable }, + }; + } + + /** + * One pass: probe proxies when due, relay one claimable row, then advance + * the submitted rows whose confirmation check is due. + * @asyncUnsafe The interval handler logs a rejection; tests await it directly. + */ + public async tick(): Promise { + if (this.ticking) return; + this.ticking = true; + try { + this.lastTickAt = new Date(this.now()).toISOString(); + await this.probeProxies(); + await this.relayOne(); + await this.trackSubmitted(); + this.lastError = null; + } catch (error) { + this.lastError = describe(error); + throw error; + } finally { + this.ticking = false; + } + } + + /** @asyncUnsafe Probe failures are recorded in proxy_reachable, never thrown. */ + public async probeProxies(force = false): Promise { + if (!force && this.now() - this.lastProbeAt < this.probeIntervalMs) return; + this.lastProbeAt = this.now(); + for (const endpoint of this.endpoints) { + try { + this.proxyReachable[endpoint.id] = await this.transport.probeProxy(endpoint); + } catch { + this.proxyReachable[endpoint.id] = false; + } + } + } + + /** @asyncUnsafe Store rejections surface through tick(). */ + private async relayOne(): Promise { + const nowMs = this.now(); + const nowIso = new Date(nowMs).toISOString(); + const row = await this.store.claimNext({ + network: this.network, + states: ['queued', 'relaying'], + workerId: this.workerId, + nowIso, + leaseUntilIso: new Date(nowMs + this.leaseMs).toISOString(), + maxAttempts: this.maxAttempts, + }); + if (!row) return; + + const endpoint = this.pickEndpoint(row); + if (!endpoint) { + await this.settleRejected(row, 'no-endpoint-for-method'); + return; + } + + const outcome = await this.transport.submit(endpoint, row.raw_tx, row.txid); + const settledAt = new Date(this.now()).toISOString(); + this.lastRelay = { endpoint_id: endpoint.id, transport: endpoint.transport, outcome: outcome.kind, at: settledAt }; + logger.info(`private relay: submission ${row.submission_id} via ${endpoint.id} -> ${outcome.kind} (attempt ${row.attempts})`); + + if (outcome.kind === 'submitted') { + await this.store.settle({ + submissionId: row.submission_id, workerId: this.workerId, fromState: 'relaying', toState: 'submitted', + nowIso: settledAt, leaseUntilIso: new Date(this.now() + this.trackingIntervalMs).toISOString(), + relayEndpointId: endpoint.id, relayedAtIso: settledAt, lastError: null, + }); + return; + } + if (outcome.kind === 'rejected') { + await this.settleRejected(row, outcome.reason, endpoint.id); + return; + } + if (row.attempts >= this.maxAttempts) { + await this.settleRejected(row, 'relay-attempts-exhausted: ' + outcome.reason, endpoint.id); + return; + } + await this.store.settle({ + submissionId: row.submission_id, workerId: this.workerId, fromState: 'relaying', toState: 'queued', + nowIso: settledAt, leaseUntilIso: new Date(this.now() + this.retryBackoffMs).toISOString(), + relayEndpointId: endpoint.id, lastError: outcome.reason, + }); + } + + /** @asyncUnsafe Store rejections surface through tick(). */ + private async settleRejected(row: PrivateRelaySubmissionRow, reason: string, endpointId?: string): Promise { + await this.store.settle({ + submissionId: row.submission_id, workerId: this.workerId, fromState: 'relaying', toState: 'rejected', + nowIso: new Date(this.now()).toISOString(), leaseUntilIso: null, + relayEndpointId: endpointId ?? row.relay_endpoint_id, lastError: reason.slice(0, 500), + }); + } + + private pickEndpoint(row: PrivateRelaySubmissionRow): PrivateRelayEndpoint | null { + const transport = transportForMethod(row.method); + if (!transport) return null; + const candidates = endpointsForTransport(this.endpoints, transport); + if (!candidates.length) return null; + const reachable = candidates.filter(endpoint => this.proxyReachable[endpoint.id] !== false); + const pool = reachable.length ? reachable : candidates; + return pool[this.roundRobin++ % pool.length]; + } + + /** @asyncUnsafe Store rejections surface through tick(); node lookups are caught per row. */ + private async trackSubmitted(): Promise { + const nowMs = this.now(); + const due = await this.store.dueForTracking(this.network, new Date(nowMs).toISOString(), 20); + for (const row of due) { + let status: { confirmed: boolean; block_height?: number } | null = null; + try { + status = await this.confirmation(row.txid); + } catch { + status = null; + } + const checkedAt = new Date(this.now()).toISOString(); + if (status?.confirmed) { + await this.store.advanceTracking({ + submissionId: row.submission_id, expectedUpdatedAtIso: row.updated_at, toState: 'confirmed', + nowIso: checkedAt, leaseUntilIso: null, confirmedBlockHeight: status.block_height ?? null, lastError: null, + }); + continue; + } + const relayedAtMs = row.relayed_at ? new Date(row.relayed_at).getTime() : nowMs; + const windowClosed = nowMs - relayedAtMs > this.trackingWindowMs; + await this.store.advanceTracking({ + submissionId: row.submission_id, expectedUpdatedAtIso: row.updated_at, toState: 'submitted', + nowIso: checkedAt, leaseUntilIso: windowClosed ? null : new Date(this.now() + this.trackingIntervalMs).toISOString(), + lastError: windowClosed ? 'confirmation-tracking-window-closed' : undefined, + }); + } + } +} + +function describe(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/backend/src/api/intelligence/private-submission/private-submission.models.ts b/backend/src/api/intelligence/private-submission/private-submission.models.ts index aece67477c8..80b7d4c2196 100644 --- a/backend/src/api/intelligence/private-submission/private-submission.models.ts +++ b/backend/src/api/intelligence/private-submission/private-submission.models.ts @@ -44,6 +44,33 @@ export interface SubmissionDiagnosisResult { acceleration_recommended: boolean; privacy_advisory: string; available_methods: SubmissionMethod[]; + /** Owned mempool package facts for the entry. */ + package?: { + effective_feerate_sats_vb: number; + ancestor_count: number; + descendant_count: number; + ancestor_fee_sats: number; + ancestor_weight: number; + cpfp_boosted: boolean; + cpfp_checked: boolean; + best_descendant_txid: string | null; + projected_block_index: number | null; + }; + policy?: { + source: string; + mempool_min_fee_sats_vb: number | null; + min_relay_fee_sats_vb: number | null; + mempool_loaded: boolean | null; + }; + replacement?: { + signals_rbf: boolean; + replaces_txids: string[]; + replaced_by_txid: string | null; + is_replacement: boolean; + }; + observed_at_utc?: string; + first_seen_utc?: string | null; + scope?: string; } export interface PrivateBroadcastRecord { @@ -52,10 +79,67 @@ export interface PrivateBroadcastRecord { method: SubmissionMethod; network: string; queued_at_utc: string; - status: 'queued' | 'acknowledged' | 'aborted' | 'broadcast_completed' | 'failed'; + status: + | 'queued' + | 'acknowledged' + | 'aborted' + | 'broadcast_completed' + | 'failed' + | 'relaying' + | 'submitted' + | 'confirmed' + | 'rejected' + | 'cancelled' + | 'abort-too-late'; retry_count: number; can_abort: boolean; last_error?: string; + /** Present exactly once, on the response that created the record. */ + owner_token?: string; + /** True when this submission matched an existing record for the same txid. */ + duplicate?: boolean; + relay_endpoint_id?: string; + relayed_at_utc?: string; + confirmed_block_height?: number; + updated_at_utc?: string; +} + +/** + * Owner authentication for private submission readback and abort. The token is + * returned once by POST /submission/private and is presented either in this + * header or, for the abort POST, as the owner_token body field. + */ +export const PRIVATE_SUBMISSION_OWNER_TOKEN_HEADER = 'x-submission-owner-token'; + +export interface PrivateRelayEndpointFact { + id: string; + transport: 'tor' | 'i2p'; + submit_host: string; + proxy_reachable: boolean | null; +} + +export interface PrivateRelayOverview { + network: string; + relay: { + configured: boolean; + reason: string | null; + endpoints: PrivateRelayEndpointFact[]; + rejected_endpoints: { id: string; reason: string }[]; + }; + queue: { + queued: number; + relaying: number; + submitted: number; + confirmed: number; + rejected: number; + cancelled: number; + }; + worker: { + running: boolean; + last_tick_at: string | null; + last_error: string | null; + last_relay: { endpoint_id: string; transport: 'tor' | 'i2p'; outcome: string; at: string } | null; + }; } export interface AcceleratorProvider { @@ -68,19 +152,44 @@ export interface AcceleratorProvider { maximum_tx_vsize: number; payment_methods: string[]; partner_mining_claims: string[]; - status_endpoint: string; - health_status: 'online' | 'degraded' | 'offline'; + /** Null: the owned directory names no health endpoint and none is probed. */ + status_endpoint: string | null; + health_status: 'online' | 'degraded' | 'offline' | 'unmeasured'; effective_from: string; - expires_at: string; - provider_signature: string; + /** Null when a directory key has no end of validity. */ + expires_at: string | null; + /** Null: the directory file is the trust root; entries carry no separate signature. */ + provider_signature: string | null; + issuer?: string; + protocol_version?: string; + keys?: AcceleratorProviderKey[]; + directory?: { source: string; revision: string; loaded_at_utc: string }; } +/** + * A provider's signed acceleration receipt. + * + * Signed payload encoding (schema_version 'universe-accelerator-receipt-v1'): + * the receipt object without `provider_signature` and `key_id`, serialised + * as JSON with object keys sorted lexicographically at every level, no + * whitespace, strings as JSON strings, numbers as JSON numbers, arrays in + * their given order. The message signed is the UTF-8 bytes of the string + * `universe-accelerator-receipt-v1\n` followed by that JSON. The + * signature is hex: 64 bytes for ed25519, 64 bytes for secp256k1 BIP-340 + * Schnorr (over the SHA-256 of the message). `key_id` names the provider + * key in the owned directory that must be valid at `submitted_at_utc`. + * The payload binds provider_id, receipt_id, txid, network, submitted_at_utc, + * expires_at_utc and the acceleration terms (target feerate, provider fee, + * claimed coverage and pools, status); altering any of them invalidates the + * signature. + */ export interface AcceleratorReceipt { schema_version: string; provider_id: string; receipt_id: string; txid: string; wtxid?: string; + network: string; submitted_at_utc: string; expires_at_utc: string; target_feerate_sats_vb: number; @@ -88,9 +197,48 @@ export interface AcceleratorReceipt { claimed_mining_coverage_pct: number; claimed_partner_pools: string[]; status: 'active' | 'included' | 'expired' | 'refunded'; + key_id: string; provider_signature: string; } +/** + * invalid: structure, completeness or signature failed (an altered field + * fails here); unsupported: an encoding this deployment cannot verify; + * unavailable-trust: no directory, or the provider or key is not in it or + * not valid at issue; wrong-network; expired; duplicate: a replay of a + * receipt already verified here; verified. + */ +export type ReceiptVerificationStage = + | 'invalid' + | 'unsupported' + | 'unavailable-trust' + | 'wrong-network' + | 'expired' + | 'duplicate' + | 'verified'; + +export interface ReceiptVerificationResult { + verified: boolean; + stage: ReceiptVerificationStage; + errors: string[]; + provider_id?: string; + receipt_id?: string; + key_id?: string; + algorithm?: string; + verified_at_utc?: string; + directory?: { source: string; revision: string; loaded_at_utc: string }; + replay?: { first_verified_at_utc: string; seen_count: number }; + scope?: string; +} + +export interface AcceleratorProviderKey { + kid: string; + algorithm: 'ed25519' | 'secp256k1-schnorr'; + publicKey: string; + validFrom: string; + validUntil: string | null; +} + export interface TransactionOrderingEvidence { txid: string; block_hash: string; @@ -107,6 +255,10 @@ export interface TransactionOrderingEvidence { is_ordering_sensitive: boolean; protocol_impact_description?: string; confidence_rating: 'high' | 'medium' | 'low'; + template_coverage?: 'template-observed' | 'no-template-observed'; + template_ids?: string[]; + template_position?: number | null; + sensor_observer_id?: string | null; } export interface SubmissionOverview { diff --git a/backend/src/api/intelligence/private-submission/private-submission.routes.ts b/backend/src/api/intelligence/private-submission/private-submission.routes.ts index b0a14c0d859..9d8c28f8b8b 100644 --- a/backend/src/api/intelligence/private-submission/private-submission.routes.ts +++ b/backend/src/api/intelligence/private-submission/private-submission.routes.ts @@ -9,6 +9,12 @@ import { Application, Request, Response } from 'express'; import privateSubmissionService, { SubmissionEvidenceError, } from './private-submission.service'; +import { PRIVATE_SUBMISSION_OWNER_TOKEN_HEADER } from './private-submission.models'; +import { PrivateRelayUnavailableError } from './private-relay.submissions'; +import { + PrivateSubmissionAuthorizationError, + PrivateSubmissionNotFoundError, +} from './private-relay.types'; /** * Turns an absent integration into a 503 that names it, and anything else into @@ -19,33 +25,60 @@ function fail(res: Response, err: unknown): Response { if (err instanceof SubmissionInputError) return res.status(400).json({ stage: 'invalid-input', error: err.message }); if (err instanceof SubmissionEvidenceError) { - return res.status(503).json({ stage: err.code, error: err.message }); + return res.status(err.status).json({ stage: err.code, error: err.message }); } return res .status(500) .json({ error: (err as Error)?.message || 'Internal error' }); } +/** + * The private relay routes add three answers to fail(): 403 when the owner + * token does not match the record, 404 for an unknown submission id, and 503 + * named after the missing store or relay. + */ +function failPrivate(res: Response, err: unknown): Response { + if (err instanceof PrivateSubmissionAuthorizationError) + return res.status(403).json({ stage: err.code, error: err.message }); + if (err instanceof PrivateSubmissionNotFoundError) + return res.status(404).json({ stage: err.code, error: err.message }); + if (err instanceof PrivateRelayUnavailableError) + return res.status(503).json({ stage: err.code, error: err.message }); + return fail(res, err); +} + +/** + * The owner token is read from the X-Submission-Owner-Token header or, on the + * abort POST, from the owner_token body field. It is never taken from the URL. + */ +function ownerTokenInput(req: Request, allowBody: boolean): string | undefined { + const header = req.get(PRIVATE_SUBMISSION_OWNER_TOKEN_HEADER); + if (typeof header === 'string' && header.trim()) return header.trim(); + if (allowBody && req.body && typeof req.body === 'object' && typeof req.body.owner_token === 'string') + return req.body.owner_token.trim(); + return undefined; +} + class PrivateSubmissionRoutes { public initRoutes(app: Application): void { app.get( '/api/v1/intelligence/submission/overview', - (_req: Request, res: Response) => { + async (_req: Request, res: Response) => { try { - res.json(privateSubmissionService.getOverview()); + res.json(await privateSubmissionService.getOverview()); } catch (err) { - fail(res, err); + failPrivate(res, err); } } ); app.get( '/api/v1/intelligence/submission/capabilities', - (_req: Request, res: Response) => { + async (_req: Request, res: Response) => { try { - res.json(privateSubmissionService.getCapabilities()); + res.json(await privateSubmissionService.getCapabilities()); } catch (err) { - fail(res, err); + failPrivate(res, err); } } ); @@ -64,43 +97,46 @@ class PrivateSubmissionRoutes { app.post( '/api/v1/intelligence/submission/private', - (req: Request, res: Response) => { + async (req: Request, res: Response) => { try { - res.json( - privateSubmissionService.submitPrivate(submissionInput(req.body)) + const record = await privateSubmissionService.submitPrivate( + submissionInput(req.body) ); + res.status(record.duplicate ? 200 : 201).json(record); } catch (err) { - fail(res, err); + failPrivate(res, err); } } ); app.get( '/api/v1/intelligence/submission/private/:submissionToken', - (req: Request, res: Response) => { + async (req: Request, res: Response) => { try { res.json( - privateSubmissionService.getPrivateSubmission( - tokenInput(req.params.submissionToken) + await privateSubmissionService.getPrivateSubmission( + tokenInput(req.params.submissionToken), + ownerTokenInput(req, false) ) ); } catch (err) { - fail(res, err); + failPrivate(res, err); } } ); app.post( '/api/v1/intelligence/submission/private/:submissionToken/abort', - (req: Request, res: Response) => { + async (req: Request, res: Response) => { try { res.json( - privateSubmissionService.abortPrivateSubmission( - tokenInput(req.params.submissionToken) + await privateSubmissionService.abortPrivateSubmission( + tokenInput(req.params.submissionToken), + ownerTokenInput(req, true) ) ); } catch (err) { - fail(res, err); + failPrivate(res, err); } } ); @@ -131,9 +167,8 @@ class PrivateSubmissionRoutes { } ); - // The one route here that can still answer, because it reads the caller's - // own payload. A malformed receipt is the caller's 400; a complete one - // cannot be verified without the registry, which is a 503. + // A malformed, altered, expired or wrong-network receipt is the caller's + // 400; a replay is a 409; an absent directory or unknown key is a 503. app.post( '/api/v1/intelligence/accelerators/receipts/verify', (req: Request, res: Response) => { @@ -141,7 +176,8 @@ class PrivateSubmissionRoutes { const result = privateSubmissionService.verifyAcceleratorReceipt( req.body || {} ); - res.status(result.stage === 'invalid' ? 400 : 503).json(result); + const status = result.verified ? 200 : result.stage === 'duplicate' ? 409 : result.stage === 'unavailable-trust' ? 503 : 400; + res.status(status).json(result); } catch (err) { fail(res, err); } diff --git a/backend/src/api/intelligence/private-submission/private-submission.service.ts b/backend/src/api/intelligence/private-submission/private-submission.service.ts index ad686db6d99..f7a8ef7346e 100644 --- a/backend/src/api/intelligence/private-submission/private-submission.service.ts +++ b/backend/src/api/intelligence/private-submission/private-submission.service.ts @@ -1,37 +1,40 @@ +import config from '../../../config'; import { SubmissionCapabilities, SubmissionDiagnosisResult, PrivateBroadcastRecord, AcceleratorProvider, AcceleratorReceipt, + ReceiptVerificationResult, TransactionOrderingEvidence, + PrivateRelayOverview, } from './private-submission.models'; +import { + abortPrivateRelaySubmission, + privateRelayCapabilities, + privateRelayOverview, + readPrivateRelaySubmission, + submitPrivateRelay, +} from './private-relay.submissions'; +import { AcceleratorDirectoryError, acceleratorDirectory, providerView, verifyAcceleratorReceipt } from './accelerator-directory'; +import { BlockOrderingEvidence, OrderingEvidenceError, OrderingEvidenceService, orderingEvidenceService, OrderingFindings } from './ordering-evidence.service'; +import { DiagnosisNotFound, DiagnosisReaders, diagnoseFromOwnedMempool, ownedDiagnosisReaders } from './transaction-diagnosis'; /** - * Raised when a read has no source behind it. The routes map the code to a - * status, so an absent integration is reported as an absent integration rather - * than as an answer. + * Raised when a read has no source behind it, or names nothing the source + * holds. The routes map the status: 503 for an absent integration, 404 for + * an absent record. */ export class SubmissionEvidenceError extends Error { constructor( public readonly code: string, - message: string + message: string, + public readonly status: number = 503 ) { super(message); } } -const relayUnavailable = - 'Private broadcast is unavailable. Queueing, relaying and abort all require the owned Tor and I2P submission relay, which is not configured on this deployment (PRE-01).'; - -const registryUnavailable = - 'The accelerator provider registry is unavailable. Provider identities, health and receipt trust cannot be determined until the owned signed provider directory is configured (PRE-04).'; - -const orderingUnavailable = - 'Ordering evidence is unavailable. Template and first-seen observation require the owned relay sensor and template recorder, which are not configured on this deployment (PRE-01).'; - -const diagnosisUnavailable = - 'Acceleration diagnosis is unavailable. Feerate, policy, RBF and CPFP findings require the owned mempool and policy source for this transaction, which is not wired to this route (PRE-01).'; /** * Private submission, acceleration and ordering evidence. @@ -55,139 +58,121 @@ const diagnosisUnavailable = * deployment does not have. */ export class PrivateSubmissionService { - public getOverview(): never { - throw new SubmissionEvidenceError('unavailable-source', relayUnavailable); + /** Test seams for the owned readers behind diagnosis and ordering. */ + public diagnosisReaders: DiagnosisReaders = ownedDiagnosisReaders; + public ordering: OrderingEvidenceService = orderingEvidenceService; + + /** @asyncUnsafe The route turns a rejection into an exact HTTP answer. */ + public async getOverview(): Promise { + return privateRelayOverview(); } - public getCapabilities(): SubmissionCapabilities { - throw new SubmissionEvidenceError('unavailable-source', relayUnavailable); + /** @asyncUnsafe The route turns a rejection into an exact HTTP answer. */ + public async getCapabilities(): Promise { + return privateRelayCapabilities(); } - public diagnoseTransaction(_rawTxOrTxid: string): SubmissionDiagnosisResult { - throw new SubmissionEvidenceError( - 'unavailable-source', - diagnosisUnavailable - ); + /** Facts of the requested transaction's own mempool entry; a transaction this mempool does not hold is a typed 404. */ + public diagnoseTransaction(rawTxOrTxid: string): SubmissionDiagnosisResult { + try { + return diagnoseFromOwnedMempool(rawTxOrTxid, this.diagnosisReaders); + } catch (e) { + if (e instanceof DiagnosisNotFound) { + throw new SubmissionEvidenceError('transaction-not-in-mempool', e.message, 404); + } + throw e; + } } - public submitPrivate(_submission: { + /** @asyncUnsafe The route turns a rejection into an exact HTTP answer. */ + public async submitPrivate(submission: { raw_tx: string; method: string; - }): PrivateBroadcastRecord { - throw new SubmissionEvidenceError('unavailable-relay', relayUnavailable); + }): Promise { + return submitPrivateRelay(submission); } - public getPrivateSubmission( - _token: string - ): PrivateBroadcastRecord | undefined { - throw new SubmissionEvidenceError('unavailable-relay', relayUnavailable); + /** @asyncUnsafe The route turns a rejection into an exact HTTP answer. */ + public async getPrivateSubmission( + submissionId: string, + ownerToken?: string + ): Promise { + return readPrivateRelaySubmission(submissionId, ownerToken); } - public abortPrivateSubmission(_token: string): { + /** @asyncUnsafe The route turns a rejection into an exact HTTP answer. */ + public async abortPrivateSubmission( + submissionId: string, + ownerToken?: string + ): Promise<{ success: boolean; status: string; - } { - throw new SubmissionEvidenceError('unavailable-relay', relayUnavailable); + }> { + return abortPrivateRelaySubmission(submissionId, ownerToken); + } + + private directory(): ReturnType { + try { + return acceleratorDirectory(); + } catch (e) { + if (e instanceof AcceleratorDirectoryError) { + throw new SubmissionEvidenceError('unavailable-registry', e.message + ' (reason: ' + e.reason + ')'); + } + throw e; + } } - public listAcceleratorProviders(): { providers: AcceleratorProvider[] } { - throw new SubmissionEvidenceError( - 'unavailable-registry', - registryUnavailable - ); + /** Providers from the owned directory that list this deployment's network. */ + public listAcceleratorProviders(): { providers: AcceleratorProvider[]; network: string; directory: { source: string; revision: string; loaded_at_utc: string } } { + const directory = this.directory(); + const network = config.MEMPOOL.NETWORK; + return { + providers: directory.providers.filter(provider => provider.networks.includes(network)).map(provider => providerView(provider, directory)), + network, + directory: { source: directory.source, revision: directory.revision, loaded_at_utc: directory.loaded_at_utc }, + }; } - public getAcceleratorProvider( - _providerId: string - ): AcceleratorProvider | undefined { - throw new SubmissionEvidenceError( - 'unavailable-registry', - registryUnavailable - ); + public getAcceleratorProvider(providerId: string): AcceleratorProvider { + const directory = this.directory(); + const provider = directory.providers.find(entry => entry.id === providerId); + if (!provider) { + throw new SubmissionEvidenceError('provider-not-in-directory', `Provider ${providerId} is not in the owned directory (revision ${directory.revision}).`, 404); + } + return providerView(provider, directory); } /** - * Reads the payload the caller supplied and reports what is missing from it. - * - * A structurally complete receipt is not a verified receipt: the signature - * has to be checked against the provider's key, and there is no registry to - * take that key from. So a complete payload reports unavailable trust rather - * than success, and an incomplete one reports its own faults, which is a - * finding this deployment can actually make. + * Structural faults are the caller's; everything past them is decided by + * the owned directory's key for the named provider: the signed payload + * encoding documented on AcceleratorReceipt, the key's validity at issue, + * the network, the expiry and a replay of a receipt already verified here. */ - public verifyAcceleratorReceipt(receipt: Partial): { - verified: false; - stage: 'invalid' | 'unavailable-registry'; - errors: string[]; - } { - const errors: string[] = []; - if ( - !receipt || - typeof receipt !== 'object' || - Array.isArray(receipt) || - JSON.stringify(receipt).length > 16384 - ) - return { - verified: false, - stage: 'invalid', - errors: ['Receipt must be a JSON object no larger than16KiB.'], - }; - for (const [key, max] of [ - ['provider_id', 256], - ['receipt_id', 256], - ['provider_signature', 8192], - ] as const) - if (typeof receipt[key] === 'string' && receipt[key]!.length > max) - errors.push(key + ' is too long'); - if (typeof receipt?.provider_id !== 'string' || !receipt.provider_id.trim()) - errors.push('provider_id is required'); - if (typeof receipt?.receipt_id !== 'string' || !receipt.receipt_id.trim()) - errors.push('receipt_id is required'); - if ( - typeof receipt?.provider_signature !== 'string' || - !receipt.provider_signature.trim() - ) - errors.push('provider_signature is required'); - if ( - typeof receipt?.txid !== 'string' || - !/^[0-9a-f]{64}$/i.test(receipt.txid) - ) - errors.push('Valid 32-byte txid is required'); - - if (errors.length > 0) { - return { verified: false, stage: 'invalid', errors }; + public verifyAcceleratorReceipt(receipt: Partial, now = Date.now()): ReceiptVerificationResult { + return verifyAcceleratorReceipt(receipt, { now }); + } + + private orderingRead(read: () => T): T { + try { + return read(); + } catch (e) { + if (e instanceof OrderingEvidenceError) { + throw new SubmissionEvidenceError(e.code, e.message, e.status); + } + throw e; } - return { - verified: false, - stage: 'unavailable-registry', - errors: [registryUnavailable], - }; } - public getTransactionOrdering( - _txid: string - ): TransactionOrderingEvidence | undefined { - throw new SubmissionEvidenceError( - 'unavailable-source', - orderingUnavailable - ); + public getTransactionOrdering(txid: string): TransactionOrderingEvidence { + return this.orderingRead(() => this.ordering.getTransactionOrdering(txid)); } - public getBlockOrdering(_blockHash: string): { - block_hash: string; - transactions: TransactionOrderingEvidence[]; - } { - throw new SubmissionEvidenceError( - 'unavailable-source', - orderingUnavailable - ); + public getBlockOrdering(blockHash: string): BlockOrderingEvidence { + return this.orderingRead(() => this.ordering.getBlockOrdering(blockHash)); } - public listOrderingFindings(): { findings: TransactionOrderingEvidence[] } { - throw new SubmissionEvidenceError( - 'unavailable-source', - orderingUnavailable - ); + public listOrderingFindings(): OrderingFindings { + return this.ordering.listOrderingFindings(); } } diff --git a/backend/src/api/intelligence/private-submission/private-submission.test.ts b/backend/src/api/intelligence/private-submission/private-submission.test.ts index 2149754fd8d..587c8f2a9bc 100644 --- a/backend/src/api/intelligence/private-submission/private-submission.test.ts +++ b/backend/src/api/intelligence/private-submission/private-submission.test.ts @@ -1,47 +1,264 @@ +import { generateKeyPairSync, sign as signEd25519 } from 'crypto'; +import { Transaction } from 'bitcoinjs-lib'; import privateSubmissionService, { SubmissionEvidenceError } from './private-submission.service'; +import { setPrivateRelayRuntime } from './private-relay.runtime'; +import { ACCELERATOR_RECEIPT_SCHEMA, AcceleratorDirectory, deterministicJson, parseAcceleratorDirectory, receiptSigningMessage, useAcceleratorDirectory } from './accelerator-directory'; +import { OrderingEvidenceService } from './ordering-evidence.service'; +import { AcceleratorReceipt } from './private-submission.models'; +import { DiagnosisReaders } from './transaction-diagnosis'; +import config from '../../../config'; +import { BlockExtended, MempoolTransactionExtended, TransactionExtended } from '../../../mempool.interfaces'; +import { CandidateTemplate } from '../templates/template-collector.service'; + +const TX = '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b'; +const PARENT = 'aa'.repeat(32); +const CHILD = 'bb'.repeat(32); +const UNSEEN = 'cc'.repeat(32); +const BLOCK = '00'.repeat(32); /** - * These assertions replace a suite that asserted the constants the service used - * to return: two accelerator providers, a diagnosis whose numbers did not - * depend on its argument, a queued broadcast nothing relayed, and a receipt - * that verified on four non-empty fields. Passing those proved the constants - * were still present, not that any of it worked. + * These assertions replace a suite that asserted the unavailable constants + * the service used to throw. The relay and overview paths are still owned + * elsewhere and keep their unavailable states; diagnosis, ordering and the + * accelerator directory now read owned sources, faked here. */ describe('PrivateSubmissionService', () => { const unavailable = (code: string) => expect.objectContaining({ code }); - it('reports the missing relay rather than queueing a broadcast nothing sends', () => { - expect(() => privateSubmissionService.submitPrivate({ raw_tx: '0200000001', method: 'privatebroadcast_tor' })) - .toThrow(unavailable('unavailable-relay')); - expect(() => privateSubmissionService.getPrivateSubmission('tok-priv-1')).toThrow(unavailable('unavailable-relay')); - expect(() => privateSubmissionService.abortPrivateSubmission('tok-priv-1')).toThrow(unavailable('unavailable-relay')); + // The private relay itself (queueing, worker, abort, overview and capabilities) + // is covered in private-relay.test.ts with fakes for the durable store and + // the transport. Here: without a database the whole feature reports the + // missing store rather than queueing a broadcast nothing sends. + it('reports the missing durable store rather than queueing a broadcast nothing sends', async () => { + setPrivateRelayRuntime({ network: 'signet', endpoints: { endpoints: [], issues: [], unconfigured: true }, store: null, worker: null, coreVersion: () => '?' }); + try { + await expect(privateSubmissionService.submitPrivate({ raw_tx: '02'.repeat(80), method: 'privatebroadcast_tor' })) + .rejects.toMatchObject({ code: 'durable-store-unavailable' }); + await expect(privateSubmissionService.getPrivateSubmission('tok-priv-1', 'ab'.repeat(32))).rejects.toMatchObject({ code: 'durable-store-unavailable' }); + await expect(privateSubmissionService.abortPrivateSubmission('tok-priv-1', 'ab'.repeat(32))).rejects.toMatchObject({ code: 'durable-store-unavailable' }); + await expect(privateSubmissionService.getOverview()).rejects.toMatchObject({ code: 'durable-store-unavailable' }); + await expect(privateSubmissionService.getCapabilities()).rejects.toMatchObject({ code: 'durable-store-unavailable' }); + } finally { + setPrivateRelayRuntime(null); + } }); - it('reports the missing capability and overview source rather than a constant', () => { - expect(() => privateSubmissionService.getOverview()).toThrow(unavailable('unavailable-source')); - expect(() => privateSubmissionService.getCapabilities()).toThrow(unavailable('unavailable-source')); + it('raises the typed error the routes map to a status', () => { + try { + privateSubmissionService.listAcceleratorProviders(); + throw new Error('expected the provider read to refuse without a directory'); + } catch (err) { + expect(err).toBeInstanceOf(SubmissionEvidenceError); + expect((err as SubmissionEvidenceError).status).toBe(503); + } + }); +}); + +describe('transaction diagnosis from the owned mempool', () => { + const entry = (overrides: Partial = {}): MempoolTransactionExtended => ({ + txid: TX, fee: 1500, vsize: 250, weight: 1000, feePerVsize: 6, effectiveFeePerVsize: 9, adjustedFeePerVsize: 9, adjustedVsize: 250, sigops: 4, order: 1, + vin: [{ sequence: 0xfffffffd, txid: PARENT, vout: 0, is_coinbase: false } as any], vout: [], version: 2, locktime: 0, size: 300, status: { confirmed: false }, + ancestors: [{ txid: PARENT, fee: 100, weight: 800 }], descendants: [], bestDescendant: null, cpfpChecked: true, position: { block: 2, vsize: 1_500_000 }, firstSeen: 1_700_000_000, + ...overrides, + } as unknown as MempoolTransactionExtended); + const readers = (held: Record, extra: Partial = {}): DiagnosisReaders => ({ + mempoolEntry: txid => held[txid], + mempoolInfo: () => ({ mempoolminfee: 0.00001, minrelaytxfee: 0.00001, loaded: true }), + replaces: () => undefined, + replacedBy: () => undefined, + ...extra, }); + afterEach(() => { privateSubmissionService.diagnosisReaders = readers({}); }); - it('reports the missing diagnosis source rather than a fixed feerate verdict', () => { - expect(() => privateSubmissionService.diagnoseTransaction( - '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b', - )).toThrow(unavailable('unavailable-source')); + it('answers a held transaction with its own fee, package, policy and replacement facts', () => { + privateSubmissionService.diagnosisReaders = readers({ [TX]: entry() }, { replaces: () => ['dd'.repeat(32)] }); + const diagnosis = privateSubmissionService.diagnoseTransaction(TX); + expect(diagnosis).toMatchObject({ + txid: TX, vsize: 250, feerate_sats_vb: 6, is_mempool_present: true, is_policy_compliant: true, rbf_eligible: true, cpfp_eligible: true, has_conflicts: true, acceleration_recommended: true, + package: { effective_feerate_sats_vb: 9, ancestor_count: 1, descendant_count: 0, ancestor_fee_sats: 100, ancestor_weight: 800, cpfp_boosted: true, cpfp_checked: true, projected_block_index: 2 }, + policy: { mempool_min_fee_sats_vb: 1, min_relay_fee_sats_vb: 1, mempool_loaded: true }, + replacement: { signals_rbf: true, replaces_txids: ['dd'.repeat(32)], replaced_by_txid: null, is_replacement: false }, + first_seen_utc: '2023-11-14T22:13:20.000Z', + }); + expect(diagnosis.available_methods).toEqual(['public_p2p']); }); - it('reports the missing provider registry rather than a directory of providers', () => { - expect(() => privateSubmissionService.listAcceleratorProviders()).toThrow(unavailable('unavailable-registry')); - expect(() => privateSubmissionService.getAcceleratorProvider('any-provider')).toThrow(unavailable('unavailable-registry')); + it('derives the facts from the entry, not the txid: a different entry gives a different diagnosis', () => { + privateSubmissionService.diagnosisReaders = readers({ [TX]: entry({ fee: 100, vsize: 400, effectiveFeePerVsize: 0.25, vin: [{ sequence: 0xffffffff, txid: PARENT, vout: 0 } as any], ancestors: [], descendants: [{ txid: CHILD, fee: 10, weight: 400 }], position: { block: 0, vsize: 1 } }) }); + const diagnosis = privateSubmissionService.diagnoseTransaction(TX); + expect(diagnosis).toMatchObject({ feerate_sats_vb: 0.25, is_policy_compliant: false, rbf_eligible: false, cpfp_eligible: false, acceleration_recommended: false, package: { descendant_count: 1, cpfp_boosted: false } }); }); - it('reports a structurally complete receipt as unverified while the trust source is absent', () => { - const complete = privateSubmissionService.verifyAcceleratorReceipt({ - provider_id: 'provider-under-test', - receipt_id: 'rcp-1', - txid: '4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b', - provider_signature: 'signature-bytes', - }); - expect(complete.verified).toBe(false); - expect(complete.stage).toBe('unavailable-registry'); + it('answers an unknown transaction with a typed 404 rather than a diagnosis', () => { + privateSubmissionService.diagnosisReaders = readers({}); + expect(() => privateSubmissionService.diagnoseTransaction(TX)).toThrow(expect.objectContaining({ code: 'transaction-not-in-mempool', status: 404 })); + }); + + it('resolves a raw transaction to its txid before the lookup', () => { + const built = new Transaction(); + built.addInput(Buffer.alloc(32, 1), 0); + built.addOutput(Buffer.from('6a', 'hex'), 0); + const raw = built.toHex(); + const seen: string[] = []; + privateSubmissionService.diagnosisReaders = readers({}, { mempoolEntry: txid => { seen.push(txid); return undefined; } }); + expect(() => privateSubmissionService.diagnoseTransaction(raw)).toThrow(/no entry/); + expect(seen).toEqual([expect.stringMatching(/^[0-9a-f]{64}$/)]); + expect(seen[0]).not.toBe(raw); + }); +}); + +describe('ordering evidence from recorded templates and first-seen observations', () => { + const template = (id: string, txids: string[], observedAt: string): CandidateTemplate => ({ + template_id: id, source_id: 'src-core-gbt', source_name: 'Bitcoin Core getblocktemplate', source_type: 'core_gbt', height: 100, prev_block_hash: PARENT, + tx_count: txids.length, total_weight: 0, total_fees_sats: 0, sigops_count: null, coinbase_value_sats: null, fingerprint_hash: 'f', observed_at_utc: observedAt, txids, + }); + const block = (): [BlockExtended, TransactionExtended[]] => [ + { id: BLOCK, height: 100, previousblockhash: PARENT, timestamp: 1_700_000_100 } as unknown as BlockExtended, + [ + { txid: 'coinbase', vin: [{ is_coinbase: true }], fee: 0, vsize: 100 }, + { txid: TX, vin: [{ txid: 'ee'.repeat(32) }], fee: 1000, vsize: 200, effectiveFeePerVsize: 5 }, + { txid: CHILD, vin: [{ txid: TX }], fee: 500, vsize: 100 }, + { txid: UNSEEN, vin: [{ txid: 'ff'.repeat(32) }], fee: 300, vsize: 150 }, + ] as unknown as TransactionExtended[], + ]; + function service(templates: CandidateTemplate[], seen: Record): OrderingEvidenceService { + return new OrderingEvidenceService({ + templatesForHeight: height => templates.filter(t => t.height === height), + firstSeen: txid => (seen[txid] ? { first_observed_utc: seen[txid], source_id: 'owned-mempool-poller-test' } : null), + }, 'signet'); + } + afterEach(() => { privateSubmissionService.ordering = new OrderingEvidenceService({ templatesForHeight: () => [], firstSeen: () => null }); }); + + it('compares the mined order with the recorded template and the first-seen observations', () => { + const ordering = service([template('tmpl-1', [UNSEEN, TX], '2023-11-14T22:14:00.000Z')], { [TX]: '2023-11-14T22:13:00.000Z', [CHILD]: '2023-11-14T22:13:30.000Z' }); + privateSubmissionService.ordering = ordering; + ordering.observeBlock(...block(), Date.parse('2023-11-14T22:15:00.000Z')); + + const first = privateSubmissionService.getTransactionOrdering(TX); + expect(first).toMatchObject({ block_hash: BLOCK, block_height: 100, block_position: 1, evidence_state: 'ordering_changed_between_template_and_block', first_sensor_seen_utc: '2023-11-14T22:13:00.000Z', first_template_seen_utc: '2023-11-14T22:14:00.000Z', template_position: 2, fee_sats_vb: 5, package_feerate_sats_vb: 5, is_ordering_sensitive: true, confidence_rating: 'high', template_coverage: 'template-observed', template_ids: ['tmpl-1'] }); + const child = privateSubmissionService.getTransactionOrdering(CHILD); + expect(child).toMatchObject({ block_position: 2, evidence_state: 'dependency_required_order', dependency_txids: [TX], template_ids: [], confidence_rating: 'high' }); + const unseen = privateSubmissionService.getTransactionOrdering(UNSEEN); + expect(unseen).toMatchObject({ block_position: 3, evidence_state: 'observed_only_in_template', first_sensor_seen_utc: undefined, template_position: 1, sensor_observer_id: null }); + + const whole = privateSubmissionService.getBlockOrdering(BLOCK); + expect(whole).toMatchObject({ block_hash: BLOCK, height: 100, network: 'signet', template_coverage: 'template-observed', templates_compared: ['tmpl-1'], sensor: { observers: 1, observer_id: 'owned-mempool-poller-test' } }); + expect(whole.transactions.map(t => t.txid)).toEqual([TX, CHILD, UNSEEN]); + + const findings = privateSubmissionService.listOrderingFindings(); + expect(findings.state).toBe('observed'); + expect(findings.findings.map(f => [f.txid, f.evidence_state])).toEqual([[TX, 'ordering_changed_between_template_and_block'], [CHILD, 'dependency_required_order'], [UNSEEN, 'observed_only_in_template']]); + expect(findings.coverage).toEqual([{ block_hash: BLOCK, height: 100, template_coverage: 'template-observed', transactions: 3, findings: 3 }]); + }); + + it('reports a height with no recorded template as no-template-observed, not as an empty success', () => { + const ordering = service([], { [TX]: '2023-11-14T22:13:00.000Z' }); + privateSubmissionService.ordering = ordering; + ordering.observeBlock(...block()); + const whole = privateSubmissionService.getBlockOrdering(BLOCK); + expect(whole.template_coverage).toBe('no-template-observed'); + expect(whole.templates_compared).toEqual([]); + expect(whole.transactions.every(t => t.evidence_state === 'insufficient_coverage' && t.template_coverage === 'no-template-observed')).toBe(true); + expect(whole.transactions[0].confidence_rating).toBe('medium'); + expect(whole.transactions[2].confidence_rating).toBe('low'); + const findings = privateSubmissionService.listOrderingFindings(); + expect(findings.findings).toEqual([]); + expect(findings.coverage[0].template_coverage).toBe('no-template-observed'); + }); + + it('ignores templates for another parent and answers unknown blocks and transactions with typed 404s', () => { + const ordering = service([{ ...template('tmpl-other', [TX], '2023-11-14T22:14:00.000Z'), prev_block_hash: 'dd'.repeat(32) }], {}); + privateSubmissionService.ordering = ordering; + expect(privateSubmissionService.listOrderingFindings()).toMatchObject({ state: 'no-blocks-observed', findings: [], coverage: [] }); + expect(() => privateSubmissionService.getBlockOrdering(BLOCK)).toThrow(expect.objectContaining({ code: 'block-not-observed', status: 404 })); + expect(() => privateSubmissionService.getTransactionOrdering(TX)).toThrow(expect.objectContaining({ code: 'transaction-not-observed', status: 404 })); + ordering.observeBlock(...block()); + expect(privateSubmissionService.getBlockOrdering(BLOCK).template_coverage).toBe('no-template-observed'); + }); +}); + +describe('accelerator provider directory and receipt verification', () => { + const { publicKey, privateKey } = generateKeyPairSync('ed25519'); + const rawPublicKey = (publicKey.export({ format: 'der', type: 'spki' }) as Buffer).subarray(-32).toString('hex'); + const directoryText = JSON.stringify({ + schema: 'universe-accelerator-directory-v1', revision: 'r1', + providers: [ + { id: 'accel-one', name: 'Accelerator One', networks: [config.MEMPOOL.NETWORK, 'signet'], issuer: 'Universe operations', protocolVersion: ACCELERATOR_RECEIPT_SCHEMA, + keys: [{ kid: 'k1', algorithm: 'ed25519', publicKey: rawPublicKey, validFrom: '2026-01-01T00:00:00.000Z', validUntil: '2027-01-01T00:00:00.000Z' }, { kid: 'k0', algorithm: 'ed25519', publicKey: rawPublicKey, validFrom: '2020-01-01T00:00:00.000Z', validUntil: '2021-01-01T00:00:00.000Z' }], + terms: { minimum_fee_sats: 1000, maximum_tx_vsize: 100000, payment_methods: ['lightning'], partner_mining_claims: ['none verified'] } }, + { id: 'elsewhere', name: 'Other network only', networks: ['other-network'], issuer: 'x', protocolVersion: ACCELERATOR_RECEIPT_SCHEMA, keys: [{ kid: 'k', algorithm: 'secp256k1-schnorr', publicKey: 'ab'.repeat(32), validFrom: '2026-01-01T00:00:00.000Z', validUntil: null }] }, + ], + }); + const directory: AcceleratorDirectory = parseAcceleratorDirectory(directoryText, '/etc/universe/accelerators.json', Date.parse('2026-09-17T00:00:00.000Z')); + const NOW = Date.parse('2026-09-17T12:00:00.000Z'); + + function receipt(overrides: Partial = {}): AcceleratorReceipt { + const payload = { + schema_version: ACCELERATOR_RECEIPT_SCHEMA, provider_id: 'accel-one', receipt_id: 'rcpt-1', txid: TX, network: config.MEMPOOL.NETWORK, + submitted_at_utc: '2026-09-17T11:00:00.000Z', expires_at_utc: '2026-09-18T11:00:00.000Z', target_feerate_sats_vb: 20, provider_fee_sats: 5000, + claimed_mining_coverage_pct: 30, claimed_partner_pools: ['pool-a'], status: 'active' as const, + ...overrides, + }; + const { provider_signature, key_id, ...signed } = payload as Partial; + void provider_signature; void key_id; + const signature = signEd25519(null, receiptSigningMessage(signed as AcceleratorReceipt), privateKey).toString('hex'); + return { ...(signed as AcceleratorReceipt), key_id: 'k1', provider_signature: signature, ...('provider_signature' in overrides ? { provider_signature: overrides.provider_signature as string } : {}), ...('key_id' in overrides ? { key_id: overrides.key_id as string } : {}) }; + } + + beforeEach(() => { useAcceleratorDirectory(directory); }); + afterEach(() => { useAcceleratorDirectory(undefined); }); + + it('lists the directory providers for this network and reads one by id', () => { + const listed = privateSubmissionService.listAcceleratorProviders(); + expect(listed.providers.map(p => p.provider_id)).toEqual(['accel-one']); + expect(listed.directory).toEqual({ source: '/etc/universe/accelerators.json', revision: 'r1', loaded_at_utc: '2026-09-17T00:00:00.000Z' }); + const one = privateSubmissionService.getAcceleratorProvider('accel-one'); + expect(one).toMatchObject({ name: 'Accelerator One', identity_key: rawPublicKey, health_status: 'unmeasured', status_endpoint: null, provider_signature: null, issuer: 'Universe operations', minimum_fee_sats: 1000, effective_from: '2020-01-01T00:00:00.000Z', expires_at: '2027-01-01T00:00:00.000Z' }); + expect(one.keys?.map(k => k.kid)).toEqual(['k1', 'k0']); + expect(() => privateSubmissionService.getAcceleratorProvider('nobody')).toThrow(expect.objectContaining({ code: 'provider-not-in-directory', status: 404 })); + }); + + it('reports the unconfigured directory rather than a directory of providers', () => { + useAcceleratorDirectory(null); + expect(() => privateSubmissionService.listAcceleratorProviders()).toThrow(expect.objectContaining({ code: 'unavailable-registry', status: 503 })); + expect(() => privateSubmissionService.listAcceleratorProviders()).toThrow(/UNIVERSE_ACCELERATOR_PROVIDER_DIRECTORY/); + const result = privateSubmissionService.verifyAcceleratorReceipt(receipt(), NOW); + expect(result).toMatchObject({ verified: false, stage: 'unavailable-trust' }); + }); + + it('verifies an authentic receipt with the directory key valid at issue and records it', () => { + const result = privateSubmissionService.verifyAcceleratorReceipt(receipt(), NOW); + expect(result).toMatchObject({ verified: true, stage: 'verified', errors: [], provider_id: 'accel-one', receipt_id: 'rcpt-1', key_id: 'k1', algorithm: 'ed25519', verified_at_utc: '2026-09-17T12:00:00.000Z', directory: { revision: 'r1' }, replay: { seen_count: 1 } }); + expect(result.scope).toMatch(/not that the acceleration was delivered/); + }); + + it('fails independently on an altered signature, txid, provider, network, expiry and terms', () => { + const cases: Array<[Partial, string, RegExp]> = [ + [receipt({ provider_signature: 'ab'.repeat(64) }), 'invalid', /does not verify/], + [{ ...receipt(), txid: 'ee'.repeat(32) }, 'invalid', /does not verify/], + [{ ...receipt(), provider_fee_sats: 1 }, 'invalid', /does not verify/], + [{ ...receipt(), provider_id: 'elsewhere' }, 'unavailable-trust', /not a directory key of provider elsewhere/], + [{ ...receipt(), provider_id: 'nobody' }, 'unavailable-trust', /not in the owned directory/], + [{ ...receipt(), key_id: 'k9' }, 'unavailable-trust', /not a directory key/], + [receipt({ key_id: 'k0' }), 'unavailable-trust', /was not valid at/], + [receipt({ network: 'other-network' }), 'wrong-network', /this deployment serves/], + [receipt({ expires_at_utc: '2026-09-17T11:30:00.000Z' }), 'expired', /expired at/], + [receipt({ schema_version: 'someone-elses-receipt-v3' }), 'unsupported', /no other receipt encoding/], + [{ ...receipt(), submitted_at_utc: undefined as unknown as string }, 'invalid', /submitted_at_utc/], + ]; + for (const [payload, stage, message] of cases) { + const result = privateSubmissionService.verifyAcceleratorReceipt(payload, NOW); + expect(result.verified).toBe(false); + expect(result.stage).toBe(stage); + expect(result.errors.join('\n')).toMatch(message); + } + }); + + it('flags a replay of a verified receipt as a duplicate', () => { + expect(privateSubmissionService.verifyAcceleratorReceipt(receipt({ receipt_id: 'rcpt-dup' }), NOW).stage).toBe('verified'); + const again = privateSubmissionService.verifyAcceleratorReceipt(receipt({ receipt_id: 'rcpt-dup' }), NOW + 1000); + expect(again).toMatchObject({ verified: false, stage: 'duplicate', replay: { first_verified_at_utc: '2026-09-17T12:00:00.000Z', seen_count: 2 } }); + expect(privateSubmissionService.verifyAcceleratorReceipt(receipt({ receipt_id: 'rcpt-other' }), NOW).stage).toBe('verified'); }); it('still names the faults in a receipt it can read', () => { @@ -50,28 +267,19 @@ describe('PrivateSubmissionService', () => { expect(malformed.stage).toBe('invalid'); expect(malformed.errors).toContain('provider_id is required'); expect(malformed.errors).toContain('Valid 32-byte txid is required'); + const nonhex = { provider_id: 'provider', receipt_id: 'receipt', provider_signature: 'ab'.repeat(64), txid: 'z'.repeat(64) }; + expect(privateSubmissionService.verifyAcceleratorReceipt(nonhex)).toMatchObject({ verified: false, stage: 'invalid' }); + expect(privateSubmissionService.verifyAcceleratorReceipt({ ...nonhex, txid: 'ab'.repeat(32) })).toMatchObject({ verified: false, stage: 'unsupported' }); }); - it('rejects a 64-character nonhex txid and never treats signature-shaped bytes as trusted', () => { - const receipt = { provider_id: 'provider', receipt_id: 'receipt', provider_signature: 'ab'.repeat(64), txid: 'z'.repeat(64) }; - expect(privateSubmissionService.verifyAcceleratorReceipt(receipt)).toMatchObject({ verified: false, stage: 'invalid' }); - expect(privateSubmissionService.verifyAcceleratorReceipt({ ...receipt, txid: 'ab'.repeat(32) })) - .toMatchObject({ verified: false, stage: 'unavailable-registry' }); - }); - - it('reports the missing ordering sensor rather than evidence for invented transactions', () => { - expect(() => privateSubmissionService.getTransactionOrdering('9b71d224bd62f3785d96d46ad3ea3d73319bfbc2770d3d5f7cc9a4744d91aafb')) - .toThrow(unavailable('unavailable-source')); - expect(() => privateSubmissionService.getBlockOrdering('00'.repeat(32))).toThrow(unavailable('unavailable-source')); - expect(() => privateSubmissionService.listOrderingFindings()).toThrow(unavailable('unavailable-source')); + it('uses the documented deterministic encoding', () => { + expect(deterministicJson({ b: 1, a: { d: [3, { z: 1, y: 2 }], c: 'x' }, u: undefined })).toBe('{"a":{"c":"x","d":[3,{"y":2,"z":1}]},"b":1}'); + expect(receiptSigningMessage({ b: 1 } as unknown as AcceleratorReceipt).toString('utf8')).toBe(ACCELERATOR_RECEIPT_SCHEMA + '\n{"b":1}'); }); - it('raises the typed error the routes map to a 503', () => { - try { - privateSubmissionService.listAcceleratorProviders(); - throw new Error('expected the registry read to refuse'); - } catch (err) { - expect(err).toBeInstanceOf(SubmissionEvidenceError); - } + it('refuses a malformed directory file', () => { + expect(() => parseAcceleratorDirectory('nope', 'f')).toThrow(/not JSON/); + expect(() => parseAcceleratorDirectory(JSON.stringify({ schema: 'other', revision: '1', providers: [] }), 'f')).toThrow(/must declare schema/); + expect(() => parseAcceleratorDirectory(JSON.stringify({ schema: 'universe-accelerator-directory-v1', revision: '1', providers: [{ id: 'p', name: 'P', networks: ['signet'], issuer: 'i', protocolVersion: 'v', keys: [{ kid: 'k', algorithm: 'rsa', publicKey: 'ab', validFrom: 'x' }] }] }), 'f')).toThrow(/supported algorithm/); }); }); diff --git a/backend/src/api/intelligence/private-submission/submission-input.test.ts b/backend/src/api/intelligence/private-submission/submission-input.test.ts index e80fbb9dbf1..037e1ae9d73 100644 --- a/backend/src/api/intelligence/private-submission/submission-input.test.ts +++ b/backend/src/api/intelligence/private-submission/submission-input.test.ts @@ -2,6 +2,8 @@ import express from 'express'; import { AddressInfo } from 'net'; import { Transaction } from 'bitcoinjs-lib'; import routes from './private-submission.routes'; +import { setPrivateRelayRuntime } from './private-relay.runtime'; +import privateSubmissionService from './private-submission.service'; import { diagnosisInput, submissionInput, @@ -43,7 +45,9 @@ describe('Private submission bounded inputs and real HTTP boundary', () => { 'bounds submission tokens', (token) => expect(() => tokenInput(token)).toThrow() ); - it('serves all12 actual HTTP operations as unavailable authorities and malformed diagnosis as400', async () => { + it('serves all12 actual HTTP operations with their exact source states and malformed diagnosis as400', async () => { + privateSubmissionService.diagnosisReaders = { mempoolEntry: () => undefined, mempoolInfo: () => null, replaces: () => undefined, replacedBy: () => undefined }; + setPrivateRelayRuntime({ network: 'signet', endpoints: { endpoints: [], issues: [], unconfigured: true }, store: null, worker: null, coreVersion: () => '?' }); const app = express(); app.use(express.json()); routes.initRoutes(app); @@ -53,18 +57,24 @@ describe('Private submission bounded inputs and real HTTP boundary', () => { 'http://127.0.0.1:' + (server.address() as AddressInfo).port + '/api/v1/intelligence'; - const requests: [string, string, any?][] = [ - ['GET', '/submission/overview'], - ['GET', '/submission/capabilities'], - ['POST', '/submission/diagnose', { txid: tx.getId() }], - ['POST', '/submission/private', { raw_tx: raw, method: 'public_p2p' }], - ['GET', '/submission/private/token'], - ['POST', '/submission/private/token/abort', {}], - ['GET', '/accelerators/providers'], - ['GET', '/accelerators/providers/provider'], + // Relay-backed routes: absent integration, 503. Owned-source routes: + // an unknown transaction or block is a 404, an empty observer answers + // its state, an unconfigured directory is a 503, an unsupported + // receipt encoding is a 400. Nothing answers success. + const requests: [string, string, number, string, any?][] = [ + ['GET', '/submission/overview', 503, 'durable-store-unavailable'], + ['GET', '/submission/capabilities', 503, 'durable-store-unavailable'], + ['POST', '/submission/diagnose', 404, 'transaction-not-in-mempool', { txid: tx.getId() }], + ['POST', '/submission/private', 503, 'durable-store-unavailable', { raw_tx: raw, method: 'privatebroadcast_tor' }], + ['GET', '/submission/private/token', 503, 'durable-store-unavailable'], + ['POST', '/submission/private/token/abort', 503, 'durable-store-unavailable', {}], + ['GET', '/accelerators/providers', 503, 'unavailable-registry'], + ['GET', '/accelerators/providers/provider', 503, 'unavailable-registry'], [ 'POST', '/accelerators/receipts/verify', + 400, + 'unsupported', { provider_id: 'p', receipt_id: 'r', @@ -72,12 +82,12 @@ describe('Private submission bounded inputs and real HTTP boundary', () => { provider_signature: 'untrusted', }, ], - ['GET', '/ordering/transactions/' + tx.getId()], - ['GET', '/ordering/blocks/' + 'ab'.repeat(32)], - ['GET', '/ordering/findings'], + ['GET', '/ordering/transactions/' + tx.getId(), 404, 'transaction-not-observed'], + ['GET', '/ordering/blocks/' + 'ab'.repeat(32), 404, 'block-not-observed'], + ['GET', '/ordering/findings', 200, 'no-blocks-observed'], ]; try { - for (const [method, url, body] of requests) { + for (const [method, url, status, stage, body] of requests) { const res = await fetch(base + url, { method, ...(body @@ -87,10 +97,11 @@ describe('Private submission bounded inputs and real HTTP boundary', () => { } : {}), }); - expect(res.status).toBe(503); + expect([url, res.status]).toEqual([url, status]); const v: any = await res.json(); - expect(v.stage).toContain('unavailable'); + expect(v.stage ?? v.state).toBe(stage); expect(v.verified).not.toBe(true); + expect(v.findings ?? []).toEqual([]); } const bad = await fetch(base + '/submission/diagnose', { method: 'POST', @@ -98,6 +109,7 @@ describe('Private submission bounded inputs and real HTTP boundary', () => { expect(bad.status).toBe(400); expect(((await bad.json()) as any).stage).toBe('invalid-input'); } finally { + setPrivateRelayRuntime(null); await new Promise((r) => server.close(() => r())); } }); diff --git a/backend/src/api/intelligence/private-submission/transaction-diagnosis.ts b/backend/src/api/intelligence/private-submission/transaction-diagnosis.ts new file mode 100644 index 00000000000..cbec8a0801f --- /dev/null +++ b/backend/src/api/intelligence/private-submission/transaction-diagnosis.ts @@ -0,0 +1,100 @@ +import { Transaction } from 'bitcoinjs-lib'; +import { IBitcoinApi } from '../../bitcoin/bitcoin-api.interface'; +import memPool from '../../mempool'; +import rbfCache from '../../rbf-cache'; +import { MempoolTransactionExtended } from '../../../mempool.interfaces'; +import { SubmissionDiagnosisResult, SubmissionMethod } from './private-submission.models'; + +/** + * Acceleration diagnosis from the owned mempool: the transaction's own + * entry (fee, size, ancestors, descendants, CPFP effect, RBF signalling, + * replacements this backend recorded) and the owned node's policy floor. + * + * A transaction this backend's mempool does not hold has no facts to + * diagnose, and says so; nothing is estimated from the txid. + */ + +export class DiagnosisNotFound extends Error { + constructor(public readonly txid: string) { + super('This backend\'s mempool holds no entry for that transaction; nothing about its fee, policy or package state is observed.'); + } +} + +export interface DiagnosisReaders { + mempoolEntry: (txid: string) => MempoolTransactionExtended | undefined; + mempoolInfo: () => Pick | null; + replaces: (txid: string) => string[] | undefined; + replacedBy: (txid: string) => string | undefined; +} + +export const ownedDiagnosisReaders: DiagnosisReaders = { + mempoolEntry: txid => memPool.getMempool()[txid], + mempoolInfo: () => memPool.getMempoolInfo(), + replaces: txid => rbfCache.getReplaces(txid), + replacedBy: txid => rbfCache.getReplacedBy(txid), +}; + +const BTC_PER_KVB_TO_SAT_PER_VB = 100_000; + +/** The txid of a raw transaction, or the input itself when it already is one. */ +export function txidOf(rawTxOrTxid: string): string { + if (/^[0-9a-f]{64}$/i.test(rawTxOrTxid)) { return rawTxOrTxid.toLowerCase(); } + return Transaction.fromHex(rawTxOrTxid).getId(); +} + +export function diagnoseFromOwnedMempool(rawTxOrTxid: string, readers: DiagnosisReaders = ownedDiagnosisReaders): SubmissionDiagnosisResult { + const txid = txidOf(rawTxOrTxid); + const entry = readers.mempoolEntry(txid); + if (!entry) { throw new DiagnosisNotFound(txid); } + const info = readers.mempoolInfo(); + const vsize = entry.vsize; + const feerate = vsize > 0 ? entry.fee / vsize : 0; + const effective = typeof entry.effectiveFeePerVsize === 'number' && Number.isFinite(entry.effectiveFeePerVsize) ? entry.effectiveFeePerVsize : feerate; + const ancestors = entry.ancestors ?? []; + const descendants = entry.descendants ?? []; + const minFloorSatVb = info && Number.isFinite(info.mempoolminfee) ? info.mempoolminfee * BTC_PER_KVB_TO_SAT_PER_VB : null; + const rbfSignalling = entry.vin.some(vin => typeof vin.sequence === 'number' && vin.sequence < 0xfffffffe); + const replaces = readers.replaces(txid) ?? []; + const replacedBy = readers.replacedBy(txid) ?? null; + const round = (value: number): number => Math.round(value * 100) / 100; + const methods: SubmissionMethod[] = ['public_p2p']; + return { + txid, + vsize, + feerate_sats_vb: round(feerate), + is_mempool_present: true, + is_policy_compliant: minFloorSatVb === null ? effective > 0 : effective >= minFloorSatVb, + rbf_eligible: rbfSignalling, + cpfp_eligible: descendants.length === 0 && (entry.bestDescendant === null || entry.bestDescendant === undefined), + has_conflicts: replaces.length > 0 || replacedBy !== null, + acceleration_recommended: entry.position ? entry.position.block > 0 : false, + privacy_advisory: 'Diagnosis is from this backend\'s own mempool entry and policy floor; no third party was consulted.', + available_methods: methods, + package: { + effective_feerate_sats_vb: round(effective), + ancestor_count: ancestors.length, + descendant_count: descendants.length, + ancestor_fee_sats: ancestors.reduce((sum, a) => sum + a.fee, 0), + ancestor_weight: ancestors.reduce((sum, a) => sum + a.weight, 0), + cpfp_boosted: effective > feerate + 1e-9, + cpfp_checked: entry.cpfpChecked === true, + best_descendant_txid: entry.bestDescendant?.txid ?? null, + projected_block_index: entry.position ? entry.position.block : null, + }, + policy: { + source: 'owned Core getmempoolinfo via this backend', + mempool_min_fee_sats_vb: minFloorSatVb === null ? null : round(minFloorSatVb), + min_relay_fee_sats_vb: info && Number.isFinite(info.minrelaytxfee) ? round(info.minrelaytxfee * BTC_PER_KVB_TO_SAT_PER_VB) : null, + mempool_loaded: info?.loaded ?? null, + }, + replacement: { + signals_rbf: rbfSignalling, + replaces_txids: replaces, + replaced_by_txid: replacedBy, + is_replacement: entry.replacement === true, + }, + observed_at_utc: new Date().toISOString(), + first_seen_utc: typeof entry.firstSeen === 'number' ? new Date(entry.firstSeen * 1000).toISOString() : null, + scope: 'Facts of one owned mempool entry at read time. Confirmation, propagation and the policy of other nodes are not measured here.', + }; +} diff --git a/backend/src/api/intelligence/query-studio/query-engine.ts b/backend/src/api/intelligence/query-studio/query-engine.ts new file mode 100644 index 00000000000..3e6ab29ff61 --- /dev/null +++ b/backend/src/api/intelligence/query-studio/query-engine.ts @@ -0,0 +1,215 @@ +import { createPool, Pool, PoolConnection } from 'mysql2/promise'; +import { QUERY_STUDIO_TABLES, RenderedQuery } from './query-grammar'; + +/** + * The analytics connection behind Query Studio. + * + * It is a separate, small pool opened from UNIVERSE_QUERY_ENGINE_DSN, which + * names a read-only account on the explorer database or a replica of it. + * The application's write pool is never used here, so a query that the + * grammar lets through still runs with the privileges of the read-only + * account. Every statement carries a deadline; when it passes, the + * connection is destroyed rather than returned, which is what makes the + * server abandon the work instead of finishing it for nobody. + */ + +export const QUERY_ENGINE_ENVIRONMENT_VARIABLE = 'UNIVERSE_QUERY_ENGINE_DSN'; + +export const QUERY_ENGINE_LIMITS = { + connections: 2, + statementDeadlineMs: 5_000, + connectTimeoutMs: 3_000, + maxRows: 1_000, + defaultRows: 100, + resultBytes: 1_048_576, +} as const; + +export interface EngineColumn { + name: string; + type: string; + nullable: boolean; + is_primary_key: boolean; +} + +export interface EngineTable { + table_name: string; + columns: EngineColumn[]; + indexes: string[]; +} + +export interface EngineRun { + rows: Array>; + columns: string[]; +} + +export interface QueryEngine { + readonly source: string; + describe(tables: string[]): Promise; + run(query: RenderedQuery, deadlineMs: number): Promise; +} + +export class QueryEngineError extends Error { + constructor(public readonly reason: 'unconfigured' | 'invalid-dsn' | 'unreachable' | 'deadline' | 'failed', message: string) { + super(message); + } +} + +export interface ParsedDsn { + host: string; + port: number; + user: string; + password: string; + database: string; + socketPath: string | null; +} + +/** mysql://user:password@host:port/database, or mysql://user:password@localhost/database?socket=/path for a unix socket. */ +export function parseQueryEngineDsn(dsn: string): ParsedDsn { + let url: URL; + try { + url = new URL(dsn); + } catch { + throw new QueryEngineError('invalid-dsn', 'UNIVERSE_QUERY_ENGINE_DSN is not a URL.'); + } + if (url.protocol !== 'mysql:' && url.protocol !== 'mariadb:') { + throw new QueryEngineError('invalid-dsn', 'UNIVERSE_QUERY_ENGINE_DSN must use the mysql:// scheme.'); + } + const database = decodeURIComponent(url.pathname.replace(/^\//, '')); + if (!database || database.includes('/')) { + throw new QueryEngineError('invalid-dsn', 'UNIVERSE_QUERY_ENGINE_DSN must name exactly one database.'); + } + if (!url.username) { + throw new QueryEngineError('invalid-dsn', 'UNIVERSE_QUERY_ENGINE_DSN must name the read-only account.'); + } + const socketPath = url.searchParams.get('socket'); + const port = url.port ? Number(url.port) : 3306; + if (!Number.isInteger(port) || port < 1 || port > 65535) { + throw new QueryEngineError('invalid-dsn', 'UNIVERSE_QUERY_ENGINE_DSN has an invalid port.'); + } + return { + host: url.hostname || '127.0.0.1', + port, + user: decodeURIComponent(url.username), + password: decodeURIComponent(url.password), + database, + socketPath: socketPath || null, + }; +} + +export class MysqlQueryEngine implements QueryEngine { + private pool: Pool | null = null; + public readonly source: string; + + constructor(private readonly dsn: ParsedDsn) { + this.source = dsn.socketPath ? `mysql://${dsn.user}@[socket]/${dsn.database}` : `mysql://${dsn.user}@${dsn.host}:${dsn.port}/${dsn.database}`; + } + + private getPool(): Pool { + if (!this.pool) { + this.pool = createPool({ + ...(this.dsn.socketPath ? { socketPath: this.dsn.socketPath } : { host: this.dsn.host, port: this.dsn.port }), + user: this.dsn.user, + password: this.dsn.password, + database: this.dsn.database, + connectionLimit: QUERY_ENGINE_LIMITS.connections, + connectTimeout: QUERY_ENGINE_LIMITS.connectTimeoutMs, + // Exact values: BIGINT and DECIMAL arrive as strings, never rounded doubles. + supportBigNumbers: true, + bigNumberStrings: true, + decimalNumbers: false, + dateStrings: true, + timezone: '+00:00', + multipleStatements: false, + }); + } + return this.pool; + } + + /** @asyncUnsafe The service turns a rejection into a typed unavailable state. */ + private async connection(): Promise { + try { + return await this.getPool().getConnection(); + } catch (e) { + throw new QueryEngineError('unreachable', 'The analytics replica did not accept a connection: ' + (e instanceof Error ? e.message : String(e))); + } + } + + /** @asyncUnsafe The service turns a rejection into a typed unavailable state. */ + public async describe(tables: string[]): Promise { + const connection = await this.connection(); + try { + const [columns] = await connection.query( + 'SELECT TABLE_NAME, COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_KEY FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME IN (?) ORDER BY TABLE_NAME, ORDINAL_POSITION', + [tables], + ); + const [indexes] = await connection.query( + 'SELECT DISTINCT TABLE_NAME, INDEX_NAME FROM information_schema.STATISTICS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME IN (?) ORDER BY TABLE_NAME, INDEX_NAME', + [tables], + ); + const byTable = new Map(); + for (const row of columns) { + const name = String(row.TABLE_NAME); + const allowed = QUERY_STUDIO_TABLES[name]; + if (!allowed || !allowed.columns.includes(String(row.COLUMN_NAME))) { continue; } + const table = byTable.get(name) ?? { table_name: name, columns: [], indexes: [] }; + table.columns.push({ + name: String(row.COLUMN_NAME), + type: String(row.COLUMN_TYPE), + nullable: String(row.IS_NULLABLE).toUpperCase() === 'YES', + is_primary_key: String(row.COLUMN_KEY).toUpperCase() === 'PRI', + }); + byTable.set(name, table); + } + for (const row of indexes) { + byTable.get(String(row.TABLE_NAME))?.indexes.push(String(row.INDEX_NAME)); + } + return [...byTable.values()]; + } catch (e) { + if (e instanceof QueryEngineError) { throw e; } + throw new QueryEngineError('failed', 'Schema discovery failed: ' + (e instanceof Error ? e.message : String(e))); + } finally { + connection.release(); + } + } + + /** @asyncUnsafe The service turns a rejection into a typed unavailable state. */ + public async run(query: RenderedQuery, deadlineMs: number): Promise { + const connection = await this.connection(); + let timer: NodeJS.Timeout | undefined; + let expired = false; + const deadline = new Promise((_, reject) => { + timer = setTimeout(() => { + expired = true; + // Destroying the socket is what cancels the statement on the server. + connection.destroy(); + reject(new QueryEngineError('deadline', `The query did not finish within ${deadlineMs} ms and was cancelled.`)); + }, deadlineMs); + }); + try { + const hinted = query.sql.replace(/^SELECT /, `SELECT /*+ MAX_EXECUTION_TIME(${deadlineMs}) */ `); + const [rows] = await Promise.race([connection.query(hinted, query.params), deadline]); + return { rows: rows.map(row => ({ ...row })), columns: query.columns }; + } catch (e) { + if (e instanceof QueryEngineError) { throw e; } + throw new QueryEngineError('failed', 'The analytics replica rejected the query: ' + (e instanceof Error ? e.message : String(e))); + } finally { + if (timer) { clearTimeout(timer); } + if (!expired) { connection.release(); } + } + } +} + +let configured: QueryEngine | null | undefined; + +/** The engine named by the environment, or null when nothing is configured. */ +export function queryEngineFromEnvironment(environment: Record = process.env): QueryEngine | null { + if (configured !== undefined) { return configured; } + const dsn = String(environment[QUERY_ENGINE_ENVIRONMENT_VARIABLE] ?? '').trim(); + configured = dsn ? new MysqlQueryEngine(parseQueryEngineDsn(dsn)) : null; + return configured; +} + +/** Test seam. */ +export function useQueryEngine(engine: QueryEngine | null | undefined): void { + configured = engine; +} diff --git a/backend/src/api/intelligence/query-studio/query-grammar.test.ts b/backend/src/api/intelligence/query-studio/query-grammar.test.ts new file mode 100644 index 00000000000..c0c65a3e6ef --- /dev/null +++ b/backend/src/api/intelligence/query-studio/query-grammar.test.ts @@ -0,0 +1,86 @@ +import { parseSelect, QUERY_GRAMMAR_LIMITS, QueryGrammarError, renderSelect } from './query-grammar'; + +const render = (sql: string, maxRows = 100, offset = 0) => renderSelect(parseSelect(sql), maxRows, offset); + +describe('Query Studio grammar', () => { + it('renders a bounded SELECT with quoted identifiers and parameterised literals', () => { + const rendered = render('SELECT height, hash, fees FROM blocks WHERE height > 800000 AND pool_id IN (1, 2) ORDER BY height DESC LIMIT 5'); + expect(rendered.sql).toBe('SELECT `height` AS `height`, `hash` AS `hash`, `fees` AS `fees` FROM `blocks` WHERE (`height` > ? AND `pool_id` IN (?, ?)) ORDER BY `height` DESC LIMIT 6'); + expect(rendered.params).toEqual([800000, 1, 2]); + expect(rendered.columns).toEqual(['height', 'hash', 'fees']); + }); + + it('caps the LIMIT at the row budget, applies the cursor offset and expands *', () => { + const rendered = render('SELECT * FROM pools LIMIT 5000', 100, 300); + expect(rendered.sql).toBe('SELECT `id`, `name`, `link`, `unique_id` FROM `pools` LIMIT 101 OFFSET 300'); + expect(render('SELECT id FROM pools OFFSET 10', 10, 5).sql).toContain('LIMIT 11 OFFSET 15'); + }); + + it('accepts aggregates with GROUP BY and rejects an unaggregated column', () => { + const rendered = render('SELECT pool_id, COUNT(*) AS mined, SUM(fees) FROM blocks GROUP BY pool_id ORDER BY pool_id'); + expect(rendered.sql).toBe('SELECT `pool_id` AS `pool_id`, COUNT(*) AS `mined`, SUM(`fees`) AS `sum_fees` FROM `blocks` GROUP BY `pool_id` ORDER BY `pool_id` ASC LIMIT 101'); + expect(() => parseSelect('SELECT height, COUNT(*) FROM blocks GROUP BY pool_id')).toThrow(/must be aggregated/); + }); + + it('treats keywords inside string literals as data', () => { + const rendered = render("SELECT name FROM pools WHERE name = 'DROP TABLE blocks; -- ''quoted'' /* comment */'"); + expect(rendered.sql).toBe('SELECT `name` AS `name` FROM `pools` WHERE `name` = ? LIMIT 101'); + expect(rendered.params).toEqual(["DROP TABLE blocks; -- 'quoted' /* comment */"]); + }); + + it.each([ + ['DROP TABLE blocks', /Expected SELECT/], + ['DELETE FROM blocks WHERE 1=1', /Expected SELECT/], + ['UPDATE blocks SET fees = 0', /Expected SELECT/], + ['INSERT INTO pools (name) VALUES (\'x\')', /Expected SELECT/], + ['SELECT * FROM blocks; DROP TABLE blocks', /Only one statement/], + ['SELECT * FROM blocks -- comment', /Comments/], + ['SELECT /* hint */ * FROM blocks', /Comments/], + ['SELECT * FROM blocks # comment', /Comments/], + ['SELECT * FROM intelligence_api_keys', /FROM must name one readable table/], + ['SELECT key_hash FROM blocks', /not readable/], + ['SELECT coinbase_raw FROM blocks', /not readable/], + ['SELECT b.height FROM blocks b', /Unknown table qualifier/], + ['SELECT height FROM blocks JOIN pools ON pools.id = blocks.pool_id', /aliases, joins/], + ['SELECT height FROM blocks WHERE pool_id IN (SELECT id FROM pools)', /Expected a literal/], + ['SELECT SLEEP(10) FROM blocks', /not readable/], + ['SELECT LOAD_FILE(\'/etc/passwd\') FROM blocks', /not readable/], + ['SELECT height FROM blocks WHERE hash = @@version', /Unexpected character/], + ['SELECT height FROM blocks INTO OUTFILE \'/tmp/x\'', /aliases, joins/], + ['SELECT height FROM blocks UNION SELECT id FROM pools', /aliases, joins/], + ['SELECT height FROM blocks WHERE hash = \'unterminated', /Unterminated string/], + ['SELECT height FROM blocks WHERE hash = \'a\\\'b\'', /Backslash/], + ['SELECT height FROM blocks LIMIT -1', /Unexpected character/], + ['SELECT height FROM blocks LIMIT 1.5', /LIMIT needs/], + ['SELECT height FROM blocks WHERE height = 99999999999999999999', /safe range/], + ['', /sql is required/], + ['SELECT height FROM blocks WHERE hash = \'x\u0000\'', /Control characters/], + ])('rejects %s', (sql, expected) => { + expect(() => parseSelect(sql)).toThrow(expected); + expect(() => parseSelect(sql)).toThrow(QueryGrammarError); + }); + + it('bounds the size of a query', () => { + expect(() => parseSelect('SELECT height FROM blocks WHERE ' + 'height = 1 OR '.repeat(600) + 'height = 2')).toThrow(/limited to 4096 characters/); + const many = 'SELECT height FROM blocks WHERE pool_id IN (' + Array.from({ length: QUERY_GRAMMAR_LIMITS.inListValues + 1 }, (_, i) => i).join(',') + ')'; + expect(() => parseSelect(many)).toThrow(/IN lists are limited/); + expect(() => parseSelect("SELECT height FROM blocks WHERE hash = '" + 'x'.repeat(QUERY_GRAMMAR_LIMITS.stringLength + 1) + "'")).toThrow(/String literals are limited/); + expect(() => parseSelect('SELECT height FROM blocks WHERE ' + '('.repeat(9) + 'height = 1' + ')'.repeat(9))).toThrow(/nests too deeply/); + }); + + it('reports the position of the offending token', () => { + try { + parseSelect('SELECT height FROM blocks WHERE hash = \'a\'; DELETE FROM blocks'); + throw new Error('accepted'); + } catch (e) { + expect(e).toBeInstanceOf(QueryGrammarError); + expect((e as QueryGrammarError).position).toBe(42); + } + }); + + it('accepts NULL tests, BETWEEN, LIKE, NOT and a trailing semicolon', () => { + const rendered = render("SELECT DISTINCT pool_id FROM blocks WHERE NOT (pool_id IS NULL) AND height BETWEEN 1 AND 10 AND hash NOT LIKE '00%' AND `hash` <> 'x';"); + expect(rendered.sql).toBe('SELECT DISTINCT `pool_id` AS `pool_id` FROM `blocks` WHERE (NOT (`pool_id` IS NULL) AND `height` BETWEEN ? AND ? AND `hash` NOT LIKE ? AND `hash` <> ?) LIMIT 101'); + expect(rendered.params).toEqual([1, 10, '00%', 'x']); + }); +}); diff --git a/backend/src/api/intelligence/query-studio/query-grammar.ts b/backend/src/api/intelligence/query-studio/query-grammar.ts new file mode 100644 index 00000000000..b08a3bcd7e3 --- /dev/null +++ b/backend/src/api/intelligence/query-studio/query-grammar.ts @@ -0,0 +1,557 @@ +/** + * The bounded read-only query language of Query Studio. + * + * A caller's SQL never reaches the engine as text. It is tokenised, parsed + * into a small tree, checked against an explicit allowlist of tables and + * columns, and rendered again with quoted identifiers and parameter + * placeholders for every literal. What the engine receives is therefore a + * statement this file composed, and the keywords it understands are the + * complete list below. Everything else (a second statement, a comment, a + * subquery, a join, a function that is not an aggregate, a write, DDL, a + * variable, an identifier outside the allowlist) is a parse error before any + * connection is touched. A keyword inside a string literal is data. + */ + +export class QueryGrammarError extends Error { + constructor(message: string, public readonly position: number | null = null) { + super(message); + } +} + +export const QUERY_GRAMMAR_LIMITS = { + sqlLength: 4096, + tokens: 512, + selectItems: 64, + inListValues: 128, + stringLength: 512, + identifierLength: 64, + predicateDepth: 8, +} as const; + +/** Columns of the explorer tables Query Studio may read. Names are the migration's. */ +export interface AllowedTable { + description: string; + columns: readonly string[]; +} + +export const QUERY_STUDIO_TABLES: Readonly> = Object.freeze({ + blocks: { + description: 'One row per indexed block: header fields, size, weight, fee totals and pool attribution.', + columns: [ + 'height', 'hash', 'blockTimestamp', 'size', 'weight', 'tx_count', 'difficulty', 'pool_id', 'fees', 'median_fee', + 'reward', 'version', 'bits', 'nonce', 'merkle_root', 'previous_block_hash', 'median_timestamp', 'coinbase_address', + 'avg_tx_size', 'total_inputs', 'total_outputs', 'total_output_amt', 'median_fee_amt', 'segwit_total_txs', + 'segwit_total_size', 'segwit_total_weight', 'utxoset_change', 'utxoset_size', 'total_input_amt', + ], + }, + pools: { + description: 'Mining pool definitions the block attribution uses.', + columns: ['id', 'name', 'link', 'unique_id'], + }, + hashrates: { + description: 'Indexed hashrate samples, network-wide (pool_id NULL) and per pool.', + columns: ['id', 'hashrate_timestamp', 'avg_hashrate', 'pool_id', 'share', 'type'], + }, + difficulty_adjustments: { + description: 'Every difficulty retarget with the height it took effect at.', + columns: ['time', 'height', 'difficulty', 'adjustment'], + }, + prices: { + description: 'Fiat price samples recorded by the owned price updater.', + columns: ['id', 'time', 'USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'], + }, + statistics: { + description: 'Mempool statistics samples: transaction counts, throughput and the fee-band vsize histogram.', + columns: [ + 'id', 'added', 'unconfirmed_transactions', 'tx_per_second', 'vbytes_per_second', 'mempool_byte_weight', 'total_fee', + 'vsize_0', 'vsize_1', 'vsize_2', 'vsize_3', 'vsize_4', 'vsize_5', 'vsize_6', 'vsize_8', 'vsize_10', 'vsize_12', + 'vsize_15', 'vsize_20', 'vsize_30', 'vsize_40', 'vsize_50', 'vsize_60', 'vsize_70', 'vsize_80', 'vsize_90', + 'vsize_100', 'vsize_125', 'vsize_150', 'vsize_175', 'vsize_200', 'vsize_250', 'vsize_300', 'vsize_350', + 'vsize_400', 'vsize_500', 'vsize_600', 'vsize_700', 'vsize_800', 'vsize_900', 'vsize_1000', 'vsize_1200', + 'vsize_1400', 'vsize_1600', 'vsize_1800', 'vsize_2000', + ], + }, + blocks_audits: { + description: 'Block audit summaries: how closely each mined block matched the projected template.', + columns: ['time', 'hash', 'height', 'match_rate', 'expected_fees', 'expected_weight', 'version'], + }, +}); + +const AGGREGATES = ['COUNT', 'SUM', 'AVG', 'MIN', 'MAX'] as const; +type Aggregate = (typeof AGGREGATES)[number]; + +const KEYWORDS = new Set([ + 'SELECT', 'DISTINCT', 'FROM', 'WHERE', 'GROUP', 'BY', 'ORDER', 'ASC', 'DESC', 'LIMIT', 'OFFSET', 'AS', 'AND', 'OR', + 'NOT', 'IS', 'NULL', 'IN', 'BETWEEN', 'LIKE', 'TRUE', 'FALSE', ...AGGREGATES, +]); + +type Token = + | { kind: 'keyword'; value: string; at: number } + | { kind: 'identifier'; value: string; at: number } + | { kind: 'number'; value: string; at: number } + | { kind: 'string'; value: string; at: number } + | { kind: 'symbol'; value: string; at: number } + | { kind: 'end'; value: ''; at: number }; + +const SYMBOLS = ['<=', '>=', '<>', '!=', '=', '<', '>', '(', ')', ',', '*', '.']; + +function tokenize(sql: string): Token[] { + const tokens: Token[] = []; + let i = 0; + while (i < sql.length) { + const ch = sql[i]; + if (/\s/.test(ch)) { i++; continue; } + if (tokens.length >= QUERY_GRAMMAR_LIMITS.tokens) { + throw new QueryGrammarError(`The query has more than ${QUERY_GRAMMAR_LIMITS.tokens} tokens.`, i); + } + if (ch === '-' && sql[i + 1] === '-' || ch === '#' || ch === '/' && sql[i + 1] === '*') { + throw new QueryGrammarError('Comments are not part of the query language.', i); + } + if (ch === ';') { + // Only a single trailing semicolon is tolerated. + if (sql.slice(i + 1).trim().length > 0) { + throw new QueryGrammarError('Only one statement is accepted.', i); + } + i++; + continue; + } + if (ch === '\'') { + let j = i + 1; + let value = ''; + let closed = false; + while (j < sql.length) { + if (sql[j] === '\'') { + if (sql[j + 1] === '\'') { value += '\''; j += 2; continue; } + closed = true; + j++; + break; + } + if (sql[j] === '\\') { + throw new QueryGrammarError('Backslash escapes are not accepted in string literals; double the quote instead.', j); + } + value += sql[j]; + j++; + } + if (!closed) { throw new QueryGrammarError('Unterminated string literal.', i); } + if (value.length > QUERY_GRAMMAR_LIMITS.stringLength) { + throw new QueryGrammarError(`String literals are limited to ${QUERY_GRAMMAR_LIMITS.stringLength} characters.`, i); + } + tokens.push({ kind: 'string', value, at: i }); + i = j; + continue; + } + if (ch === '`') { + const end = sql.indexOf('`', i + 1); + if (end < 0) { throw new QueryGrammarError('Unterminated quoted identifier.', i); } + const value = sql.slice(i + 1, end); + if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(value) || value.length > QUERY_GRAMMAR_LIMITS.identifierLength) { + throw new QueryGrammarError('Quoted identifiers may only contain letters, digits and underscores.', i); + } + tokens.push({ kind: 'identifier', value, at: i }); + i = end + 1; + continue; + } + const number = /^\d+(\.\d+)?/.exec(sql.slice(i)); + if (number) { + if (/[A-Za-z_]/.test(sql[i + number[0].length] ?? '')) { + throw new QueryGrammarError('Unexpected character after a number.', i); + } + tokens.push({ kind: 'number', value: number[0], at: i }); + i += number[0].length; + continue; + } + const word = /^[A-Za-z_][A-Za-z0-9_]*/.exec(sql.slice(i)); + if (word) { + if (word[0].length > QUERY_GRAMMAR_LIMITS.identifierLength) { + throw new QueryGrammarError('Identifier too long.', i); + } + const upper = word[0].toUpperCase(); + tokens.push(KEYWORDS.has(upper) ? { kind: 'keyword', value: upper, at: i } : { kind: 'identifier', value: word[0], at: i }); + i += word[0].length; + continue; + } + const symbol = SYMBOLS.find(candidate => sql.startsWith(candidate, i)); + if (symbol) { + tokens.push({ kind: 'symbol', value: symbol, at: i }); + i += symbol.length; + continue; + } + throw new QueryGrammarError(`Unexpected character ${JSON.stringify(ch)}.`, i); + } + tokens.push({ kind: 'end', value: '', at: sql.length }); + return tokens; +} + +type Literal = { kind: 'literal'; value: string | number | boolean | null }; +type ColumnRef = { kind: 'column'; name: string }; +type AggregateCall = { kind: 'aggregate'; fn: Aggregate; column: string | '*' }; +type Operand = Literal | ColumnRef | AggregateCall; + +type Predicate = + | { kind: 'compare'; left: Operand; op: string; right: Operand } + | { kind: 'null'; operand: Operand; negated: boolean } + | { kind: 'in'; operand: Operand; values: Literal[]; negated: boolean } + | { kind: 'between'; operand: Operand; low: Literal; high: Literal } + | { kind: 'like'; operand: Operand; pattern: Literal; negated: boolean } + | { kind: 'not'; inner: Predicate } + | { kind: 'and'; parts: Predicate[] } + | { kind: 'or'; parts: Predicate[] }; + +export interface ParsedSelect { + table: string; + distinct: boolean; + items: Array<{ expr: ColumnRef | AggregateCall; alias: string | null }> | '*'; + where: Predicate | null; + groupBy: string[]; + orderBy: Array<{ column: string; direction: 'ASC' | 'DESC' }>; + limit: number | null; + offset: number | null; +} + +/** A statement the engine can run: placeholders for every literal, quoted identifiers for every name. */ +export interface RenderedQuery { + sql: string; + params: Array; + table: string; + columns: string[]; +} + +class Parser { + private index = 0; + private depth = 0; + private table = ''; + + constructor(private readonly tokens: Token[]) {} + + private peek(offset = 0): Token { return this.tokens[Math.min(this.index + offset, this.tokens.length - 1)]; } + private next(): Token { return this.tokens[this.index++]; } + + private isKeyword(value: string, offset = 0): boolean { + const token = this.peek(offset); + return token.kind === 'keyword' && token.value === value; + } + + private isSymbol(value: string, offset = 0): boolean { + const token = this.peek(offset); + return token.kind === 'symbol' && token.value === value; + } + + private expectKeyword(value: string): void { + const token = this.next(); + if (token.kind !== 'keyword' || token.value !== value) { + throw new QueryGrammarError(`Expected ${value}.`, token.at); + } + } + + private expectSymbol(value: string): void { + const token = this.next(); + if (token.kind !== 'symbol' || token.value !== value) { + throw new QueryGrammarError(`Expected ${JSON.stringify(value)}.`, token.at); + } + } + + private identifier(): { value: string; at: number } { + const token = this.next(); + if (token.kind !== 'identifier') { + throw new QueryGrammarError('Expected an identifier.', token.at); + } + return { value: token.value, at: token.at }; + } + + private column(): ColumnRef { + const first = this.identifier(); + let name = first.value; + if (this.isSymbol('.')) { + // table.column is accepted only for the one table in the query. + if (first.value !== this.table) { + throw new QueryGrammarError(`Unknown table qualifier ${first.value}.`, first.at); + } + this.next(); + name = this.identifier().value; + } + if (!QUERY_STUDIO_TABLES[this.table].columns.includes(name)) { + throw new QueryGrammarError(`Column ${name} is not readable on ${this.table}.`, first.at); + } + return { kind: 'column', name }; + } + + private literal(): Literal { + const token = this.next(); + if (token.kind === 'string') { return { kind: 'literal', value: token.value }; } + if (token.kind === 'number') { return { kind: 'literal', value: token.value.includes('.') ? Number(token.value) : safeInteger(token.value, token.at) }; } + if (token.kind === 'symbol' && token.value === '-' ) { throw new QueryGrammarError('Negative literals are written as 0 - n.', token.at); } + if (token.kind === 'keyword') { + if (token.value === 'NULL') { return { kind: 'literal', value: null }; } + if (token.value === 'TRUE') { return { kind: 'literal', value: true }; } + if (token.value === 'FALSE') { return { kind: 'literal', value: false }; } + } + throw new QueryGrammarError('Expected a literal value.', token.at); + } + + private aggregate(): AggregateCall { + const fn = this.next().value as Aggregate; + this.expectSymbol('('); + let column: string | '*'; + if (this.isSymbol('*')) { + if (fn !== 'COUNT') { throw new QueryGrammarError(`${fn}(*) is not accepted; name a column.`, this.peek().at); } + this.next(); + column = '*'; + } else { + column = this.column().name; + } + this.expectSymbol(')'); + return { kind: 'aggregate', fn, column }; + } + + private operand(): Operand { + const token = this.peek(); + if (token.kind === 'keyword' && (AGGREGATES as readonly string[]).includes(token.value)) { return this.aggregate(); } + if (token.kind === 'identifier') { return this.column(); } + return this.literal(); + } + + private predicateOr(): Predicate { + if (++this.depth > QUERY_GRAMMAR_LIMITS.predicateDepth) { + throw new QueryGrammarError('The WHERE clause nests too deeply.', this.peek().at); + } + try { + const parts = [this.predicateAnd()]; + while (this.isKeyword('OR')) { this.next(); parts.push(this.predicateAnd()); } + return parts.length === 1 ? parts[0] : { kind: 'or', parts }; + } finally { + this.depth--; + } + } + + private predicateAnd(): Predicate { + const parts = [this.predicateNot()]; + while (this.isKeyword('AND')) { this.next(); parts.push(this.predicateNot()); } + return parts.length === 1 ? parts[0] : { kind: 'and', parts }; + } + + private predicateNot(): Predicate { + if (this.isKeyword('NOT')) { this.next(); return { kind: 'not', inner: this.predicateNot() }; } + if (this.isSymbol('(')) { + this.next(); + const inner = this.predicateOr(); + this.expectSymbol(')'); + return inner; + } + return this.comparison(); + } + + private comparison(): Predicate { + const left = this.operand(); + const token = this.peek(); + if (token.kind === 'symbol' && ['=', '!=', '<>', '<', '<=', '>', '>='].includes(token.value)) { + this.next(); + return { kind: 'compare', left, op: token.value === '!=' ? '<>' : token.value, right: this.operand() }; + } + if (this.isKeyword('IS')) { + this.next(); + let negated = false; + if (this.isKeyword('NOT')) { this.next(); negated = true; } + this.expectKeyword('NULL'); + return { kind: 'null', operand: left, negated }; + } + let negated = false; + if (this.isKeyword('NOT')) { this.next(); negated = true; } + if (this.isKeyword('IN')) { + this.next(); + this.expectSymbol('('); + const values: Literal[] = [this.literal()]; + while (this.isSymbol(',')) { + this.next(); + if (values.length >= QUERY_GRAMMAR_LIMITS.inListValues) { + throw new QueryGrammarError(`IN lists are limited to ${QUERY_GRAMMAR_LIMITS.inListValues} values.`, this.peek().at); + } + values.push(this.literal()); + } + this.expectSymbol(')'); + return { kind: 'in', operand: left, values, negated }; + } + if (this.isKeyword('LIKE')) { + this.next(); + const pattern = this.literal(); + if (typeof pattern.value !== 'string') { throw new QueryGrammarError('LIKE needs a string pattern.', token.at); } + return { kind: 'like', operand: left, pattern, negated }; + } + if (this.isKeyword('BETWEEN')) { + if (negated) { throw new QueryGrammarError('NOT BETWEEN is not accepted; use two comparisons.', token.at); } + this.next(); + const low = this.literal(); + this.expectKeyword('AND'); + const high = this.literal(); + return { kind: 'between', operand: left, low, high }; + } + throw new QueryGrammarError('Expected a comparison.', token.at); + } + + private bound(keyword: 'LIMIT' | 'OFFSET'): number { + const token = this.next(); + if (token.kind !== 'number' || token.value.includes('.')) { + throw new QueryGrammarError(`${keyword} needs a non-negative integer.`, token.at); + } + return safeInteger(token.value, token.at); + } + + public statement(): ParsedSelect { + this.expectKeyword('SELECT'); + let distinct = false; + if (this.isKeyword('DISTINCT')) { this.next(); distinct = true; } + + // The table is needed before the select list can be checked, so find FROM first. + let fromIndex = -1; + for (let i = this.index; i < this.tokens.length; i++) { + const token = this.tokens[i]; + if (token.kind === 'keyword' && token.value === 'FROM') { fromIndex = i; break; } + } + if (fromIndex < 0) { throw new QueryGrammarError('A query needs FROM and one of the readable tables.', this.peek().at); } + const tableToken = this.tokens[fromIndex + 1]; + if (!tableToken || tableToken.kind !== 'identifier' || !Object.prototype.hasOwnProperty.call(QUERY_STUDIO_TABLES, tableToken.value)) { + throw new QueryGrammarError( + `FROM must name one readable table: ${Object.keys(QUERY_STUDIO_TABLES).join(', ')}.`, + tableToken ? tableToken.at : this.peek().at, + ); + } + this.table = tableToken.value; + + let items: ParsedSelect['items']; + if (this.isSymbol('*')) { + this.next(); + items = '*'; + } else { + items = []; + for (;;) { + if (items.length >= QUERY_GRAMMAR_LIMITS.selectItems) { + throw new QueryGrammarError(`Select lists are limited to ${QUERY_GRAMMAR_LIMITS.selectItems} items.`, this.peek().at); + } + const expr = this.operand(); + if (expr.kind === 'literal') { throw new QueryGrammarError('Select items must be columns or aggregates.', this.peek(-1).at); } + let alias: string | null = null; + if (this.isKeyword('AS')) { this.next(); alias = this.identifier().value; } + items.push({ expr, alias }); + if (this.isSymbol(',')) { this.next(); continue; } + break; + } + } + this.expectKeyword('FROM'); + this.next(); // the table, checked above + if (this.isSymbol('.') || this.isKeyword('AS') || this.peek().kind === 'identifier') { + throw new QueryGrammarError('Table aliases, joins and schema qualifiers are not accepted.', this.peek().at); + } + + let where: Predicate | null = null; + if (this.isKeyword('WHERE')) { this.next(); where = this.predicateOr(); } + + const groupBy: string[] = []; + if (this.isKeyword('GROUP')) { + this.next(); + this.expectKeyword('BY'); + groupBy.push(this.column().name); + while (this.isSymbol(',')) { this.next(); groupBy.push(this.column().name); } + } + + const orderBy: ParsedSelect['orderBy'] = []; + if (this.isKeyword('ORDER')) { + this.next(); + this.expectKeyword('BY'); + for (;;) { + const column = this.column().name; + let direction: 'ASC' | 'DESC' = 'ASC'; + if (this.isKeyword('ASC') || this.isKeyword('DESC')) { direction = this.next().value as 'ASC' | 'DESC'; } + orderBy.push({ column, direction }); + if (this.isSymbol(',')) { this.next(); continue; } + break; + } + } + + let limit: number | null = null; + let offset: number | null = null; + if (this.isKeyword('LIMIT')) { this.next(); limit = this.bound('LIMIT'); } + if (this.isKeyword('OFFSET')) { this.next(); offset = this.bound('OFFSET'); } + + const end = this.next(); + if (end.kind !== 'end') { throw new QueryGrammarError('Unexpected input after the end of the query.', end.at); } + + if (groupBy.length && items !== '*') { + for (const item of items) { + if (item.expr.kind === 'column' && !groupBy.includes(item.expr.name)) { + throw new QueryGrammarError(`Column ${item.expr.name} must be aggregated or listed in GROUP BY.`, 0); + } + } + } + if (groupBy.length && items === '*') { + throw new QueryGrammarError('SELECT * cannot be combined with GROUP BY.', 0); + } + return { table: this.table, distinct, items, where, groupBy, orderBy, limit, offset }; + } +} + +function safeInteger(text: string, at: number): number { + const value = Number(text); + if (!Number.isSafeInteger(value)) { throw new QueryGrammarError('Integer literal is outside the safe range.', at); } + return value; +} + +export function parseSelect(sql: string): ParsedSelect { + if (typeof sql !== 'string' || sql.trim().length === 0) { throw new QueryGrammarError('sql is required.', 0); } + if (sql.length > QUERY_GRAMMAR_LIMITS.sqlLength) { + throw new QueryGrammarError(`Queries are limited to ${QUERY_GRAMMAR_LIMITS.sqlLength} characters.`, QUERY_GRAMMAR_LIMITS.sqlLength); + } + for (let i = 0; i < sql.length; i++) { + const code = sql.charCodeAt(i); + if ((code < 32 && code !== 9 && code !== 10 && code !== 13) || code === 127) { throw new QueryGrammarError('Control characters are not accepted.', i); } + } + return new Parser(tokenize(sql)).statement(); +} + +const quote = (identifier: string): string => '`' + identifier + '`'; + +/** + * Renders the tree back to SQL. `maxRows` replaces or caps the caller's LIMIT + * (one extra row is requested so truncation is detected) and `offset` is the + * cursor position. + */ +export function renderSelect(parsed: ParsedSelect, maxRows: number, offset: number): RenderedQuery { + const params: RenderedQuery['params'] = []; + const operand = (value: Operand): string => { + if (value.kind === 'column') { return quote(value.name); } + if (value.kind === 'aggregate') { return `${value.fn}(${value.column === '*' ? '*' : quote(value.column)})`; } + params.push(value.value); + return '?'; + }; + const predicate = (node: Predicate): string => { + switch (node.kind) { + case 'compare': return `${operand(node.left)} ${node.op} ${operand(node.right)}`; + case 'null': return `${operand(node.operand)} IS ${node.negated ? 'NOT ' : ''}NULL`; + case 'in': return `${operand(node.operand)} ${node.negated ? 'NOT ' : ''}IN (${node.values.map(operand).join(', ')})`; + case 'between': return `${operand(node.operand)} BETWEEN ${operand(node.low)} AND ${operand(node.high)}`; + case 'like': return `${operand(node.operand)} ${node.negated ? 'NOT ' : ''}LIKE ${operand(node.pattern)}`; + case 'not': return `NOT (${predicate(node.inner)})`; + case 'and': return '(' + node.parts.map(predicate).join(' AND ') + ')'; + case 'or': return '(' + node.parts.map(predicate).join(' OR ') + ')'; + } + }; + const columns: string[] = []; + let selectList: string; + if (parsed.items === '*') { + columns.push(...QUERY_STUDIO_TABLES[parsed.table].columns); + selectList = columns.map(quote).join(', '); + } else { + selectList = parsed.items.map(item => { + const label = item.alias ?? (item.expr.kind === 'column' ? item.expr.name : `${item.expr.fn.toLowerCase()}_${item.expr.column === '*' ? 'all' : item.expr.column}`); + columns.push(label); + return `${operand(item.expr)} AS ${quote(label)}`; + }).join(', '); + } + let sql = `SELECT ${parsed.distinct ? 'DISTINCT ' : ''}${selectList} FROM ${quote(parsed.table)}`; + if (parsed.where) { sql += ` WHERE ${predicate(parsed.where)}`; } + if (parsed.groupBy.length) { sql += ` GROUP BY ${parsed.groupBy.map(quote).join(', ')}`; } + if (parsed.orderBy.length) { sql += ` ORDER BY ${parsed.orderBy.map(entry => `${quote(entry.column)} ${entry.direction}`).join(', ')}`; } + const effectiveLimit = parsed.limit === null ? maxRows : Math.min(parsed.limit, maxRows); + sql += ` LIMIT ${effectiveLimit + 1}`; + const effectiveOffset = (parsed.offset ?? 0) + offset; + if (effectiveOffset > 0) { sql += ` OFFSET ${effectiveOffset}`; } + return { sql, params, table: parsed.table, columns }; +} diff --git a/backend/src/api/intelligence/query-studio/query-studio.routes.ts b/backend/src/api/intelligence/query-studio/query-studio.routes.ts index 14ad9108769..0fb6f74c565 100644 --- a/backend/src/api/intelligence/query-studio/query-studio.routes.ts +++ b/backend/src/api/intelligence/query-studio/query-studio.routes.ts @@ -1,13 +1,17 @@ import { Application, Request, Response } from 'express'; -import { QueryStudioEvidenceError, queryStudioService, usageMetricsUnavailable } from './query-studio.service'; -import { developerIdentity, IdentityError } from '../identity/developer-identity'; -import { ownerOf, requireOwner, sendIdentityError } from '../identity/owner-auth'; +import { QueryPolicyError, QueryStudioEvidenceError, queryStudioService } from './query-studio.service'; +import { AuthenticatedOwner, developerIdentity, IdentityError } from '../identity/developer-identity'; +import { bearerFromRequest, ownerOf, recordOwnerUsage, requireOwner, sendIdentityError } from '../identity/owner-auth'; import { handleError } from '../../../utils/api'; /** An absent source is a 503 that names the source, never a 500 and never an invented row. */ function fail(req: Request, res: Response, e: unknown, status: number, fallback: string): void { if (e instanceof QueryStudioEvidenceError) { - res.status(e.status).json({ stage: e.code, error: e.message }); + res.status(e.status).json({ stage: e.code, reason: e.reason, error: e.message }); + return; + } + if (e instanceof QueryPolicyError) { + res.status(400).json({ stage: 'rejected-by-grammar', error: e.message, position: e.position }); return; } if (e instanceof IdentityError) { @@ -17,6 +21,22 @@ function fail(req: Request, res: Response, e: unknown, status: number, fallback: handleError(req, res, status, e instanceof Error ? e.message : fallback); } +/** + * Execution and history accept, but do not require, an API key: with one the + * execution is recorded in that owner's history and usage; without one nothing + * is retained. A presented key that does not authenticate is a 401, never an + * anonymous fallback. + * @asyncUnsafe Callers turn a rejection into an exact HTTP answer. + */ +async function optionalOwner(req: Request, res: Response): Promise { + const secret = bearerFromRequest(req); + if (!secret) { return null; } + const owner = await developerIdentity.authenticateKey(secret); + if (!owner) { throw new IdentityError('unauthenticated', 'The presented API key is not valid.', 401); } + recordOwnerUsage(res, owner); + return owner; +} + /** * Developer platform and Query Studio. * @@ -87,7 +107,7 @@ class QueryStudioRoutes { private async $getUsage(req: Request, res: Response): Promise { try { - throw new QueryStudioEvidenceError('unavailable-usage-metrics', usageMetricsUnavailable); + res.json(await queryStudioService.getUsage(ownerOf(res))); } catch (e) { fail(req, res, e, 500, 'Failed to fetch usage metrics'); } @@ -133,7 +153,8 @@ class QueryStudioRoutes { res.status(400).json({ error: 'sql parameter required.' }); return; } - const result = queryStudioService.executeQuery(sql, maxRows); + const owner = await optionalOwner(req, res); + const result = await queryStudioService.executeQuery(sql, maxRows, req.body?.cursor, owner); res.json(result); } catch (e) { fail(req, res, e, 400, 'Query execution error'); @@ -142,8 +163,8 @@ class QueryStudioRoutes { private async $getSchema(req: Request, res: Response): Promise { try { - const schema = queryStudioService.getSchema(); - res.json({ tables: schema, count: schema.length }); + const schema = await queryStudioService.getSchema(); + res.json({ ...schema, count: schema.tables.length }); } catch (e) { fail(req, res, e, 500, 'Failed to fetch schema'); } @@ -151,10 +172,11 @@ class QueryStudioRoutes { private async $getHistory(req: Request, res: Response): Promise { try { - const history = queryStudioService.getHistory(); - res.json({ history, count: history.length }); + const owner = await optionalOwner(req, res); + const history = queryStudioService.getHistory(owner); + res.json({ history, count: history.length, scope: owner ? 'owner' : 'unauthenticated: history is kept per API-key owner only' }); } catch (e) { - handleError(req, res, 500, e instanceof Error ? e.message : 'Failed to fetch query history'); + fail(req, res, e, 500, 'Failed to fetch query history'); } } diff --git a/backend/src/api/intelligence/query-studio/query-studio.service.ts b/backend/src/api/intelligence/query-studio/query-studio.service.ts index ed31d91b08e..e0f1d660279 100644 --- a/backend/src/api/intelligence/query-studio/query-studio.service.ts +++ b/backend/src/api/intelligence/query-studio/query-studio.service.ts @@ -1,7 +1,11 @@ +import { createHash } from 'crypto'; import { EventEnvelopeValidator } from '../events/event-envelope'; import config from '../../../config'; import { AuthenticatedOwner, IdentityError } from '../identity/developer-identity'; import { ownerStore } from '../identity/owner-store'; +import { OwnerUsageLedger, ownerUsageLedger, OwnerUsageObservation, UsageLedgerCoverage } from '../identity/owner-usage-ledger'; +import { parseSelect, QUERY_STUDIO_TABLES, QueryGrammarError, renderSelect } from './query-grammar'; +import { QUERY_ENGINE_ENVIRONMENT_VARIABLE, QUERY_ENGINE_LIMITS, QueryEngine, QueryEngineError, queryEngineFromEnvironment } from './query-engine'; /** * Raised when a read has no source behind it. The routes map the code to a @@ -9,25 +13,33 @@ import { ownerStore } from '../identity/owner-store'; * than as an answer. */ export class QueryStudioEvidenceError extends Error { - constructor(public readonly code: string, message: string, public readonly status = 503) { + constructor(public readonly code: string, message: string, public readonly status = 503, public readonly reason: string | null = null) { super(message); } } -const queryEngineUnavailable = - 'Query Studio results are unavailable. Query execution and the sandbox schema require the owned read-only analytics replica (UNIVERSE_QUERY_ENGINE_DSN) fed by the owned mempool and relay sensor tables, which is not connected on this deployment.'; +/** A query the grammar refuses. The routes answer it with a 400. */ +export class QueryPolicyError extends Error { + constructor(message: string, public readonly position: number | null = null) { + super('Security policy violation: ' + message); + } +} -export const usageMetricsUnavailable = - 'Developer usage metrics are unavailable. Request counts, quota and latency require the owned API gateway metrics store (UNIVERSE_API_GATEWAY_METRICS_ORIGIN), which is not connected on this deployment.'; +const queryEngineUnconfigured = + `Query Studio results are unavailable. Query execution and the sandbox schema require the owned read-only analytics replica named by ${QUERY_ENGINE_ENVIRONMENT_VARIABLE}, which is not configured on this deployment.`; export interface QueryExecutionResult { query_id: string; sql: string; + executed_sql: string; columns: string[]; rows: Array>; row_count: number; execution_time_ms: number; truncated: boolean; + next_cursor: string | null; + source: { engine: string; table: string; network: string; deadline_ms: number; max_rows: number; result_bytes_limit: number }; + precision: string; } export interface TableColumnSchema { @@ -44,6 +56,26 @@ export interface TableSchemaInfo { indexes: string[]; } +export interface QuerySchemaResult { + network: string; + source: string; + observed_at: string; + tables: TableSchemaInfo[]; + /** Allowlisted tables the replica does not currently carry. */ + missing_tables: string[]; + grammar: string; +} + +export interface QueryHistoryEntry { + query_id: string; + owner_id: string; + sql: string; + executed_at: string; + duration_ms: number; + row_count: number; + truncated: boolean; +} + export interface SavedQueryRecord { query_id: string; owner_id: string; @@ -53,22 +85,53 @@ export interface SavedQueryRecord { updated_at: string; } +export interface DeveloperUsageResult { + owner_id: string; + network: string; + state: 'observed' | 'no-observations'; + usage: Omit | null; + quota: { keys: Array<{ key_id: string; name: string; rate_limit_per_minute: number; last_used_at: string | null; revoked: boolean }>; source: string }; + coverage: UsageLedgerCoverage; +} + const SAVED_QUERY_LIMITS = { perOwner: 200, titleLength: 128, sqlLength: 16_384 } as const; +const HISTORY_LIMITS = { perOwner: 100, owners: 5_000 } as const; + +const GRAMMAR_DESCRIPTION = + 'One SELECT over one readable table: columns or COUNT/SUM/AVG/MIN/MAX, WHERE with comparisons, IS NULL, IN, BETWEEN, LIKE, AND/OR/NOT, GROUP BY, ORDER BY, LIMIT and OFFSET. No joins, subqueries, comments, other functions, variables, writes or second statements.'; + +function cursorFor(sqlHash: string, offset: number): string { + return Buffer.from(JSON.stringify({ h: sqlHash, o: offset }), 'utf8').toString('base64url'); +} + +function offsetFromCursor(cursor: unknown, sqlHash: string): number { + if (cursor === undefined || cursor === null || cursor === '') { return 0; } + if (typeof cursor !== 'string' || cursor.length > 256) { throw new QueryPolicyError('The cursor is not one this query issued.'); } + try { + const parsed = JSON.parse(Buffer.from(cursor, 'base64url').toString('utf8')); + if (parsed?.h !== sqlHash || !Number.isSafeInteger(parsed?.o) || parsed.o < 0) { throw new Error('mismatch'); } + return parsed.o; + } catch { + throw new QueryPolicyError('The cursor is not one this query issued.'); + } +} /** * Query Studio. * - * Execution used to answer every SELECT with the same two invented rows, and - * the schema described a sandbox that was never connected. Both now report - * the analytics replica they would need. The SQL policy check stays because - * it is a check on the caller's input, and saved queries and history stay - * because they hold what callers submitted, minus the seeded example. + * Execution parses the caller's SQL with the bounded grammar, renders the + * statement itself and runs it on the read-only analytics engine with a + * deadline, a row limit and a byte limit; the schema is discovered from the + * same engine and intersected with the allowlist. Neither answers without + * the engine. History is appended on every authenticated execution and read + * back per owner. Saved queries are the owner's data, as before. */ export class QueryStudioService { private static instance: QueryStudioService; - private queryHistory: Array<{ query_id: string; sql: string; executed_at: string; duration_ms: number }> = []; + private queryHistory = new Map(); + private engineOverride: QueryEngine | null | undefined; - private constructor() {} + private constructor(private readonly usageLedger: OwnerUsageLedger = ownerUsageLedger) {} public static getInstance(): QueryStudioService { if (!QueryStudioService.instance) { @@ -77,43 +140,143 @@ export class QueryStudioService { return QueryStudioService.instance; } - public getSchema(): TableSchemaInfo[] { - throw new QueryStudioEvidenceError('unavailable-query-engine', queryEngineUnavailable); + /** Test seam: an engine other than the one named by the environment. */ + public useEngine(engine: QueryEngine | null | undefined): void { + this.engineOverride = engine; + this.queryHistory.clear(); + } + + private engine(): QueryEngine { + let engine: QueryEngine | null; + try { + engine = this.engineOverride !== undefined ? this.engineOverride : queryEngineFromEnvironment(); + } catch (e) { + throw new QueryStudioEvidenceError('unavailable-query-engine', e instanceof Error ? e.message : String(e), 503, 'invalid-dsn'); + } + if (!engine) { + throw new QueryStudioEvidenceError('unavailable-query-engine', queryEngineUnconfigured, 503, 'unconfigured'); + } + return engine; + } + + private static translate(e: unknown): never { + if (e instanceof QueryEngineError) { + throw new QueryStudioEvidenceError('unavailable-query-engine', e.message, e.reason === 'deadline' ? 504 : 503, e.reason); + } + throw e; } - public executeQuery(sql: string, maxRows = 100): QueryExecutionResult { - void maxRows; - const trimmed = sql.trim(); + /** @asyncUnsafe Callers turn a rejection into an exact HTTP answer. */ + public async getSchema(): Promise { + const engine = this.engine(); + const allowed = Object.keys(QUERY_STUDIO_TABLES); + let discovered; + try { + discovered = await engine.describe(allowed); + } catch (e) { + QueryStudioService.translate(e); + } + const present = new Set(discovered.map(table => table.table_name)); + return { + network: config.MEMPOOL.NETWORK, + source: engine.source, + observed_at: new Date().toISOString(), + tables: discovered.map(table => ({ ...table, description: QUERY_STUDIO_TABLES[table.table_name].description })), + missing_tables: allowed.filter(name => !present.has(name)), + grammar: GRAMMAR_DESCRIPTION, + }; + } - // Strict validation: Only SELECT permitted - const forbiddenKeywords = ['drop', 'delete', 'update', 'insert', 'alter', 'truncate', 'grant', 'revoke', 'create', 'execute', 'exec']; - const normalized = trimmed.toLowerCase(); + /** @asyncUnsafe Callers turn a rejection into an exact HTTP answer. */ + public async executeQuery(sql: string, maxRows: number = QUERY_ENGINE_LIMITS.defaultRows, cursor?: unknown, owner: AuthenticatedOwner | null = null): Promise { + let parsed; + try { + parsed = parseSelect(sql); + } catch (e) { + if (e instanceof QueryGrammarError) { throw new QueryPolicyError(e.message, e.position); } + throw e; + } + if (!Number.isSafeInteger(maxRows) || maxRows < 1) { throw new QueryPolicyError('max_rows must be a positive integer.'); } + const rowLimit = Math.min(maxRows, QUERY_ENGINE_LIMITS.maxRows); + const sqlHash = createHash('sha256').update(sql).digest('hex').slice(0, 16); + const offset = offsetFromCursor(cursor, sqlHash); + const rendered = renderSelect(parsed, rowLimit, offset); + const engine = this.engine(); - if (!normalized.startsWith('select')) { - throw new Error('Security policy violation: Only SELECT queries are permitted in Query Studio.'); + const startedAt = Date.now(); + let run; + try { + run = await engine.run(rendered, QUERY_ENGINE_LIMITS.statementDeadlineMs); + } catch (e) { + QueryStudioService.translate(e); } + const durationMs = Date.now() - startedAt; - if (trimmed.includes(';')) { - const parts = trimmed.split(';').map((p) => p.trim()).filter(Boolean); - if (parts.length > 1) { - throw new Error('Security policy violation: Multiple statements are not permitted.'); - } + // Row and byte limits: one row past the limit means more exist; bytes are + // counted as the rows are serialised so a wide row set stops early. + const rows: Array> = []; + let bytes = 2; + let truncated = run.rows.length > rowLimit; + for (const row of run.rows.slice(0, rowLimit)) { + bytes += Buffer.byteLength(JSON.stringify(row), 'utf8') + 1; + if (bytes > QUERY_ENGINE_LIMITS.resultBytes) { truncated = true; break; } + rows.push(row); } + const queryId = EventEnvelopeValidator.generateUuidV7(); + if (owner) { this.appendHistory(owner.owner_id, { query_id: queryId, owner_id: owner.owner_id, sql, executed_at: new Date(startedAt).toISOString(), duration_ms: durationMs, row_count: rows.length, truncated }); } + return { + query_id: queryId, + sql, + executed_sql: rendered.sql, + columns: run.columns, + rows, + row_count: rows.length, + execution_time_ms: durationMs, + truncated, + next_cursor: truncated && rows.length > 0 ? cursorFor(sqlHash, offset + rows.length) : null, + source: { engine: engine.source, table: rendered.table, network: config.MEMPOOL.NETWORK, deadline_ms: QUERY_ENGINE_LIMITS.statementDeadlineMs, max_rows: rowLimit, result_bytes_limit: QUERY_ENGINE_LIMITS.resultBytes }, + precision: 'BIGINT and DECIMAL values are returned as decimal strings; DOUBLE and FLOAT columns as IEEE numbers; timestamps as UTC strings.', + }; + } - for (const kw of forbiddenKeywords) { - const regex = new RegExp(`\\b${kw}\\b`, 'i'); - if (regex.test(normalized)) { - throw new Error(`Security policy violation: Disallowed keyword '${kw}' detected.`); + private appendHistory(ownerId: string, entry: QueryHistoryEntry): void { + let entries = this.queryHistory.get(ownerId); + if (!entries) { + if (this.queryHistory.size >= HISTORY_LIMITS.owners) { + const oldest = this.queryHistory.keys().next().value; + if (oldest !== undefined) { this.queryHistory.delete(oldest); } } + entries = []; } + this.queryHistory.delete(ownerId); + this.queryHistory.set(ownerId, entries); + entries.unshift(entry); + if (entries.length > HISTORY_LIMITS.perOwner) { entries.length = HISTORY_LIMITS.perOwner; } + } - // The policy above is a check on the caller's SQL; the rows would come from - // the analytics replica, and no replica is connected. - throw new QueryStudioEvidenceError('unavailable-query-engine', queryEngineUnavailable); + /** The owner's own executions, newest first. Unauthenticated executions are not retained. */ + public getHistory(owner: AuthenticatedOwner | null): QueryHistoryEntry[] { + if (!owner) { return []; } + return (this.queryHistory.get(owner.owner_id) ?? []).map(entry => ({ ...entry })); } - public getHistory(): Array<{ query_id: string; sql: string; executed_at: string; duration_ms: number }> { - return this.queryHistory; + /** @asyncUnsafe Callers turn a rejection into an exact HTTP answer. */ + public async getUsage(owner: AuthenticatedOwner): Promise { + const keys = await ownerStore().listApiKeys(owner.owner_id, config.MEMPOOL.NETWORK); + const observation = this.usageLedger.read(owner.owner_id); + const { state, owner_id, ...usage } = observation as OwnerUsageObservation; + void owner_id; + return { + owner_id: owner.owner_id, + network: config.MEMPOOL.NETWORK, + state, + usage: state === 'observed' ? usage : null, + quota: { + keys: keys.map(key => ({ key_id: key.key_id, name: key.name, rate_limit_per_minute: key.rate_limit, last_used_at: key.last_used_at, revoked: key.revoked_at !== null })), + source: 'Per-key rate limits and last-use timestamps from the durable owner store.', + }, + coverage: this.usageLedger.coverage(), + }; } /** @asyncUnsafe Saved queries are the owner's data: durable, owner and network scoped, bounded. */ diff --git a/backend/src/api/intelligence/query-studio/query-studio.test.ts b/backend/src/api/intelligence/query-studio/query-studio.test.ts index 658ede951f0..b48874efe4c 100644 --- a/backend/src/api/intelligence/query-studio/query-studio.test.ts +++ b/backend/src/api/intelligence/query-studio/query-studio.test.ts @@ -1,73 +1,142 @@ import { Application, Request, Response } from 'express'; +import { EventEmitter } from 'events'; import queryStudioRoutes from './query-studio.routes'; -import { QueryStudioEvidenceError, queryStudioService } from './query-studio.service'; +import { QueryPolicyError, QueryStudioEvidenceError, queryStudioService } from './query-studio.service'; +import { EngineRun, EngineTable, parseQueryEngineDsn, QueryEngine, QueryEngineError, QUERY_ENGINE_LIMITS } from './query-engine'; +import { RenderedQuery } from './query-grammar'; import { developerIdentity, AuthenticatedOwner } from '../identity/developer-identity'; import { MemoryOwnerStore, useOwnerStore } from '../identity/owner-store'; +import { ownerUsageLedger } from '../identity/owner-usage-ledger'; -/** - * These assertions replace a suite that asserted the constants the service used - * to return: two invented mempool rows for any SELECT, a schema for a sandbox - * that was never connected, and a seeded saved query owned by 'dev-admin'. - * Passing those proved the constants were present, not that any query ran. - */ -describe('Product 9: Developer Data Platform and Query Studio', () => { - const unavailable = (code: string) => expect.objectContaining({ code, status: 503 }); - - it('reports the missing analytics replica rather than a schema for an unconnected sandbox', () => { - expect(() => queryStudioService.getSchema()).toThrow(unavailable('unavailable-query-engine')); +/** A fake analytics connection: records the statement it was handed and answers with fixture rows. */ +class FakeEngine implements QueryEngine { + public readonly source = 'mysql://readonly@fake/explorer'; + public statements: RenderedQuery[] = []; + public tables: EngineTable[] = [ + { table_name: 'blocks', columns: [{ name: 'height', type: 'int(11) unsigned', nullable: false, is_primary_key: true }, { name: 'hash', type: 'varchar(65)', nullable: false, is_primary_key: false }, { name: 'total_output_amt', type: 'bigint unsigned', nullable: false, is_primary_key: false }], indexes: ['PRIMARY', 'pool_id'] }, + { table_name: 'pools', columns: [{ name: 'id', type: 'smallint unsigned', nullable: false, is_primary_key: true }, { name: 'name', type: 'varchar(50)', nullable: false, is_primary_key: false }], indexes: ['PRIMARY'] }, + ]; + constructor(private readonly answer: (query: RenderedQuery) => EngineRun | Promise) {} + /** @asyncSafe */ + public async describe(): Promise { return this.tables; } + /** @asyncUnsafe test double */ + public async run(query: RenderedQuery): Promise { this.statements.push(query); return this.answer(query); } +} + +const fixtureRows = [ + { height: 200000, hash: '00'.repeat(32), total_output_amt: '9223372036854775807' }, + { height: 199999, hash: '11'.repeat(32), total_output_amt: '12345678901234567890' }, +]; + +async function owners(): Promise<{ analyst: AuthenticatedOwner; other: AuthenticatedOwner; analystKey: string }> { + useOwnerStore(new MemoryOwnerStore()); + developerIdentity.resetForTests(); + ownerUsageLedger.resetForTests(); + const a = await developerIdentity.bootstrapOwner('analyst', '203.0.113.1'); + const b = await developerIdentity.bootstrapOwner('other', '203.0.113.2'); + return { analyst: (await developerIdentity.authenticateKey(a.secret_key))!, other: (await developerIdentity.authenticateKey(b.secret_key))!, analystKey: a.secret_key }; +} + +describe('Query Studio connector', () => { + afterEach(() => { queryStudioService.useEngine(undefined); }); + + it('reports the unconfigured engine with its reason and never an empty result', async () => { + queryStudioService.useEngine(null); + for (const read of [() => queryStudioService.getSchema(), () => queryStudioService.executeQuery('SELECT height FROM blocks')]) { + await expect(read()).rejects.toMatchObject({ code: 'unavailable-query-engine', reason: 'unconfigured', status: 503 }); + await expect(read()).rejects.toBeInstanceOf(QueryStudioEvidenceError); + } + expect(queryStudioService.getHistory(null)).toEqual([]); + }); + + it('rejects writes, stacked statements, comments and oversized queries before touching the engine', async () => { + const engine = new FakeEngine(() => ({ rows: [], columns: [] })); + queryStudioService.useEngine(engine); + for (const sql of ['DROP TABLE blocks', 'DELETE FROM blocks WHERE 1=1', 'UPDATE blocks SET fees = 0', 'SELECT * FROM blocks; DROP TABLE blocks', 'SELECT height FROM blocks -- x', 'SELECT * FROM intelligence_api_keys', 'SELECT height FROM blocks WHERE ' + 'height = 1 OR '.repeat(600) + 'height = 2']) { + await expect(queryStudioService.executeQuery(sql)).rejects.toBeInstanceOf(QueryPolicyError); + await expect(queryStudioService.executeQuery(sql)).rejects.toThrow(/Security policy violation/); + } + await expect(queryStudioService.executeQuery('SELECT height FROM blocks', 0)).rejects.toThrow(/max_rows/); + expect(engine.statements).toEqual([]); }); - it('reports the missing analytics replica rather than invented rows for a policy-compliant SELECT', () => { - const sql = 'SELECT txid, fee_sats, vsize, feerate FROM mempool_transactions WHERE feerate > 10 LIMIT 10'; - expect(() => queryStudioService.executeQuery(sql)).toThrow(unavailable('unavailable-query-engine')); - expect(queryStudioService.getHistory()).toEqual([]); + it('runs a parsed SELECT on the engine, keeps exact integers and appends owner-scoped history', async () => { + const { analyst, other } = await owners(); + const engine = new FakeEngine(() => ({ rows: fixtureRows, columns: ['height', 'hash', 'total_output_amt'] })); + queryStudioService.useEngine(engine); + const sql = "SELECT height, hash, total_output_amt FROM blocks WHERE hash <> 'DROP TABLE blocks' ORDER BY height DESC LIMIT 2"; + const result = await queryStudioService.executeQuery(sql, 100, undefined, analyst); + expect(engine.statements).toEqual([{ + sql: 'SELECT `height` AS `height`, `hash` AS `hash`, `total_output_amt` AS `total_output_amt` FROM `blocks` WHERE `hash` <> ? ORDER BY `height` DESC LIMIT 3', + params: ['DROP TABLE blocks'], table: 'blocks', columns: ['height', 'hash', 'total_output_amt'], + }]); + expect(result.rows).toEqual(fixtureRows); + expect(result.rows[1].total_output_amt).toBe('12345678901234567890'); + expect(result.row_count).toBe(2); + expect(result.truncated).toBe(false); + expect(result.next_cursor).toBeNull(); + expect(result.source).toMatchObject({ engine: engine.source, table: 'blocks', deadline_ms: QUERY_ENGINE_LIMITS.statementDeadlineMs }); + + const history = queryStudioService.getHistory(analyst); + expect(history).toHaveLength(1); + expect(history[0]).toMatchObject({ query_id: result.query_id, owner_id: analyst.owner_id, sql, row_count: 2, truncated: false }); + expect(queryStudioService.getHistory(other)).toEqual([]); + expect(queryStudioService.getHistory(null)).toEqual([]); }); - it('still rejects non-SELECT queries and dangerous SQL keywords before naming a source', () => { - expect(() => { - queryStudioService.executeQuery('DROP TABLE mempool_transactions'); - }).toThrow(/Security policy violation/); + it('marks truncation when the engine returns more than the row budget and issues a cursor bound to the query', async () => { + const engine = new FakeEngine(query => ({ rows: Array.from({ length: 3 }, (_, i) => ({ height: i })), columns: query.columns })); + queryStudioService.useEngine(engine); + const first = await queryStudioService.executeQuery('SELECT height FROM blocks', 2); + expect(first.truncated).toBe(true); + expect(first.row_count).toBe(2); + expect(first.next_cursor).toEqual(expect.any(String)); + const second = await queryStudioService.executeQuery('SELECT height FROM blocks', 2, first.next_cursor); + expect(engine.statements[1].sql).toContain('LIMIT 3 OFFSET 2'); + expect(second.row_count).toBe(2); + await expect(queryStudioService.executeQuery('SELECT hash FROM blocks', 2, first.next_cursor)).rejects.toThrow(/cursor/); + }); - expect(() => { - queryStudioService.executeQuery('DELETE FROM mempool_transactions WHERE 1=1'); - }).toThrow(/Security policy violation/); + it('stops at the byte budget', async () => { + const wide = 'x'.repeat(400_000); + const engine = new FakeEngine(() => ({ rows: [{ hash: wide }, { hash: wide }, { hash: wide }], columns: ['hash'] })); + queryStudioService.useEngine(engine); + const result = await queryStudioService.executeQuery('SELECT hash FROM blocks', 10); + expect(result.row_count).toBe(2); + expect(result.truncated).toBe(true); + }); - expect(() => { - queryStudioService.executeQuery('UPDATE mempool_transactions SET fee_sats = 0'); - }).toThrow(/Security policy violation/); + it('maps engine faults to typed unavailable states', async () => { + queryStudioService.useEngine(new FakeEngine(() => { throw new QueryEngineError('deadline', 'cancelled'); })); + await expect(queryStudioService.executeQuery('SELECT height FROM blocks')).rejects.toMatchObject({ code: 'unavailable-query-engine', reason: 'deadline', status: 504 }); + queryStudioService.useEngine(new FakeEngine(() => { throw new QueryEngineError('unreachable', 'refused'); })); + await expect(queryStudioService.executeQuery('SELECT height FROM blocks')).rejects.toMatchObject({ reason: 'unreachable', status: 503 }); + }); - expect(() => { - queryStudioService.executeQuery('SELECT * FROM mempool_transactions; DROP TABLE mempool_checkpoints'); - }).toThrow(/Multiple statements/); + it('discovers the schema from the engine, intersected with the allowlist', async () => { + const engine = new FakeEngine(() => ({ rows: [], columns: [] })); + queryStudioService.useEngine(engine); + const schema = await queryStudioService.getSchema(); + expect(schema.tables.map(table => table.table_name)).toEqual(['blocks', 'pools']); + expect(schema.tables[0].description).toMatch(/indexed block/); + expect(schema.tables[0].columns[2]).toEqual({ name: 'total_output_amt', type: 'bigint unsigned', nullable: false, is_primary_key: false }); + expect(schema.missing_tables).toEqual(['hashrates', 'difficulty_adjustments', 'prices', 'statistics', 'blocks_audits']); + expect(schema.source).toBe(engine.source); }); - it('never resolves an absent source as an empty result set', () => { - for (const read of [ - () => queryStudioService.getSchema(), - () => queryStudioService.executeQuery('SELECT 1'), - ]) { - let resolved: unknown = 'unresolved'; - try { - resolved = read(); - } catch (e) { - expect(e).toBeInstanceOf(QueryStudioEvidenceError); - continue; - } - throw new Error(`resolved with ${JSON.stringify(resolved)}`); + it('parses the DSN into a read-only pool configuration and refuses malformed ones', () => { + expect(parseQueryEngineDsn('mysql://ro:s3cret@10.0.0.5:3307/mempool')).toEqual({ host: '10.0.0.5', port: 3307, user: 'ro', password: 's3cret', database: 'mempool', socketPath: null }); + expect(parseQueryEngineDsn('mysql://ro@localhost/mempool?socket=/run/mysqld/mysqld.sock').socketPath).toBe('/run/mysqld/mysqld.sock'); + for (const bad of ['not a url', 'postgres://ro@h/db', 'mysql://ro@h/', 'mysql://h/db', 'mysql://ro@h:99999/db']) { + expect(() => parseQueryEngineDsn(bad)).toThrow(QueryEngineError); } }); it('keeps saved queries per authenticated owner, durably and bounded', async () => { - useOwnerStore(new MemoryOwnerStore()); - developerIdentity.resetForTests(); - const a = await developerIdentity.bootstrapOwner('analyst', '203.0.113.1'); - const b = await developerIdentity.bootstrapOwner('other', '203.0.113.2'); - const analyst: AuthenticatedOwner = (await developerIdentity.authenticateKey(a.secret_key))!; - const other: AuthenticatedOwner = (await developerIdentity.authenticateKey(b.secret_key))!; + const { analyst, other } = await owners(); expect(await queryStudioService.getSavedQueries(analyst)).toEqual([]); - - const title = 'High Priority Mempool Packages'; - const sql = 'SELECT txid, feerate FROM mempool_transactions ORDER BY feerate DESC LIMIT 50'; + const title = 'High fee blocks'; + const sql = 'SELECT height, fees FROM blocks ORDER BY fees DESC LIMIT 50'; const saved = await queryStudioService.saveQuery(analyst, title, sql); expect(saved.owner_id).toBe(analyst.owner_id); expect((await queryStudioService.getSavedQueries(analyst)).map((q) => q.query_id)).toEqual([saved.query_id]); @@ -92,6 +161,28 @@ describe('Product 9: Developer Data Platform and Query Studio', () => { }); }); +describe('Developer usage metrics', () => { + it('answers no-observations for an owner the ledger has not seen, and only that owner\'s counts afterwards', async () => { + const { analyst, other } = await owners(); + const before = await queryStudioService.getUsage(analyst); + expect(before.state).toBe('no-observations'); + expect(before.usage).toBeNull(); + expect(before.quota.keys).toHaveLength(1); + expect(before.quota.keys[0].rate_limit_per_minute).toBe(1000); + expect(before.coverage.persistence).toBe('process-memory-only'); + + ownerUsageLedger.record(analyst.owner_id, analyst.key_id, 200, 12); + ownerUsageLedger.record(analyst.owner_id, analyst.key_id, 429, 3); + ownerUsageLedger.record(other.owner_id, other.key_id, 200, 50); + const after = await queryStudioService.getUsage(analyst); + expect(after.state).toBe('observed'); + expect(after.usage).toMatchObject({ requests_total: 2, responses_2xx: 1, responses_4xx: 1, rate_limited: 1, latency_ms: { max: 12, samples: 2 } }); + expect(after.usage?.keys).toEqual([expect.objectContaining({ key_id: analyst.key_id, requests: 2 })]); + const foreign = await queryStudioService.getUsage(other); + expect(foreign.usage?.requests_total).toBe(1); + }); +}); + describe('Query Studio HTTP responses', () => { type Handler = (req: Request, res: Response) => Promise; @@ -107,31 +198,69 @@ describe('Query Studio HTTP responses', () => { return { gets, posts }; } - it.each([ - ['/api/v1/intelligence/developer/usage', 'unavailable-usage-metrics'], - ['/api/v1/intelligence/query/schema', 'unavailable-query-engine'], - ])('answers %s with a 503 that names the missing source', async (path, stage) => { - const { gets } = mount(); - const res = { status: jest.fn().mockReturnThis(), json: jest.fn() }; - await gets.get(path)!({ query: {} } as unknown as Request, res as unknown as Response); + function response(locals: Record = {}) { + const res = Object.assign(new EventEmitter(), { statusCode: 200, locals, status: jest.fn(), json: jest.fn(), send: jest.fn() }); + res.status.mockImplementation((code: number) => { res.statusCode = code; return res; }); + return res; + } + + afterEach(() => { queryStudioService.useEngine(undefined); }); + + it('answers schema and execute with a 503 naming the unconfigured engine', async () => { + queryStudioService.useEngine(null); + const { gets, posts } = mount(); + const res = response(); + await gets.get('/api/v1/intelligence/query/schema')!({ query: {}, headers: {} } as unknown as Request, res as unknown as Response); expect(res.status).toHaveBeenCalledWith(503); - const body = res.json.mock.calls[0][0]; - expect(body.stage).toBe(stage); - expect(body).not.toHaveProperty('tables'); - expect(body).not.toHaveProperty('requests_total'); + expect(res.json.mock.calls[0][0]).toMatchObject({ stage: 'unavailable-query-engine', reason: 'unconfigured' }); + expect(res.json.mock.calls[0][0]).not.toHaveProperty('tables'); + + const exec = response(); + await posts.get('/api/v1/intelligence/query/execute')!({ body: { sql: 'SELECT height FROM blocks' }, headers: {} } as unknown as Request, exec as unknown as Response); + expect(exec.status).toHaveBeenCalledWith(503); + expect(exec.json.mock.calls[0][0]).not.toHaveProperty('rows'); }); - it('answers a policy-compliant execute with a 503 and a policy violation with a 400', async () => { - const { posts } = mount(); + it('answers a grammar rejection with a 400 and a fixture query with rows, recording history for the key holder', async () => { + const { analystKey, analyst } = await owners(); + queryStudioService.useEngine(new FakeEngine(() => ({ rows: fixtureRows, columns: ['height', 'hash', 'total_output_amt'] }))); + const { gets, posts } = mount(); const execute = posts.get('/api/v1/intelligence/query/execute')!; - const res = { status: jest.fn().mockReturnThis(), json: jest.fn() }; - await execute({ body: { sql: 'SELECT 1' } } as Request, res as unknown as Response); - expect(res.status).toHaveBeenCalledWith(503); - expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ stage: 'unavailable-query-engine' })); - expect(res.json.mock.calls[0][0]).not.toHaveProperty('rows'); - const bad = { status: jest.fn().mockReturnThis(), json: jest.fn(), send: jest.fn() }; - await execute({ body: { sql: 'DROP TABLE x' }, accepts: () => true } as unknown as Request, bad as unknown as Response); + const bad = response(); + await execute({ body: { sql: 'DROP TABLE blocks' }, headers: {} } as unknown as Request, bad as unknown as Response); expect(bad.status).toHaveBeenCalledWith(400); + expect(bad.json.mock.calls[0][0]).toMatchObject({ stage: 'rejected-by-grammar' }); + + const ok = response(); + await execute({ body: { sql: 'SELECT height, hash, total_output_amt FROM blocks LIMIT 2' }, headers: { authorization: 'Bearer ' + analystKey } } as unknown as Request, ok as unknown as Response); + expect(ok.status).not.toHaveBeenCalledWith(503); + expect(ok.json.mock.calls[0][0]).toMatchObject({ row_count: 2, rows: fixtureRows }); + + const history = response(); + await gets.get('/api/v1/intelligence/query/history')!({ headers: { authorization: 'Bearer ' + analystKey } } as unknown as Request, history as unknown as Response); + expect(history.json.mock.calls[0][0]).toMatchObject({ count: 1, scope: 'owner' }); + expect(history.json.mock.calls[0][0].history[0].owner_id).toBe(analyst.owner_id); + + const anonymous = response(); + await gets.get('/api/v1/intelligence/query/history')!({ headers: {} } as unknown as Request, anonymous as unknown as Response); + expect(anonymous.json.mock.calls[0][0]).toMatchObject({ count: 0 }); + + const wrongKey = response(); + await execute({ body: { sql: 'SELECT height FROM blocks' }, headers: { authorization: 'Bearer uip_live_nope' } } as unknown as Request, wrongKey as unknown as Response); + expect(wrongKey.status).toHaveBeenCalledWith(401); + }); + + it('answers usage for the authenticated owner only and 401 without one', async () => { + const { analyst } = await owners(); + const { gets } = mount(); + const usage = gets.get('/api/v1/intelligence/developer/usage')!; + const unauthenticated = response(); + await usage({ query: {}, headers: {} } as unknown as Request, unauthenticated as unknown as Response); + expect(unauthenticated.status).toHaveBeenCalledWith(401); + const res = response({ owner: analyst }); + await usage({ query: {}, headers: {} } as unknown as Request, res as unknown as Response); + expect(res.json.mock.calls[0][0]).toMatchObject({ owner_id: analyst.owner_id, state: 'no-observations' }); + expect(res.json.mock.calls[0][0]).not.toHaveProperty('requests_total'); }); }); diff --git a/backend/src/api/intelligence/verification-integrity.test.ts b/backend/src/api/intelligence/verification-integrity.test.ts index 332ec84287a..ff7892bda6a 100644 --- a/backend/src/api/intelligence/verification-integrity.test.ts +++ b/backend/src/api/intelligence/verification-integrity.test.ts @@ -23,14 +23,16 @@ const participants = [ ]; describe('verification evidence integrity', () => { - it('cannot verify an unknown snapshot without a trusted manifest or snapshot bytes', () => { - expect(() => bootstrapService.verifySnapshot({ snapshot_id: 'unknown', file_sha256: '00'.repeat(32), - base_height: 840000, expected_txoutset_hash: '00'.repeat(32) })).toThrow(/unavailable/i); + it('cannot verify an unknown snapshot without a trusted manifest or snapshot bytes', /** @asyncUnsafe Jest awaits this test and reports its rejection. */ async () => { + await expect(bootstrapService.verifySnapshot({ snapshot_id: 'unknown', file_sha256: '00'.repeat(32), + base_height: 840000, expected_txoutset_hash: '00'.repeat(32) })).rejects.toThrow(/unavailable/i); }); - it('cannot start a node job when no authorized executor or durable job store exists', () => { - expect(() => bootstrapService.createOperatorJob({ job_type: 'load_snapshot', node_id: 'missing', snapshot_id: 'unknown' })) - .toThrow(/unavailable/i); + it('cannot start a node job without a verified control-plane request, and not without the operator prerequisites either', /** @asyncUnsafe Jest awaits this test and reports its rejection. */ async () => { + await expect(bootstrapService.createOperatorJob({ job_type: 'load_snapshot', node_id: 'missing', snapshot_id: 'unknown', idempotency_key: 'idem-12345678', confirm: 'load_snapshot' }, undefined)) + .rejects.toThrow(expect.objectContaining({ code: 'UNAUTHORIZED', status: 401 })); + await expect(bootstrapService.createOperatorJob({ job_type: 'load_snapshot', node_id: 'missing', snapshot_id: 'unknown', idempotency_key: 'idem-12345678', confirm: 'load_snapshot' }, { keyId: 'ops', elevated: true })) + .rejects.toThrow(/unavailable/i); }); it('does not manufacture calendar receipts, proof upgrades or Bitcoin attestations', /** @asyncUnsafe Jest awaits this test and reports its rejection. */ async () => { diff --git a/backend/src/api/network-observatory/network-observatory.service.test.ts b/backend/src/api/network-observatory/network-observatory.service.test.ts index 8ed22e55fe7..7c198afcba5 100644 --- a/backend/src/api/network-observatory/network-observatory.service.test.ts +++ b/backend/src/api/network-observatory/network-observatory.service.test.ts @@ -1,39 +1,117 @@ +jest.mock('../bitcoin/bitcoin-client', () => ({ __esModule: true, default: {} })); +jest.mock('../mempool-blocks', () => ({ __esModule: true, default: { getMempoolBlocksWithTransactions: () => [] } })); + import { Application, Request, Response } from 'express'; +import config from '../../config'; +import { RelayCollectorService } from '../intelligence/relay/relay-collector.service'; +import { templateCollectorService } from '../intelligence/templates/template-collector.service'; import networkObservatoryRoutes from './network-observatory.routes'; -import { NetworkObservatoryEvidenceError, networkObservatoryService } from './network-observatory.service'; +import { NetworkObservatoryEvidenceError, NetworkObservatoryService, ObservatoryReaders } from './network-observatory.service'; + +const TX = 'ab'.repeat(32); +const OTHER = 'cd'.repeat(32); /** - * These assertions replace a suite that asserted the constants the service - * used to return: at least four nodes online, a propagation timeline whose - * median latency was positive because the deltas were literals, and a - * template comparison at a height above 800000. Passing those proved the - * constants were present, not that any node had reported. + * The collectors are the real ones with their sources stubbed: a relay + * collector fed poll deltas, and the template collector fed getblocktemplate + * answers. Nothing here invents a second observer. */ +function setup(local: string[] | null = [TX]) { + let now = 100_000; + const snapshot = jest.fn(async () => ({ + network: config.MEMPOOL.NETWORK, observed_at_utc: new Date(now).toISOString(), age_ms: 0, freshness_limit_ms: 30000, + peers: [{ addr: '192.0.2.1:8333', id: 4, transport_protocol_type: 'v2' }, { addr: '192.0.2.2:8333', id: 5, transport_protocol_type: 'v1' }], + info: { subversion: '/Satoshi:30.0.0/', protocolversion: 70016, relayfee: 0.00001, networks: [] }, + } as any)); + const relay = new RelayCollectorService({ now: () => now, snapshotReader: snapshot, policyReader: async () => ({ fullrbf: true }) }); + templateCollectorService.resetForTests(); + templateCollectorService.readProjection = () => null; + const readers: ObservatoryReaders = { relay, templates: templateCollectorService, localMempoolTxids: () => (local ? new Set(local) : null) }; + const service = new NetworkObservatoryService(readers); + return { service, relay, snapshot, setNow: (value: number) => { now = value; }, now: () => now }; +} + describe('NetworkObservatoryService', () => { - const unavailable = (code: string) => expect.objectContaining({ code, status: 503 }); + it('reports the single owned observer as one node with unmeasured dimensions unknown', async () => { + const { service, relay } = setup(); + relay.observeMempoolPoll([{ txid: TX }], [], true, 100_000); + const nodes = await service.$getNodes(); + expect(nodes).toHaveLength(1); + expect(nodes[0]).toMatchObject({ + id: expect.stringContaining('owned-mempool-poller-'), region: 'unknown', clientVersion: '/Satoshi:30.0.0/', protocolVersion: 70016, fullRbf: true, + minRelayFeeRate: 1, clockOffsetMs: null, connectedPeers: 2, mempoolTxCount: 1, status: 'online', network: config.MEMPOOL.NETWORK, policySourceAvailable: true, + }); + expect(nodes[0].scope).toMatch(/One observer/); + expect(JSON.stringify(nodes)).not.toMatch(/192\.0\.2/); + }); - it('reports the missing observer fleet rather than invented nodes, timelines and templates', async () => { - await expect(networkObservatoryService.$getNodes()).rejects.toThrow(unavailable('unavailable-observer-fleet')); - await expect(networkObservatoryService.$getPropagation()).rejects.toThrow(unavailable('unavailable-observer-fleet')); - await expect(networkObservatoryService.$getPropagation('ab'.repeat(32))).rejects.toThrow(unavailable('unavailable-observer-fleet')); - await expect(networkObservatoryService.$getTemplates()).rejects.toThrow(unavailable('unavailable-observer-fleet')); + it('marks the observer degraded when its polls are stale and reports the owned node outage as unavailable', async () => { + const { service, relay, snapshot, setNow } = setup(); + relay.observeMempoolPoll([], [], true, 100_000); + setNow(140_000); + expect((await service.$getNodes())[0].status).toBe('degraded'); + snapshot.mockRejectedValueOnce(new Error('node unavailable')); + await expect(service.$getNodes()).rejects.toMatchObject({ code: 'owned-node-unavailable', status: 503 }); + snapshot.mockRejectedValueOnce(new Error('node unavailable')); + await expect(service.$getPropagation(TX)).rejects.toBeInstanceOf(NetworkObservatoryEvidenceError); }); - it('never resolves an absent source as an empty directory', async () => { - for (const read of [ - () => networkObservatoryService.$getNodes(), - () => networkObservatoryService.$getPropagation(), - () => networkObservatoryService.$getTemplates(), - ]) { - let resolved: unknown = 'unresolved'; - try { - resolved = await read(); - } catch (e) { - expect(e).toBeInstanceOf(NetworkObservatoryEvidenceError); - continue; - } - throw new Error(`resolved with ${JSON.stringify(resolved)}`); - } + it('returns a transaction\'s recorded lifecycle from the one observer, with observer identity, clock uncertainty and retention', async () => { + const { service, relay, setNow } = setup(); + relay.observeMempoolPoll([], [], true, 100_000); + relay.observeMempoolPoll([{ txid: TX }], [], true, 101_000); + relay.observeMempoolPoll([], [{ txid: TX }], true, 103_000); + setNow(103_000); + const propagation = await service.$getPropagation(TX); + expect(propagation).toMatchObject({ + txid: TX, state: 'observed', network: config.MEMPOOL.NETWORK, firstSeenTimestamp: 101_000, lastSeenTimestamp: 103_000, + medianLatencyMs: null, p95LatencyMs: null, spreadDeltaMs: null, + observer: { observers: 1, clockOffsetMs: null, clockUncertaintyMs: null }, + window: { collection: 'observing', retainedTransactions: 1, lastCompletePollUtc: new Date(103_000).toISOString() }, + }); + expect(propagation.nodeObservations).toEqual([ + expect.objectContaining({ arrivedAt: 101_000, deltaFromFirstMs: 0, accepted: true, presence: 'present', previousCompletePollUtc: new Date(100_000).toISOString() }), + expect.objectContaining({ arrivedAt: 103_000, deltaFromFirstMs: 2_000, accepted: false, presence: 'left_mempool' }), + ]); + expect(new Set(propagation.nodeObservations.map(o => o.nodeId)).size).toBe(1); + expect(propagation.expiresAtUtc).toBe(new Date(103_000 + 24 * 60 * 60 * 1000).toISOString()); + }); + + it('answers an unobserved transaction and an empty observer with explicit states rather than a timeline', async () => { + const { service, relay } = setup(); + const empty = await service.$getPropagation(); + expect(empty).toMatchObject({ txid: null, state: 'no-observations', nodeObservations: [], window: { collection: 'not_started' } }); + relay.observeMempoolPoll([{ txid: OTHER }], [], true, 100_000); + const unknown = await service.$getPropagation(TX); + expect(unknown).toMatchObject({ txid: TX, state: 'not-observed', nodeObservations: [], firstSeenTimestamp: null }); + const latest = await service.$getPropagation(); + expect(latest).toMatchObject({ txid: OTHER, state: 'observed' }); + await expect(service.$getPropagation('not-a-txid')).rejects.toMatchObject({ code: 'invalid-txid', status: 400 }); + }); + + it('compares the recorded candidate templates for the latest height with each other and the local mempool', async () => { + const { service } = setup([TX]); + templateCollectorService.fetchCoreTemplate = async () => ({ height: 100, previousblockhash: 'p'.repeat(64), transactions: [{ txid: TX, hash: TX, fee: 100, weight: 400 }, { txid: OTHER, hash: OTHER, fee: 50, weight: 400 }], coinbasevalue: 5000 }); + await templateCollectorService.collectCoreTemplate(1_000); + templateCollectorService.readProjection = () => ({ transactionIds: [TX], totalFees: 100, blockVSize: 100, nTx: 1 }); + templateCollectorService.collectProjection(2_000); + const comparison = await service.$getTemplates(); + expect(comparison).toMatchObject({ state: 'observed', blockHeight: 100, generatedAt: 2_000, consensusMempoolTxCount: 1, missingFromLocalCount: 1, feeRateSpreadSatVb: null, observer: { observers: 1 } }); + expect(comparison.candidateTemplates).toHaveLength(2); + const core = comparison.candidateTemplates.find(t => t.sourceType === 'core_gbt')!; + expect(core).toMatchObject({ poolName: 'Bitcoin Core getblocktemplate', txCount: 2, totalFeesSats: '150', uniqueTxids: [OTHER], missingFromLocalMempool: 1, expectedMedianFeeRate: null }); + const projection = comparison.candidateTemplates.find(t => t.sourceType === 'mempool_projection')!; + expect(projection).toMatchObject({ txCount: 1, uniqueTxids: [], missingFromLocalMempool: 0 }); + expect(comparison.sources.map(s => s.status)).toEqual(['active', 'active']); + }); + + it('reports no templates observed with the sources\' own status instead of an empty list of pools', async () => { + const { service } = setup(null); + templateCollectorService.fetchCoreTemplate = async () => { throw new Error('rpc offline'); }; + await templateCollectorService.collectCoreTemplate(1_000); + const comparison = await service.$getTemplates(); + expect(comparison).toMatchObject({ state: 'no-templates-observed', blockHeight: null, candidateTemplates: [], consensusMempoolTxCount: null, missingFromLocalCount: null }); + expect(comparison.sources.find(s => s.sourceId === 'src-core-gbt')).toMatchObject({ status: 'offline', lastError: 'rpc offline' }); }); }); @@ -49,19 +127,23 @@ describe('Network observatory HTTP responses', () => { return gets; } - it('answers every read with a 503 that names the missing source', async () => { + it('mounts the four routes and maps a source outage to a 503 that names it', async () => { const gets = mount(); - expect(gets.size).toBe(4); - for (const handler of gets.values()) { + expect([...gets.keys()]).toEqual([ + config.MEMPOOL.API_URL_PREFIX + 'network/nodes', + config.MEMPOOL.API_URL_PREFIX + 'network/propagation', + config.MEMPOOL.API_URL_PREFIX + 'network/propagation/:txid', + config.MEMPOOL.API_URL_PREFIX + 'network/templates', + ]); + // The default service reads the process-wide relay collector, whose owned node is not reachable in a unit test. + for (const path of [config.MEMPOOL.API_URL_PREFIX + 'network/nodes', config.MEMPOOL.API_URL_PREFIX + 'network/propagation/:txid']) { const res = { status: jest.fn().mockReturnThis(), json: jest.fn() }; - await handler({ params: { txid: 'ab'.repeat(32) } } as unknown as Request, res as unknown as Response); + await gets.get(path)!({ params: { txid: TX } } as unknown as Request, res as unknown as Response); expect(res.status).toHaveBeenCalledWith(503); const body = res.json.mock.calls[0][0]; - expect(body.stage).toMatch(/^unavailable-/); - expect(typeof body.error).toBe('string'); + expect(body.stage).toMatch(/owned-node-unavailable|unavailable/); expect(body).not.toHaveProperty('nodes'); expect(body).not.toHaveProperty('nodeObservations'); - expect(body).not.toHaveProperty('candidateTemplates'); } }); }); diff --git a/backend/src/api/network-observatory/network-observatory.service.ts b/backend/src/api/network-observatory/network-observatory.service.ts index 5d40b12f5cd..5eadb5da379 100644 --- a/backend/src/api/network-observatory/network-observatory.service.ts +++ b/backend/src/api/network-observatory/network-observatory.service.ts @@ -1,5 +1,13 @@ +import config from '../../config'; +import memPool from '../mempool'; +import { RELAY_LIMITS, RelayEvidenceError, relayCollectorService, RelayCollectorService, TransactionRelayLifecycle } from '../intelligence/relay/relay-collector.service'; +import { TEMPLATE_LIMITS, templateCollectorService, TemplateCollectorService } from '../intelligence/templates/template-collector.service'; import { BlockTemplateComparison, + CandidateTemplate, + NodeArrival, + ObservationWindow, + ObserverIdentity, ObserverNode, PropagationObservation, } from './network-observatory.types'; @@ -15,33 +23,219 @@ export class NetworkObservatoryEvidenceError extends Error { } } -const fleetUnavailable = - 'Cross-node observations are unavailable. Node, propagation and template reads require the owned observer fleet (the Universe Bitcoin nodes with their first-seen and getblocktemplate telemetry export), which is not connected on this deployment.'; +/** The owned telemetry this deployment has; injectable for tests. */ +export interface ObservatoryReaders { + relay: Pick; + templates: Pick; + /** The local mempool's txids, or null when the mempool is not enabled here. */ + localMempoolTxids: () => Set | null; +} + +type RelayOverview = Awaited>; + +const SCOPE = + 'One observer: this backend\'s mempool poll of its owned Bitcoin Core node. Detection time is local poll time, not peer arrival time. Regions, clock offsets, inter-node latency and multi-observer propagation are not measured and are reported as null or unknown.'; /** - * Observer fleet, propagation and block-template evidence. + * Observer, propagation and block-template evidence, read from the owned + * telemetry this backend already collects: the relay collector's first-seen + * mempool poll lifecycles and the template collector's candidate templates. * - * Every read here needs the owned observer fleet reporting first-seen times - * and candidate templates. None is connected, so each read reports the - * absent source. The revision this replaces answered from constants: four - * nodes marked online in four regions, a propagation timeline computed from - * the request clock for any txid (and a fixed txid when none was given), and - * three candidate templates with pool names nobody had polled. + * The revision this replaces threw for every read; the one before that + * answered from constants (four nodes in four regions, a timeline computed + * from the request clock, three pool templates nobody had polled). This one + * answers with what one local observer has actually recorded and labels it + * as exactly that. */ export class NetworkObservatoryService { - /** @asyncSafe */ + constructor(private readonly readers: ObservatoryReaders = { + relay: relayCollectorService, + templates: templateCollectorService, + localMempoolTxids: () => (config.MEMPOOL.ENABLED ? new Set(Object.keys(memPool.getMempool())) : null), + + }) {} + + /** @asyncSafe Every rejection is translated to the typed error. */ + private async overview(): Promise { + try { + return await this.readers.relay.getOverview(); + } catch (e) { + if (e instanceof RelayEvidenceError) { + throw new NetworkObservatoryEvidenceError(e.code, e.message, e.status); + } + throw new NetworkObservatoryEvidenceError('unavailable-owned-node', 'The owned node observation could not be read: ' + (e instanceof Error ? e.message : String(e))); + } + } + + private identity(overview: RelayOverview): ObserverIdentity { + return { + observerId: overview.collection.observer_id, + observers: 1, + clockOffsetMs: null, + clockUncertaintyMs: null, + method: overview.source.method, + }; + } + + private window(overview: RelayOverview): ObservationWindow { + return { + observedAtUtc: overview.source.observed_at_utc, + ageMs: overview.source.age_ms, + freshnessLimitMs: overview.source.freshness_limit_ms, + retentionMs: RELAY_LIMITS.retentionMs, + retainedTransactions: overview.collection.retained_transactions, + lastPollUtc: overview.collection.last_poll_utc, + lastCompletePollUtc: overview.collection.last_complete_poll_utc, + collection: overview.collection.status as ObservationWindow['collection'], + }; + } + + /** @asyncUnsafe Callers turn a rejection into an exact HTTP answer. */ public async $getNodes(): Promise { - throw new NetworkObservatoryEvidenceError('unavailable-observer-fleet', fleetUnavailable); + const overview = await this.overview(); + const local = this.readers.localMempoolTxids(); + return overview.sensors.map(sensor => ({ + id: sensor.id, + name: sensor.name, + region: 'unknown', + clientVersion: sensor.client_version, + protocolVersion: sensor.protocol_version, + fullRbf: sensor.full_rbf, + minRelayFeeRate: sensor.min_relay_feerate, + clockOffsetMs: null, + connectedPeers: sensor.connected_peers_count, + mempoolTxCount: local ? local.size : null, + status: overview.collection.status === 'observing' ? 'online' : 'degraded', + network: sensor.network, + observedAtUtc: sensor.last_heartbeat, + policySourceAvailable: sensor.policy_source_available, + scope: SCOPE, + })); + } + + private arrivals(record: TransactionRelayLifecycle, observerName: string): NodeArrival[] { + const first = Date.parse(record.first_observed_utc); + return record.events.map(event => ({ + nodeId: event.source_id, + nodeName: observerName, + arrivedAt: Date.parse(event.observed_at_utc), + deltaFromFirstMs: Math.max(0, Date.parse(event.observed_at_utc) - first), + accepted: event.payload.presence === 'present', + presence: event.payload.presence, + completePoll: event.payload.complete_poll, + previousCompletePollUtc: event.payload.previous_complete_poll_utc, + })); } - /** @asyncSafe */ - public async $getPropagation(_txid?: string): Promise { - throw new NetworkObservatoryEvidenceError('unavailable-observer-fleet', fleetUnavailable); + /** + * The lifecycle of one transaction as this observer saw it, or the most + * recently observed one when no txid is given. A transaction this + * observer never retained is 'not-observed'; an observer that has not + * polled yet is 'no-observations'. Neither is invented. + * @asyncUnsafe Callers turn a rejection into an exact HTTP answer. + */ + public async $getPropagation(txid?: string): Promise { + const overview = await this.overview(); + const observer = this.identity(overview); + const window = this.window(overview); + const base = { network: config.MEMPOOL.NETWORK, observer, window, medianLatencyMs: null, p95LatencyMs: null, spreadDeltaMs: null, scope: SCOPE } as const; + let record: TransactionRelayLifecycle | null = null; + if (txid !== undefined) { + try { + record = this.readers.relay.getPropagationForTx(txid); + } catch (e) { + if (e instanceof RelayEvidenceError && e.code === 'transaction-not-observed') { + return { ...base, txid: txid.toLowerCase(), state: 'not-observed', firstSeenTimestamp: null, lastSeenTimestamp: null, nodeObservations: [], expiresAtUtc: null }; + } + if (e instanceof RelayEvidenceError) { throw new NetworkObservatoryEvidenceError(e.code, e.message, e.status); } + throw e; + } + } else { + record = overview.recent_propagation_sample[0] ?? null; + if (!record) { + return { ...base, txid: null, state: 'no-observations', firstSeenTimestamp: null, lastSeenTimestamp: null, nodeObservations: [], expiresAtUtc: null }; + } + } + const observerName = overview.sensors[0]?.name ?? 'Owned Bitcoin Core node'; + return { + ...base, + txid: record.txid, + state: 'observed', + firstSeenTimestamp: Date.parse(record.first_observed_utc), + lastSeenTimestamp: Date.parse(record.last_observed_utc), + nodeObservations: this.arrivals(record, observerName), + expiresAtUtc: record.retention.expires_at_utc, + }; } - /** @asyncSafe */ + /** + * The candidate templates recorded for the latest height, compared with + * each other and with the local mempool. A deployment whose collector has + * not recorded a template yet says so rather than listing none. + * @asyncUnsafe Callers turn a rejection into an exact HTTP answer. + */ public async $getTemplates(): Promise { - throw new NetworkObservatoryEvidenceError('unavailable-observer-fleet', fleetUnavailable); + const sources = this.readers.templates.getSources().map(source => ({ + sourceId: source.source_id, name: source.name, status: source.status, lastTemplateAt: source.last_template_at, lastError: source.last_error, + })); + const observer: ObserverIdentity = { + observerId: 'owned-template-collector-' + config.MEMPOOL.NETWORK, + observers: 1, + clockOffsetMs: null, + clockUncertaintyMs: null, + method: 'owned Core getblocktemplate on a cold schedule and this backend\'s next-block projection', + }; + const retention = { templates: TEMPLATE_LIMITS.templates, txidsPerTemplate: TEMPLATE_LIMITS.txidsInResponse }; + const scope = 'Templates from the owned Core node and this backend\'s projection only; no pool template is observed. Fee-rate spread across pools is not measured.'; + const all = this.readers.templates.getTemplatesForHeight(); + if (!all.length) { + return { network: config.MEMPOOL.NETWORK, state: 'no-templates-observed', blockHeight: null, generatedAt: null, candidateTemplates: [], consensusMempoolTxCount: null, missingFromLocalCount: null, feeRateSpreadSatVb: null, sources, observer, retention, scope }; + } + const height = Math.max(...all.map(template => template.height)); + // The latest template per source at the latest height; older heights are history, not candidates. + const latestBySource = new Map(); + for (const template of all) { + if (template.height !== height) { continue; } + const previous = latestBySource.get(template.source_id); + if (!previous || Date.parse(template.observed_at_utc) >= Date.parse(previous.observed_at_utc)) { latestBySource.set(template.source_id, template); } + } + const candidates = [...latestBySource.values()]; + const local = this.readers.localMempoolTxids(); + const counts = new Map(); + for (const template of candidates) { for (const id of template.txids) { counts.set(id, (counts.get(id) ?? 0) + 1); } } + const missing = new Set(); + const candidateTemplates: CandidateTemplate[] = candidates.map(template => { + const notLocal = local ? template.txids.filter(id => !local.has(id)) : null; + for (const id of notLocal ?? []) { missing.add(id); } + return { + poolName: template.source_name, + templateId: template.template_id, + sourceId: template.source_id, + sourceType: template.source_type, + observedAtUtc: template.observed_at_utc, + prevBlockHash: template.prev_block_hash, + txCount: template.tx_count, + totalWeight: template.total_weight, + totalFeesSats: String(template.total_fees_sats), + expectedMedianFeeRate: null, + uniqueTxids: template.txids.filter(id => counts.get(id) === 1).slice(0, 200), + missingFromLocalMempool: notLocal ? notLocal.length : null, + }; + }); + return { + network: config.MEMPOOL.NETWORK, + state: 'observed', + blockHeight: height, + generatedAt: Math.max(...candidates.map(template => Date.parse(template.observed_at_utc))), + candidateTemplates, + consensusMempoolTxCount: local ? local.size : null, + missingFromLocalCount: local ? missing.size : null, + feeRateSpreadSatVb: null, + sources, + observer, + retention, + scope, + }; } } diff --git a/backend/src/api/network-observatory/network-observatory.types.ts b/backend/src/api/network-observatory/network-observatory.types.ts index f2c1266bd2c..3d4f00029cc 100644 --- a/backend/src/api/network-observatory/network-observatory.types.ts +++ b/backend/src/api/network-observatory/network-observatory.types.ts @@ -1,28 +1,66 @@ /** * Types for the Cross-Node Mempool, Relay, Policy, and Block-Template Observatory. + * + * Every read carries the identity of the observer that produced it and the + * time it was produced. This deployment has one observer: this backend's + * own mempool poll and its owned Core node. Dimensions a single local + * observer cannot measure (regions, clock offsets, inter-node latency, + * per-pool templates) are `null` or 'unknown', never estimated. */ +/** Who observed, and how far its clock can be trusted. */ +export interface ObserverIdentity { + readonly observerId: string; + readonly observers: 1; + readonly clockOffsetMs: number | null; + readonly clockUncertaintyMs: number | null; + readonly method: string; +} + +/** Freshness and bound of what is retained. */ +export interface ObservationWindow { + readonly observedAtUtc: string; + readonly ageMs: number; + readonly freshnessLimitMs: number; + readonly retentionMs: number; + readonly retainedTransactions: number; + readonly lastPollUtc: string | null; + readonly lastCompletePollUtc: string | null; + readonly collection: 'not_started' | 'observing' | 'interrupted' | 'stale'; +} + export interface ObserverNode { readonly id: string; readonly name: string; readonly region: string; readonly clientVersion: string; - readonly protocolVersion: number; - readonly fullRbf: boolean; - readonly minRelayFeeRate: number; - readonly clockOffsetMs: number; + readonly protocolVersion: number | null; + readonly fullRbf: boolean | null; + readonly minRelayFeeRate: number | null; + readonly clockOffsetMs: number | null; readonly connectedPeers: number; - readonly mempoolTxCount: number; + readonly mempoolTxCount: number | null; readonly status: 'online' | 'syncing' | 'degraded'; + readonly network: string; + readonly observedAtUtc: string; + readonly policySourceAvailable: boolean; + readonly scope: string; } export interface PropagationObservation { - readonly txid: string; - readonly firstSeenTimestamp: number; + readonly txid: string | null; + readonly network: string; + readonly state: 'observed' | 'not-observed' | 'no-observations'; + readonly firstSeenTimestamp: number | null; + readonly lastSeenTimestamp: number | null; readonly nodeObservations: readonly NodeArrival[]; - readonly medianLatencyMs: number; - readonly p95LatencyMs: number; - readonly spreadDeltaMs: number; + readonly medianLatencyMs: number | null; + readonly p95LatencyMs: number | null; + readonly spreadDeltaMs: number | null; + readonly observer: ObserverIdentity; + readonly window: ObservationWindow; + readonly expiresAtUtc: string | null; + readonly scope: string; } export interface NodeArrival { @@ -32,22 +70,45 @@ export interface NodeArrival { readonly deltaFromFirstMs: number; readonly accepted: boolean; readonly rejectionReason?: string; + readonly presence: 'present' | 'left_mempool'; + readonly completePoll: boolean; + readonly previousCompletePollUtc: string | null; } export interface BlockTemplateComparison { - readonly blockHeight: number; - readonly generatedAt: number; + readonly network: string; + readonly state: 'observed' | 'no-templates-observed'; + readonly blockHeight: number | null; + readonly generatedAt: number | null; readonly candidateTemplates: readonly CandidateTemplate[]; - readonly consensusMempoolTxCount: number; - readonly missingFromLocalCount: number; - readonly feeRateSpreadSatVb: number; + readonly consensusMempoolTxCount: number | null; + readonly missingFromLocalCount: number | null; + readonly feeRateSpreadSatVb: number | null; + readonly sources: readonly TemplateSourceStatus[]; + readonly observer: ObserverIdentity; + readonly retention: { readonly templates: number; readonly txidsPerTemplate: number }; + readonly scope: string; +} + +export interface TemplateSourceStatus { + readonly sourceId: string; + readonly name: string; + readonly status: 'active' | 'degraded' | 'offline' | 'not_collected'; + readonly lastTemplateAt: string | null; + readonly lastError: string | null; } export interface CandidateTemplate { readonly poolName: string; + readonly templateId: string; + readonly sourceId: string; + readonly sourceType: 'core_gbt' | 'mempool_projection'; + readonly observedAtUtc: string; + readonly prevBlockHash: string; readonly txCount: number; readonly totalWeight: number; readonly totalFeesSats: string; - readonly expectedMedianFeeRate: number; + readonly expectedMedianFeeRate: number | null; readonly uniqueTxids: readonly string[]; + readonly missingFromLocalMempool: number | null; } diff --git a/backend/src/api/prices/prices.routes.ts b/backend/src/api/prices/prices.routes.ts index cdf435f3280..08b2f7613cd 100644 --- a/backend/src/api/prices/prices.routes.ts +++ b/backend/src/api/prices/prices.routes.ts @@ -14,7 +14,9 @@ class PricesRoutes { res.header('Cache-control', 'public'); res.setHeader('Expires', new Date(Date.now() + 360_0000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR).toUTCString()); - res.json(pricesUpdater.getLatestPrices()); + // The served price plus its provenance: a disabled or stale observation + // is marked as such rather than presented as the current quote. + res.json(pricesUpdater.getAdvertisedPrices()); } } diff --git a/backend/src/api/redis-cache.ts b/backend/src/api/redis-cache.ts index 80a947c12fd..8cbd768f9e7 100644 --- a/backend/src/api/redis-cache.ts +++ b/backend/src/api/redis-cache.ts @@ -435,6 +435,24 @@ class RedisCache { public setIgnoreBlocksCache(): void { this.ignoreBlocksCache = true; } + + /** + * Atomically claims a key for ttlMs (SET NX PX). Returns true when this call + * made the claim and false when the key already existed. Throws when Redis + * is disabled or not connected, so the caller can fail closed. + * + * @asyncUnsafe The caller turns a rejection into an unavailable answer. + */ + public async $claimOnce(key: string, ttlMs: number): Promise { + if (!config.REDIS.ENABLED) { + throw new Error('Redis is disabled'); + } + if (!this.connected) { + throw new Error('Redis is not connected'); + } + const answer = await this.client.set(key, '1', { NX: true, PX: Math.max(1, Math.round(ttlMs)) }); + return answer === 'OK'; + } } export default new RedisCache(); diff --git a/backend/src/api/tx-selection-worker.ts b/backend/src/api/tx-selection-worker.ts index 98dd429096f..fcc1347921f 100644 --- a/backend/src/api/tx-selection-worker.ts +++ b/backend/src/api/tx-selection-worker.ts @@ -32,7 +32,7 @@ if (parentPort) { * Build projected mempool blocks using an approximation of the transaction selection algorithm from Bitcoin Core * (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp) */ -function makeBlockTemplates(mempool: Map) +export function makeBlockTemplates(mempool: Map) : { blocks: number[][], rates: Map, clusters: Map } { const start = Date.now(); const auditPool: Map = new Map(); @@ -85,7 +85,7 @@ function makeBlockTemplates(mempool: Map) // (i.e. the package rooted in the transaction with the best ancestor score) const blocks: number[][] = []; let blockWeight = 4000; - const blockSigops = 0; + let blockSigops = 0; let transactions: AuditTransaction[] = []; const modified: PairingHeap = new PairingHeap((a, b): boolean => { if (a.score === b.score) { @@ -152,6 +152,10 @@ function makeBlockTemplates(mempool: Map) mempoolTx.cpfpChecked = true; transactions.push(ancestor); blockWeight += ancestor.weight; + // Each ancestor is counted once, here, as it is placed. The package + // check above used ancestorSigops, which updateDescendants trims as + // ancestors get placed, so nothing is counted twice. + blockSigops += ancestor.sigops; used.push(ancestor); } @@ -184,6 +188,7 @@ function makeBlockTemplates(mempool: Map) // reset for the next block transactions = []; blockWeight = 4000; + blockSigops = 0; // 'overflow' packages didn't fit in this block, but are valid candidates for the next for (const overflowTx of overflow.reverse()) { diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 55545cec62d..76b96e39f59 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -38,6 +38,22 @@ import bitcoinSecondClient from './bitcoin/bitcoin-second-client'; import { calculateMempoolTxCpfp } from './cpfp'; import stratumApi, { StratumJob } from './services/stratum'; +/** Most transaction ids one connection may track at once. */ +const MAX_TRACKED_TXS = 100; +/** Authoritative status lookups in flight for one track-txs request. */ +const TRACKED_TX_LOOKUP_CONCURRENCY = 4; +/** Upper bound on one authoritative status lookup. */ +const TRACKED_TX_LOOKUP_DEADLINE_MS = 10_000; + +function isTransactionNotFound(e: unknown): boolean { + const error = e as { response?: { status?: number }; message?: string } | null; + if (error?.response?.status === 404) { + return true; + } + const message = typeof error?.message === 'string' ? error.message : ''; + return message.indexOf('No such mempool or blockchain transaction') > -1; +} + // valid 'want' subscriptions const wantable = [ 'blocks', @@ -99,7 +115,7 @@ class WebsocketHandler { 'mempoolInfo': memPool.getMempoolInfo(), 'vBytesPerSecond': memPool.getVBytesPerSecond(), 'blocks': _blocks, - 'conversions': priceUpdater.getLatestPrices(), + 'conversions': priceUpdater.getAdvertisedPrices(), 'mempool-blocks': mempoolBlocks.getMempoolBlocks(), 'transactions': memPool.getLatestTransactions(), 'backendInfo': backendInfo.getBackendInfo(), @@ -229,45 +245,27 @@ class WebsocketHandler { const txids: string[] = []; if (Array.isArray(parsedMessage['track-txs'])) { for (const txid of parsedMessage['track-txs']) { - if (/^[a-fA-F0-9]{64}$/.test(txid)) { + if (typeof txid === 'string' && /^[a-fA-F0-9]{64}$/.test(txid) && !txids.includes(txid)) { txids.push(txid); } } } - const txs: { [txid: string]: TxTrackingInfo } = {}; - for (const txid of txids) { - const txInfo: TxTrackingInfo = { - confirmed: true, - }; - const rbfCacheTxid = rbfCache.getReplacedBy(txid); - if (rbfCacheTxid) { - txInfo.replacedBy = rbfCacheTxid; - txInfo.confirmed = false; - } - const tx = memPool.getMempool()[txid]; - if (tx && tx.position) { - txInfo.position = { - ...tx.position - }; - if (tx.acceleration) { - txInfo.accelerated = tx.acceleration; - } - } - if (tx) { - txInfo.confirmed = false; - } - txs[txid] = txInfo; - } - - if (txids.length) { - client['track-txs'] = txids; - } else { + if (txids.length > MAX_TRACKED_TXS) { + response['track-txs-error'] = `"too many transactions requested, this connection supports tracking a maximum of ${MAX_TRACKED_TXS} transactions"`; client['track-txs'] = null; - } + } else { + const txs = await this.$resolveTrackedTxs(txids); + + if (txids.length) { + client['track-txs'] = txids; + } else { + client['track-txs'] = null; + } - if (Object.keys(txs).length) { - response['tracked-txs'] = JSON.stringify(txs); + if (Object.keys(txs).length) { + response['tracked-txs'] = JSON.stringify(txs); + } } } @@ -466,14 +464,99 @@ class WebsocketHandler { } } + /** + * The initial status of every tracked transaction, resolved with bounded + * concurrency. Absence from this process's mempool is never treated as + * confirmation: a transaction that is neither here nor replaced is looked + * up at the node, and only a real block height and hash may confirm it. + * + * @asyncSafe Every lookup failure becomes an unavailable status. + */ + private async $resolveTrackedTxs(txids: string[]): Promise<{ [txid: string]: TxTrackingInfo }> { + const txs: { [txid: string]: TxTrackingInfo } = {}; + const queue = txids.slice(); + const worker = async (): Promise => { + while (queue.length) { + const txid = queue.shift(); + if (!txid) { + return; + } + try { + txs[txid] = await this.$trackedTxStatus(txid); + } catch (e) { + logger.debug(`Could not resolve tracked transaction ${txid}: ` + (e instanceof Error ? e.message : e)); + txs[txid] = { status: 'unavailable' }; + } + } + }; + try { + await Promise.all(Array.from({ length: Math.min(TRACKED_TX_LOOKUP_CONCURRENCY, txids.length) }, worker)); + } catch (e) { + // Each worker answers its own failures; this only guards the join. + logger.debug('Tracked transaction lookups did not complete: ' + (e instanceof Error ? e.message : e)); + } + return txs; + } + + /** @asyncSafe */ + private async $trackedTxStatus(txid: string): Promise { + const rbfCacheTxid = rbfCache.getReplacedBy(txid); + if (rbfCacheTxid) { + return { replacedBy: rbfCacheTxid, confirmed: false, status: 'replaced' }; + } + const tx = memPool.getMempool()[txid]; + if (tx) { + const txInfo: TxTrackingInfo = { confirmed: false, status: 'mempool' }; + if (tx.position) { + txInfo.position = { ...tx.position }; + if (tx.acceleration) { + txInfo.accelerated = tx.acceleration; + } + } + return txInfo; + } + let deadline: NodeJS.Timeout | undefined; + try { + const lookup = bitcoinApi.$getRawTransaction(txid, true, false); + // A lookup that outlives the deadline must not surface as an unhandled rejection. + lookup.catch(() => undefined); + const expired = new Promise((_resolve, reject) => { + deadline = setTimeout(() => reject(new Error('transaction status lookup timed out')), TRACKED_TX_LOOKUP_DEADLINE_MS); + }); + const found = await Promise.race([lookup, expired]); + const status = found?.status; + if ( + status?.confirmed === true + && typeof status.block_height === 'number' && Number.isInteger(status.block_height) && status.block_height >= 0 + && typeof status.block_hash === 'string' && /^[a-fA-F0-9]{64}$/.test(status.block_hash) + ) { + return { confirmed: true, status: 'confirmed', blockHeight: status.block_height, blockHash: status.block_hash }; + } + // Known to the node but not yet in a block, or in a block the node + // could not identify: either way it is not confirmed. + return { confirmed: false, status: 'mempool' }; + } catch (e) { + if (isTransactionNotFound(e)) { + return { confirmed: false, status: 'unknown' }; + } + logger.debug(`Could not resolve tracked transaction ${txid}: ` + (e instanceof Error ? e.message : e)); + return { status: 'unavailable' }; + } finally { + if (deadline !== undefined) { + clearTimeout(deadline); + } + } + } + handleNewConversionRates(conversionRates: ApiPrice) { if (!this.webSocketServers.length) { throw new Error('No WebSocket.Server have been set'); } - this.updateSocketDataFields({ 'conversions': conversionRates }); + const advertised = { ...conversionRates, ...priceUpdater.getPriceObservation() }; + this.updateSocketDataFields({ 'conversions': advertised }); - const response = JSON.stringify({ conversions: conversionRates }); + const response = JSON.stringify({ conversions: advertised }); // TODO - Fix indentation after PR is merged for (const server of this.webSocketServers) { server.clients.forEach((client) => { diff --git a/backend/src/config.ts b/backend/src/config.ts index 2f31e694903..87d60912283 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -309,7 +309,7 @@ const defaults: IConfig = { 'EXTERNAL_DATA_SERVER': { // Unset by default: this deployment never calls a hosted third-party API. 'MEMPOOL_API': '', - 'MEMPOOL_ONION': 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1', + 'MEMPOOL_ONION': '', 'LIQUID_API': 'https://liquid.network/api/v1', 'LIQUID_ONION': 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1' }, diff --git a/backend/src/index.ts b/backend/src/index.ts index 78786f2bc26..dade4e48f53 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -91,8 +91,10 @@ import { timeMachineService } from './api/intelligence/time-machine/time-machine import { relayCollectorService } from './api/intelligence/relay/relay-collector.service'; import { boundedHistoryFlush } from './api/intelligence/time-machine/history-shutdown'; import { templateCollectorService } from './api/intelligence/templates/template-collector.service'; +import { orderingEvidenceService } from './api/intelligence/private-submission/ordering-evidence.service'; import { globalNetworkService } from './api/intelligence/global-network/global-network.service'; import { developerIdentity } from './api/intelligence/identity/developer-identity'; +import { startPrivateRelayWorker } from './api/intelligence/private-submission/private-relay.runtime'; import rbfCache from './api/rbf-cache'; import globalNetworkRoutes from './api/intelligence/global-network/global-network.routes'; import lightningReliabilityRoutes from './api/intelligence/lightning/lightning-reliability.routes'; @@ -228,6 +230,10 @@ class Server { next(); }) .use('/internal/admin/v1', adminAdapterJsonParser()) + // Bootstrap operator jobs reuse the admin adapter signature, which is + // computed over the raw body: the raw bytes must be captured before the + // general JSON parser consumes them. + .use('/api/v1/intelligence/bootstrap/operator', adminAdapterJsonParser()) .use(express.urlencoded({ extended: true, limit: '10mb' })) .use(express.text({ type: ['text/plain', 'application/base64'], limit: '10mb' })) .use(express.json({ limit: '10mb' })) @@ -461,6 +467,8 @@ class Server { else timeMachineService.markObservationFailure(); }); blockObservationHub.subscribe('templates', (block, transactions) => { templateCollectorService.observeBlock(block, transactions); }); + // Ordering evidence compares the mined order with the templates recorded for the height, so it reads after the template collector. + blockObservationHub.subscribe('ordering-evidence', (block, transactions) => { orderingEvidenceService.observeBlock(block, transactions); }); // The hub contains each observer's failure; the matcher's own rejection is reported there. blockObservationHub.subscribe('watchlist-matcher', (block, transactions) => watchlistMatcher.observeBlock(block, transactions).then(() => undefined)); blocks.setNewAsyncBlockCallback((block, _txIds, transactions) => blockObservationHub.dispatch(block, transactions).then(() => undefined)); @@ -479,6 +487,9 @@ class Server { developerIdentity.startOutboxWorker(); // Cold schedule: one getblocktemplate every two minutes plus one after each block. templateCollectorService.startPolling(); + // Leased relay worker: reclaims expired leases on start, so a restart + // mid-relay resumes instead of stranding the row. + startPrivateRelayWorker(); globalNetworkService.startSnapshots(10 * 60_000, () => blocks.getCurrentBlockHeight()); } websocketHandler.setupConnectionHandling(); diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 327bbbebf0a..17ee671f526 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -21,6 +21,30 @@ export interface CoreIndex { type TaskName = 'blocksPrices' | 'coinStatsIndex'; +export type SingleTaskStatus = + | 'completed' + | 'already-running' + | 'disabled' + | 'deferred' + | 'network-inapplicable' + | 'failed'; + +/** + * What actually happened when a single task was asked to run. A caller that + * needs authoritative completion (the admin adapter) must only trust + * `completed`, and even then confirm the persisted rows itself. + */ +export interface SingleTaskOutcome { + task: TaskName; + status: SingleTaskStatus; + /** Why the task did not run, for every status other than completed. */ + reason?: string; + /** The failure message, for status failed. */ + error?: string; + /** Wall-clock bounds of the work, for status completed. */ + checkpoint?: { startedAt: string; finishedAt: string }; +} + class Indexer { private runIndexer = true; private indexerRunning = false; @@ -77,14 +101,20 @@ class Indexer { return null; } - public reindex(): void { - if (Common.indexingEnabled()) { - if (this.reindexTimeout) { - clearTimeout(this.reindexTimeout); - this.reindexTimeout = undefined; - } - this.runIndexer = true; + /** + * Releases the indexing loop to run again. Returns whether the request was + * accepted; it is refused when indexing is disabled in this deployment. + */ + public reindex(): boolean { + if (!Common.indexingEnabled()) { + return false; + } + if (this.reindexTimeout) { + clearTimeout(this.reindexTimeout); + this.reindexTimeout = undefined; } + this.runIndexer = true; + return true; } private scheduleNextRun(timeout: number): void { @@ -110,17 +140,26 @@ class Indexer { return; } else { // debounce clearTimeout(this.tasksScheduled[task]); + delete this.tasksScheduled[task]; } } - this.tasksScheduled[task] = setTimeout(async () => { + const handle = setTimeout(async () => { + // Release the slot before the work starts so the task can schedule its + // own successor. Identity-compared: a replaced timer's callback never + // fires, but it must not be able to clear a newer handle either way. + if (this.tasksScheduled[task] === handle) { + delete this.tasksScheduled[task]; + } try { - await this.runSingleTask(task); + const outcome = await this.runSingleTask(task); + if (outcome.status === 'failed') { + logger.err(`Scheduled task ${task} failed: ${outcome.error ?? 'unknown error'}`); + } } catch (e) { logger.err(`Unexpected error in scheduled task ${task}: ` + (e instanceof Error ? e.message : e)); - } finally { - clearTimeout(this.tasksScheduled[task]); } }, timeout); + this.tasksScheduled[task] = handle; } /** @@ -130,41 +169,55 @@ class Indexer { * * @asyncSafe */ - public async runSingleTask(task: TaskName): Promise { - if (!Common.indexingEnabled() || this.tasksRunning[task]) { - return; + public async runSingleTask(task: TaskName): Promise { + if (!Common.indexingEnabled()) { + return { task, status: 'disabled', reason: 'Indexing is disabled in this deployment.' }; + } + if (this.tasksRunning[task]) { + return { task, status: 'already-running', reason: `The ${task} task is already running.` }; } this.tasksRunning[task] = true; + const startedAt = new Date().toISOString(); - switch (task) { - case 'blocksPrices': { - if (!['testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK) && config.FIAT_PRICE.ENABLED) { - let lastestPriceId; + try { + switch (task) { + case 'blocksPrices': { + if (['testnet', 'signet', 'testnet4', 'regtest'].includes(config.MEMPOOL.NETWORK)) { + return { task, status: 'network-inapplicable', reason: `Coins on ${config.MEMPOOL.NETWORK} have no fiat price.` }; + } + if (!config.FIAT_PRICE.ENABLED) { + return { task, status: 'disabled', reason: 'Fiat prices are switched off in this deployment.' }; + } + let lastestPriceId: number | null | undefined; try { lastestPriceId = await PricesRepository.$getLatestPriceId(); } catch (e) { logger.debug('failed to fetch latest price id from db: ' + (e instanceof Error ? e.message : e)); - } if (priceUpdater.historyInserted === false || lastestPriceId === null) { + } + if (priceUpdater.historyInserted === false || lastestPriceId === null || lastestPriceId === undefined) { logger.debug(`Blocks prices indexer is waiting for the price updater to complete`, logger.tags.mining); this.scheduleSingleTask(task, 10000); - } else { - logger.debug(`Blocks prices indexer will run now`, logger.tags.mining); - await mining.$indexBlockPrices(); + return { task, status: 'deferred', reason: 'Price history is not available yet; the task was rescheduled.' }; } + logger.debug(`Blocks prices indexer will run now`, logger.tags.mining); + await mining.$indexBlockPrices(); + return { task, status: 'completed', checkpoint: { startedAt, finishedAt: new Date().toISOString() } }; } - } break; - case 'coinStatsIndex': { - logger.debug(`Indexing coinStatsIndex now`); - try { + case 'coinStatsIndex': { + logger.debug(`Indexing coinStatsIndex now`); await mining.$indexCoinStatsIndex(); - } catch (e) { - logger.debug(`failed to index coinstatsindex: ` + (e instanceof Error ? e.message : e)); + return { task, status: 'completed', checkpoint: { startedAt, finishedAt: new Date().toISOString() } }; } - } break; + } + return { task, status: 'failed', error: `Unknown task ${String(task)}.` }; + } catch (e) { + const error = e instanceof Error ? e.message : String(e); + logger.debug(`failed to run ${task}: ` + error); + return { task, status: 'failed', error }; + } finally { + this.tasksRunning[task] = false; } - - this.tasksRunning[task] = false; } /** @asyncSafe */ diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 4064d957878..2a8a89f33eb 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -474,7 +474,11 @@ export interface TxTrackingInfo { acceleratedBy?: number[], acceleratedAt?: number, feeDelta?: number, - confirmed?: boolean + confirmed?: boolean, + /** Additive: how the initial status was established. Absent on later delta frames. */ + status?: 'confirmed' | 'mempool' | 'replaced' | 'unknown' | 'unavailable', + blockHeight?: number, + blockHash?: string, } export interface WebsocketResponse { diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index 34646e30f75..464da2ddf8e 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -34,6 +34,35 @@ export interface ConversionRates { [currency: string]: number } +/** Networks whose coins have no fiat value; nothing prices them. */ +const UNPRICED_NETWORKS = ['testnet', 'signet', 'testnet4', 'regtest']; + +/** + * How old a stored observation may be and still be advertised as the + * current price. Matches the staleness bound the capability report and the + * admin snapshot already apply to the newest price. + */ +export const PRICE_MAX_AGE_SECONDS = 3 * 60 * 60; + +export type PriceObservationState = 'live' | 'stale' | 'disabled' | 'network-inapplicable' | 'unavailable'; + +/** + * The provenance of whatever price is being served. Additive to the price + * object so an old consumer keeps reading currency fields while a new one can + * tell a live quote from a retained observation. + */ +export interface PriceObservation { + state: PriceObservationState; + source: 'feed' | 'database' | null; + /** Unix seconds of the observation being reported, live or retained. */ + observedAt: number | null; + ageSeconds: number | null; + stale: boolean; + disabled: boolean; + /** The newest retained observation when it is not being served as live. */ + storedObservation: ApiPrice | null; +} + function getMedian(arr: number[]): number { const sortedArr = arr.slice().sort((a, b) => a - b); const mid = Math.floor(sortedArr.length / 2); @@ -57,6 +86,9 @@ class PriceUpdater { private currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY']; private latestPrices: ApiPrice; private latestGoodPrices: ApiPrice; + /** The newest observation read from the database, kept with its timestamp even when it is not served. */ + private storedPrices: ApiPrice | null = null; + private liveSource: 'feed' | 'database' | null = null; private currencyConversionFeed: ConversionFeed | undefined; private newCurrencies: string[] = ['BGN', 'BRL', 'CNY', 'CZK', 'DKK', 'HKD', 'HRK', 'HUF', 'IDR', 'ILS', 'INR', 'ISK', 'KRW', 'MXN', 'MYR', 'NOK', 'NZD', 'PHP', 'PLN', 'RON', 'RUB', 'SEK', 'SGD', 'THB', 'TRY', 'ZAR']; private lastTimeConversionsRatesFetched: number = 0; @@ -77,10 +109,69 @@ class PriceUpdater { this.setCyclePosition(); } + /** Only a price this deployment may serve as current; every field is -1 otherwise. */ public getLatestPrices(): ApiPrice { return this.latestGoodPrices; } + /** Whether this deployment prices its coin at all. */ + public pricingApplies(): boolean { + return config.FIAT_PRICE.ENABLED === true && !UNPRICED_NETWORKS.includes(config.MEMPOOL.NETWORK); + } + + private ageSeconds(observation: ApiPrice | null, nowMs: number): number | null { + if (!observation || typeof observation.time !== 'number' || observation.time <= 0) { + return null; + } + return Math.max(0, Math.round(nowMs / 1000 - observation.time)); + } + + /** The state of the served price, computed from what is actually held. */ + public getPriceObservation(nowMs: number = Date.now()): PriceObservation { + const disabled = config.FIAT_PRICE.ENABLED !== true; + const networkInapplicable = UNPRICED_NETWORKS.includes(config.MEMPOOL.NETWORK); + const live = this.latestGoodPrices.USD > 0; + const liveAge = this.ageSeconds(this.latestGoodPrices, nowMs); + const stored = this.storedPrices && this.storedPrices.USD > 0 ? this.storedPrices : null; + if (disabled || networkInapplicable) { + return { + state: networkInapplicable ? 'network-inapplicable' : 'disabled', + source: stored ? 'database' : null, + observedAt: stored?.time ?? null, + ageSeconds: this.ageSeconds(stored, nowMs), + stale: stored !== null, + disabled: true, + storedObservation: stored, + }; + } + if (live) { + const stale = liveAge !== null && liveAge > PRICE_MAX_AGE_SECONDS; + return { + state: stale ? 'stale' : 'live', + source: this.liveSource, + observedAt: this.latestGoodPrices.time ?? null, + ageSeconds: liveAge, + stale, + disabled: false, + storedObservation: null, + }; + } + return { + state: stored ? 'stale' : 'unavailable', + source: stored ? 'database' : null, + observedAt: stored?.time ?? null, + ageSeconds: this.ageSeconds(stored, nowMs), + stale: stored !== null, + disabled: false, + storedObservation: stored, + }; + } + + /** What the prices route and the websocket advertise: the served price plus its provenance. */ + public getAdvertisedPrices(nowMs: number = Date.now()): ApiPrice & PriceObservation { + return { ...this.getLatestPrices(), ...this.getPriceObservation(nowMs) }; + } + public getEmptyPricesObj(): ApiPrice { return { time: 0, @@ -130,9 +221,25 @@ class PriceUpdater { * * @asyncUnsafe */ - public async $initializeLatestPriceWithDb(): Promise { - this.latestPrices = await PricesRepository.$getLatestConversionRates(); - this.latestGoodPrices = JSON.parse(JSON.stringify(this.latestPrices)); + public async $initializeLatestPriceWithDb(nowMs: number = Date.now()): Promise { + const stored = await PricesRepository.$getLatestConversionRates(); + this.storedPrices = stored.USD > 0 ? stored : null; + if (!this.pricingApplies()) { + if (this.storedPrices) { + logger.info(`Fiat prices are not served on this deployment; the stored observation from ${new Date((this.storedPrices.time ?? 0) * 1000).toISOString()} is retained but not advertised as current.`); + } + return; + } + const age = this.ageSeconds(this.storedPrices, nowMs); + if (!this.storedPrices || age === null || age > PRICE_MAX_AGE_SECONDS) { + if (this.storedPrices) { + logger.warn(`The stored fiat price is ${Math.round((age ?? 0) / 3600)} hours old and is not served as current until the feed answers.`); + } + return; + } + this.latestPrices = JSON.parse(JSON.stringify(stored)); + this.latestGoodPrices = JSON.parse(JSON.stringify(stored)); + this.liveSource = 'database'; } /** @asyncSafe */ @@ -190,6 +297,7 @@ class PriceUpdater { if (price > 0) { this.latestGoodPrices[currency] = price; this.latestGoodPrices.time = Math.round(new Date().getTime() / 1000); + this.liveSource = 'feed'; } } diff --git a/docker/README.md b/docker/README.md index 2658914eb60..de758221ee7 100644 --- a/docker/README.md +++ b/docker/README.md @@ -7,10 +7,44 @@ If you are looking to use these Docker images to deploy your own instance of Mem See a video guide of this installation method by k3tan [on BitcoinTV.com](https://bitcointv.com/w/8fpAx6rf5CQ16mMhospwjg). Jump to a section in this doc: +- [Owned endpoint prerequisites](#owned-endpoint-prerequisites) - [Configure with Bitcoin Core Only](#configure-with-bitcoin-core-only) - [Configure with Bitcoin Core + Electrum Server](#configure-with-bitcoin-core--electrum-server) - [Further Configuration](#further-configuration) +## Owned endpoint prerequisites + +The images never contact a third-party blockchain service on their own. Every +endpoint default is empty or same-origin, so an unconfigured feature fails +visibly with the name of the prerequisite instead of quietly reading from +somewhere else. The start scripts (`backend/start.sh`, `frontend/entrypoint.sh`) +accept an endpoint only when it is empty, a `/path` on the same origin, or an +`http(s)` URL without credentials whose host is loopback, a single-label +container name, or under `bitcoinuniverse.io`; anything else stops the +container with exit code 78, naming the field and the host but never the value. + +| Feature | Backend variable | Frontend variable | Default and unconfigured behaviour | +| --- | --- | --- | --- | +| About page (contributors, donations, translators) | `EXTERNAL_DATA_SERVER_MEMPOOL_API` | | empty; the routes are not mounted | +| Accounts, accelerations, invoices, faucet, Lightning metadata, proofs | `MEMPOOL_SERVICES_API` | `SERVICES_API` | empty; capabilities report the dependency as not configured and the frontend stays on its own origin | +| The same over Tor | `EXTERNAL_DATA_SERVER_MEMPOOL_ONION` | `ONION_SERVICES_API` | empty; both must name this deployment's own `.onion` host, and the page hostname alone never switches providers | +| Services proxied through nginx | | `PROXIED_SERVICES=true` with `PROXIED_SERVICES_HOST` | off; `true` without a host refuses to start | +| Fiat prices | `FIAT_PRICE_ENABLED` | | `false`; `/api/v1/prices` answers `-1` until an owned price source is configured | +| Mining pool table refresh | `MEMPOOL_AUTOMATIC_POOLS_UPDATE` with `MEMPOOL_POOLS_JSON_URL` and `MEMPOOL_POOLS_JSON_TREE_URL` | | `false` and empty; the bundled `pools-v2.json` is used | +| Cross-network links | | `MEMPOOL_WEBSITE_URL` | `https://explorer.bitcoinuniverse.io` | + +The one permitted external runtime category is **Liquid asset catalogue +metadata**, read from `EXTERNAL_DATA_SERVER_LIQUID_API` and +`EXTERNAL_DATA_SERVER_LIQUID_ONION` (defaults: `liquid.network` and its onion). +It covers asset names, tickers, precision, icons and issuer details only. +Balances, ownership, transfers and history come from the owned Elements node and +indexer; those two fields accept no other host than the catalogue or an owned +one. + +`scripts/universe/docker-runtime-defaults.test.mjs` runs both start scripts +against the shipped templates and proves the rendered configuration for a +minimal environment, for explicit owned endpoints, and for rejected input. + ## Configure with Bitcoin Core Only _Note: address lookups require an Electrum Server and will not work with this configuration. [Add an Electrum Server](#configure-with-bitcoin-core--electrum-server) to your backend for full functionality._ @@ -107,8 +141,8 @@ Below we list all settings from `mempool-config.json` and the corresponding over "STDOUT_LOG_MIN_PRIORITY": "info", "INDEXING_BLOCKS_AMOUNT": false, "AUTOMATIC_POOLS_UPDATE": false, - "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json", - "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master", + "POOLS_JSON_URL": "", + "POOLS_JSON_TREE_URL": "", "POOLS_UPDATE_DELAY": 604800, "CPFP_INDEXING": false, "MAX_BLOCKS_BULK_QUERY": 0, diff --git a/docker/backend/Dockerfile b/docker/backend/Dockerfile index 049f13f6fbb..8ee1e1de470 100644 --- a/docker/backend/Dockerfile +++ b/docker/backend/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.84-bookworm AS builder +FROM rust:1.89.0-bookworm AS builder ARG commitHash ENV MEMPOOL_COMMIT_HASH=${commitHash} @@ -8,7 +8,7 @@ WORKDIR /build RUN apt-get update && \ apt-get install -y curl ca-certificates && \ curl -fsSL https://deb.nodesource.com/setup_24.x | bash - && \ - apt-get install -y nodejs=24.13.0-1nodesource1 build-essential python3 pkg-config && \ + apt-get install -y nodejs=24.19.0-1nodesource1 build-essential python3 pkg-config && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* @@ -19,20 +19,27 @@ ENV PATH="/usr/local/cargo/bin:$PATH" COPY --from=backend . . COPY --from=rustgbt . ../rust/ ENV FD=/build/rust-gbt -RUN npm install --omit=dev --omit=optional +# The pinned runtime pair and the toolchain that builds the native engines; +# a base image that drifts fails here, not later. +RUN node --version | grep -x v24.19.0 && npm --version | grep -x 11.17.0 && cargo --version | grep -F 1.89.0 +# Locked install (preinstall builds rust-gbt with this toolchain); the +# vendored first-party tarball in backend/vendor is part of the build context. +RUN npm ci --omit=dev --omit=optional WORKDIR /build RUN npm run package -FROM rust:1.84-bookworm AS runtime +FROM rust:1.89.0-bookworm AS runtime RUN apt-get update && \ apt-get install -y curl ca-certificates && \ curl -fsSL https://deb.nodesource.com/setup_24.x | bash - && \ - apt-get install -y nodejs=24.13.0-1nodesource1 && \ + apt-get install -y nodejs=24.19.0-1nodesource1 && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* +RUN node --version | grep -x v24.19.0 && npm --version | grep -x 11.17.0 + WORKDIR /backend RUN chown 1000:1000 ./ diff --git a/docker/backend/start.sh b/docker/backend/start.sh index cfd782977d9..f996626a6c7 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -27,9 +27,11 @@ __MEMPOOL_EXTERNAL_MAX_RETRY__=${MEMPOOL_EXTERNAL_MAX_RETRY:=1} __MEMPOOL_EXTERNAL_RETRY_INTERVAL__=${MEMPOOL_EXTERNAL_RETRY_INTERVAL:=0} __MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool} __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info} -__MEMPOOL_AUTOMATIC_POOLS_UPDATE__=${MEMPOOL_AUTOMATIC_POOLS_UPDATE:=true} -__MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json} -__MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master} +# The mining pool table ships with the backend (POOLS_JSON_FILE). Fetching a +# newer table at runtime is opt-in and needs an explicit source. +__MEMPOOL_AUTOMATIC_POOLS_UPDATE__=${MEMPOOL_AUTOMATIC_POOLS_UPDATE:=false} +__MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=""} +__MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=""} __MEMPOOL_POOLS_JSON_FILE__=${MEMPOOL_POOLS_JSON_FILE:=tasks/pools/pools-v2.json} __MEMPOOL_POOLS_UPDATE_DELAY__=${MEMPOOL_POOLS_UPDATE_DELAY:=604800} __MEMPOOL_AUDIT__=${MEMPOOL_AUDIT:=false} @@ -109,8 +111,11 @@ __SOCKS5PROXY_USERNAME__=${SOCKS5PROXY_USERNAME:=""} __SOCKS5PROXY_PASSWORD__=${SOCKS5PROXY_PASSWORD:=""} # EXTERNAL_DATA_SERVER -__EXTERNAL_DATA_SERVER_MEMPOOL_API__=${EXTERNAL_DATA_SERVER_MEMPOOL_API:=https://mempool.space/api/v1} -__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__=${EXTERNAL_DATA_SERVER_MEMPOOL_ONION:=http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1} +# Unset by default: the about-page and acceleration routes that read these +# stay unmounted until an operator names a Bitcoin Universe endpoint. The +# Liquid asset catalogue is the one documented external metadata source. +__EXTERNAL_DATA_SERVER_MEMPOOL_API__=${EXTERNAL_DATA_SERVER_MEMPOOL_API:=""} +__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__=${EXTERNAL_DATA_SERVER_MEMPOOL_ONION:=""} __EXTERNAL_DATA_SERVER_LIQUID_API__=${EXTERNAL_DATA_SERVER_LIQUID_API:=https://liquid.network/api/v1} __EXTERNAL_DATA_SERVER_LIQUID_ONION__=${EXTERNAL_DATA_SERVER_LIQUID_ONION:=http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1} @@ -148,7 +153,9 @@ __REPLICATION_STATISTICS_START_TIME__=${REPLICATION_STATISTICS_START_TIME:=14819 __REPLICATION_SERVERS__=${REPLICATION_SERVERS:=[]} # MEMPOOL_SERVICES -__MEMPOOL_SERVICES_API__=${MEMPOOL_SERVICES_API:="https://mempool.space/api/v1/services"} +# Unset by default: accounts, accelerations and the other services routes +# report "unconfigured" until an operator names a Bitcoin Universe endpoint. +__MEMPOOL_SERVICES_API__=${MEMPOOL_SERVICES_API:=""} __MEMPOOL_SERVICES_ACCELERATIONS__=${MEMPOOL_SERVICES_ACCELERATIONS:=false} # STRATUM @@ -161,10 +168,69 @@ __REDIS_UNIX_SOCKET_PATH__=${REDIS_UNIX_SOCKET_PATH:=""} __REDIS_BATCH_QUERY_BASE_SIZE__=${REDIS_BATCH_QUERY_BASE_SIZE:=5000} # FIAT_PRICE -__FIAT_PRICE_ENABLED__=${FIAT_PRICE_ENABLED:=true} +# Off by default: the price task calls public exchange APIs from the backend. +# Turn it on only for a deployment whose price source is its own. +__FIAT_PRICE_ENABLED__=${FIAT_PRICE_ENABLED:=false} __FIAT_PRICE_PAID__=${FIAT_PRICE_PAID:=false} __FIAT_PRICE_API_KEY__=${FIAT_PRICE_API_KEY:=""} +# Runtime data comes from Bitcoin Universe infrastructure only. An endpoint is +# empty, a same-origin /path, or an http(s) URL without credentials whose host +# is loopback, a single-label container name, or under bitcoinuniverse.io. +# Onion fields may name a .onion host, never the upstream explorer's. Liquid +# fields may name the Liquid asset catalogue (metadata only). Anything else +# stops the container before configuration is rendered, and the value is not +# echoed: only the field name and, when it is not a credential problem, the +# host. +universe_endpoint_check() { + name="$1" + value="$2" + extra="$3" + [ -z "${value}" ] && return 0 + case "${value}" in + /*) return 0 ;; + http://*|https://*) ;; + *) + echo "start.sh: ${name} must be empty, a /path, or an http(s) URL; refusing to start" >&2 + return 1 ;; + esac + case "${value}" in + *[\"\'\\\!\|\ \ ]*) + echo "start.sh: ${name} contains a character no endpoint URL uses (quote, backslash, space, ! or |); refusing to start" >&2 + return 1 ;; + esac + hostport=${value#*://} + hostport=${hostport%%/*} + case "${hostport}" in + *@*) + echo "start.sh: ${name} must not carry credentials in the URL; refusing to start" >&2 + return 1 ;; + esac + host=$(printf '%s' "${hostport%%:*}" | tr 'A-Z' 'a-z' | sed 's/\.*$//') + case "${host}" in + ""|*[!a-z0-9.-]*) + echo "start.sh: ${name} has a malformed host; refusing to start" >&2 + return 1 ;; + localhost|127.*|bitcoinuniverse.io|*.bitcoinuniverse.io) return 0 ;; + mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion) ;; # origin-gate:deny + liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion) [ "${extra}" = "liquid" ] && return 0 ;; + *.onion) case "${extra}" in onion|liquid) return 0 ;; esac ;; + liquid.network|*.liquid.network) [ "${extra}" = "liquid" ] && return 0 ;; + *.*) ;; + *) return 0 ;; + esac + echo "start.sh: ${name} points outside Bitcoin Universe infrastructure (host: ${host}); refusing to start" >&2 + return 1 +} + +universe_endpoint_check EXTERNAL_DATA_SERVER_MEMPOOL_API "${__EXTERNAL_DATA_SERVER_MEMPOOL_API__}" || exit 78 +universe_endpoint_check EXTERNAL_DATA_SERVER_MEMPOOL_ONION "${__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__}" onion || exit 78 +universe_endpoint_check EXTERNAL_DATA_SERVER_LIQUID_API "${__EXTERNAL_DATA_SERVER_LIQUID_API__}" liquid || exit 78 +universe_endpoint_check EXTERNAL_DATA_SERVER_LIQUID_ONION "${__EXTERNAL_DATA_SERVER_LIQUID_ONION__}" liquid || exit 78 +universe_endpoint_check MEMPOOL_SERVICES_API "${__MEMPOOL_SERVICES_API__}" || exit 78 +universe_endpoint_check MEMPOOL_POOLS_JSON_URL "${__MEMPOOL_POOLS_JSON_URL__}" || exit 78 +universe_endpoint_check MEMPOOL_POOLS_JSON_TREE_URL "${__MEMPOOL_POOLS_JSON_TREE_URL__}" || exit 78 + mkdir -p "${__MEMPOOL_CACHE_DIR__}" sed -i "s!__MEMPOOL_NETWORK__!${__MEMPOOL_NETWORK__}!g" mempool-config.json diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 2ffa0be5bcd..619e0427373 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -20,7 +20,14 @@ services: start_period: 20s api: environment: - MEMPOOL_BACKEND: "none" + # The fork refuses BACKEND=none (address lookups would fail while the site + # still offers them). The address index is an owned Esplora reached over a + # loopback endpoint inside the container: bind-mount its Unix socket to + # /backend/esplora.sock and set ESPLORA_UNIX_SOCKET_PATH, or publish the + # index on the container loopback. A missing index is reported as + # unreachable, never substituted by a third party. + MEMPOOL_BACKEND: "esplora" + ESPLORA_REST_API_URL: "http://127.0.0.1:3000" CORE_RPC_HOST: "172.27.0.1" CORE_RPC_PORT: "8332" CORE_RPC_USERNAME: "mempool" diff --git a/docker/frontend/Dockerfile b/docker/frontend/Dockerfile index 6bbb2d5f326..3f937cb2ddb 100644 --- a/docker/frontend/Dockerfile +++ b/docker/frontend/Dockerfile @@ -1,4 +1,4 @@ -FROM node:24.13-bookworm-slim AS builder +FROM node:24.19.0-bookworm-slim AS builder ARG commitHash ENV DOCKER_COMMIT_HASH=${commitHash} @@ -8,10 +8,16 @@ WORKDIR /build COPY . . RUN apt-get update RUN apt-get install -y build-essential rsync +# The pinned runtime pair; a base image that drifts fails here, not later. +RUN node --version | grep -x v24.19.0 && npm --version | grep -x 11.17.0 RUN cp mempool-frontend-config.sample.json mempool-frontend-config.json -RUN npm install --omit=dev --omit=optional +# Locked install; the build tools are dev dependencies and nothing from this +# stage but dist/ reaches the nginx image. +RUN npm ci -RUN npm run build +# The fork's production build: English only, and no asset synchronization +# from third parties at build time. +RUN npm run build:universe FROM nginx:1.27.0-alpine diff --git a/docker/frontend/entrypoint.sh b/docker/frontend/entrypoint.sh index c1a99944f2a..c4fd3413061 100644 --- a/docker/frontend/entrypoint.sh +++ b/docker/frontend/entrypoint.sh @@ -3,19 +3,77 @@ __MEMPOOL_BACKEND_MAINNET_HTTP_HOST__=${BACKEND_MAINNET_HTTP_HOST:=127.0.0.1} __MEMPOOL_BACKEND_MAINNET_HTTP_PORT__=${BACKEND_MAINNET_HTTP_PORT:=8999} __MEMPOOL_FRONTEND_HTTP_PORT__=${FRONTEND_HTTP_PORT:=8080} +# File locations, overridable so the rendering can be exercised outside the image. +__NGINX_ETC__=${NGINX_ETC:=/etc/nginx} +__PATCH_DIR__=${PATCH_DIR:=/patch} +__MEMPOOL_WWW__=${MEMPOOL_WWW:=/var/www/mempool} + +# Runtime data comes from Bitcoin Universe infrastructure only. A services +# endpoint is empty, a same-origin /path, or an http(s) URL without +# credentials whose host is loopback, a single-label container name, or under +# bitcoinuniverse.io. Anything else stops the container before configuration +# is rendered; the value is not echoed, only the field name and, when it is not +# a credential problem, the host. +universe_endpoint_check() { + name="$1" + value="$2" + extra="$3" + [ -z "${value}" ] && return 0 + case "${value}" in + /*) return 0 ;; + http://*|https://*) ;; + *) + echo "entrypoint.sh: ${name} must be empty, a /path, or an http(s) URL; refusing to start" >&2 + return 1 ;; + esac + case "${value}" in + *[\"\'\\\!\|\ \ ]*) + echo "entrypoint.sh: ${name} contains a character no endpoint URL uses (quote, backslash, space, ! or |); refusing to start" >&2 + return 1 ;; + esac + hostport=${value#*://} + hostport=${hostport%%/*} + case "${hostport}" in + *@*) + echo "entrypoint.sh: ${name} must not carry credentials in the URL; refusing to start" >&2 + return 1 ;; + esac + host=$(printf '%s' "${hostport%%:*}" | tr 'A-Z' 'a-z' | sed 's/\.*$//') + case "${host}" in + ""|*[!a-z0-9.-]*) + echo "entrypoint.sh: ${name} has a malformed host; refusing to start" >&2 + return 1 ;; + localhost|127.*|bitcoinuniverse.io|*.bitcoinuniverse.io) return 0 ;; + mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion) ;; # origin-gate:deny + *.onion) [ "${extra}" = "onion" ] && return 0 ;; + *.*) ;; + *) return 0 ;; + esac + echo "entrypoint.sh: ${name} points outside Bitcoin Universe infrastructure (host: ${host}); refusing to start" >&2 + return 1 +} + +# Proxying the services API through nginx needs an explicit owned host; the +# nginx template carries a __PROXIED_SERVICES_HOST__ placeholder where it +# offers that route. __PROXIED_SERVICES__=${PROXIED_SERVICES:=false} -__PROXIED_SERVICES_HOST__=${PROXIED_SERVICES_HOST:=https://mempool.space} +__PROXIED_SERVICES_HOST__=${PROXIED_SERVICES_HOST:=""} if [ "${__PROXIED_SERVICES__}" = "true" ]; then - sed -i "s|proxy_pass https://mempool.space;|proxy_pass ${__PROXIED_SERVICES_HOST__};|g" /etc/nginx/conf.d/nginx-mempool.conf + if [ -z "${__PROXIED_SERVICES_HOST__}" ]; then + echo "entrypoint.sh: PROXIED_SERVICES=true needs PROXIED_SERVICES_HOST (a Bitcoin Universe services origin); refusing to start" >&2 + exit 78 + fi + universe_endpoint_check PROXIED_SERVICES_HOST "${__PROXIED_SERVICES_HOST__}" || exit 78 + sed -i "s|__PROXIED_SERVICES_HOST__|${__PROXIED_SERVICES_HOST__}|g" "${__NGINX_ETC__}/conf.d/nginx-mempool.conf" fi -sed -i "s/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__/${__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__}/g" /etc/nginx/conf.d/nginx-mempool.conf -sed -i "s/__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/${__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__}/g" /etc/nginx/conf.d/nginx-mempool.conf +sed -i "s/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__/${__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__}/g" "${__NGINX_ETC__}/conf.d/nginx-mempool.conf" +sed -i "s/__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/${__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__}/g" "${__NGINX_ETC__}/conf.d/nginx-mempool.conf" -cp /etc/nginx/nginx.conf /patch/nginx.conf -sed -i "s/__MEMPOOL_FRONTEND_HTTP_PORT__/${__MEMPOOL_FRONTEND_HTTP_PORT__}/g" /patch/nginx.conf -cat /patch/nginx.conf > /etc/nginx/nginx.conf +cp "${__NGINX_ETC__}/nginx.conf" "${__PATCH_DIR__}/nginx.conf" +sed -i "s/__MEMPOOL_FRONTEND_HTTP_PORT__/${__MEMPOOL_FRONTEND_HTTP_PORT__}/g" "${__PATCH_DIR__}/nginx.conf" +cat "${__PATCH_DIR__}/nginx.conf" > "${__NGINX_ETC__}/nginx.conf" if [ "${LIGHTNING_DETECTED_PORT}" != "" ];then export LIGHTNING=true @@ -39,7 +97,7 @@ __BLOCK_WEIGHT_UNITS__=${BLOCK_WEIGHT_UNITS:=4000000} __MEMPOOL_BLOCKS_AMOUNT__=${MEMPOOL_BLOCKS_AMOUNT:=8} __BASE_MODULE__=${BASE_MODULE:=mempool} __ROOT_NETWORK__=${ROOT_NETWORK:=} -__MEMPOOL_WEBSITE_URL__=${MEMPOOL_WEBSITE_URL:=https://mempool.space} +__MEMPOOL_WEBSITE_URL__=${MEMPOOL_WEBSITE_URL:=https://explorer.bitcoinuniverse.io} __LIQUID_WEBSITE_URL__=${LIQUID_WEBSITE_URL:=https://liquid.network} __MINING_DASHBOARD__=${MINING_DASHBOARD:=true} __LIGHTNING__=${LIGHTNING:=false} @@ -56,11 +114,24 @@ __SIGNET_TX_FIRST_SEEN_START_HEIGHT__=${SIGNET_TX_FIRST_SEEN_START_HEIGHT:=0} __REGTEST_TX_FIRST_SEEN_START_HEIGHT__=${REGTEST_TX_FIRST_SEEN_START_HEIGHT:=0} __ACCELERATOR__=${ACCELERATOR:=false} __ACCELERATOR_BUTTON__=${ACCELERATOR_BUTTON:=true} -__SERVICES_API__=${SERVICES_API:=https://mempool.space/api/v1/services} +# Unset by default, matching the frontend: account and acceleration calls +# then never leave this origin. Name a Bitcoin Universe services endpoint to +# turn them on. +__SERVICES_API__=${SERVICES_API:=""} +universe_endpoint_check SERVICES_API "${__SERVICES_API__}" || exit 78 +# The services endpoint used when this deployment is reached over Tor. Only +# an explicitly configured .onion host is used; the page hostname alone never +# switches providers. +__ONION_SERVICES_API__=${ONION_SERVICES_API:=""} +universe_endpoint_check ONION_SERVICES_API "${__ONION_SERVICES_API__}" onion || exit 78 __PUBLIC_ACCELERATIONS__=${PUBLIC_ACCELERATIONS:=false} __HISTORICAL_PRICE__=${HISTORICAL_PRICE:=true} __ADDITIONAL_CURRENCIES__=${ADDITIONAL_CURRENCIES:=false} __STRATUM_ENABLED__=${STRATUM_ENABLED:=false} +# Which network each non-Bitcoin chain is read from, as a JSON object such as +# {"dogecoin":"testnet"}. Unlisted chains read mainnet; Bitcoin follows the +# network selector. The frontend parses this string. +__UNIVERSE_CHAIN_NETWORKS__=${UNIVERSE_CHAIN_NETWORKS:={}} # Export as environment variables to be used by envsubst export __MAINNET_ENABLED__ @@ -97,12 +168,14 @@ export __REGTEST_TX_FIRST_SEEN_START_HEIGHT__ export __ACCELERATOR__ export __ACCELERATOR_BUTTON__ export __SERVICES_API__ +export __ONION_SERVICES_API__ export __PUBLIC_ACCELERATIONS__ export __HISTORICAL_PRICE__ export __ADDITIONAL_CURRENCIES__ export __STRATUM_ENABLED__ +export __UNIVERSE_CHAIN_NETWORKS__ -folder=$(find /var/www/mempool -name "config.js" | xargs dirname) +folder=$(find "${__MEMPOOL_WWW__}" -name "config.js" | xargs dirname) echo ${folder} envsubst < ${folder}/config.template.js > ${folder}/config.js diff --git a/docs/operations/CONFIGURATION.md b/docs/operations/CONFIGURATION.md index f3e83fc3301..f91b934b8b9 100644 --- a/docs/operations/CONFIGURATION.md +++ b/docs/operations/CONFIGURATION.md @@ -135,7 +135,7 @@ purpose: the `all` range is the whole series. ### `FIAT_PRICE` -`ENABLED` (default `true`), `PAID`, `API_KEY`. +`ENABLED` (backend default `true`, Docker default `false`), `PAID`, `API_KEY`. Worth understanding before you turn it on: the price task calls public exchange APIs directly from the backend. Those are market data sources, not blockchain @@ -163,12 +163,38 @@ credentials. Routes the backend's outbound requests through a SOCKS5 proxy. | Section | State | Why | | --- | --- | --- | | `LIGHTNING`, `LND`, `CLIGHTNING` | off, and off by default | The inherited Lightning product is not part of the Universe explorer. `/api/v1/backend-info` reports `lightning: false` on the public deployment | -| `MEMPOOL_SERVICES` | `API` empty, `ACCELERATIONS` false | Upstream's hosted services. Leave both unset | -| `EXTERNAL_DATA_SERVER` | `MEMPOOL_API` empty | **Universe change.** Upstream ships a hosted endpoint here. The remaining onion and Liquid entries are upstream defaults this fork has not cleared; they are not read by a mainnet Bitcoin deployment, and nothing in the Universe deployment path calls them. Do not enable them | +| `MEMPOOL_SERVICES` | `API` empty, `ACCELERATIONS` false | Accounts, accelerations, invoices and the other services routes report `unconfigured` until `API` names a Bitcoin Universe services endpoint. The Docker start script refuses any other host | +| `EXTERNAL_DATA_SERVER` | `MEMPOOL_API` and `MEMPOOL_ONION` empty | **Universe change.** Upstream ships hosted endpoints here; the about-page routes stay unmounted until `MEMPOOL_API` names a Bitcoin Universe endpoint, and `MEMPOOL_ONION` is the onion host of that same deployment, never a provider switch. `LIQUID_API` / `LIQUID_ONION` keep the Liquid asset catalogue (see the metadata exception below) | | `REPLICATION` | off | Replicates audit and statistics data from other mempool servers | | `MAXMIND` | off | GeoIP databases, used only by the Lightning product | | `WALLETS`, `STRATUM` | off | Upstream features this fork does not use | +### Owned endpoint prerequisites and the metadata exception + +Every runtime endpoint the backend or the frontend calls is either a relative +path on its own origin, loopback, a container name, or a host under +`bitcoinuniverse.io`. `scripts/universe/check-origins.mjs` enforces that on the +source tree, on the built bundle, and (through +`scripts/universe/docker-runtime-defaults.test.mjs`) on the configuration the +container start scripts render. The scripts themselves refuse to start on any +other host, on a URL carrying credentials, or on a malformed value, and they +never echo the value they refused, only the field name and the host. + +| Feature | Prerequisite | Unconfigured behaviour | +| --- | --- | --- | +| About-page contributors, donations, translators | `EXTERNAL_DATA_SERVER.MEMPOOL_API` (Docker: `EXTERNAL_DATA_SERVER_MEMPOOL_API`) | routes not mounted, `404` | +| Accounts, accelerations, invoices, faucet, Lightning metadata, proofs | `MEMPOOL_SERVICES.API` (Docker: `MEMPOOL_SERVICES_API`) and the frontend `SERVICES_API` (Docker frontend: `SERVICES_API`) | capabilities report the dependency as not configured; the frontend stays on its own origin | +| The same over Tor | `EXTERNAL_DATA_SERVER.MEMPOOL_ONION` and the frontend `ONION_SERVICES_API`, both naming this deployment's own `.onion` host | clearnet endpoints are used; the page hostname alone never switches providers | +| Fiat prices | `FIAT_PRICE.ENABLED` (Docker: `FIAT_PRICE_ENABLED`, default `false`) | `/api/v1/prices` answers `-1` for every currency | +| Mining pool table refresh | `MEMPOOL.AUTOMATIC_POOLS_UPDATE` with an explicit `POOLS_JSON_URL` and `POOLS_JSON_TREE_URL` (Docker defaults are empty) | the bundled `pools-v2.json` is used | + +The one permitted external runtime category is **Liquid asset catalogue +metadata**: asset names, tickers, precision, icons and issuer details read from +`EXTERNAL_DATA_SERVER.LIQUID_API` / `LIQUID_ONION` (`liquid.network` and its +onion). It covers collection metadata only. Balances, ownership, transfers, +history and every other chain fact come from the owned Elements node and +indexer, and the origin gate rejects any attempt to widen the exception. + ## Gateway `scripts/universe/gateway.mjs`. Every variable is optional and every default is @@ -226,6 +252,17 @@ publish an authority's lag without publishing where it lives. time and by the dev proxy. It is optional; without it the build uses its defaults. +`UNIVERSE_CHAIN_NETWORKS` (Docker frontend: `UNIVERSE_CHAIN_NETWORKS`, a JSON string) +names which network each non-Bitcoin chain is read from, for example +`{"dogecoin":"testnet"}`. Values are `mainnet`, `testnet` or `regtest` per +chain; the default `{}` reads every chain from mainnet. Bitcoin is never listed: +it follows the network selector, and the selector never implies a Dogecoin or +Zcash network. An entry the frontend cannot use is ignored with a console +warning and that chain reads mainnet. The overlay must serve the named scope +(`UNIVERSE_DOGECOIN_NETWORKS` for Dogecoin, the indexer's declared network for +Zcash); a scope it does not serve is shown as unavailable under that network, +never as mainnet data. + The dev server proxy table is `frontend/proxy.conf.local.js`, and it targets `http://localhost:8999`. To point the dev server at a different backend, edit the targets there. There is no environment variable for it. diff --git a/docs/operations/INSTALL.md b/docs/operations/INSTALL.md index 07ac9ba9eb9..b09f84e1576 100644 --- a/docs/operations/INSTALL.md +++ b/docs/operations/INSTALL.md @@ -19,7 +19,7 @@ them. This page covers what is different, in order. | --- | --- | --- | | Node.js | 24.19.0 | pinned in `.nvmrc`; the toolchain the CI fleet and the release artifact use | | npm | 11.17.0 | pinned alongside Node | -| Rust and `cargo` | 1.84 | `rust/gbt/rust-toolchain`. The backend's `preinstall` script builds `rust/gbt` before anything else, so `npm ci` in `backend/` fails without `cargo` on the path | +| Rust and `cargo` | 1.89.0 | `rust/gbt/rust-toolchain`. The locked engine dependencies need edition 2024, which 1.84 lacks. The backend's `preinstall` script builds `rust/gbt` before anything else, so `npm ci` in `backend/` fails without `cargo` on the path | | Bitcoin Core | a `txindex=1` node with RPC enabled | the only source of Bitcoin chain data | | MariaDB or MySQL | MariaDB 10.5 or later; the Universe fleet standard is MySQL 8.4 LTS | blocks, mining, statistics, and price history | | An address index | optional | without one, address and unspent-output lookups report that they cannot be served | diff --git a/docs/protocols/PROTOCOL-COVERAGE.json b/docs/protocols/PROTOCOL-COVERAGE.json index 666281716b6..2ea64c5d5c5 100644 --- a/docs/protocols/PROTOCOL-COVERAGE.json +++ b/docs/protocols/PROTOCOL-COVERAGE.json @@ -2,8 +2,8 @@ "schemaVersion": "universe-explorer-protocol-manifest-v1", "registryVersion": "1.1.0", "sourceRepository": "bitcoinuniverseio/backend-apis", - "sourceSha": "04ff6efd3ea2dd841742ecf48932f0b10a8b5dd0", - "generatedAt": "2026-09-05T22:45:43.323Z", + "sourceSha": "7ec4602e7a5dcd6495268ae64698280657cc9c73", + "generatedAt": "2026-09-17T05:45:55.810Z", "primaryStrip": [ "ORDINALS", "RUNES", @@ -1839,13 +1839,14 @@ "route": "/api/v1/dogecoin/protocols/doginals", "authorityPath": "/api/v1/inscriptions", "evidence": "source-contract", - "acceptance": "NOT TESTED" + "acceptance": "NOT TESTED", + "pagination": "cursor" }, { "id": "chain-detail", "method": "GET", - "route": "/api/v1/dogecoin/protocols/doginals/:reference", - "authorityPath": "/inscription/:reference", + "route": "/api/v1/dogecoin/protocols/doginals/:id", + "authorityPath": "/inscription/:id", "evidence": "source-contract", "acceptance": "NOT TESTED" } @@ -1895,23 +1896,25 @@ "route": "/api/v1/dogecoin/protocols/drc20", "authorityPath": "/api/v1/drc20/tokens", "evidence": "source-contract", - "acceptance": "NOT TESTED" + "acceptance": "NOT TESTED", + "pagination": "cursor" }, { "id": "chain-detail", "method": "GET", - "route": "/api/v1/dogecoin/protocols/drc20/:reference", - "authorityPath": "/api/v1/drc20/tokens/:reference", + "route": "/api/v1/dogecoin/protocols/drc20/:ticker", + "authorityPath": "/api/v1/drc20/tokens/:ticker", "evidence": "source-contract", "acceptance": "NOT TESTED" }, { "id": "holders", "method": "GET", - "route": "/api/v1/dogecoin/protocols/drc20/:reference/holders", - "authorityPath": "/api/v1/drc20/tokens/:reference/holders", + "route": "/api/v1/dogecoin/protocols/drc20/:ticker/holders", + "authorityPath": "/api/v1/drc20/tokens/:ticker/holders", "evidence": "source-contract", - "acceptance": "NOT TESTED" + "acceptance": "NOT TESTED", + "pagination": "cursor" } ], "authorizedReadOperations": [ @@ -1943,7 +1946,11 @@ "visualToken": "protocol-tap-doge", "implementedReadOperations": [ "registry", - "activity" + "activity", + "chain-list", + "chain-detail", + "holders", + "events" ], "readOperationDescriptors": [ { @@ -1961,11 +1968,50 @@ "authorityPath": "/token-explorer/doge-tap", "evidence": "source-contract", "acceptance": "NOT TESTED" + }, + { + "id": "chain-list", + "method": "GET", + "route": "/api/v1/dogecoin/protocols/doge-tap", + "authorityPath": "/token-explorer/doge-tap/assets", + "evidence": "source-contract", + "acceptance": "NOT TESTED", + "pagination": "offset" + }, + { + "id": "chain-detail", + "method": "GET", + "route": "/api/v1/dogecoin/protocols/doge-tap/:ref", + "authorityPath": "/token-explorer/doge-tap/assets/:ref", + "evidence": "source-contract", + "acceptance": "NOT TESTED" + }, + { + "id": "holders", + "method": "GET", + "route": "/api/v1/dogecoin/protocols/doge-tap/:ref/holders", + "authorityPath": "/token-explorer/doge-tap/assets/:ref/holders", + "evidence": "source-contract", + "acceptance": "NOT TESTED", + "pagination": "offset" + }, + { + "id": "events", + "method": "GET", + "route": "/api/v1/dogecoin/protocols/doge-tap/:ref/events", + "authorityPath": "/token-explorer/doge-tap/assets/:ref/events", + "evidence": "source-contract", + "acceptance": "NOT TESTED", + "pagination": "offset" } ], "authorizedReadOperations": [ "registry", - "activity" + "activity", + "chain-list", + "chain-detail", + "holders", + "events" ], "releaseStatus": "BLOCKED", "indexerAuthority": "index-doge-tap", @@ -2004,13 +2050,14 @@ "route": "/api/v1/dogecoin/protocols/dunes", "authorityPath": "/api/v1/dunes/tokens", "evidence": "source-contract", - "acceptance": "NOT TESTED" + "acceptance": "NOT TESTED", + "pagination": "cursor" }, { "id": "chain-detail", "method": "GET", - "route": "/api/v1/dogecoin/protocols/dunes/:reference", - "authorityPath": "/api/v1/dunes/tokens/:reference", + "route": "/api/v1/dogecoin/protocols/dunes/:dune", + "authorityPath": "/api/v1/dunes/tokens/:dune", "evidence": "source-contract", "acceptance": "NOT TESTED" } @@ -2064,8 +2111,8 @@ { "id": "chain-detail", "method": "GET", - "route": "/api/v1/zcash/protocols/zerdinals/:reference", - "authorityPath": "/zcash-metaprotocols/inscriptions/:reference", + "route": "/api/v1/zcash/protocols/zerdinals/:id", + "authorityPath": "/zcash-metaprotocols/inscriptions/:id", "evidence": "source-contract", "acceptance": "NOT TESTED" } @@ -2117,8 +2164,8 @@ { "id": "chain-detail", "method": "GET", - "route": "/api/v1/zcash/protocols/zrunes/:reference", - "authorityPath": "/zcash-metaprotocols/zrunes/:reference", + "route": "/api/v1/zcash/protocols/zrunes/:id", + "authorityPath": "/zcash-metaprotocols/zrunes/:id", "evidence": "source-contract", "acceptance": "NOT TESTED" } @@ -2170,8 +2217,8 @@ { "id": "chain-detail", "method": "GET", - "route": "/api/v1/zcash/protocols/zrc20/:reference", - "authorityPath": "/zcash-metaprotocols/tokens/:reference", + "route": "/api/v1/zcash/protocols/zrc20/:tick", + "authorityPath": "/zcash-metaprotocols/tokens/:tick", "evidence": "source-contract", "acceptance": "NOT TESTED" } diff --git a/docs/protocols/PROTOCOL-COVERAGE.md b/docs/protocols/PROTOCOL-COVERAGE.md index 58ee92751f7..72dd18b78c8 100644 --- a/docs/protocols/PROTOCOL-COVERAGE.md +++ b/docs/protocols/PROTOCOL-COVERAGE.md @@ -20,9 +20,9 @@ A protocol never silently disappears from this table: `PROTOCOL-ROSTER.lock` records every id that has been published, and the gate fails when one of them stops appearing. -Pinned from bitcoinuniverseio/backend-apis at commit 04ff6efd3ea2dd841742ecf48932f0b10a8b5dd0, +Pinned from bitcoinuniverseio/backend-apis at commit 7ec4602e7a5dcd6495268ae64698280657cc9c73, manifest schema universe-explorer-protocol-manifest-v1, registry version 1.1.0, -recorded 2026-09-05T22:45:43.323Z. +recorded 2026-09-17T05:45:55.810Z. 39 protocol identities are retained. 7 carry historical readable declarations; these are not current runtime or E2E passes. Operation descriptors identify implemented public reads and their owned authority routes; configuration and acceptance are separate. The registry is not the complete application operation inventory. @@ -62,7 +62,7 @@ recorded 2026-09-05T22:45:43.323Z. | anima | OTHER | bitcoin | index-anima | BLOCKED | unknown | registry, status, transitions, transition, items, item, item-history | | doginals | OTHER | dogecoin | ord-dogecoin | BLOCKED | unknown | registry, chain-list, chain-detail | | drc20 | OTHER | dogecoin | ord-dogecoin | BLOCKED | unknown | registry, chain-list, chain-detail, holders | -| tap_doge | OTHER | dogecoin | index-doge-tap | BLOCKED | unknown | registry, activity | +| tap_doge | OTHER | dogecoin | index-doge-tap | BLOCKED | unknown | registry, activity, chain-list, chain-detail, holders, events | | dunes | OTHER | dogecoin | ord-dogecoin | BLOCKED | unknown | registry, chain-list, chain-detail | | zerdinals | OTHER | zcash | index-zcash-metaprotocols | VERIFIED READ ONLY | complete | registry, chain-list, chain-detail | | zrunes | OTHER | zcash | index-zcash-metaprotocols | VERIFIED READ ONLY | complete | registry, chain-list, chain-detail | diff --git a/frontend/generate-config.js b/frontend/generate-config.js index 4485c2a953e..d8f5d834234 100644 --- a/frontend/generate-config.js +++ b/frontend/generate-config.js @@ -98,14 +98,21 @@ if (process.env.DOCKER_COMMIT_HASH) { const newConfig = `(function (window) { window.__env = window.__env || {};${settings.reduce((str, obj) => `${str} - window.__env.${obj.key} = ${typeof obj.value === 'string' ? `'${obj.value}'` : obj.value};`, '')} + window.__env.${obj.key} = ${typeof obj.value === 'string' ? `'${obj.value}'` : isJsonObject(obj.value) ? JSON.stringify(obj.value) : obj.value};`, '')} window.__env.GIT_COMMIT_HASH = '${gitCommitHash}'; window.__env.PACKAGE_JSON_VERSION = '${packetJsonVersion}'; }((typeof global !== 'undefined') ? global : this));`; +// An object setting (UNIVERSE_CHAIN_NETWORKS) is substituted as a quoted JSON +// string in the Docker template, so an unset variable renders '' rather than +// a syntax error; the frontend parses the string. +function isJsonObject(value) { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + const newConfigTemplate = `(function (window) { window.__env = window.__env || {};${settings.reduce((str, obj) => `${str} - window.__env.${obj.key} = ${typeof obj.value === 'string' ? `'\${__${obj.key}__}'` : `\${__${obj.key}__}`};`, '')} + window.__env.${obj.key} = ${typeof obj.value === 'string' || isJsonObject(obj.value) ? `'\${__${obj.key}__}'` : `\${__${obj.key}__}`};`, '')} window.__env.GIT_COMMIT_HASH = '${gitCommitHash}'; window.__env.PACKAGE_JSON_VERSION = '${packetJsonVersion}'; }(this));`; diff --git a/frontend/mempool-frontend-config.sample.json b/frontend/mempool-frontend-config.sample.json index 069d0ae64f6..b50c84f6a3a 100644 --- a/frontend/mempool-frontend-config.sample.json +++ b/frontend/mempool-frontend-config.sample.json @@ -36,5 +36,7 @@ "ACCELERATOR_BUTTON": true, "PUBLIC_ACCELERATIONS": false, "STRATUM_ENABLED": false, - "SERVICES_API": "/services" + "SERVICES_API": "/services", + "ONION_SERVICES_API": "", + "UNIVERSE_CHAIN_NETWORKS": {} } diff --git a/frontend/src/app/components/master-page/master-page.component.ts b/frontend/src/app/components/master-page/master-page.component.ts index cb46c560889..9b72995f6f2 100644 --- a/frontend/src/app/components/master-page/master-page.component.ts +++ b/frontend/src/app/components/master-page/master-page.component.ts @@ -7,11 +7,12 @@ import { EnterpriseService } from '@app/services/enterprise.service'; import { NavigationService } from '@app/services/navigation.service'; import { StorageService } from '@app/services/storage.service'; import { ChainHealthService, ChainHealthState } from '@app/universe/chain-health.service'; +import { chainNetwork } from '@app/universe/chain-network'; import { healthServiceSummary, nodeHealthLabel, readHealth } from '@app/universe/multichain-explorer/chain-health'; import { UniverseLocalService } from '@app/universe/universe-local.service'; import { mainReady } from '@app/universe/main-ready'; import { UniverseViewportService } from '@app/universe/universe-viewport.service'; -import { ChainCapabilityEnvelope, ExplorerChain } from '@app/universe/universe.types'; +import { ChainCapabilityEnvelope, ExplorerChain, ExplorerNetwork } from '@app/universe/universe.types'; import { formatExactInteger, } from '@app/universe/multichain-explorer/multichain-view'; @@ -243,7 +244,7 @@ export class MasterPageComponent implements OnInit, AfterViewInit, OnDestroy { } chainCapability(capabilities: ChainCapabilityEnvelope[], chain: ExplorerChain): ChainCapabilityEnvelope | undefined { - return capabilities.find((capability) => capability.chain === chain && capability.network === (chain === 'bitcoin' ? (this.stateService.network || 'mainnet') : 'mainnet')); + return capabilities.find((capability) => capability.chain === chain && capability.network === this.resolvedNetwork(chain)); } chainState(capability: ChainCapabilityEnvelope | undefined): string { @@ -251,7 +252,12 @@ export class MasterPageComponent implements OnInit, AfterViewInit, OnDestroy { } chainNetwork(chain: ExplorerChain): string { - return this.networkLabel(chain === 'bitcoin' ? (this.stateService.network || 'mainnet') : 'mainnet'); + return this.networkLabel(this.resolvedNetwork(chain)); + } + + /** Bitcoin follows the selector; every other chain reads its configured network. */ + private resolvedNetwork(chain: ExplorerChain): string { + return chainNetwork(chain, (this.stateService.network || 'mainnet') as ExplorerNetwork, this.stateService.env); } chainDetail(capability: ChainCapabilityEnvelope | undefined): string { diff --git a/frontend/src/app/docs/api-docs/api-docs-data.ts b/frontend/src/app/docs/api-docs/api-docs-data.ts index 5e930187a46..050fd6130ac 100644 --- a/frontend/src/app/docs/api-docs/api-docs-data.ts +++ b/frontend/src/app/docs/api-docs/api-docs-data.ts @@ -1296,7 +1296,7 @@ export const wsApiDocsData = [ fragment: 'track-txs', title: 'Track Transactions', description: { - default: 'Subscribe to multiple transactions to receive live updates on their status and position in the mempool. Limits on the maximum number of tracked addresses apply.' + default: 'Subscribe to multiple transactions to receive live updates on their status and position in the mempool. Limits on the maximum number of tracked transactions apply. The initial tracked-txs response carries a status per transaction: mempool (with its projected position), replaced (with replacedBy), confirmed (only together with blockHeight and blockHash), unknown (the node does not know the transaction) or unavailable (the status lookup failed). confirmed is true only for the confirmed status; a missing or false flag is never a confirmation. Later delta frames carry no status.' }, payload: `{ "track-txs": [ @@ -1316,6 +1316,8 @@ export const wsApiDocsData = [ response: `{ "tracked-txs": { "8a4666c6d22ce74fa47e1c4fdb09af556a234cc6a606539a75caf66ba44a2d07": { + "confirmed": false, + "status": "mempool", "position": { "block": 0, "vsize": 434494 @@ -1342,10 +1344,22 @@ export const wsApiDocsData = [ } }, "941df06064c290b4627e92bdbf3bff7c0e97aab33e273c2a20404f9cfd21b607": { + "confirmed": false, + "status": "mempool", "position": { "block": 2, "vsize": 932479.5 } + }, + "e6a1b3d6c0e4b6b72f7a8b0b3c2a5d7e0f1c3b5a7d9e1f3a5c7e9b1d3f5a7c9e": { + "confirmed": true, + "status": "confirmed", + "blockHeight": 840000, + "blockHash": "0000000000000000000320283a032748cef8227873ff4872689bf23f1cda83a5" + }, + "b7c9d1e3f5a7c9e1b3d5f7a9c1e3b5d7f9a1c3e5b7d9f1a3c5e7b9d1f3a5c7e9": { + "confirmed": false, + "status": "unknown" } } }` @@ -1357,6 +1371,8 @@ export const wsApiDocsData = [ response: `{ "tracked-txs": { "8a4666c6d22ce74fa47e1c4fdb09af556a234cc6a606539a75caf66ba44a2d07": { + "confirmed": false, + "status": "mempool", "position": { "block": 0, "vsize": 434494 @@ -1383,10 +1399,22 @@ export const wsApiDocsData = [ } }, "941df06064c290b4627e92bdbf3bff7c0e97aab33e273c2a20404f9cfd21b607": { + "confirmed": false, + "status": "mempool", "position": { "block": 2, "vsize": 932479.5 } + }, + "e6a1b3d6c0e4b6b72f7a8b0b3c2a5d7e0f1c3b5a7d9e1f3a5c7e9b1d3f5a7c9e": { + "confirmed": true, + "status": "confirmed", + "blockHeight": 840000, + "blockHash": "0000000000000000000320283a032748cef8227873ff4872689bf23f1cda83a5" + }, + "b7c9d1e3f5a7c9e1b3d5f7a9c1e3b5d7f9a1c3e5b7d9f1a3c5e7b9d1f3a5c7e9": { + "confirmed": false, + "status": "unknown" } } }` @@ -1398,6 +1426,8 @@ export const wsApiDocsData = [ response: `{ "tracked-txs": { "8a4666c6d22ce74fa47e1c4fdb09af556a234cc6a606539a75caf66ba44a2d07": { + "confirmed": false, + "status": "mempool", "position": { "block": 0, "vsize": 434494 @@ -1424,10 +1454,22 @@ export const wsApiDocsData = [ } }, "941df06064c290b4627e92bdbf3bff7c0e97aab33e273c2a20404f9cfd21b607": { + "confirmed": false, + "status": "mempool", "position": { "block": 2, "vsize": 932479.5 } + }, + "e6a1b3d6c0e4b6b72f7a8b0b3c2a5d7e0f1c3b5a7d9e1f3a5c7e9b1d3f5a7c9e": { + "confirmed": true, + "status": "confirmed", + "blockHeight": 840000, + "blockHash": "0000000000000000000320283a032748cef8227873ff4872689bf23f1cda83a5" + }, + "b7c9d1e3f5a7c9e1b3d5f7a9c1e3b5d7f9a1c3e5b7d9f1a3c5e7b9d1f3a5c7e9": { + "confirmed": false, + "status": "unknown" } } }` @@ -1439,6 +1481,8 @@ export const wsApiDocsData = [ response: `{ "tracked-txs": { "8a4666c6d22ce74fa47e1c4fdb09af556a234cc6a606539a75caf66ba44a2d07": { + "confirmed": false, + "status": "mempool", "position": { "block": 0, "vsize": 434494 @@ -1465,10 +1509,22 @@ export const wsApiDocsData = [ } }, "941df06064c290b4627e92bdbf3bff7c0e97aab33e273c2a20404f9cfd21b607": { + "confirmed": false, + "status": "mempool", "position": { "block": 2, "vsize": 932479.5 } + }, + "e6a1b3d6c0e4b6b72f7a8b0b3c2a5d7e0f1c3b5a7d9e1f3a5c7e9b1d3f5a7c9e": { + "confirmed": true, + "status": "confirmed", + "blockHeight": 840000, + "blockHash": "0000000000000000000320283a032748cef8227873ff4872689bf23f1cda83a5" + }, + "b7c9d1e3f5a7c9e1b3d5f7a9c1e3b5d7f9a1c3e5b7d9f1a3c5e7b9d1f3a5c7e9": { + "confirmed": false, + "status": "unknown" } } }` diff --git a/frontend/src/app/services/state.service.ts b/frontend/src/app/services/state.service.ts index eaf4ce54fd7..7ee89a9fa43 100644 --- a/frontend/src/app/services/state.service.ts +++ b/frontend/src/app/services/state.service.ts @@ -100,6 +100,13 @@ export interface Env { PACKAGE_JSON_VERSION_MEMPOOL_SPACE?: string; STRATUM_ENABLED: boolean; SERVICES_API?: string; + ONION_SERVICES_API?: string; + /** + * Which network each non-Bitcoin chain is read from, as a JSON object such + * as {"dogecoin":"testnet"} (or that object as a JSON string). Unlisted + * chains read mainnet; Bitcoin always follows the network selector. + */ + UNIVERSE_CHAIN_NETWORKS?: Record | string; customize?: Customization; PROD_DOMAINS: string[]; } @@ -151,6 +158,9 @@ const defaultEnv: Env = { // The hosted services API is an upstream product. It is unset here, so the // account and acceleration calls that use it never leave this origin. 'SERVICES_API': '', + // The onion services endpoint a Tor-served deployment names for itself. + 'ONION_SERVICES_API': '', + 'UNIVERSE_CHAIN_NETWORKS': {}, 'PROD_DOMAINS': [], }; @@ -273,8 +283,11 @@ export class StateService { this.env.MINING_DASHBOARD = false; } - if (document.location.hostname.endsWith('.onion')) { - this.env.SERVICES_API = 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1/services'; + // Onion transport is explicit configuration: a deployment served over Tor + // names its own onion services endpoint. The page hostname alone never + // switches providers. + if (document.location.hostname.endsWith('.onion') && this.env.ONION_SERVICES_API) { + this.env.SERVICES_API = this.env.ONION_SERVICES_API; } if (this.isBrowser) { diff --git a/frontend/src/app/universe/bootstrap/bootstrap-boundaries.spec.ts b/frontend/src/app/universe/bootstrap/bootstrap-boundaries.spec.ts index 7c9fe529ceb..ad10ef3679b 100644 --- a/frontend/src/app/universe/bootstrap/bootstrap-boundaries.spec.ts +++ b/frontend/src/app/universe/bootstrap/bootstrap-boundaries.spec.ts @@ -9,13 +9,14 @@ describe('bootstrap source and request boundaries',()=>{ const networks=new BehaviorSubject('signet');const pending=new Subject(); const api={network:'signet',networkChanged$:networks,verifySnapshotChecksum$:()=>pending,generateBootstrapPlan$:()=>pending}; const c:any=kind==='verify'?new BootstrapVerifyComponent(api as any,{markForCheck:vi.fn()} as any):new BootstrapPlannerComponent(api as any,{markForCheck:vi.fn()} as any); + if(kind==='verify')c.snapshotHeight=10; c.ngOnInit();const run=()=>kind==='verify'?c.verifyChecksum():c.calculatePlan();run();c.clear();expect(pending.observed).toBe(false); run();networks.next('regtest');expect(pending.observed).toBe(false);run();c.ngOnDestroy();expect(pending.observed).toBe(false); }); it('does not promote truthy validity to safe-to-load or convert503 into mismatch',()=>{ - const pending=new Subject();const c=new BootstrapVerifyComponent({verifySnapshotChecksum$:()=>pending} as any,{markForCheck:vi.fn()} as any); - c.verifyChecksum();pending.next({valid:'true',status:'pinned_core_verified'});expect(c.report).toBeNull();expect(c.failure).toContain('Supplied hashes'); - pending.error({status:503,error:{error:'Trusted snapshot source unavailable'}});expect(c.report).toBeNull();expect(c.failure).toContain('unavailable'); + const pending=new Subject();const c=new BootstrapVerifyComponent({network:'signet',verifySnapshotChecksum$:()=>pending} as any,{markForCheck:vi.fn()} as any); + c.snapshotHeight=10;c.verifyChecksum();pending.next({valid:'true',status:'pinned_core_verified'});expect(c.report).toBeNull();expect(c.failure?.message).toContain('Supplied hashes'); + pending.error({status:503,error:{error:'Trusted snapshot source unavailable'}});expect(c.report).toBeNull();expect(c.failure?.message).toContain('unavailable');expect(c.failure?.kind).toBe('unavailable'); }); it('clears detail on route/network and rejects wrong reference or absent network',()=>{ const networks=new BehaviorSubject('signet');const params=new BehaviorSubject(convertToParamMap({heightOrHash:'10'}));const route={paramMap:params,snapshot:{paramMap:params.value}}; diff --git a/frontend/src/app/universe/bootstrap/bootstrap-observation.spec.ts b/frontend/src/app/universe/bootstrap/bootstrap-observation.spec.ts index 4591569c5d9..a5c4034c6c8 100644 --- a/frontend/src/app/universe/bootstrap/bootstrap-observation.spec.ts +++ b/frontend/src/app/universe/bootstrap/bootstrap-observation.spec.ts @@ -5,6 +5,24 @@ import { BootstrapChainstatesComponent } from './bootstrap-chainstates.component import { BootstrapOverviewComponent } from './bootstrap-overview.component'; afterEach(() => vi.useRealTimers()); +describe('Bootstrap overview subfeature statuses', () => { + it('renders each subfeature status with the reason the backend gave', () => { + const component = new BootstrapOverviewComponent({} as any, { markForCheck: vi.fn() } as any); + const rows = component.subfeatures({ + snapshot_catalogue_status: 'unavailable', snapshot_catalogue_reason: 'No catalogue file is configured.', + verification_store_status: 'unavailable', verification_store_reason: 'Verification runs need the durable MySQL store (DATABASE.ENABLED).', + planning_status: 'unavailable', operator_status: 'available', operator_reason: null, + } as any); + expect(rows.map(r => [r.label, r.status])).toEqual([ + ['Snapshot catalogue', 'unavailable'], ['Verification store', 'unavailable'], ['Planning', 'unavailable'], ['Operator jobs', 'available'], + ]); + expect(rows[0].reason).toContain('catalogue'); + expect(rows[1].reason).toContain('MySQL'); + expect(rows[2].reason).toContain('verification store'); + expect(rows[3].reason).toBeNull(); + expect(component.subfeatures({} as any).every(r => r.status === 'unavailable')).toBe(true); + }); +}); describe('Bootstrap owned observation lifecycle', () => { it('uses selected network for every API operation', () => { const http = { get: vi.fn(() => of([])), post: vi.fn(() => of({})) }; diff --git a/frontend/src/app/universe/bootstrap/bootstrap-overview.component.ts b/frontend/src/app/universe/bootstrap/bootstrap-overview.component.ts index ff4f2d320ac..d213d555902 100644 --- a/frontend/src/app/universe/bootstrap/bootstrap-overview.component.ts +++ b/frontend/src/app/universe/bootstrap/bootstrap-overview.component.ts @@ -77,11 +77,17 @@ import { RelativeUrlPipe } from '@app/shared/pipes/relative-url/relative-url.pip {{ error }} -
- {{ overview?.snapshot_catalogue_reason }} +
+
+
+
+ {{ f.label }} + {{ f.status }} +
+
{{ f.reason }}
+
No reason was reported.
+
+
@@ -94,7 +100,7 @@ import { RelativeUrlPipe } from '@app/shared/pipes/relative-url/relative-url.pip : overview.total_snapshots }}
-
Pinned in Bitcoin Core
+
In the trusted catalogue
@@ -128,7 +134,7 @@ import { RelativeUrlPipe } from '@app/shared/pipes/relative-url/relative-url.pip
-

Pinned AssumeUTXO Snapshots

+

Snapshots verified over their bytes

- {{ s.coins_count | number }} + {{ s.coins_count === null ? 'not pinned' : (s.coins_count | number) }} {{ (s.file_size_bytes / 1073741824).toFixed(2) }} GB @@ -187,6 +193,9 @@ import { RelativeUrlPipe } from '@app/shared/pipes/relative-url/relative-url.pip + + No catalogue snapshot has a verification run that reached valid on this network. +
@@ -232,6 +241,14 @@ import { RelativeUrlPipe } from '@app/shared/pipes/relative-url/relative-url.pip ], }) export class BootstrapOverviewComponent implements OnInit, OnDestroy { + subfeatures(overview: BootstrapOverview): { label: string; status: string; reason: string | null }[] { + return [ + { label: 'Snapshot catalogue', status: overview.snapshot_catalogue_status ?? 'unavailable', reason: overview.snapshot_catalogue_reason ?? null }, + { label: 'Verification store', status: overview.verification_store_status ?? 'unavailable', reason: overview.verification_store_reason ?? null }, + { label: 'Planning', status: overview.planning_status ?? 'unavailable', reason: overview.planning_status === 'available' ? null : 'Planning needs the catalogue and the verification store.' }, + { label: 'Operator jobs', status: overview.operator_status ?? 'unavailable', reason: overview.operator_reason ?? null }, + ]; + } loading = true; error: string | null = null; overview: BootstrapOverview | null = null; diff --git a/frontend/src/app/universe/bootstrap/bootstrap-planner.component.spec.ts b/frontend/src/app/universe/bootstrap/bootstrap-planner.component.spec.ts index 5a54bd01645..91d33f5c42f 100644 --- a/frontend/src/app/universe/bootstrap/bootstrap-planner.component.spec.ts +++ b/frontend/src/app/universe/bootstrap/bootstrap-planner.component.spec.ts @@ -1,31 +1,62 @@ import { ChangeDetectorRef } from '@angular/core'; -import { Subject, throwError } from 'rxjs'; +import { BehaviorSubject, Subject, of, throwError } from 'rxjs'; import { describe, expect, it, vi } from 'vitest'; import { BootstrapPlannerComponent } from './bootstrap-planner.component'; -import { BootstrapApiService } from './bootstrap.service'; +import { BootstrapApiService, NodeBootstrapPlan } from './bootstrap.service'; + +function planner(api: Partial): BootstrapPlannerComponent { + return new BootstrapPlannerComponent({ network: 'signet', networkChanged$: new BehaviorSubject('signet'), ...api } as unknown as BootstrapApiService, { markForCheck: vi.fn() } as unknown as ChangeDetectorRef); +} + +const measuredPlan: NodeBootstrapPlan = { + plan_id: 'plan-1', + network: 'signet', + node_id: 'core-1', + node_version: '29.0.0', + target_height: 10, + measurement_status: 'measured', + measured: { + node_observed_at: '2026-09-17T00:00:00.000Z', + current_phase: 'traditional_ibd', + tip_height: 5, + headers: 12, + blocks_on_disk_bytes: 1073741824, + capacity: { method: 'statfs', measured_at: '2026-09-17T00:00:00.000Z', free_bytes: 10737418240, total_bytes: null }, + supports_loadtxoutset: true, + }, + selected_snapshot: { snapshot_id: 'signet-10', height: 10, block_hash: 'aa'.repeat(32), core_version: '29.0.0', size_bytes: 2147483648, sha256: 'bb'.repeat(32), utxo_commitment: 'cc'.repeat(32), verification_id: 'run-1', verified_at: '2026-09-16T00:00:00.000Z' }, + requirements: { snapshot_file_bytes: 2147483648, chainstate_estimate_bytes: 2147483648, headroom_bytes: 429496729, required_bytes: 4724464025, free_bytes: 10737418240, feasible: true }, + assumptions: ['No bandwidth was declared, so no download time is estimated.'], + caller_declared: { available_disk_gb: null, bandwidth_mbps: null, matches_measurement: null }, + estimates: { snapshot_download_hours: null, ibd_download_hours: null, basis: 'not estimated' }, + expected_transitions: ['snapshot_loading', 'snapshot_active_syncing_to_tip', 'background_validation', 'fully_validated'], + actions_not_executed: ['loadtxoutset utxo-10.dat on core-1 through an authorized operator job'], + rollback_instructions: ['Stop the node and remove the chainstate_snapshot directory.'], + created_at: '2026-09-17T00:00:01.000Z', +}; describe('bootstrap planner source failures', () => { it('does not fabricate a projection when the owned planner is unavailable', () => { - const api = { generateBootstrapPlan$: () => throwError(() => ({ status: 503 })) } as unknown as BootstrapApiService; - const component = new BootstrapPlannerComponent(api, { markForCheck: vi.fn() } as unknown as ChangeDetectorRef); + const component = planner({ generateBootstrapPlan$: () => throwError(() => ({ status: 503, error: { stage: 'unavailable-node-source', error: 'Bootstrap evidence is unavailable.' } })) }); component.calculatePlan(); expect(component.plan).toBeNull(); expect(component.calculating).toBe(false); expect(component.failure).toBeTruthy(); + expect(component.rejection?.stage).toBe('unavailable-node-source'); + expect(component.rejectionTitle(component.rejection!)).toBe('Planning source unavailable'); }); - it('clears a previous failure while retrying and rejects an unmeasured projection', () => { + it('clears a previous failure while retrying and rejects a projection that is not a measured plan', () => { const response = new Subject(); - const api = { generateBootstrapPlan$: vi.fn() + const component = planner({ generateBootstrapPlan$: vi.fn() .mockReturnValueOnce(throwError(() => ({ status: 503 }))) - .mockReturnValueOnce(response) } as unknown as BootstrapApiService; - const component = new BootstrapPlannerComponent(api, { markForCheck: vi.fn() } as unknown as ChangeDetectorRef); + .mockReturnValueOnce(response) }); component.calculatePlan(); component.calculatePlan(); expect(component.failure).toBeNull(); expect(component.plan).toBeNull(); expect(component.calculating).toBe(true); - const plan = { assumeutxo_ready_hours: 4, traditional_ibd_hours: 20, background_validation_hours: 25 }; + const plan = { network: 'signet', measurement_status: 'measured', assumeutxo_ready_hours: 4, traditional_ibd_hours: 20, background_validation_hours: 25 }; response.next(plan); response.complete(); expect(component.plan).toBeNull(); @@ -33,3 +64,55 @@ describe('bootstrap planner source failures', () => { expect(component.calculating).toBe(false); }); }); + +describe('bootstrap planner measured plans', () => { + it('sends only declared inputs and renders the feasibility document without inventing hours', () => { + const generateBootstrapPlan$ = vi.fn(() => of(measuredPlan)); + const component = planner({ generateBootstrapPlan$ }); + component.calculatePlan(); + expect(generateBootstrapPlan$).toHaveBeenCalledWith({}); + expect(component.plan?.plan_id).toBe('plan-1'); + expect(component.rejection).toBeNull(); + expect(component.hours(null)).toBe('not estimated'); + expect(component.hours(1.5)).toContain('assumed'); + expect(component.gb(measuredPlan.requirements.required_bytes)).toBe('4.40'); + }); + + it('binds the plan to the requested height and network', () => { + const wrongHeight = planner({ generateBootstrapPlan$: () => of({ ...measuredPlan, target_height: 11 }) }); + wrongHeight.targetHeight = 10; + wrongHeight.bandwidthMbps = 100; + wrongHeight.calculatePlan(); + expect(wrongHeight.plan).toBeNull(); + expect(wrongHeight.rejection?.message).toContain('network-bound'); + + const wrongNetwork = planner({ generateBootstrapPlan$: () => of({ ...measuredPlan, network: 'mainnet' }) }); + wrongNetwork.calculatePlan(); + expect(wrongNetwork.plan).toBeNull(); + + const sends = vi.fn(() => of(measuredPlan)); + const declared = planner({ generateBootstrapPlan$: sends }); + declared.targetHeight = 10; + declared.availableDiskGb = 50; + declared.bandwidthMbps = 100; + declared.calculatePlan(); + expect(sends).toHaveBeenCalledWith({ target_height: 10, available_disk_gb: 50, bandwidth_mbps: 100 }); + expect(declared.plan?.plan_id).toBe('plan-1'); + }); + + it.each([ + [409, 'no-compatible-snapshot', 'No compatible snapshot'], + [409, 'snapshot-not-verified', 'not verified'], + [409, 'insufficient-capacity', 'insufficient'], + [409, 'chainstate-not-eligible', 'not eligible'], + [409, 'stale-measurement', 'stale'], + [404, 'node-not-observed', 'not observed'], + [503, 'durable-store-unavailable', 'unavailable'], + ])('renders the typed rejection %s %s', (status, stage, title) => { + const component = planner({ generateBootstrapPlan$: () => throwError(() => ({ status, error: { stage, error: 'reason text' } })) }); + component.calculatePlan(); + expect(component.plan).toBeNull(); + expect(component.rejection).toEqual({ stage, httpStatus: status, message: 'reason text' }); + expect(component.rejectionTitle(component.rejection!)).toContain(title); + }); +}); diff --git a/frontend/src/app/universe/bootstrap/bootstrap-planner.component.ts b/frontend/src/app/universe/bootstrap/bootstrap-planner.component.ts index 847de46367b..62f7ded8a76 100644 --- a/frontend/src/app/universe/bootstrap/bootstrap-planner.component.ts +++ b/frontend/src/app/universe/bootstrap/bootstrap-planner.component.ts @@ -4,9 +4,19 @@ import { CommonModule } from '@angular/common'; import { RouterModule } from '@angular/router'; import { FormsModule } from '@angular/forms'; import { classifyLoadFailure, loadFailureMessage } from '@app/shared/load-state'; -import { BootstrapApiService } from './bootstrap.service'; +import { BootstrapApiService, BootstrapPlanRequest, NodeBootstrapPlan } from './bootstrap.service'; import { RelativeUrlPipe } from '@app/shared/pipes/relative-url/relative-url.pipe'; +/** A plan the backend refused, with the typed reason it named. */ +export interface BootstrapPlanRejection { + /** The backend's stage code, for example no-compatible-snapshot, or null when the request never got an answer body. */ + stage: string | null; + httpStatus: number | null; + message: string; +} + +const GIB = 1073741824; + @Component({ selector: 'app-bootstrap-planner', standalone: true, @@ -17,10 +27,10 @@ import { RelativeUrlPipe } from '@app/shared/pipes/relative-url/relative-url.pip

Node Bootstrap Planner

- Planning inputs + Feasibility from measurements

- Simulates Initial Block Download (IBD) sync duration comparing traditional genesis-to-tip validation vs AssumeUTXO snapshot fast bootstrap. + Asks the owned node whether an AssumeUTXO load is feasible: measured node capabilities and disk capacity against a verified, pinned, compatible snapshot. Nothing is executed and no sync duration is estimated.

@@ -134,11 +189,16 @@ import { IntelligenceApiService } from './intelligence-api.service'; `], }) export class QueryStudioComponent implements OnInit, OnDestroy { - schema: any[] = []; + schema: TableSchemaInfo[] = []; + schemaInfo: QuerySchemaResult | null = null; + schemaLoading = false; + schemaError: QueryFailure | null = null; sqlQuery = ''; loading = false; - queryError: string | null = null; - queryResult: any = null; + queryError: QueryFailure | null = null; + /** The latest page; `rows` accumulates every page of the same statement. */ + queryResult: QueryExecutionResult | null = null; + rows: Array> = []; private sub?: Subscription; private schemaSub?: Subscription; @@ -156,40 +216,73 @@ export class QueryStudioComponent implements OnInit, OnDestroy { ngOnInit(): void { this.loadSchema();this.context.add(this.state.networkChanged$?.subscribe(()=>this.reset()));this.context.add(this.owner.key$.subscribe(()=>this.reset())); } - invalidate(): void {this.generation++;this.sub?.unsubscribe();this.queryResult=null;this.queryError=null;this.loading=false;this.cdr.markForCheck();} + invalidate(): void {this.generation++;this.sub?.unsubscribe();this.queryResult=null;this.rows=[];this.queryError=null;this.loading=false;this.cdr.markForCheck();} private reset(): void {this.invalidate();this.loadSchema();} private loadSchema(): void { - this.schemaSub?.unsubscribe();this.schema=[];const generation=this.generation; - this.schemaSub=this.api.getQuerySchema$().subscribe({next:res=>{if(this.destroyed||generation!==this.generation)return;this.schema=Array.isArray(res?.tables)?res.tables:[];this.cdr.markForCheck();},error:()=>{if(this.destroyed||generation!==this.generation)return;this.schema=[];this.queryError='Analytics source unavailable.';this.cdr.markForCheck();}}); + this.schemaSub?.unsubscribe();this.schema=[];this.schemaInfo=null;this.schemaError=null;this.schemaLoading=true;const generation=this.generation; + this.schemaSub=this.api.getQuerySchema$().subscribe({ + next:res=>{if(this.destroyed||generation!==this.generation)return;this.schemaLoading=false;this.schema=Array.isArray(res?.tables)?res.tables:[];this.schemaInfo=res&&Array.isArray(res.tables)?res:null;this.cdr.markForCheck();}, + error:err=>{if(this.destroyed||generation!==this.generation)return;this.schemaLoading=false;this.schema=[];this.schemaInfo=null;this.schemaError=classifyQueryFailure(err,'Schema discovery failed.');this.cdr.markForCheck();}}); + } + + failureTitle(failure: QueryFailure): string { + switch (failure.kind) { + case 'grammar': return 'Rejected by the query grammar.'; + case 'engine': return failure.reason === 'unconfigured' ? 'No analytics replica is configured on this deployment.' : 'The analytics engine is unavailable.'; + case 'deadline': return 'The query did not finish inside its deadline.'; + default: return 'The request failed.'; + } + } + + /** Exact strings stay exact; null is shown as NULL rather than an empty cell. */ + cell(value: unknown): string { + if (value === null || value === undefined) return 'NULL'; + return typeof value === 'object' ? JSON.stringify(value) : String(value); } useSaved(sql: string): void { this.invalidate(); this.sqlQuery=sql; this.cdr.markForCheck(); } loadSampleQuery(): void { this.sqlQuery = 'SELECT txid, fee_sats, feerate FROM mempool_transactions LIMIT 10'; - this.sub?.unsubscribe(); this.queryResult=null; this.queryError=null; this.loading=false; this.cdr.markForCheck(); + this.invalidate(); } executeQuery(): void { this.invalidate(); if (!this.sqlQuery.trim()) return; + this.run(undefined); + } + + loadMore(): void { + const cursor = this.queryResult?.next_cursor; + if (!cursor || this.loading) return; + this.run(cursor); + } + + private run(cursor: string | undefined): void { const query=this.sqlQuery.trim(),generation=this.generation; this.loading = true; this.queryError = null; this.cdr.markForCheck(); this.sub?.unsubscribe(); - this.sub = this.api.executeDevQuery$(this.sqlQuery.trim()).subscribe({ + this.sub = this.api.executeDevQuery$(query, 100, cursor).subscribe({ next: (res) => { if(this.destroyed||generation!==this.generation||query!==this.sqlQuery.trim())return; - this.queryResult = res; this.loading = false; + if (!res || !Array.isArray(res.rows) || !Array.isArray(res.columns)) { + this.queryResult = null; this.rows = []; + this.queryError = { kind: 'other', stage: null, reason: null, position: null, message: 'The engine answered without rows or columns.' }; + } else { + this.queryResult = res; + this.rows = cursor ? this.rows.concat(res.rows) : res.rows; + } this.cdr.markForCheck(); }, error: (err) => { if(this.destroyed||generation!==this.generation)return; - this.queryResult=null; - this.queryError = err?.error?.error || err?.message || 'Query execution failed'; + if (!cursor) { this.queryResult=null; this.rows=[]; } + this.queryError = classifyQueryFailure(err, 'Query execution failed'); this.loading = false; this.cdr.markForCheck(); }, diff --git a/frontend/src/app/universe/intelligence-platform/saved-query-panel.component.spec.ts b/frontend/src/app/universe/intelligence-platform/saved-query-panel.component.spec.ts index ccbefe501ca..03ac2b56b31 100644 --- a/frontend/src/app/universe/intelligence-platform/saved-query-panel.component.spec.ts +++ b/frontend/src/app/universe/intelligence-platform/saved-query-panel.component.spec.ts @@ -24,6 +24,6 @@ import { QueryStudioComponent } from './query-studio.component'; describe('query execution lifecycle',()=>{ it('clears prior results on edits, owner changes and failures without executing a sample',()=>{ const calls:Subject[]=[];const api:any={getQuerySchema$:()=>of({tables:[]}),executeDevQuery$:vi.fn(()=>{const s=new Subject();calls.push(s);return s;})};const owner:any={key$:new BehaviorSubject('first')},state:any={networkChanged$:new Subject()}; - const page=new QueryStudioComponent(api,{markForCheck:()=>{}} as ChangeDetectorRef,state,owner);page.ngOnInit();page.loadSampleQuery();expect(api.executeDevQuery$).not.toHaveBeenCalled();page.executeQuery();calls[0].next({rows:[{value:1}]});expect(page.queryResult).not.toBeNull();page.invalidate();expect(page.queryResult).toBeNull();page.executeQuery();owner.key$.next('second');expect(calls[1].observed).toBe(false);expect(page.queryResult).toBeNull();page.executeQuery();calls[2].error(new Error('unavailable'));expect(page.queryResult).toBeNull();expect(page.queryError).toBeTruthy();page.ngOnDestroy(); + const page=new QueryStudioComponent(api,{markForCheck:()=>{}} as ChangeDetectorRef,state,owner);page.ngOnInit();page.loadSampleQuery();expect(api.executeDevQuery$).not.toHaveBeenCalled();page.executeQuery();calls[0].next({rows:[{value:1}],columns:["value"]});expect(page.queryResult).not.toBeNull();page.invalidate();expect(page.queryResult).toBeNull();page.executeQuery();owner.key$.next('second');expect(calls[1].observed).toBe(false);expect(page.queryResult).toBeNull();page.executeQuery();calls[2].error(new Error('unavailable'));expect(page.queryResult).toBeNull();expect(page.queryError).toBeTruthy();page.ngOnDestroy(); }); }); diff --git a/frontend/src/app/universe/multichain-explorer/multichain-explorer.component.ts b/frontend/src/app/universe/multichain-explorer/multichain-explorer.component.ts index 0b73f0df799..9fbaa645f76 100644 --- a/frontend/src/app/universe/multichain-explorer/multichain-explorer.component.ts +++ b/frontend/src/app/universe/multichain-explorer/multichain-explorer.component.ts @@ -448,7 +448,7 @@ export class MultichainExplorerComponent implements OnInit, OnDestroy { } this.saved = this.local.toggleBookmark({ chain: this.chain, - network: 'mainnet', + network: this.api.chainNetwork(this.chain), kind, value: this.reference, path: this.router.url.split('?')[0], @@ -722,7 +722,7 @@ export class MultichainExplorerComponent implements OnInit, OnDestroy { ) : of({ chain: this.chain, - network: 'mainnet', + network: this.api.chainNetwork(this.chain), state: 'unavailable', reason: 'section-not-supported', }); @@ -736,7 +736,7 @@ export class MultichainExplorerComponent implements OnInit, OnDestroy { ) : of({ chain: this.chain, - network: 'mainnet', + network: this.api.chainNetwork(this.chain), state: 'unavailable', reason: 'section-not-supported', }); @@ -757,7 +757,7 @@ export class MultichainExplorerComponent implements OnInit, OnDestroy { } this.local.recordVisit({ chain: this.chain, - network: 'mainnet', + network: this.api.chainNetwork(this.chain), kind, value: this.reference, path: this.router.url.split('?')[0], @@ -770,7 +770,7 @@ export class MultichainExplorerComponent implements OnInit, OnDestroy { return ( !!kind && !!this.reference && - this.local.isBookmarked(kind, this.reference, this.chain, 'mainnet') + this.local.isBookmarked(kind, this.reference, this.chain, this.api.chainNetwork(this.chain)) ); } @@ -822,6 +822,15 @@ export class MultichainExplorerComponent implements OnInit, OnDestroy { return $localize`:@@universe.chain.error-not-found:The ${this.chainName}:CHAIN: authority has no record of this object. It may not exist, or it may be outside the range this indexer covers.`; } if (status === '503') { + // The configured network is not one this overlay serves. Name it: the + // page must never read as mainnet data under another network's name. + const body = (error as { error?: unknown }).error; + const reasons = typeof body === 'string' ? [body] + : typeof body === 'object' && body !== null ? [(body as { error?: unknown }).error, (body as { message?: unknown }).message] : []; + if (reasons.includes(`${this.chain}-network-unavailable`)) { + const network = this.api.chainNetwork(this.chain); + return $localize`:@@universe.chain.error-network-unavailable:The ${this.chainName}:CHAIN: authority does not serve ${network}:NETWORK:, the network this deployment is configured to read. Nothing is shown from another network.`; + } return $localize`:@@universe.chain.error-unavailable:The ${this.chainName}:CHAIN: authority is not answering right now. Nothing here is stale data presented as current: the page shows no facts rather than old ones.`; } if (status === '400') { diff --git a/frontend/src/app/universe/network-observatory/network-observatory.component.html b/frontend/src/app/universe/network-observatory/network-observatory.component.html index fc5b11e31d4..3b742ba9b06 100644 --- a/frontend/src/app/universe/network-observatory/network-observatory.component.html +++ b/frontend/src/app/universe/network-observatory/network-observatory.component.html @@ -7,48 +7,59 @@

Cross-Node Mempool, Relay, Policy & Block-Template Observatory

- Observer fleet topology, first-seen timelines, multi-node acceptance matrix, local vs pool block-template comparisons. + What this deployment's own observer recorded: its owned node's relay policy, the mempool lifecycle it retained for a transaction, and the block templates it collected. One observer; nothing it cannot measure is estimated.

-

Reading cross-node observer metrics...

+

Reading the owned observer...

+
+ +
+

Observer Unavailable

+

The owned node observation behind these reads could not be obtained. No fleet, timeline or template is shown in its place.

+

{{ vm.message }}

+

Reason code: {{ vm.stage }}

+

Network Observatory Error

Could not fetch node propagation or block-template data.

{{ vm.message }}

+

Reason code: {{ vm.stage }}

+
- Observer Nodes - {{ vm.nodes?.length }} Online - Global vantage fleet + Observers + {{ vm.propagation.observer.observers }} + Single observer: {{ vm.propagation.observer.observerId }}
- Median Latency - {{ vm.propagation.medianLatencyMs }} ms - P95: {{ vm.propagation.p95LatencyMs }} ms + Inter-node Latency + {{ measured(vm.propagation.medianLatencyMs, 'ms') }} + Median and P95 need more than one observer
- Propagation Spread - {{ vm.propagation.spreadDeltaMs }} ms - Global delta + Observation Window + {{ vm.propagation.window.collection }} + {{ vm.propagation.window.retainedTransactions }} retained, {{ vm.propagation.window.retentionMs / 1000 }} s retention, age {{ vm.propagation.window.ageMs }} ms
Template Spread - {{ vm.templates.feeRateSpreadSatVb }} sat/vB - {{ vm.templates.candidateTemplates.length }} Candidates + {{ measured(vm.templates.feeRateSpreadSatVb, 'sat/vB') }} + {{ vm.templates.candidateTemplates.length }} candidate template(s) retained
-

Universe Observer Fleet & Relay Policies

+

Owned Observer & Relay Policy

+

One observer. Regions, clock offsets and multi-node propagation are not measured by a single local observer and are shown as unknown.

@@ -57,21 +68,24 @@

Universe Observer Fleet & Relay Policies

- + - - + + - - - - - + + + + +
Client Software Full-RBF Min Relay FeeClock DriftClock Offset Mempool Size Status
{{ n.name }}{{ n.region }}{{ n.name }} ({{ n.network }}){{ n.region === 'unknown' ? 'unknown (not measured)' : n.region }} {{ n.clientVersion }}{{ n.fullRbf ? 'Active' : 'Opt-in' }}{{ n.minRelayFeeRate }} sat/vB{{ n.clockOffsetMs }} ms{{ n.mempoolTxCount }} txs{{ n.status }} + unknown + {{ n.fullRbf ? 'Active' : 'Opt-in' }} + {{ measured(n.minRelayFeeRate, 'sat/vB') }}{{ measured(n.clockOffsetMs, 'ms') }}{{ n.mempoolTxCount === null ? 'not measured' : n.mempoolTxCount + ' txs' }}{{ n.status }}
@@ -79,29 +93,37 @@

Universe Observer Fleet & Relay Policies

-

Sample Transaction Propagation Timeline ({{ vm.propagation.txid.slice(0, 16) }}...)

- - - - - - - - - - - - - - - -
Observer NodeArrival Delta (ms)Accepted
{{ o.nodeName }}+{{ o.deltaFromFirstMs }} msAccepted
+

Observed Mempool Lifecycle ({{ vm.propagation.txid.slice(0, 16) }}...)

+

Observed Mempool Lifecycle

+

This observer has not completed a mempool poll yet, so no transaction lifecycle has been recorded. Nothing is shown in its place.

+

This observer has no retained detection for that transaction. Unknown transactions are not sampled.

+ +

Detection time is this observer's local poll time, not peer arrival time. Observed by {{ vm.propagation.observer.observerId }} ({{ vm.propagation.observer.method }}); last poll {{ vm.propagation.window.lastPollUtc ?? 'none' }}, retained until {{ vm.propagation.expiresAtUtc }}.

+ + + + + + + + + + + + + + + +
ObserverDelta from first detection (ms)Presence
{{ o.nodeName }} (partial poll)+{{ o.deltaFromFirstMs }} ms{{ o.presence === 'present' ? 'In mempool' : 'Left mempool' }}
+
-

Candidate Block Templates (Height #{{ vm.templates.blockHeight }})

- +

Candidate Block Templates (Height #{{ vm.templates.blockHeight }})

+

Candidate Block Templates

+

No block template has been observed by this backend's own collector yet. Per-pool templates are not collected here.

+
@@ -113,15 +135,35 @@

Candidate Block Templates (Height #{{ vm.templates.blockHeight }})

- + - + + + +
Template Source
{{ t.poolName }}{{ t.poolName }} ({{ t.sourceType }}) {{ t.txCount }} {{ t.totalWeight }} WU {{ t.totalFeesSats }}{{ t.expectedMedianFeeRate }} sat/vB{{ measured(t.expectedMedianFeeRate, 'sat/vB') }}
+ + + + + + + + + + + + + + +
Collector SourceStatusLast TemplateLast Error
{{ s.name }}{{ s.status }}{{ s.lastTemplateAt ?? 'none' }}{{ s.lastError ?? '' }}
+ +

{{ vm.propagation.scope }}

diff --git a/frontend/src/app/universe/network-observatory/network-observatory.component.spec.ts b/frontend/src/app/universe/network-observatory/network-observatory.component.spec.ts index ae5e5635576..9d093e03fc1 100644 --- a/frontend/src/app/universe/network-observatory/network-observatory.component.spec.ts +++ b/frontend/src/app/universe/network-observatory/network-observatory.component.spec.ts @@ -1,24 +1,36 @@ import { HttpErrorResponse } from '@angular/common/http'; -import { of, throwError } from 'rxjs'; +import { Subject, of, throwError } from 'rxjs'; import { describe, expect, it, vi } from 'vitest'; import { SeoService } from '@app/services/seo.service'; import { UniverseApiService } from '@app/universe/universe-api.service'; import { NetworkObservatoryComponent } from './network-observatory.component'; const unavailable = () => throwError(() => new HttpErrorResponse({ status: 503 })); +const ownedNodeDown = () => throwError(() => new HttpErrorResponse({ status: 503, error: { stage: 'owned-node-unavailable', error: 'Owned Core peer/network observation is unavailable, stale or on the wrong network.' } })); -function observe(api: Partial): any { +const observer = { observerId: 'backend-1', observers: 1 as const, clockOffsetMs: null, clockUncertaintyMs: null, method: 'mempool poll' }; +const window = { observedAtUtc: '2026-09-17T00:00:00.000Z', ageMs: 5, freshnessLimitMs: 60000, retentionMs: 3600000, retainedTransactions: 2, lastPollUtc: '2026-09-17T00:00:00.000Z', lastCompletePollUtc: '2026-09-17T00:00:00.000Z', collection: 'observing' as const }; +const node = { id: 'n1', name: 'owned core', region: 'unknown', clientVersion: '/Satoshi:29.0.0/', protocolVersion: 70016, fullRbf: null, minRelayFeeRate: 1, clockOffsetMs: null, connectedPeers: 8, mempoolTxCount: null, status: 'online' as const, network: 'signet', observedAtUtc: '2026-09-17T00:00:00.000Z', policySourceAvailable: true, scope: 'one observer' }; +const noTemplates = { network: 'signet', state: 'no-templates-observed' as const, blockHeight: null, generatedAt: null, candidateTemplates: [], consensusMempoolTxCount: null, missingFromLocalCount: null, feeRateSpreadSatVb: null, sources: [{ sourceId: 'core', name: 'Core GBT', status: 'not_collected' as const, lastTemplateAt: null, lastError: null }], observer, retention: { templates: 4, txidsPerTemplate: 100 }, scope: 'one observer' }; + +function build(api: Partial): NetworkObservatoryComponent { const view = new NetworkObservatoryComponent(api as UniverseApiService, { setTitle: vi.fn() } as unknown as SeoService); view.ngOnInit(); + return view; +} + +function observe(api: Partial): any { + const view = build(api); let observed: any; view.vm$.subscribe((value) => { observed = value; }).unsubscribe(); + view.ngOnDestroy(); return observed; } describe('Network observatory on an absent source', () => { it('shows the error state with the reason when the fleet read fails', () => { const vm = observe({ getObserverNodes$: unavailable, getPropagationObservation$: unavailable, getBlockTemplateComparison$: unavailable }); - expect(vm.kind).toBe('error'); + expect(vm.kind).toBe('unavailable'); expect(vm.message).toBeTruthy(); expect(vm.propagation).toBeUndefined(); }); @@ -26,10 +38,64 @@ describe('Network observatory on an absent source', () => { it('does not show a timeline over an empty fleet table when only the node read fails', () => { const vm = observe({ getObserverNodes$: unavailable, - getPropagationObservation$: () => of({ txid: 'ab'.repeat(32), nodeObservations: [] }), - getBlockTemplateComparison$: () => of({ blockHeight: 1, candidateTemplates: [] }), + getPropagationObservation$: () => of({ txid: 'ab'.repeat(32), nodeObservations: [] } as any), + getBlockTemplateComparison$: () => of({ blockHeight: 1, candidateTemplates: [] } as any), }); - expect(vm.kind).toBe('error'); + expect(vm.kind).toBe('unavailable'); expect(vm.nodes).toBeUndefined(); }); + + it('renders owned-node-unavailable as an unavailable state with its reason code, and retry re-reads', () => { + const nodes = vi.fn().mockReturnValueOnce(ownedNodeDown()).mockReturnValue(of({ nodes: [node], total: 1 })); + const view = build({ + getObserverNodes$: nodes, + getPropagationObservation$: () => of({ txid: null, network: 'signet', state: 'no-observations', firstSeenTimestamp: null, lastSeenTimestamp: null, nodeObservations: [], medianLatencyMs: null, p95LatencyMs: null, spreadDeltaMs: null, observer, window: { ...window, collection: 'not_started' }, expiresAtUtc: null, scope: 'one observer' } as any), + getBlockTemplateComparison$: () => of(noTemplates as any), + }); + const seen: any[] = []; + view.vm$.subscribe((value) => seen.push(value)); + expect(seen.at(-1).kind).toBe('unavailable'); + expect(seen.at(-1).stage).toBe('owned-node-unavailable'); + expect(seen.at(-1).message).toContain('Owned Core'); + view.retry(); + expect(seen.at(-1).kind).toBe('ready'); + expect(seen.at(-1).propagation.state).toBe('no-observations'); + expect(seen.at(-1).templates.state).toBe('no-templates-observed'); + expect(nodes).toHaveBeenCalledTimes(2); + view.ngOnDestroy(); + }); + + it('cancels an in-flight attempt when retried and stops on destroy', () => { + const first = new Subject(); + const second = new Subject(); + const nodes = vi.fn().mockReturnValueOnce(first).mockReturnValue(second); + const view = build({ getObserverNodes$: nodes, getPropagationObservation$: () => of({} as any), getBlockTemplateComparison$: () => of({} as any) }); + expect(first.observed).toBe(true); + view.retry(); + expect(first.observed).toBe(false); + expect(second.observed).toBe(true); + view.ngOnDestroy(); + expect(second.observed).toBe(false); + }); + + it('keeps a 404 as an error rather than an unavailable observer', () => { + const vm = observe({ getObserverNodes$: () => throwError(() => new HttpErrorResponse({ status: 404 })), getPropagationObservation$: () => of({} as any), getBlockTemplateComparison$: () => of({} as any) }); + expect(vm.kind).toBe('error'); + }); +}); + +describe('Network observatory single-observer facts', () => { + it('passes observed lifecycles through with no latency percentiles and labels unmeasured values', () => { + const propagation = { txid: 'ab'.repeat(32), network: 'signet', state: 'observed', firstSeenTimestamp: 1, lastSeenTimestamp: 2, nodeObservations: [{ nodeId: 'n1', nodeName: 'owned core', arrivedAt: 1, deltaFromFirstMs: 0, accepted: true, presence: 'present', completePoll: true, previousCompletePollUtc: null }], medianLatencyMs: null, p95LatencyMs: null, spreadDeltaMs: null, observer, window, expiresAtUtc: '2026-09-17T01:00:00.000Z', scope: 'one observer' }; + const view = build({ getObserverNodes$: () => of({ nodes: [node], total: 1 }), getPropagationObservation$: () => of(propagation as any), getBlockTemplateComparison$: () => of(noTemplates as any) }); + let vm: any; + view.vm$.subscribe((value) => { vm = value; }).unsubscribe(); + expect(vm.kind).toBe('ready'); + expect(vm.nodes[0].region).toBe('unknown'); + expect(vm.propagation.observer.observers).toBe(1); + expect(vm.propagation.medianLatencyMs).toBeNull(); + expect(view.measured(null, 'ms')).toBe('not measured'); + expect(view.measured(0, 'ms')).toBe('0 ms'); + view.ngOnDestroy(); + }); }); diff --git a/frontend/src/app/universe/network-observatory/network-observatory.component.ts b/frontend/src/app/universe/network-observatory/network-observatory.component.ts index a2f5d0a25fb..67beb3b77c4 100644 --- a/frontend/src/app/universe/network-observatory/network-observatory.component.ts +++ b/frontend/src/app/universe/network-observatory/network-observatory.component.ts @@ -1,7 +1,7 @@ -import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core'; +import { ChangeDetectionStrategy, Component, OnDestroy, OnInit } from '@angular/core'; import { CommonModule } from '@angular/common'; import { RouterModule } from '@angular/router'; -import { BehaviorSubject, Observable, catchError, combineLatest, map, of } from 'rxjs'; +import { BehaviorSubject, Observable, Subject, Subscription, catchError, combineLatest, map, of, startWith, switchMap } from 'rxjs'; import { classifyLoadFailure, loadFailureMessage } from '@app/shared/load-state'; import { SeoService } from '@app/services/seo.service'; import { UniverseApiService } from '@app/universe/universe-api.service'; @@ -13,12 +13,19 @@ import { PropagationObservation, } from '@app/universe/universe.types'; +/** + * unavailable: the owned node observation behind every read is down (503 + * owned-node-unavailable or another 5xx) and a retry can help; error: any + * other failure, including a route this deployment does not serve. + */ interface NetworkViewModel { - readonly kind: 'loading' | 'ready' | 'error'; + readonly kind: 'loading' | 'ready' | 'unavailable' | 'error'; readonly nodes?: ObserverNode[]; readonly propagation?: PropagationObservation; readonly templates?: BlockTemplateComparison; readonly message?: string; + /** The backend's typed reason, for example owned-node-unavailable. */ + readonly stage?: string; } @Component({ @@ -29,8 +36,10 @@ interface NetworkViewModel { imports: [RelativeUrlPipe, CommonModule, RouterModule], changeDetection: ChangeDetectionStrategy.OnPush, }) -export class NetworkObservatoryComponent implements OnInit { +export class NetworkObservatoryComponent implements OnInit, OnDestroy { private readonly state = new BehaviorSubject({ kind: 'loading' }); + private readonly retry$ = new Subject(); + private subscription?: Subscription; readonly vm$: Observable = this.state.asObservable(); constructor( @@ -43,18 +52,44 @@ export class NetworkObservatoryComponent implements OnInit { ngOnInit(): void { // No per-read fallback: a fleet table the source could not answer is an // error with its reason, not an empty table under a live-looking timeline. - combineLatest([ - this.api.getObserverNodes$(), - this.api.getPropagationObservation$(), - this.api.getBlockTemplateComparison$(), - ]).pipe( - map(([nodesData, propagation, templates]): NetworkViewModel => ({ - kind: 'ready', - nodes: nodesData.nodes, - propagation, - templates, - })), - catchError((error) => of({ kind: 'error', message: loadFailureMessage(classifyLoadFailure(error)) })), + // Retry re-runs all three reads; a click while one attempt is in flight + // cancels it. + this.subscription = this.retry$.pipe( + startWith(undefined), + switchMap(() => combineLatest([ + this.api.getObserverNodes$(), + this.api.getPropagationObservation$(), + this.api.getBlockTemplateComparison$(), + ]).pipe( + map(([nodesData, propagation, templates]): NetworkViewModel => ({ + kind: 'ready', + nodes: nodesData.nodes, + propagation, + templates, + })), + catchError((error) => of(this.failure(error))), + startWith({ kind: 'loading' }), + )), ).subscribe((vm) => this.state.next(vm)); } + + ngOnDestroy(): void { + this.subscription?.unsubscribe(); + } + + retry(): void { + this.retry$.next(); + } + + private failure(error: any): NetworkViewModel { + const reason = classifyLoadFailure(error); + const stage = typeof error?.error?.stage === 'string' ? error.error.stage : undefined; + const message = typeof error?.error?.error === 'string' ? error.error.error : loadFailureMessage(reason); + return { kind: reason === 'unavailable' || reason === 'timeout' || reason === 'network' ? 'unavailable' : 'error', message, stage }; + } + + /** A value one local observer did not measure is shown as exactly that. */ + measured(value: number | null | undefined, unit: string): string { + return value === null || value === undefined ? 'not measured' : `${value} ${unit}`; + } } diff --git a/frontend/src/app/universe/private-submission/private-submission-accelerators.component.ts b/frontend/src/app/universe/private-submission/private-submission-accelerators.component.ts index ed4545c6048..d5e9bcd12ed 100644 --- a/frontend/src/app/universe/private-submission/private-submission-accelerators.component.ts +++ b/frontend/src/app/universe/private-submission/private-submission-accelerators.component.ts @@ -8,6 +8,7 @@ import { loadFailureMessage, } from '@app/shared/load-state'; import { + AcceleratorDirectoryRef, AcceleratorProvider, PrivateSubmissionApiService, } from './private-submission.service'; @@ -25,7 +26,7 @@ import { RelativeUrlPipe } from '@app/shared/pipes/relative-url/relative-url.pip

Accelerator Providers

- Signed provider directory: claimed pools and published fees. + The owned provider directory is the trust root: identities, signing keys and their validity windows, and the terms each provider declares. Nothing here is probed or paid.

-
Available Acceleration Services
+
Providers in the owned directory
+
+ {{ directory.source }}, revision {{ directory.revision }}, loaded {{ directory.loaded_at_utc }} for {{ network }}. +
Provider Networks + Keys and validity Claimed partner pools Minimum fee Max vsize + Payment methods Health Action - - The provider directory is empty. + + The owned directory lists no provider for this network. @@ -78,8 +84,10 @@ import { RelativeUrlPipe } from '@app/shared/pipes/relative-url/relative-url.pip > {{ p.name }} +
{{ p.provider_id }}
+
issuer {{ p.issuer }}, protocol {{ p.protocol_version }}
- directory entry expired {{ p.expires_at }} + every directory key expired {{ p.expires_at }}
@@ -89,6 +97,13 @@ import { RelativeUrlPipe } from '@app/shared/pipes/relative-url/relative-url.pip >{{ network }} + +
+ {{ k.kid }} ({{ k.algorithm }}): {{ k.validFrom }} to {{ k.validUntil ?? 'no end of validity' }} +
+
identity key {{ p.identity_key }}
+
valid from {{ p.effective_from }} until {{ p.expires_at ?? 'no end of validity' }}
+ {{ p.maximum_tx_vsize | number }} vB + + {{ m }} + none declared + {{ - isExpired(p) - ? 'EXPIRED / HEALTH UNKNOWN' - : 'Reported: ' + p.health_status - }}{{ healthLabel(p) }} @@ -149,6 +164,8 @@ export class PrivateSubmissionAcceleratorsComponent this.networkSub?.unsubscribe(); } public providers: AcceleratorProvider[] = []; + public directory: AcceleratorDirectoryRef | null = null; + public network: string | null = null; public loaded = false; public loadError: string | null = null; @@ -158,6 +175,8 @@ export class PrivateSubmissionAcceleratorsComponent this.networkSub = this.api.network$.subscribe(() => { this.request?.unsubscribe(); this.providers = []; + this.directory = null; + this.network = null; this.loaded = false; this.loadError = null; this.request = this.api.listAccelerators$().subscribe({ @@ -174,6 +193,8 @@ export class PrivateSubmissionAcceleratorsComponent return; } this.providers = res.providers; + this.directory = res.directory ?? null; + this.network = res.network ?? null; this.loaded = true; this.loadError = null; }, @@ -187,15 +208,28 @@ export class PrivateSubmissionAcceleratorsComponent }); } + /** A null expires_at is a key with no end of validity, not an expired one. */ public isExpired(provider: AcceleratorProvider): boolean { + if (provider.expires_at === null || provider.expires_at === undefined) { + return false; + } const expires = Date.parse(provider.expires_at); return !Number.isFinite(expires) || expires <= Date.now(); } + public healthLabel(provider: AcceleratorProvider): string { + if (this.isExpired(provider)) { + return 'EXPIRED / HEALTH UNKNOWN'; + } + return provider.health_status === 'unmeasured' + ? 'Not measured: no endpoint is probed' + : 'Reported: ' + provider.health_status; + } + public healthClass( health: AcceleratorProvider['health_status'] | null ): string { - if (health === null) { + if (health === null || health === 'unmeasured') { return 'bg-secondary'; } if (health === 'online') { diff --git a/frontend/src/app/universe/private-submission/private-submission-broadcast.component.spec.ts b/frontend/src/app/universe/private-submission/private-submission-broadcast.component.spec.ts index f8f377180ed..2c26e3db9cd 100644 --- a/frontend/src/app/universe/private-submission/private-submission-broadcast.component.spec.ts +++ b/frontend/src/app/universe/private-submission/private-submission-broadcast.component.spec.ts @@ -165,11 +165,11 @@ describe('private broadcast: the receipt shows the exact backend status', () => }); it('refresh and abort use the submission token and re-read the record', () => { - const get = vi.fn(() => of(record('aborted'))); - const abort = vi.fn(() => of({ success: true, status: 'aborted' })); + const get = vi.fn(() => of(record('cancelled'))); + const abort = vi.fn(() => of({ success: true, status: 'cancelled' })); const c = component({ getCapabilities$: () => of(capabilities), - submitPrivate$: () => of(record('queued')), + submitPrivate$: () => of({ ...record('queued'), owner_token: 'a'.repeat(64) }), getPrivateSubmission$: get, abortPrivate$: abort, }); @@ -178,9 +178,9 @@ describe('private broadcast: the receipt shows the exact backend status', () => c.submitPrivate(); expect(c.broadcastReceipt?.can_abort).toBe(true); c.abort(); - expect(abort).toHaveBeenCalledWith('tok'); - expect(get).toHaveBeenCalledWith('tok'); - expect(c.broadcastReceipt?.status).toBe('aborted'); + expect(abort).toHaveBeenCalledWith('tok', 'a'.repeat(64)); + expect(get).toHaveBeenCalledWith('tok', 'a'.repeat(64)); + expect(c.broadcastReceipt?.status).toBe('cancelled'); }); }); diff --git a/frontend/src/app/universe/private-submission/private-submission-broadcast.component.ts b/frontend/src/app/universe/private-submission/private-submission-broadcast.component.ts index f1750408c66..d053be928e5 100644 --- a/frontend/src/app/universe/private-submission/private-submission-broadcast.component.ts +++ b/frontend/src/app/universe/private-submission/private-submission-broadcast.component.ts @@ -11,6 +11,7 @@ import { import { PrivateBroadcastRecord, PrivateBroadcastStatus, + PRIVATE_BROADCAST_STATUSES, PrivateSubmissionApiService, SubmissionCapabilities, SubmissionMethod, @@ -97,6 +98,42 @@ export function statusPresentation(status: PrivateBroadcastStatus): { }; case 'failed': return { heading: 'Broadcast failed', badge: 'FAILED', tone: 'danger' }; + case 'relaying': + return { + heading: 'Relaying through the owned private endpoint', + badge: 'RELAYING', + tone: 'warning', + }; + case 'submitted': + return { + heading: 'Accepted by the owned node; awaiting confirmation', + badge: 'SUBMITTED', + tone: 'warning', + }; + case 'confirmed': + return { + heading: 'Confirmed on the owned node', + badge: 'CONFIRMED', + tone: 'success', + }; + case 'rejected': + return { + heading: 'Rejected by the owned node', + badge: 'REJECTED', + tone: 'danger', + }; + case 'cancelled': + return { + heading: 'Cancelled before relay', + badge: 'CANCELLED', + tone: 'secondary', + }; + case 'abort-too-late': + return { + heading: 'Relay had already started; nothing was undone', + badge: 'ABORT TOO LATE', + tone: 'warning', + }; default: return { heading: 'Unknown state', @@ -311,6 +348,7 @@ export class PrivateSubmissionBroadcastComponent implements OnInit, OnDestroy { method: SubmissionMethod; network: string; token?: string; + ownerToken?: string; }; constructor(private api: PrivateSubmissionApiService) {} ngOnInit(): void { @@ -361,7 +399,14 @@ export class PrivateSubmissionBroadcastComponent implements OnInit, OnDestroy { return statusPresentation(this.broadcastReceipt?.status ?? 'failed'); } isTerminal(s: PrivateBroadcastStatus): boolean { - return ['broadcast_completed', 'aborted', 'failed'].includes(s); + return [ + 'broadcast_completed', + 'aborted', + 'failed', + 'confirmed', + 'rejected', + 'cancelled', + ].includes(s); } submitPrivate(): void { if (!this.canSubmit || !this.method) return; @@ -398,7 +443,9 @@ export class PrivateSubmissionBroadcastComponent implements OnInit, OnDestroy { this.request?.unsubscribe(); this.refreshing = true; this.loadError = null; - this.request = this.api.getPrivateSubmission$(token).subscribe({ + this.request = this.api + .getPrivateSubmission$(token, this.expected.ownerToken) + .subscribe({ next: (r) => { this.broadcastReceipt = this.acceptRecord(r); this.refreshing = false; @@ -417,11 +464,18 @@ export class PrivateSubmissionBroadcastComponent implements OnInit, OnDestroy { this.request?.unsubscribe(); this.refreshing = false; this.aborting = true; - this.request = this.api.abortPrivate$(token).subscribe({ + this.request = this.api + .abortPrivate$(token, this.expected?.ownerToken) + .subscribe({ next: (r) => { this.aborting = false; - if (r?.success !== true || r.status !== 'aborted') { - this.loadError = 'The relay did not confirm an abort.'; + if (r?.success !== true || !['aborted', 'cancelled'].includes(r.status)) { + // abort-too-late is a truthful answer: the relay already started and + // nothing was reversed. Refresh status shows the record's real state. + this.loadError = + r?.status === 'abort-too-late' + ? 'The relay had already started; the abort changed nothing.' + : 'The relay did not confirm an abort.'; return; } this.refreshStatus(); @@ -449,13 +503,7 @@ export class PrivateSubmissionBroadcastComponent implements OnInit, OnDestroy { r.network !== e.network || this.api.network !== e.network || (e.token !== undefined && r.submission_token !== e.token) || - ![ - 'queued', - 'acknowledged', - 'aborted', - 'broadcast_completed', - 'failed', - ].includes(r.status) || + !PRIVATE_BROADCAST_STATUSES.includes(r.status) || typeof r.can_abort !== 'boolean' || !Number.isSafeInteger(r.retry_count) || r.retry_count < 0 || @@ -466,6 +514,10 @@ export class PrivateSubmissionBroadcastComponent implements OnInit, OnDestroy { return null; } e.token = r.submission_token; + if (typeof r.owner_token === 'string' && /^[0-9a-f]{64}$/i.test(r.owner_token)) { + // Kept in memory only for this page's readback and abort; never rendered. + e.ownerToken = r.owner_token; + } return r; } ngOnDestroy(): void { diff --git a/frontend/src/app/universe/private-submission/private-submission-receipts.component.spec.ts b/frontend/src/app/universe/private-submission/private-submission-receipts.component.spec.ts index 5f89baf225e..e7772c48982 100644 --- a/frontend/src/app/universe/private-submission/private-submission-receipts.component.spec.ts +++ b/frontend/src/app/universe/private-submission/private-submission-receipts.component.spec.ts @@ -1,12 +1,31 @@ import { describe, expect, it, vi } from 'vitest'; -import { of } from 'rxjs'; +import { BehaviorSubject, of, throwError } from 'rxjs'; import { PrivateSubmissionApiService } from './private-submission.service'; import { PrivateSubmissionReceiptsComponent } from './private-submission-receipts.component'; +import { PrivateSubmissionAcceleratorsComponent } from './private-submission-accelerators.component'; +import { validProvider } from './submission-validation'; const receipt = { receipt_id: 'receipt', provider_id: 'provider', txid: 'ab'.repeat(32), + provider_fee_sats: 1200, + submitted_at_utc: '2026-09-17T00:00:00.000Z', +}; + +const directory = { source: '/etc/universe/accelerators.json', revision: 'r7', loaded_at_utc: '2026-09-17T00:00:00.000Z' }; + +const verified = { + verified: true, + stage: 'verified', + errors: [], + provider_id: 'provider', + receipt_id: 'receipt', + key_id: 'k1', + algorithm: 'ed25519', + verified_at_utc: '2026-09-17T00:00:01.000Z', + directory, + scope: 'Signature over the declared terms only.', }; describe('receipt verification presentation', () => { @@ -14,14 +33,10 @@ describe('receipt verification presentation', () => { {}, { verified: false }, { verified: true, signature_valid: false }, - { - ...receipt, - verified: true, - signature_valid: true, - txid: 'cd'.repeat(32), - }, + { verified: true, stage: 'verified', errors: [] }, + { ...verified, receipt_id: 'other' }, ])( - 'cannot badge an unverified or mismatched HTTP 200 as signature valid', + 'cannot badge an unverified, incomplete or mismatched HTTP 200 as signature valid', (response) => { const api = { verifyReceipt$: vi.fn(() => of(response)), @@ -30,7 +45,8 @@ describe('receipt verification presentation', () => { component.receiptJson = JSON.stringify(receipt); component.verifyReceipt(); expect(component.verificationResult).toBeNull(); - expect(component.loadError).toContain('did not verify'); + expect(component.verdict).toBeNull(); + expect(component.loadError).toMatch(/did not verify|different receipt/); } ); @@ -44,4 +60,125 @@ describe('receipt verification presentation', () => { expect(component.verificationResult).toBeNull(); expect(component.loadError).toContain('authenticated provider trust'); }); + + it('renders a 200 verified result with the directory key and the pasted terms as inputs', () => { + const component = new PrivateSubmissionReceiptsComponent({ + verifyReceipt$: () => of(verified), + } as unknown as PrivateSubmissionApiService); + component.receiptJson = JSON.stringify(receipt); + component.verifyReceipt(); + expect(component.loadError).toBeNull(); + expect(component.verdict?.stage).toBe('verified'); + expect(component.verdict?.httpStatus).toBe(200); + expect(component.verificationResult?.key_id).toBe('k1'); + expect(component.requested?.['provider_fee_sats']).toBe(1200); + expect(component.stageTitle('verified')).toContain('owned directory'); + expect(component.stageExplanation(component.verdict!)).toContain('remain unverified'); + }); + + it.each([ + [400, 'invalid', ['provider_signature does not verify.']], + [400, 'unsupported', ['secp256k1-schnorr is not verifiable here.']], + [400, 'wrong-network', ['Receipt names mainnet; this backend serves signet.']], + [400, 'expired', ['expires_at_utc has passed.']], + [409, 'duplicate', []], + [503, 'unavailable-trust', ['No accelerator directory is configured.']], + ])('renders the %s %s stage distinctly, never as verified', (status, stage, errors) => { + const body: any = { verified: false, stage, errors, provider_id: 'provider', receipt_id: 'receipt' }; + if (stage === 'duplicate') body.replay = { first_verified_at_utc: '2026-09-16T00:00:00.000Z', seen_count: 2 }; + const component = new PrivateSubmissionReceiptsComponent({ + verifyReceipt$: () => throwError(() => ({ status, error: body })), + } as unknown as PrivateSubmissionApiService); + component.receiptJson = JSON.stringify(receipt); + component.verifyReceipt(); + expect(component.verificationResult).toBeNull(); + expect(component.loadError).toBeNull(); + expect(component.verdict?.stage).toBe(stage); + expect(component.verdict?.httpStatus).toBe(status); + expect(component.verdict?.result.errors).toEqual(errors); + expect(component.stageTitle(stage as any)).not.toContain('verified against'); + expect(component.stageClass(stage as any)).not.toBe('bg-success'); + if (stage === 'duplicate') expect(component.verdict?.result.replay?.seen_count).toBe(2); + }); + + it('reports a failure without a typed body as the transport failure', () => { + const component = new PrivateSubmissionReceiptsComponent({ + verifyReceipt$: () => throwError(() => ({ status: 0 })), + } as unknown as PrivateSubmissionApiService); + component.receiptJson = JSON.stringify(receipt); + component.verifyReceipt(); + expect(component.verdict).toBeNull(); + expect(component.loadError).toBeTruthy(); + }); + + it('clears a verdict on a network switch', () => { + const network$ = new BehaviorSubject('signet'); + const component = new PrivateSubmissionReceiptsComponent({ + network$, + verifyReceipt$: () => of(verified), + } as unknown as PrivateSubmissionApiService); + component.ngOnInit(); + component.receiptJson = JSON.stringify(receipt); + component.verifyReceipt(); + expect(component.verdict?.stage).toBe('verified'); + network$.next('testnet'); + expect(component.verdict).toBeNull(); + component.ngOnDestroy(); + }); +}); + +describe('accelerator directory presentation', () => { + const provider = { + provider_id: 'acc-1', + identity_key: 'aa'.repeat(32), + name: 'Owned Accelerator', + supported_networks: ['signet'], + submission_modes: ['configured_accelerator'], + minimum_fee_sats: 1000, + maximum_tx_vsize: 100000, + payment_methods: ['lightning'], + partner_mining_claims: [], + status_endpoint: null, + health_status: 'unmeasured', + effective_from: '2026-01-01T00:00:00.000Z', + expires_at: null, + provider_signature: null, + issuer: 'universe', + protocol_version: '1', + keys: [{ kid: 'k1', algorithm: 'ed25519', publicKey: 'aa'.repeat(32), validFrom: '2026-01-01T00:00:00.000Z', validUntil: null }], + directory, + }; + + it('accepts the owned directory view and renders identity, keys, validity and terms', () => { + expect(validProvider(provider)).toBe(true); + expect(validProvider({ ...provider, health_status: 'probed' })).toBe(false); + expect(validProvider({ ...provider, keys: [{ kid: 'k', algorithm: 'rsa', publicKey: 'x', validFrom: 'a', validUntil: null }] })).toBe(false); + const component = new PrivateSubmissionAcceleratorsComponent({ + network$: new BehaviorSubject('signet'), + listAccelerators$: () => of({ providers: [provider], network: 'signet', directory }), + } as unknown as PrivateSubmissionApiService); + component.ngOnInit(); + expect(component.loadError).toBeNull(); + expect(component.providers).toHaveLength(1); + expect(component.directory?.revision).toBe('r7'); + expect(component.network).toBe('signet'); + expect(component.isExpired(provider as any)).toBe(false); + expect(component.healthLabel(provider as any)).toContain('Not measured'); + expect(component.healthClass('unmeasured')).toBe('bg-secondary'); + component.ngOnDestroy(); + }); + + it('reports an unconfigured directory as the typed failure and an expired key window as expired', () => { + const component = new PrivateSubmissionAcceleratorsComponent({ + network$: new BehaviorSubject('signet'), + listAccelerators$: () => throwError(() => ({ status: 503, error: { stage: 'unavailable-registry', error: 'No accelerator directory is configured (reason: unconfigured)' } })), + } as unknown as PrivateSubmissionApiService); + component.ngOnInit(); + expect(component.providers).toEqual([]); + expect(component.loaded).toBe(false); + expect(component.loadError).toContain('unconfigured'); + expect(component.isExpired({ ...provider, expires_at: '2020-01-01T00:00:00.000Z' } as any)).toBe(true); + expect(component.healthLabel({ ...provider, expires_at: '2020-01-01T00:00:00.000Z' } as any)).toContain('EXPIRED'); + component.ngOnDestroy(); + }); }); diff --git a/frontend/src/app/universe/private-submission/private-submission-receipts.component.ts b/frontend/src/app/universe/private-submission/private-submission-receipts.component.ts index 16c3c6ddc04..e19ea0028d7 100644 --- a/frontend/src/app/universe/private-submission/private-submission-receipts.component.ts +++ b/frontend/src/app/universe/private-submission/private-submission-receipts.component.ts @@ -7,9 +7,25 @@ import { classifyLoadFailure, loadFailureMessage, } from '@app/shared/load-state'; -import { PrivateSubmissionApiService } from './private-submission.service'; +import { + PrivateSubmissionApiService, + RECEIPT_VERIFICATION_STAGES, + ReceiptVerificationResult, + ReceiptVerificationStage, +} from './private-submission.service'; import { RelativeUrlPipe } from '@app/shared/pipes/relative-url/relative-url.pipe'; +/** + * What the verifier answered: one of its typed stages with the result body, + * whether it came as a 200 (verified), 409 (duplicate), 503 + * (unavailable-trust) or 400 (every other stage). + */ +export interface ReceiptVerdict { + stage: ReceiptVerificationStage; + httpStatus: number | null; + result: ReceiptVerificationResult; +} + @Component({ selector: 'app-private-submission-receipts', standalone: true, @@ -27,8 +43,10 @@ import { RelativeUrlPipe } from '@app/shared/pipes/relative-url/relative-url.pip Accelerator Receipts & Cryptographic Proof Verification

- Request provider receipt signature checks. A signature alone does - not prove payment, mining coverage or acceleration. + Checks a provider receipt's signature against the key the owned + directory holds for that provider. A verified signature proves the + provider signed these terms; it does not prove payment, mining + coverage or acceleration.

{{ verifying - ? 'Checking configured receipt trust...' + ? 'Checking against the owned directory...' : 'Verify Cryptographic Receipt' }} @@ -72,48 +90,72 @@ import { RelativeUrlPipe } from '@app/shared/pipes/relative-url/relative-url.pip
-
Receipt response
- UNVERIFIED +
{{ stageTitle(v.stage) }}
+ {{ v.stage | uppercase }}
+

{{ stageExplanation(v) }}

+
    +
  • {{ e }}
    • +
+
Receipt ID
- {{ verificationResult.receipt_id }} + {{ v.result.receipt_id ?? requested?.receipt_id ?? 'unknown' }}
Provider
- {{ verificationResult.provider_id }} + {{ v.result.provider_id ?? requested?.provider_id ?? 'unknown' }}
Accelerated Txid
- {{ verificationResult.txid }} + {{ requested?.txid ?? 'unknown' }}
-
Declared provider fee
-
- {{ verificationResult.provider_fee_sats | number }} sats -
+ +
Signing key
+
{{ v.result.key_id }} ({{ v.result.algorithm }})
-
Timestamp
-
- {{ verificationResult.submitted_at_utc }} -
+
Verified at
+
{{ v.result.verified_at_utc }}
+ +
Directory
+
{{ v.result.directory?.source }} revision {{ v.result.directory?.revision }}, loaded {{ v.result.directory?.loaded_at_utc }}
+ +
Declared provider fee
+
+ {{ requested?.provider_fee_sats ?? 'unknown' }} sats (declared by the provider, not a payment record) +
+ +
Submitted at
+
+ {{ requested?.submitted_at_utc ?? 'unknown' }} +
+ + + +
First verified
+
{{ v.result.replay.first_verified_at_utc }}
+
Presented
+
{{ v.result.replay.seen_count }} times
+
+

{{ v.result.scope }}

- Paste receipt JSON payload to verify signature authenticity and - non-repudiation. + Paste a receipt JSON payload to check its provider signature + against the owned directory.

@@ -129,7 +171,8 @@ export class PrivateSubmissionReceiptsComponent implements OnInit, OnDestroy { } reset(): void { this.request?.unsubscribe(); - this.verificationResult = null; + this.verdict = null; + this.requested = null; this.loadError = null; this.verifying = false; } @@ -139,11 +182,52 @@ export class PrivateSubmissionReceiptsComponent implements OnInit, OnDestroy { } public receiptJson = ''; public verifying = false; - public verificationResult: any = null; + /** The verifier's typed answer, whichever HTTP status carried it. */ + public verdict: ReceiptVerdict | null = null; + /** The receipt as pasted; shown as the caller's input, never as the verdict. */ + public requested: Record | null = null; public loadError: string | null = null; + /** Kept for callers and tests: the result body only when it verified. */ + get verificationResult(): ReceiptVerificationResult | null { + return this.verdict?.stage === 'verified' ? this.verdict.result : null; + } + constructor(private api: PrivateSubmissionApiService) {} + stageTitle(stage: ReceiptVerificationStage): string { + switch (stage) { + case 'verified': return 'Signature verified against the owned directory'; + case 'duplicate': return 'Receipt already verified here (replay)'; + case 'unavailable-trust': return 'No trust root for this receipt'; + case 'wrong-network': return 'Receipt is for another network'; + case 'expired': return 'Receipt has expired'; + case 'unsupported': return 'Signature encoding not supported here'; + default: return 'Receipt invalid'; + } + } + + stageClass(stage: ReceiptVerificationStage): string { + return stage === 'verified' ? 'bg-success' : stage === 'duplicate' || stage === 'unavailable-trust' ? 'bg-warning text-dark' : 'bg-danger'; + } + + stageExplanation(verdict: ReceiptVerdict): string { + switch (verdict.stage) { + case 'verified': return 'The provider key named by key_id, valid at the receipt\'s issue time in the owned directory, signed exactly these terms. Payment, mining coverage and acceleration remain unverified.'; + case 'duplicate': return 'This exact receipt was verified here before; a replay is reported, not re-verified (HTTP 409).'; + case 'unavailable-trust': return 'The owned directory is absent, or the provider or key is not in it or not valid at issue. Nothing could be checked (HTTP 503).'; + case 'wrong-network': return 'The receipt names a network other than the selected one.'; + case 'expired': return 'The receipt\'s expires_at_utc has passed.'; + case 'unsupported': return 'The receipt uses an encoding this deployment cannot verify.'; + default: return 'The structure, completeness or signature failed; an altered field fails here.'; + } + } + + private isResult(body: unknown): body is ReceiptVerificationResult { + const r = body as ReceiptVerificationResult; + return !!r && typeof r.verified === 'boolean' && RECEIPT_VERIFICATION_STAGES.includes(r.stage) && Array.isArray(r.errors); + } + public verifyReceipt(): void { this.reset(); if (!this.receiptJson) return; @@ -158,30 +242,48 @@ export class PrivateSubmissionReceiptsComponent implements OnInit, OnDestroy { // Unreadable input is the reader's error to see. Sending an empty body // instead, as the revision this replaces did, verified nothing and then // rendered whatever came back as the verdict on their receipt. - this.verificationResult = null; + this.verdict = null; this.loadError = $localize`:@@submission.receipt.malformed:This is not readable JSON.`; return; } if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { - this.verificationResult = null; + this.verdict = null; this.loadError = $localize`:@@submission.receipt.object:Enter a receipt JSON object.`; return; } const requested = parsed as Record; this.verifying = true; this.loadError = null; - this.verificationResult = null; + this.verdict = null; this.request = this.api.verifyReceipt$(parsed).subscribe({ next: (res) => { - this.verificationResult = null; - this.loadError = - 'The service did not verify the receipt against an implemented authenticated provider trust contract. Payment and acceleration remain unverified.'; this.verifying = false; + // A 200 is only verified when the body says so with the verified + // stage; anything else on a 200 is not a verdict this page can show. + if (!this.isResult(res) || res.verified !== true || res.stage !== 'verified' || !res.key_id || !res.verified_at_utc) { + this.verdict = null; + this.loadError = + 'The service did not verify the receipt against an implemented authenticated provider trust contract. Payment and acceleration remain unverified.'; + return; + } + if (res.receipt_id !== undefined && res.receipt_id !== requested['receipt_id']) { + this.verdict = null; + this.loadError = 'The service answered for a different receipt than the one submitted; nothing is verified.'; + return; + } + this.requested = requested; + this.verdict = { stage: 'verified', httpStatus: 200, result: res }; }, error: (err) => { - this.verificationResult = null; - this.loadError = loadFailureMessage(classifyLoadFailure(err)); this.verifying = false; + const body = err?.error; + if (this.isResult(body) && body.verified === false && body.stage !== 'verified') { + this.requested = requested; + this.verdict = { stage: body.stage, httpStatus: typeof err?.status === 'number' ? err.status : null, result: body }; + return; + } + this.verdict = null; + this.loadError = (typeof body?.error === 'string' ? body.error : null) || loadFailureMessage(classifyLoadFailure(err)); }, }); } diff --git a/frontend/src/app/universe/private-submission/private-submission.models.ts b/frontend/src/app/universe/private-submission/private-submission.models.ts index 33940f55d95..74b3f83c682 100644 --- a/frontend/src/app/universe/private-submission/private-submission.models.ts +++ b/frontend/src/app/universe/private-submission/private-submission.models.ts @@ -44,6 +44,33 @@ export interface SubmissionDiagnosisResult { acceleration_recommended: boolean; privacy_advisory: string; available_methods: SubmissionMethod[]; + /** Owned mempool package facts for the entry. */ + package?: { + effective_feerate_sats_vb: number; + ancestor_count: number; + descendant_count: number; + ancestor_fee_sats: number; + ancestor_weight: number; + cpfp_boosted: boolean; + cpfp_checked: boolean; + best_descendant_txid: string | null; + projected_block_index: number | null; + }; + policy?: { + source: string; + mempool_min_fee_sats_vb: number | null; + min_relay_fee_sats_vb: number | null; + mempool_loaded: boolean | null; + }; + replacement?: { + signals_rbf: boolean; + replaces_txids: string[]; + replaced_by_txid: string | null; + is_replacement: boolean; + }; + observed_at_utc?: string; + first_seen_utc?: string | null; + scope?: string; } export interface PrivateBroadcastRecord { @@ -53,12 +80,61 @@ export interface PrivateBroadcastRecord { network: string; queued_at_utc: string; status: - 'queued' | 'acknowledged' | 'aborted' | 'broadcast_completed' | 'failed'; + | 'queued' + | 'acknowledged' + | 'aborted' + | 'broadcast_completed' + | 'failed' + | 'relaying' + | 'submitted' + | 'confirmed' + | 'rejected' + | 'cancelled' + | 'abort-too-late'; retry_count: number; can_abort: boolean; last_error?: string; + /** Returned exactly once, by the POST that created the record; needed for readback and abort. */ + owner_token?: string; + duplicate?: boolean; + relay_endpoint_id?: string; + relayed_at_utc?: string; + confirmed_block_height?: number; + updated_at_utc?: string; } +export const PRIVATE_BROADCAST_STATUSES: readonly PrivateBroadcastRecord['status'][] = [ + 'queued', + 'acknowledged', + 'aborted', + 'broadcast_completed', + 'failed', + 'relaying', + 'submitted', + 'confirmed', + 'rejected', + 'cancelled', + 'abort-too-late', +]; + +/** Header carrying the owner token on private submission readback and abort. */ +export const PRIVATE_SUBMISSION_OWNER_TOKEN_HEADER = 'X-Submission-Owner-Token'; + +export interface AcceleratorProviderKey { + kid: string; + algorithm: 'ed25519' | 'secp256k1-schnorr'; + publicKey: string; + validFrom: string; + validUntil: string | null; +} + +export interface AcceleratorDirectoryRef { + source: string; + revision: string; + loaded_at_utc: string; +} + +/** A provider from the owned directory file, which is the trust root. */ export interface AcceleratorProvider { provider_id: string; identity_key: string; @@ -69,11 +145,18 @@ export interface AcceleratorProvider { maximum_tx_vsize: number; payment_methods: string[]; partner_mining_claims: string[]; - status_endpoint: string; - health_status: 'online' | 'degraded' | 'offline'; + /** Null: the owned directory names no health endpoint and none is probed. */ + status_endpoint: string | null; + health_status: 'online' | 'degraded' | 'offline' | 'unmeasured'; effective_from: string; - expires_at: string; - provider_signature: string; + /** Null when a directory key has no end of validity. */ + expires_at: string | null; + /** Null: entries carry no separate signature; the directory file is the trust root. */ + provider_signature: string | null; + issuer?: string; + protocol_version?: string; + keys?: AcceleratorProviderKey[]; + directory?: AcceleratorDirectoryRef; } export interface AcceleratorReceipt { @@ -82,6 +165,7 @@ export interface AcceleratorReceipt { receipt_id: string; txid: string; wtxid?: string; + network: string; submitted_at_utc: string; expires_at_utc: string; target_feerate_sats_vb: number; @@ -89,9 +173,50 @@ export interface AcceleratorReceipt { claimed_mining_coverage_pct: number; claimed_partner_pools: string[]; status: 'active' | 'included' | 'expired' | 'refunded'; + key_id: string; provider_signature: string; } +/** + * invalid: structure, completeness or signature failed; unsupported: an + * encoding this deployment cannot verify; unavailable-trust: no directory, or + * the provider or key is not in it or not valid at issue; wrong-network; + * expired; duplicate: a replay of a receipt already verified here; verified. + * HTTP: 200 verified, 409 duplicate, 503 unavailable-trust, 400 otherwise. + */ +export type ReceiptVerificationStage = + | 'invalid' + | 'unsupported' + | 'unavailable-trust' + | 'wrong-network' + | 'expired' + | 'duplicate' + | 'verified'; + +export const RECEIPT_VERIFICATION_STAGES: readonly ReceiptVerificationStage[] = [ + 'invalid', + 'unsupported', + 'unavailable-trust', + 'wrong-network', + 'expired', + 'duplicate', + 'verified', +]; + +export interface ReceiptVerificationResult { + verified: boolean; + stage: ReceiptVerificationStage; + errors: string[]; + provider_id?: string; + receipt_id?: string; + key_id?: string; + algorithm?: string; + verified_at_utc?: string; + directory?: AcceleratorDirectoryRef; + replay?: { first_verified_at_utc: string; seen_count: number }; + scope?: string; +} + export interface TransactionOrderingEvidence { txid: string; block_hash: string; @@ -108,12 +233,65 @@ export interface TransactionOrderingEvidence { is_ordering_sensitive: boolean; protocol_impact_description?: string; confidence_rating: 'high' | 'medium' | 'low'; + template_coverage?: 'template-observed' | 'no-template-observed'; + template_ids?: string[]; + template_position?: number | null; + sensor_observer_id?: string | null; } -export interface SubmissionOverview { - capabilities: SubmissionCapabilities; - active_accelerator_providers: AcceleratorProvider[]; - recent_ordering_findings_count: number; - total_private_broadcasts_24h: number; - average_queue_duration_seconds: number; +export interface BlockOrderingEvidence { + block_hash: string; + height: number; + mined_timestamp_utc: string; + network: string; + template_coverage: 'template-observed' | 'no-template-observed'; + templates_compared: string[]; + sensor: { observer_id: string | null; observers: 1 }; + transactions: TransactionOrderingEvidence[]; + truncated: boolean; + scope: string; +} + +/** The findings envelope; no-blocks-observed means this observer has recorded no block yet. */ +export interface OrderingFindings { + network: string; + state: 'observed' | 'no-blocks-observed'; + findings: TransactionOrderingEvidence[]; + coverage: Array<{ block_hash: string; height: number; template_coverage: 'template-observed' | 'no-template-observed'; transactions: number; findings: number }>; + retention: { blocks: number; retained_blocks: number }; + scope: string; } + +export interface PrivateRelayEndpointFact { + id: string; + transport: 'tor' | 'i2p'; + submit_host: string; + proxy_reachable: boolean | null; +} + +/** What GET submission/overview answers: the owned relay, its queue and its worker. */ +export interface PrivateRelayOverview { + network: string; + relay: { + configured: boolean; + reason: string | null; + endpoints: PrivateRelayEndpointFact[]; + rejected_endpoints: { id: string; reason: string }[]; + }; + queue: { + queued: number; + relaying: number; + submitted: number; + confirmed: number; + rejected: number; + cancelled: number; + }; + worker: { + running: boolean; + last_tick_at: string | null; + last_error: string | null; + last_relay: { endpoint_id: string; transport: 'tor' | 'i2p'; outcome: string; at: string } | null; + }; +} + +export type SubmissionOverview = PrivateRelayOverview; diff --git a/frontend/src/app/universe/private-submission/private-submission.service.ts b/frontend/src/app/universe/private-submission/private-submission.service.ts index e9760c4f828..de52d1da8b6 100644 --- a/frontend/src/app/universe/private-submission/private-submission.service.ts +++ b/frontend/src/app/universe/private-submission/private-submission.service.ts @@ -6,10 +6,21 @@ import { StateService } from '@app/services/state.service'; import { SubmissionOverview, SubmissionCapabilities, + SubmissionDiagnosisResult, SubmissionMethod, PrivateBroadcastRecord, AcceleratorProvider, + AcceleratorDirectoryRef, + BlockOrderingEvidence, + OrderingFindings, + ReceiptVerificationResult, + TransactionOrderingEvidence, + PRIVATE_SUBMISSION_OWNER_TOKEN_HEADER, } from './private-submission.models'; + +function ownerHeaders(ownerToken?: string): Record { + return ownerToken ? { [PRIVATE_SUBMISSION_OWNER_TOKEN_HEADER]: ownerToken } : {}; +} export * from './private-submission.models'; export type PrivateBroadcastStatus = PrivateBroadcastRecord['status']; /** @@ -65,8 +76,9 @@ export class PrivateSubmissionApiService { ); } - public diagnose$(txidOrHex: string): Observable { - return this.http.post( + /** The transaction's own mempool facts, or a 404 transaction-not-in-mempool. */ + public diagnose$(txidOrHex: string): Observable { + return this.http.post( `${this.baseUrl}/submission/diagnose`, /^[0-9a-f]{64}$/i.test(txidOrHex) ? { txid: txidOrHex } @@ -90,26 +102,35 @@ export class PrivateSubmissionApiService { ); } + /** + * Readback and abort are owner-authenticated: the backend returns the owner + * token once, on creation, and requires it back in a header. The token is + * never placed in the URL. + */ public getPrivateSubmission$( - token: string + token: string, + ownerToken?: string ): Observable { return this.http.get( - `${this.baseUrl}/submission/private/${encodeURIComponent(token)}` + `${this.baseUrl}/submission/private/${encodeURIComponent(token)}`, + { headers: ownerHeaders(ownerToken) } ); } public abortPrivate$( - token: string + token: string, + ownerToken?: string ): Observable<{ success: boolean; status: string }> { return this.http.post<{ success: boolean; status: string }>( `${this.baseUrl}/submission/private/${encodeURIComponent(token)}/abort`, - {} + {}, + { headers: ownerHeaders(ownerToken) } ); } - /** The service contract is an envelope, { providers: [...] }, never a bare array. */ - public listAccelerators$(): Observable<{ providers: AcceleratorProvider[] }> { - return this.http.get<{ providers: AcceleratorProvider[] }>( + /** The service contract is an envelope, { providers, network, directory }, never a bare array. */ + public listAccelerators$(): Observable<{ providers: AcceleratorProvider[]; network?: string; directory?: AcceleratorDirectoryRef }> { + return this.http.get<{ providers: AcceleratorProvider[]; network?: string; directory?: AcceleratorDirectoryRef }>( `${this.baseUrl}/accelerators/providers` ); } @@ -120,31 +141,39 @@ export class PrivateSubmissionApiService { ); } - public verifyReceipt$(receiptPayload: any): Observable { - return this.http.post( + /** + * 200 only when the signature verified against the owned directory key; + * 409 duplicate, 503 unavailable-trust and 400 for the other stages carry + * the same result body in the error. + */ + public verifyReceipt$(receiptPayload: any): Observable { + return this.http.post( `${this.baseUrl}/accelerators/receipts/verify`, receiptPayload ); } - public getTxOrdering$(txid: string): Observable { - return this.http.get( + /** 404 transaction-not-observed when no retained block holds the transaction. */ + public getTxOrdering$(txid: string): Observable { + return this.http.get( `${this.baseUrl}/ordering/transactions/${encodeURIComponent(txid)}` ); } - public getBlockOrdering$(blockHash: string): Observable { - return this.http.get( + /** 404 block-not-observed when this backend did not record the block. */ + public getBlockOrdering$(blockHash: string): Observable { + return this.http.get( `${this.baseUrl}/ordering/blocks/${encodeURIComponent(blockHash)}` ); } - public listOrderingFindings$(): Observable { - return this.http.get(`${this.baseUrl}/ordering/findings`).pipe( + /** The findings envelope with its state, coverage and retention. */ + public listOrderingFindings$(): Observable { + return this.http.get(`${this.baseUrl}/ordering/findings`).pipe( map((x) => { - if (!x || !Array.isArray(x.findings)) + if (!x || !Array.isArray(x.findings) || !['observed', 'no-blocks-observed'].includes(x.state)) throw Error('Malformed findings envelope.'); - return x.findings; + return x; }) ); } diff --git a/frontend/src/app/universe/private-submission/submission-evidence.component.html b/frontend/src/app/universe/private-submission/submission-evidence.component.html index 08c0e5d99cb..96cbf0b1eca 100644 --- a/frontend/src/app/universe/private-submission/submission-evidence.component.html +++ b/frontend/src/app/universe/private-submission/submission-evidence.component.html @@ -11,38 +11,124 @@

{{ title }}

revenue.

Loading source evidence…

-
-

Reported source fields

-
+ +

Private relay

+

+ {{ data.relay.configured ? 'configured' : 'not configured' }} + {{ data.relay.reason }} +

+ + + + + + + + + + +
EndpointTransportSubmit hostProxy reachable
{{ e.id }}{{ e.transport }}{{ e.submit_host }}{{ e.proxy_reachable === null ? 'not probed yet' : e.proxy_reachable ? 'yes' : 'no' }}
+

+ Rejected endpoint configuration: + {{ r.id }}: {{ r.reason }}; +

+

Queue

+
+ +
{{ f.key }}
+
{{ display(f.value) }}
+ +
+

Relay worker

+
+
running
{{ data.worker.running ? 'yes' : 'no' }}
+
last tick
{{ data.worker.last_tick_at ?? 'never' }}
+
last error
{{ data.worker.last_error ?? 'none' }}
+
last relay
+
+ {{ data.worker.last_relay.at }} {{ data.worker.last_relay.endpoint_id }} ({{ data.worker.last_relay.transport }}): {{ data.worker.last_relay.outcome }} + none +
+
+ + +

Reported source fields

+

+ {{ display(data.evidence_state) }}: {{ orderingLabel(data.evidence_state) }} + No collected template covers this block. + Observer: {{ data.sensor_observer_id }}. +

+
{{ f.key }}
{{ display(f.value) }}
-
+ + +

Block {{ data.height }} as this observer recorded it

+

+ Mined {{ data.mined_timestamp_utc }} on {{ data.network }}. Template coverage: {{ data.template_coverage }} ({{ data.templates_compared.join(', ') }}). Observer: {{ data.sensor?.observer_id ?? 'unknown' }} ({{ data.sensor?.observers ?? 1 }} observer). + The transaction list is truncated. +

+

No transaction of this block was recorded.

+
+ {{ row.txid }}

- Evidence: {{ display(row.evidence_state) }} · Confidence: - {{ display(row.confidence_rating) }} + Position {{ display(row.block_position) }} · {{ display(row.evidence_state) }}: {{ orderingLabel(row.evidence_state) }} · Confidence: {{ display(row.confidence_rating) }}

+
+

{{ data.scope }}

+ + +

+ No blocks observed. This backend has not recorded a block yet, so there is no ordering evidence. Nothing is shown in its place. +

+

- Observed block position: {{ display(row.block_position) }} · Fee: - {{ display(row.fee_sats_vb) }} sat/vB + {{ data.retention.retained_blocks }} of the last {{ data.retention.blocks }} blocks retained on {{ data.network }}.

- 
{{ row | json }}
- -

- The source returned no findings. -

+ + + + + + + + + + + +
BlockHeightTemplate coverageTransactionsFindings
{{ c.block_hash.slice(0, 16) }}…{{ c.height }}{{ c.template_coverage }}{{ c.transactions }}{{ c.findings }}
+
+ {{ row.txid }} +

+ Evidence: {{ display(row.evidence_state) }} ({{ orderingLabel(row.evidence_state) }}) · Confidence: + {{ display(row.confidence_rating) }} +

+

+ Observed block position: {{ display(row.block_position) }} · Fee: + {{ display(row.fee_sats_vb) }} sat/vB + · Template coverage: {{ row.template_coverage }} +

+ 
{{ row | json }}
+
+

+ The retained blocks produced no ordering finding. +

+ +

{{ data.scope }}

+

Provider declarations and timestamps are not independent payment, reach or current health proof. @@ -50,6 +136,7 @@

Reported source fields

Transaction diagnosis

+

The transaction's own entry in the owned mempool: package feerate, ancestors and descendants, replacement signals and the node's policy limits. A transaction this mempool does not hold has no facts to report.