From e087153f8c9061a3d641a8973a4bd9726fc80d63 Mon Sep 17 00:00:00 2001 From: "Bill Gates (ops agent)" Date: Fri, 18 Sep 2026 17:57:46 +0000 Subject: [PATCH 1/2] feat(x402): count payment outcomes and settled USDC 402 responses are already countable via http_request_duration_seconds_count{status_code="402"}, but nothing records what happens after one. An operator cannot currently tell a paywall nobody pays from one whose payments are failing to settle. That is not hypothetical. config.ts already warns that a mainnet deployment with incomplete CDP credentials silently falls back to X_402_USDC_FACILITATOR_URL, where the commonly configured facilitators (facilitator.x402.rs, x402.org) support testnets only -- "EVERY payment will fail verification and the gateway will earn nothing while still returning 402s". Today that state is invisible in metrics; it looks exactly like nobody paying. Adds two counters, incremented in processPaymentAndTopUp so both the browser paywall redirect and the data path are covered: - x402_payment_total{outcome,target} -- outcome is the stage that ended the attempt: no_payment_header, invalid_target, missing_host, verify_failed, unsupported_processor, unsupported_payload, settle_failed, topup_failed, error, settled. - x402_payment_settled_usdc_total{target} -- USDC actually settled, converted from atomic units (6 decimals), guarded against a non-numeric amount poisoning the counter. Documented in docs/x402-and-rate-limiting.md with example queries, including settled-per-402 conversion. Tests: a settled payment counts once and records its amount; settlement failure and verification failure are counted distinctly; a request with no payment header is counted; nothing is added to the USDC counter unless a payment settled. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012NWDKc9pST69qTEha4AGaB --- docs/x402-and-rate-limiting.md | 38 +++++ src/metrics.ts | 25 +++ src/payments/payment-processor-utils.test.ts | 166 +++++++++++++++++++ src/payments/payment-processor-utils.ts | 25 +++ 4 files changed, 254 insertions(+) create mode 100644 src/payments/payment-processor-utils.test.ts diff --git a/docs/x402-and-rate-limiting.md b/docs/x402-and-rate-limiting.md index c87f09680..23c24cf37 100644 --- a/docs/x402-and-rate-limiting.md +++ b/docs/x402-and-rate-limiting.md @@ -12,6 +12,7 @@ traffic management and content monetization. - [x402 Payment Protocol Deep Dive](#x402-payment-protocol-deep-dive) - [Integration Topics](#integration-topics) - [Reference](#reference) +- [Metrics](#metrics) - [Troubleshooting](#troubleshooting) - [Examples](#examples) @@ -2033,6 +2034,43 @@ curl -X POST \ -d '{"tokens": 1000000, "tokenType": "paid"}' ``` +## Metrics + +Payment attempts are counted at `/ar-io/__gateway_metrics`: + +| Metric | Labels | Meaning | +| --- | --- | --- | +| `x402_payment_total` | `outcome`, `target` | Payment attempts by the stage that ended them | +| `x402_payment_settled_usdc_total` | `target` | USDC actually settled (atomic units converted; USDC has 6 decimals) | + +`outcome` values: `no_payment_header`, `invalid_target`, `missing_host`, +`verify_failed`, `unsupported_processor`, `unsupported_payload`, +`settle_failed`, `topup_failed`, `error`, `settled`. + +402 responses themselves are already countable without these, via +`http_request_duration_seconds_count{status_code="402"}`. The counters above +cover what happens *after* a 402 — whether anyone pays, and whether their +payments settle. + +That distinction matters in one specific failure: a mainnet deployment with +incomplete CDP credentials silently falls back to `X_402_USDC_FACILITATOR_URL`, +and the commonly configured facilitators there support testnets only. Every +payment then fails verification while the gateway keeps advertising x402 and +serving 402s. Without these metrics that is indistinguishable from a paywall +nobody has paid yet: + +```promql +# paid attempts that never settled +sum by (outcome) (rate(x402_payment_total{outcome!="settled"}[1h])) + +# revenue actually settled +sum(increase(x402_payment_settled_usdc_total[24h])) + +# conversion: settled payments per 402 served +sum(increase(x402_payment_total{outcome="settled"}[24h])) + / sum(increase(http_request_duration_seconds_count{status_code="402"}[24h])) +``` + ## Troubleshooting ### Rate Limiter Issues diff --git a/src/metrics.ts b/src/metrics.ts index 4f28e414a..875b8fbd8 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -1571,6 +1571,31 @@ export const rateLimitTokensConsumedTotal = new promClient.Counter({ labelNames: ['bucket_type', 'token_type', 'domain'], }); +/** + * x402 payment funnel. 402 responses are already countable via + * http_request_duration_seconds_count{status_code="402"}; what was missing is + * everything after one: whether a payment verified, settled, and topped up a + * bucket. Without it an operator cannot tell a paywall nobody pays from one + * whose settlements are failing — which matters because a mainnet deployment + * with incomplete CDP credentials silently falls back to a facilitator that + * cannot settle, and keeps serving 402s while earning nothing. + * + * `outcome` is the stage that ended the attempt: no_payment_header, + * invalid_target, missing_host, verify_failed, unsupported_processor, + * unsupported_payload, settle_failed, topup_failed, or settled. + */ +export const x402PaymentCounter = new promClient.Counter({ + name: 'x402_payment_total', + help: 'x402 payment attempts by outcome and top-up target', + labelNames: ['outcome', 'target'], +}); + +export const x402PaymentSettledUsdcCounter = new promClient.Counter({ + name: 'x402_payment_settled_usdc_total', + help: 'Total USDC settled through x402 (converted from atomic units; USDC has 6 decimals)', + labelNames: ['target'], +}); + // // Root TX Index metrics // diff --git a/src/payments/payment-processor-utils.test.ts b/src/payments/payment-processor-utils.test.ts new file mode 100644 index 000000000..ede625346 --- /dev/null +++ b/src/payments/payment-processor-utils.test.ts @@ -0,0 +1,166 @@ +/** + * AR.IO Gateway + * Copyright (C) 2022-2025 Permanent Data Solutions, Inc. All Rights Reserved. + * + * SPDX-License-Identifier: AGPL-3.0-or-later + */ +import assert from 'node:assert'; +import { afterEach, describe, it, mock } from 'node:test'; +import { Request } from 'express'; + +import { createTestLogger } from '../../test/test-logger.js'; +import * as metrics from '../metrics.js'; +import { RateLimiter } from '../limiter/types.js'; +import { processPaymentAndTopUp } from './payment-processor-utils.js'; +import { X402UsdcProcessor } from './x402-usdc-processor.js'; + +const log = createTestLogger({ suite: 'processPaymentAndTopUp' }); + +const req = (headers: Record = { host: 'example.com' }) => { + const mockRequest = { + method: 'GET', + originalUrl: '/ar-io/x402/test', + protocol: 'https', + headers, + header: (name: string) => headers[name.toLowerCase()], + }; + return mockRequest as unknown as Request; +}; + +const rateLimiter = { + topOffPaidTokens: async () => undefined, + topOffPaidTokensForResource: async () => undefined, +} as unknown as RateLimiter; + +// A processor that passes the `instanceof X402UsdcProcessor` checks while its +// network calls are stubbed. Payment amounts are USDC atomic units (6 decimals). +function makeProcessor(overrides: Record = {}) { + const processor = new X402UsdcProcessor({ + walletAddress: '0x1234567890123456789012345678901234567890', + network: 'base', + perBytePrice: 0.000001, + minPrice: 0.001, + maxPrice: 1.0, + facilitatorUrl: 'https://facilitator.example.com', + settleTimeoutMs: 5000, + version: 1, + log, + } as any); + Object.assign(processor, { + extractPayment: () => ({ + network: 'base', + scheme: 'exact', + payload: { authorization: { value: '250000' } }, // $0.25 + }), + calculateRequirements: () => ({}) as any, + verifyPayment: async () => ({ isValid: true }), + settlePayment: async () => ({ success: true, responseHeader: 'header' }), + paymentToContentSize: () => 1000, + paymentToTokens: () => 10, + ...overrides, + }); + return processor; +} + +async function outcomeCount(outcome: string, target = 'ip') { + const { values } = await metrics.x402PaymentCounter.get(); + return ( + values.find( + (v: any) => v.labels.outcome === outcome && v.labels.target === target, + )?.value ?? 0 + ); +} + +async function settledUsdc(target = 'ip') { + const { values } = await metrics.x402PaymentSettledUsdcCounter.get(); + return values.find((v: any) => v.labels.target === target)?.value ?? 0; +} + +describe('processPaymentAndTopUp metrics', () => { + afterEach(() => mock.restoreAll()); + + it('counts a settled payment and its USDC amount', async () => { + const before = await outcomeCount('settled'); + const beforeUsdc = await settledUsdc(); + + const result = await processPaymentAndTopUp( + rateLimiter, + makeProcessor() as any, + req(), + log, + { type: 'ip' }, + ); + + assert.equal(result.success, true); + assert.equal(await outcomeCount('settled'), before + 1); + // 250000 atomic units = $0.25 + assert.ok(Math.abs((await settledUsdc()) - (beforeUsdc + 0.25)) < 1e-9); + }); + + // The case that motivated this: a mainnet deployment whose facilitator cannot + // settle keeps serving 402s and earning nothing. 402 counts alone look + // identical to a healthy paywall nobody has paid yet. + it('counts a settlement failure separately from a verification failure', async () => { + const beforeSettle = await outcomeCount('settle_failed'); + const beforeVerify = await outcomeCount('verify_failed'); + + await processPaymentAndTopUp( + rateLimiter, + makeProcessor({ + settlePayment: async () => ({ + success: false, + errorReason: 'facilitator does not support this network', + }), + }) as any, + req(), + log, + { type: 'ip' }, + ); + assert.equal(await outcomeCount('settle_failed'), beforeSettle + 1); + + await processPaymentAndTopUp( + rateLimiter, + makeProcessor({ + verifyPayment: async () => ({ + isValid: false, + invalidReason: 'insufficient_funds', + }), + }) as any, + req(), + log, + { type: 'ip' }, + ); + assert.equal(await outcomeCount('verify_failed'), beforeVerify + 1); + }); + + it('counts a request that carried no payment header', async () => { + const before = await outcomeCount('no_payment_header'); + + const result = await processPaymentAndTopUp( + rateLimiter, + makeProcessor({ extractPayment: () => undefined }) as any, + req(), + log, + { type: 'ip' }, + ); + + assert.equal(result.success, false); + assert.equal(await outcomeCount('no_payment_header'), before + 1); + }); + + it('does not record settled USDC when nothing settled', async () => { + const beforeUsdc = await settledUsdc(); + + await processPaymentAndTopUp( + rateLimiter, + makeProcessor({ + settlePayment: async () => ({ success: false, errorReason: 'nope' }), + }) as any, + req(), + log, + { type: 'ip' }, + ); + + assert.equal(await settledUsdc(), beforeUsdc); + }); +}); diff --git a/src/payments/payment-processor-utils.ts b/src/payments/payment-processor-utils.ts index ff0dea88f..425ed7ee4 100644 --- a/src/payments/payment-processor-utils.ts +++ b/src/payments/payment-processor-utils.ts @@ -10,6 +10,7 @@ import { PaymentRequirements } from 'x402/types'; import * as config from '../config.js'; import { RateLimiter } from '../limiter/types.js'; import { PaymentProcessor } from './types.js'; +import * as metrics from '../metrics.js'; import { X402UsdcProcessor } from './x402-usdc-processor.js'; export interface PaymentTopUpTarget { @@ -63,11 +64,16 @@ export async function processPaymentAndTopUp( target: PaymentTopUpTarget, contentSizeOverride?: number, ): Promise { + // Records where an attempt ended, so the funnel from 402 -> settled payment is + // visible. Every early return below goes through this. + const countOutcome = (outcome: string) => + metrics.x402PaymentCounter.inc({ outcome, target: target.type }); try { // Extract payment from headers const payment = paymentProcessor.extractPayment(req); if (payment === undefined) { + countOutcome('no_payment_header'); return { success: false, error: 'No payment found in headers', @@ -81,6 +87,7 @@ export async function processPaymentAndTopUp( target.host === undefined || target.path === undefined ) { + countOutcome('invalid_target'); return { success: false, error: 'Resource top-up requires method, host, and path', @@ -107,6 +114,7 @@ export async function processPaymentAndTopUp( // Validate host header is present const host = req.headers.host; if (host === undefined || host === '') { + countOutcome('missing_host'); return { success: false, error: 'Missing Host header - required for payment processing', @@ -131,6 +139,7 @@ export async function processPaymentAndTopUp( ); if (!verifyResult.isValid) { + countOutcome('verify_failed'); return { success: false, error: `Payment verification failed: ${verifyResult.invalidReason}`, @@ -144,6 +153,7 @@ export async function processPaymentAndTopUp( network: payment.network, scheme: payment.scheme, }); + countOutcome('unsupported_processor'); return { success: false, error: `Unsupported payment processor type: ${paymentProcessor.constructor.name}`, @@ -159,6 +169,7 @@ export async function processPaymentAndTopUp( hasAuthorization: 'authorization' in payment.payload, hasTransaction: 'transaction' in payment.payload, }); + countOutcome('unsupported_payload'); return { success: false, error: @@ -174,6 +185,7 @@ export async function processPaymentAndTopUp( ); if (!settlementResult.success) { + countOutcome('settle_failed'); return { success: false, error: `Payment settlement failed: ${settlementResult.errorReason}`, @@ -216,6 +228,7 @@ export async function processPaymentAndTopUp( tokensAdded = tokens * multiplierApplied; } else { log.error('Invalid target type', { target }); + countOutcome('topup_failed'); return { success: false, error: `Invalid target type: ${target.type}`, @@ -232,6 +245,17 @@ export async function processPaymentAndTopUp( multiplierApplied, }); + countOutcome('settled'); + // USDC has 6 decimals; parseInt of a non-numeric value would poison the + // counter, so only record a finite result. + const settledUsdc = parseInt(paymentAmount, 10) / 1_000_000; + if (Number.isFinite(settledUsdc)) { + metrics.x402PaymentSettledUsdcCounter.inc( + { target: target.type }, + settledUsdc, + ); + } + return { success: true, tokensAdded, @@ -240,6 +264,7 @@ export async function processPaymentAndTopUp( responseHeader: settlementResult.responseHeader, }; } catch (error: any) { + countOutcome('error'); log.error('Error processing payment and top-up', { error: error.message, stack: error.stack, From 20a3fbc088bb01c8d226ed1ab1ff305f6383aaa3 Mon Sep 17 00:00:00 2001 From: "Bill Gates (ops agent)" Date: Fri, 18 Sep 2026 19:01:30 +0000 Subject: [PATCH 2/2] fix(x402): record settled USDC at settlement, and attribute top-up failures Addresses CodeRabbit's review. The counters were both recorded after the rate-limiter top-up, so a payment that settled on-chain and then failed to grant access was filed as outcome="error" with no revenue recorded at all. That is the worst case to misreport: the funds moved. It also defeated the purpose of the change, since the resulting metrics understated income and hid the failure. - x402_payment_settled_usdc_total is now incremented immediately after a successful settlement, which is when the funds actually move. A later failure cannot erase revenue that was really collected. - A throwing top-up is attributed to outcome="topup_failed" rather than falling through to the generic catch. - countOutcome records that an outcome was already set, so the outer catch only reports "error" when nothing more specific was recorded. No attempt is counted twice. The consequence is a deliberate asymmetry, documented in both the TSDoc and docs/x402-and-rate-limiting.md: revenue counts every payment collected, while outcome="settled" additionally requires access to have been granted. The gap between them, outcome="topup_failed", is money taken without access -- worth alerting on. Also documents the "error" outcome, which the metric TSDoc omitted. Test: a settlement followed by a throwing top-up records the USDC, counts topup_failed, and counts neither "error" nor "settled". Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012NWDKc9pST69qTEha4AGaB --- docs/x402-and-rate-limiting.md | 12 +++- src/metrics.ts | 14 ++++- src/payments/payment-processor-utils.test.ts | 34 +++++++++++ src/payments/payment-processor-utils.ts | 61 ++++++++++++++------ 4 files changed, 98 insertions(+), 23 deletions(-) diff --git a/docs/x402-and-rate-limiting.md b/docs/x402-and-rate-limiting.md index 23c24cf37..d9c0e13c8 100644 --- a/docs/x402-and-rate-limiting.md +++ b/docs/x402-and-rate-limiting.md @@ -2045,7 +2045,14 @@ Payment attempts are counted at `/ar-io/__gateway_metrics`: `outcome` values: `no_payment_header`, `invalid_target`, `missing_host`, `verify_failed`, `unsupported_processor`, `unsupported_payload`, -`settle_failed`, `topup_failed`, `error`, `settled`. +`settle_failed`, `topup_failed`, `error` (an unexpected throw with no more +specific stage), `settled`. + +`x402_payment_settled_usdc_total` is recorded **at settlement** — the point the +funds move — while `outcome="settled"` also requires the access top-up to have +succeeded. So revenue counts every payment collected, and +`outcome="topup_failed"` counts payments taken where access was not granted. +That second number is worth alerting on: it is money owed back. 402 responses themselves are already countable without these, via `http_request_duration_seconds_count{status_code="402"}`. The counters above @@ -2066,6 +2073,9 @@ sum by (outcome) (rate(x402_payment_total{outcome!="settled"}[1h])) # revenue actually settled sum(increase(x402_payment_settled_usdc_total[24h])) +# paid but not granted access — alert on this +sum(increase(x402_payment_total{outcome="topup_failed"}[1h])) + # conversion: settled payments per 402 served sum(increase(x402_payment_total{outcome="settled"}[24h])) / sum(increase(http_request_duration_seconds_count{status_code="402"}[24h])) diff --git a/src/metrics.ts b/src/metrics.ts index 875b8fbd8..bab763278 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -1580,9 +1580,17 @@ export const rateLimitTokensConsumedTotal = new promClient.Counter({ * with incomplete CDP credentials silently falls back to a facilitator that * cannot settle, and keeps serving 402s while earning nothing. * - * `outcome` is the stage that ended the attempt: no_payment_header, - * invalid_target, missing_host, verify_failed, unsupported_processor, - * unsupported_payload, settle_failed, topup_failed, or settled. + * `outcome` is the stage that ended the attempt: `no_payment_header`, + * `invalid_target`, `missing_host`, `verify_failed`, `unsupported_processor`, + * `unsupported_payload`, `settle_failed`, `topup_failed`, `error` (an + * unexpected throw with no more specific stage), or `settled`. + * + * Note that `x402_payment_settled_usdc_total` is recorded at settlement, which + * is when the funds actually move, while `outcome="settled"` requires the + * access top-up to have succeeded as well. Revenue therefore counts every + * payment collected even when a later step failed, and + * `sum(x402_payment_total{outcome="topup_failed"})` is the count of payments + * taken without access granted — an amount owed back, and worth alerting on. */ export const x402PaymentCounter = new promClient.Counter({ name: 'x402_payment_total', diff --git a/src/payments/payment-processor-utils.test.ts b/src/payments/payment-processor-utils.test.ts index ede625346..7634098f3 100644 --- a/src/payments/payment-processor-utils.test.ts +++ b/src/payments/payment-processor-utils.test.ts @@ -133,6 +133,40 @@ describe('processPaymentAndTopUp metrics', () => { assert.equal(await outcomeCount('verify_failed'), beforeVerify + 1); }); + // A payment that settles on-chain and then fails to grant access is the worst + // case to misreport: the funds moved. It must not be filed as a generic + // error, and the revenue must still be counted. + it('records settled USDC and topup_failed when the top-up throws after settlement', async () => { + const beforeUsdc = await settledUsdc(); + const beforeTopup = await outcomeCount('topup_failed'); + const beforeError = await outcomeCount('error'); + const beforeSettled = await outcomeCount('settled'); + + const failingLimiter = { + topOffPaidTokens: async () => { + throw new Error('redis unavailable'); + }, + topOffPaidTokensForResource: async () => undefined, + } as unknown as RateLimiter; + + const result = await processPaymentAndTopUp( + failingLimiter, + makeProcessor() as any, + req(), + log, + { type: 'ip' }, + ); + + assert.equal(result.success, false); + // The payment settled, so the money is counted... + assert.ok(Math.abs((await settledUsdc()) - (beforeUsdc + 0.25)) < 1e-9); + // ...attributed to the stage that actually failed... + assert.equal(await outcomeCount('topup_failed'), beforeTopup + 1); + // ...and not double-counted as a generic error or a clean success. + assert.equal(await outcomeCount('error'), beforeError); + assert.equal(await outcomeCount('settled'), beforeSettled); + }); + it('counts a request that carried no payment header', async () => { const before = await outcomeCount('no_payment_header'); diff --git a/src/payments/payment-processor-utils.ts b/src/payments/payment-processor-utils.ts index 425ed7ee4..5d2ec1b12 100644 --- a/src/payments/payment-processor-utils.ts +++ b/src/payments/payment-processor-utils.ts @@ -65,9 +65,13 @@ export async function processPaymentAndTopUp( contentSizeOverride?: number, ): Promise { // Records where an attempt ended, so the funnel from 402 -> settled payment is - // visible. Every early return below goes through this. - const countOutcome = (outcome: string) => + // visible. Every early return below goes through this, and the outer catch + // only records `error` if nothing more specific was recorded first. + let outcomeRecorded = false; + const countOutcome = (outcome: string) => { + outcomeRecorded = true; metrics.x402PaymentCounter.inc({ outcome, target: target.type }); + }; try { // Extract payment from headers const payment = paymentProcessor.extractPayment(req); @@ -192,6 +196,20 @@ export async function processPaymentAndTopUp( }; } + // Recorded here rather than after the top-up: settlement is the point the + // funds actually move, so a later failure granting access must not erase + // revenue that was really collected. USDC has 6 decimals; parseInt of a + // non-numeric value would poison the counter, so only a finite result is + // recorded. + const settledUsdc = + parseInt(payment.payload.authorization.value.toString(), 10) / 1_000_000; + if (Number.isFinite(settledUsdc)) { + metrics.x402PaymentSettledUsdcCounter.inc( + { target: target.type }, + settledUsdc, + ); + } + // Convert payment amount to tokens and top up bucket // Payment has been settled successfully and validations passed, now grant access tokens @@ -214,17 +232,29 @@ export async function processPaymentAndTopUp( if (target.type === 'ip') { // For IP bucket, use existing method with request - await rateLimiter.topOffPaidTokens(req, tokens); + try { + await rateLimiter.topOffPaidTokens(req, tokens); + } catch (error: any) { + // The payment settled; only the access grant failed. Attribute it + // precisely instead of letting the outer catch call it an error. + countOutcome('topup_failed'); + throw error; + } tokensAdded = tokens * multiplierApplied; } else if (target.type === 'resource') { // For resource bucket, use new method with explicit params // Note: target validation already done at function entry - await rateLimiter.topOffPaidTokensForResource( - target.method!, - target.host!, - target.path!, - tokens, - ); + try { + await rateLimiter.topOffPaidTokensForResource( + target.method!, + target.host!, + target.path!, + tokens, + ); + } catch (error: any) { + countOutcome('topup_failed'); + throw error; + } tokensAdded = tokens * multiplierApplied; } else { log.error('Invalid target type', { target }); @@ -246,15 +276,6 @@ export async function processPaymentAndTopUp( }); countOutcome('settled'); - // USDC has 6 decimals; parseInt of a non-numeric value would poison the - // counter, so only record a finite result. - const settledUsdc = parseInt(paymentAmount, 10) / 1_000_000; - if (Number.isFinite(settledUsdc)) { - metrics.x402PaymentSettledUsdcCounter.inc( - { target: target.type }, - settledUsdc, - ); - } return { success: true, @@ -264,7 +285,9 @@ export async function processPaymentAndTopUp( responseHeader: settlementResult.responseHeader, }; } catch (error: any) { - countOutcome('error'); + if (!outcomeRecorded) { + countOutcome('error'); + } log.error('Error processing payment and top-up', { error: error.message, stack: error.stack,