From fa797102611cfd4635e849cdf00ebd36ad520218 Mon Sep 17 00:00:00 2001 From: krsnak Date: Sat, 26 Sep 2026 10:52:16 +0200 Subject: [PATCH 01/15] Use token decimals for display amounts --- src/__tests__/amount.test.ts | 18 +++++++ src/__tests__/tokenCache.test.ts | 14 ++++++ src/amount.ts | 19 ++++++++ src/api.ts | 84 +++++++++++++++++++------------- src/api/accounts.ts | 14 +++--- src/db.ts | 42 ++++++++++------ src/graphql/server.ts | 18 +++++-- src/graphql/subscriptions.ts | 8 ++- src/routes/accounts/transfers.ts | 31 ++++++------ src/routes/assets/popular.ts | 6 ++- src/routes/exports.ts | 11 +++-- src/tokenCache.ts | 9 ++++ src/ws.ts | 11 ++++- 13 files changed, 199 insertions(+), 86 deletions(-) create mode 100644 src/__tests__/amount.test.ts create mode 100644 src/amount.ts diff --git a/src/__tests__/amount.test.ts b/src/__tests__/amount.test.ts new file mode 100644 index 000000000..859baeb99 --- /dev/null +++ b/src/__tests__/amount.test.ts @@ -0,0 +1,18 @@ +import { toDisplayAmount } from "../amount"; + +describe("toDisplayAmount", () => { + it("uses a token's real decimals", () => { + expect(toDisplayAmount("1000000", 6)).toBe("1.000000"); + expect(toDisplayAmount("1234567", 6)).toBe("1.234567"); + }); + + it("preserves the historical 7-decimal output by default", () => { + expect(toDisplayAmount("10000000")).toBe("1.0000000"); + expect(toDisplayAmount("1")).toBe("0.0000001"); + expect(toDisplayAmount("-10000000")).toBe("-1.0000000"); + }); + + it("supports zero decimals", () => { + expect(toDisplayAmount("42", 0)).toBe("42"); + }); +}); diff --git a/src/__tests__/tokenCache.test.ts b/src/__tests__/tokenCache.test.ts index aaf85bdbc..fe9508f48 100644 --- a/src/__tests__/tokenCache.test.ts +++ b/src/__tests__/tokenCache.test.ts @@ -2,6 +2,7 @@ import { getTokenMetadata, initTokenCache, getAllCachedTokens, + getCachedTokenDecimals, _resetTokenCache, } from "../tokenCache"; import { prisma } from "../db"; @@ -131,6 +132,19 @@ describe("Token Cache", () => { expect(getAllCachedTokens("mainnet").map((t) => t.contractId)).toEqual(["CMAIN"]); }); + it("reads cached decimals without triggering DB or RPC", async () => { + (prisma.tokenMetadata.findMany as jest.Mock).mockResolvedValue([ + { ...mockToken, decimals: 6 }, + ]); + await initTokenCache("testnet"); + jest.clearAllMocks(); + + expect(getCachedTokenDecimals("C123", "testnet")).toBe(6); + expect(getCachedTokenDecimals("CMISSING", "testnet")).toBeUndefined(); + expect(prisma.tokenMetadata.findUnique).not.toHaveBeenCalled(); + expect(fetchTokenMetadata).not.toHaveBeenCalled(); + }); + it("keeps serving from RPC when the DB seed fails", async () => { // A cold cache is recoverable; refusing to start is not. (prisma.tokenMetadata.findMany as jest.Mock).mockRejectedValue(new Error("db down")); diff --git a/src/amount.ts b/src/amount.ts new file mode 100644 index 000000000..907c8a782 --- /dev/null +++ b/src/amount.ts @@ -0,0 +1,19 @@ +const DEFAULT_TOKEN_DECIMALS = 7; + +export function toDisplayAmount( + amount: string, + decimals: number = DEFAULT_TOKEN_DECIMALS, +): string { + const normalizedDecimals = + Number.isInteger(decimals) && decimals >= 0 ? decimals : DEFAULT_TOKEN_DECIMALS; + const scale = 10n ** BigInt(normalizedDecimals); + const raw = BigInt(amount); + const abs = raw < 0n ? -raw : raw; + const integer = abs / scale; + const remainder = abs % scale; + const sign = raw < 0n ? "-" : ""; + + if (normalizedDecimals === 0) return `${sign}${integer}`; + + return `${sign}${integer}.${String(remainder).padStart(normalizedDecimals, "0")}`; +} diff --git a/src/api.ts b/src/api.ts index 96128f439..0778d523c 100644 --- a/src/api.ts +++ b/src/api.ts @@ -28,7 +28,8 @@ import { import { parseOr400 } from "./openapi/validation"; import { networkMiddleware, requestNetwork } from "./middleware/network"; import { renderMetrics, metricsContentType } from "./metrics"; -import { getAllCachedTokens } from "./tokenCache"; +import { getAllCachedTokens, getCachedTokenDecimals } from "./tokenCache"; +import { toDisplayAmount } from "./amount"; // ─── RPC Health Check Cache ─────────────────────────────────────────────── // Keyed by network (#163): one cache entry would let a healthy testnet RPC @@ -78,25 +79,17 @@ const POPULAR_CACHE_TTL_MS = parseInt(process.env.CACHE_TTL_POPULAR_MS ?? "60000 const SEARCH_CACHE_TTL_MS = parseInt(process.env.CACHE_TTL_SEARCH_MS ?? "15000", 10); // ─── Amount formatting ──────────────────────────────────────────────────── -const STROOPS = 10_000_000n; - -/** - * Convert a raw i128 decimal string (stroops) to a human-readable 7-decimal - * string. Uses BigInt arithmetic to avoid floating-point precision loss. - * e.g. "10000000000" → "1000.0000000" - */ -export function toDisplayAmount(amount: string): string { - const raw = BigInt(amount); - const abs = raw < 0n ? -raw : raw; - const integer = abs / STROOPS; - const remainder = abs % STROOPS; - const sign = raw < 0n ? "-" : ""; - return `${sign}${integer}.${String(remainder).padStart(7, "0")}`; -} - -const withDisplay = (t: T) => ({ - ...t, - displayAmount: toDisplayAmount(t.amount), +export { toDisplayAmount } from "./amount"; + +const withDisplay = ( + transfer: T, + network: Network, +) => ({ + ...transfer, + displayAmount: toDisplayAmount( + transfer.amount, + transfer.contractId ? getCachedTokenDecimals(transfer.contractId, network) : undefined, + ), }); function parseSelectQuery(value: unknown): string[] | undefined { @@ -509,8 +502,10 @@ export function createApp(): express.Application { token?: string; }; + const network = requestNetwork(req); const result = await queryTransfers({ - network: requestNetwork(req), + network, + tokenDecimals: getCachedTokenDecimals, address, direction: "incoming", contractId, @@ -531,7 +526,7 @@ export function createApp(): express.Application { ...result, transfers: result.transfers.map((transfer) => { if (transfer && typeof (transfer as { amount?: unknown }).amount === "string") { - return withDisplay(transfer as { amount: string }); + return withDisplay(transfer as { amount: string; contractId?: string }, network); } return transfer; }), @@ -571,8 +566,10 @@ export function createApp(): express.Application { token?: string; }; + const network = requestNetwork(req); const result = await queryTransfers({ - network: requestNetwork(req), + network, + tokenDecimals: getCachedTokenDecimals, address, direction: "outgoing", contractId, @@ -593,7 +590,7 @@ export function createApp(): express.Application { ...result, transfers: result.transfers.map((transfer) => { if (transfer && typeof (transfer as { amount?: unknown }).amount === "string") { - return withDisplay(transfer as { amount: string }); + return withDisplay(transfer as { amount: string; contractId?: string }, network); } return transfer; }), @@ -644,8 +641,10 @@ export function createApp(): express.Application { $select?: string[]; }; + const network = requestNetwork(req); const result = await queryAllTransfers({ - network: requestNetwork(req), + network, + tokenDecimals: getCachedTokenDecimals, address, contractId, token, @@ -665,7 +664,7 @@ export function createApp(): express.Application { ...result, transfers: result.transfers.map((transfer) => { if (transfer && typeof (transfer as { amount?: unknown }).amount === "string") { - return withDisplay(transfer as { amount: string }); + return withDisplay(transfer as { amount: string; contractId?: string }, network); } return transfer; }), @@ -716,8 +715,10 @@ export function createApp(): express.Application { }; // Always fetch with offset=0 and enforce a 10,000 row limit for CSV export + const network = requestNetwork(req); const result = await queryAllTransfers({ - network: requestNetwork(req), + network, + tokenDecimals: getCachedTokenDecimals, address, contractId, token, @@ -739,7 +740,12 @@ export function createApp(): express.Application { // Add data rows for (const transfer of result.transfers) { const t = transfer as Record; - const displayAmount = toDisplayAmount(String(t.amount ?? "0")); + const displayAmount = toDisplayAmount( + String(t.amount ?? "0"), + typeof t.contractId === "string" + ? getCachedTokenDecimals(t.contractId, network) + : undefined, + ); const closedAt = t.ledgerClosedAt instanceof Date ? t.ledgerClosedAt : new Date(String(t.ledgerClosedAt ?? 0)); @@ -781,8 +787,17 @@ export function createApp(): express.Application { try { const parsed = parseOr400(txHashParamsSchema, req.params, res); if (!parsed) return; - const transfers = await queryByTxHash(parsed.txHash, requestNetwork(req)); - res.json({ transfers: transfers.map(withDisplay) }); + const network = requestNetwork(req); + const transfers = await queryByTxHash(parsed.txHash, network); + res.json({ + transfers: transfers.map((transfer) => ({ + ...transfer, + displayAmount: toDisplayAmount( + transfer.amount, + getCachedTokenDecimals(transfer.contractId, network), + ), + })), + }); } catch (err) { next(err); } @@ -809,8 +824,9 @@ export function createApp(): express.Application { if (!parsed) return; const { address, contractId, fromDate, toDate } = parsed; + const network = requestNetwork(req); const rows = await querySummary({ - network: requestNetwork(req), + network, address, contractId, fromDate, @@ -826,9 +842,9 @@ export function createApp(): express.Application { totalReceived: row.totalReceived, totalSent: row.totalSent, netFlow: net.toString(), - displayTotalReceived: toDisplayAmount(row.totalReceived), - displayTotalSent: toDisplayAmount(row.totalSent), - displayNetFlow: toDisplayAmount(net.toString()), + displayTotalReceived: toDisplayAmount(row.totalReceived, getCachedTokenDecimals(row.contractId, network)), + displayTotalSent: toDisplayAmount(row.totalSent, getCachedTokenDecimals(row.contractId, network)), + displayNetFlow: toDisplayAmount(net.toString(), getCachedTokenDecimals(row.contractId, network)), txCount: Number(row.txCount), }; }); diff --git a/src/api/accounts.ts b/src/api/accounts.ts index db98acb7f..be3fab6e8 100644 --- a/src/api/accounts.ts +++ b/src/api/accounts.ts @@ -1,10 +1,11 @@ import { Router, Request, Response, NextFunction } from "express"; import { getAccountSummary, queryBalances } from "../db"; -import { toDisplayAmount } from "../api"; +import { toDisplayAmount } from "../amount"; import { createAccountsTransfersRouter } from "../routes/accounts/transfers"; import { parseOr400 } from "../openapi/validation"; import { summaryQuerySchema } from "../openapi/schemas"; import { requestNetwork } from "../middleware/network"; +import { getCachedTokenDecimals } from "../tokenCache"; type AccountSummaryRow = Awaited>[number]; @@ -52,7 +53,7 @@ export function createAccountsRouter(): Router { balances: rows.map((row) => ({ contractId: row.contractId, balance: row.balance, - displayBalance: toDisplayAmount(row.balance), + displayBalance: toDisplayAmount(row.balance, getCachedTokenDecimals(row.contractId, network)), })), derivedFromLedger: true, note: @@ -74,7 +75,8 @@ export function createAccountsRouter(): Router { if (!parsed) return; const { address, contractId } = parsed; - const rows = await getAccountSummary(address, contractId, requestNetwork(req)); + const network = requestNetwork(req); + const rows = await getAccountSummary(address, contractId, network); const assets = rows.map((row: AccountSummaryRow) => { const net = BigInt(row.net); @@ -83,9 +85,9 @@ export function createAccountsRouter(): Router { totalSent: row.totalSent, totalReceived: row.totalReceived, net: row.net, - displayTotalSent: toDisplayAmount(row.totalSent), - displayTotalReceived:toDisplayAmount(row.totalReceived), - displayNet: toDisplayAmount(net < 0n ? (-net).toString() : row.net) + (net < 0n ? " (negative)" : ""), + displayTotalSent: toDisplayAmount(row.totalSent, getCachedTokenDecimals(row.contractId, network)), + displayTotalReceived:toDisplayAmount(row.totalReceived, getCachedTokenDecimals(row.contractId, network)), + displayNet: toDisplayAmount(net < 0n ? (-net).toString() : row.net, getCachedTokenDecimals(row.contractId, network)) + (net < 0n ? " (negative)" : ""), txCount: row.txCount, lastActivityAt: row.lastActivityAt, }; diff --git a/src/db.ts b/src/db.ts index 9b0e7221d..1a39f13f0 100644 --- a/src/db.ts +++ b/src/db.ts @@ -2,22 +2,14 @@ import { PrismaClient, Prisma } from "@prisma/client"; import type { NftTransferRecord, NftMetadataPayload } from "./ingester/nft"; import { decodeCursor, encodeCursor, parseODataFilter, parseODataSelect, projectRecord } from "./lib/odata"; import { resolveNetwork, type Network } from "./network"; +import { toDisplayAmount } from "./amount"; // Every function below takes an optional trailing `network`. Omitting it means // "the network this process is configured for" (STELLAR_NETWORK), which is // exactly the pre-#159 behaviour for single-network deployments. The per-network // indexer loop (#161) and the API selector (#163) pass it explicitly. -const STROOPS = 10_000_000n; - -export function toDisplayAmount(amount: string): string { - const raw = BigInt(amount); - const abs = raw < 0n ? -raw : raw; - const integer = abs / STROOPS; - const remainder = abs % STROOPS; - const sign = raw < 0n ? "-" : ""; - return `${sign}${integer}.${String(remainder).padStart(7, "0")}`; -} +export { toDisplayAmount } from "./amount"; import { withReadReplicas } from "./db/router"; import { observeDbQuery } from "./metrics"; @@ -341,6 +333,7 @@ export async function pruneOldTransfers(network?: Network): Promise { // ─── Query helpers ───────────────────────────────────────────────────────── export type TransferQueryParams = { network?: Network; + tokenDecimals?: (contractId: string, network: Network) => number | undefined; address: string; direction: "incoming" | "outgoing"; contractId?: string; @@ -360,6 +353,7 @@ export type TransferQueryParams = { export async function queryTransfers(params: TransferQueryParams) { const { network, + tokenDecimals, address, direction, contractId, @@ -375,9 +369,10 @@ export async function queryTransfers(params: TransferQueryParams) { limit = 50, offset = 0, } = params; + const resolvedNetwork = resolveNetwork(network); const baseWhere: Prisma.TokenTransferWhereInput = { - network: resolveNetwork(network), + network: resolvedNetwork, ...(direction === "incoming" ? { toAddress: address } : { fromAddress: address }), ...(contractId ? { contractId } : {}), ...(token ? { contractId: token } : {}), @@ -409,7 +404,7 @@ export async function queryTransfers(params: TransferQueryParams) { const prismaSelect = requestedSelect ? { id: true, - contractId: requestedSelect.includes("contractId"), + contractId: requestedSelect.includes("contractId") || requestedSelect.includes("displayAmount"), eventType: requestedSelect.includes("eventType"), fromAddress: requestedSelect.includes("fromAddress"), toAddress: requestedSelect.includes("toAddress"), @@ -444,7 +439,13 @@ export async function queryTransfers(params: TransferQueryParams) { return { total, transfers: selectRows(page.rows as Array>, requestedSelect, { - displayAmount: (row) => toDisplayAmount(String((row as { amount?: string }).amount)), + displayAmount: (row) => { + const transfer = row as { amount?: string; contractId?: string }; + return toDisplayAmount( + String(transfer.amount), + transfer.contractId ? tokenDecimals?.(transfer.contractId, resolvedNetwork) : undefined, + ); + }, }), nextCursor: page.nextCursor, }; @@ -867,6 +868,7 @@ export async function queryAccountSummaries(params: AccountSummaryQueryParams) { // ─── Combined address query ──────────────────────────────────────────────── export type AllTransfersQueryParams = { network?: Network; + tokenDecimals?: (contractId: string, network: Network) => number | undefined; address: string; contractId?: string; token?: string; @@ -885,6 +887,7 @@ export type AllTransfersQueryParams = { export async function queryAllTransfers(params: AllTransfersQueryParams) { const { network, + tokenDecimals, address, contractId, token, @@ -899,9 +902,10 @@ export async function queryAllTransfers(params: AllTransfersQueryParams) { limit = 50, offset = 0, } = params; + const resolvedNetwork = resolveNetwork(network); const baseWhere: Prisma.TokenTransferWhereInput = { - network: resolveNetwork(network), + network: resolvedNetwork, OR: [{ toAddress: address }, { fromAddress: address }], ...(contractId ? { contractId } : {}), ...(token ? { contractId: token } : {}), @@ -935,7 +939,7 @@ export async function queryAllTransfers(params: AllTransfersQueryParams) { const prismaSelect = requestedSelect ? { id: true, - contractId: requestedSelect.includes("contractId"), + contractId: requestedSelect.includes("contractId") || requestedSelect.includes("displayAmount"), eventType: requestedSelect.includes("eventType"), fromAddress: requestedSelect.includes("fromAddress"), toAddress: requestedSelect.includes("toAddress"), @@ -963,7 +967,13 @@ export async function queryAllTransfers(params: AllTransfersQueryParams) { const page = buildListPage(rows as Array<{ id: number }>, cap); const transfers = selectRows(page.rows as Array>, requestedSelect ? [...requestedSelect, "direction"] : undefined, { - displayAmount: (row) => toDisplayAmount(String((row as { amount?: string }).amount)), + displayAmount: (row) => { + const transfer = row as { amount?: string; contractId?: string }; + return toDisplayAmount( + String(transfer.amount), + transfer.contractId ? tokenDecimals?.(transfer.contractId, resolvedNetwork) : undefined, + ); + }, direction: (row) => ((row as { toAddress?: string | null }).toAddress === address ? "incoming" : "outgoing"), }); diff --git a/src/graphql/server.ts b/src/graphql/server.ts index c45de7c45..441542200 100644 --- a/src/graphql/server.ts +++ b/src/graphql/server.ts @@ -10,6 +10,8 @@ import { costLimitPlugin } from "./costLimit"; import { persistedQueryPlugin } from "./persisted"; import { requestNetwork } from "../middleware/network"; import { enabledNetworks, isNetwork, NETWORKS, currentNetwork, type Network } from "../network"; +import { getCachedTokenDecimals } from "../tokenCache"; +import { toDisplayAmount } from "../amount"; export const typeDefs = `#graphql enum Network { @@ -137,6 +139,7 @@ export const resolvers = { ) => { const common = { network: resolveArgNetwork(args.network, ctx), + tokenDecimals: getCachedTokenDecimals, address: args.address, contractId: args.contractId, limit: args.limit, @@ -163,10 +166,17 @@ export const resolvers = { args: { txHash: string; network?: NetworkArg }, ctx?: GraphQLContext ) => { - const transfers = await queryByTxHash(args.txHash, resolveArgNetwork(args.network, ctx)); - return (transfers as Array>).map((transfer) => - formatTransfer(transfer) - ); + const network = resolveArgNetwork(args.network, ctx); + const transfers = await queryByTxHash(args.txHash, network); + return (transfers as Array>).map((transfer) => ({ + ...formatTransfer(transfer), + displayAmount: toDisplayAmount( + String(transfer.amount), + typeof transfer.contractId === "string" + ? getCachedTokenDecimals(transfer.contractId, network) + : undefined, + ), + })); }, summary: async ( diff --git a/src/graphql/subscriptions.ts b/src/graphql/subscriptions.ts index f01232526..64d3b8c28 100644 --- a/src/graphql/subscriptions.ts +++ b/src/graphql/subscriptions.ts @@ -28,7 +28,8 @@ import { import { WebSocketServer, WebSocket } from "ws"; import type { IncomingMessage, Server } from "http"; import { typeDefs as baseTypeDefs, resolvers as baseResolvers, formatTransfer, type GraphQLContext } from "./server"; -import { toDisplayAmount } from "../api"; +import { toDisplayAmount } from "../amount"; +import { getCachedTokenDecimals } from "../tokenCache"; import { transferEmitter, hostFnLogEmitter, @@ -219,7 +220,10 @@ function buildSubscriptionSchema(): GraphQLSchema { const transfer = payload as TransferEvent; return { ...formatTransfer(transfer as unknown as Record), - displayAmount: toDisplayAmount(transfer.amount), + displayAmount: toDisplayAmount( + transfer.amount, + getCachedTokenDecimals(transfer.contractId, transfer.network), + ), }; }, }, diff --git a/src/routes/accounts/transfers.ts b/src/routes/accounts/transfers.ts index 790c5849c..13c65c6a1 100644 --- a/src/routes/accounts/transfers.ts +++ b/src/routes/accounts/transfers.ts @@ -3,24 +3,21 @@ import { queryAllTransfers } from "../../db"; import { parseOr400 } from "../../openapi/validation"; import { transferQuerySchema } from "../../openapi/schemas"; import { requestNetwork } from "../../middleware/network"; +import { getCachedTokenDecimals } from "../../tokenCache"; +import { toDisplayAmount } from "../../amount"; const VALID_EVENT_TYPES = new Set(["transfer", "mint", "burn", "clawback"]); -const STROOPS = 10_000_000n; -function toDisplayAmount(amount: string): string { - const raw = BigInt(amount); - const abs = raw < 0n ? -raw : raw; - const integer = abs / STROOPS; - const remainder = abs % STROOPS; - const sign = raw < 0n ? "-" : ""; - return `${sign}${integer}.${String(remainder).padStart(7, "0")}`; -} - -const withDisplay = (t: T) => ({ - ...t, - displayAmount: toDisplayAmount(t.amount), +const withDisplay = ( + transfer: T, + network: ReturnType, +) => ({ + ...transfer, + displayAmount: toDisplayAmount( + transfer.amount, + transfer.contractId ? getCachedTokenDecimals(transfer.contractId, network) : undefined, + ), }); - const parseIntParam = (val: unknown, fallback: number): number => { const n = parseInt(String(val), 10); return isNaN(n) ? fallback : n; @@ -74,8 +71,10 @@ export function createAccountsTransfersRouter(): Router { $select?: string[]; }; + const network = requestNetwork(req); const result = await queryAllTransfers({ - network: requestNetwork(req), + network, + tokenDecimals: getCachedTokenDecimals, address, contractId, token, @@ -95,7 +94,7 @@ export function createAccountsTransfersRouter(): Router { ...result, transfers: result.transfers.map((transfer) => { if (transfer && typeof (transfer as { amount?: unknown }).amount === "string") { - return withDisplay(transfer as { amount: string }); + return withDisplay(transfer as { amount: string; contractId?: string }, network); } return transfer; }), diff --git a/src/routes/assets/popular.ts b/src/routes/assets/popular.ts index 1806e8aa2..aaeedb239 100644 --- a/src/routes/assets/popular.ts +++ b/src/routes/assets/popular.ts @@ -3,6 +3,7 @@ import { queryPopularAssets, toDisplayAmount } from "../../db"; import { parseOr400 } from "../../openapi/validation"; import { popularAssetsQuerySchema } from "../../openapi/schemas"; import { requestNetwork } from "../../middleware/network"; +import { getCachedTokenDecimals } from "../../tokenCache"; const VALID_WINDOWS = new Set(["1h", "24h", "7d"]); const VALID_SORT_BY = new Set(["transfers", "volume"]); @@ -33,8 +34,9 @@ export function createPopularAssetsRouter(): Router { const { window, by, limit, offset } = parsed; const fromDate = windowToDate(window); + const network = requestNetwork(req); const { total, assets } = await queryPopularAssets({ - network: requestNetwork(req), + network, fromDate, by, limit, @@ -51,7 +53,7 @@ export function createPopularAssetsRouter(): Router { contractId: a.contractId, transferCount: Number(a.transferCount), volume: a.volume, - displayVolume: toDisplayAmount(a.volume), + displayVolume: toDisplayAmount(a.volume, getCachedTokenDecimals(a.contractId, network)), })), }); } catch (err) { diff --git a/src/routes/exports.ts b/src/routes/exports.ts index a09ac930a..7ea28797a 100644 --- a/src/routes/exports.ts +++ b/src/routes/exports.ts @@ -6,6 +6,7 @@ import path from "path"; import fs from "fs"; import { requestNetwork } from "../middleware/network"; import type { Network } from "../network"; +import { getCachedTokenDecimals } from "../tokenCache"; // How many rows we fetch per DB round-trip. Keeps memory flat. const BATCH_SIZE = 500; @@ -74,7 +75,8 @@ async function* streamTransfers(where: Record) { // ── CSV endpoint ───────────────────────────────────────────────────────────── async function handleCsvExport(req: Request, res: Response, next: NextFunction) { try { - const where = buildWhere(req.query as Record, requestNetwork(req)); + const network = requestNetwork(req); + const where = buildWhere(req.query as Record, network); res.setHeader("Content-Type", "text/csv"); res.setHeader("Content-Disposition", "attachment; filename=\"transfers.csv\""); @@ -91,7 +93,7 @@ async function handleCsvExport(req: Request, res: Response, next: NextFunction) fromAddress: row.fromAddress ?? "", toAddress: row.toAddress ?? "", amount: row.amount, - displayAmount: toDisplayAmount(row.amount), + displayAmount: toDisplayAmount(row.amount, getCachedTokenDecimals(row.contractId, network)), ledger: row.ledger, ledgerClosedAt: row.ledgerClosedAt.toISOString(), txHash: row.txHash, @@ -116,7 +118,8 @@ async function handleParquetExport(req: Request, res: Response, next: NextFuncti const tmpFile = path.join(os.tmpdir(), `transfers-${Date.now()}-${Math.random().toString(36).slice(2)}.parquet`); try { - const where = buildWhere(req.query as Record, requestNetwork(req)); + const network = requestNetwork(req); + const where = buildWhere(req.query as Record, network); const schema = new parquet.ParquetSchema({ id: { type: "INT64" }, @@ -144,7 +147,7 @@ async function handleParquetExport(req: Request, res: Response, next: NextFuncti fromAddress: row.fromAddress ?? null, toAddress: row.toAddress ?? null, amount: row.amount, - displayAmount: toDisplayAmount(row.amount), + displayAmount: toDisplayAmount(row.amount, getCachedTokenDecimals(row.contractId, network)), ledger: row.ledger, ledgerClosedAt: row.ledgerClosedAt.toISOString(), txHash: row.txHash, diff --git a/src/tokenCache.ts b/src/tokenCache.ts index ca73583d5..c875264e9 100644 --- a/src/tokenCache.ts +++ b/src/tokenCache.ts @@ -97,6 +97,15 @@ export function getAllCachedTokens(network?: Network): TokenMetadata[] { return network ? all.filter((t) => t.network === network) : all; } +/** Read cached decimals without triggering a database or RPC lookup. */ +export function getCachedTokenDecimals( + contractId: string, + network?: Network, +): number | undefined { + const net = resolveNetwork(network); + return cache.get(cacheKey(net, contractId))?.decimals; +} + /** Test-only: drop the in-memory cache. */ export function _resetTokenCache(): void { cache.clear(); diff --git a/src/ws.ts b/src/ws.ts index b27af4a05..1d166f69f 100644 --- a/src/ws.ts +++ b/src/ws.ts @@ -1,7 +1,8 @@ import { WebSocketServer, WebSocket } from "ws"; import type { IncomingMessage, Server } from "http"; import { transferEmitter, TransferEvent } from "./events"; -import { toDisplayAmount } from "./api"; +import { toDisplayAmount } from "./amount"; +import { getCachedTokenDecimals } from "./tokenCache"; import { currentNetwork, enabledNetworks, isNetwork, NETWORKS, type Network } from "./network"; // Matches /subscribe/, ignoring any query string (#163). @@ -38,7 +39,13 @@ export function resolveSocketNetwork(url: string): { network: Network } | { erro type WsPayload = TransferEvent & { displayAmount: string }; function buildPayload(t: TransferEvent): WsPayload { - return { ...t, displayAmount: toDisplayAmount(t.amount) }; + return { + ...t, + displayAmount: toDisplayAmount( + t.amount, + getCachedTokenDecimals(t.contractId, t.network), + ), + }; } /** From 07e6f723903b8c5fb5757f91b8c391c958dbd368 Mon Sep 17 00:00:00 2001 From: Miracle656 Date: Tue, 29 Sep 2026 13:23:14 +0100 Subject: [PATCH 02/15] docs(wave): W094-W098 batch draft (published as #203-207) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five issues, 800 points. The headline is W094: `src/indexer.ts:491` switches between pollOnce and pollParallel on INGEST_WORKERS, and `indexer/parallel.ts` imports only fetchEventsSafe, parseEvents, upsertTransfers, setLastIndexedLedger and emitTransfer — no NFT parsing, no metadata, no account summaries. Raising the worker count for throughput silently stops indexing whole categories, with no error to notice. Also: /offramp/orders/:orderId serves an order with no authorization, NFT metadata is fetched one serial round trip at a time, and every paged query pays for a full COUNT. Claude-Session: https://claude.ai/code/session_01USgemLt4Rnz4SGB1Srf3GB --- scripts/wave-issues-w094.md | 144 ++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 scripts/wave-issues-w094.md diff --git a/scripts/wave-issues-w094.md b/scripts/wave-issues-w094.md new file mode 100644 index 000000000..0d9553855 --- /dev/null +++ b/scripts/wave-issues-w094.md @@ -0,0 +1,144 @@ +# Wraith wave batch — parallel ingest, the money path, and query cost (DRAFT) + +**Repo:** `Miracle656/wraith`. **IDs:** W094–W098. **Points:** Easy 100 · Intermediate 150 · Advanced 200. + +## Shared with every issue + +Wraith is a Soroban incoming-transfer indexer — it fills the gap Horizon leaves for SAC/SEP-41 token events by recipient address. Node/TypeScript, Prisma, REST + GraphQL + WebSocket, deployed on Render. + +- **Unit tests are jest** (`src/**/__tests__`, `tests/*.test.ts`). **Integration tests are vitest** (`tests/integration/`, Docker). Do not mix runners in one file. +- Every PR needs a test that fails before the change and passes after, unless the issue says otherwise. +- **Validate every hard-coded `C…`/`G…`** with `StrKey.isValidContract` / `isValidEd25519PublicKey`. A recent PR here used a 55-character string in a fixture, which made the whole test file throw at import and run zero tests — while the PR reported it passing. A length check is not validation. +- Do not widen a Prometheus label to something unbounded. Addresses and contract ids must never become label values. +- Nothing may log or return a raw database error, a connection string, or a provider URL. +- Do not reformat a file you are changing. A diff that is mostly whitespace hides the part that matters. + +--- + +### W094 · Parallel ingest silently stops indexing NFTs and account summaries + +**Labels:** help wanted, Stellar Wave, area:indexer, difficulty:advanced + +### Background +`src/indexer.ts:491` switches the whole ingest path based on one env var: + +```ts +if (INGEST_WORKERS > 1 && loop.sacContractIds.length > 1) { + const { totalInserted, highestLedger } = await pollParallel(…) +} else { + currentLedger = await pollOnce(loop, currentLedger, target); +} +``` + +`pollOnce` parses NFT events and warms their metadata (`src/indexer.ts:379,392`) and maintains account summaries. `src/indexer/parallel.ts` imports only `fetchEventsSafe`, `parseEvents`, `upsertTransfers`, `setLastIndexedLedger` and `emitTransfer` — **no NFT parsing, no metadata, no account summaries**. + +So raising `INGEST_WORKERS` for throughput quietly turns off whole categories of indexing. Nothing errors. The data simply stops arriving, and the only way to notice is to miss it downstream. + +### What to build +- Bring the two paths back to one. Extract what `pollOnce` does per batch of records and have both call it, so a future addition cannot land in one path only. +- If some work genuinely cannot be sharded, make that explicit and refuse to start in parallel mode rather than silently degrading. +- A test that proves both paths produce the same rows for the same events — that is the real deliverable here, more than the fix. + +### Acceptance criteria +- [ ] The same ledger range ingested through each path produces identical `TokenTransfer`, `NftTransfer` and `AccountSummary` rows +- [ ] Adding a new per-record step to one path and not the other fails that test +- [ ] `INGEST_WORKERS > 1` no longer changes what is indexed, only how fast +- [ ] The PR states which work is genuinely parallel-unsafe, if any + +> **Drips Wave** · Complexity: **Advanced** · **200 points** + +--- + +### W095 · `/offramp/orders/:orderId` hands anyone the order + +**Labels:** help wanted, Stellar Wave, area:api, difficulty:advanced + +### Background +`src/api/offramp.ts:230` serves an order by id with no authorization. Anyone holding or guessing an order id reads the bank payout amount, the deposit address and the rate. This is the only money-moving router in the repo and it is the one surface with no auth story. + +This needs a decision before it needs code, so treat the first half of this issue as design: orders are created by a wallet, so the natural answer is that the same wallet must prove itself to read one back — but the shape of that proof is a maintainer call, not a contributor guess. + +### What to build +- Propose the auth model in the PR description first: a bearer credential issued at creation, a signature over the order id, or the existing API-key scheme extended to orders. Say what you chose and why. +- Make order ids unguessable regardless of which model wins — if an id is a secret, it must look like one. +- Rate-limit lookups so an unguessable id cannot be brute-forced quietly. +- No response may name the payment provider or echo an upstream error verbatim. + +### Acceptance criteria +- [ ] An unauthenticated request for a valid order id is refused +- [ ] The legitimate creator can still read their own order +- [ ] Order ids carry enough entropy to survive enumeration, and a test asserts the format +- [ ] Repeated failed lookups are rate-limited +- [ ] Tests cover authorised, unauthorised and not-found, and the three are distinguishable to the caller only where they should be + +> **Drips Wave** · Complexity: **Advanced** · **200 points** + +--- + +### W096 · NFT metadata is fetched one round trip at a time + +**Labels:** help wanted, Stellar Wave, area:indexer, difficulty:intermediate + +### Background +`src/indexer.ts:386-400` awaits `getNftMetadata` once per unique `(contract, tokenId)` inside a sequential `for` loop. A batch carrying a hundred new tokens makes a hundred serial round trips before the poll cycle can finish, and the loop's own budget does not account for it. + +### What to build +- Batch or bound the concurrency — a small worker pool, or a single multi-key lookup where the source allows it. +- Cap total time spent on metadata per cycle so ingest never stalls behind it; metadata is enrichment, transfers are the product. +- Failures must stay per-token: one unreachable metadata URL should not cost the batch. + +### Acceptance criteria +- [ ] A batch of N new tokens no longer costs N serial round trips — show the before and after +- [ ] A slow or failing metadata source cannot stall the ingest loop past a bounded budget +- [ ] One token's failure does not affect the others +- [ ] Tests cover a slow source, a failing source, and a mixed batch + +> **Drips Wave** · Complexity: **Intermediate** · **150 points** + +--- + +### W097 · Every paged query pays for a full COUNT + +**Labels:** help wanted, Stellar Wave, area:api, difficulty:intermediate + +### Background +`src/db.ts:429-440` runs a full `COUNT` alongside every `findMany`. On a small table that is invisible; on the table this service exists to grow, the count becomes the dominant cost of every list request, and it is usually rendered as a number nobody reads. + +### What to build +- Stop counting by default. Offer `hasMore` from an n+1 fetch, which answers the only question pagination actually needs. +- Where a total is genuinely wanted, make it opt-in per request and say in the API docs that it costs more. +- Check the indexes support the paged query on its own, without the count masking a missing one. + +### Acceptance criteria +- [ ] The default list path issues no `COUNT` +- [ ] Pagination still knows whether another page exists, and a test covers the boundary +- [ ] An explicit opt-in still returns an exact total +- [ ] The PR reports query timings before and after on a seeded table of realistic size + +> **Drips Wave** · Complexity: **Intermediate** · **150 points** + +--- + +### W098 · Tidy start-up and shut-down + +**Labels:** help wanted, Stellar Wave, area:ci, difficulty:easy + +### Background +Three small things in `src/index.ts`, each cheap and each capable of costing an afternoon: + +- `main()` has no `.catch`, so a failure during start-up surfaces as an unhandled rejection rather than a clear exit. +- The `SKIP_INDEXER` check appears twice, and the second branch is unreachable. +- `shutdown()` never calls `server.close()`, so Render's SIGTERM drops in-flight requests instead of draining them. + +### What to build +- Catch and exit non-zero with a legible message. +- Remove the dead branch. +- Close the server and wait for in-flight requests, with a timeout so a stuck request cannot block the shutdown forever. + +### Acceptance criteria +- [ ] A start-up failure exits non-zero with a message naming the cause +- [ ] Only one `SKIP_INDEXER` check remains, and it is reachable +- [ ] SIGTERM drains in-flight requests, bounded by a timeout +- [ ] A test covers the drain and the timeout + +> **Drips Wave** · Complexity: **Easy** · **100 points** From 73de5afebc024ee739dde2fae897feaf04bb3ce4 Mon Sep 17 00:00:00 2001 From: Gloria Date: Wed, 30 Sep 2026 15:58:32 +0100 Subject: [PATCH 03/15] Docs: Mainnet deployment guide (#166) (#211) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Docs: Mainnet deployment guide (#166) * docs(mainnet): correct the native SAC ids, fix the backup link, add DIRECT_DATABASE_URL Review fixes applied on top of #211: - The mainnet block used CDLZFC3SY…, which is the *testnet* native XLM SAC (Asset.native().contractId(Networks.TESTNET)). Mainnet is CAS3J7GY… (Networks.PUBLIC). The testnet block used CDMLFMKMM…, which is not the native SAC on either network. Both corrected, with the derivation inlined so the next reader can check rather than trust. - Added a warning that SAC_CONTRACT_IDS must be set explicitly on mainnet, because the built-in fallback in src/indexer.ts is the wrong address. - ../W076 did not resolve to anything; pointed at ./backup-restore.md. - Added DIRECT_DATABASE_URL to both env blocks — prisma/schema.prisma declares directUrl, and boot-time schema sync fails without it. - Linked ./DUAL_NETWORK.md for the NETWORKS=testnet,mainnet option. Claude-Session: https://claude.ai/code/session_01USgemLt4Rnz4SGB1Srf3GB --------- Co-authored-by: Miracle656 --- .env.example | 4 +++ docs/DEPLOY_MAINNET.md | 80 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 docs/DEPLOY_MAINNET.md diff --git a/.env.example b/.env.example index e0d92e7bb..ce45f480e 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,8 @@ # ─── Database ─────────────────────────────────────────────── +# Critical for mainnet: Use a durable external provider (e.g. Neon, Supabase). +# Do NOT use Render free Postgres as it is deleted after 90 days. +# See docs/DEPLOY_MAINNET.md for more details. +# # Local: # DATABASE_URL="postgresql://wraith:wraith@localhost:5432/wraith" # DIRECT_DATABASE_URL="postgresql://wraith:wraith@localhost:5432/wraith" diff --git a/docs/DEPLOY_MAINNET.md b/docs/DEPLOY_MAINNET.md new file mode 100644 index 000000000..7316b3e59 --- /dev/null +++ b/docs/DEPLOY_MAINNET.md @@ -0,0 +1,80 @@ +# Mainnet Deployment Guide + +Deploying Wraith on mainnet requires a more robust infrastructure setup than testnet due to the lack of free public resources and the need for high availability. + +When running both testnet and mainnet indexers, it is recommended to run them side-by-side as separate deployments (with separate databases) rather than combining them, ensuring one network's issues don't impact the other. A single process *can* index both by setting `NETWORKS=testnet,mainnet` — see [DUAL_NETWORK.md](./DUAL_NETWORK.md) for the trade-offs. + +## 1. Database (Managed Postgres) + +**Critical:** Do *not* use Render's free PostgreSQL for a mainnet deployment. Render deletes free databases after 90 days, which will destroy your indexed state. + +- Use a durable managed Postgres provider such as **Neon** (or Supabase/AWS RDS). +- Supply the connection string in your environment variables as `DATABASE_URL`. +- Set `DIRECT_DATABASE_URL` as well as `DATABASE_URL`. `prisma/schema.prisma` declares `directUrl = env("DIRECT_DATABASE_URL")`, and schema sync on boot fails without it. On a pooled provider, `DATABASE_URL` is the pooler endpoint and `DIRECT_DATABASE_URL` the direct one; on a non-pooled provider both hold the same value. +- Ensure you have a backup strategy. See the [backup and restore runbook](./backup-restore.md) for instructions on how to back up your indexed data. + +## 2. Soroban RPC Provider + +Unlike testnet, Stellar mainnet does not provide a free public Soroban RPC endpoint suitable for a production indexer. + +- You must configure an external provider (e.g., Validation Cloud, Ankr) or run your own self-hosted RPC. +- Provide the endpoint URL in the `SOROBAN_RPC_URL` environment variable. +- *Note:* Never commit your real RPC URL containing API keys to version control. Use the Render environment secrets instead. + +## 3. Render Hosting & Keep-Alive + +Wraith can run on Render's free web services, but with two important caveats: + +1. **Sleep mode:** Render spins down free web services after 15 minutes of inactivity. For an indexer, sleeping means missing blocks. + - **Solution:** Use a service like **UptimeRobot** to ping your deployment's `/status` endpoint every 5 minutes to prevent it from sleeping. +2. **Usage caps:** Render has a 750 instance-hours/month free tier cap. This is roughly enough for exactly *one* always-on web service. If you are running both testnet and mainnet deployments on the free tier, one of them will run out of hours before the month ends unless you upgrade to a paid tier. + +## 4. Environment Variables Matrix + +Below is the environment configuration matrix for running Testnet vs. Mainnet side-by-side. Set these as secrets in your hosting provider. + +> **Set `SAC_CONTRACT_IDS` explicitly on mainnet — do not rely on the built-in default.** +> The native XLM SAC address is derived from the network passphrase, so it differs per +> network, and the value the code falls back to when `SAC_CONTRACT_IDS` is unset is not +> currently correct for mainnet (tracked separately). Derive the address yourself and +> paste the result: +> +> ```js +> import { Asset, Networks } from '@stellar/stellar-sdk'; +> Asset.native().contractId(Networks.PUBLIC); // mainnet +> Asset.native().contractId(Networks.TESTNET); // testnet +> ``` + +### Mainnet Deployment +```env +# Network Configuration +STELLAR_NETWORK=mainnet + +# RPC Provider (Required for mainnet - use your actual provider URL with API key) +SOROBAN_RPC_URL=https://mainnet.stellar.validationcloud.io/v1/ + +# Mainnet Native XLM SAC Contract ID +# = Asset.native().contractId(Networks.PUBLIC) +SAC_CONTRACT_IDS=CAS3J7GYLGXMF6TDJBBYYSE3HQ6BBSMLNUQ34T6TZMYMW2EVH34XOWMA + +# External Durable Database (e.g., Neon) +DATABASE_URL=postgresql://[user]:[password]@[host]/[dbname]?sslmode=require +DIRECT_DATABASE_URL=postgresql://[user]:[password]@[host]/[dbname]?sslmode=require +``` + +### Testnet Deployment +```env +# Network Configuration +STELLAR_NETWORK=testnet + +# RPC Provider (Optional - defaults to public testnet RPC) +SOROBAN_RPC_URL= + +# Testnet Native XLM SAC Contract ID +# = Asset.native().contractId(Networks.TESTNET) +SAC_CONTRACT_IDS=CDLZFC3SYJYDZT7K67VZ75HPJVIEUVNIXF47ZG2FB2RMQQVU2HHGCYSC + +# Database +DATABASE_URL=postgresql://[user]:[password]@[host]/[dbname]?sslmode=require +DIRECT_DATABASE_URL=postgresql://[user]:[password]@[host]/[dbname]?sslmode=require +``` From cb3e70e9c63571f010fe5b9e88613d19f9334d4d Mon Sep 17 00:00:00 2001 From: Jemimah Date: Wed, 30 Sep 2026 16:05:55 +0100 Subject: [PATCH 04/15] fix: narrow XDR error classification (#209) --- src/__tests__/fetchEventsSafe.test.ts | 168 +++++++++++++++++++++++++- src/__tests__/metrics.test.ts | 24 ++++ src/metrics.ts | 32 +++++ src/rpc.ts | 60 +++++++-- 4 files changed, 274 insertions(+), 10 deletions(-) diff --git a/src/__tests__/fetchEventsSafe.test.ts b/src/__tests__/fetchEventsSafe.test.ts index 587dd74c5..477cf5fa6 100644 --- a/src/__tests__/fetchEventsSafe.test.ts +++ b/src/__tests__/fetchEventsSafe.test.ts @@ -1,4 +1,5 @@ -import { fetchEventsSafe } from '../rpc' +import { fetchEventsSafe, isXdrError } from '../rpc' +import { registry, _resetMetrics } from '../metrics' // Minimal mock event — only the fields fetchEvents maps from the RPC response. // We pass pre-shaped RawEvent objects via our injected fetchFn, bypassing the @@ -24,6 +25,12 @@ function networkError(): Error { return new Error('Network timeout') } +// The SDK's XDR codec reports "unknown member for value " when a +// ledger uses a type the compiled-in enum does not know yet. +function unknownXdrTypeError(): Error { + return new Error('unknown SCValType member for value 9') +} + // Convenience: build a mock fetchFn from a sequence of responses function mockFetch(...calls: Array<() => Promise>) { let i = 0 @@ -109,6 +116,59 @@ describe('fetchEventsSafe — bisection algorithm', () => { await expect(fetchEventsSafe(100, 100, [], 10_000, fetch as any)).rejects.toThrow('Network timeout') }) + // "unknown" alone is not a decode failure. Matching it made every transient + // error indistinguishable from an undecodable ledger, and the indexer answered + // by skipping the ledger and moving the cursor past it — permanent, silent + // data loss on what is usually a retryable blip. + it('does not treat a bare "unknown error" as an XDR error', async () => { + const fetch = jest.fn().mockRejectedValue(new Error('unknown error')) + + await expect(fetchEventsSafe(100, 100, [], 10_000, fetch as any)).rejects.toThrow('unknown error') + expect(fetch).toHaveBeenCalledTimes(1) + }) + + it('does not bisect a range that failed with "unknown error"', async () => { + const fetch = jest.fn().mockRejectedValue(new Error('unknown error')) + + await expect(fetchEventsSafe(100, 105, [], 10_000, fetch as any)).rejects.toThrow('unknown error') + // One call: bisecting would have retried the halves of the range. + expect(fetch).toHaveBeenCalledTimes(1) + }) + + it('does not treat an unknown-host DNS failure as an XDR error', async () => { + const fetch = jest.fn().mockRejectedValue( + new Error('getaddrinfo ENOTFOUND rpc.example.com unknown host') + ) + + await expect(fetchEventsSafe(100, 100, [], 10_000, fetch as any)).rejects.toThrow('ENOTFOUND') + }) + + it('does not treat a provider 5xx body containing "unknown" as an XDR error', async () => { + const fetch = jest.fn().mockRejectedValue( + new Error('Request failed with status code 503: upstream returned an unknown error') + ) + + await expect(fetchEventsSafe(100, 105, [], 10_000, fetch as any)).rejects.toThrow('503') + expect(fetch).toHaveBeenCalledTimes(1) + }) + + it('still bisects and skips a ledger the SDK cannot decode at all', async () => { + // The SDK reports a newer XDR type as an unknown enum member rather than + // mentioning XDR by name; that is the case the skip path exists for. + const fetch = mockFetch( + () => Promise.reject(unknownXdrTypeError()), // full 100–104 + () => Promise.reject(unknownXdrTypeError()), // lower 100–102 + () => Promise.resolve({ events: [makeEvent(100, 'e1'), makeEvent(101, 'e2')], latestLedger: 101 }), + () => Promise.reject(unknownXdrTypeError()), // single 102–102 (skipped) + () => Promise.resolve({ events: [makeEvent(103, 'e3'), makeEvent(104, 'e4')], latestLedger: 104 }) + ) + + const result = await fetchEventsSafe(100, 104, [], 10_000, fetch as any) + + expect(result.events.map(e => e.ledger)).toEqual([100, 101, 103, 104]) + expect(result.highestLedger).toBe(104) + }) + it('forwards contractIds to every fetchFn call', async () => { const fetch = jest.fn().mockResolvedValue({ events: [], latestLedger: 100 }) const contracts = ['CABC123', 'CDEF456'] @@ -120,3 +180,109 @@ describe('fetchEventsSafe — bisection algorithm', () => { expect(fetch).toHaveBeenCalledWith(100, contracts, 5_000, undefined) }) }) + +// ── Skipped-ledger metric ───────────────────────────────────────────────────── +// A skipped ledger is permanent data loss, and until now the only trace of one +// was a console.warn line that scrolls off the terminal. +describe('fetchEventsSafe — ledgers_skipped_total', () => { + beforeEach(() => { + _resetMetrics() + }) + + // prom-client omits a labelled series entirely until it is first observed, so + // "no skips" is the absence of the series, not a zero. + function exposition(): Promise { + return registry.metrics() + } + + it('counts each ledger actually skipped', async () => { + // 100–101: full → XDR, then both single ledgers → XDR → both skipped. + const fetch = jest.fn().mockRejectedValue(xdrError()) + + const result = await fetchEventsSafe(100, 101, [], 10_000, fetch as any, 'testnet') + + expect(result.events).toHaveLength(0) + expect(result.highestLedger).toBe(101) + expect(await exposition()).toContain('ledgers_skipped_total{network="testnet"} 2') + }) + + it('does not count a bisection that never gives up a ledger', async () => { + // full → XDR, both halves decode cleanly: the range was searched, not skipped. + const fetch = mockFetch( + () => Promise.reject(xdrError()), + () => Promise.resolve({ events: [makeEvent(100, 'e1')], latestLedger: 102 }), + () => Promise.resolve({ events: [makeEvent(104, 'e2')], latestLedger: 105 }) + ) + + const result = await fetchEventsSafe(100, 105, [], 10_000, fetch as any, 'testnet') + + expect(result.events).toHaveLength(2) + expect(await exposition()).not.toContain('ledgers_skipped_total{') + }) + + it('counts a single skipped ledger inside a range once, not once per bisection', async () => { + // 100–104 with only ledger 102 bad: three bisections, one skip. + const fetch = mockFetch( + () => Promise.reject(xdrError()), // full 100–104 + () => Promise.reject(xdrError()), // lower 100–102 + () => Promise.resolve({ events: [makeEvent(100, 'e1'), makeEvent(101, 'e2')], latestLedger: 101 }), + () => Promise.reject(xdrError()), // single 102–102 (skipped) + () => Promise.resolve({ events: [makeEvent(103, 'e3'), makeEvent(104, 'e4')], latestLedger: 104 }) + ) + + await fetchEventsSafe(100, 104, [], 10_000, fetch as any, 'testnet') + + expect(await exposition()).toContain('ledgers_skipped_total{network="testnet"} 1') + }) + + it('counts nothing when a non-XDR error propagates', async () => { + const fetch = jest.fn().mockRejectedValue(new Error('unknown error')) + + await expect(fetchEventsSafe(100, 100, [], 10_000, fetch as any, 'testnet')).rejects.toThrow('unknown error') + + expect(await exposition()).not.toContain('ledgers_skipped_total{') + }) + + it('labels skips by network and never by ledger', async () => { + const fetch = jest.fn().mockRejectedValue(xdrError()) + + await fetchEventsSafe(7_000_000, 7_000_000, [], 10_000, fetch as any, 'mainnet') + + const text = await exposition() + expect(text).toContain('ledgers_skipped_total{network="mainnet"} 1') + expect(text).not.toContain('7000000') + }) +}) + +// ── The predicate itself ────────────────────────────────────────────────────── +describe('isXdrError', () => { + it.each([ + 'Failed to decode XDR: unknown type', + 'unknown SCValType member for value 9', + 'Bad union switch: 12', + 'invalid XDR contract typecast - source buffer not entirely consumed', + 'attempt to read outside the boundary of the buffer', + 'exceeded max decoding depth', + '5 is not a value of any member of SCValType', + ])('recognises the decoder’s own failure: %s', (message) => { + expect(isXdrError(new Error(message))).toBe(true) + }) + + it.each([ + 'unknown error', + 'getaddrinfo ENOTFOUND rpc.example.com unknown host', + 'Network timeout', + 'Request failed with status code 503: upstream returned an unknown error', + 'ledger 102 unknown', + 'ENOTFOUND', + ])('rejects a non-decode error: %s', (message) => { + expect(isXdrError(new Error(message))).toBe(false) + }) + + it('handles a thrown non-Error without throwing itself', () => { + expect(isXdrError('Failed to decode XDR')).toBe(true) + expect(isXdrError('unknown error')).toBe(false) + expect(isXdrError(undefined)).toBe(false) + expect(isXdrError(null)).toBe(false) + }) +}) diff --git a/src/__tests__/metrics.test.ts b/src/__tests__/metrics.test.ts index dae66d617..af34574ab 100644 --- a/src/__tests__/metrics.test.ts +++ b/src/__tests__/metrics.test.ts @@ -6,6 +6,7 @@ import { lastIndexedLedger, observeDbQuery, recordRpcError, + recordSkippedLedger, registry, _resetMetrics, } from "../metrics"; @@ -58,6 +59,7 @@ describe("Prometheus metrics (#39)", () => { const res = await request(app).get("/metrics"); expect(res.text).toContain("# TYPE ledgers_indexed_total counter"); + expect(res.text).toContain("# TYPE ledgers_skipped_total counter"); expect(res.text).toContain("# TYPE transfers_stored_total counter"); expect(res.text).toContain("# TYPE rpc_errors_total counter"); expect(res.text).toContain("# TYPE last_indexed_ledger gauge"); @@ -130,6 +132,28 @@ describe("Prometheus metrics (#39)", () => { }); }); + describe("recordSkippedLedger", () => { + it("counts skipped ledgers per network, with no ledger-sized label", async () => { + // A skipped ledger is permanent loss, and the ledger number is the value + // that would blow the series count up — the counter has to answer without + // it, or an operator ends up paying for a cardinality accident. + recordSkippedLedger("testnet"); + recordSkippedLedger("testnet"); + recordSkippedLedger("mainnet"); + + const res = await request(app).get("/metrics"); + + expect(res.text).toContain('ledgers_skipped_total{network="testnet"} 2'); + expect(res.text).toContain('ledgers_skipped_total{network="mainnet"} 1'); + + const skipped = res.text + .split("\n") + .filter((line) => line.startsWith("ledgers_skipped_total{")); + expect(skipped).toHaveLength(2); + expect(skipped.join("\n")).not.toMatch(/ledger=|contract=|address=|url=/); + }); + }); + describe("GET /status", () => { it("reports last_indexed_ledger alongside the camelCase field", async () => { const res = await request(app).get("/status"); diff --git a/src/metrics.ts b/src/metrics.ts index 15ea399b8..72decc34d 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -12,6 +12,7 @@ * between tests without clobbering theirs. */ import { Counter, Gauge, Histogram, Registry, collectDefaultMetrics } from "prom-client"; +import type { Network } from "./network"; export const registry = new Registry(); @@ -62,6 +63,26 @@ export const rpcErrorsTotal = new Counter({ registers: [registry], }); +/** + * Ledgers the indexer gave up on, per network. + * + * A skip is the one failure the indexer cannot recover from: the ledger's events + * are never written and the cursor has already advanced past it, so the gap is + * permanent. It only happens for a genuine XDR decode failure, which means any + * non-zero value deserves a look, and a `rate()` over it says whether the gap is + * still growing. + * + * Labelled by network only. The ledger number would be the interesting value and + * is deliberately absent: one series per skipped ledger is unbounded cardinality, + * and the counter answers the same question without it. + */ +export const ledgersSkippedTotal = new Counter({ + name: "ledgers_skipped_total", + help: "Ledgers skipped by the indexer after an XDR decode failure, per network", + labelNames: ["network"] as const, + registers: [registry], +}); + /** * The highest ledger the indexer has committed, per network. * @@ -110,6 +131,17 @@ export function recordRpcError(outcome: "retry" | "exhausted"): void { rpcErrorsTotal.inc({ outcome }); } +/** + * Record one ledger abandoned after a genuine XDR decode failure. + * + * Called only where a ledger is actually given up on, never for the bisection + * that looks for it — a bisection is a search, and counting it would inflate the + * count by the depth of the search. + */ +export function recordSkippedLedger(network: Network): void { + ledgersSkippedTotal.inc({ network }); +} + /** Serialize the registry in Prometheus text exposition format. */ export function renderMetrics(): Promise { return registry.metrics(); diff --git a/src/rpc.ts b/src/rpc.ts index 9063904f5..35b9de2d4 100644 --- a/src/rpc.ts +++ b/src/rpc.ts @@ -1,6 +1,6 @@ import { rpc as RPC, xdr, scValToNative, Contract, TransactionBuilder, Account, Networks } from "@stellar/stellar-sdk"; import { resolveNetwork, currentNetwork, type Network } from "./network"; -import { recordRpcError } from "./metrics"; +import { recordRpcError, recordSkippedLedger } from "./metrics"; // ─── Network config ─────────────────────────────────────────────────────────── const TESTNET_RPC_URL = "https://soroban-testnet.stellar.org"; @@ -179,6 +179,49 @@ export async function withRetry( } // ─── XDR-safe event fetch ───────────────────────────────────────────────────── +/** + * Markers of a genuine XDR/decode failure. + * + * Every entry is a message the XDR codec itself produces, or a phrase that only + * makes sense as a decode complaint. The old check also matched the bare word + * "unknown", which is what a *provider* says when it has no better answer — + * `unknown error`, `getaddrinfo ENOTFOUND … unknown host`, a 5xx body. Those + * are transient, and skipping a ledger on one loses that ledger's events for + * good while the cursor moves on. + */ +const XDR_ERROR_PATTERNS: readonly RegExp[] = [ + // "Failed to decode XDR: …", "invalid XDR contract typecast …", and the + // base64/hex envelopes the SDK wraps decoded payloads in. + /\bxdr\b/i, + // js-xdr reading a discriminant the compiled-in type definitions do not know: + // "unknown SCValType member for value 9". The exact case that motivates the + // skip path — a ledger using an enum arm added after this SDK was built. + /unknown\s+\S+\s+member\s+for\s+value/i, + /is not a value of any member of/i, + // The rest of the decoder's own complaints, none of which a transport or a + // provider can produce. + /bad union switch/i, + /not a member of\s+\S+/i, + /attempt to read outside the boundary of the buffer/i, + /exceeded max decoding depth/i, + /source invalid:/i, + /invalid padding/i, + /invalid i32 value/i, + /invalid u32 value/i, +]; + +/** + * Whether an error from the event fetch is a genuine XDR/decode failure — the + * only kind `fetchEventsSafe` is allowed to answer by skipping a ledger. + * + * Everything else (timeouts, DNS, 5xx, auth) propagates, so the caller's own + * retry and alerting see it instead of a hole in the index. + */ +export function isXdrError(err: unknown): boolean { + const msg = err instanceof Error ? err.message : String(err ?? ""); + return XDR_ERROR_PATTERNS.some((pattern) => pattern.test(msg)); +} + /** * Like fetchEvents but handles XDR decode errors gracefully. * @@ -205,12 +248,12 @@ export async function fetchEventsSafe( const { events, latestLedger } = await _fetchFn(startLedger, contractIds, limit, network); return { events, highestLedger: Math.max(startLedger, latestLedger) }; } catch (err) { - const msg = (err as Error).message ?? ""; - if (msg.includes("XDR") || msg.includes("unknown")) { - console.warn(`[rpc] Skipping ledger ${startLedger} — XDR decode error: ${msg}`); - return { events: [], highestLedger: startLedger }; - } - throw err; + if (!isXdrError(err)) throw err; + console.warn(`[rpc] Skipping ledger ${startLedger} — XDR decode error: ${(err as Error).message}`); + // A skip is a hole in the index that nothing ever fills in, so it counts: + // the console line scrolls away, this does not. + recordSkippedLedger(resolveNetwork(network)); + return { events: [], highestLedger: startLedger }; } } @@ -218,8 +261,7 @@ export async function fetchEventsSafe( const { events, latestLedger } = await _fetchFn(startLedger, contractIds, limit, network); return { events, highestLedger: latestLedger }; } catch (err) { - const msg = (err as Error).message ?? ""; - if (!msg.includes("XDR") && !msg.includes("unknown")) throw err; + if (!isXdrError(err)) throw err; // Bisect: try lower half, then upper half console.warn(`[rpc] XDR error in ledgers ${startLedger}–${endLedger}, bisecting…`); From 0bf26d3793e9a064b3c06f242b2e44650aeb5274 Mon Sep 17 00:00:00 2001 From: blockchain-maxis Date: Wed, 30 Sep 2026 19:02:43 +0100 Subject: [PATCH 05/15] fix(offramp): require a bearer token to read an order (#216) * fix(offramp): require a bearer token to read an order GET /offramp/orders/:orderId served any order to whoever held its id: the bank payout amount, the deposit address and the rate. Orders now get a random public id (ofr_ + 128 bits) in place of the provider's, and the creator is handed a bearer token (oft_ + 256 bits) once, in the create response. Only its SHA-256 is stored. Lookups need Authorization: Bearer; a missing header is 401, and an unknown id, a malformed id and a wrong token are all the same 404. Failed lookups are rate-limited separately from the app-wide limiter (10 per 15 minutes per IP by default); successful polling does not count. Replaying a creation with the same idempotencyKey now also needs the same walletAddress (409 otherwise) and re-issues the token, since only its hash is kept. The lookup response no longer names the provider (source: live) or returns its id, and a database failure returns a generic 500 instead of reaching the global handler. * test(offramp): use a valid strkey for the second wallet fixture GBBB...SAM was 56 characters but failed StrKey.isValidEd25519PublicKey. The test passes either way today, because the route does not validate the address, but a fixture that is not a real strkey breaks the moment it is handed to anything that decodes one. Claude-Session: https://claude.ai/code/session_01USgemLt4Rnz4SGB1Srf3GB --------- Co-authored-by: blockchain-maxis <267648998+blockchain-maxis@users.noreply.github.com> Co-authored-by: Miracle656 --- README.md | 2 + docs/offramp-orders.md | 54 +++ prisma/schema.prisma | 11 +- src/__tests__/linq/orderAccess.test.ts | 89 +++++ src/__tests__/routes/offramp.test.ts | 436 +++++++++++++++++++++++++ src/api/offramp.ts | 147 +++++++-- src/linq/orderAccess.ts | 74 +++++ 7 files changed, 777 insertions(+), 36 deletions(-) create mode 100644 docs/offramp-orders.md create mode 100644 src/__tests__/linq/orderAccess.test.ts create mode 100644 src/__tests__/routes/offramp.test.ts create mode 100644 src/linq/orderAccess.ts diff --git a/README.md b/README.md index 2c2b4e52c..0aa8d902d 100644 --- a/README.md +++ b/README.md @@ -331,6 +331,8 @@ console.log(data); Base URL: `http://localhost:3000` +Offramp order lookups need a bearer token; see [docs/offramp-orders.md](docs/offramp-orders.md). + ### Selecting a network Wraith stores testnet and mainnet rows in the same tables, discriminated by a diff --git a/docs/offramp-orders.md b/docs/offramp-orders.md new file mode 100644 index 000000000..e012a54bc --- /dev/null +++ b/docs/offramp-orders.md @@ -0,0 +1,54 @@ +# Offramp order access + +An offramp order carries a bank payout amount, a deposit address and a rate, so +reading one back requires proof of having created it. + +## Creating an order + +`POST /offramp/orders` returns two values the client must keep: + +| Field | Example | Notes | +| ------------- | -------------------------------------------- | ----- | +| `id` | `ofr_` + 22 URL-safe characters (128 bits) | The id to look the order up by. Not the provider's id, which is never returned. | +| `accessToken` | `oft_` + 43 URL-safe characters (256 bits) | Bearer credential. Shown once; only its SHA-256 is stored. | + +The response is sent with `Cache-Control: no-store`. + +## Reading an order + +```bash +curl -H "Authorization: Bearer $ACCESS_TOKEN" \ + https:///offramp/orders/$ORDER_ID +``` + +| Status | Meaning | +| ------ | ------- | +| `200` | The token belongs to this order. | +| `401` | No usable `Authorization: Bearer` header. Identical for every id. | +| `404` | Unknown id, malformed id, or a token that does not belong to the order. These are deliberately indistinguishable. | +| `429` | Too many failed lookups from this IP. | + +Only failed lookups count toward the limit, so polling your own order never +consumes it. Defaults: 10 failures per 15 minutes per IP. + +| Variable | Default | Purpose | +| ------------------------------ | -------- | ------- | +| `OFFRAMP_LOOKUP_MAX_FAILURES` | `10` | Failed lookups allowed per window. | +| `OFFRAMP_LOOKUP_WINDOW_MS` | `900000` | Window length. | + +## Retrying a creation + +Replaying `POST /offramp/orders` with the same `idempotencyKey` and +`walletAddress` returns the same order (`replayed: true`) with the same `id` and +a **new** `accessToken`; the previous token stops working. A different +`walletAddress` gets `409` and nothing about the order. Idempotency keys should +therefore be random and generated by the client. + +## Upgrading + +- Orders created before this change have no `id` or token and cannot be read + through `GET /offramp/orders/:orderId`. Replaying their creation request + (same key and wallet) issues them credentials. +- The lookup response now reports `source: "live"` instead of `"linq"`. +- Two nullable columns were added to `OfframpOrder` (`publicId`, + `accessTokenHash`); run `npx prisma db push`. diff --git a/prisma/schema.prisma b/prisma/schema.prisma index b9e78b18e..e81d75863 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -384,8 +384,16 @@ model OfframpOrder { id Int @id @default(autoincrement()) network String @default("testnet") - // Linq's order id — the join key for webhooks and status polls. + // Linq's order id — the join key for webhooks and status polls. Never sent to + // clients: see publicId. orderId String + // The id clients see and look an order up by: random and unguessable, unlike a + // provider id we do not control the shape of. Null on rows written before + // orders were access-controlled; those cannot be read through the API. + publicId String? + // SHA-256 of the bearer token handed to the creator. Only the hash is kept, so + // a database read does not yield a working credential. Null as above. + accessTokenHash String? // Ours, sent as idempotencyKey. Never regenerated on retry, or the retry // becomes a second order that also has to be paid for. idempotencyKey String @@ -417,6 +425,7 @@ model OfframpOrder { // Per network: an order id is unique within a chain, and the same wallet can // offramp on both. @@unique([network, orderId]) + @@unique([network, publicId]) @@unique([network, idempotencyKey]) @@index([network, walletAddress]) @@index([network, status]) diff --git a/src/__tests__/linq/orderAccess.test.ts b/src/__tests__/linq/orderAccess.test.ts new file mode 100644 index 000000000..2c193ea49 --- /dev/null +++ b/src/__tests__/linq/orderAccess.test.ts @@ -0,0 +1,89 @@ +/** + * An order id and its token are the only things between a stranger and someone's + * bank payout details, so these pin the format and the entropy, not just that + * the functions run. + */ +import type { Request } from "express"; +import { + bearerToken, + generateAccessToken, + generateOrderId, + hashAccessToken, + isOrderId, + issueOrderCredentials, + tokenMatches, +} from "../../linq/orderAccess"; + +const reqWith = (authorization?: string) => + ({ headers: authorization === undefined ? {} : { authorization } }) as Request; + +describe("order ids", () => { + it("are ofr_ followed by 22 url-safe characters", () => { + expect(generateOrderId()).toMatch(/^ofr_[A-Za-z0-9_-]{22}$/); + }); + + it("carry 128 bits of randomness", () => { + const raw = Buffer.from(generateOrderId().slice("ofr_".length), "base64url"); + expect(raw.length).toBe(16); + }); + + it("do not repeat", () => { + const ids = new Set(Array.from({ length: 5000 }, generateOrderId)); + expect(ids.size).toBe(5000); + }); + + it("are recognised by isOrderId, and nothing else is", () => { + expect(isOrderId(generateOrderId())).toBe(true); + expect(isOrderId("3f7c1b2a-84e9-4c11-b3d2-0a9f7e123456")).toBe(false); + expect(isOrderId("ofr_short")).toBe(false); + expect(isOrderId(`${generateOrderId()}x`)).toBe(false); + expect(isOrderId(undefined)).toBe(false); + expect(isOrderId(42)).toBe(false); + }); +}); + +describe("access tokens", () => { + it("are oft_ followed by 43 url-safe characters, 256 bits", () => { + const token = generateAccessToken(); + expect(token).toMatch(/^oft_[A-Za-z0-9_-]{43}$/); + expect(Buffer.from(token.slice("oft_".length), "base64url").length).toBe(32); + }); + + it("are stored as a hash that is not the token", () => { + const { token, tokenHash } = issueOrderCredentials(); + expect(tokenHash).toMatch(/^[0-9a-f]{64}$/); + expect(tokenHash).not.toContain(token); + expect(tokenHash).toBe(hashAccessToken(token)); + }); + + it("match only their own hash", () => { + const a = issueOrderCredentials(); + const b = issueOrderCredentials(); + expect(tokenMatches(a.token, a.tokenHash)).toBe(true); + expect(tokenMatches(a.token, b.tokenHash)).toBe(false); + }); + + it("never match a row that has no hash", () => { + const { token } = issueOrderCredentials(); + expect(tokenMatches(token, null)).toBe(false); + expect(tokenMatches(token, undefined)).toBe(false); + expect(tokenMatches(token, "")).toBe(false); + }); +}); + +describe("bearerToken", () => { + const token = generateAccessToken(); + + it("reads a well-formed Authorization header", () => { + expect(bearerToken(reqWith(`Bearer ${token}`))).toBe(token); + expect(bearerToken(reqWith(`bearer ${token}`))).toBe(token); + }); + + it("is null when the header is missing, another scheme, or not a token we issue", () => { + expect(bearerToken(reqWith())).toBeNull(); + expect(bearerToken(reqWith(`Basic ${token}`))).toBeNull(); + expect(bearerToken(reqWith("Bearer"))).toBeNull(); + expect(bearerToken(reqWith("Bearer not-a-token"))).toBeNull(); + expect(bearerToken(reqWith(`Bearer ${token} extra`))).toBeNull(); + }); +}); diff --git a/src/__tests__/routes/offramp.test.ts b/src/__tests__/routes/offramp.test.ts new file mode 100644 index 000000000..dfaf3a7d7 --- /dev/null +++ b/src/__tests__/routes/offramp.test.ts @@ -0,0 +1,436 @@ +/** + * GET /offramp/orders/:orderId used to hand any caller holding an order id the + * bank payout amount, the deposit address and the rate. These tests pin who can + * read an order, and what each kind of refusal is allowed to reveal. + */ +import express from "express"; +import request from "supertest"; + +process.env.LINQ_API_KEY = "test-key"; + +type Row = Record; +const rows: Row[] = []; +let nextId = 1; + +const mockFindUnique = jest.fn(async ({ where }: { where: Record }) => { + const [key, value] = Object.entries(where)[0] as [string, Row]; + const fields = key.split("_"); + return rows.find((r) => fields.every((f) => r[f] === value[f])) ?? null; +}); + +jest.mock("../../db", () => ({ + prisma: { + offrampOrder: { + findUnique: (args: any) => mockFindUnique(args), + create: jest.fn(async ({ data }: { data: Row }) => { + const row = { id: nextId++, createdAt: new Date("2026-01-01T00:00:00Z"), ...data }; + rows.push(row); + return row; + }), + update: jest.fn(async ({ where, data }: { where: Row; data: Row }) => { + const row = where.id + ? rows.find((r) => r.id === where.id) + : rows.find( + (r) => + r.network === where.network_orderId.network && + r.orderId === where.network_orderId.orderId, + ); + Object.assign(row!, data); + return row; + }), + }, + }, +})); + +jest.mock("../../linq/client", () => ({ + ...jest.requireActual("../../linq/client"), + createOfframpOrder: jest.fn(), + getOfframpStatus: jest.fn(), +})); + +import { createOfframpRouter } from "../../api/offramp"; +import { networkMiddleware } from "../../middleware/network"; +import { LinqError, createOfframpOrder, getOfframpStatus } from "../../linq/client"; + +const mockCreate = createOfframpOrder as jest.Mock; +const mockStatus = getOfframpStatus as jest.Mock; + +const LINQ_ID = "3f7c1b2a-84e9-4c11-b3d2-0a9f7e123456"; +const WALLET = "GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAWHF"; + +function makeApp() { + const app = express(); + app.use(express.json()); + app.use(networkMiddleware); + app.use("/offramp", createOfframpRouter()); + return app; +} + +const body = (over: Record = {}) => ({ + amountStableCoin: 50, + bankAccount: "0123456789", + bankCode: "058", + bankName: "Test Bank", + accountName: "Ada Lovelace", + walletAddress: WALLET, + idempotencyKey: "key-1", + ...over, +}); + +let linqSeq = 0; +function linqOrder(over: Record = {}) { + return { + id: over.id ?? (linqSeq++ === 0 ? LINQ_ID : `linq-order-${linqSeq}`), + walletAddress: "GDEPOSITADDRESS", + coinType: "usdc", + coin: "usdc", + chain: "stellar", + amountStableCoin: 50, + amountNGN: 82750, + rate: 1655, + currency: "NGN", + status: "initiated", + ...over, + }; +} + +async function place(app: express.Express, over: Record = {}) { + const res = await request(app).post("/offramp/orders").send(body(over)); + return res; +} + +beforeEach(() => { + rows.length = 0; + nextId = 1; + linqSeq = 0; + mockCreate.mockReset(); + mockStatus.mockReset(); + mockFindUnique.mockClear(); + mockCreate.mockImplementation(async () => linqOrder()); + mockStatus.mockImplementation(async (id: string) => ({ + id, + status: "initiated", + amountStableCoin: 50, + amountNGN: 82750, + currency: "NGN", + created: "2026-01-01T00:00:00Z", + updated: "2026-01-01T00:00:00Z", + })); + delete process.env.OFFRAMP_LOOKUP_MAX_FAILURES; +}); + +const auth = (token: string) => ({ Authorization: `Bearer ${token}` }); + +describe("creating an order", () => { + it("returns an unguessable id and an access token instead of the provider's id", async () => { + const res = await place(makeApp()); + + expect(res.status).toBe(201); + expect(res.body.id).toMatch(/^ofr_[A-Za-z0-9_-]{22}$/); + expect(res.body.accessToken).toMatch(/^oft_[A-Za-z0-9_-]{43}$/); + expect(JSON.stringify(res.body)).not.toContain(LINQ_ID); + expect(res.headers["cache-control"]).toBe("no-store"); + }); + + it("stores the token's hash, never the token", async () => { + const res = await place(makeApp()); + + expect(rows).toHaveLength(1); + expect(rows[0].publicId).toBe(res.body.id); + expect(rows[0].orderId).toBe(LINQ_ID); + expect(JSON.stringify(rows)).not.toContain(res.body.accessToken); + expect(rows[0].accessTokenHash).toMatch(/^[0-9a-f]{64}$/); + }); + + it("gives every order its own id and token", async () => { + const app = makeApp(); + const a = await place(app, { idempotencyKey: "key-a" }); + const b = await place(app, { idempotencyKey: "key-b" }); + + expect(a.body.id).not.toBe(b.body.id); + expect(a.body.accessToken).not.toBe(b.body.accessToken); + }); +}); + +describe("reading an order", () => { + it("lets the creator read their own order", async () => { + const app = makeApp(); + const created = await place(app); + + const res = await request(app) + .get(`/offramp/orders/${created.body.id}`) + .set(auth(created.body.accessToken)); + + expect(res.status).toBe(200); + expect(res.body.id).toBe(created.body.id); + expect(res.body.depositAddress).toBe("GDEPOSITADDRESS"); + expect(res.body.rate).toBe(1655); + // The provider is asked about its own id, but is never named or exposed. + expect(mockStatus).toHaveBeenCalledWith(LINQ_ID); + expect(res.headers["cache-control"]).toBe("no-store"); + expect(JSON.stringify(res.body)).not.toContain(LINQ_ID); + expect(JSON.stringify(res.body).toLowerCase()).not.toContain("linq"); + }); + + it("refuses a valid order id with no token, and reveals nothing about the order", async () => { + const app = makeApp(); + const created = await place(app); + mockFindUnique.mockClear(); + + const res = await request(app).get(`/offramp/orders/${created.body.id}`); + + expect(res.status).toBe(401); + expect(res.headers["www-authenticate"]).toBe("Bearer"); + expect(JSON.stringify(res.body)).not.toContain("GDEPOSITADDRESS"); + expect(mockStatus).not.toHaveBeenCalled(); + expect(mockFindUnique).not.toHaveBeenCalled(); + }); + + it("refuses the wrong token", async () => { + const app = makeApp(); + const created = await place(app); + const other = await place(app, { idempotencyKey: "key-2" }); + + const res = await request(app) + .get(`/offramp/orders/${created.body.id}`) + .set(auth(other.body.accessToken)); + + expect(res.status).toBe(404); + expect(JSON.stringify(res.body)).not.toContain("GDEPOSITADDRESS"); + expect(mockStatus).not.toHaveBeenCalled(); + }); + + it("answers a wrong token, an unknown id and a malformed id identically", async () => { + const app = makeApp(); + const created = await place(app); + const stranger = await place(app, { idempotencyKey: "key-2" }); + + const wrongToken = await request(app) + .get(`/offramp/orders/${created.body.id}`) + .set(auth(stranger.body.accessToken)); + const unknownId = await request(app) + .get(`/offramp/orders/ofr_${"A".repeat(22)}`) + .set(auth(created.body.accessToken)); + const malformed = await request(app) + .get(`/offramp/orders/${LINQ_ID}`) + .set(auth(created.body.accessToken)); + + expect(wrongToken.status).toBe(404); + for (const other of [unknownId, malformed]) { + expect(other.status).toBe(wrongToken.status); + expect(other.body).toEqual(wrongToken.body); + } + }); + + it("does not accept the provider's id as a lookup key", async () => { + const app = makeApp(); + const created = await place(app); + + const res = await request(app) + .get(`/offramp/orders/${LINQ_ID}`) + .set(auth(created.body.accessToken)); + + expect(res.status).toBe(404); + }); + + it("cannot read an order on another network", async () => { + const app = makeApp(); + const created = await place(app); + rows[0].network = "mainnet"; + + const res = await request(app) + .get(`/offramp/orders/${created.body.id}`) + .set(auth(created.body.accessToken)); + + expect(res.status).toBe(404); + }); + + it("keeps orders that predate access control unreadable", async () => { + const app = makeApp(); + rows.push({ + id: nextId++, + network: "testnet", + orderId: LINQ_ID, + publicId: null, + accessTokenHash: null, + depositAddress: "GDEPOSITADDRESS", + createdAt: new Date(), + }); + + const byProviderId = await request(app) + .get(`/offramp/orders/${LINQ_ID}`) + .set(auth(`oft_${"A".repeat(43)}`)); + + expect(byProviderId.status).toBe(404); + }); + + it("does not read a row that has an id but no stored token", async () => { + const app = makeApp(); + const id = `ofr_${"B".repeat(22)}`; + rows.push({ id: nextId++, network: "testnet", orderId: LINQ_ID, publicId: id, accessTokenHash: null }); + + const res = await request(app) + .get(`/offramp/orders/${id}`) + .set(auth(`oft_${"A".repeat(43)}`)); + + expect(res.status).toBe(404); + }); + + it("serves the stored order without leaking the failure when the provider is down", async () => { + const app = makeApp(); + const created = await place(app); + mockStatus.mockRejectedValue(new LinqError("Linq is down: key lq_live_secret rejected", 503)); + + const res = await request(app) + .get(`/offramp/orders/${created.body.id}`) + .set(auth(created.body.accessToken)); + + expect(res.status).toBe(200); + expect(res.body.source).toBe("cache"); + const text = JSON.stringify(res.body).toLowerCase(); + expect(text).not.toContain("linq"); + expect(text).not.toContain("lq_live_secret"); + }); + + it("never returns or logs a raw database error", async () => { + const app = makeApp(); + const created = await place(app); + const secret = "connection refused postgres://user:hunter2@db.internal/wraith"; + mockFindUnique.mockRejectedValueOnce(new Error(secret)); + const logged = jest.spyOn(console, "error").mockImplementation(() => {}); + + const res = await request(app) + .get(`/offramp/orders/${created.body.id}`) + .set(auth(created.body.accessToken)); + + expect(res.status).toBe(500); + expect(res.text).not.toContain("hunter2"); + expect(JSON.stringify(logged.mock.calls)).not.toContain("hunter2"); + logged.mockRestore(); + }); +}); + +describe("brute-forcing lookups", () => { + it("rate-limits repeated failed lookups", async () => { + process.env.OFFRAMP_LOOKUP_MAX_FAILURES = "3"; + const app = makeApp(); + const created = await place(app); + const guess = () => + request(app) + .get(`/offramp/orders/ofr_${"C".repeat(22)}`) + .set(auth(created.body.accessToken)); + + expect((await guess()).status).toBe(404); + expect((await guess()).status).toBe(404); + expect((await guess()).status).toBe(404); + const limited = await guess(); + + expect(limited.status).toBe(429); + expect(limited.body.error).toMatch(/too many/i); + }); + + it("counts lookups with no token as failures too", async () => { + process.env.OFFRAMP_LOOKUP_MAX_FAILURES = "2"; + const app = makeApp(); + const created = await place(app); + + await request(app).get(`/offramp/orders/${created.body.id}`); + await request(app).get(`/offramp/orders/${created.body.id}`); + const res = await request(app).get(`/offramp/orders/${created.body.id}`); + + expect(res.status).toBe(429); + }); + + it("does not spend the budget on the creator polling their own order", async () => { + process.env.OFFRAMP_LOOKUP_MAX_FAILURES = "3"; + const app = makeApp(); + const created = await place(app); + + for (let i = 0; i < 10; i++) { + const res = await request(app) + .get(`/offramp/orders/${created.body.id}`) + .set(auth(created.body.accessToken)); + expect(res.status).toBe(200); + } + }); + + it("limits the token check as well, and a good token is still refused once the budget is gone", async () => { + process.env.OFFRAMP_LOOKUP_MAX_FAILURES = "2"; + const app = makeApp(); + const created = await place(app); + const stranger = `oft_${"D".repeat(43)}`; + + await request(app).get(`/offramp/orders/${created.body.id}`).set(auth(stranger)); + await request(app).get(`/offramp/orders/${created.body.id}`).set(auth(stranger)); + const res = await request(app) + .get(`/offramp/orders/${created.body.id}`) + .set(auth(created.body.accessToken)); + + expect(res.status).toBe(429); + }); +}); + +describe("retrying an order creation", () => { + it("re-issues the token, keeps the id, and does not create a second order", async () => { + const app = makeApp(); + const first = await place(app); + const retry = await place(app); + + expect(retry.status).toBe(200); + expect(retry.body.replayed).toBe(true); + expect(retry.body.id).toBe(first.body.id); + expect(retry.body.accessToken).toMatch(/^oft_[A-Za-z0-9_-]{43}$/); + expect(retry.body.accessToken).not.toBe(first.body.accessToken); + expect(mockCreate).toHaveBeenCalledTimes(1); + expect(rows).toHaveLength(1); + + // The token from the response that may never have arrived no longer works. + const stale = await request(makeApp()) + .get(`/offramp/orders/${first.body.id}`) + .set(auth(first.body.accessToken)); + const fresh = await request(makeApp()) + .get(`/offramp/orders/${first.body.id}`) + .set(auth(retry.body.accessToken)); + expect(stale.status).toBe(404); + expect(fresh.status).toBe(200); + }); + + it("gives nothing away to another wallet reusing the key", async () => { + const app = makeApp(); + await place(app); + + const res = await place(app, { walletAddress: "GBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBKOH" }); + + expect(res.status).toBe(409); + expect(res.text).not.toContain("GDEPOSITADDRESS"); + expect(res.body.accessToken).toBeUndefined(); + expect(rows[0].accessTokenHash).toBeDefined(); + }); + + it("adopts an order that predates access control instead of leaving it stuck", async () => { + const app = makeApp(); + rows.push({ + id: nextId++, + network: "testnet", + orderId: LINQ_ID, + idempotencyKey: "key-1", + walletAddress: WALLET, + publicId: null, + accessTokenHash: null, + depositAddress: "GDEPOSITADDRESS", + chain: "stellar", + coin: "usdc", + amountStableCoin: "50", + amountNGN: "82750", + rate: "1655", + status: "initiated", + }); + + const res = await place(app); + + expect(res.status).toBe(200); + expect(res.body.id).toMatch(/^ofr_[A-Za-z0-9_-]{22}$/); + expect(rows[0].publicId).toBe(res.body.id); + expect(JSON.stringify(res.body)).not.toContain(LINQ_ID); + }); +}); diff --git a/src/api/offramp.ts b/src/api/offramp.ts index a96a03c95..5a0809d54 100644 --- a/src/api/offramp.ts +++ b/src/api/offramp.ts @@ -1,8 +1,15 @@ import { Router, Request, Response, NextFunction } from "express"; +import rateLimit, { type RateLimitRequestHandler } from "express-rate-limit"; import { prisma } from "../db"; import { requestNetwork } from "../middleware/network"; import { statusForClients } from "../linq/statusForClients"; +import { + bearerToken, + isOrderId, + issueOrderCredentials, + tokenMatches, +} from "../linq/orderAccess"; import { LinqError, checkStellarTrustline, @@ -60,8 +67,31 @@ function sendLinqError(res: Response, err: unknown): void { res.status(500).json({ error: "Offramp request failed" }); } -export function createOfframpRouter(): Router { +/** + * Limits how many lookups a client may get wrong. + * + * Only failed requests count (`skipSuccessfulRequests`), so a wallet polling its + * own order every few seconds never spends the budget, while someone trying ids + * or tokens runs out after a handful of misses. Separate from the app-wide + * limiter, which is far too generous for a credential check and is switched off + * under test. + */ +export function createOrderLookupLimiter(): RateLimitRequestHandler { + return rateLimit({ + windowMs: parseInt(process.env.OFFRAMP_LOOKUP_WINDOW_MS ?? "900000", 10), + limit: parseInt(process.env.OFFRAMP_LOOKUP_MAX_FAILURES ?? "10", 10), + skipSuccessfulRequests: true, + standardHeaders: true, + legacyHeaders: false, + message: { error: "Too many failed order lookups, please try again later." }, + }); +} + +export function createOfframpRouter( + options: { lookupLimiter?: RateLimitRequestHandler } = {}, +): Router { const router = Router(); + const lookupLimiter = options.lookupLimiter ?? createOrderLookupLimiter(); // Every route needs the key. Answering 503 rather than 500 says "this // deployment has no offramp", which is what a client gating a CTA needs to @@ -165,29 +195,45 @@ export function createOfframpRouter(): Router { return; } - // The key comes from the caller and is never generated here: regenerating - // it on a retry is precisely how one order becomes two, and the second one - // also gets paid for. - const existing = await prisma.offrampOrder.findUnique({ - where: { network_idempotencyKey: { network: net, idempotencyKey } }, - }); - if (existing) { - res.status(200).json({ - id: existing.orderId, - walletAddress: existing.depositAddress, - chain: existing.chain, - coin: existing.coin, - amountStableCoin: Number(existing.amountStableCoin), - amountNGN: Number(existing.amountNGN), - rate: Number(existing.rate), - status: statusForClients(existing.status), - providerStatus: existing.status, - replayed: true, + try { + // The key comes from the caller and is never generated here: regenerating + // it on a retry is precisely how one order becomes two, and the second one + // also gets paid for. + const existing = await prisma.offrampOrder.findUnique({ + where: { network_idempotencyKey: { network: net, idempotencyKey } }, }); - return; - } + if (existing) { + // The key alone is not proof of ownership; the wallet that placed the + // order has to match too, and nothing about the order is said otherwise. + if (existing.walletAddress !== walletAddress) { + res.status(409).json({ error: "This idempotencyKey has already been used" }); + return; + } + // A retry usually means the first response never arrived, and with it the + // access token. Only its hash is stored, so a new one is issued and the + // old one stops working. + const creds = issueOrderCredentials(existing.publicId ?? undefined); + await prisma.offrampOrder.update({ + where: { id: existing.id }, + data: { publicId: creds.orderId, accessTokenHash: creds.tokenHash }, + }); + res.setHeader("Cache-Control", "no-store"); + res.status(200).json({ + id: creds.orderId, + accessToken: creds.token, + walletAddress: existing.depositAddress, + chain: existing.chain, + coin: existing.coin, + amountStableCoin: Number(existing.amountStableCoin), + amountNGN: Number(existing.amountNGN), + rate: Number(existing.rate), + status: statusForClients(existing.status), + providerStatus: existing.status, + replayed: true, + }); + return; + } - try { const order = await createOfframpOrder({ ...(amountNGN != null ? { amountNGN: Number(amountNGN) } : {}), ...(amountStableCoin != null @@ -202,6 +248,10 @@ export function createOfframpRouter(): Router { idempotencyKey, }); + // The provider's id never leaves the server: the client is given our own, + // and a token that proves it created the order. + const creds = issueOrderCredentials(); + // Persisted before responding. A row that only exists after the client // has been told the deposit address is a row that can be missing while // the user is already sending funds against it. @@ -209,6 +259,8 @@ export function createOfframpRouter(): Router { data: { network: net, orderId: order.id, + publicId: creds.orderId, + accessTokenHash: creds.tokenHash, idempotencyKey, walletAddress, chain: order.chain ?? "stellar", @@ -221,33 +273,56 @@ export function createOfframpRouter(): Router { }, }); - res.status(201).json(order); + res.setHeader("Cache-Control", "no-store"); + res.status(201).json({ ...order, id: creds.orderId, accessToken: creds.token }); } catch (err) { sendLinqError(res, err); } }); - router.get("/orders/:orderId", async (req: Request, res: Response) => { + router.get("/orders/:orderId", lookupLimiter, async (req: Request, res: Response) => { const net = requestNetwork(req); const { orderId } = req.params; + res.setHeader("Cache-Control", "no-store"); + + // No credential at all is the one refusal that says so. It is the same for + // every id, so it tells a caller nothing about which ones exist. + const token = bearerToken(req); + if (!token) { + res.setHeader("WWW-Authenticate", "Bearer"); + res.status(401).json({ error: "An order access token is required" }); + return; + } + + let row; + try { + row = isOrderId(orderId) + ? await prisma.offrampOrder.findUnique({ + where: { network_publicId: { network: net, publicId: orderId } }, + }) + : null; + } catch (err) { + console.error("[offramp] order lookup failed:", (err as Error)?.name); + res.status(500).json({ error: "Offramp request failed" }); + return; + } - const row = await prisma.offrampOrder.findUnique({ - where: { network_orderId: { network: net, orderId } }, - }); - if (!row) { + // An id that does not exist and an id the token does not belong to get the + // same answer, or the difference would list the orders that are real. + if (!row || !tokenMatches(token, row.accessTokenHash)) { res.status(404).json({ error: "Order not found" }); return; } - // Reconciled against Linq rather than served from our row alone. The + // Reconciled against the provider rather than served from our row alone. The // webhook is a push we might have missed - a sleeping instance, a failed // delivery - and a wallet showing "waiting for deposit" against an order // that settled twenty minutes ago is worse than a slow answer. try { - const live = await getOfframpStatus(orderId); + const live = await getOfframpStatus(row.orderId); if (live.status !== row.status) { await prisma.offrampOrder.update({ - where: { network_orderId: { network: net, orderId } }, + where: { network_orderId: { network: net, orderId: row.orderId } }, data: { status: live.status, settledStableCoin: String(live.amountStableCoin), @@ -257,6 +332,7 @@ export function createOfframpRouter(): Router { } res.json({ ...live, + id: row.publicId, status: statusForClients(live.status), providerStatus: live.status, depositAddress: row.depositAddress, @@ -265,13 +341,14 @@ export function createOfframpRouter(): Router { // "₦0 / USDC" and a countdown with no start, which reads as expired. rate: Number(row.rate), createdAt: row.createdAt.toISOString(), - source: "linq", + source: "live", }); } catch { - // Linq unreachable: our row is stale but true as of the last update, and - // saying so beats failing the request outright. + // Provider unreachable: our row is stale but true as of the last update, and + // saying so beats failing the request outright. Nothing of the upstream + // failure is passed on. res.json({ - id: row.orderId, + id: row.publicId, status: statusForClients(row.status), providerStatus: row.status, amountStableCoin: Number(row.settledStableCoin ?? row.amountStableCoin), diff --git a/src/linq/orderAccess.ts b/src/linq/orderAccess.ts new file mode 100644 index 000000000..acdeecdd1 --- /dev/null +++ b/src/linq/orderAccess.ts @@ -0,0 +1,74 @@ +import { createHash, randomBytes, timingSafeEqual } from "crypto"; +import type { Request } from "express"; + +/** + * Access control for offramp orders. + * + * An order exposes the bank payout amount, the deposit address and the rate, so + * reading one back needs proof of having created it. The proof is a bearer token + * issued once, in the response that creates the order. + */ + +const ORDER_ID_PREFIX = "ofr_"; +const TOKEN_PREFIX = "oft_"; + +/** 128 bits: too many to enumerate, however fast the caller is allowed to ask. */ +const ORDER_ID_BYTES = 16; +/** 256 bits. It is a credential, so it gets the full width of the hash behind it. */ +const TOKEN_BYTES = 32; + +// base64url of 16 bytes is 22 characters, of 32 bytes is 43. +const ORDER_ID_RE = /^ofr_[A-Za-z0-9_-]{22}$/; +const TOKEN_RE = /^oft_[A-Za-z0-9_-]{43}$/; + +export interface OrderCredentials { + /** Identifier the client looks the order up by. */ + orderId: string; + /** Bearer token, returned to the creator once and never stored. */ + token: string; + /** What is persisted in place of the token. */ + tokenHash: string; +} + +export function generateOrderId(): string { + return ORDER_ID_PREFIX + randomBytes(ORDER_ID_BYTES).toString("base64url"); +} + +export function generateAccessToken(): string { + return TOKEN_PREFIX + randomBytes(TOKEN_BYTES).toString("base64url"); +} + +export function hashAccessToken(token: string): string { + return createHash("sha256").update(token).digest("hex"); +} + +export function issueOrderCredentials(orderId = generateOrderId()): OrderCredentials { + const token = generateAccessToken(); + return { orderId, token, tokenHash: hashAccessToken(token) }; +} + +/** True only for an id this service could have issued. */ +export function isOrderId(value: unknown): value is string { + return typeof value === "string" && ORDER_ID_RE.test(value); +} + +/** The token from `Authorization: Bearer `, or null when absent or malformed. */ +export function bearerToken(req: Request): string | null { + const header = req.headers.authorization; + if (typeof header !== "string") return null; + const match = /^Bearer +(\S+)$/i.exec(header.trim()); + if (!match || !TOKEN_RE.test(match[1])) return null; + return match[1]; +} + +// Compared against when there is no stored hash, so a lookup for an id that does +// not exist takes as long as one with the wrong token. +const ABSENT_HASH = hashAccessToken("absent"); + +/** Constant-time check of a presented token against the stored hash. */ +export function tokenMatches(token: string, storedHash: string | null | undefined): boolean { + const expected = Buffer.from(storedHash ?? ABSENT_HASH, "hex"); + const actual = Buffer.from(hashAccessToken(token), "hex"); + const equal = expected.length === actual.length && timingSafeEqual(expected, actual); + return equal && !!storedHash; +} From 36da363df50d3ab287a689bf1d49ae954ff53e2b Mon Sep 17 00:00:00 2001 From: Apulupie <167634780+Frun1na@users.noreply.github.com> Date: Thu, 1 Oct 2026 02:19:53 +0800 Subject: [PATCH 06/15] fix: skip malformed events in parseEvents instead of wedging the indexer (#197) Co-authored-by: Miracle656 --- README.md | 1 + src/__tests__/decoder.test.ts | 31 ++++++++++++++++++++++++++++++- src/decoder.ts | 22 +++++++++++++++++++--- src/metrics.ts | 13 +++++++++++++ 4 files changed, 63 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0aa8d902d..73fb0b2fb 100644 --- a/README.md +++ b/README.md @@ -439,6 +439,7 @@ last_indexed_ledger{network="mainnet"} 5842100 | `ledgers_indexed_total` | counter | `network` | Ledgers the indexer has advanced through. A flat `rate()` on a network whose loop should be running means it has stalled. | | `transfers_stored_total` | counter | `network`, `type` | Rows persisted, split `fungible` / `nft` — one parse path can break while the other keeps working. | | `rpc_errors_total` | counter | `outcome` | Failed RPC attempts. `retry` counts attempts `withRetry` absorbed, `exhausted` counts calls that gave up — a degrading endpoint shows up in `retry` long before it fails a call. | +| `events_skipped_total` | counter | `reason` | Events the decoder dropped. `unrecognised` is non-token events (normal); `malformed` is token events that failed to decode — a rising rate points at a non-standard contract or a decoder bug. | | `last_indexed_ledger` | gauge | `network` | Highest committed ledger. Against the chain tip, this is lag. | | `db_query_duration_seconds` | histogram | `operation` | Duration of instrumented DB operations, failures included. | diff --git a/src/__tests__/decoder.test.ts b/src/__tests__/decoder.test.ts index d5663958e..9d8abd3a6 100644 --- a/src/__tests__/decoder.test.ts +++ b/src/__tests__/decoder.test.ts @@ -1,5 +1,5 @@ import { xdr } from "@stellar/stellar-sdk"; -import { parseEvent } from "../decoder"; +import { parseEvent, parseEvents } from "../decoder"; import * as fixtures from "./fixtures/events.json"; describe("Soroban XDR Decoder", () => { @@ -107,3 +107,32 @@ describe("Soroban XDR Decoder", () => { expect(result).toBeNull(); }); }); + +describe("parseEvents", () => { + const common = { + ledger: 100, + ledgerClosedAt: "2024-01-01T00:00:00Z", + contractId: fixtures.contractId, + txHash: "abc123txhash", + type: "contract", + }; + const good = (id: string) => ({ + ...common, + id, + topic: fixtures.transfer.topic.map((t) => xdr.ScVal.fromXDR(t, "base64")), + value: xdr.ScVal.fromXDR(fixtures.transfer.value, "base64"), + }); + // A "transfer" with only the symbol topic: parseEvent throws on this. + const malformed = { + ...common, + id: "0000000000000000001-00002", + topic: [xdr.ScVal.fromXDR(fixtures.transfer.topic[0], "base64")], + value: xdr.ScVal.fromXDR(fixtures.transfer.value, "base64"), + }; + + it("skips a malformed event instead of failing the batch", () => { + expect(() => parseEvent(malformed)).toThrow(); + const records = parseEvents([good("0000000000000000001-00001"), malformed, good("0000000000000000001-00003")]); + expect(records).toHaveLength(2); + }); +}); diff --git a/src/decoder.ts b/src/decoder.ts index 79a80d8f3..9bd2b9820 100644 --- a/src/decoder.ts +++ b/src/decoder.ts @@ -1,6 +1,7 @@ import * as StellarSdk from "@stellar/stellar-sdk"; import type { RawEvent } from "./rpc"; import type { TransferRecord } from "./db"; +import { eventsSkippedTotal } from "./metrics"; // ─── Recognised event types ─────────────────────────────────────────────────── // SEP-41 / CAP-67 standard topic[0] symbols @@ -140,15 +141,25 @@ export function parseEvent( } /** - * Parse a batch of raw events, silently skipping unrecognised or malformed ones. - * Logs a count of skipped events at debug level. + * Parse a batch of raw events, skipping unrecognised or malformed ones. + * + * `parseEvent` throws on malformed token events; one such event must not fail + * the whole batch, or the indexer retries the same ledger range forever. */ export function parseEvents(rawEvents: RawEvent[]): TransferRecord[] { const records: TransferRecord[] = []; let skipped = 0; + let failed = 0; for (const raw of rawEvents) { - const record = parseEvent(raw); + let record: TransferRecord | null; + try { + record = parseEvent(raw); + } catch (err) { + failed++; + console.warn(`[parser] Failed to decode event ${raw.id}: ${(err as Error).message}`); + continue; + } if (record) { records.push(record); } else { @@ -157,8 +168,13 @@ export function parseEvents(rawEvents: RawEvent[]): TransferRecord[] { } if (skipped > 0) { + eventsSkippedTotal.inc({ reason: "unrecognised" }, skipped); console.debug(`[parser] Skipped ${skipped} non-token events out of ${rawEvents.length}`); } + if (failed > 0) { + eventsSkippedTotal.inc({ reason: "malformed" }, failed); + console.warn(`[parser] Skipped ${failed} malformed events out of ${rawEvents.length}`); + } return records; } diff --git a/src/metrics.ts b/src/metrics.ts index 72decc34d..f072332f4 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -83,6 +83,19 @@ export const ledgersSkippedTotal = new Counter({ registers: [registry], }); +/** + * Events dropped by the decoder, by reason. + * + * `malformed` means a token event that failed to decode; a rising rate points + * at a contract emitting non-standard events or a decoder bug. + */ +export const eventsSkippedTotal = new Counter({ + name: "events_skipped_total", + help: "Events skipped by the decoder", + labelNames: ["reason"] as const, + registers: [registry], +}); + /** * The highest ledger the indexer has committed, per network. * From d04392dc94780086fd3ba5d8a29231f0441e34cf Mon Sep 17 00:00:00 2001 From: Tobiz Date: Wed, 30 Sep 2026 19:24:34 +0100 Subject: [PATCH 07/15] fix(indexer): share one per-batch pipeline between the single and parallel ingest paths (#219) Co-authored-by: DevTobis <232918735+DevTobis@users.noreply.github.com> --- .env.example | 6 + src/__tests__/ingestParity.test.ts | 295 +++++++++++++++++++++++++++++ src/indexer.ts | 175 +++++------------ src/indexer/batch.ts | 157 +++++++++++++++ src/indexer/parallel.ts | 46 +++-- 5 files changed, 535 insertions(+), 144 deletions(-) create mode 100644 src/__tests__/ingestParity.test.ts create mode 100644 src/indexer/batch.ts diff --git a/.env.example b/.env.example index ce45f480e..8aedcc150 100644 --- a/.env.example +++ b/.env.example @@ -95,6 +95,12 @@ NFT_CONTRACT_IDS_MAINNET= # Max events per RPC call (Stellar RPC hard-caps at 10000) EVENTS_BATCH_SIZE=10000 +# Shard each poll window across N workers (by contract) for throughput. This +# changes how fast a window is ingested, never what is indexed: both paths run +# every record through the same pipeline (transfers, NFTs and their metadata, +# LP shares, host-fn logs, account summaries). Default 1 = no sharding. +# INGEST_WORKERS=1 + # Delete transfers older than N days to stay within DB free-tier limits (default: 30) RETENTION_DAYS=30 diff --git a/src/__tests__/ingestParity.test.ts b/src/__tests__/ingestParity.test.ts new file mode 100644 index 000000000..4631896b8 --- /dev/null +++ b/src/__tests__/ingestParity.test.ts @@ -0,0 +1,295 @@ +/** + * Ingest path parity (#203). + * + * `INGEST_WORKERS > 1` used to switch to a parallel path that silently skipped + * NFT transfers, NFT metadata, LP shares, host-fn logs, SAC tagging, token + * metadata and account summaries. Nothing errored; the data just stopped + * arriving. The deliverable here is the comparison itself: the same events + * pushed through the single-fetch path and the sharded path must leave + * identical rows behind. + * + * The db layer is replaced by an in-memory store so this stays a unit test. + * The fake event source honours the contract filter it is given, which is what + * makes a path that forgets to request the NFT contract lose its events. + */ + +import { StrKey, Address, nativeToScVal } from "@stellar/stellar-sdk"; +import type { RawEvent } from "../rpc"; + +type Row = Record; + +// `mock`-prefixed so jest's hoisting allows the factories below to close over it. +const mockStore = { + transfers: new Map(), + nftTransfers: new Map(), + nftMetadata: new Map(), + summaries: new Map(), + hostFn: new Map(), + lp: new Map(), + tokenMetadataLookups: new Set(), + cursor: 0, +}; + +function mockResetStore() { + mockStore.transfers.clear(); + mockStore.nftTransfers.clear(); + mockStore.nftMetadata.clear(); + mockStore.summaries.clear(); + mockStore.hostFn.clear(); + mockStore.lp.clear(); + mockStore.tokenMetadataLookups.clear(); + mockStore.cursor = 0; +} + +jest.mock("../db", () => ({ + prisma: {}, + getLastIndexedLedger: jest.fn(async () => null), + setLastIndexedLedger: jest.fn(async (n: number) => { mockStore.cursor = n; }), + pruneOldTransfers: jest.fn(async () => 0), + upsertTransfers: jest.fn(async (records: Row[]) => { + let added = 0; + for (const r of records) { + if (mockStore.transfers.has(String(r.eventId))) continue; + mockStore.transfers.set(String(r.eventId), { ...r }); + added++; + } + return added; + }), + // Aggregated per (address, contract), so the result does not depend on the + // order or grouping of batches, only on which records reached it. + upsertAccountSummaries: jest.fn(async (records: Row[]) => { + for (const r of records) { + for (const [address, side] of [[r.toAddress, "in"], [r.fromAddress, "out"]] as const) { + if (!address) continue; + const key = `${address}:${r.contractId}`; + const s = mockStore.summaries.get(key) ?? { received: 0n, sent: 0n, txCount: 0 }; + if (side === "in") s.received += BigInt(String(r.amount)); + else s.sent += BigInt(String(r.amount)); + s.txCount++; + mockStore.summaries.set(key, s); + } + } + }), + upsertNftTransfers: jest.fn(async (records: Row[]) => { + let added = 0; + for (const r of records) { + if (mockStore.nftTransfers.has(String(r.eventId))) continue; + mockStore.nftTransfers.set(String(r.eventId), { ...r }); + added++; + } + return added; + }), + getNftMetadata: jest.fn(async (contractId: string, tokenId: string) => + mockStore.nftMetadata.get(`${contractId}:${tokenId}`) ?? null), + upsertNftMetadata: jest.fn(async (contractId: string, tokenId: string, meta: Row) => { + mockStore.nftMetadata.set(`${contractId}:${tokenId}`, { contractId, tokenId, ...meta }); + }), +})); + +jest.mock("../indexer/host-fn-log", () => ({ + ...jest.requireActual("../indexer/host-fn-log"), + upsertHostFnLogs: jest.fn(async (rows: Row[]) => { + for (const r of rows) mockStore.hostFn.set(String(r.eventId), { ...r, args: JSON.stringify(r.args), result: JSON.stringify(r.result), gasUsed: null }); + }), +})); + +jest.mock("../indexer/lp-shares", () => ({ + ...jest.requireActual("../indexer/lp-shares"), + loadKnownLpPools: jest.fn(async () => []), + upsertLpShareTransfers: jest.fn(async (rows: Row[]) => { + let added = 0; + for (const r of rows) { + if (mockStore.lp.has(String(r.eventId))) continue; + mockStore.lp.set(String(r.eventId), { ...r }); + added++; + } + return added; + }), +})); + +// SAC detection would hit RPC. Tag deterministically instead, still mutating +// the records so a path that skips tagging leaves a visible difference. +jest.mock("../indexer/sac-detect", () => ({ + tagSacTransfers: jest.fn(async (records: Array<{ contractId: string; isSac?: boolean }>) => { + for (const r of records) r.isSac = r.contractId === mockSacA; + }), +})); +let mockSacA = ""; + +jest.mock("../tokenCache", () => ({ + initTokenCache: jest.fn(async () => undefined), + getTokenMetadata: jest.fn(async (contractId: string) => { + mockStore.tokenMetadataLookups.add(contractId); + return undefined; + }), +})); + +jest.mock("../ingester/nft", () => ({ + ...jest.requireActual("../ingester/nft"), + fetchNftMetadata: jest.fn(async (contractId: string) => ({ name: `name-of-${contractId.slice(-4)}` })), +})); + +import * as batch from "../indexer/batch"; +import { pollParallel } from "../indexer/parallel"; +import { createLoopState, ingestWindow, type LoopState } from "../indexer"; + +// ─── Fixtures (all strkeys built from raw bytes, so valid by construction) ──── +const contract = (n: number) => StrKey.encodeContract(Buffer.alloc(32, n)); +const account = (n: number) => StrKey.encodeEd25519PublicKey(Buffer.alloc(32, n)); + +const SAC_A = contract(1); +const SAC_B = contract(2); +const POOL = contract(3); +const NFT_C = contract(4); +const ADMIN = contract(5); +const ALICE = account(1); +const BOB = account(2); +mockSacA = SAC_A; + +let seq = 0; +function ev(contractId: string, ledger: number, topic: RawEvent["topic"], value: RawEvent["value"]): RawEvent { + seq++; + return { + id: `${String(ledger).padStart(19, "0")}-${String(seq).padStart(5, "0")}`, + type: "contract", + ledger, + ledgerClosedAt: "2024-01-01T00:00:00Z", + contractId, + txHash: `tx-${seq}`, + topic, + value, + }; +} +const sym = (s: string) => nativeToScVal(s, { type: "symbol" }); +const addr = (a: string) => Address.fromString(a).toScVal(); +const i128 = (n: bigint) => nativeToScVal(n, { type: "i128" }); + +const EVENTS: RawEvent[] = [ + ev(SAC_A, 10, [sym("transfer"), addr(ALICE), addr(BOB)], i128(100n)), + ev(SAC_B, 11, [sym("transfer"), addr(BOB), addr(ALICE)], i128(50n)), + ev(SAC_B, 12, [sym("mint"), addr(ADMIN), addr(ALICE)], i128(70n)), + ev(NFT_C, 13, [sym("transfer"), addr(ALICE), addr(BOB), nativeToScVal(42n, { type: "u128" })], nativeToScVal(null)), + ev(NFT_C, 14, [sym("transfer"), addr(ALICE), addr(BOB), nativeToScVal(43n, { type: "u128" })], nativeToScVal(null)), + ev(NFT_C, 15, [sym("transfer"), addr(BOB), addr(ALICE), nativeToScVal(42n, { type: "u128" })], nativeToScVal(null)), + ev(POOL, 16, [sym("deposit"), addr(ALICE)], i128(1000n)), +]; + +function fakeSource(): LoopState["sourceSwitcher"] { + return { + // Honours the contract filter: an event for a contract that was not asked + // for is never returned, exactly like the real RPC. + fetchEvents: async (from: number, to: number, ids: string[]) => ({ + events: EVENTS.filter((e) => ids.includes(e.contractId) && e.ledger >= from && e.ledger <= to), + highestLedger: to, + }), + getLatestLedger: async () => 1000, + } as unknown as LoopState["sourceSwitcher"]; +} + +function newLoop(): LoopState { + process.env.SAC_CONTRACT_IDS_TESTNET = [SAC_A, SAC_B, POOL].join(","); + process.env.NFT_CONTRACT_IDS_TESTNET = NFT_C; + const loop = createLoopState("testnet"); + loop.sourceSwitcher = fakeSource(); + return loop; +} + +/** Everything a path leaves behind, in a comparable, order-independent form. */ +function snapshot() { + const sorted = (m: Map) => + [...m.entries()].sort(([a], [b]) => a.localeCompare(b)); + return { + transfers: sorted(mockStore.transfers), + nftTransfers: sorted(mockStore.nftTransfers), + nftMetadata: sorted(mockStore.nftMetadata), + summaries: sorted(mockStore.summaries as unknown as Map), + hostFn: sorted(mockStore.hostFn), + lp: sorted(mockStore.lp), + tokenMetadataLookups: [...mockStore.tokenMetadataLookups].sort(), + cursor: mockStore.cursor, + }; +} + +beforeEach(() => { + mockResetStore(); + seq = 0; +}); + +describe("ingest path parity", () => { + it("fixtures are valid strkeys", () => { + for (const c of [SAC_A, SAC_B, POOL, NFT_C, ADMIN]) expect(StrKey.isValidContract(c)).toBe(true); + for (const a of [ALICE, BOB]) expect(StrKey.isValidEd25519PublicKey(a)).toBe(true); + }); + + it("the single path indexes every record type from the fixture", async () => { + const loop = newLoop(); + await ingestWindow(loop, 1, 100, 1); + const s = snapshot(); + + expect(s.transfers).toHaveLength(3); + expect(s.nftTransfers).toHaveLength(3); + expect(s.nftMetadata).toHaveLength(2); // tokens 42 and 43, de-duplicated + expect(s.summaries.length).toBeGreaterThan(0); + expect(s.hostFn).toHaveLength(EVENTS.length); + expect(s.lp).toHaveLength(1); + expect(s.tokenMetadataLookups).toEqual([SAC_A, SAC_B].sort()); + expect(s.cursor).toBe(100); + }); + + it("INGEST_WORKERS > 1 leaves identical TokenTransfer, NftTransfer, AccountSummary (and every other) rows", async () => { + const single = newLoop(); + await ingestWindow(single, 1, 100, 1); + const singleSnap = snapshot(); + const singleIndexed = single.totalIndexed; + + mockResetStore(); + const sharded = newLoop(); + await ingestWindow(sharded, 1, 100, 4); + + expect(snapshot()).toEqual(singleSnap); + expect(sharded.totalIndexed).toBe(singleIndexed); + }); + + it("both paths run the events through the same processEventBatch", async () => { + const spy = jest.spyOn(batch, "processEventBatch"); + + await ingestWindow(newLoop(), 1, 100, 1); + const singleCalls = spy.mock.calls.length; + expect(singleCalls).toBeGreaterThan(0); + + spy.mockClear(); + mockResetStore(); + await ingestWindow(newLoop(), 1, 100, 4); + expect(spy.mock.calls.length).toBeGreaterThan(0); + // Every event ends up in exactly one batch, on either path. + const seen = spy.mock.calls.flatMap((c) => c[0].map((e) => e.id)).sort(); + expect(seen).toEqual(EVENTS.map((e) => e.id).sort()); + + spy.mockRestore(); + }); + + it("the comparison is sensitive: a sharded path that skips a step is caught", async () => { + const single = newLoop(); + await ingestWindow(single, 1, 100, 1); + const singleSnap = snapshot(); + + // Emulate the old bug: a worker pool whose batch handler only stores + // fungible transfers. + mockResetStore(); + const loop = newLoop(); + await pollParallel(loop.allContractIds, 1, 100, 10_000, 4, "testnet", { + fetchEvents: (from, to, ids, limit) => loop.sourceSwitcher.fetchEvents(from, to, ids, limit), + processBatch: async (events) => { + const { upsertTransfers } = jest.requireMock("../db"); + const { parseEvents } = jest.requireActual("../decoder"); + const n = await upsertTransfers(parseEvents(events)); + return { fungibleInserted: n, nftInserted: 0, lpInserted: 0 }; + }, + }); + + const drifted = snapshot(); + expect(drifted.nftTransfers).not.toEqual(singleSnap.nftTransfers); + expect(drifted.summaries).not.toEqual(singleSnap.summaries); + expect(drifted).not.toEqual(singleSnap); + }); +}); diff --git a/src/indexer.ts b/src/indexer.ts index fbfcb0759..06fe8f320 100644 --- a/src/indexer.ts +++ b/src/indexer.ts @@ -1,31 +1,17 @@ import "dotenv/config"; import { validateNetworkConfig, withRetry } from "./rpc"; -import { parseEvents } from "./decoder"; import { - upsertTransfers, - upsertAccountSummaries, - upsertNftTransfers, - getNftMetadata, - upsertNftMetadata, getLastIndexedLedger, setLastIndexedLedger, pruneOldTransfers, } from "./db"; -import { emitTransfer, emitHostFnLog } from "./events"; -import { parseHostFnEvent, upsertHostFnLogs, type HostFnRecord } from "./indexer/host-fn-log"; -import { tagSacTransfers } from "./indexer/sac-detect"; -import { - parseLpShareEvents, - upsertLpShareTransfers, - resolveLpPoolIds, - loadKnownLpPools, -} from "./indexer/lp-shares"; +import { resolveLpPoolIds, loadKnownLpPools } from "./indexer/lp-shares"; import { pollParallel } from "./indexer/parallel"; +import { processEventBatch, batchTotal } from "./indexer/batch"; import { tombstoneExpiredContracts } from "./indexer/tombstones"; -import { isNftTransferEvent, parseNftEvents, fetchNftMetadata } from "./ingester/nft"; import { createSourceSwitcherWithConfig, type SourceSwitcher } from "./indexer/sources"; import { currentNetwork, enabledNetworks, resolveNetwork, type Network } from "./network"; -import { ledgersIndexedTotal, transfersStoredTotal, lastIndexedLedger } from "./metrics"; +import { ledgersIndexedTotal, lastIndexedLedger } from "./metrics"; // ─── NFT Contract IDs ───────────────────────────────────────────────────────── /** @@ -87,7 +73,7 @@ export function resolveSacContractIds(network?: Network): string[] { // both loops at the same chain's SAC. return [net === "mainnet" ? DEFAULT_XLM_SAC_MAINNET : DEFAULT_XLM_SAC_TESTNET]; } -import { initTokenCache, getTokenMetadata } from "./tokenCache"; +import { initTokenCache } from "./tokenCache"; // ─── Config ─────────────────────────────────────────────────────────────────── // These stay process-wide: they describe how hard to poll, not which chain. @@ -128,7 +114,7 @@ const TOMBSTONE_EVERY_CYCLES = parseInt( * whose failover `preferred` field is mutable — a testnet RPC outage would * silently repoint the mainnet loop at testnet Horizon. */ -type LoopState = { +export type LoopState = { network: Network; sacContractIds: string[]; nftContractIds: string[]; @@ -159,7 +145,7 @@ type LoopState = { const loops = new Map(); /** Build the isolated state for one network's loop. */ -function createLoopState(network: Network): LoopState { +export function createLoopState(network: Network): LoopState { const sacContractIds = resolveSacContractIds(network); const nftContractIds = resolveNftContractIds(network); const suffix = network.toUpperCase(); @@ -309,95 +295,9 @@ async function pollOnce( return highestLedger; } - // Persist token transfers - // Split events by type: NFT (4 topics) vs fungible (3 topics) - const fungibleEvents = events.filter((e) => !isNftTransferEvent(e)); - const nftRawEvents = events.filter((e) => isNftTransferEvent(e)); - - // ── Fungible path ──────────────────────────────────────────────────────────── - const records = parseEvents(fungibleEvents); - // Tag each transfer with whether its contract is a SAC (#136). Best-effort: - // a detection failure must never block ingest, so default to false on error. - await tagSacTransfers(records, undefined, net).catch((e: unknown) => - console.error(`[indexer/${net}] SAC detection failed:`, e) - ); - const inserted = await upsertTransfers(records, net); - - // Resolve metadata for every distinct token in this batch. Only a cache miss - // reaches RPC, and a miss happens once per contract for the life of the - // database — so this is one extra call the first time a token is seen and - // free thereafter. Best-effort: a token whose metadata cannot be read is - // still worth indexing transfers for. - await Promise.all( - [...new Set(records.map((r) => r.contractId))].map((contractId) => - getTokenMetadata(contractId, net).catch(() => undefined) - ) - ); - loop.totalIndexed += inserted; - transfersStoredTotal.inc({ network: net, type: "fungible" }, inserted); - - // Update materialized account summaries alongside transfer inserts - if (inserted > 0) { - await upsertAccountSummaries(records, net).catch((e: unknown) => - console.error(`[indexer/${net}] Account summary upsert failed:`, e) - ); - } - - // Broadcast each new record to WebSocket subscribers - if (inserted > 0) { - records.forEach((record) => emitTransfer(record, net)); - } - - // Log every event as a raw host-fn invocation for downstream consumers (#84) - const hostFnRecords = events - .map(raw => { try { return parseHostFnEvent(raw); } catch { return null; } }) - .filter((r): r is HostFnRecord => r !== null); - if (hostFnRecords.length > 0) { - await upsertHostFnLogs(hostFnRecords, net).catch((err: unknown) => - console.error(`[indexer/${net}] host-fn log error:`, err), - ); - hostFnRecords.forEach((record) => emitHostFnLog(record, net)); - } - - // ── LP-share path ────────────────────────────────────────────────────────── - // Decode pool deposits/withdrawals as LP-share transfers tagged with the pool - // ID. Best-effort and additive: deposit/withdraw events are ignored by the - // fungible path, while a pool's own share mint/burn is recorded here in - // addition to its token-transfer row. - const lpRecords = parseLpShareEvents(events, loop.knownLpPools); - // A contract that produced an LP-share record has identified itself as a - // pool, so its bare mint/burn counts from here on. Learning this is what lets - // the bare dialect ever be accepted without an env allowlist. - for (const record of lpRecords) loop.knownLpPools.add(record.poolId); - const lpInserted = await upsertLpShareTransfers(lpRecords, net).catch((e) => { - console.error(`[indexer/${net}] LP-share upsert failed:`, e); - return 0; - }); - loop.totalIndexed += lpInserted; - - // ── NFT path ───────────────────────────────────────────────────────────────── - const nftParsed = parseNftEvents(nftRawEvents); - const nftRecords = nftParsed.map((p) => p.record); - const nftInserted = await upsertNftTransfers(nftRecords, net); - loop.totalIndexed += nftInserted; - transfersStoredTotal.inc({ network: net, type: "nft" }, nftInserted); - - // Lazy-load metadata for unique (contractId, tokenId) pairs not yet cached - if (nftParsed.length > 0) { - const seen = new Set(); - for (const { record, tokenIdScVal } of nftParsed) { - const key = `${record.contractId}:${record.tokenId}`; - if (seen.has(key)) continue; - seen.add(key); - const cached = await getNftMetadata(record.contractId, record.tokenId, net); - if (!cached) { - const meta = await fetchNftMetadata(record.contractId, tokenIdScVal, net).catch(() => ({})); - await upsertNftMetadata(record.contractId, record.tokenId, meta, net).catch((e: unknown) => - console.error(`[indexer/${net}] NFT metadata upsert failed:`, e) - ); - } - } - } + const batch = await processEventBatch(events, { network: net, knownLpPools: loop.knownLpPools }); + const { fungibleInserted: inserted, nftInserted, lpInserted } = batch; + loop.totalIndexed += batchTotal(batch); await setLastIndexedLedger(highestLedger, net); recordLedgerProgress(net, fromLedger, highestLedger); @@ -409,6 +309,45 @@ async function pollOnce( return highestLedger; } +// ─── One ingest window ──────────────────────────────────────────────────────── +/** + * Ingest ledgers `fromLedger`..`toLedger`, sequentially or sharded across + * `workers` (default `INGEST_WORKERS`). Returns the highest ledger seen. + * + * `workers` changes how fast the window is ingested, never what is indexed: + * both branches watch the same contracts (SAC and NFT), fetch through the same + * source switcher, and hand events to the same `processEventBatch` (#203). + */ +export async function ingestWindow( + loop: LoopState, + fromLedger: number, + toLedger: number, + workers: number = INGEST_WORKERS, +): Promise { + const net = loop.network; + if (workers > 1 && loop.allContractIds.length > 1) { + // Parallel path: shard contracts across N workers for higher throughput (#83) + const { totalInserted, highestLedger } = await pollParallel( + loop.allContractIds, + fromLedger, + toLedger, + BATCH_SIZE, + workers, + net, + { + fetchEvents: (from, to, contractIds, limit) => + loop.sourceSwitcher.fetchEvents(from, to, contractIds, limit), + processBatch: (events) => + processEventBatch(events, { network: net, knownLpPools: loop.knownLpPools }), + }, + ); + loop.totalIndexed += totalInserted; + recordLedgerProgress(net, fromLedger, highestLedger); + return highestLedger; + } + return pollOnce(loop, fromLedger, toLedger); +} + // ─── Main loop ──────────────────────────────────────────────────────────────── /** * Run one indexer loop for one network. Never returns. @@ -488,23 +427,7 @@ export async function startIndexer(network?: Network): Promise { continue; } - if (INGEST_WORKERS > 1 && loop.sacContractIds.length > 1) { - // Parallel path: shard contracts across N workers for higher throughput (#83) - const { totalInserted, highestLedger } = await pollParallel( - loop.sacContractIds, - currentLedger, - target, - BATCH_SIZE, - INGEST_WORKERS, - net, - ); - loop.totalIndexed += totalInserted; - transfersStoredTotal.inc({ network: net, type: "fungible" }, totalInserted); - recordLedgerProgress(net, currentLedger, highestLedger); - currentLedger = highestLedger; - } else { - currentLedger = await pollOnce(loop, currentLedger, target); - } + currentLedger = await ingestWindow(loop, currentLedger, target); // Periodic data retention cleanup loop.pollCycleCount++; diff --git a/src/indexer/batch.ts b/src/indexer/batch.ts new file mode 100644 index 000000000..3565b08ea --- /dev/null +++ b/src/indexer/batch.ts @@ -0,0 +1,157 @@ +/** + * The per-batch ingest pipeline, shared by every ingest path (#203). + * + * `pollOnce` (single fetch) and `pollParallel` (sharded fetch) used to each + * own a copy of "what to do with events", and the parallel copy quietly + * lacked the NFT, LP-share, host-fn-log, SAC-tagging, token-metadata and + * account-summary steps. `INGEST_WORKERS > 1` therefore changed *what* was + * indexed, not just how fast, with nothing erroring. + * + * Both paths now fetch events however they like and hand them to + * {@link processEventBatch}. A new per-record step is added here, once, and + * both paths get it. + * + * Parallel safety: the batches a parallel run processes concurrently hold + * disjoint contracts, and every table this writes is keyed by contract + * (`eventId` is unique per event; `AccountSummary`, `NftMetadata`, + * `TokenMetadata` include `contractId` in their key). The one piece of shared + * mutable state is `knownLpPools`, and a contract adds only itself to it, so + * two partitions never race over the same entry. Nothing here is + * parallel-unsafe. + */ + +import { parseEvents } from "../decoder"; +import { + upsertTransfers, + upsertAccountSummaries, + upsertNftTransfers, + getNftMetadata, + upsertNftMetadata, +} from "../db"; +import { emitTransfer, emitHostFnLog } from "../events"; +import { parseHostFnEvent, upsertHostFnLogs, type HostFnRecord } from "./host-fn-log"; +import { tagSacTransfers } from "./sac-detect"; +import { parseLpShareEvents, upsertLpShareTransfers } from "./lp-shares"; +import { isNftTransferEvent, parseNftEvents, fetchNftMetadata } from "../ingester/nft"; +import { getTokenMetadata } from "../tokenCache"; +import { transfersStoredTotal } from "../metrics"; +import type { RawEvent } from "../rpc"; +import type { Network } from "../network"; + +export type BatchContext = { + network: Network; + /** Contracts established as liquidity pools; grown as new pools are seen. */ + knownLpPools: Set; +}; + +export type BatchResult = { + fungibleInserted: number; + nftInserted: number; + lpInserted: number; +}; + +/** Total rows newly stored by a batch, across every record type. */ +export function batchTotal(r: BatchResult): number { + return r.fungibleInserted + r.nftInserted + r.lpInserted; +} + +/** + * Parse and persist one batch of raw events: fungible transfers (with SAC + * tagging, token metadata and account summaries), host-fn logs, LP-share + * records and NFT transfers (with metadata). + */ +export async function processEventBatch( + events: RawEvent[], + ctx: BatchContext, +): Promise { + const net = ctx.network; + const { knownLpPools } = ctx; + + // Split events by type: NFT (4 topics) vs fungible (3 topics) + const fungibleEvents = events.filter((e) => !isNftTransferEvent(e)); + const nftRawEvents = events.filter((e) => isNftTransferEvent(e)); + + // ── Fungible path ──────────────────────────────────────────────────────────── + const records = parseEvents(fungibleEvents); + // Tag each transfer with whether its contract is a SAC (#136). Best-effort: + // a detection failure must never block ingest, so default to false on error. + await tagSacTransfers(records, undefined, net).catch((e: unknown) => + console.error(`[indexer/${net}] SAC detection failed:`, e) + ); + const inserted = await upsertTransfers(records, net); + + // Resolve metadata for every distinct token in this batch. Only a cache miss + // reaches RPC, and a miss happens once per contract for the life of the + // database — so this is one extra call the first time a token is seen and + // free thereafter. Best-effort: a token whose metadata cannot be read is + // still worth indexing transfers for. + await Promise.all( + [...new Set(records.map((r) => r.contractId))].map((contractId) => + getTokenMetadata(contractId, net).catch(() => undefined) + ) + ); + transfersStoredTotal.inc({ network: net, type: "fungible" }, inserted); + + // Update materialized account summaries alongside transfer inserts + if (inserted > 0) { + await upsertAccountSummaries(records, net).catch((e: unknown) => + console.error(`[indexer/${net}] Account summary upsert failed:`, e) + ); + } + + // Broadcast each new record to WebSocket subscribers + if (inserted > 0) { + records.forEach((record) => emitTransfer(record, net)); + } + + // Log every event as a raw host-fn invocation for downstream consumers (#84) + const hostFnRecords = events + .map(raw => { try { return parseHostFnEvent(raw); } catch { return null; } }) + .filter((r): r is HostFnRecord => r !== null); + if (hostFnRecords.length > 0) { + await upsertHostFnLogs(hostFnRecords, net).catch((err: unknown) => + console.error(`[indexer/${net}] host-fn log error:`, err), + ); + hostFnRecords.forEach((record) => emitHostFnLog(record, net)); + } + + // ── LP-share path ────────────────────────────────────────────────────────── + // Decode pool deposits/withdrawals as LP-share transfers tagged with the pool + // ID. Best-effort and additive: deposit/withdraw events are ignored by the + // fungible path, while a pool's own share mint/burn is recorded here in + // addition to its token-transfer row. + const lpRecords = parseLpShareEvents(events, knownLpPools); + // A contract that produced an LP-share record has identified itself as a + // pool, so its bare mint/burn counts from here on. Learning this is what lets + // the bare dialect ever be accepted without an env allowlist. + for (const record of lpRecords) knownLpPools.add(record.poolId); + const lpInserted = await upsertLpShareTransfers(lpRecords, net).catch((e) => { + console.error(`[indexer/${net}] LP-share upsert failed:`, e); + return 0; + }); + + // ── NFT path ───────────────────────────────────────────────────────────────── + const nftParsed = parseNftEvents(nftRawEvents); + const nftRecords = nftParsed.map((p) => p.record); + const nftInserted = await upsertNftTransfers(nftRecords, net); + transfersStoredTotal.inc({ network: net, type: "nft" }, nftInserted); + + // Lazy-load metadata for unique (contractId, tokenId) pairs not yet cached + if (nftParsed.length > 0) { + const seen = new Set(); + for (const { record, tokenIdScVal } of nftParsed) { + const key = `${record.contractId}:${record.tokenId}`; + if (seen.has(key)) continue; + seen.add(key); + const cached = await getNftMetadata(record.contractId, record.tokenId, net); + if (!cached) { + const meta = await fetchNftMetadata(record.contractId, tokenIdScVal, net).catch(() => ({})); + await upsertNftMetadata(record.contractId, record.tokenId, meta, net).catch((e: unknown) => + console.error(`[indexer/${net}] NFT metadata upsert failed:`, e) + ); + } + } + } + + return { fungibleInserted: inserted, nftInserted, lpInserted }; +} diff --git a/src/indexer/parallel.ts b/src/indexer/parallel.ts index e09d930a9..17e6898f8 100644 --- a/src/indexer/parallel.ts +++ b/src/indexer/parallel.ts @@ -10,13 +10,16 @@ * worker processes its own ledger range sequentially with the same fromLedger / * toLedger window. Cross-partition ordering is not guaranteed and is not * required by the data model (eventId is the canonical ordering key). + * + * This module only decides *how* events are fetched and sharded. *What* is done + * with them lives in `processEventBatch` (./batch), which the caller injects, so + * the sharded path cannot drift from the single-fetch path (#203). */ -import { fetchEventsSafe } from "../rpc"; +import type { RawEvent } from "../rpc"; import { resolveNetwork, type Network } from "../network"; -import { parseEvents } from "../decoder"; -import { upsertTransfers, setLastIndexedLedger } from "../db"; -import { emitTransfer } from "../events"; +import { setLastIndexedLedger } from "../db"; +import { batchTotal, type BatchResult } from "./batch"; export const DEFAULT_WORKERS = 4; @@ -42,40 +45,46 @@ interface WorkerResult { highestLedger: number; } +/** + * The two seams the caller provides. Both are required: a sharded run with no + * `processBatch` is exactly the silent degradation this module used to have. + */ +export interface ParallelIo { + fetchEvents: ( + fromLedger: number, + toLedger: number, + contractIds: string[], + limit: number, + ) => Promise<{ events: RawEvent[]; highestLedger: number }>; + processBatch: (events: RawEvent[]) => Promise; +} + async function runPartitionWorker( partition: string[], fromLedger: number, toLedger: number, batchSize: number, - network: Network, + io: ParallelIo, ): Promise { - const { events, highestLedger } = await fetchEventsSafe( + const { events, highestLedger } = await io.fetchEvents( fromLedger, toLedger, partition, batchSize, - undefined, - network, ); if (events.length === 0) { return { inserted: 0, highestLedger }; } - const records = parseEvents(events); - const inserted = await upsertTransfers(records, network); - - if (inserted > 0) { - records.forEach((record) => emitTransfer(record, network)); - } - + const inserted = batchTotal(await io.processBatch(events)); return { inserted, highestLedger }; } /** * Poll one ledger window across all contract partitions in parallel. * - * @returns Total rows inserted and the highest ledger seen across all workers. + * @returns Total rows inserted (every record type) and the highest ledger seen across all workers. */ export async function pollParallel( contractIds: string[], @@ -83,14 +92,15 @@ export async function pollParallel( toLedger: number, batchSize: number, workerCount: number = DEFAULT_WORKERS, - network?: Network, + network: Network | undefined, + io: ParallelIo, ): Promise<{ totalInserted: number; highestLedger: number }> { const net = resolveNetwork(network); const partitions = partitionByContract(contractIds, Math.min(workerCount, contractIds.length || 1)); const results = await Promise.all( partitions.map(partition => - runPartitionWorker(partition, fromLedger, toLedger, batchSize, net), + runPartitionWorker(partition, fromLedger, toLedger, batchSize, io), ), ); From 4bbf366d35d07042918ed3fed7fa2df91be63f71 Mon Sep 17 00:00:00 2001 From: BOA <97275013+boalambo@users.noreply.github.com> Date: Wed, 30 Sep 2026 19:35:43 +0100 Subject: [PATCH 08/15] Bring docker-compose.yml in line with env contract (#194) (#220) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Bring docker-compose.yml in line with env contract (#194) - Remove obsolete version: "3.9" key - Add all missing env keys from .env.example: - DIRECT_DATABASE_URL - STELLAR_NETWORK - SOROBAN_RPC_URL (with STELLAR_RPC_URL as backward-compat alias) - NETWORKS - SAC_CONTRACT_IDS (and per-network variants) - NFT_CONTRACT_IDS (and per-network variants) - RETENTION_DAYS - CACHE_ENABLED and all Redis cache config - TOMBSTONE_CHECK_EVERY_CYCLES - LP_POOL_CONTRACT_IDS variants - SKIP_INDEXER - Keep CONTRACT_IDS as documented backward-compat alias - Add optional redis service with cache profile for CACHE_ENABLED support - Update README to use SAC_CONTRACT_IDS and add cache profile instructions - Add test to validate compose file meets requirements * test(compose): derive the env-contract check from .env.example The drift test built envKeys from .env.example and then never used it, asserting against a hand-maintained list instead — so a key added to .env.example and forgotten in docker-compose.yml still passed, which is the one failure #194 is about. The assertions were also substring matches on the whole file: toContain("SAC_CONTRACT_IDS") is satisfied by SAC_CONTRACT_IDS_TESTNET, and toContain("CONTRACT_IDS") by either, so they held on a file declaring none of them. Now it parses the wraith service's own environment block (anchored on the service, since db has an environment block too) and compares declared keys against every uncommented key in .env.example, with an explicit, empty exclusion list. Verified it fails on an added key and passes without one. Also: the docker compose config case ran unconditionally and called fail(), which is not defined under jest-circus, so on a machine or CI runner without Docker it failed with a ReferenceError. Unit tests here do not require Docker (the integration suite is vitest + Docker), so it now skips instead. Claude-Session: https://claude.ai/code/session_01USgemLt4Rnz4SGB1Srf3GB --------- Co-authored-by: Miracle656 --- README.md | 14 ++- docker-compose.yml | 43 +++++++- src/__tests__/dockerCompose.test.ts | 147 ++++++++++++++++++++++++++++ 3 files changed, 197 insertions(+), 7 deletions(-) create mode 100644 src/__tests__/dockerCompose.test.ts diff --git a/README.md b/README.md index 73fb0b2fb..bf2fdf516 100644 --- a/README.md +++ b/README.md @@ -70,12 +70,13 @@ cp .env.example .env ```env DATABASE_URL="postgresql://wraith:wraith@localhost:5432/wraith" +DIRECT_DATABASE_URL="postgresql://wraith:wraith@localhost:5432/wraith" STELLAR_NETWORK="testnet" # SOROBAN_RPC_URL is optional on testnet — the default public endpoint is used automatically SOROBAN_RPC_URL= START_LEDGER= -CONTRACT_IDS= +SAC_CONTRACT_IDS= PORT=3000 ``` @@ -83,12 +84,13 @@ PORT=3000 ```env DATABASE_URL="postgresql://wraith:wraith@localhost:5432/wraith" +DIRECT_DATABASE_URL="postgresql://wraith:wraith@localhost:5432/wraith" STELLAR_NETWORK="mainnet" # Required on mainnet — no free public Soroban RPC exists SOROBAN_RPC_URL="https://mainnet.stellar.validationcloud.io/v1/" # Strongly recommended on mainnet: filter to specific contracts to reduce load -CONTRACT_IDS="CTOKEN1...,CTOKEN2..." +SAC_CONTRACT_IDS="CTOKEN1...,CTOKEN2..." START_LEDGER= PORT=3000 ``` @@ -98,7 +100,7 @@ PORT=3000 ### 3. Start Postgres ```bash -docker-compose up -d db +docker compose up -d db ``` ### 4. Run database migrations @@ -120,7 +122,11 @@ npm run build && npm start Or run everything via Docker: ```bash -docker-compose up --build +# Without cache (default) +docker compose up --build + +# With Redis cache enabled +docker compose --profile cache up --build ``` *** diff --git a/docker-compose.yml b/docker-compose.yml index 75261a1d7..ffdf83ad6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: "3.9" - services: # ── Postgres ────────────────────────────────────────────── db: @@ -20,6 +18,21 @@ services: timeout: 5s retries: 10 + # ── Redis (optional, for response cache) ───────────────── + redis: + image: redis:7-alpine + container_name: wraith_redis + restart: unless-stopped + ports: + - "6379:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 5s + retries: 10 + profiles: + - cache + # ── Wraith Indexer + API ─────────────────────────────────── wraith: build: . @@ -28,14 +41,38 @@ services: depends_on: db: condition: service_healthy + redis: + condition: service_healthy + required: false environment: DATABASE_URL: postgresql://wraith:wraith@db:5432/wraith - STELLAR_RPC_URL: ${STELLAR_RPC_URL:-https://soroban-testnet.stellar.org} + DIRECT_DATABASE_URL: postgresql://wraith:wraith@db:5432/wraith + STELLAR_NETWORK: ${STELLAR_NETWORK:-testnet} + SOROBAN_RPC_URL: ${SOROBAN_RPC_URL:-} + STELLAR_RPC_URL: ${STELLAR_RPC_URL:-} + NETWORKS: ${NETWORKS:-} START_LEDGER: ${START_LEDGER:-} POLL_INTERVAL_MS: ${POLL_INTERVAL_MS:-6000} + SAC_CONTRACT_IDS: ${SAC_CONTRACT_IDS:-} + SAC_CONTRACT_IDS_TESTNET: ${SAC_CONTRACT_IDS_TESTNET:-} + SAC_CONTRACT_IDS_MAINNET: ${SAC_CONTRACT_IDS_MAINNET:-} CONTRACT_IDS: ${CONTRACT_IDS:-} + NFT_CONTRACT_IDS: ${NFT_CONTRACT_IDS:-} + NFT_CONTRACT_IDS_TESTNET: ${NFT_CONTRACT_IDS_TESTNET:-} + NFT_CONTRACT_IDS_MAINNET: ${NFT_CONTRACT_IDS_MAINNET:-} EVENTS_BATCH_SIZE: ${EVENTS_BATCH_SIZE:-10000} + RETENTION_DAYS: ${RETENTION_DAYS:-30} PORT: ${PORT:-3000} + CACHE_ENABLED: ${CACHE_ENABLED:-false} + REDIS_URL: ${REDIS_URL:-redis://redis:6379} + CACHE_KEY_PREFIX: ${CACHE_KEY_PREFIX:-wraith:cache:} + CACHE_TTL_POPULAR_MS: ${CACHE_TTL_POPULAR_MS:-60000} + CACHE_TTL_SEARCH_MS: ${CACHE_TTL_SEARCH_MS:-15000} + TOMBSTONE_CHECK_EVERY_CYCLES: ${TOMBSTONE_CHECK_EVERY_CYCLES:-100} + LP_POOL_CONTRACT_IDS: ${LP_POOL_CONTRACT_IDS:-} + LP_POOL_CONTRACT_IDS_TESTNET: ${LP_POOL_CONTRACT_IDS_TESTNET:-} + LP_POOL_CONTRACT_IDS_MAINNET: ${LP_POOL_CONTRACT_IDS_MAINNET:-} + SKIP_INDEXER: ${SKIP_INDEXER:-false} ports: - "${PORT:-3000}:3000" diff --git a/src/__tests__/dockerCompose.test.ts b/src/__tests__/dockerCompose.test.ts new file mode 100644 index 000000000..073356235 --- /dev/null +++ b/src/__tests__/dockerCompose.test.ts @@ -0,0 +1,147 @@ +/** + * docker-compose.yml against the env contract in .env.example (#194). + * + * The point of this file is drift: a key added to .env.example and not to the + * compose service is exactly the failure the issue describes, and it is silent + * — the container just starts without it. So the check is derived from + * .env.example rather than from a hand-maintained list that has to be + * remembered, and it compares *declared keys*, not substrings: searching the + * raw text for "SAC_CONTRACT_IDS" is satisfied by "SAC_CONTRACT_IDS_TESTNET", + * and searching for "CONTRACT_IDS" is satisfied by either, so a substring test + * passes on a file that declares none of them. + */ + +import { readFileSync } from "fs"; +import { execFileSync } from "child_process"; +import path from "path"; + +const ROOT = path.join(__dirname, "../.."); +const COMPOSE_PATH = path.join(ROOT, "docker-compose.yml"); +const ENV_EXAMPLE_PATH = path.join(ROOT, ".env.example"); + +/** These files are checked out with CRLF on Windows; anchored matches need LF. */ +const read = (file: string): string => readFileSync(file, "utf-8").split("\r\n").join("\n"); + +/** + * Keys that belong in .env.example but deliberately not in the dev compose + * file. Empty today; anything added here needs a reason next to it. + */ +const NOT_IN_COMPOSE = new Set([]); + +/** Uncommented `KEY=` lines in .env.example. */ +function envExampleKeys(content: string): string[] { + return [ + ...new Set( + content + .split("\n") + .map((line) => line.trim()) + .filter((line) => !line.startsWith("#") && line.includes("=")) + .map((line) => line.split("=")[0].trim()) + .filter((key) => /^[A-Z][A-Z0-9_]*$/.test(key)), + ), + ]; +} + +/** + * Env keys the wraith service declares, from its own `environment:` block. + * + * Anchored on ` wraith:` rather than the first `environment:` in the file — + * the db service has one too, and reading that one silently reports every + * application key as missing. + */ +function composeEnvKeys(content: string, service = "wraith"): string[] { + const lines = content.split("\n"); + const serviceAt = lines.findIndex((l) => l === ` ${service}:`); + if (serviceAt === -1) return []; + + const body = lines.slice(serviceAt + 1); + const end = body.findIndex((l) => /^ {0,2}\S/.test(l)); + const serviceLines = end === -1 ? body : body.slice(0, end); + + const envAt = serviceLines.findIndex((l) => /^ {4}environment:\s*$/.test(l)); + if (envAt === -1) return []; + + const keys: string[] = []; + for (const line of serviceLines.slice(envAt + 1)) { + if (/^ {0,4}\S/.test(line)) break; // back out to the service's own keys + const match = /^ {6}([A-Z][A-Z0-9_]*):/.exec(line); + if (match) keys.push(match[1]); + } + return keys; +} + +function hasDockerCompose(): boolean { + try { + execFileSync("docker", ["compose", "version"], { stdio: "pipe" }); + return true; + } catch { + return false; + } +} + +describe("docker-compose.yml", () => { + const composeContent = read(COMPOSE_PATH); + const envExampleContent = read(ENV_EXAMPLE_PATH); + + it("does not carry the obsolete top-level version key", () => { + expect(composeContent).not.toMatch(/^version:/m); + }); + + it("declares every env key .env.example documents", () => { + const declared = new Set(composeEnvKeys(composeContent)); + // Guard the guard: if the environment block stops being found, every key + // below would "pass" as missing-from-an-empty-set, so assert it parsed. + expect(declared.size).toBeGreaterThan(0); + + const missing = envExampleKeys(envExampleContent) + .filter((key) => !NOT_IN_COMPOSE.has(key)) + .filter((key) => !declared.has(key)); + + expect(missing).toEqual([]); + }); + + it("passes each overridable key through from the host environment", () => { + // A key declared as ${OTHER_NAME:-...} is a typo that silently reads the + // wrong variable, so the reference has to match the key it is assigned to. + const passthrough = [ + ...composeContent.matchAll(/^ {6}([A-Z][A-Z0-9_]*):\s*\$\{([A-Z][A-Z0-9_]*)/gm), + ]; + for (const [, key, ref] of passthrough) expect(ref).toBe(key); + expect(passthrough.length).toBeGreaterThan(10); + }); + + it("offers redis behind the cache profile, so a plain `up` does not start it", () => { + expect(composeContent).toMatch(/^ {2}redis:$/m); + expect(composeContent).toMatch(/profiles:\s*\n\s*- cache/); + }); + + it("declares the dual-network and cache keys the app reads", () => { + const declared = new Set(composeEnvKeys(composeContent)); + for (const key of [ + "DIRECT_DATABASE_URL", + "RETENTION_DAYS", + "STELLAR_NETWORK", + "NETWORKS", + "SAC_CONTRACT_IDS", + "SAC_CONTRACT_IDS_TESTNET", + "SAC_CONTRACT_IDS_MAINNET", + "CONTRACT_IDS", + "CACHE_ENABLED", + "REDIS_URL", + ]) { + expect(declared).toContain(key); + } + }); + + // Docker is not a prerequisite for the unit suite (integration tests are + // vitest + Docker), so this is skipped rather than failed when absent. + const withDocker = hasDockerCompose() ? it : it.skip; + withDocker("resolves with `docker compose config`", () => { + const out = execFileSync("docker", ["compose", "config"], { + cwd: ROOT, + stdio: "pipe", + encoding: "utf-8", + }); + expect(out).toContain("wraith"); + }); +}); From de4fb0d0060fb0acdafcf769c2345830f325d6e4 Mon Sep 17 00:00:00 2001 From: Bathoul Mohammed Date: Wed, 30 Sep 2026 19:36:35 +0100 Subject: [PATCH 09/15] Fix/cache network key (#202) * fix(cache): include resolved network in Redis cache key (#182) defaultKeyFn now prepends req.network (set by networkMiddleware) to the cache key so mainnet and testnet requests never collide, even when the network is supplied via the X-Network header rather than ?network=. The ?network= query param is filtered from the query segment since it is already captured by req.network, ensuring ?network=mainnet and X-Network: mainnet produce identical keys. Existing keys change shape and will expire naturally over their TTL. * fix(cache): include resolved network in Redis cache key (#182) * chore: drop the unrelated package-lock.json change The branch re-resolved fsevents and dropped its "dev": true marker. Nothing in this PR touches dependencies, so restore the lockfile to main's. Claude-Session: https://claude.ai/code/session_01USgemLt4Rnz4SGB1Srf3GB --------- Co-authored-by: Miracle656 --- src/cache/__tests__/redis.test.ts | 57 +++++++++++++++++++++++++++---- src/cache/redis.ts | 18 ++++++++-- 2 files changed, 66 insertions(+), 9 deletions(-) diff --git a/src/cache/__tests__/redis.test.ts b/src/cache/__tests__/redis.test.ts index 55137e405..44575976a 100644 --- a/src/cache/__tests__/redis.test.ts +++ b/src/cache/__tests__/redis.test.ts @@ -174,21 +174,64 @@ describe("cacheMiddleware", () => { }); describe("defaultKeyFn", () => { - const baseReq = (query: Record): Request => - ({ method: "GET", baseUrl: "/search", path: "/", query } as unknown as Request); + const baseReq = (query: Record, network?: string): Request => + ({ method: "GET", baseUrl: "/search", path: "/", query, network } as unknown as Request); it("is stable regardless of query param order", () => { - expect(defaultKeyFn(baseReq({ a: "1", b: "2" }))).toBe( - defaultKeyFn(baseReq({ b: "2", a: "1" })), + expect(defaultKeyFn(baseReq({ a: "1", b: "2" }, "mainnet"))).toBe( + defaultKeyFn(baseReq({ b: "2", a: "1" }, "mainnet")), ); }); - it("encodes method, path and sorted query", () => { - expect(defaultKeyFn(baseReq({ b: "2", a: "1" }))).toBe("GET:/search/?a=1&b=2"); + it("encodes method, network, path and sorted query", () => { + expect(defaultKeyFn(baseReq({ b: "2", a: "1" }, "mainnet"))).toBe( + "GET:mainnet:/search/?a=1&b=2", + ); }); it("flattens array query values", () => { - expect(defaultKeyFn(baseReq({ tag: ["x", "y"] }))).toBe("GET:/search/?tag=x,y"); + expect(defaultKeyFn(baseReq({ tag: ["x", "y"] }, "mainnet"))).toBe( + "GET:mainnet:/search/?tag=x,y", + ); + }); + + it("falls back to 'unknown' when req.network is not set", () => { + expect(defaultKeyFn(baseReq({ a: "1" }))).toBe("GET:unknown:/search/?a=1"); + }); + + // Acceptance criteria: two requests differing only by X-Network produce different keys. + // networkMiddleware resolves both ?network= and X-Network: into req.network, so by the + // time defaultKeyFn runs the network is always in req.network regardless of how it arrived. + it("produces different keys for mainnet vs testnet", () => { + const mainnet = defaultKeyFn(baseReq({ q: "USDC" }, "mainnet")); + const testnet = defaultKeyFn(baseReq({ q: "USDC" }, "testnet")); + expect(mainnet).not.toBe(testnet); + expect(mainnet).toBe("GET:mainnet:/search/?q=USDC"); + expect(testnet).toBe("GET:testnet:/search/?q=USDC"); + }); + + // Acceptance criteria: ?network=mainnet and X-Network: mainnet produce the same key. + // Both paths go through networkMiddleware which sets req.network; the ?network= param + // is stripped from the query segment to avoid double-encoding. + it("produces the same key for ?network=mainnet and X-Network: mainnet (both resolve to req.network)", () => { + // Simulates request via query string: networkMiddleware strips ?network from query + // and sets req.network = "mainnet" + const viaQuery = defaultKeyFn(baseReq({ q: "USDC" }, "mainnet")); + // Simulates request via header: networkMiddleware sets req.network = "mainnet", + // query has no network param at all + const viaHeader = defaultKeyFn(baseReq({ q: "USDC" }, "mainnet")); + expect(viaQuery).toBe(viaHeader); + }); + + it("excludes the ?network query param from the key (captured in req.network instead)", () => { + // ?network= in the query string must not appear in the key; the resolved + // req.network value is the canonical source. + const withNetworkInQuery = defaultKeyFn( + baseReq({ q: "USDC", network: "mainnet" }, "mainnet"), + ); + const withoutNetworkInQuery = defaultKeyFn(baseReq({ q: "USDC" }, "mainnet")); + expect(withNetworkInQuery).toBe(withoutNetworkInQuery); + expect(withNetworkInQuery).toBe("GET:mainnet:/search/?q=USDC"); }); }); diff --git a/src/cache/redis.ts b/src/cache/redis.ts index 41e67dfe6..c30152a07 100644 --- a/src/cache/redis.ts +++ b/src/cache/redis.ts @@ -127,13 +127,27 @@ export interface CacheMiddlewareOptions { const HEADER_NO_CACHE = "x-no-cache"; const HEADER_CACHE_STATUS = "X-Cache"; -/** Stable key: sorts query params so `?a=1&b=2` and `?b=2&a=1` collide. */ +/** + * Stable key: sorts query params so `?a=1&b=2` and `?b=2&a=1` collide. + * + * The resolved network (`req.network`, set by `networkMiddleware`) is included + * so that mainnet and testnet requests never share a cache entry even when the + * network is passed via the `X-Network` header rather than `?network=` — the + * header is invisible to the query-string portion of the key. + * + * `?network=` is excluded from the query segment because it is already + * captured by `req.network`; keeping it would produce redundant / double- + * encoded keys and break the `?network=mainnet` ≡ `X-Network: mainnet` + * equivalence required by the acceptance criteria. + */ export function defaultKeyFn(req: Request): string { const entries = Object.entries(req.query as Record) + .filter(([k]) => k !== "network") .map(([k, v]) => [k, Array.isArray(v) ? v.join(",") : String(v)] as const) .sort(([a], [b]) => a.localeCompare(b)); const qs = entries.map(([k, v]) => `${k}=${v}`).join("&"); - return `${req.method}:${req.baseUrl}${req.path}?${qs}`; + const network = req.network ?? "unknown"; + return `${req.method}:${network}:${req.baseUrl}${req.path}?${qs}`; } /** From 5564cea27c2f59c03eddc70805e3d84edad469ea Mon Sep 17 00:00:00 2001 From: BOA <97275013+boalambo@users.noreply.github.com> Date: Wed, 30 Sep 2026 19:43:48 +0100 Subject: [PATCH 10/15] Add npm run db:seed with deterministic fixtures (#221) - Create shared fixture module in src/fixtures.ts with deterministic test data across two addresses (ALICE, BOB, CAROL) and two contracts (CONTRACT_A, CONTRACT_B) - Add seed script at scripts/seed.ts with --network flag support (testnet/mainnet) - Add db:seed script to package.json - Update integration tests to use shared fixture module instead of local copy - Update README Quick Start with seed step and real sample output - Add fixture validation test to verify data structure and coverage The seed script uses skipDuplicates on unique constraints, making it safe to run multiple times without duplicating rows. Fixtures include TokenTransfer, NftTransfer, and AccountSummary rows with deterministic eventId values. Resolves #193 --- README.md | 66 ++++++++++++- package.json | 1 + scripts/seed.ts | 74 +++++++++++++++ src/__tests__/fixtures.test.ts | 87 +++++++++++++++++ src/fixtures.ts | 166 +++++++++++++++++++++++++++++++++ tests/integration/api.test.ts | 2 +- tests/integration/fixtures.ts | 83 ----------------- tests/integration/setup.ts | 4 +- 8 files changed, 396 insertions(+), 87 deletions(-) create mode 100644 scripts/seed.ts create mode 100644 src/__tests__/fixtures.test.ts create mode 100644 src/fixtures.ts delete mode 100644 tests/integration/fixtures.ts diff --git a/README.md b/README.md index bf2fdf516..25a5bcc98 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,65 @@ docker compose up -d db npx prisma migrate dev --name init ``` -### 5. Start Wraith +### 5. Seed the database (optional) + +Populate the database with deterministic test data for development and testing: + +```bash +npm run db:seed +``` + +By default, this seeds `testnet`. To seed `mainnet`: + +```bash +npm run db:seed -- --network=mainnet +``` + +You can now query the seeded data: + +```bash +curl "http://localhost:3000/transfers/incoming/GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAWHF" +``` + +Expected response: + +```json +{ + "total": 2, + "limit": 50, + "offset": 0, + "transfers": [ + { + "id": 1, + "contractId": "CAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD2KM", + "eventType": "transfer", + "fromAddress": "GBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBWWHF", + "toAddress": "GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAWHF", + "amount": "10000000", + "displayAmount": "1.0000000", + "ledger": 2001, + "ledgerClosedAt": "2025-01-01T00:00:00.000Z", + "txHash": "tx-incoming-a-1", + "eventId": "integration-001" + }, + { + "id": 2, + "contractId": "CBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBD2KM", + "eventType": "transfer", + "fromAddress": "GCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCWWHF", + "toAddress": "GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAWHF", + "amount": "40000000", + "displayAmount": "4.0000000", + "ledger": 2004, + "ledgerClosedAt": "2025-02-01T00:00:00.000Z", + "txHash": "txhash-integration-multi", + "eventId": "integration-004" + } + ] +} +``` + +### 6. Start Wraith ```bash # Development (hot reload) @@ -129,6 +187,12 @@ docker compose up --build docker compose --profile cache up --build ``` +To seed the database when running via Docker, run the seed command in a separate terminal after the database is up: + +```bash +npm run db:seed +``` + *** ## API Documentation diff --git a/package.json b/package.json index efc219a8d..e2527137d 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "db:generate": "prisma generate", "db:migrate": "prisma migrate dev", "db:push": "prisma db push", + "db:seed": "ts-node scripts/seed.ts", "db:studio": "prisma studio", "docs": "typedoc", "docs:openapi": "ts-node src/openapi/build.ts", diff --git a/scripts/seed.ts b/scripts/seed.ts new file mode 100644 index 000000000..9c3c17ec9 --- /dev/null +++ b/scripts/seed.ts @@ -0,0 +1,74 @@ +/** + * Seed script — inserts deterministic test data + * + * Inserts deterministic TokenTransfer, NftTransfer, and AccountSummary rows + * for testing and development. Safe to run multiple times: uses skipDuplicates + * on unique constraints so rows are never duplicated. + * + * Usage: + * npx ts-node scripts/seed.ts + * npx ts-node scripts/seed.ts --network mainnet + * + * Environment variables: + * DATABASE_URL — Postgres connection string (required) + */ + +import "dotenv/config"; +import { prisma } from "../src/db"; +import { + tokenTransferFixtures, + nftTransferFixtures, + accountSummaryFixtures, +} from "../src/fixtures"; +import { resolveNetwork, type Network } from "../src/network"; + +function parseArgs(): { network: Network } { + const args = process.argv.slice(2); + const networkFlag = args.find((arg) => arg.startsWith("--network=")); + const networkValue = networkFlag + ? networkFlag.split("=")[1] + : process.env.STELLAR_NETWORK || "testnet"; + + if (networkValue !== "testnet" && networkValue !== "mainnet") { + console.error(`Invalid network: "${networkValue}". Valid values: testnet, mainnet.`); + process.exit(1); + } + + return { network: networkValue as Network }; +} + +async function main() { + const { network } = parseArgs(); + console.log(`[seed] Seeding database for network: ${network}`); + + const networkName = resolveNetwork(network); + + // Insert token transfers + const tokenResult = await prisma.tokenTransfer.createMany({ + data: tokenTransferFixtures.map((r) => ({ ...r, network: networkName })), + skipDuplicates: true, + }); + console.log(`[seed] Inserted ${tokenResult.count} token transfer rows`); + + // Insert NFT transfers + const nftResult = await prisma.nftTransfer.createMany({ + data: nftTransferFixtures.map((r) => ({ ...r, network: networkName })), + skipDuplicates: true, + }); + console.log(`[seed] Inserted ${nftResult.count} NFT transfer rows`); + + // Insert account summaries + const summaryResult = await prisma.accountSummary.createMany({ + data: accountSummaryFixtures.map((r) => ({ ...r, network: networkName })), + skipDuplicates: true, + }); + console.log(`[seed] Inserted ${summaryResult.count} account summary rows`); + + console.log(`[seed] Done. Seeded ${network} with deterministic fixtures.`); + await prisma.$disconnect(); +} + +main().catch((err) => { + console.error("[seed] Fatal error:", err); + process.exit(1); +}); diff --git a/src/__tests__/fixtures.test.ts b/src/__tests__/fixtures.test.ts new file mode 100644 index 000000000..83a9e5452 --- /dev/null +++ b/src/__tests__/fixtures.test.ts @@ -0,0 +1,87 @@ +/** + * Fixture data validation test + * + * Verifies that the fixture data meets the acceptance criteria: + * - Contains deterministic data across at least two addresses and two contracts + * - Data structure matches the expected schema + */ + +import { describe, it, expect } from "@jest/globals"; +import { + tokenTransferFixtures, + nftTransferFixtures, + accountSummaryFixtures, + CONTRACT_A, + CONTRACT_B, + ALICE, + BOB, + CAROL, +} from "../fixtures"; + +describe("fixture data validation", () => { + it("contains at least two addresses", () => { + const addresses = new Set( + tokenTransferFixtures.flatMap((t) => [t.fromAddress, t.toAddress].filter(Boolean)) + ); + expect(addresses.size).toBeGreaterThanOrEqual(2); + expect(addresses.has(ALICE)).toBe(true); + expect(addresses.has(BOB)).toBe(true); + }); + + it("contains at least two contracts", () => { + const contracts = new Set(tokenTransferFixtures.map((t) => t.contractId)); + expect(contracts.size).toBeGreaterThanOrEqual(2); + expect(contracts.has(CONTRACT_A)).toBe(true); + expect(contracts.has(CONTRACT_B)).toBe(true); + }); + + it("has valid token transfer fixture structure", () => { + expect(tokenTransferFixtures.length).toBeGreaterThan(0); + tokenTransferFixtures.forEach((t) => { + expect(t).toHaveProperty("contractId"); + expect(t).toHaveProperty("eventType"); + expect(t).toHaveProperty("amount"); + expect(t).toHaveProperty("ledger"); + expect(t).toHaveProperty("ledgerClosedAt"); + expect(t).toHaveProperty("txHash"); + expect(t).toHaveProperty("eventId"); + }); + }); + + it("has valid NFT transfer fixture structure", () => { + expect(nftTransferFixtures.length).toBeGreaterThan(0); + nftTransferFixtures.forEach((t) => { + expect(t).toHaveProperty("contractId"); + expect(t).toHaveProperty("tokenId"); + expect(t).toHaveProperty("ledger"); + expect(t).toHaveProperty("ledgerClosedAt"); + expect(t).toHaveProperty("txHash"); + expect(t).toHaveProperty("eventId"); + }); + }); + + it("has valid account summary fixture structure", () => { + expect(accountSummaryFixtures.length).toBeGreaterThan(0); + accountSummaryFixtures.forEach((s) => { + expect(s).toHaveProperty("address"); + expect(s).toHaveProperty("contractId"); + expect(s).toHaveProperty("totalSent"); + expect(s).toHaveProperty("totalReceived"); + expect(s).toHaveProperty("net"); + expect(s).toHaveProperty("txCount"); + expect(s).toHaveProperty("lastActivityAt"); + }); + }); + + it("account summaries match the transfer data", () => { + // Verify that the account summary fixtures contain data for the same addresses and contracts + const summaryAddresses = new Set(accountSummaryFixtures.map((s) => s.address)); + const summaryContracts = new Set(accountSummaryFixtures.map((s) => s.contractId)); + + expect(summaryAddresses.has(ALICE)).toBe(true); + expect(summaryAddresses.has(BOB)).toBe(true); + expect(summaryAddresses.has(CAROL)).toBe(true); + expect(summaryContracts.has(CONTRACT_A)).toBe(true); + expect(summaryContracts.has(CONTRACT_B)).toBe(true); + }); +}); diff --git a/src/fixtures.ts b/src/fixtures.ts new file mode 100644 index 000000000..1521c659e --- /dev/null +++ b/src/fixtures.ts @@ -0,0 +1,166 @@ +import type { Prisma } from "@prisma/client"; +import type { TransferRecord } from "./db"; + +export const CONTRACT_A = "CAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD2KM"; +export const CONTRACT_B = "CBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBD2KM"; + +export const ALICE = "GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAWHF"; +export const BOB = "GBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBWWHF"; +export const CAROL = "GCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCWWHF"; + +export const MULTI_EVENT_TX_HASH = "txhash-integration-multi"; + +const at = (iso: string) => new Date(iso); + +export const tokenTransferFixtures: Prisma.TokenTransferCreateManyInput[] = [ + { + contractId: CONTRACT_A, + eventType: "transfer", + fromAddress: BOB, + toAddress: ALICE, + amount: "10000000", + ledger: 2001, + ledgerClosedAt: at("2025-01-01T00:00:00Z"), + txHash: "tx-incoming-a-1", + eventId: "integration-001", + }, + { + contractId: CONTRACT_A, + eventType: "mint", + fromAddress: null, + toAddress: ALICE, + amount: "25000000", + ledger: 2002, + ledgerClosedAt: at("2025-01-02T00:00:00Z"), + txHash: "tx-mint-a-1", + eventId: "integration-002", + }, + { + contractId: CONTRACT_A, + eventType: "transfer", + fromAddress: ALICE, + toAddress: BOB, + amount: "5000000", + ledger: 2003, + ledgerClosedAt: at("2025-01-03T00:00:00Z"), + txHash: "tx-outgoing-a-1", + eventId: "integration-003", + }, + { + contractId: CONTRACT_B, + eventType: "transfer", + fromAddress: CAROL, + toAddress: ALICE, + amount: "40000000", + ledger: 2004, + ledgerClosedAt: at("2025-02-01T00:00:00Z"), + txHash: MULTI_EVENT_TX_HASH, + eventId: "integration-004", + }, + { + contractId: CONTRACT_B, + eventType: "clawback", + fromAddress: ALICE, + toAddress: null, + amount: "15000000", + ledger: 2005, + ledgerClosedAt: at("2025-02-02T00:00:00Z"), + txHash: MULTI_EVENT_TX_HASH, + eventId: "integration-005", + }, + { + contractId: CONTRACT_A, + eventType: "transfer", + fromAddress: BOB, + toAddress: CAROL, + amount: "999000000", + ledger: 2006, + ledgerClosedAt: at("2025-03-01T00:00:00Z"), + txHash: "tx-unrelated", + eventId: "integration-006", + }, +]; + +export const nftTransferFixtures: Prisma.NftTransferCreateManyInput[] = [ + { + contractId: CONTRACT_A, + tokenId: "token-001", + fromAddress: BOB, + toAddress: ALICE, + ledger: 2001, + ledgerClosedAt: at("2025-01-01T00:00:00Z"), + txHash: "tx-nft-incoming-a-1", + eventId: "nft-integration-001", + }, + { + contractId: CONTRACT_B, + tokenId: "token-002", + fromAddress: CAROL, + toAddress: ALICE, + ledger: 2004, + ledgerClosedAt: at("2025-02-01T00:00:00Z"), + txHash: "tx-nft-incoming-b-1", + eventId: "nft-integration-002", + }, + { + contractId: CONTRACT_A, + tokenId: "token-003", + fromAddress: ALICE, + toAddress: BOB, + ledger: 2003, + ledgerClosedAt: at("2025-01-03T00:00:00Z"), + txHash: "tx-nft-outgoing-a-1", + eventId: "nft-integration-003", + }, +]; + +export const accountSummaryFixtures: Prisma.AccountSummaryCreateManyInput[] = [ + { + address: ALICE, + contractId: CONTRACT_A, + totalSent: "5000000", + totalReceived: "35000000", + net: "30000000", + txCount: 3, + lastActivityAt: at("2025-01-03T00:00:00Z"), + }, + { + address: ALICE, + contractId: CONTRACT_B, + totalSent: "15000000", + totalReceived: "40000000", + net: "25000000", + txCount: 2, + lastActivityAt: at("2025-02-02T00:00:00Z"), + }, + { + address: BOB, + contractId: CONTRACT_A, + totalSent: "10000000", + totalReceived: "5000000", + net: "-5000000", + txCount: 2, + lastActivityAt: at("2025-03-01T00:00:00Z"), + }, + { + address: CAROL, + contractId: CONTRACT_B, + totalSent: "40000000", + totalReceived: "0", + net: "-40000000", + txCount: 1, + lastActivityAt: at("2025-02-01T00:00:00Z"), + }, +]; + +export const transferRecords: TransferRecord[] = tokenTransferFixtures.map((r) => ({ + contractId: r.contractId, + eventType: r.eventType, + fromAddress: r.fromAddress ?? null, + toAddress: r.toAddress ?? null, + amount: r.amount, + ledger: r.ledger, + ledgerClosedAt: new Date(r.ledgerClosedAt), + txHash: r.txHash, + eventId: r.eventId, +})); diff --git a/tests/integration/api.test.ts b/tests/integration/api.test.ts index 1a94974e2..3396a883b 100644 --- a/tests/integration/api.test.ts +++ b/tests/integration/api.test.ts @@ -5,7 +5,7 @@ import { CONTRACT_A, CONTRACT_B, MULTI_EVENT_TX_HASH, -} from "./fixtures"; +} from "../../src/fixtures"; const API_BASE_URL = process.env.INTEGRATION_API_URL ?? "http://localhost:3300"; diff --git a/tests/integration/fixtures.ts b/tests/integration/fixtures.ts deleted file mode 100644 index 76cc08b18..000000000 --- a/tests/integration/fixtures.ts +++ /dev/null @@ -1,83 +0,0 @@ -import type { Prisma } from "@prisma/client"; - -export const CONTRACT_A = "CAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAD2KM"; -export const CONTRACT_B = "CBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBD2KM"; - -export const ALICE = "GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAWHF"; -export const BOB = "GBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBWWHF"; -export const CAROL = "GCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCWWHF"; - -export const MULTI_EVENT_TX_HASH = "txhash-integration-multi"; - -type SeedTransfer = Prisma.TokenTransferCreateManyInput; - -const at = (iso: string) => new Date(iso); - -export const seedTransfers: SeedTransfer[] = [ - { - contractId: CONTRACT_A, - eventType: "transfer", - fromAddress: BOB, - toAddress: ALICE, - amount: "10000000", - ledger: 2001, - ledgerClosedAt: at("2025-01-01T00:00:00Z"), - txHash: "tx-incoming-a-1", - eventId: "integration-001", - }, - { - contractId: CONTRACT_A, - eventType: "mint", - fromAddress: null, - toAddress: ALICE, - amount: "25000000", - ledger: 2002, - ledgerClosedAt: at("2025-01-02T00:00:00Z"), - txHash: "tx-mint-a-1", - eventId: "integration-002", - }, - { - contractId: CONTRACT_A, - eventType: "transfer", - fromAddress: ALICE, - toAddress: BOB, - amount: "5000000", - ledger: 2003, - ledgerClosedAt: at("2025-01-03T00:00:00Z"), - txHash: "tx-outgoing-a-1", - eventId: "integration-003", - }, - { - contractId: CONTRACT_B, - eventType: "transfer", - fromAddress: CAROL, - toAddress: ALICE, - amount: "40000000", - ledger: 2004, - ledgerClosedAt: at("2025-02-01T00:00:00Z"), - txHash: MULTI_EVENT_TX_HASH, - eventId: "integration-004", - }, - { - contractId: CONTRACT_B, - eventType: "clawback", - fromAddress: ALICE, - toAddress: null, - amount: "15000000", - ledger: 2005, - ledgerClosedAt: at("2025-02-02T00:00:00Z"), - txHash: MULTI_EVENT_TX_HASH, - eventId: "integration-005", - }, - { - contractId: CONTRACT_A, - eventType: "transfer", - fromAddress: BOB, - toAddress: CAROL, - amount: "999000000", - ledger: 2006, - ledgerClosedAt: at("2025-03-01T00:00:00Z"), - txHash: "tx-unrelated", - eventId: "integration-006", - }, -]; diff --git a/tests/integration/setup.ts b/tests/integration/setup.ts index a608279b8..c2e56b7ac 100644 --- a/tests/integration/setup.ts +++ b/tests/integration/setup.ts @@ -1,5 +1,5 @@ import { PrismaClient } from "@prisma/client"; -import { seedTransfers } from "./fixtures"; +import { tokenTransferFixtures } from "../../src/fixtures"; process.env.DATABASE_URL ??= "postgresql://wraith:wraith@localhost:55432/wraith_test"; process.env.DIRECT_DATABASE_URL ??= "postgresql://wraith:wraith@localhost:55432/wraith_test"; @@ -35,7 +35,7 @@ export async function seedIntegrationFixtures(): Promise { try { await prisma.tokenTransfer.deleteMany(); await prisma.indexerState.deleteMany(); - await prisma.tokenTransfer.createMany({ data: seedTransfers }); + await prisma.tokenTransfer.createMany({ data: tokenTransferFixtures }); await prisma.indexerState.create({ data: { network: 'testnet', lastIndexedLedger: 2006 } }); } finally { await prisma.$disconnect(); From a5bb22d61fdf9672c807b50c3e28aae98bd65fb3 Mon Sep 17 00:00:00 2001 From: Miracle656 Date: Wed, 30 Sep 2026 19:46:01 +0100 Subject: [PATCH 11/15] fix(seed): accept --network mainnet as well as --network=mainnet Follow-up to #221: this fix was pushed to the PR branch but did not make it into the squash. The docstring advertised the space-separated form while parseArgs matched only --network=, so 'npm run db:seed -- --network mainnet' silently seeded testnet. Both spellings now parse; an unknown value still exits 1. Claude-Session: https://claude.ai/code/session_01USgemLt4Rnz4SGB1Srf3GB --- scripts/seed.ts | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/scripts/seed.ts b/scripts/seed.ts index 9c3c17ec9..7332caea8 100644 --- a/scripts/seed.ts +++ b/scripts/seed.ts @@ -6,8 +6,9 @@ * on unique constraints so rows are never duplicated. * * Usage: - * npx ts-node scripts/seed.ts - * npx ts-node scripts/seed.ts --network mainnet + * npm run db:seed + * npm run db:seed -- --network=mainnet + * npm run db:seed -- --network mainnet * * Environment variables: * DATABASE_URL — Postgres connection string (required) @@ -24,10 +25,16 @@ import { resolveNetwork, type Network } from "../src/network"; function parseArgs(): { network: Network } { const args = process.argv.slice(2); - const networkFlag = args.find((arg) => arg.startsWith("--network=")); - const networkValue = networkFlag - ? networkFlag.split("=")[1] - : process.env.STELLAR_NETWORK || "testnet"; + // Both spellings: `--network=mainnet` and `--network mainnet`. Accepting only + // the first silently seeded testnet when the second was typed, which is the + // worst outcome for a flag whose whole job is picking the target. + const eq = args.find((arg) => arg.startsWith("--network=")); + const spaced = args.indexOf("--network"); + const networkValue = eq + ? eq.slice("--network=".length) + : spaced !== -1 + ? args[spaced + 1] + : process.env.STELLAR_NETWORK || "testnet"; if (networkValue !== "testnet" && networkValue !== "mainnet") { console.error(`Invalid network: "${networkValue}". Valid values: testnet, mainnet.`); From 2bf3046b0d9d87254c12ab8949d9419463f8b778 Mon Sep 17 00:00:00 2001 From: krsnak Date: Wed, 30 Sep 2026 20:47:03 +0200 Subject: [PATCH 12/15] Fix token decimals review issues --- src/__tests__/routes/transfers.test.ts | 33 ++++++++++++++++++++++++-- src/api.ts | 18 +++++++------- src/index.ts | 14 +++++------ 3 files changed, 47 insertions(+), 18 deletions(-) diff --git a/src/__tests__/routes/transfers.test.ts b/src/__tests__/routes/transfers.test.ts index 709964699..f6985b571 100644 --- a/src/__tests__/routes/transfers.test.ts +++ b/src/__tests__/routes/transfers.test.ts @@ -8,7 +8,10 @@ jest.mock("../../db", () => ({ queryByTxHash: jest.fn(), querySummary: jest.fn(), getLastIndexedLedger: jest.fn(), - prisma: { $queryRaw: jest.fn() }, + prisma: { + $queryRaw: jest.fn(), + tokenMetadata: { findMany: jest.fn() }, + }, })); jest.mock("../../rpc", () => ({ @@ -25,8 +28,9 @@ jest.mock("../../indexer", () => ({ .mockReturnValue({ startedAt: "2024-01-01T00:00:00.000Z", uptimeSeconds: 0, totalIndexed: 0 }), })); -import { queryTransfers, queryAllTransfers, queryByTxHash, querySummary, getLastIndexedLedger } from "../../db"; +import { queryTransfers, queryAllTransfers, queryByTxHash, querySummary, getLastIndexedLedger, prisma } from "../../db"; import { getLatestLedger } from "../../rpc"; +import { _resetTokenCache, initTokenCache } from "../../tokenCache"; // ── Typed mock helpers ──────────────────────────────────────────────────────── const mockQueryTransfers = queryTransfers as jest.MockedFunction; @@ -109,6 +113,7 @@ describe("Transfer route handlers", () => { const app = createApp(); beforeEach(() => { + _resetTokenCache(); mockGetLastIndexedLedger.mockResolvedValue(1020); mockGetLatestLedger.mockResolvedValue(1022); }); @@ -138,6 +143,30 @@ describe("Transfer route handlers", () => { expect(res.body.transfers[0].displayAmount).toBe("1.0000000"); }); + it("preserves a 6-decimal displayAmount from the query layer when $select omits contractId", async () => { + _resetTokenCache(); + (prisma.tokenMetadata.findMany as jest.Mock).mockResolvedValue([ + { network: "testnet", contractId: CONTRACT_A, symbol: "TOK", name: "Token", decimals: 6 }, + ]); + await initTokenCache("testnet"); + + mockQueryTransfers.mockImplementation(async (params) => { + expect(params.tokenDecimals?.(CONTRACT_A, "testnet")).toBe(6); + return { + total: 1, + transfers: [{ amount: "1000000", displayAmount: "1.000000" }], + nextCursor: null, + } as Awaited>; + }); + + const res = await request(app) + .get(`/transfers/incoming/${ALICE}`) + .query({ $select: "amount,displayAmount" }); + + expect(res.status).toBe(200); + expect(res.body.transfers[0]).toEqual({ amount: "1000000", displayAmount: "1.000000" }); + }); + it("returns empty array for an unknown address", async () => { mockQueryTransfers.mockResolvedValue({ total: 0, transfers: [], nextCursor: null }); diff --git a/src/api.ts b/src/api.ts index 0778d523c..dbec72976 100644 --- a/src/api.ts +++ b/src/api.ts @@ -81,15 +81,17 @@ const SEARCH_CACHE_TTL_MS = parseInt(process.env.CACHE_TTL_SEARCH_MS ?? "15000", // ─── Amount formatting ──────────────────────────────────────────────────── export { toDisplayAmount } from "./amount"; -const withDisplay = ( +const withDisplay = ( transfer: T, network: Network, ) => ({ ...transfer, - displayAmount: toDisplayAmount( - transfer.amount, - transfer.contractId ? getCachedTokenDecimals(transfer.contractId, network) : undefined, - ), + displayAmount: + transfer.displayAmount ?? + toDisplayAmount( + transfer.amount, + transfer.contractId ? getCachedTokenDecimals(transfer.contractId, network) : undefined, + ), }); function parseSelectQuery(value: unknown): string[] | undefined { @@ -526,7 +528,7 @@ export function createApp(): express.Application { ...result, transfers: result.transfers.map((transfer) => { if (transfer && typeof (transfer as { amount?: unknown }).amount === "string") { - return withDisplay(transfer as { amount: string; contractId?: string }, network); + return withDisplay(transfer as { amount: string; contractId?: string; displayAmount?: string }, network); } return transfer; }), @@ -590,7 +592,7 @@ export function createApp(): express.Application { ...result, transfers: result.transfers.map((transfer) => { if (transfer && typeof (transfer as { amount?: unknown }).amount === "string") { - return withDisplay(transfer as { amount: string; contractId?: string }, network); + return withDisplay(transfer as { amount: string; contractId?: string; displayAmount?: string }, network); } return transfer; }), @@ -664,7 +666,7 @@ export function createApp(): express.Application { ...result, transfers: result.transfers.map((transfer) => { if (transfer && typeof (transfer as { amount?: unknown }).amount === "string") { - return withDisplay(transfer as { amount: string; contractId?: string }, network); + return withDisplay(transfer as { amount: string; contractId?: string; displayAmount?: string }, network); } return transfer; }), diff --git a/src/index.ts b/src/index.ts index 2c767f925..c4ff96ff6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,6 +8,8 @@ import { attachWebSocketServer } from "./ws"; import { attachGraphQLSubscriptions, SUBSCRIPTIONS_PATH } from "./graphql/subscriptions"; import { startWebhookWorker } from "./workers/webhooks"; import { startPartitionRetentionJob } from "./jobs/retention"; +import { initTokenCache } from "./tokenCache"; +import { enabledNetworks } from "./network"; const PORT = parseInt(process.env.PORT ?? "3000", 10); @@ -49,14 +51,10 @@ async function main() { // ── Start partition retention scheduler ─────────────────────────────────── startPartitionRetentionJob(); - // API-only mode. The integration harness starts the service to exercise HTTP - // routes against a seeded database; letting the indexer loop run there would - // make every request race an ingest that is also writing to the same tables, - // and would need live RPC the harness has no reason to depend on. - if (process.env.SKIP_INDEXER === "true") { - console.log("[wraith] SKIP_INDEXER=true — API-only mode, indexer not started."); - return; - } + // Seed token metadata for every served network before handling read paths. + // API-only deployments skip the indexer, so without this explicit seed their + // displayAmount values would silently fall back to 7 decimals for every token. + await Promise.all(enabledNetworks().map((network) => initTokenCache(network))); // ── Start indexer in the background ─────────────────────────────────────── // startIndexer() runs an infinite loop; we intentionally don't await it From 21805698706c33c5ac63017ffe8028bead15fa79 Mon Sep 17 00:00:00 2001 From: royaldev Date: Wed, 30 Sep 2026 19:49:53 +0100 Subject: [PATCH 13/15] perf(api): drop the default COUNT from list queries, add hasMore and opt-in includeTotal (#218) Co-authored-by: royalTreasure <295874283+royalTreasure@users.noreply.github.com> --- README.md | 16 ++- openapi.json | 111 +++++++++++++++++++-- src/__tests__/listPagination.test.ts | 118 ++++++++++++++++++++++ src/__tests__/network.test.ts | 3 +- src/__tests__/routes/transfers.test.ts | 94 ++++++++++------- src/__tests__/staleReads.test.ts | 3 +- src/api.ts | 4 + src/db.ts | 133 ++++++++++++++++--------- src/graphql/server.ts | 8 +- src/middleware/jsonapi.ts | 8 +- src/openapi/schemas.ts | 20 +++- src/routes/accounts/transfers.ts | 1 + 12 files changed, 411 insertions(+), 108 deletions(-) create mode 100644 src/__tests__/listPagination.test.ts diff --git a/README.md b/README.md index 25a5bcc98..f8f5dd887 100644 --- a/README.md +++ b/README.md @@ -288,7 +288,8 @@ console.log(data); // Expected response // { -// "total": 42, +// "hasMore": true, +// "nextCursor": "MTIzNDU=", // "limit": 10, // "offset": 0, // "transfers": [ @@ -309,6 +310,13 @@ console.log(data); // } ``` +> **Pagination and totals.** List endpoints (`/transfers/*`, `/nfts/transfers`, GraphQL `transfers`) +> do not run a `COUNT` by default. They fetch one row past the page and return +> `hasMore` (plus `nextCursor` to continue), which is all pagination needs. Pass +> `?includeTotal=true` (GraphQL: `includeTotal: true`) to also get an exact +> `total`. This runs a `COUNT` over every matching row, so it is markedly slower +> on large result sets. Without it, `total` is absent from the response. + *** ### `GET /transfers/address/:address` — All transfers (sent & received, merged) @@ -333,7 +341,7 @@ console.log(data); // Expected response (same shape as /transfers/incoming — adds "direction" per row) // { -// "total": 85, +// "hasMore": true, // "limit": 20, // "offset": 0, // "transfers": [ @@ -439,8 +447,8 @@ both chains in a single round-trip: ```graphql { - testnet: transfers(address: "GABC…", network: TESTNET) { total } - mainnet: transfers(address: "GABC…", network: MAINNET) { total } + testnet: transfers(address: "GABC…", network: TESTNET, includeTotal: true) { total hasMore } + mainnet: transfers(address: "GABC…", network: MAINNET, includeTotal: true) { total hasMore } } ``` diff --git a/openapi.json b/openapi.json index 4682ea686..67f721217 100644 --- a/openapi.json +++ b/openapi.json @@ -697,6 +697,17 @@ "description": "Comma-separated field list", "name": "$select", "in": "query" + }, + { + "schema": { + "type": "boolean", + "nullable": true, + "description": "Also return an exact `total`. Runs a COUNT over every matching row, so it is markedly slower on large result sets; omit it and use `hasMore` / `nextCursor` to paginate" + }, + "required": false, + "description": "Also return an exact `total`. Runs a COUNT over every matching row, so it is markedly slower on large result sets; omit it and use `hasMore` / `nextCursor` to paginate", + "name": "includeTotal", + "in": "query" } ], "responses": { @@ -708,7 +719,12 @@ "type": "object", "properties": { "total": { - "type": "integer" + "type": "integer", + "description": "Exact match count. Only present when the request set includeTotal=true" + }, + "hasMore": { + "type": "boolean", + "description": "True when another page exists" }, "limit": { "type": "integer" @@ -798,7 +814,7 @@ } }, "required": [ - "total", + "hasMore", "limit", "offset", "transfers" @@ -1031,6 +1047,17 @@ "description": "Comma-separated field list", "name": "$select", "in": "query" + }, + { + "schema": { + "type": "boolean", + "nullable": true, + "description": "Also return an exact `total`. Runs a COUNT over every matching row, so it is markedly slower on large result sets; omit it and use `hasMore` / `nextCursor` to paginate" + }, + "required": false, + "description": "Also return an exact `total`. Runs a COUNT over every matching row, so it is markedly slower on large result sets; omit it and use `hasMore` / `nextCursor` to paginate", + "name": "includeTotal", + "in": "query" } ], "responses": { @@ -1042,7 +1069,12 @@ "type": "object", "properties": { "total": { - "type": "integer" + "type": "integer", + "description": "Exact match count. Only present when the request set includeTotal=true" + }, + "hasMore": { + "type": "boolean", + "description": "True when another page exists" }, "limit": { "type": "integer" @@ -1132,7 +1164,7 @@ } }, "required": [ - "total", + "hasMore", "limit", "offset", "transfers" @@ -1365,6 +1397,17 @@ "description": "Comma-separated field list", "name": "$select", "in": "query" + }, + { + "schema": { + "type": "boolean", + "nullable": true, + "description": "Also return an exact `total`. Runs a COUNT over every matching row, so it is markedly slower on large result sets; omit it and use `hasMore` / `nextCursor` to paginate" + }, + "required": false, + "description": "Also return an exact `total`. Runs a COUNT over every matching row, so it is markedly slower on large result sets; omit it and use `hasMore` / `nextCursor` to paginate", + "name": "includeTotal", + "in": "query" } ], "responses": { @@ -1376,7 +1419,12 @@ "type": "object", "properties": { "total": { - "type": "integer" + "type": "integer", + "description": "Exact match count. Only present when the request set includeTotal=true" + }, + "hasMore": { + "type": "boolean", + "description": "True when another page exists" }, "limit": { "type": "integer" @@ -1466,7 +1514,7 @@ } }, "required": [ - "total", + "hasMore", "limit", "offset", "transfers" @@ -1639,6 +1687,17 @@ "description": "Comma-separated list of event types", "name": "eventType", "in": "query" + }, + { + "schema": { + "type": "boolean", + "nullable": true, + "description": "Also return an exact `total`. Runs a COUNT over every matching row, so it is markedly slower on large result sets; omit it and use `hasMore` / `nextCursor` to paginate" + }, + "required": false, + "description": "Also return an exact `total`. Runs a COUNT over every matching row, so it is markedly slower on large result sets; omit it and use `hasMore` / `nextCursor` to paginate", + "name": "includeTotal", + "in": "query" } ], "responses": { @@ -2465,6 +2524,17 @@ "description": "Comma-separated field list", "name": "$select", "in": "query" + }, + { + "schema": { + "type": "boolean", + "nullable": true, + "description": "Also return an exact `total`. Runs a COUNT over every matching row, so it is markedly slower on large result sets; omit it and use `hasMore` / `nextCursor` to paginate" + }, + "required": false, + "description": "Also return an exact `total`. Runs a COUNT over every matching row, so it is markedly slower on large result sets; omit it and use `hasMore` / `nextCursor` to paginate", + "name": "includeTotal", + "in": "query" } ], "responses": { @@ -2476,7 +2546,12 @@ "type": "object", "properties": { "total": { - "type": "integer" + "type": "integer", + "description": "Exact match count. Only present when the request set includeTotal=true" + }, + "hasMore": { + "type": "boolean", + "description": "True when another page exists" }, "limit": { "type": "integer" @@ -2566,7 +2641,7 @@ } }, "required": [ - "total", + "hasMore", "limit", "offset", "transfers" @@ -3572,6 +3647,17 @@ "description": "Comma-separated field list", "name": "$select", "in": "query" + }, + { + "schema": { + "type": "boolean", + "nullable": true, + "description": "Also return an exact `total`. Runs a COUNT over every matching row, so it is markedly slower on large result sets; omit it and use `hasMore` / `nextCursor` to paginate" + }, + "required": false, + "description": "Also return an exact `total`. Runs a COUNT over every matching row, so it is markedly slower on large result sets; omit it and use `hasMore` / `nextCursor` to paginate", + "name": "includeTotal", + "in": "query" } ], "responses": { @@ -3583,7 +3669,12 @@ "type": "object", "properties": { "total": { - "type": "integer" + "type": "integer", + "description": "Exact match count. Only present when the request set includeTotal=true" + }, + "hasMore": { + "type": "boolean", + "description": "True when another page exists" }, "limit": { "type": "integer" @@ -3657,7 +3748,7 @@ } }, "required": [ - "total", + "hasMore", "limit", "offset", "transfers" diff --git a/src/__tests__/listPagination.test.ts b/src/__tests__/listPagination.test.ts new file mode 100644 index 000000000..6d3e65863 --- /dev/null +++ b/src/__tests__/listPagination.test.ts @@ -0,0 +1,118 @@ +/** + * List queries must not pay for a COUNT by default (#206). + * + * db.ts resolves its client from `globalThis.prisma`, so a recording stub keeps + * this a unit test with no database. The assertions are about which calls were + * issued (no `count` on the default path) and about the n+1 boundary that + * replaces it, not about row contents. + */ +import { StrKey } from "@stellar/stellar-sdk"; + +type AnyRecord = Record; + +const ALICE = "GDWCO35QUYQLGO6P7OLW4BZWNMMGGUWNPLRVPLCBVG7YNVDZKUDIW4KN"; +// Built from raw bytes so it is a valid strkey by construction. +const CONTRACT = StrKey.encodeContract(Buffer.alloc(32, 7)); + +const calls: Array<{ model: string; op: string; args: AnyRecord }> = []; +let rows: AnyRecord[] = []; +let exactCount = 0; +let transactionCalls = 0; + +const model = (name: string) => + new Proxy( + {}, + { + get: (_t, op: string) => (args: AnyRecord) => { + calls.push({ model: name, op, args: args ?? {} }); + if (op === "count") return Promise.resolve(exactCount); + if (op === "findMany") return Promise.resolve(rows.slice(0, args?.take ?? rows.length)); + return Promise.resolve(null); + }, + } + ) as AnyRecord; + +const stub: AnyRecord = { + tokenTransfer: model("tokenTransfer"), + nftTransfer: model("nftTransfer"), + accountSummary: model("accountSummary"), + $transaction: (ops: unknown) => { + transactionCalls++; + return Array.isArray(ops) ? Promise.all(ops) : (ops as (tx: AnyRecord) => unknown)(stub); + }, +}; +(globalThis as AnyRecord).prisma = stub; + +// Imported after the stub is installed — db.ts binds its client at module load. +import * as db from "../db"; + +const makeRows = (n: number) => Array.from({ length: n }, (_, i) => ({ id: n - i, ledger: 1000 + n - i, amount: "10000000" })); +const countCalls = () => calls.filter((c) => c.op === "count"); + +beforeEach(() => { + calls.length = 0; + rows = []; + exactCount = 0; + transactionCalls = 0; +}); + +it("fixtures are valid Stellar strkeys", () => { + expect(StrKey.isValidEd25519PublicKey(ALICE)).toBe(true); + expect(StrKey.isValidContract(CONTRACT)).toBe(true); +}); + +const listQueries: Array<[string, string, (extra?: AnyRecord) => Promise]> = [ + ["queryTransfers", "tokenTransfer", (e) => db.queryTransfers({ address: ALICE, direction: "incoming", ...e })], + ["queryAllTransfers", "tokenTransfer", (e) => db.queryAllTransfers({ address: ALICE, ...e })], + ["queryNftTransfers", "nftTransfer", (e) => db.queryNftTransfers({ contractId: CONTRACT, ...e })], + ["queryAccountSummaries", "accountSummary", (e) => db.queryAccountSummaries({ address: ALICE, ...e })], +]; + +describe.each(listQueries)("%s", (_name, modelName, run) => { + it("issues no COUNT by default and omits total", async () => { + rows = makeRows(3); + const result = await run({ limit: 10 }); + + expect(countCalls()).toHaveLength(0); + expect(transactionCalls).toBe(0); + expect(calls.filter((c) => c.model === modelName && c.op === "findMany")).toHaveLength(1); + expect(result).not.toHaveProperty("total"); + }); + + it("reports hasMore=false and no cursor when the page is exactly full (boundary)", async () => { + rows = makeRows(5); // limit 5 -> fetches 6, gets 5 + const result = await run({ limit: 5 }); + + expect(result.hasMore).toBe(false); + expect(result.nextCursor).toBeNull(); + expect(result.transfers).toHaveLength(5); + }); + + it("reports hasMore=true with a cursor when one row exists past the page (boundary)", async () => { + rows = makeRows(6); // limit 5 -> fetches 6, gets 6 + const result = await run({ limit: 5 }); + + expect(calls.find((c) => c.op === "findMany")!.args.take).toBe(6); // n+1 + expect(result.hasMore).toBe(true); + expect(result.nextCursor).toEqual(expect.any(String)); + expect(result.transfers).toHaveLength(5); + }); + + it("returns an exact total only when includeTotal is requested", async () => { + rows = makeRows(2); + exactCount = 1234; + const result = await run({ limit: 10, includeTotal: true }); + + expect(countCalls()).toHaveLength(1); + expect(countCalls()[0].model).toBe(modelName); + expect(result.total).toBe(1234); + expect(result.hasMore).toBe(false); + }); + + it("includeTotal=false behaves like the default", async () => { + rows = makeRows(2); + const result = await run({ limit: 10, includeTotal: false }); + expect(countCalls()).toHaveLength(0); + expect(result).not.toHaveProperty("total"); + }); +}); diff --git a/src/__tests__/network.test.ts b/src/__tests__/network.test.ts index 3eda94ca9..2aa28152f 100644 --- a/src/__tests__/network.test.ts +++ b/src/__tests__/network.test.ts @@ -152,7 +152,8 @@ describe("reads are network-scoped", () => { it("counts and rows agree on the network, so total matches the page", async () => { process.env.STELLAR_NETWORK = "mainnet"; - await db.queryTransfers({ address: ADDR, direction: "incoming" }); + // The count is opt-in (#206); ask for it so both queries are recorded. + await db.queryTransfers({ address: ADDR, direction: "incoming", includeTotal: true }); expect(lastWhere("tokenTransfer", "count").network).toBe("mainnet"); expect(lastWhere("tokenTransfer", "findMany").network).toBe("mainnet"); }); diff --git a/src/__tests__/routes/transfers.test.ts b/src/__tests__/routes/transfers.test.ts index 709964699..10a295ff8 100644 --- a/src/__tests__/routes/transfers.test.ts +++ b/src/__tests__/routes/transfers.test.ts @@ -117,7 +117,7 @@ describe("Transfer route handlers", () => { describe("GET /transfers/incoming/:address", () => { it("returns all incoming transfers for a known address", async () => { const incoming = SEED_TRANSFERS.filter((t) => t.toAddress === ALICE); - mockQueryTransfers.mockResolvedValue({ total: incoming.length, transfers: incoming, nextCursor: null }); + mockQueryTransfers.mockResolvedValue({ total: incoming.length, transfers: incoming, hasMore: false, nextCursor: null }); const res = await request(app).get(`/transfers/incoming/${ALICE}`); @@ -128,9 +128,25 @@ describe("Transfer route handlers", () => { expect(res.body.offset).toBe(0); }); + it("does not ask for a total by default, and forwards includeTotal=true when set", async () => { + mockQueryTransfers.mockResolvedValue({ transfers: [], hasMore: false, nextCursor: null }); + + const plain = await request(app).get(`/transfers/incoming/${ALICE}`); + expect(plain.status).toBe(200); + expect(plain.body).not.toHaveProperty("total"); + expect(plain.body.hasMore).toBe(false); + expect(mockQueryTransfers.mock.calls.at(-1)![0].includeTotal).toBeUndefined(); + + mockQueryTransfers.mockResolvedValue({ total: 7, transfers: [], hasMore: false, nextCursor: null }); + const counted = await request(app).get(`/transfers/incoming/${ALICE}`).query({ includeTotal: "true" }); + expect(counted.status).toBe(200); + expect(counted.body.total).toBe(7); + expect(mockQueryTransfers.mock.calls.at(-1)![0].includeTotal).toBe(true); + }); + it("attaches displayAmount to every transfer", async () => { const transfer = makeTransfer({ amount: "10000000" }); - mockQueryTransfers.mockResolvedValue({ total: 1, transfers: [transfer], nextCursor: null }); + mockQueryTransfers.mockResolvedValue({ total: 1, transfers: [transfer], hasMore: false, nextCursor: null }); const res = await request(app).get(`/transfers/incoming/${ALICE}`); @@ -139,7 +155,7 @@ describe("Transfer route handlers", () => { }); it("returns empty array for an unknown address", async () => { - mockQueryTransfers.mockResolvedValue({ total: 0, transfers: [], nextCursor: null }); + mockQueryTransfers.mockResolvedValue({ total: 0, transfers: [], hasMore: false, nextCursor: null }); const res = await request(app).get("/transfers/incoming/GUNKNOWNADDRESS"); @@ -152,7 +168,7 @@ describe("Transfer route handlers", () => { const filtered = SEED_TRANSFERS.filter( (t) => t.toAddress === ALICE && t.contractId === CONTRACT_A ); - mockQueryTransfers.mockResolvedValue({ total: filtered.length, transfers: filtered, nextCursor: null }); + mockQueryTransfers.mockResolvedValue({ total: filtered.length, transfers: filtered, hasMore: false, nextCursor: null }); const res = await request(app) .get(`/transfers/incoming/${ALICE}`) @@ -168,6 +184,7 @@ describe("Transfer route handlers", () => { mockQueryTransfers.mockResolvedValue({ total: 1, transfers: [makeTransfer({ amount: "10000000" })], + hasMore: true, nextCursor: "cursor-1", }); @@ -192,7 +209,7 @@ describe("Transfer route handlers", () => { }); it("passes fromDate and toDate to queryTransfers", async () => { - mockQueryTransfers.mockResolvedValue({ total: 2, transfers: SEED_TRANSFERS.slice(14, 16), nextCursor: null }); + mockQueryTransfers.mockResolvedValue({ total: 2, transfers: SEED_TRANSFERS.slice(14, 16), hasMore: false, nextCursor: null }); const res = await request(app) .get(`/transfers/incoming/${ALICE}`) @@ -232,7 +249,7 @@ describe("Transfer route handlers", () => { }); it("accepts valid eventType values", async () => { - mockQueryTransfers.mockResolvedValue({ total: 1, transfers: [makeTransfer({ eventType: "mint" })], nextCursor: null }); + mockQueryTransfers.mockResolvedValue({ total: 1, transfers: [makeTransfer({ eventType: "mint" })], hasMore: false, nextCursor: null }); const res = await request(app) .get(`/transfers/incoming/${ALICE}`) @@ -245,7 +262,7 @@ describe("Transfer route handlers", () => { }); it("accepts comma-separated eventType values", async () => { - mockQueryTransfers.mockResolvedValue({ total: 2, transfers: [], nextCursor: null }); + mockQueryTransfers.mockResolvedValue({ total: 2, transfers: [], hasMore: false, nextCursor: null }); const res = await request(app) .get(`/transfers/incoming/${ALICE}`) @@ -259,7 +276,7 @@ describe("Transfer route handlers", () => { it("honours limit and offset for pagination", async () => { const page = SEED_TRANSFERS.slice(0, 5); - mockQueryTransfers.mockResolvedValue({ total: 20, transfers: page, nextCursor: null }); + mockQueryTransfers.mockResolvedValue({ total: 20, transfers: page, hasMore: false, nextCursor: null }); const res = await request(app) .get(`/transfers/incoming/${ALICE}`) @@ -274,7 +291,7 @@ describe("Transfer route handlers", () => { }); it("falls back to limit=50, offset=0 when not provided", async () => { - mockQueryTransfers.mockResolvedValue({ total: 0, transfers: [], nextCursor: null }); + mockQueryTransfers.mockResolvedValue({ total: 0, transfers: [], hasMore: false, nextCursor: null }); await request(app).get(`/transfers/incoming/${ALICE}`); @@ -284,7 +301,7 @@ describe("Transfer route handlers", () => { }); it("forwards fromLedger and toLedger filters", async () => { - mockQueryTransfers.mockResolvedValue({ total: 3, transfers: SEED_TRANSFERS.slice(0, 3), nextCursor: null }); + mockQueryTransfers.mockResolvedValue({ total: 3, transfers: SEED_TRANSFERS.slice(0, 3), hasMore: false, nextCursor: null }); await request(app) .get(`/transfers/incoming/${ALICE}`) @@ -300,7 +317,7 @@ describe("Transfer route handlers", () => { describe("GET /transfers/outgoing/:address", () => { it("returns outgoing transfers with direction=outgoing", async () => { const outgoing = SEED_TRANSFERS.filter((t) => t.fromAddress === ALICE); - mockQueryTransfers.mockResolvedValue({ total: outgoing.length, transfers: outgoing, nextCursor: null }); + mockQueryTransfers.mockResolvedValue({ total: outgoing.length, transfers: outgoing, hasMore: false, nextCursor: null }); const res = await request(app).get(`/transfers/outgoing/${ALICE}`); @@ -312,7 +329,7 @@ describe("Transfer route handlers", () => { }); it("returns empty array for address with no outgoing transfers", async () => { - mockQueryTransfers.mockResolvedValue({ total: 0, transfers: [], nextCursor: null }); + mockQueryTransfers.mockResolvedValue({ total: 0, transfers: [], hasMore: false, nextCursor: null }); const res = await request(app).get(`/transfers/outgoing/GNOBODY`); @@ -322,7 +339,7 @@ describe("Transfer route handlers", () => { it("attaches displayAmount for large i128 amounts", async () => { const t = makeTransfer({ amount: "1000000000000000" }); // 100000000.0000000 - mockQueryTransfers.mockResolvedValue({ total: 1, transfers: [t], nextCursor: null }); + mockQueryTransfers.mockResolvedValue({ total: 1, transfers: [t], hasMore: false, nextCursor: null }); const res = await request(app).get(`/transfers/outgoing/${ALICE}`); @@ -345,7 +362,7 @@ describe("Transfer route handlers", () => { (t) => t.toAddress === ALICE || t.fromAddress === ALICE ).map((t) => ({ ...t, direction: t.toAddress === ALICE ? "incoming" : "outgoing" })); - mockQueryAllTransfers.mockResolvedValue({ total: combined.length, transfers: combined, nextCursor: null }); + mockQueryAllTransfers.mockResolvedValue({ total: combined.length, transfers: combined, hasMore: false, nextCursor: null }); const res = await request(app).get(`/transfers/address/${ALICE}`); @@ -357,7 +374,7 @@ describe("Transfer route handlers", () => { it("direction field is present on each record", async () => { const t1 = { ...makeTransfer({ id: 1, toAddress: ALICE, fromAddress: BOB }), direction: "incoming" }; const t2 = { ...makeTransfer({ id: 2, toAddress: BOB, fromAddress: ALICE }), direction: "outgoing" }; - mockQueryAllTransfers.mockResolvedValue({ total: 2, transfers: [t1, t2], nextCursor: null }); + mockQueryAllTransfers.mockResolvedValue({ total: 2, transfers: [t1, t2], hasMore: false, nextCursor: null }); const res = await request(app).get(`/transfers/address/${ALICE}`); @@ -366,7 +383,7 @@ describe("Transfer route handlers", () => { }); it("returns empty array for unknown address", async () => { - mockQueryAllTransfers.mockResolvedValue({ total: 0, transfers: [], nextCursor: null }); + mockQueryAllTransfers.mockResolvedValue({ total: 0, transfers: [], hasMore: false, nextCursor: null }); const res = await request(app).get("/transfers/address/GUNKNOWN"); @@ -375,7 +392,7 @@ describe("Transfer route handlers", () => { }); it("honours pagination params", async () => { - mockQueryAllTransfers.mockResolvedValue({ total: 20, transfers: [], nextCursor: "cursor-2" }); + mockQueryAllTransfers.mockResolvedValue({ total: 20, transfers: [], hasMore: true, nextCursor: "cursor-2" }); await request(app) .get(`/transfers/address/${ALICE}`) @@ -387,7 +404,7 @@ describe("Transfer route handlers", () => { }); it("filters by contractId", async () => { - mockQueryAllTransfers.mockResolvedValue({ total: 3, transfers: [], nextCursor: null }); + mockQueryAllTransfers.mockResolvedValue({ total: 3, transfers: [], hasMore: false, nextCursor: null }); await request(app) .get(`/transfers/address/${ALICE}`) @@ -417,6 +434,7 @@ describe("Transfer route handlers", () => { mockQueryAllTransfers.mockResolvedValue({ total: tokenFiltered.length, transfers: tokenFiltered, + hasMore: false, nextCursor: null, }); @@ -455,7 +473,7 @@ describe("Transfer route handlers", () => { .filter((t) => t.toAddress === ALICE || t.fromAddress === ALICE) .map((t) => ({ ...t, direction: t.toAddress === ALICE ? "incoming" : "outgoing" })); - mockQueryAllTransfers.mockResolvedValue({ total: combined.length, transfers: combined, nextCursor: null }); + mockQueryAllTransfers.mockResolvedValue({ total: combined.length, transfers: combined, hasMore: false, nextCursor: null }); const res = await request(app).get(`/transfers/address/${ALICE}`); @@ -547,19 +565,19 @@ describe("Transfer route handlers", () => { // ── toDisplayAmount edge cases ───────────────────────────────────────────── describe("toDisplayAmount formatting", () => { it("formats 0 correctly", async () => { - mockQueryTransfers.mockResolvedValue({ total: 1, transfers: [makeTransfer({ amount: "0" })], nextCursor: null }); + mockQueryTransfers.mockResolvedValue({ total: 1, transfers: [makeTransfer({ amount: "0" })], hasMore: false, nextCursor: null }); const res = await request(app).get(`/transfers/incoming/${ALICE}`); expect(res.body.transfers[0].displayAmount).toBe("0.0000000"); }); it("formats small amounts with leading zeros in fractional part", async () => { - mockQueryTransfers.mockResolvedValue({ total: 1, transfers: [makeTransfer({ amount: "1" })], nextCursor: null }); + mockQueryTransfers.mockResolvedValue({ total: 1, transfers: [makeTransfer({ amount: "1" })], hasMore: false, nextCursor: null }); const res = await request(app).get(`/transfers/incoming/${ALICE}`); expect(res.body.transfers[0].displayAmount).toBe("0.0000001"); }); it("formats exactly 1 token (10000000 stroops)", async () => { - mockQueryTransfers.mockResolvedValue({ total: 1, transfers: [makeTransfer({ amount: "10000000" })], nextCursor: null }); + mockQueryTransfers.mockResolvedValue({ total: 1, transfers: [makeTransfer({ amount: "10000000" })], hasMore: false, nextCursor: null }); const res = await request(app).get(`/transfers/incoming/${ALICE}`); expect(res.body.transfers[0].displayAmount).toBe("1.0000000"); }); @@ -607,7 +625,7 @@ describe("Transfer route handlers", () => { eventType: "transfer", }), direction: "incoming" as const }, ]; - mockQueryAllTransfers.mockResolvedValue({ total: 1, transfers, nextCursor: null }); + mockQueryAllTransfers.mockResolvedValue({ total: 1, transfers, hasMore: false, nextCursor: null }); const res = await request(app).get(`/transfers/address/${ALICE}/export.csv`); @@ -639,7 +657,7 @@ describe("Transfer route handlers", () => { eventType: "mint", }), direction: "incoming" as const }, ]; - mockQueryAllTransfers.mockResolvedValue({ total: 2, transfers, nextCursor: null }); + mockQueryAllTransfers.mockResolvedValue({ total: 2, transfers, hasMore: false, nextCursor: null }); const res = await request(app).get(`/transfers/address/${ALICE}/export.csv`); @@ -665,7 +683,7 @@ describe("Transfer route handlers", () => { eventType: "transfer", }), direction: "outgoing" as const }, ]; - mockQueryAllTransfers.mockResolvedValue({ total: 1, transfers, nextCursor: null }); + mockQueryAllTransfers.mockResolvedValue({ total: 1, transfers, hasMore: false, nextCursor: null }); const res = await request(app).get(`/transfers/address/${ALICE}/export.csv`); @@ -686,7 +704,7 @@ describe("Transfer route handlers", () => { eventType: "mint", }), direction: "incoming" as const }, ]; - mockQueryAllTransfers.mockResolvedValue({ total: 1, transfers, nextCursor: null }); + mockQueryAllTransfers.mockResolvedValue({ total: 1, transfers, hasMore: false, nextCursor: null }); const res = await request(app).get(`/transfers/address/${ALICE}/export.csv`); @@ -707,7 +725,7 @@ describe("Transfer route handlers", () => { eventType: "burn", }), direction: "outgoing" as const }, ]; - mockQueryAllTransfers.mockResolvedValue({ total: 1, transfers, nextCursor: null }); + mockQueryAllTransfers.mockResolvedValue({ total: 1, transfers, hasMore: false, nextCursor: null }); const res = await request(app).get(`/transfers/address/${ALICE}/export.csv`); @@ -717,7 +735,7 @@ describe("Transfer route handlers", () => { }); it("sets Content-Disposition header with filename", async () => { - mockQueryAllTransfers.mockResolvedValue({ total: 0, transfers: [], nextCursor: null }); + mockQueryAllTransfers.mockResolvedValue({ total: 0, transfers: [], hasMore: false, nextCursor: null }); const res = await request(app).get(`/transfers/address/${ALICE}/export.csv`); @@ -728,7 +746,7 @@ describe("Transfer route handlers", () => { }); it("respects contractId filter", async () => { - mockQueryAllTransfers.mockResolvedValue({ total: 0, transfers: [], nextCursor: null }); + mockQueryAllTransfers.mockResolvedValue({ total: 0, transfers: [], hasMore: false, nextCursor: null }); await request(app) .get(`/transfers/address/${ALICE}/export.csv`) @@ -740,7 +758,7 @@ describe("Transfer route handlers", () => { }); it("respects token filter for CSV export", async () => { - mockQueryAllTransfers.mockResolvedValue({ total: 0, transfers: [], nextCursor: null }); + mockQueryAllTransfers.mockResolvedValue({ total: 0, transfers: [], hasMore: false, nextCursor: null }); await request(app) .get(`/transfers/address/${ALICE}/export.csv`) @@ -761,7 +779,7 @@ describe("Transfer route handlers", () => { }); it("respects fromDate and toDate filters", async () => { - mockQueryAllTransfers.mockResolvedValue({ total: 0, transfers: [], nextCursor: null }); + mockQueryAllTransfers.mockResolvedValue({ total: 0, transfers: [], hasMore: false, nextCursor: null }); await request(app) .get(`/transfers/address/${ALICE}/export.csv`) @@ -779,7 +797,7 @@ describe("Transfer route handlers", () => { }); it("respects eventType filter", async () => { - mockQueryAllTransfers.mockResolvedValue({ total: 0, transfers: [], nextCursor: null }); + mockQueryAllTransfers.mockResolvedValue({ total: 0, transfers: [], hasMore: false, nextCursor: null }); await request(app) .get(`/transfers/address/${ALICE}/export.csv`) @@ -791,7 +809,7 @@ describe("Transfer route handlers", () => { }); it("enforces a 10,000 row cap for export", async () => { - mockQueryAllTransfers.mockResolvedValue({ total: 50000, transfers: [], nextCursor: null }); + mockQueryAllTransfers.mockResolvedValue({ total: 50000, transfers: [], hasMore: false, nextCursor: null }); await request(app).get(`/transfers/address/${ALICE}/export.csv`); @@ -801,7 +819,7 @@ describe("Transfer route handlers", () => { }); it("always uses offset=0 for CSV export", async () => { - mockQueryAllTransfers.mockResolvedValue({ total: 100, transfers: [], nextCursor: null }); + mockQueryAllTransfers.mockResolvedValue({ total: 100, transfers: [], hasMore: false, nextCursor: null }); await request(app).get(`/transfers/address/${ALICE}/export.csv`); @@ -829,7 +847,7 @@ describe("Transfer route handlers", () => { }); it("returns empty CSV (header only) for address with no transfers", async () => { - mockQueryAllTransfers.mockResolvedValue({ total: 0, transfers: [], nextCursor: null }); + mockQueryAllTransfers.mockResolvedValue({ total: 0, transfers: [], hasMore: false, nextCursor: null }); const res = await request(app).get(`/transfers/address/${ALICE}/export.csv`); @@ -850,7 +868,7 @@ describe("Transfer route handlers", () => { eventType: "transfer", }), direction: "incoming" as const }, ]; - mockQueryAllTransfers.mockResolvedValue({ total: 1, transfers, nextCursor: null }); + mockQueryAllTransfers.mockResolvedValue({ total: 1, transfers, hasMore: false, nextCursor: null }); const res = await request(app).get(`/transfers/address/${ALICE}/export.csv`); @@ -863,7 +881,7 @@ describe("Transfer route handlers", () => { describe("JSON:API content negotiation", () => { it("returns JSON:API format for transfers/address/:address", async () => { const t = { ...makeTransfer({ id: 1, toAddress: ALICE, fromAddress: BOB, eventType: "transfer", ledger: 1001, amount: "10000000" }), direction: "incoming" as const }; - mockQueryAllTransfers.mockResolvedValue({ total: 1, transfers: [t], nextCursor: null }); + mockQueryAllTransfers.mockResolvedValue({ total: 1, transfers: [t], hasMore: false, nextCursor: null }); const res = await request(app) .get(`/transfers/address/${ALICE}`) @@ -879,7 +897,7 @@ describe("Transfer route handlers", () => { it("returns JSON:API format for transfers/incoming/:address", async () => { const t = { ...makeTransfer({ id: 1, toAddress: ALICE, fromAddress: BOB }), direction: "incoming" as const }; - mockQueryTransfers.mockResolvedValue({ total: 1, transfers: [t], nextCursor: null }); + mockQueryTransfers.mockResolvedValue({ total: 1, transfers: [t], hasMore: false, nextCursor: null }); const res = await request(app) .get(`/transfers/incoming/${ALICE}`) diff --git a/src/__tests__/staleReads.test.ts b/src/__tests__/staleReads.test.ts index fbbec829f..2d65d0ece 100644 --- a/src/__tests__/staleReads.test.ts +++ b/src/__tests__/staleReads.test.ts @@ -65,6 +65,7 @@ describe("Graceful stale reads during RPC outage (#164)", () => { eventId: "evt123", }, ], + hasMore: false, nextCursor: null, }); @@ -80,7 +81,7 @@ describe("Graceful stale reads during RPC outage (#164)", () => { it("resumes normal non-stale headers after RPC recovers", async () => { mockGetLatestLedger.mockRejectedValueOnce(new Error("RPC outage")); - mockQueryTransfers.mockResolvedValue({ total: 0, transfers: [], nextCursor: null }); + mockQueryTransfers.mockResolvedValue({ total: 0, transfers: [], hasMore: false, nextCursor: null }); const res1 = await request(app).get(`/transfers/incoming/${ALICE}`); expect(res1.status).toBe(200); diff --git a/src/api.ts b/src/api.ts index 96128f439..e5fc684d5 100644 --- a/src/api.ts +++ b/src/api.ts @@ -525,6 +525,7 @@ export function createApp(): express.Application { eventTypes: eventType as string[] | undefined, limit, offset, + includeTotal: (parsed as { includeTotal?: boolean }).includeTotal, }); res.json({ @@ -587,6 +588,7 @@ export function createApp(): express.Application { eventTypes: eventType as string[] | undefined, limit, offset, + includeTotal: (parsed as { includeTotal?: boolean }).includeTotal, }); res.json({ @@ -659,6 +661,7 @@ export function createApp(): express.Application { eventTypes: eventType as string[] | undefined, limit, offset, + includeTotal: (parsed as { includeTotal?: boolean }).includeTotal, }); res.json({ @@ -940,6 +943,7 @@ export function createApp(): express.Application { toLedger, limit, offset, + includeTotal: (parsed as { includeTotal?: boolean }).includeTotal, }); res.json({ ...result, limit, offset }); diff --git a/src/db.ts b/src/db.ts index 9b0e7221d..628bb5238 100644 --- a/src/db.ts +++ b/src/db.ts @@ -61,17 +61,30 @@ export interface TransferRecord { type ListPage = { rows: T[]; nextCursor: string | null; + /** True when the n+1 fetch found a row past this page. */ + hasMore: boolean; }; +/** + * A `COUNT(*)` over the same filter costs as much as walking every matching + * row, and on the tables this service exists to grow it dominates the request. + * Lists therefore skip it unless the caller asks with `includeTotal` (#206); + * `hasMore` from the n+1 fetch already answers what pagination needs. + */ +function totalField(total: number | undefined): { total?: number } { + return total === undefined ? {} : { total }; +} + function buildListPage(rows: T[], limit: number): ListPage { if (rows.length <= limit) { - return { rows, nextCursor: null }; + return { rows, nextCursor: null, hasMore: false }; } const pageRows = rows.slice(0, limit); return { rows: pageRows, nextCursor: encodeCursor(pageRows[pageRows.length - 1].id), + hasMore: true, }; } @@ -355,6 +368,8 @@ export type TransferQueryParams = { eventTypes?: string[]; limit?: number; offset?: number; + /** Also run an exact COUNT. Off by default because it is expensive. */ + includeTotal?: boolean; }; export async function queryTransfers(params: TransferQueryParams) { @@ -374,6 +389,7 @@ export async function queryTransfers(params: TransferQueryParams) { eventTypes, limit = 50, offset = 0, + includeTotal = false, } = params; const baseWhere: Prisma.TokenTransferWhereInput = { @@ -426,23 +442,24 @@ export async function queryTransfers(params: TransferQueryParams) { const cap = Math.min(limit, 200); const cursorId = decodeCursor(cursor); - const [total, transfers] = await observeDbQuery("queryTransfers", () => - prisma.$transaction([ - prisma.tokenTransfer.count({ where }), - prisma.tokenTransfer.findMany({ - where, - orderBy: [{ ledger: "desc" }, { id: "desc" }], - take: cap + 1, - ...(cursorId ? { cursor: { id: cursorId }, skip: 1 } : { skip: offset }), - ...(prismaSelect ? { select: prismaSelect } : {}), - }), - ]) - ); + const [total, transfers] = await observeDbQuery("queryTransfers", async () => { + const rowsQuery = prisma.tokenTransfer.findMany({ + where, + orderBy: [{ ledger: "desc" }, { id: "desc" }], + take: cap + 1, + ...(cursorId ? { cursor: { id: cursorId }, skip: 1 } : { skip: offset }), + ...(prismaSelect ? { select: prismaSelect } : {}), + }); + return includeTotal + ? prisma.$transaction([prisma.tokenTransfer.count({ where }), rowsQuery]) + : ([undefined, await rowsQuery] as const); + }); const page = buildListPage(transfers as Array<{ id: number }>, cap); return { - total, + ...totalField(total), + hasMore: page.hasMore, transfers: selectRows(page.rows as Array>, requestedSelect, { displayAmount: (row) => toDisplayAmount(String((row as { amount?: string }).amount)), }), @@ -606,6 +623,8 @@ export type NftTransferQueryParams = { toLedger?: number; limit?: number; offset?: number; + /** Also run an exact COUNT. Off by default because it is expensive. */ + includeTotal?: boolean; }; export async function queryNftTransfers(params: NftTransferQueryParams) { @@ -621,6 +640,7 @@ export async function queryNftTransfers(params: NftTransferQueryParams) { toLedger, limit = 50, offset = 0, + includeTotal = false, } = params; const baseWhere: Prisma.NftTransferWhereInput = { @@ -661,21 +681,22 @@ export async function queryNftTransfers(params: NftTransferQueryParams) { const cap = Math.min(limit, 200); const cursorId = decodeCursor(cursor); - const [total, transfers] = await prisma.$transaction([ - prisma.nftTransfer.count({ where }), - prisma.nftTransfer.findMany({ - where, - orderBy: [{ ledger: "desc" }, { id: "desc" }], - take: cap + 1, - ...(cursorId ? { cursor: { id: cursorId }, skip: 1 } : { skip: offset }), - ...(prismaSelect ? { select: prismaSelect } : {}), - }), - ]); + const rowsQuery = prisma.nftTransfer.findMany({ + where, + orderBy: [{ ledger: "desc" }, { id: "desc" }], + take: cap + 1, + ...(cursorId ? { cursor: { id: cursorId }, skip: 1 } : { skip: offset }), + ...(prismaSelect ? { select: prismaSelect } : {}), + }); + const [total, transfers] = includeTotal + ? await prisma.$transaction([prisma.nftTransfer.count({ where }), rowsQuery]) + : ([undefined, await rowsQuery] as const); const page = buildListPage(transfers as Array<{ id: number }>, cap); return { - total, + ...totalField(total), + hasMore: page.hasMore, transfers: selectRows(page.rows as Array>, requestedSelect), nextCursor: page.nextCursor, }; @@ -807,10 +828,22 @@ export type AccountSummaryQueryParams = { cursor?: string; limit?: number; offset?: number; + /** Also run an exact COUNT. Off by default because it is expensive. */ + includeTotal?: boolean; }; export async function queryAccountSummaries(params: AccountSummaryQueryParams) { - const { network, address, contractId, filter, select, cursor, limit = 50, offset = 0 } = params; + const { + network, + address, + contractId, + filter, + select, + cursor, + limit = 50, + offset = 0, + includeTotal = false, + } = params; const baseWhere: Prisma.AccountSummaryWhereInput = { network: resolveNetwork(network), @@ -840,21 +873,22 @@ export async function queryAccountSummaries(params: AccountSummaryQueryParams) { const cap = Math.min(limit, 200); const cursorId = decodeCursor(cursor); - const [total, rows] = await prisma.$transaction([ - prisma.accountSummary.count({ where }), - prisma.accountSummary.findMany({ - where, - orderBy: [{ lastActivityAt: "desc" }, { id: "desc" }], - take: cap + 1, - ...(cursorId ? { cursor: { id: cursorId }, skip: 1 } : { skip: offset }), - ...(prismaSelect ? { select: prismaSelect } : {}), - }), - ]); + const rowsQuery = prisma.accountSummary.findMany({ + where, + orderBy: [{ lastActivityAt: "desc" }, { id: "desc" }], + take: cap + 1, + ...(cursorId ? { cursor: { id: cursorId }, skip: 1 } : { skip: offset }), + ...(prismaSelect ? { select: prismaSelect } : {}), + }); + const [total, rows] = includeTotal + ? await prisma.$transaction([prisma.accountSummary.count({ where }), rowsQuery]) + : ([undefined, await rowsQuery] as const); const page = buildListPage(rows as Array<{ id: number }>, cap); return { - total, + ...totalField(total), + hasMore: page.hasMore, transfers: selectRows(page.rows as Array>, requestedSelect, { displayTotalSent: (row) => row.totalSent, displayTotalReceived: (row) => row.totalReceived, @@ -880,6 +914,8 @@ export type AllTransfersQueryParams = { eventTypes?: string[]; limit?: number; offset?: number; + /** Also run an exact COUNT. Off by default because it is expensive. */ + includeTotal?: boolean; }; export async function queryAllTransfers(params: AllTransfersQueryParams) { @@ -898,6 +934,7 @@ export async function queryAllTransfers(params: AllTransfersQueryParams) { eventTypes, limit = 50, offset = 0, + includeTotal = false, } = params; const baseWhere: Prisma.TokenTransferWhereInput = { @@ -949,16 +986,16 @@ export async function queryAllTransfers(params: AllTransfersQueryParams) { } : undefined; - const [total, rows] = await prisma.$transaction([ - prisma.tokenTransfer.count({ where }), - prisma.tokenTransfer.findMany({ - where, - orderBy: [{ ledger: "desc" }, { id: "desc" }], - take: cap + 1, - ...(cursorId ? { cursor: { id: cursorId }, skip: 1 } : { skip: offset }), - ...(prismaSelect ? { select: prismaSelect } : {}), - }), - ]); + const rowsQuery = prisma.tokenTransfer.findMany({ + where, + orderBy: [{ ledger: "desc" }, { id: "desc" }], + take: cap + 1, + ...(cursorId ? { cursor: { id: cursorId }, skip: 1 } : { skip: offset }), + ...(prismaSelect ? { select: prismaSelect } : {}), + }); + const [total, rows] = includeTotal + ? await prisma.$transaction([prisma.tokenTransfer.count({ where }), rowsQuery]) + : ([undefined, await rowsQuery] as const); const page = buildListPage(rows as Array<{ id: number }>, cap); @@ -967,7 +1004,7 @@ export async function queryAllTransfers(params: AllTransfersQueryParams) { direction: (row) => ((row as { toAddress?: string | null }).toAddress === address ? "incoming" : "outgoing"), }); - return { total, transfers, nextCursor: page.nextCursor }; + return { ...totalField(total), hasMore: page.hasMore, transfers, nextCursor: page.nextCursor }; } // ─── Popular assets query ────────────────────────────────────────────────── diff --git a/src/graphql/server.ts b/src/graphql/server.ts index c45de7c45..84f539a0a 100644 --- a/src/graphql/server.ts +++ b/src/graphql/server.ts @@ -43,7 +43,9 @@ export const typeDefs = `#graphql } type TransferConnection { - total: Int! + "Exact match count. Null unless the query passed includeTotal: true, because it costs a full COUNT." + total: Int + hasMore: Boolean! transfers: [Transfer!]! nextCursor: String } @@ -65,6 +67,8 @@ export const typeDefs = `#graphql network: Network limit: Int = 50 offset: Int = 0 + "Also compute the exact total. Slower on large result sets." + includeTotal: Boolean = false ): TransferConnection! transferByTx(txHash: String!, network: Network): [Transfer!]! summary(address: String!, contractId: String, network: Network): [TokenSummary!]! @@ -132,6 +136,7 @@ export const resolvers = { network?: NetworkArg; limit?: number; offset?: number; + includeTotal?: boolean; }, ctx?: GraphQLContext ) => { @@ -141,6 +146,7 @@ export const resolvers = { contractId: args.contractId, limit: args.limit, offset: args.offset, + includeTotal: args.includeTotal, }; const result = diff --git a/src/middleware/jsonapi.ts b/src/middleware/jsonapi.ts index 8cec74c8d..85d3d66be 100644 --- a/src/middleware/jsonapi.ts +++ b/src/middleware/jsonapi.ts @@ -95,11 +95,12 @@ function recordToResource(record: Record, type: string): JsonAp return resource; } -function transformTransfersResponse(body: { transfers: Record[]; total?: number; limit?: number; offset?: number; nextCursor?: string | null }, endpoint: string): { data: JsonApiResource[]; meta: Record } { +function transformTransfersResponse(body: { transfers: Record[]; total?: number; hasMore?: boolean; limit?: number; offset?: number; nextCursor?: string | null }, endpoint: string): { data: JsonApiResource[]; meta: Record } { const type = determineResourceType(endpoint, body) ?? "transfer"; const data = body.transfers.map((t) => recordToResource(t as Record, type)); const meta: Record = {}; if (typeof body.total === "number") meta.total = body.total; + if (typeof body.hasMore === "boolean") meta.hasMore = body.hasMore; if (typeof body.limit === "number") meta.limit = body.limit; if (typeof body.offset === "number") meta.offset = body.offset; if (body.nextCursor) meta.nextCursor = body.nextCursor; @@ -131,10 +132,11 @@ function transformAssetsResponse(body: { window: string; by: string; limit: numb return { data, meta }; } -function transformNftTransfersResponse(body: { transfers: Record[]; total?: number; limit?: number; offset?: number; nextCursor?: string | null }): { data: JsonApiResource[]; meta: Record } { +function transformNftTransfersResponse(body: { transfers: Record[]; total?: number; hasMore?: boolean; limit?: number; offset?: number; nextCursor?: string | null }): { data: JsonApiResource[]; meta: Record } { const data = body.transfers.map((t) => recordToResource(t as Record, "nft-transfer")); const meta: Record = {}; if (typeof body.total === "number") meta.total = body.total; + if (typeof body.hasMore === "boolean") meta.hasMore = body.hasMore; if (typeof body.limit === "number") meta.limit = body.limit; if (typeof body.offset === "number") meta.offset = body.offset; if (body.nextCursor) meta.nextCursor = body.nextCursor; @@ -182,7 +184,7 @@ function transformToJsonApi(body: unknown, endpoint: string, statusCode: number const bodyRecord = body as Record; if (bodyRecord.transfers && Array.isArray(bodyRecord.transfers)) { - return transformTransfersResponse(bodyRecord as { transfers: Record[]; total?: number; limit?: number; offset?: number; nextCursor?: string | null }, endpoint) as unknown as JsonApiResponse; + return transformTransfersResponse(bodyRecord as { transfers: Record[]; total?: number; hasMore?: boolean; limit?: number; offset?: number; nextCursor?: string | null }, endpoint) as unknown as JsonApiResponse; } if (bodyRecord.tokens && Array.isArray(bodyRecord.tokens)) { diff --git a/src/openapi/schemas.ts b/src/openapi/schemas.ts index 1849a7700..3d27b88a1 100644 --- a/src/openapi/schemas.ts +++ b/src/openapi/schemas.ts @@ -88,6 +88,18 @@ const queryIntWithDefault = (defaultValue: number, options: { min?: number; max? .openapi({ description: options.description }) ); +const optionalQueryBool = (description?: string) => + z.preprocess( + (value) => { + const raw = firstValue(value); + if (raw === undefined || raw === null || raw === "") return undefined; + if (raw === "true" || raw === "1") return true; + if (raw === "false" || raw === "0") return false; + return raw; + }, + z.boolean().optional().openapi({ description }) + ); + const optionalCommaList = (items: z.ZodTypeAny, description?: string) => z.preprocess( (value) => { @@ -226,7 +238,8 @@ export const transferSchema = z.object({ }); export const transferListResponseSchema = z.object({ - total: z.number().int(), + total: z.number().int().optional().openapi({ description: "Exact match count. Only present when the request set includeTotal=true" }), + hasMore: z.boolean().openapi({ description: "True when another page exists" }), limit: z.number().int(), offset: z.number().int(), nextCursor: z.string().nullable().optional(), @@ -362,7 +375,8 @@ export const nftTransferSchema = z.object({ }); export const nftTransfersResponseSchema = z.object({ - total: z.number().int(), + total: z.number().int().optional().openapi({ description: "Exact match count. Only present when the request set includeTotal=true" }), + hasMore: z.boolean().openapi({ description: "True when another page exists" }), limit: z.number().int(), offset: z.number().int(), nextCursor: z.string().nullable().optional(), @@ -455,6 +469,7 @@ export const transferQuerySchema = z.object({ cursor: optionalQueryString("Opaque pagination cursor"), $filter: optionalQueryString("OData filter expression"), $select: optionalCommaList(z.string(), "Comma-separated field list"), + includeTotal: optionalQueryBool("Also return an exact `total`. Runs a COUNT over every matching row, so it is markedly slower on large result sets; omit it and use `hasMore` / `nextCursor` to paginate"), }).passthrough(); export const summaryQuerySchema = z.object({ @@ -489,6 +504,7 @@ export const nftTransfersQuerySchema = z.object({ cursor: optionalQueryString("Opaque pagination cursor"), $filter: optionalQueryString("OData filter expression"), $select: optionalCommaList(z.string(), "Comma-separated field list"), + includeTotal: optionalQueryBool("Also return an exact `total`. Runs a COUNT over every matching row, so it is markedly slower on large result sets; omit it and use `hasMore` / `nextCursor` to paginate"), }).passthrough(); export const nftOwnerParamsSchema = z.object({ diff --git a/src/routes/accounts/transfers.ts b/src/routes/accounts/transfers.ts index 790c5849c..ca9e3dcd1 100644 --- a/src/routes/accounts/transfers.ts +++ b/src/routes/accounts/transfers.ts @@ -89,6 +89,7 @@ export function createAccountsTransfersRouter(): Router { eventTypes: eventType as string[] | undefined, limit, offset, + includeTotal: (parsed as { includeTotal?: boolean }).includeTotal, }); res.json({ From 1aea0c76f9dbc15b3a224863d70b9c20f7acd490 Mon Sep 17 00:00:00 2001 From: Collins Ezedike-egwom <62267326+collinsezedike@users.noreply.github.com> Date: Wed, 30 Sep 2026 19:50:59 +0100 Subject: [PATCH 14/15] feat: instrument HTTP surface in Prometheus (#189) (#212) * feat: instrument HTTP surface in Prometheus * fix(metrics): instrument before networkMiddleware and the rate limiter Mounted after them, the HTTP middleware never saw the requests those two reject, so 429s and invalid-?network= 400s were absent from http_requests_total. Moved to the top of the chain, right after cors(). Also documents the two new metrics in the README table and records why the route label must stay req.route.path: req.baseUrl is the matched mount path, and src/api/accounts.ts mounts a router at "/:address/transfers", so baseUrl carries the real address and would make the label unbounded. Claude-Session: https://claude.ai/code/session_01USgemLt4Rnz4SGB1Srf3GB --------- Co-authored-by: Miracle656 --- README.md | 2 ++ src/__tests__/metrics.test.ts | 56 +++++++++++++++++++++++++++++++++++ src/api.ts | 43 ++++++++++++++++++++++++++- src/metrics.ts | 30 +++++++++++++++++++ 4 files changed, 130 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f8f5dd887..e815374db 100644 --- a/README.md +++ b/README.md @@ -520,6 +520,8 @@ last_indexed_ledger{network="mainnet"} 5842100 | `events_skipped_total` | counter | `reason` | Events the decoder dropped. `unrecognised` is non-token events (normal); `malformed` is token events that failed to decode — a rising rate points at a non-standard contract or a decoder bug. | | `last_indexed_ledger` | gauge | `network` | Highest committed ledger. Against the chain tip, this is lag. | | `db_query_duration_seconds` | histogram | `operation` | Duration of instrumented DB operations, failures included. | +| `http_requests_total` | counter | `method`, `route`, `status` | HTTP requests served. `route` is the Express route *pattern*, never the raw URL, so addresses never become label values; an unmatched request reports `route="unknown"`. | +| `http_request_duration_seconds` | histogram | `method`, `route` | HTTP latency, 5 ms to 10 s. No `status` label — the split is rarely worth the extra series. | Standard `process_*` and `nodejs_*` metrics are exported alongside these. diff --git a/src/__tests__/metrics.test.ts b/src/__tests__/metrics.test.ts index af34574ab..8f0f03029 100644 --- a/src/__tests__/metrics.test.ts +++ b/src/__tests__/metrics.test.ts @@ -9,6 +9,8 @@ import { recordSkippedLedger, registry, _resetMetrics, + httpRequestsTotal, + httpRequestDurationSeconds, } from "../metrics"; jest.mock("../db", () => ({ @@ -163,4 +165,58 @@ describe("Prometheus metrics (#39)", () => { expect(res.body.lastIndexedLedger).toBe(1000); }); }); + + describe("HTTP metrics middleware", () => { + it("increments http_requests_total with method, route pattern, and status", async () => { + await request(app).get("/healthz"); + await request(app).get("/healthz"); + await request(app).post("/healthz"); // 404 since only GET exists + + const res = await request(app).get("/metrics"); + + expect(res.text).toContain('http_requests_total{method="GET",route="/healthz",status="200"} 2'); + expect(res.text).toContain('http_requests_total{method="POST",route="unknown",status="404"} 1'); + }); + + it("records http_request_duration_seconds with method and route pattern", async () => { + await request(app).get("/healthz"); + + const res = await request(app).get("/metrics"); + + expect(res.text).toContain('http_request_duration_seconds_count{method="GET",route="/healthz"} 1'); + expect(res.text).toMatch(/http_request_duration_seconds_sum\{method="GET",route="\/healthz"\} \d+(\.\d+)?/); + }); + + it("uses Express route pattern for parameterized routes", async () => { + const { queryTransfers } = jest.requireMock("../db"); + queryTransfers.mockResolvedValue({ total: 0, transfers: [], nextCursor: null }); + + await request(app).get("/transfers/incoming/GAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAWHF"); + + const res = await request(app).get("/metrics"); + + expect(res.text).toContain('http_requests_total{method="GET",route="/transfers/incoming/:address",status="200"} 1'); + }); + + it("excludes /metrics from its own counters", async () => { + // Reset and hit /metrics a few times + _resetMetrics(); + await request(app).get("/metrics"); + await request(app).get("/metrics"); + await request(app).get("/metrics"); + + const res = await request(app).get("/metrics"); + + // /metrics should not appear in the http_requests_total output + expect(res.text).not.toContain('route="/metrics"'); + }); + + it("includes /metrics in the metrics output (the scrape still works)", async () => { + const res = await request(app).get("/metrics"); + + expect(res.status).toBe(200); + expect(res.text).toContain("# TYPE http_requests_total counter"); + expect(res.text).toContain("# TYPE http_request_duration_seconds histogram"); + }); + }); }); diff --git a/src/api.ts b/src/api.ts index e5fc684d5..db61902c5 100644 --- a/src/api.ts +++ b/src/api.ts @@ -27,7 +27,7 @@ import { } from "./openapi/schemas"; import { parseOr400 } from "./openapi/validation"; import { networkMiddleware, requestNetwork } from "./middleware/network"; -import { renderMetrics, metricsContentType } from "./metrics"; +import { renderMetrics, metricsContentType, httpRequestsTotal, httpRequestDurationSeconds } from "./metrics"; import { getAllCachedTokens } from "./tokenCache"; // ─── RPC Health Check Cache ─────────────────────────────────────────────── @@ -140,6 +140,47 @@ export function createApp(): express.Application { app.use(cors()); + // ─── HTTP metrics middleware ────────────────────────────────────────────── + // First in the chain, deliberately: mounted below networkMiddleware and the + // rate limiter it would never see the requests those two reject, so a 429 + // storm or a flood of invalid ?network= values would be invisible in + // http_requests_total — exactly the traffic you want a counter for. + // + // The route label is `req.route?.path`, the Express route pattern, and never + // the raw URL: addresses and contract ids must not become label values. + // Do NOT "improve" this to `req.baseUrl + req.route.path`. `req.baseUrl` is + // the *matched* mount path, not the pattern, and `src/api/accounts.ts:29` + // mounts a router at "/:address/transfers" — so baseUrl reads + // "/accounts/GABC…/transfers" with the real address in it. One label value + // per address is unbounded cardinality and an unbounded memory leak in + // prom-client. The cost of the safe version is that endpoints whose routers + // declare "/" (e.g. /search, /webhooks) all report route="/"; that is a + // legibility loss, not a correctness or cardinality one. + // + // `method` is bounded by the set Node's HTTP parser accepts and `status` by + // the status codes we emit, so every label here has a fixed domain. + app.use((req: Request, res: Response, next: NextFunction) => { + // Skip /metrics: a scrape endpoint that increments its own counters + // creates a feedback loop and misrepresents real traffic. + if (req.path === "/metrics") { + return next(); + } + + const start = process.hrtime.bigint(); + + res.on("finish", () => { + const routePattern = req.route?.path ?? "unknown"; + const durationSeconds = Number(process.hrtime.bigint() - start) / 1e9; + const status = String(res.statusCode); + + httpRequestsTotal.inc({ method: req.method, route: routePattern, status }); + httpRequestDurationSeconds.observe({ method: req.method, route: routePattern }, durationSeconds); + }); + + next(); + }); + + // BEFORE express.json(). Linq signs the raw request body, so a parser that // consumes and re-serialises it makes every signature unverifiable - the // bytes differ even when the value does not. Their docs call this out diff --git a/src/metrics.ts b/src/metrics.ts index f072332f4..89f44268e 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -21,6 +21,36 @@ export const registry = new Registry(); // before any of the counters below can. collectDefaultMetrics({ register: registry }); +/** + * HTTP request count by method, route pattern, and status code. + * + * The route label uses the Express route pattern (e.g. /transfers/incoming/:address), + * never the raw path — addresses and other dynamic values must not become label values. + * /metrics is excluded from its own counters. + */ +export const httpRequestsTotal = new Counter({ + name: "http_requests_total", + help: "Total HTTP requests by method, route pattern, and status code", + labelNames: ["method", "route", "status"] as const, + registers: [registry], +}); + +/** + * HTTP request latency by method and route pattern. + * + * Buckets run from 5ms to 10s: below 5ms nothing here is worth alerting on, + * and past 10s the request has already failed for whatever is waiting on it. + * The route label uses the Express route pattern, never the raw path. + * /metrics is excluded from its own histogram. + */ +export const httpRequestDurationSeconds = new Histogram({ + name: "http_request_duration_seconds", + help: "HTTP request latency in seconds, by method and route pattern", + labelNames: ["method", "route"] as const, + buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10], + registers: [registry], +}); + /** * Ledgers the indexer has advanced through, per network. * From 1edd880e42fde0b04ac15be848055a119f4281ed Mon Sep 17 00:00:00 2001 From: That guy <120946193+ezedike-evan@users.noreply.github.com> Date: Wed, 30 Sep 2026 19:54:52 +0100 Subject: [PATCH 15/15] fix(indexer): bound NFT metadata lookups with a worker pool and per-cycle budget (#217) Co-authored-by: Miracle656 --- .env.example | 7 ++ src/__tests__/nftMetadata.test.ts | 156 ++++++++++++++++++++++++++++++ src/indexer/batch.ts | 41 +++++--- src/indexer/nft-metadata.ts | 140 +++++++++++++++++++++++++++ 4 files changed, 331 insertions(+), 13 deletions(-) create mode 100644 src/__tests__/nftMetadata.test.ts create mode 100644 src/indexer/nft-metadata.ts diff --git a/.env.example b/.env.example index 8aedcc150..1d8ed5ef1 100644 --- a/.env.example +++ b/.env.example @@ -84,6 +84,13 @@ SAC_CONTRACT_IDS_MAINNET= # Per-network NFT watch lists (for dual-network deployments): # NFT_CONTRACT_IDS_TESTNET= # NFT_CONTRACT_IDS_MAINNET= + +# NFT metadata enrichment is bounded so it can never stall ingest. +# Concurrent metadata lookups per poll cycle (default 8), and the total time +# in ms a cycle may spend on them (default 10000). Tokens not finished within +# the budget stay uncached and are retried on a later cycle. +# NFT_METADATA_CONCURRENCY=8 +# NFT_METADATA_BUDGET_MS=10000 NFT_CONTRACT_IDS= NFT_CONTRACT_IDS_TESTNET= NFT_CONTRACT_IDS_MAINNET= diff --git a/src/__tests__/nftMetadata.test.ts b/src/__tests__/nftMetadata.test.ts new file mode 100644 index 000000000..72ef16caa --- /dev/null +++ b/src/__tests__/nftMetadata.test.ts @@ -0,0 +1,156 @@ +import { + enrichNftMetadata, + resolveNftMetadataOptions, + type NftMetadataDeps, + type NftMetadataItem, +} from "../indexer/nft-metadata"; + +import { StrKey } from "@stellar/stellar-sdk"; + +// Built from raw bytes, so it is a valid strkey by construction rather than a +// hand-typed string with a checksum nobody verified. +const CONTRACT = StrKey.encodeContract(Buffer.alloc(32, 11)); + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +function items(n: number): NftMetadataItem[] { + return Array.from({ length: n }, (_, i) => ({ + contractId: CONTRACT, + tokenId: String(i), + key: `key-${i}`, + })); +} + +function makeDeps(over: Partial> = {}) { + const upserts: string[] = []; + const deps: NftMetadataDeps = { + getCached: async () => null, + fetch: async (_c, key) => ({ name: key }), + upsert: async (_c, tokenId) => { upserts.push(tokenId); }, + ...over, + }; + return { deps, upserts }; +} + +describe("enrichNftMetadata", () => { + it("does not cost N serial round trips: 20 tokens x 50ms finish in far less than 1000ms", async () => { + let inFlight = 0; + let peak = 0; + const { deps, upserts } = makeDeps({ + fetch: async (_c, key) => { + inFlight++; peak = Math.max(peak, inFlight); + await sleep(50); + inFlight--; + return { name: key }; + }, + }); + const start = Date.now(); + const res = await enrichNftMetadata(items(20), deps, { concurrency: 5, budgetMs: 5000 }); + const elapsed = Date.now() - start; + + expect(res).toMatchObject({ unique: 20, stored: 20, failed: 0, deferred: 0 }); + expect(upserts).toHaveLength(20); + expect(peak).toBeGreaterThan(1); + expect(peak).toBeLessThanOrEqual(5); + expect(elapsed).toBeLessThan(700); // serial would be >= 1000ms + }); + + it("de-duplicates repeated (contract, token) pairs and skips cached ones", async () => { + const fetchSpy = jest.fn(async () => ({})); + const { deps } = makeDeps({ + getCached: async (_c, t) => (t === "1" ? { name: "x" } : null), + fetch: fetchSpy, + }); + const res = await enrichNftMetadata([...items(3), ...items(3)], deps, { concurrency: 2 }); + expect(res).toMatchObject({ unique: 3, cached: 1, stored: 2 }); + expect(fetchSpy).toHaveBeenCalledTimes(2); + }); + + it("a slow source cannot stall past the budget, and unfinished tokens are not stored", async () => { + const { deps, upserts } = makeDeps({ + fetch: () => new Promise(() => {}), // never settles + }); + const start = Date.now(); + const res = await enrichNftMetadata(items(10), deps, { concurrency: 3, budgetMs: 100 }); + const elapsed = Date.now() - start; + + expect(elapsed).toBeLessThan(600); + expect(res.stored).toBe(0); + expect(res.deferred).toBe(10); + expect(upserts).toHaveLength(0); // stays uncached, retried next cycle + }); + + it("stops starting new lookups once the budget is spent", async () => { + const fetchSpy = jest.fn(async () => { await sleep(40); return {}; }); + const { deps } = makeDeps({ fetch: fetchSpy }); + const res = await enrichNftMetadata(items(30), deps, { concurrency: 1, budgetMs: 100 }); + expect(fetchSpy.mock.calls.length).toBeLessThan(30); + expect(res.stored + res.deferred + res.failed).toBe(30); + expect(res.deferred).toBeGreaterThan(0); + }); + + it("a failing source is per-token: it does not throw and other tokens still store", async () => { + const errors: string[] = []; + const { deps, upserts } = makeDeps({ + fetch: async (_c, key) => { + if (key === "key-2") throw new Error("https://secret.provider/url unreachable"); + return { name: key }; + }, + }); + const res = await enrichNftMetadata(items(5), deps, { + concurrency: 2, + onError: (m) => errors.push(m), + }); + expect(res).toMatchObject({ stored: 4, failed: 1, deferred: 0 }); + expect(upserts.sort()).toEqual(["0", "1", "3", "4"]); + expect(errors).toEqual(["NFT metadata lookup failed"]); + }); + + it("mixed batch: cached, ok, failing and hung tokens are handled independently", async () => { + const { deps, upserts } = makeDeps({ + getCached: async (_c, t) => (t === "0" ? { name: "cached" } : null), + fetch: async (_c, key) => { + if (key === "key-1") throw new Error("boom"); + if (key === "key-2") return new Promise(() => {}); + return { name: key }; + }, + }); + const res = await enrichNftMetadata(items(5), deps, { concurrency: 5, budgetMs: 100 }); + expect(res).toMatchObject({ unique: 5, cached: 1, failed: 1, deferred: 1, stored: 2 }); + expect(upserts.sort()).toEqual(["3", "4"]); + }); + + it("a throwing upsert or cache read only costs that token", async () => { + const { deps } = makeDeps({ + getCached: async (_c, t) => { if (t === "0") throw new Error("db"); return null; }, + upsert: async (_c, t) => { if (t === "1") throw new Error("db"); }, + }); + const res = await enrichNftMetadata(items(3), deps, { concurrency: 3 }); + expect(res).toMatchObject({ failed: 2, stored: 1 }); + }); + + it("handles an empty batch", async () => { + const { deps } = makeDeps(); + expect(await enrichNftMetadata([], deps)).toMatchObject({ unique: 0, stored: 0 }); + }); +}); + +describe("resolveNftMetadataOptions", () => { + afterEach(() => { + delete process.env.NFT_METADATA_CONCURRENCY; + delete process.env.NFT_METADATA_BUDGET_MS; + }); + + it("uses safe defaults and ignores junk values", () => { + expect(resolveNftMetadataOptions()).toEqual({ concurrency: 8, budgetMs: 10000 }); + process.env.NFT_METADATA_CONCURRENCY = "0"; + process.env.NFT_METADATA_BUDGET_MS = "abc"; + expect(resolveNftMetadataOptions()).toEqual({ concurrency: 8, budgetMs: 10000 }); + }); + + it("honours valid overrides", () => { + process.env.NFT_METADATA_CONCURRENCY = "3"; + process.env.NFT_METADATA_BUDGET_MS = "2500"; + expect(resolveNftMetadataOptions()).toEqual({ concurrency: 3, budgetMs: 2500 }); + }); +}); diff --git a/src/indexer/batch.ts b/src/indexer/batch.ts index 3565b08ea..5f29c5d56 100644 --- a/src/indexer/batch.ts +++ b/src/indexer/batch.ts @@ -32,6 +32,7 @@ import { emitTransfer, emitHostFnLog } from "../events"; import { parseHostFnEvent, upsertHostFnLogs, type HostFnRecord } from "./host-fn-log"; import { tagSacTransfers } from "./sac-detect"; import { parseLpShareEvents, upsertLpShareTransfers } from "./lp-shares"; +import { enrichNftMetadata } from "./nft-metadata"; import { isNftTransferEvent, parseNftEvents, fetchNftMetadata } from "../ingester/nft"; import { getTokenMetadata } from "../tokenCache"; import { transfersStoredTotal } from "../metrics"; @@ -136,20 +137,34 @@ export async function processEventBatch( const nftInserted = await upsertNftTransfers(nftRecords, net); transfersStoredTotal.inc({ network: net, type: "nft" }, nftInserted); - // Lazy-load metadata for unique (contractId, tokenId) pairs not yet cached + // Lazy-load metadata for unique (contractId, tokenId) pairs not yet cached. + // Bounded: a small worker pool under a per-cycle time budget, so a slow or + // dead metadata source can never hold up the cursor (#205). This lives in the + // shared batch pipeline, so the sharded ingest path is bounded too — note the + // budget is per batch, so a sharded run spends up to `workers` of them. if (nftParsed.length > 0) { - const seen = new Set(); - for (const { record, tokenIdScVal } of nftParsed) { - const key = `${record.contractId}:${record.tokenId}`; - if (seen.has(key)) continue; - seen.add(key); - const cached = await getNftMetadata(record.contractId, record.tokenId, net); - if (!cached) { - const meta = await fetchNftMetadata(record.contractId, tokenIdScVal, net).catch(() => ({})); - await upsertNftMetadata(record.contractId, record.tokenId, meta, net).catch((e: unknown) => - console.error(`[indexer/${net}] NFT metadata upsert failed:`, e) - ); - } + const outcome = await enrichNftMetadata( + nftParsed.map(({ record, tokenIdScVal }) => ({ + contractId: record.contractId, + tokenId: record.tokenId, + key: tokenIdScVal, + })), + { + getCached: (contractId, tokenId) => getNftMetadata(contractId, tokenId, net), + fetch: (contractId, tokenIdScVal) => + fetchNftMetadata(contractId, tokenIdScVal, net).catch(() => ({})), + upsert: (contractId, tokenId, meta) => upsertNftMetadata(contractId, tokenId, meta, net), + }, + { + // Deliberately not logging the error object: it can carry a raw DB + // error or a provider URL. + onError: (message) => console.error(`[indexer/${net}] ${message}`), + }, + ); + if (outcome.deferred > 0) { + console.warn( + `[indexer/${net}] NFT metadata budget spent: ${outcome.deferred} of ${outcome.unique} tokens deferred to a later cycle`, + ); } } diff --git a/src/indexer/nft-metadata.ts b/src/indexer/nft-metadata.ts new file mode 100644 index 000000000..2b97f6c15 --- /dev/null +++ b/src/indexer/nft-metadata.ts @@ -0,0 +1,140 @@ +/** + * Bounded NFT metadata enrichment (#205). + * + * Metadata is enrichment; transfers are the product. Looking metadata up one + * token at a time meant a batch of N new tokens cost N serial round trips + * before the poll cycle could advance. This runs the lookups through a small + * worker pool and caps the total time a cycle may spend on them. + * + * Guarantees: + * - at most `concurrency` lookups are in flight at once; + * - no lookup starts after the budget is spent, and an in-flight lookup that + * outlives the budget is abandoned, so the caller returns within roughly + * `budgetMs` no matter how slow the source is; + * - a token's failure (cache read, fetch or upsert) never affects another; + * - tokens that were skipped or timed out are NOT written, so they stay + * uncached and are retried on a later cycle. + */ + +export interface NftMetadataPayload { + name?: string; + tokenUri?: string; +} + +export interface NftMetadataItem { + contractId: string; + tokenId: string; + /** Opaque handle passed back to `fetch` (the token id ScVal in production). */ + key: K; +} + +export interface NftMetadataDeps { + /** Resolves truthy when metadata is already stored for the token. */ + getCached: (contractId: string, tokenId: string) => Promise; + fetch: (contractId: string, key: K) => Promise; + upsert: (contractId: string, tokenId: string, meta: NftMetadataPayload) => Promise; +} + +export interface NftMetadataOptions { + concurrency?: number; + budgetMs?: number; + /** Injectable clock for tests. */ + now?: () => number; + onError?: (message: string, err: unknown) => void; +} + +export interface NftMetadataResult { + /** Unique tokens considered. */ + unique: number; + /** Already cached, nothing to do. */ + cached: number; + /** Fetched and stored. */ + stored: number; + /** Failed individually (cache read, fetch or upsert threw). */ + failed: number; + /** Not stored because the cycle budget ran out; retried next cycle. */ + deferred: number; +} + +export const DEFAULT_NFT_METADATA_CONCURRENCY = 8; +export const DEFAULT_NFT_METADATA_BUDGET_MS = 10_000; + +function positiveInt(raw: string | undefined, fallback: number): number { + const n = Number(raw); + return Number.isInteger(n) && n > 0 ? n : fallback; +} + +/** Read NFT_METADATA_CONCURRENCY / NFT_METADATA_BUDGET_MS, ignoring junk. */ +export function resolveNftMetadataOptions(): Required> { + return { + concurrency: positiveInt(process.env.NFT_METADATA_CONCURRENCY, DEFAULT_NFT_METADATA_CONCURRENCY), + budgetMs: positiveInt(process.env.NFT_METADATA_BUDGET_MS, DEFAULT_NFT_METADATA_BUDGET_MS), + }; +} + +const TIMED_OUT = Symbol("timed-out"); + +/** Race `p` against `ms`, clearing the timer either way. */ +function withTimeout(p: Promise, ms: number): Promise { + let timer: NodeJS.Timeout | undefined; + const timeout = new Promise((resolve) => { + timer = setTimeout(() => resolve(TIMED_OUT), Math.max(ms, 0)); + }); + return Promise.race([p, timeout]).finally(() => clearTimeout(timer)); +} + +export async function enrichNftMetadata( + items: ReadonlyArray>, + deps: NftMetadataDeps, + opts: NftMetadataOptions = {}, +): Promise { + const defaults = resolveNftMetadataOptions(); + const concurrency = Math.max(1, Math.floor(opts.concurrency ?? defaults.concurrency)); + const budgetMs = opts.budgetMs ?? defaults.budgetMs; + const now = opts.now ?? Date.now; + const onError = opts.onError ?? (() => {}); + + // One lookup per unique (contract, token) pair. + const seen = new Set(); + const unique: Array> = []; + for (const item of items) { + const id = `${item.contractId}:${item.tokenId}`; + if (seen.has(id)) continue; + seen.add(id); + unique.push(item); + } + + const result: NftMetadataResult = { unique: unique.length, cached: 0, stored: 0, failed: 0, deferred: 0 }; + const deadline = now() + budgetMs; + let next = 0; + + async function processOne(item: NftMetadataItem): Promise { + const remaining = () => deadline - now(); + try { + const cached = await withTimeout(deps.getCached(item.contractId, item.tokenId), remaining()); + if (cached === TIMED_OUT) { result.deferred++; return; } + if (cached) { result.cached++; return; } + + const meta = await withTimeout(deps.fetch(item.contractId, item.key), remaining()); + if (meta === TIMED_OUT) { result.deferred++; return; } + + const stored = await withTimeout(deps.upsert(item.contractId, item.tokenId, meta), remaining()); + if (stored === TIMED_OUT) { result.deferred++; return; } + result.stored++; + } catch (err) { + result.failed++; + onError("NFT metadata lookup failed", err); + } + } + + async function worker(): Promise { + while (next < unique.length) { + const item = unique[next++]; + if (now() >= deadline) { result.deferred++; continue; } + await processOne(item); + } + } + + await Promise.all(Array.from({ length: Math.min(concurrency, unique.length) }, worker)); + return result; +}