From 90a6151c7f0fbe95b2307d7539e71d6e3ef52643 Mon Sep 17 00:00:00 2001 From: Jemimah Ekong Date: Sat, 17 Oct 2026 19:23:46 +0100 Subject: [PATCH] feat(fees): record every fee quote with the policy behind it (W085) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GET /fees tells a client what to charge, and the fee then rides on-chain in the user's own transaction. Nothing on this side recorded that it happened, so there was no answer to what had been charged, in which asset, on which day, or under which policy — and a misconfigured percent, floor or cap would overcharge every user until someone read a block explorer. - FeeQuote: one row per asset per quote — amount priced against, fee, the rule that decided it (percent | floor | cap | flat | no_amount), the fee as a percent of the amount, an alarm flag, and the policy id. No IP, user agent or sender address; the only address is the published destination. - FeePolicy: keyed by a hash of the policy, so a change is a new row with its own firstSeenAt and every quote names the policy that priced it. The policy is taken from the entry the client was sent, never re-read from env. - Recording runs after the response and never throws: a ledger outage is logged and counted (veil_fee_quote_record_failures_total), not a failed send. - Alarm: a fee above VEIL_FEE_ALARM_PERCENT (default 10) of its amount flags the row, increments veil_fee_quote_alarms_total and logs a line — raised before the write, so a database outage cannot silence it. - GET /fees/report?from=&to=: per UTC day and asset — quotes, fees quoted, alarms, policies in force, and what the destination actually received per the index (null, with a reason, when the asset is not indexed). Behind VEIL_FEE_REPORT_TOKEN; 503 when unset. Documented as x-internal. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LvL6Yz2Zfg7spM1F9DXXtz --- .env.example | 10 + docker-compose.yml | 2 + openapi.json | 277 ++++++++++++++++ prisma/schema.prisma | 52 +++ src/__tests__/feeLedger.test.ts | 570 ++++++++++++++++++++++++++++++++ src/api/feeLedger.ts | 416 +++++++++++++++++++++++ src/api/fees.ts | 104 ++++++ src/metrics.ts | 29 ++ src/openapi/build.ts | 23 ++ src/openapi/schemas.ts | 48 +++ 10 files changed, 1531 insertions(+) create mode 100644 src/__tests__/feeLedger.test.ts create mode 100644 src/api/feeLedger.ts diff --git a/.env.example b/.env.example index 1d8ed5ef1..6b62043ca 100644 --- a/.env.example +++ b/.env.example @@ -151,3 +151,13 @@ LP_POOL_CONTRACT_IDS_MAINNET= # HTTP tests run against a seeded database without an ingest racing them, and # without needing live RPC. Not for production. SKIP_INDEXER=false + +# ─── Platform fee ledger (GET /fees/report) ────────────────────────────────── +# Every fee GET /fees quotes is recorded with the policy that produced it. +# A quote whose fee exceeds this percent of the amount it priced raises an +# alarm: the row is flagged, `veil_fee_quote_alarms_total` increments, and a +# line is logged. Default 10 — the rate the fee policy itself calls stark. +VEIL_FEE_ALARM_PERCENT=10 +# Bearer token for GET /fees/report. Unset means the report answers 503, so it +# is never served open by accident. Use a long random value. +VEIL_FEE_REPORT_TOKEN= diff --git a/docker-compose.yml b/docker-compose.yml index ffdf83ad6..528a42926 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,6 +73,8 @@ services: LP_POOL_CONTRACT_IDS_TESTNET: ${LP_POOL_CONTRACT_IDS_TESTNET:-} LP_POOL_CONTRACT_IDS_MAINNET: ${LP_POOL_CONTRACT_IDS_MAINNET:-} SKIP_INDEXER: ${SKIP_INDEXER:-false} + VEIL_FEE_ALARM_PERCENT: ${VEIL_FEE_ALARM_PERCENT:-10} + VEIL_FEE_REPORT_TOKEN: ${VEIL_FEE_REPORT_TOKEN:-} ports: - "${PORT:-3000}:3000" diff --git a/openapi.json b/openapi.json index f807dbbbb..6ce845880 100644 --- a/openapi.json +++ b/openapi.json @@ -16,6 +16,12 @@ } ], "components": { + "securitySchemes": { + "bearerAuth": { + "type": "http", + "scheme": "bearer" + } + }, "schemas": {}, "parameters": {} }, @@ -5990,6 +5996,277 @@ } } } + }, + "/fees/report": { + "get": { + "summary": "Platform fees charged, per UTC day and asset, with the policy behind them", + "description": "Internal endpoint. Not part of the public data API: an operator report over the fee ledger, behind the VEIL_FEE_REPORT_TOKEN bearer token. Nothing in it identifies a person.", + "x-internal": true, + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "schema": { + "type": "string", + "nullable": true, + "description": "First UTC date to report, YYYY-MM-DD. Defaults to today." + }, + "required": false, + "description": "First UTC date to report, YYYY-MM-DD. Defaults to today.", + "name": "from", + "in": "query" + }, + { + "schema": { + "type": "string", + "nullable": true, + "description": "Last UTC date to report, inclusive, YYYY-MM-DD. Defaults to `from`. At most 92 days after it." + }, + "required": false, + "description": "Last UTC date to report, inclusive, YYYY-MM-DD. Defaults to `from`. At most 92 days after it.", + "name": "to", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "network": { + "type": "string" + }, + "from": { + "type": "string" + }, + "to": { + "type": "string" + }, + "alarmPercent": { + "type": "number", + "description": "VEIL_FEE_ALARM_PERCENT in force for this report." + }, + "days": { + "type": "array", + "items": { + "type": "object", + "properties": { + "date": { + "type": "string" + }, + "assets": { + "type": "array", + "items": { + "type": "object", + "properties": { + "assetCode": { + "type": "string" + }, + "assetIssuer": { + "type": "string", + "nullable": true + }, + "quotes": { + "type": "integer" + }, + "feesQuoted": { + "type": "string", + "description": "Sum of every fee quoted — what clients were told to charge." + }, + "alarms": { + "type": "integer" + }, + "policyIds": { + "type": "array", + "items": { + "type": "string" + } + }, + "collected": { + "type": "string", + "nullable": true, + "description": "What the fee destination received in this asset that day, per the index. Null when the asset is not indexed — which is not zero." + }, + "collectedNote": { + "type": "string" + } + }, + "required": [ + "assetCode", + "assetIssuer", + "quotes", + "feesQuoted", + "alarms", + "policyIds", + "collected" + ] + } + } + }, + "required": [ + "date", + "assets" + ] + } + }, + "policies": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "firstSeenAt": { + "type": "string" + }, + "policy": { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": [ + "flat", + "percent" + ] + }, + "assetCode": { + "type": "string" + }, + "assetIssuer": { + "type": "string", + "nullable": true + }, + "destination": { + "type": "string", + "description": "The fee destination — the address `/fees` already publishes." + }, + "percent": { + "type": "string", + "nullable": true + }, + "min": { + "type": "string", + "nullable": true + }, + "max": { + "type": "string", + "nullable": true + }, + "amount": { + "type": "string", + "nullable": true, + "description": "Flat mode only: the fixed fee." + } + }, + "required": [ + "mode", + "assetCode", + "assetIssuer", + "destination", + "percent", + "min", + "max", + "amount" + ] + } + }, + "required": [ + "firstSeenAt", + "policy" + ] + } + } + }, + "required": [ + "network", + "from", + "to", + "alarmPercent", + "days", + "policies" + ] + } + } + } + }, + "400": { + "description": "Bad date range", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ] + } + } + } + }, + "401": { + "description": "Missing or wrong report token", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ] + } + } + } + }, + "500": { + "description": "The fee ledger could not be read", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ] + } + } + } + }, + "503": { + "description": "Fee reporting is not configured on this deployment", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ] + } + } + } + } + } + } } } } diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 4d9130e30..bd69ea45c 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -495,3 +495,55 @@ model NgnOrder { @@index([network, status]) @@schema("wraith") } + +// ─── Platform fee ledger ───────────────────────────────────────────────────── + +/// One fee policy for one asset, as `GET /fees` applied it. +/// +/// Keyed by a hash of the policy itself, so the same configuration is the same +/// row however many quotes it prices, and any change — a new percent, floor, +/// cap, flat amount or destination — is a new row with its own `firstSeenAt`. +/// That is what lets a fee be explained after the fact: every `FeeQuote` names +/// the policy that produced it, and this table says when that policy appeared. +model FeePolicy { + /// sha256 of the canonical policy JSON, hex, first 32 characters. + id String @id + /// `{ mode, assetCode, assetIssuer, destination, percent, min, max, amount }` + /// — everything that decides a fee, and nothing about who asked. + policy Json + firstSeenAt DateTime @default(now()) + + @@schema("wraith") +} + +/// One fee quoted by `GET /fees`, per asset. +/// +/// **Nothing here identifies a person.** No IP, no user agent, no sender +/// address — the request carries none, and none is taken. The only address +/// is the fee destination, held inside the policy, which `/fees` publishes to +/// anyone who asks. The amount and the time are what a quote *is*; neither +/// names who wanted it. +model FeeQuote { + id Int @id @default(autoincrement()) + network String + assetCode String + /// Null for native XLM. + assetIssuer String? + /// The amount priced against, in whole units of the asset. Null when the + /// client sent none — an old client, which is told the floor. + quotedAgainst String? + /// What the client was told to charge. + fee Decimal @db.Decimal(38, 7) + /// Which rule decided the fee: percent | floor | cap | flat | no_amount. + basis String + /// The fee as a percentage of `quotedAgainst`. Null without an amount. + feePercent Decimal? @db.Decimal(20, 7) + /// The fee exceeded VEIL_FEE_ALARM_PERCENT of the amount. + alarm Boolean @default(false) + policyId String + quotedAt DateTime @default(now()) + + @@index([network, quotedAt]) + @@index([network, alarm, quotedAt]) + @@schema("wraith") +} diff --git a/src/__tests__/feeLedger.test.ts b/src/__tests__/feeLedger.test.ts new file mode 100644 index 000000000..2a22264c1 --- /dev/null +++ b/src/__tests__/feeLedger.test.ts @@ -0,0 +1,570 @@ +/** + * W085 — the platform fee ledger. + * + * `GET /fees` tells a client what to charge, and the fee then rides on-chain + * in the user's own transaction. Nothing on this side knew it had happened. + * These pin the four things the issue asks for: + * + * - every quote that returns a fee is recorded, with the policy behind it; + * - totals can be reported per asset and per day; + * - a policy change is visible, so a fee can be explained after the fact; + * - nothing recorded identifies a person. + * + * Plus the alarm, and the property that matters most in production: a ledger + * failure never fails the quote a person is in the middle of paying. + * + * The database is mocked throughout; nothing here touches a provider. + */ +import express from "express"; +import request from "supertest"; +import { Asset, Networks } from "@stellar/stellar-sdk"; + +jest.mock("../db", () => ({ + prisma: { + feePolicy: { createMany: jest.fn(), findMany: jest.fn() }, + feeQuote: { createMany: jest.fn() }, + $queryRaw: jest.fn(), + }, +})); +jest.mock("../indexer", () => ({ resolveSacContractIds: jest.fn() })); + +import { prisma } from "../db"; +import { resolveSacContractIds } from "../indexer"; +import { networkMiddleware } from "../middleware/network"; +import { feeBasis, feesRouter, reportRange, type PlatformFee } from "../api/fees"; +import { + DEFAULT_ALARM_PERCENT, + _forgetKnownPolicies, + alarmPercent, + feeReport, + policyId, + policyOf, + quoteRow, + recordFeeQuotes, +} from "../api/feeLedger"; +import { feeQuoteAlarmsTotal, feeQuoteRecordFailuresTotal, _resetMetrics } from "../metrics"; + +const db = prisma as unknown as { + feePolicy: { createMany: jest.Mock; findMany: jest.Mock }; + feeQuote: { createMany: jest.Mock }; + $queryRaw: jest.Mock; +}; + +const DESTINATION = "GB3JS2AJZ3FKIGVOOPXDEYYR3XOHQA474WGCYAMNFSGX6OW5HGWCKUZW"; +const ISSUER = "GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN"; +const TOKEN = "report-token-for-tests"; + +const FEE_ENV = [ + "VEIL_FEE_DESTINATION", + "VEIL_FEE_ASSET_CODE", + "VEIL_FEE_ASSET_ISSUER", + "VEIL_FEE_AMOUNT", + "VEIL_FEE_AMOUNT_XLM", + "VEIL_FEE_PERCENT", + "VEIL_FEE_MIN", + "VEIL_FEE_MAX", + "VEIL_FEE_MIN_XLM", + "VEIL_FEE_MAX_XLM", + "VEIL_FEE_ALARM_PERCENT", + "VEIL_FEE_REPORT_TOKEN", +]; + +/** The exact columns a quote row may have. A new one must be argued for here. */ +const QUOTE_COLUMNS = [ + "alarm", + "assetCode", + "assetIssuer", + "basis", + "fee", + "feePercent", + "network", + "policyId", + "quotedAgainst", +]; + +const percentFee = (over: Partial = {}): PlatformFee => ({ + amount: "0.5", + assetCode: "USDC", + assetIssuer: ISSUER, + destination: DESTINATION, + percent: "0.5", + min: "0.02", + max: "2", + ...over, +}); + +const flatFee = (over: Partial = {}): PlatformFee => ({ + amount: "0.1", + assetCode: "XLM", + assetIssuer: null, + destination: DESTINATION, + ...over, +}); + +function makeApp() { + const app = express(); + app.use(networkMiddleware); + app.use("/fees", feesRouter()); + return app; +} + +/** Let the fire-and-forget write after a response finish. */ +const settle = () => new Promise((r) => setImmediate(r)); + +const quotesWritten = () => db.feeQuote.createMany.mock.calls.flatMap((c) => c[0].data); +const policiesWritten = () => db.feePolicy.createMany.mock.calls.flatMap((c) => c[0].data); + +beforeEach(() => { + for (const key of FEE_ENV) delete process.env[key]; + process.env.NETWORKS = "testnet,mainnet"; + jest.clearAllMocks(); + db.feePolicy.createMany.mockResolvedValue({ count: 1 }); + db.feeQuote.createMany.mockResolvedValue({ count: 1 }); + db.feePolicy.findMany.mockResolvedValue([]); + db.$queryRaw.mockResolvedValue([]); + (resolveSacContractIds as jest.Mock).mockReturnValue([]); + _forgetKnownPolicies(); + _resetMetrics(); + jest.spyOn(console, "error").mockImplementation(() => {}); +}); + +afterEach(() => jest.restoreAllMocks()); + +afterAll(() => { + for (const key of FEE_ENV) delete process.env[key]; +}); + +// ─── Which rule decided the fee ───────────────────────────────────────────── + +describe("feeBasis", () => { + it("is flat when the policy has no percent", () => { + expect(feeBasis(flatFee(), 10)).toBe("flat"); + }); + + it("is no_amount when an old client sent nothing to price", () => { + expect(feeBasis(percentFee({ amount: "0.02" }))).toBe("no_amount"); + }); + + it("is percent when neither bound applied", () => { + // 0.5% of 100 = 0.5, between 0.02 and 2. + expect(feeBasis(percentFee({ amount: "0.5" }), 100)).toBe("percent"); + }); + + it("is floor when the floor overrode a smaller percent", () => { + // 0.5% of 1 = 0.005, under the 0.02 floor. + expect(feeBasis(percentFee({ amount: "0.02" }), 1)).toBe("floor"); + }); + + it("is cap when the cap overrode a larger percent", () => { + // 0.5% of 10,000 = 50, over the 2 cap. + expect(feeBasis(percentFee({ amount: "2" }), 10_000)).toBe("cap"); + }); + + it("is percent, not floor, when the percent lands exactly on the floor", () => { + // 0.5% of 4 = 0.02 — the floor did not change anything. + expect(feeBasis(percentFee({ amount: "0.02" }), 4)).toBe("percent"); + }); +}); + +// ─── Policies ─────────────────────────────────────────────────────────────── + +describe("policyOf / policyId", () => { + it("leaves a percent quote's computed amount out of the policy", () => { + // Otherwise every distinct send would look like a policy change. + const a = policyOf(percentFee({ amount: "0.5" })); + const b = policyOf(percentFee({ amount: "1.25" })); + expect(a.amount).toBeNull(); + expect(policyId(a)).toBe(policyId(b)); + }); + + it("keeps a flat fee's amount, because there it is the policy", () => { + expect(policyOf(flatFee()).amount).toBe("0.1"); + expect(policyId(policyOf(flatFee({ amount: "0.1" })))).not.toBe( + policyId(policyOf(flatFee({ amount: "0.2" }))), + ); + }); + + it.each([ + ["percent", { percent: "1" }], + ["floor", { min: "0.05" }], + ["cap", { max: "5" }], + ["destination", { destination: "GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN" }], + ["asset", { assetCode: "EURC" }], + ])("changes when the %s changes", (_what, change) => { + expect(policyId(policyOf(percentFee(change)))).not.toBe(policyId(policyOf(percentFee()))); + }); + + it("does not depend on the order the fields were built in", () => { + const p = policyOf(percentFee()); + const reordered = { amount: p.amount, max: p.max, min: p.min, percent: p.percent, destination: p.destination, assetIssuer: p.assetIssuer, assetCode: p.assetCode, mode: p.mode }; + expect(policyId(reordered)).toBe(policyId(p)); + }); +}); + +// ─── The row, and the alarm ───────────────────────────────────────────────── + +describe("quoteRow", () => { + it("records the amount priced, the fee, the basis and the policy", () => { + const { data } = quoteRow("mainnet", 100, { fee: percentFee({ amount: "0.5" }), basis: "percent" }); + expect(data).toMatchObject({ + network: "mainnet", + assetCode: "USDC", + assetIssuer: ISSUER, + quotedAgainst: "100", + fee: "0.5", + basis: "percent", + feePercent: "0.5", + alarm: false, + policyId: policyId(policyOf(percentFee())), + }); + }); + + // The whole of the "nothing identifies a person" criterion, as a shape: the + // row has exactly these columns, and none of them can hold an IP, a user + // agent or a sender. The destination is in the policy, and `/fees` + // publishes it to anyone. + it("has no column that could identify a person", () => { + const { data } = quoteRow("mainnet", 100, { fee: percentFee(), basis: "percent" }); + expect(Object.keys(data).sort()).toEqual(QUOTE_COLUMNS); + }); + + it("records no amount and raises no alarm when none was sent", () => { + const { data } = quoteRow("mainnet", undefined, { fee: percentFee({ amount: "0.02" }), basis: "no_amount" }); + expect(data.quotedAgainst).toBeNull(); + expect(data.feePercent).toBeNull(); + expect(data.alarm).toBe(false); + }); + + it("alarms only above the limit, not at it", () => { + const at = quoteRow("mainnet", 1, { fee: flatFee({ amount: "0.1" }), basis: "flat" }, 10); + const over = quoteRow("mainnet", 1, { fee: flatFee({ amount: "0.1000001" }), basis: "flat" }, 10); + expect(at.data.alarm).toBe(false); + expect(over.data.alarm).toBe(true); + }); + + it("catches a floor that dwarfs a tiny send", () => { + // A 0.02 floor on a 0.1 send is 20% of it. + const { data } = quoteRow("mainnet", 0.1, { fee: percentFee({ amount: "0.02" }), basis: "floor" }); + expect(data.feePercent).toBe("20"); + expect(data.alarm).toBe(true); + }); +}); + +describe("alarmPercent", () => { + it("defaults to 10", () => { + expect(alarmPercent()).toBe(DEFAULT_ALARM_PERCENT); + expect(DEFAULT_ALARM_PERCENT).toBe(10); + }); + + it("reads VEIL_FEE_ALARM_PERCENT", () => { + process.env.VEIL_FEE_ALARM_PERCENT = "2.5"; + expect(alarmPercent()).toBe(2.5); + }); + + it.each(["0", "-1", "101", "abc", ""])("falls back to the default for %p", (value) => { + process.env.VEIL_FEE_ALARM_PERCENT = value; + expect(alarmPercent()).toBe(DEFAULT_ALARM_PERCENT); + }); +}); + +// ─── Recording ────────────────────────────────────────────────────────────── + +describe("recordFeeQuotes", () => { + it("writes each quote with its policy, and the policy only once", async () => { + const quoted = [{ fee: percentFee(), basis: "percent" as const }]; + await recordFeeQuotes("mainnet", 100, quoted); + await recordFeeQuotes("mainnet", 200, quoted); + + expect(db.feePolicy.createMany).toHaveBeenCalledTimes(1); + expect(db.feePolicy.createMany.mock.calls[0][0].skipDuplicates).toBe(true); + expect(policiesWritten()).toEqual([ + { id: policyId(policyOf(percentFee())), policy: policyOf(percentFee()) }, + ]); + expect(quotesWritten()).toHaveLength(2); + }); + + it("does nothing when no fee was returned", async () => { + await recordFeeQuotes("mainnet", 100, []); + expect(db.feeQuote.createMany).not.toHaveBeenCalled(); + expect(db.feePolicy.createMany).not.toHaveBeenCalled(); + }); + + it("raises the alarm: counter, log line, and the row's flag", async () => { + await recordFeeQuotes("mainnet", 0.1, [{ fee: percentFee({ amount: "0.02" }), basis: "floor" }]); + + const counted = await feeQuoteAlarmsTotal.get(); + expect(counted.values).toEqual([ + expect.objectContaining({ value: 1, labels: { network: "mainnet", asset: "USDC" } }), + ]); + expect(console.error).toHaveBeenCalledWith(expect.stringMatching(/ALARM mainnet USDC: fee 0\.02 is 20% of 0\.1/)); + expect(quotesWritten()[0].alarm).toBe(true); + }); + + // The person is mid-payment. A ledger outage must be loud, not fatal. + it("never throws when the database fails, and counts the failure", async () => { + db.feeQuote.createMany.mockRejectedValue(new Error("connection refused")); + + await expect( + recordFeeQuotes("mainnet", 100, [{ fee: percentFee(), basis: "percent" }]), + ).resolves.toBeUndefined(); + + const failures = await feeQuoteRecordFailuresTotal.get(); + expect(failures.values).toEqual([expect.objectContaining({ value: 1, labels: { network: "mainnet" } })]); + expect(console.error).toHaveBeenCalledWith(expect.stringMatching(/could not record 1 fee quote/)); + }); + + it("still raises the alarm when the write fails", async () => { + db.feePolicy.createMany.mockRejectedValue(new Error("down")); + await recordFeeQuotes("mainnet", 0.1, [{ fee: percentFee({ amount: "0.02" }), basis: "floor" }]); + expect((await feeQuoteAlarmsTotal.get()).values[0]?.value).toBe(1); + }); + + it("retries a policy write that failed, rather than caching it as known", async () => { + db.feePolicy.createMany.mockRejectedValueOnce(new Error("down")); + const quoted = [{ fee: percentFee(), basis: "percent" as const }]; + await recordFeeQuotes("mainnet", 100, quoted); + await recordFeeQuotes("mainnet", 100, quoted); + expect(db.feePolicy.createMany).toHaveBeenCalledTimes(2); + }); +}); + +// ─── Through the route ────────────────────────────────────────────────────── + +describe("GET /fees records what it quoted", () => { + beforeEach(() => { + process.env.VEIL_FEE_DESTINATION = DESTINATION; + process.env.VEIL_FEE_ASSET_ISSUER = ISSUER; + process.env.VEIL_FEE_PERCENT = "0.5"; + process.env.VEIL_FEE_MIN = "0.02"; + process.env.VEIL_FEE_MAX = "2"; + process.env.VEIL_FEE_AMOUNT_XLM = "0.1"; + }); + + it("records every fee it returned, each with the policy that produced it", async () => { + const res = await request(makeApp()).get("/fees?amount=100").set("x-network", "mainnet"); + await settle(); + + expect(res.status).toBe(200); + const returned = res.body.fees as Record; + const rows = quotesWritten(); + expect(rows.map((r) => r.assetCode).sort()).toEqual(Object.keys(returned).sort()); + + const policies = new Map(policiesWritten().map((p) => [p.id, p.policy])); + for (const row of rows) { + const fee = returned[row.assetCode]; + expect(row.fee).toBe(fee.amount); + expect(row.network).toBe("mainnet"); + expect(row.quotedAgainst).toBe("100"); + // The policy recorded is the one the client was sent, field for field. + expect(policies.get(row.policyId)).toEqual(policyOf(fee)); + } + // VEIL_FEE_PERCENT applies to every asset; XLM's flat amount is its floor. + expect(rows.map((r) => r.basis)).toEqual(["percent", "percent"]); + }); + + it("records a quote with no amount as the floor it told the client", async () => { + await request(makeApp()).get("/fees").set("x-network", "mainnet"); + await settle(); + const usdc = quotesWritten().find((r) => r.assetCode === "USDC"); + expect(usdc).toMatchObject({ basis: "no_amount", quotedAgainst: null, fee: "0.02", alarm: false }); + }); + + it("records nothing when no fee is configured", async () => { + delete process.env.VEIL_FEE_DESTINATION; + const res = await request(makeApp()).get("/fees?amount=100"); + await settle(); + expect(res.body.enabled).toBe(false); + expect(db.feeQuote.createMany).not.toHaveBeenCalled(); + }); + + it("still answers when the ledger is down", async () => { + db.feePolicy.createMany.mockRejectedValue(new Error("down")); + db.feeQuote.createMany.mockRejectedValue(new Error("down")); + const res = await request(makeApp()).get("/fees?amount=100"); + await settle(); + expect(res.status).toBe(200); + expect(res.body.enabled).toBe(true); + }); + + it("makes a policy change visible: a new policy row, and quotes that name it", async () => { + await request(makeApp()).get("/fees?amount=100"); + await settle(); + process.env.VEIL_FEE_PERCENT = "1"; + await request(makeApp()).get("/fees?amount=100"); + await settle(); + + const usdc = quotesWritten().filter((r) => r.assetCode === "USDC"); + expect(usdc).toHaveLength(2); + expect(usdc[0].policyId).not.toBe(usdc[1].policyId); + expect(usdc.map((r) => r.fee)).toEqual(["0.5", "1"]); + + const percents = policiesWritten() + .filter((p) => p.policy.assetCode === "USDC") + .map((p) => p.policy.percent); + expect(percents).toEqual(["0.5", "1"]); + // The percent is shared, so XLM's policy changed too — and is recorded. + expect(policiesWritten().filter((p) => p.policy.assetCode === "XLM")).toHaveLength(2); + }); + + it("writes an unchanged policy once, however many quotes it prices", async () => { + for (const amount of [1, 50, 100, 10_000]) { + await request(makeApp()).get(`/fees?amount=${amount}`); + await settle(); + } + expect(quotesWritten()).toHaveLength(8); + expect(policiesWritten()).toHaveLength(2); + // ...and each basis is the rule that actually decided that fee. + expect(quotesWritten().filter((r) => r.assetCode === "USDC").map((r) => r.basis)).toEqual([ + "floor", + "percent", + "percent", + "cap", + ]); + }); + + it("takes nothing from the request beyond the amount", async () => { + await request(makeApp()) + .get("/fees?amount=100&address=GBUO4RL4RTGRFSUDUMRFMC75EWYCRTX5OE3PBIXJVDCZULOXQ2TKDR4Z") + .set("user-agent", "VeilWallet/1.2 (Pixel 7)") + .set("x-forwarded-for", "203.0.113.7"); + await settle(); + const written = JSON.stringify([quotesWritten(), policiesWritten()]); + expect(written).not.toContain("GBUO4RL4"); + expect(written).not.toContain("Pixel"); + expect(written).not.toContain("203.0.113.7"); + }); +}); + +// ─── Reporting ────────────────────────────────────────────────────────────── + +describe("reportRange", () => { + it("accepts a single day and an inclusive range", () => { + expect(reportRange("2026-10-17", "2026-10-17")).toEqual({ from: "2026-10-17", to: "2026-10-17" }); + expect(reportRange("2026-10-01", "2026-10-17")).toEqual({ from: "2026-10-01", to: "2026-10-17" }); + }); + + it.each([ + ["not a date", "yesterday", "2026-10-17"], + ["an impossible date", "2026-02-30", "2026-03-01"], + ["a reversed range", "2026-10-17", "2026-10-01"], + ["more than 92 days", "2026-01-01", "2026-12-31"], + ])("refuses %s", (_what, from, to) => { + expect(typeof reportRange(from, to)).toBe("string"); + }); +}); + +describe("GET /fees/report", () => { + const auth = (r: request.Test) => r.set("authorization", `Bearer ${TOKEN}`).set("x-network", "mainnet"); + + it("answers 503 when no report token is configured, so it is never open by accident", async () => { + const res = await request(makeApp()).get("/fees/report").set("authorization", "Bearer anything"); + expect(res.status).toBe(503); + expect(db.$queryRaw).not.toHaveBeenCalled(); + }); + + it.each([ + ["no token", undefined], + ["a wrong token", "Bearer nope"], + ["the right token without the scheme", TOKEN], + ])("refuses %s", async (_what, header) => { + process.env.VEIL_FEE_REPORT_TOKEN = TOKEN; + const req = request(makeApp()).get("/fees/report"); + const res = await (header ? req.set("authorization", header) : req); + expect(res.status).toBe(401); + expect(res.headers["www-authenticate"]).toBe("Bearer"); + expect(db.$queryRaw).not.toHaveBeenCalled(); + }); + + it("refuses a bad range with 400", async () => { + process.env.VEIL_FEE_REPORT_TOKEN = TOKEN; + const res = await auth(request(makeApp()).get("/fees/report?from=2026-10-17&to=2026-10-01")); + expect(res.status).toBe(400); + }); + + it("answers 500, not a stack trace, when the ledger cannot be read", async () => { + process.env.VEIL_FEE_REPORT_TOKEN = TOKEN; + db.$queryRaw.mockRejectedValue(new Error("relation does not exist")); + const res = await auth(request(makeApp()).get("/fees/report?from=2026-10-17")); + expect(res.status).toBe(500); + expect(JSON.stringify(res.body)).not.toContain("relation"); + }); + + it("defaults to today, UTC", async () => { + process.env.VEIL_FEE_REPORT_TOKEN = TOKEN; + const today = new Date().toISOString().slice(0, 10); + const res = await auth(request(makeApp()).get("/fees/report")); + expect(res.status).toBe(200); + expect(res.body).toMatchObject({ from: today, to: today, days: [] }); + }); + + it("reports per day and asset, with the policies and what was collected", async () => { + process.env.VEIL_FEE_REPORT_TOKEN = TOKEN; + const usdcPolicy = policyOf(percentFee()); + const xlmPolicy = policyOf(flatFee()); + const usdcId = policyId(usdcPolicy); + const xlmId = policyId(xlmPolicy); + const xlmSac = Asset.native().contractId(Networks.PUBLIC); + // XLM's contract is indexed; USDC's is not. + (resolveSacContractIds as jest.Mock).mockReturnValue([xlmSac]); + + db.$queryRaw + .mockResolvedValueOnce([ + { day: "2026-10-16", assetCode: "USDC", assetIssuer: ISSUER, quotes: 3, feesQuoted: "1.0400000", alarms: 1, policyIds: [usdcId] }, + { day: "2026-10-16", assetCode: "XLM", assetIssuer: null, quotes: 2, feesQuoted: "0.2000000", alarms: 0, policyIds: [xlmId] }, + { day: "2026-10-17", assetCode: "XLM", assetIssuer: null, quotes: 1, feesQuoted: "0.1000000", alarms: 0, policyIds: [xlmId] }, + ]) + .mockResolvedValueOnce([ + { day: "2026-10-16", total: "2000000" }, + { day: "2026-10-15", total: "15000000" }, + ]); + db.feePolicy.findMany.mockResolvedValue([ + { id: usdcId, policy: usdcPolicy, firstSeenAt: new Date("2026-10-01T09:00:00Z") }, + { id: xlmId, policy: xlmPolicy, firstSeenAt: new Date("2026-09-30T12:00:00Z") }, + ]); + + const res = await auth(request(makeApp()).get("/fees/report?from=2026-10-15&to=2026-10-17")); + + expect(res.status).toBe(200); + expect(res.body.network).toBe("mainnet"); + expect(res.body.alarmPercent).toBe(10); + expect(res.body.policies[usdcId]).toEqual({ firstSeenAt: "2026-10-01T09:00:00.000Z", policy: usdcPolicy }); + expect(res.body.days.map((d: { date: string }) => d.date)).toEqual(["2026-10-15", "2026-10-16", "2026-10-17"]); + + const [d15, d16, d17] = res.body.days; + // Received with no quote that day: shown, not dropped. + expect(d15.assets).toEqual([ + expect.objectContaining({ assetCode: "XLM", quotes: 0, feesQuoted: "0", collected: "1.5" }), + ]); + expect(d16.assets).toEqual([ + { + assetCode: "USDC", + assetIssuer: ISSUER, + quotes: 3, + feesQuoted: "1.04", + alarms: 1, + policyIds: [usdcId], + collected: null, + collectedNote: expect.stringMatching(/not indexed/), + }, + { assetCode: "XLM", assetIssuer: null, quotes: 2, feesQuoted: "0.2", alarms: 0, policyIds: [xlmId], collected: "0.2" }, + ]); + // Indexed and quoted, but nothing arrived: zero is a real answer here. + expect(d17.assets[0]).toMatchObject({ assetCode: "XLM", collected: "0" }); + + // Scoped to the network and the range asked for, with the end exclusive. + const [, ...values] = db.$queryRaw.mock.calls[0]; + expect(values).toEqual(["mainnet", new Date("2026-10-15T00:00:00Z"), new Date("2026-10-18T00:00:00Z")]); + // Collection is read only for the destination the policies named. + const collectedValues = db.$queryRaw.mock.calls[1].slice(1); + expect(collectedValues).toEqual( + expect.arrayContaining(["mainnet", xlmSac, [DESTINATION]]), + ); + }); + + it("feeReport reads no policies and no transfers for an empty range", async () => { + const report = await feeReport("mainnet", "2026-10-17", "2026-10-17"); + expect(report.days).toEqual([]); + expect(db.feePolicy.findMany).not.toHaveBeenCalled(); + expect(db.$queryRaw).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/api/feeLedger.ts b/src/api/feeLedger.ts new file mode 100644 index 000000000..9fbcf6c81 --- /dev/null +++ b/src/api/feeLedger.ts @@ -0,0 +1,416 @@ +/** + * The platform fee ledger: what `GET /fees` quoted, under which policy. + * + * ## Why this exists + * + * The fee is collected on-chain, in the same transaction as the user's own + * payment, and the money never passes through wraith. So until this, nothing on + * this side knew it had happened: there was no answer to how much had been + * charged, in which asset, on which day, and nothing tied a payment on the + * treasury account to the request that priced it. And because the fee is now + * policy — percent, floor and cap — rather than a constant, a misconfiguration + * would overcharge every user until somebody read a block explorer. + * + * ## What is recorded, and what deliberately is not + * + * One row per asset per quote: the amount priced against, the asset, the fee + * returned, the rule that decided it, and the policy that was in force. The + * policy is taken from the very entry the client was sent — never by reading + * the environment a second time — so the record cannot disagree with what the + * user was told. + * + * **Nothing recorded identifies a person.** No IP, no user agent, no sender or + * wallet address. The only address anywhere in here is the fee destination, + * which `/fees` publishes to anyone. Do not add a column that changes that: + * the amount and the time of a quote are not personal data on their own, but + * either joined to a requester would be. + * + * ## Failure + * + * Recording never throws and is never awaited by the response. The person is + * in the middle of a payment; a ledger outage must not become a failed send. + * A write that fails is logged and counted in + * `veil_fee_quote_record_failures_total`, so a gap is visible rather than a + * quietly low total. + */ + +import { createHash } from "crypto"; +import { Asset, Networks } from "@stellar/stellar-sdk"; + +import { prisma } from "../db"; +import { toDisplayAmount } from "../amount"; +import { resolveSacContractIds } from "../indexer"; +import { feeQuoteAlarmsTotal, feeQuoteRecordFailuresTotal } from "../metrics"; +import type { Network } from "../network"; +import type { FeeBasis, PlatformFee } from "./fees"; + +// ─── Policy ────────────────────────────────────────────────────────────────── + +/** Everything that decides a fee for one asset, and nothing about who asked. */ +export interface FeePolicyShape { + mode: "flat" | "percent"; + assetCode: string; + assetIssuer: string | null; + destination: string; + /** Percent mode only. */ + percent: string | null; + min: string | null; + max: string | null; + /** Flat mode only: the fixed fee. */ + amount: string | null; +} + +/** + * The policy behind one `/fees` entry. + * + * In percent mode `amount` is the *result* for this quote, not policy, so it is + * left out; otherwise every distinct amount would look like a policy change. + */ +export function policyOf(fee: PlatformFee): FeePolicyShape { + const percentMode = fee.percent !== undefined; + return { + mode: percentMode ? "percent" : "flat", + assetCode: fee.assetCode, + assetIssuer: fee.assetIssuer, + destination: fee.destination, + percent: percentMode ? fee.percent! : null, + min: percentMode ? (fee.min ?? null) : null, + max: percentMode ? (fee.max ?? null) : null, + amount: percentMode ? null : fee.amount, + }; +} + +/** + * A stable id for a policy: the same configuration always hashes the same, + * any change to it does not. Keys are written in a fixed order so the id does + * not depend on how the object happened to be built. + */ +export function policyId(policy: FeePolicyShape): string { + const canonical = JSON.stringify([ + policy.mode, + policy.assetCode, + policy.assetIssuer, + policy.destination, + policy.percent, + policy.min, + policy.max, + policy.amount, + ]); + return createHash("sha256").update(canonical).digest("hex").slice(0, 32); +} + +// ─── Alarm ─────────────────────────────────────────────────────────────────── + +/** + * The default ceiling: a fee above 10% of what it was quoted against. + * + * The fee policy's own rationale calls 10% of a send "stark" — it is the + * number that made the flat fee indefensible. A sane percent is well under a + * tenth of that, so crossing it means either a floor on a very small send or + * a misconfigured policy. Both are worth a human looking. + */ +export const DEFAULT_ALARM_PERCENT = 10; + +/** `VEIL_FEE_ALARM_PERCENT`, or the default when unset or not a sane number. */ +export function alarmPercent(): number { + const raw = Number(process.env.VEIL_FEE_ALARM_PERCENT?.trim() || NaN); + return Number.isFinite(raw) && raw > 0 && raw <= 100 ? raw : DEFAULT_ALARM_PERCENT; +} + +// ─── Recording ─────────────────────────────────────────────────────────────── + +export interface QuotedFee { + fee: PlatformFee; + basis: FeeBasis; +} + +const DP = 7; + +/** A positive decimal, at Stellar's seven places, trailing zeros removed. */ +function decimal(value: number): string { + if (!Number.isFinite(value)) return String(value); + // toFixed switches to exponent notation from 1e21 on; such an amount is not + // sendable anyway, but the record should still say what was asked. + if (Math.abs(value) >= 1e21) return String(value); + return value.toFixed(DP).replace(/0+$/, "").replace(/\.$/, "") || "0"; +} + +/** Strip trailing zeros from a decimal string that Postgres returned. */ +function trimDecimal(value: string): string { + return value.includes(".") ? value.replace(/0+$/, "").replace(/\.$/, "") : value; +} + +/** The row this quote becomes, before anything is written. */ +export function quoteRow( + network: Network, + sent: number | undefined, + { fee, basis }: QuotedFee, + limitPercent = alarmPercent(), +) { + const policy = policyOf(fee); + const charged = Number(fee.amount); + const percentOfAmount = sent !== undefined && sent > 0 ? (charged / sent) * 100 : null; + return { + policy, + data: { + network, + assetCode: fee.assetCode, + assetIssuer: fee.assetIssuer, + quotedAgainst: sent !== undefined ? decimal(sent) : null, + fee: fee.amount, + basis, + feePercent: percentOfAmount !== null ? decimal(percentOfAmount) : null, + alarm: percentOfAmount !== null && percentOfAmount > limitPercent, + policyId: policyId(policy), + }, + }; +} + +/** + * Policies already written by this process, so a steady stream of quotes + * under one configuration does not re-insert the same row every time. + */ +const knownPolicies = new Set(); + +/** Test-only. */ +export function _forgetKnownPolicies(): void { + knownPolicies.clear(); +} + +/** + * Write every fee in one `/fees` answer to the ledger. + * + * Never throws — see the module header. The alarm is raised before the write, + * so a database outage cannot also silence an overcharge. + */ +export async function recordFeeQuotes( + network: Network, + sent: number | undefined, + quoted: QuotedFee[], +): Promise { + if (quoted.length === 0) return; + try { + const limit = alarmPercent(); + const rows = quoted.map((q) => quoteRow(network, sent, q, limit)); + + for (const { data } of rows) { + if (!data.alarm) continue; + feeQuoteAlarmsTotal.inc({ network, asset: data.assetCode }); + // Amount, fee and policy only — none of it identifies anyone. + console.error( + `[fees] ALARM ${network} ${data.assetCode}: fee ${data.fee} is ${data.feePercent}% of ` + + `${data.quotedAgainst}, over the ${limit}% limit (policy ${data.policyId}, basis ${data.basis})`, + ); + } + + const newPolicies = new Map(); + for (const { policy, data } of rows) { + if (!knownPolicies.has(data.policyId)) newPolicies.set(data.policyId, policy); + } + if (newPolicies.size > 0) { + // skipDuplicates: two instances seeing a new policy at once is a race we + // expect, and the first writer's `firstSeenAt` is the true one. + await prisma.feePolicy.createMany({ + data: [...newPolicies].map(([id, policy]) => ({ id, policy: { ...policy } })), + skipDuplicates: true, + }); + for (const id of newPolicies.keys()) knownPolicies.add(id); + } + + await prisma.feeQuote.createMany({ data: rows.map((r) => r.data) }); + } catch (err) { + feeQuoteRecordFailuresTotal.inc({ network }); + console.error( + `[fees] could not record ${quoted.length} fee quote(s): ${err instanceof Error ? err.message : String(err)}`, + ); + } +} + +// ─── Reporting ─────────────────────────────────────────────────────────────── + +interface QuotedDayRow { + day: string; + assetCode: string; + assetIssuer: string | null; + quotes: number; + feesQuoted: string; + alarms: number; + policyIds: string[]; +} + +export interface FeeReportAsset { + assetCode: string; + assetIssuer: string | null; + /** Quotes that returned a fee in this asset. */ + quotes: number; + /** The sum of every fee quoted — what clients were told to charge. */ + feesQuoted: string; + /** Quotes over the alarm limit. */ + alarms: number; + /** The policies that priced them. Look each up in `policies`. */ + policyIds: string[]; + /** + * What the fee destination actually received in this asset that day, + * according to the index — or null when this asset is not indexed, which is + * not the same thing as zero. + */ + collected: string | null; + collectedNote?: string; +} + +export interface FeeReport { + network: Network; + from: string; + to: string; + alarmPercent: number; + days: Array<{ date: string; assets: FeeReportAsset[] }>; + policies: Record; +} + +/** The asset's Stellar Asset Contract id on `network`, or null if it has none. */ +function sacId(network: Network, code: string, issuer: string | null): string | null { + try { + const asset = issuer ? new Asset(code, issuer) : Asset.native(); + return asset.contractId(network === "mainnet" ? Networks.PUBLIC : Networks.TESTNET); + } catch { + return null; + } +} + +const assetKey = (code: string, issuer: string | null) => `${code}:${issuer ?? ""}`; + +/** + * What was charged on each day from `from` to `to` (inclusive, UTC dates), + * per asset, and under which policies. + * + * "Collected" comes from the indexed transfers into the destinations those + * policies named. It counts everything the destination received in that + * asset, so it is a true collected total only while the destination is used + * for fees alone — which is how it should be configured. + */ +export async function feeReport(network: Network, from: string, to: string): Promise { + const start = new Date(`${from}T00:00:00.000Z`); + const end = new Date(`${to}T00:00:00.000Z`); + end.setUTCDate(end.getUTCDate() + 1); + + const quoted = await prisma.$queryRaw` + SELECT to_char(date_trunc('day', "quotedAt"), 'YYYY-MM-DD') AS "day", + "assetCode", + "assetIssuer", + count(*)::int AS "quotes", + COALESCE(sum("fee"), 0)::text AS "feesQuoted", + (count(*) FILTER (WHERE "alarm"))::int AS "alarms", + array_agg(DISTINCT "policyId") AS "policyIds" + FROM "wraith"."FeeQuote" + WHERE "network" = ${network} + AND "quotedAt" >= ${start} + AND "quotedAt" < ${end} + GROUP BY 1, 2, 3 + ORDER BY 1, 2, 3`; + + const ids = [...new Set(quoted.flatMap((r) => r.policyIds))]; + const policyRows = ids.length + ? await prisma.feePolicy.findMany({ where: { id: { in: ids } } }) + : []; + const policies: FeeReport["policies"] = {}; + for (const row of policyRows) { + policies[row.id] = { + firstSeenAt: row.firstSeenAt.toISOString(), + policy: row.policy as unknown as FeePolicyShape, + }; + } + + // Collected, per asset that was quoted in the range. + const indexed = new Set(resolveSacContractIds(network)); + const collected = new Map>(); + const notes = new Map(); + const assets = new Map(); + for (const r of quoted) assets.set(assetKey(r.assetCode, r.assetIssuer), { code: r.assetCode, issuer: r.assetIssuer }); + + for (const [key, { code, issuer }] of assets) { + const contractId = sacId(network, code, issuer); + if (!contractId || !indexed.has(contractId)) { + notes.set(key, "This asset's contract is not indexed by this deployment, so collection cannot be read."); + continue; + } + const destinations = [ + ...new Set( + quoted + .filter((r) => assetKey(r.assetCode, r.assetIssuer) === key) + .flatMap((r) => r.policyIds) + .map((id) => policies[id]?.policy.destination) + .filter((d): d is string => typeof d === "string"), + ), + ]; + if (destinations.length === 0) { + notes.set(key, "The policies behind these quotes could not be read, so their destination is unknown."); + continue; + } + const received = await prisma.$queryRaw>` + SELECT to_char(date_trunc('day', "ledgerClosedAt"), 'YYYY-MM-DD') AS "day", + COALESCE(sum("amount"::numeric), 0)::text AS "total" + FROM "wraith"."TokenTransfer" + WHERE "network" = ${network} + AND "contractId" = ${contractId} + AND "eventType" = 'transfer' + AND "toAddress" = ANY(${destinations}) + AND "ledgerClosedAt" >= ${start} + AND "ledgerClosedAt" < ${end} + GROUP BY 1`; + const byDay = new Map(); + for (const row of received) { + // SAC amounts are integer stroops at seven decimals. + byDay.set(row.day, trimDecimal(toDisplayAmount(row.total.split(".")[0], DP))); + } + collected.set(key, byDay); + } + + // One entry per (day, asset): every quoted pair, plus any day the + // destination received funds in a quoted asset without a quote that day. + const days = new Map>(); + const slot = (day: string, code: string, issuer: string | null): FeeReportAsset => { + if (!days.has(day)) days.set(day, new Map()); + const perDay = days.get(day)!; + const key = assetKey(code, issuer); + if (!perDay.has(key)) { + const note = notes.get(key); + perDay.set(key, { + assetCode: code, + assetIssuer: issuer, + quotes: 0, + feesQuoted: "0", + alarms: 0, + policyIds: [], + collected: collected.has(key) ? (collected.get(key)!.get(day) ?? "0") : null, + ...(note ? { collectedNote: note } : {}), + }); + } + return perDay.get(key)!; + }; + + for (const r of quoted) { + const entry = slot(r.day, r.assetCode, r.assetIssuer); + entry.quotes = r.quotes; + entry.feesQuoted = trimDecimal(r.feesQuoted); + entry.alarms = r.alarms; + entry.policyIds = [...r.policyIds].sort(); + } + for (const [key, byDay] of collected) { + const { code, issuer } = assets.get(key)!; + for (const day of byDay.keys()) slot(day, code, issuer); + } + + return { + network, + from, + to, + alarmPercent: alarmPercent(), + days: [...days.keys()].sort().map((date) => ({ + date, + assets: [...days.get(date)!.values()].sort((a, b) => + assetKey(a.assetCode, a.assetIssuer).localeCompare(assetKey(b.assetCode, b.assetIssuer)), + ), + })), + policies, + }; +} diff --git a/src/api/fees.ts b/src/api/fees.ts index 23de81c1b..631705c20 100644 --- a/src/api/fees.ts +++ b/src/api/fees.ts @@ -1,5 +1,9 @@ +import { createHash, timingSafeEqual } from "crypto"; import { Router, type Request, type Response } from "express"; +import { requestNetwork } from "../middleware/network"; +import { feeReport, recordFeeQuotes } from "./feeLedger"; + /** * What Veil charges, answered by the server so the number is never in an APK. * @@ -74,6 +78,27 @@ export function feeForAmount( return fee.toFixed(STROOP_DP).replace(/0+$/, '').replace(/\.$/, ''); } +/** + * Which rule decided a fee — what the ledger records so a charge can be + * explained later without re-deriving it from a policy that may have changed. + * + * `no_amount` is the floor told to a client that sent no amount; `floor` is + * the floor overriding a percent that came out smaller. + */ +export type FeeBasis = "percent" | "floor" | "cap" | "flat" | "no_amount"; + +export function feeBasis(fee: PlatformFee, sent?: number): FeeBasis { + if (fee.percent === undefined) return "flat"; + if (sent === undefined) return "no_amount"; + const charged = Number(fee.amount); + const uncapped = Number( + feeForAmount(sent, { percent: Number(fee.percent), min: null, max: null }), + ); + if (fee.max !== undefined && charged === Number(fee.max) && uncapped > charged) return "cap"; + if (fee.min !== undefined && charged === Number(fee.min) && uncapped < charged) return "floor"; + return "percent"; +} + function validAmount(value: string | undefined): string | null { const amount = value?.trim(); if (!amount) return null; @@ -162,6 +187,38 @@ export function configuredFees(sentAmount?: number): Record return fees; } +/** The longest range one report may cover, so a request cannot scan the whole ledger. */ +export const MAX_REPORT_DAYS = 92; + +/** `YYYY-MM-DD` that is also a real calendar date. */ +function isUtcDate(value: string): boolean { + if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) return false; + const parsed = new Date(`${value}T00:00:00.000Z`); + return !Number.isNaN(parsed.getTime()) && parsed.toISOString().slice(0, 10) === value; +} + +/** A validated inclusive date range, or the reason it is not one. */ +export function reportRange(from: string, to: string): { from: string; to: string } | string { + if (!isUtcDate(from) || !isUtcDate(to)) return "from and to must be dates in YYYY-MM-DD form (UTC)"; + const days = (Date.parse(to) - Date.parse(from)) / 86_400_000 + 1; + if (days < 1) return "from must not be after to"; + if (days > MAX_REPORT_DAYS) return `A report covers at most ${MAX_REPORT_DAYS} days`; + return { from, to }; +} + +function bearer(req: Request): string { + const match = /^Bearer +(\S+)$/i.exec(String(req.headers.authorization ?? "").trim()); + return match ? match[1] : ""; +} + +/** Constant-time comparison; hashing first makes the lengths equal. */ +function sameSecret(given: string, expected: string): boolean { + if (!given) return false; + const a = createHash("sha256").update(given).digest(); + const b = createHash("sha256").update(expected).digest(); + return timingSafeEqual(a, b); +} + export function feesRouter(): Router { const router = Router(); @@ -182,6 +239,53 @@ export function feesRouter(): Router { // from "this backend is too old to know about fees" and does not read a // deliberate waiver as a failure. res.json(assets.length > 0 ? { enabled: true, fees } : { enabled: false, fees: {} }); + + // Recorded after the answer is sent, and never awaited: the person is in + // the middle of a payment, and a ledger outage must not become a failed + // send. `recordFeeQuotes` cannot throw. See `./feeLedger`. + void recordFeeQuotes( + requestNetwork(req), + sent, + Object.values(fees).map((fee) => ({ fee, basis: feeBasis(fee, sent) })), + ); + }); + + /** + * What was charged, per UTC day and asset, and under which policy. + * + * Behind a bearer token, `VEIL_FEE_REPORT_TOKEN`: nothing in it identifies a + * person, but collected totals are the operator's business volume, not + * something to publish. Without the token configured the route answers 503, + * so a deployment never serves it open by accident. + */ + router.get("/report", async (req: Request, res: Response) => { + res.setHeader("Cache-Control", "no-store"); + const expected = process.env.VEIL_FEE_REPORT_TOKEN?.trim(); + if (!expected) { + res.status(503).json({ error: "Fee reporting is not configured on this deployment" }); + return; + } + if (!sameSecret(bearer(req), expected)) { + res.setHeader("WWW-Authenticate", "Bearer"); + res.status(401).json({ error: "A valid report token is required" }); + return; + } + + const today = new Date().toISOString().slice(0, 10); + const from = req.query["from"] === undefined ? today : String(req.query["from"]); + const to = req.query["to"] === undefined ? from : String(req.query["to"]); + const range = reportRange(from, to); + if (typeof range === "string") { + res.status(400).json({ error: range }); + return; + } + + try { + res.json(await feeReport(requestNetwork(req), range.from, range.to)); + } catch (err) { + console.error(`[fees] report failed: ${err instanceof Error ? err.message : String(err)}`); + res.status(500).json({ error: "Could not read the fee ledger" }); + } }); return router; diff --git a/src/metrics.ts b/src/metrics.ts index 89f44268e..f1a8d76ab 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -185,6 +185,35 @@ export function recordSkippedLedger(network: Network): void { ledgersSkippedTotal.inc({ network }); } +/** + * Fee quotes whose fee exceeded `VEIL_FEE_ALARM_PERCENT` of the amount priced. + * + * The fee is policy — percent, floor and cap — so a misconfiguration + * overcharges every user at once, and until now the only way to notice was + * reading the treasury's history on a block explorer. Alert on any increase. + * Labelled by asset and network only: never an amount, never an address. + */ +export const feeQuoteAlarmsTotal = new Counter({ + name: "veil_fee_quote_alarms_total", + help: "Fee quotes whose fee exceeded VEIL_FEE_ALARM_PERCENT of the amount priced", + labelNames: ["network", "asset"] as const, + registers: [registry], +}); + +/** + * Fee quotes that could not be written to the ledger. + * + * The quote itself still succeeds — the user is mid-payment — so a failed + * write is otherwise silent, and a ledger with gaps reports totals that are + * quietly too low. Alert on any increase. + */ +export const feeQuoteRecordFailuresTotal = new Counter({ + name: "veil_fee_quote_record_failures_total", + help: "Fee quotes that could not be written to the fee ledger", + labelNames: ["network"] as const, + registers: [registry], +}); + /** Serialize the registry in Prometheus text exposition format. */ export function renderMetrics(): Promise { return registry.metrics(); diff --git a/src/openapi/build.ts b/src/openapi/build.ts index 7486c938c..e776f95fd 100644 --- a/src/openapi/build.ts +++ b/src/openapi/build.ts @@ -52,6 +52,8 @@ import { offrampErrorResponseSchema, ngnCustomerStatusQuerySchema, ngnCustomerStatusResponseSchema, + feeReportQuerySchema, + feeReportResponseSchema, } from "./schemas"; const registry = new OpenAPIRegistry(); @@ -537,6 +539,27 @@ registry.registerPath({ }, }); +registry.registerComponent("securitySchemes", "bearerAuth", { type: "http", scheme: "bearer" }); + +registry.registerPath({ + method: "get", + path: "/fees/report", + summary: "Platform fees charged, per UTC day and asset, with the policy behind them", + description: + "Internal endpoint. Not part of the public data API: an operator report over the fee " + + "ledger, behind the VEIL_FEE_REPORT_TOKEN bearer token. Nothing in it identifies a person.", + "x-internal": true, + security: [{ bearerAuth: [] }], + request: { query: feeReportQuerySchema }, + responses: { + 200: { description: "OK", content: { "application/json": { schema: feeReportResponseSchema } } }, + 400: { description: "Bad date range", content: { "application/json": { schema: errorResponseSchema } } }, + 401: { description: "Missing or wrong report token", content: { "application/json": { schema: errorResponseSchema } } }, + 500: { description: "The fee ledger could not be read", content: { "application/json": { schema: errorResponseSchema } } }, + 503: { description: "Fee reporting is not configured on this deployment", content: { "application/json": { schema: errorResponseSchema } } }, + }, +}); + const generator = new OpenApiGeneratorV3(registry.definitions); const document = generator.generateDocument({ openapi: "3.0.3", diff --git a/src/openapi/schemas.ts b/src/openapi/schemas.ts index 6e7bdc178..9dc0d47d4 100644 --- a/src/openapi/schemas.ts +++ b/src/openapi/schemas.ts @@ -755,3 +755,51 @@ export const ngnCustomerStatusResponseSchema = z.object({ status: z.string().optional(), }); + +export const feeReportQuerySchema = z.object({ + from: optionalQueryString("First UTC date to report, YYYY-MM-DD. Defaults to today."), + to: optionalQueryString("Last UTC date to report, inclusive, YYYY-MM-DD. Defaults to `from`. At most 92 days after it."), +}); + +const feePolicySchema = z.object({ + mode: z.enum(["flat", "percent"]), + assetCode: z.string(), + assetIssuer: z.string().nullable(), + destination: z.string().openapi({ description: "The fee destination — the address `/fees` already publishes." }), + percent: z.string().nullable(), + min: z.string().nullable(), + max: z.string().nullable(), + amount: z.string().nullable().openapi({ description: "Flat mode only: the fixed fee." }), +}); + +export const feeReportResponseSchema = z.object({ + network: z.string(), + from: z.string(), + to: z.string(), + alarmPercent: z.number().openapi({ description: "VEIL_FEE_ALARM_PERCENT in force for this report." }), + days: z.array( + z.object({ + date: z.string(), + assets: z.array( + z.object({ + assetCode: z.string(), + assetIssuer: z.string().nullable(), + quotes: z.number().int(), + feesQuoted: z.string().openapi({ description: "Sum of every fee quoted — what clients were told to charge." }), + alarms: z.number().int(), + policyIds: z.array(z.string()), + collected: z.string().nullable().openapi({ + description: + "What the fee destination received in this asset that day, per the index. " + + "Null when the asset is not indexed — which is not zero.", + }), + collectedNote: z.string().optional(), + }), + ), + }), + ), + policies: z.record( + z.string(), + z.object({ firstSeenAt: z.string(), policy: feePolicySchema }), + ), +});