diff --git a/.env.example b/.env.example index 1d8ed5ef1..6b62043ca 100644 --- a/.env.example +++ b/.env.example @@ -151,3 +151,13 @@ LP_POOL_CONTRACT_IDS_MAINNET= # HTTP tests run against a seeded database without an ingest racing them, and # without needing live RPC. Not for production. SKIP_INDEXER=false + +# ─── Platform fee ledger (GET /fees/report) ────────────────────────────────── +# Every fee GET /fees quotes is recorded with the policy that produced it. +# A quote whose fee exceeds this percent of the amount it priced raises an +# alarm: the row is flagged, `veil_fee_quote_alarms_total` increments, and a +# line is logged. Default 10 — the rate the fee policy itself calls stark. +VEIL_FEE_ALARM_PERCENT=10 +# Bearer token for GET /fees/report. Unset means the report answers 503, so it +# is never served open by accident. Use a long random value. +VEIL_FEE_REPORT_TOKEN= diff --git a/docker-compose.yml b/docker-compose.yml index ffdf83ad6..528a42926 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,6 +73,8 @@ services: LP_POOL_CONTRACT_IDS_TESTNET: ${LP_POOL_CONTRACT_IDS_TESTNET:-} LP_POOL_CONTRACT_IDS_MAINNET: ${LP_POOL_CONTRACT_IDS_MAINNET:-} SKIP_INDEXER: ${SKIP_INDEXER:-false} + VEIL_FEE_ALARM_PERCENT: ${VEIL_FEE_ALARM_PERCENT:-10} + VEIL_FEE_REPORT_TOKEN: ${VEIL_FEE_REPORT_TOKEN:-} ports: - "${PORT:-3000}:3000" diff --git a/openapi.json b/openapi.json index f807dbbbb..6ce845880 100644 --- a/openapi.json +++ b/openapi.json @@ -16,6 +16,12 @@ } ], "components": { + "securitySchemes": { + "bearerAuth": { + "type": "http", + "scheme": "bearer" + } + }, "schemas": {}, "parameters": {} }, @@ -5990,6 +5996,277 @@ } } } + }, + "/fees/report": { + "get": { + "summary": "Platform fees charged, per UTC day and asset, with the policy behind them", + "description": "Internal endpoint. Not part of the public data API: an operator report over the fee ledger, behind the VEIL_FEE_REPORT_TOKEN bearer token. Nothing in it identifies a person.", + "x-internal": true, + "security": [ + { + "bearerAuth": [] + } + ], + "parameters": [ + { + "schema": { + "type": "string", + "nullable": true, + "description": "First UTC date to report, YYYY-MM-DD. Defaults to today." + }, + "required": false, + "description": "First UTC date to report, YYYY-MM-DD. Defaults to today.", + "name": "from", + "in": "query" + }, + { + "schema": { + "type": "string", + "nullable": true, + "description": "Last UTC date to report, inclusive, YYYY-MM-DD. Defaults to `from`. At most 92 days after it." + }, + "required": false, + "description": "Last UTC date to report, inclusive, YYYY-MM-DD. Defaults to `from`. At most 92 days after it.", + "name": "to", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "network": { + "type": "string" + }, + "from": { + "type": "string" + }, + "to": { + "type": "string" + }, + "alarmPercent": { + "type": "number", + "description": "VEIL_FEE_ALARM_PERCENT in force for this report." + }, + "days": { + "type": "array", + "items": { + "type": "object", + "properties": { + "date": { + "type": "string" + }, + "assets": { + "type": "array", + "items": { + "type": "object", + "properties": { + "assetCode": { + "type": "string" + }, + "assetIssuer": { + "type": "string", + "nullable": true + }, + "quotes": { + "type": "integer" + }, + "feesQuoted": { + "type": "string", + "description": "Sum of every fee quoted — what clients were told to charge." + }, + "alarms": { + "type": "integer" + }, + "policyIds": { + "type": "array", + "items": { + "type": "string" + } + }, + "collected": { + "type": "string", + "nullable": true, + "description": "What the fee destination received in this asset that day, per the index. Null when the asset is not indexed — which is not zero." + }, + "collectedNote": { + "type": "string" + } + }, + "required": [ + "assetCode", + "assetIssuer", + "quotes", + "feesQuoted", + "alarms", + "policyIds", + "collected" + ] + } + } + }, + "required": [ + "date", + "assets" + ] + } + }, + "policies": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "firstSeenAt": { + "type": "string" + }, + "policy": { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": [ + "flat", + "percent" + ] + }, + "assetCode": { + "type": "string" + }, + "assetIssuer": { + "type": "string", + "nullable": true + }, + "destination": { + "type": "string", + "description": "The fee destination — the address `/fees` already publishes." + }, + "percent": { + "type": "string", + "nullable": true + }, + "min": { + "type": "string", + "nullable": true + }, + "max": { + "type": "string", + "nullable": true + }, + "amount": { + "type": "string", + "nullable": true, + "description": "Flat mode only: the fixed fee." + } + }, + "required": [ + "mode", + "assetCode", + "assetIssuer", + "destination", + "percent", + "min", + "max", + "amount" + ] + } + }, + "required": [ + "firstSeenAt", + "policy" + ] + } + } + }, + "required": [ + "network", + "from", + "to", + "alarmPercent", + "days", + "policies" + ] + } + } + } + }, + "400": { + "description": "Bad date range", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ] + } + } + } + }, + "401": { + "description": "Missing or wrong report token", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ] + } + } + } + }, + "500": { + "description": "The fee ledger could not be read", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ] + } + } + } + }, + "503": { + "description": "Fee reporting is not configured on this deployment", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ] + } + } + } + } + } + } } } } diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 4d9130e30..bd69ea45c 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -495,3 +495,55 @@ model NgnOrder { @@index([network, status]) @@schema("wraith") } + +// ─── Platform fee ledger ───────────────────────────────────────────────────── + +/// One fee policy for one asset, as `GET /fees` applied it. +/// +/// Keyed by a hash of the policy itself, so the same configuration is the same +/// row however many quotes it prices, and any change — a new percent, floor, +/// cap, flat amount or destination — is a new row with its own `firstSeenAt`. +/// That is what lets a fee be explained after the fact: every `FeeQuote` names +/// the policy that produced it, and this table says when that policy appeared. +model FeePolicy { + /// sha256 of the canonical policy JSON, hex, first 32 characters. + id String @id + /// `{ mode, assetCode, assetIssuer, destination, percent, min, max, amount }` + /// — everything that decides a fee, and nothing about who asked. + policy Json + firstSeenAt DateTime @default(now()) + + @@schema("wraith") +} + +/// One fee quoted by `GET /fees`, per asset. +/// +/// **Nothing here identifies a person.** No IP, no user agent, no sender +/// address — the request carries none, and none is taken. The only address +/// is the fee destination, held inside the policy, which `/fees` publishes to +/// anyone who asks. The amount and the time are what a quote *is*; neither +/// names who wanted it. +model FeeQuote { + id Int @id @default(autoincrement()) + network String + assetCode String + /// Null for native XLM. + assetIssuer String? + /// The amount priced against, in whole units of the asset. Null when the + /// client sent none — an old client, which is told the floor. + quotedAgainst String? + /// What the client was told to charge. + fee Decimal @db.Decimal(38, 7) + /// Which rule decided the fee: percent | floor | cap | flat | no_amount. + basis String + /// The fee as a percentage of `quotedAgainst`. Null without an amount. + feePercent Decimal? @db.Decimal(20, 7) + /// The fee exceeded VEIL_FEE_ALARM_PERCENT of the amount. + alarm Boolean @default(false) + policyId String + quotedAt DateTime @default(now()) + + @@index([network, quotedAt]) + @@index([network, alarm, quotedAt]) + @@schema("wraith") +} diff --git a/src/__tests__/feeLedger.test.ts b/src/__tests__/feeLedger.test.ts new file mode 100644 index 000000000..2a22264c1 --- /dev/null +++ b/src/__tests__/feeLedger.test.ts @@ -0,0 +1,570 @@ +/** + * W085 — the platform fee ledger. + * + * `GET /fees` tells a client what to charge, and the fee then rides on-chain + * in the user's own transaction. Nothing on this side knew it had happened. + * These pin the four things the issue asks for: + * + * - every quote that returns a fee is recorded, with the policy behind it; + * - totals can be reported per asset and per day; + * - a policy change is visible, so a fee can be explained after the fact; + * - nothing recorded identifies a person. + * + * Plus the alarm, and the property that matters most in production: a ledger + * failure never fails the quote a person is in the middle of paying. + * + * The database is mocked throughout; nothing here touches a provider. + */ +import express from "express"; +import request from "supertest"; +import { Asset, Networks } from "@stellar/stellar-sdk"; + +jest.mock("../db", () => ({ + prisma: { + feePolicy: { createMany: jest.fn(), findMany: jest.fn() }, + feeQuote: { createMany: jest.fn() }, + $queryRaw: jest.fn(), + }, +})); +jest.mock("../indexer", () => ({ resolveSacContractIds: jest.fn() })); + +import { prisma } from "../db"; +import { resolveSacContractIds } from "../indexer"; +import { networkMiddleware } from "../middleware/network"; +import { feeBasis, feesRouter, reportRange, type PlatformFee } from "../api/fees"; +import { + DEFAULT_ALARM_PERCENT, + _forgetKnownPolicies, + alarmPercent, + feeReport, + policyId, + policyOf, + quoteRow, + recordFeeQuotes, +} from "../api/feeLedger"; +import { feeQuoteAlarmsTotal, feeQuoteRecordFailuresTotal, _resetMetrics } from "../metrics"; + +const db = prisma as unknown as { + feePolicy: { createMany: jest.Mock; findMany: jest.Mock }; + feeQuote: { createMany: jest.Mock }; + $queryRaw: jest.Mock; +}; + +const DESTINATION = "GB3JS2AJZ3FKIGVOOPXDEYYR3XOHQA474WGCYAMNFSGX6OW5HGWCKUZW"; +const ISSUER = "GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN"; +const TOKEN = "report-token-for-tests"; + +const FEE_ENV = [ + "VEIL_FEE_DESTINATION", + "VEIL_FEE_ASSET_CODE", + "VEIL_FEE_ASSET_ISSUER", + "VEIL_FEE_AMOUNT", + "VEIL_FEE_AMOUNT_XLM", + "VEIL_FEE_PERCENT", + "VEIL_FEE_MIN", + "VEIL_FEE_MAX", + "VEIL_FEE_MIN_XLM", + "VEIL_FEE_MAX_XLM", + "VEIL_FEE_ALARM_PERCENT", + "VEIL_FEE_REPORT_TOKEN", +]; + +/** The exact columns a quote row may have. A new one must be argued for here. */ +const QUOTE_COLUMNS = [ + "alarm", + "assetCode", + "assetIssuer", + "basis", + "fee", + "feePercent", + "network", + "policyId", + "quotedAgainst", +]; + +const percentFee = (over: Partial = {}): PlatformFee => ({ + amount: "0.5", + assetCode: "USDC", + assetIssuer: ISSUER, + destination: DESTINATION, + percent: "0.5", + min: "0.02", + max: "2", + ...over, +}); + +const flatFee = (over: Partial = {}): PlatformFee => ({ + amount: "0.1", + assetCode: "XLM", + assetIssuer: null, + destination: DESTINATION, + ...over, +}); + +function makeApp() { + const app = express(); + app.use(networkMiddleware); + app.use("/fees", feesRouter()); + return app; +} + +/** Let the fire-and-forget write after a response finish. */ +const settle = () => new Promise((r) => setImmediate(r)); + +const quotesWritten = () => db.feeQuote.createMany.mock.calls.flatMap((c) => c[0].data); +const policiesWritten = () => db.feePolicy.createMany.mock.calls.flatMap((c) => c[0].data); + +beforeEach(() => { + for (const key of FEE_ENV) delete process.env[key]; + process.env.NETWORKS = "testnet,mainnet"; + jest.clearAllMocks(); + db.feePolicy.createMany.mockResolvedValue({ count: 1 }); + db.feeQuote.createMany.mockResolvedValue({ count: 1 }); + db.feePolicy.findMany.mockResolvedValue([]); + db.$queryRaw.mockResolvedValue([]); + (resolveSacContractIds as jest.Mock).mockReturnValue([]); + _forgetKnownPolicies(); + _resetMetrics(); + jest.spyOn(console, "error").mockImplementation(() => {}); +}); + +afterEach(() => jest.restoreAllMocks()); + +afterAll(() => { + for (const key of FEE_ENV) delete process.env[key]; +}); + +// ─── Which rule decided the fee ───────────────────────────────────────────── + +describe("feeBasis", () => { + it("is flat when the policy has no percent", () => { + expect(feeBasis(flatFee(), 10)).toBe("flat"); + }); + + it("is no_amount when an old client sent nothing to price", () => { + expect(feeBasis(percentFee({ amount: "0.02" }))).toBe("no_amount"); + }); + + it("is percent when neither bound applied", () => { + // 0.5% of 100 = 0.5, between 0.02 and 2. + expect(feeBasis(percentFee({ amount: "0.5" }), 100)).toBe("percent"); + }); + + it("is floor when the floor overrode a smaller percent", () => { + // 0.5% of 1 = 0.005, under the 0.02 floor. + expect(feeBasis(percentFee({ amount: "0.02" }), 1)).toBe("floor"); + }); + + it("is cap when the cap overrode a larger percent", () => { + // 0.5% of 10,000 = 50, over the 2 cap. + expect(feeBasis(percentFee({ amount: "2" }), 10_000)).toBe("cap"); + }); + + it("is percent, not floor, when the percent lands exactly on the floor", () => { + // 0.5% of 4 = 0.02 — the floor did not change anything. + expect(feeBasis(percentFee({ amount: "0.02" }), 4)).toBe("percent"); + }); +}); + +// ─── Policies ─────────────────────────────────────────────────────────────── + +describe("policyOf / policyId", () => { + it("leaves a percent quote's computed amount out of the policy", () => { + // Otherwise every distinct send would look like a policy change. + const a = policyOf(percentFee({ amount: "0.5" })); + const b = policyOf(percentFee({ amount: "1.25" })); + expect(a.amount).toBeNull(); + expect(policyId(a)).toBe(policyId(b)); + }); + + it("keeps a flat fee's amount, because there it is the policy", () => { + expect(policyOf(flatFee()).amount).toBe("0.1"); + expect(policyId(policyOf(flatFee({ amount: "0.1" })))).not.toBe( + policyId(policyOf(flatFee({ amount: "0.2" }))), + ); + }); + + it.each([ + ["percent", { percent: "1" }], + ["floor", { min: "0.05" }], + ["cap", { max: "5" }], + ["destination", { destination: "GA5ZSEJYB37JRC5AVCIA5MOP4RHTM335X2KGX3IHOJAPP5RE34K4KZVN" }], + ["asset", { assetCode: "EURC" }], + ])("changes when the %s changes", (_what, change) => { + expect(policyId(policyOf(percentFee(change)))).not.toBe(policyId(policyOf(percentFee()))); + }); + + it("does not depend on the order the fields were built in", () => { + const p = policyOf(percentFee()); + const reordered = { amount: p.amount, max: p.max, min: p.min, percent: p.percent, destination: p.destination, assetIssuer: p.assetIssuer, assetCode: p.assetCode, mode: p.mode }; + expect(policyId(reordered)).toBe(policyId(p)); + }); +}); + +// ─── The row, and the alarm ───────────────────────────────────────────────── + +describe("quoteRow", () => { + it("records the amount priced, the fee, the basis and the policy", () => { + const { data } = quoteRow("mainnet", 100, { fee: percentFee({ amount: "0.5" }), basis: "percent" }); + expect(data).toMatchObject({ + network: "mainnet", + assetCode: "USDC", + assetIssuer: ISSUER, + quotedAgainst: "100", + fee: "0.5", + basis: "percent", + feePercent: "0.5", + alarm: false, + policyId: policyId(policyOf(percentFee())), + }); + }); + + // The whole of the "nothing identifies a person" criterion, as a shape: the + // row has exactly these columns, and none of them can hold an IP, a user + // agent or a sender. The destination is in the policy, and `/fees` + // publishes it to anyone. + it("has no column that could identify a person", () => { + const { data } = quoteRow("mainnet", 100, { fee: percentFee(), basis: "percent" }); + expect(Object.keys(data).sort()).toEqual(QUOTE_COLUMNS); + }); + + it("records no amount and raises no alarm when none was sent", () => { + const { data } = quoteRow("mainnet", undefined, { fee: percentFee({ amount: "0.02" }), basis: "no_amount" }); + expect(data.quotedAgainst).toBeNull(); + expect(data.feePercent).toBeNull(); + expect(data.alarm).toBe(false); + }); + + it("alarms only above the limit, not at it", () => { + const at = quoteRow("mainnet", 1, { fee: flatFee({ amount: "0.1" }), basis: "flat" }, 10); + const over = quoteRow("mainnet", 1, { fee: flatFee({ amount: "0.1000001" }), basis: "flat" }, 10); + expect(at.data.alarm).toBe(false); + expect(over.data.alarm).toBe(true); + }); + + it("catches a floor that dwarfs a tiny send", () => { + // A 0.02 floor on a 0.1 send is 20% of it. + const { data } = quoteRow("mainnet", 0.1, { fee: percentFee({ amount: "0.02" }), basis: "floor" }); + expect(data.feePercent).toBe("20"); + expect(data.alarm).toBe(true); + }); +}); + +describe("alarmPercent", () => { + it("defaults to 10", () => { + expect(alarmPercent()).toBe(DEFAULT_ALARM_PERCENT); + expect(DEFAULT_ALARM_PERCENT).toBe(10); + }); + + it("reads VEIL_FEE_ALARM_PERCENT", () => { + process.env.VEIL_FEE_ALARM_PERCENT = "2.5"; + expect(alarmPercent()).toBe(2.5); + }); + + it.each(["0", "-1", "101", "abc", ""])("falls back to the default for %p", (value) => { + process.env.VEIL_FEE_ALARM_PERCENT = value; + expect(alarmPercent()).toBe(DEFAULT_ALARM_PERCENT); + }); +}); + +// ─── Recording ────────────────────────────────────────────────────────────── + +describe("recordFeeQuotes", () => { + it("writes each quote with its policy, and the policy only once", async () => { + const quoted = [{ fee: percentFee(), basis: "percent" as const }]; + await recordFeeQuotes("mainnet", 100, quoted); + await recordFeeQuotes("mainnet", 200, quoted); + + expect(db.feePolicy.createMany).toHaveBeenCalledTimes(1); + expect(db.feePolicy.createMany.mock.calls[0][0].skipDuplicates).toBe(true); + expect(policiesWritten()).toEqual([ + { id: policyId(policyOf(percentFee())), policy: policyOf(percentFee()) }, + ]); + expect(quotesWritten()).toHaveLength(2); + }); + + it("does nothing when no fee was returned", async () => { + await recordFeeQuotes("mainnet", 100, []); + expect(db.feeQuote.createMany).not.toHaveBeenCalled(); + expect(db.feePolicy.createMany).not.toHaveBeenCalled(); + }); + + it("raises the alarm: counter, log line, and the row's flag", async () => { + await recordFeeQuotes("mainnet", 0.1, [{ fee: percentFee({ amount: "0.02" }), basis: "floor" }]); + + const counted = await feeQuoteAlarmsTotal.get(); + expect(counted.values).toEqual([ + expect.objectContaining({ value: 1, labels: { network: "mainnet", asset: "USDC" } }), + ]); + expect(console.error).toHaveBeenCalledWith(expect.stringMatching(/ALARM mainnet USDC: fee 0\.02 is 20% of 0\.1/)); + expect(quotesWritten()[0].alarm).toBe(true); + }); + + // The person is mid-payment. A ledger outage must be loud, not fatal. + it("never throws when the database fails, and counts the failure", async () => { + db.feeQuote.createMany.mockRejectedValue(new Error("connection refused")); + + await expect( + recordFeeQuotes("mainnet", 100, [{ fee: percentFee(), basis: "percent" }]), + ).resolves.toBeUndefined(); + + const failures = await feeQuoteRecordFailuresTotal.get(); + expect(failures.values).toEqual([expect.objectContaining({ value: 1, labels: { network: "mainnet" } })]); + expect(console.error).toHaveBeenCalledWith(expect.stringMatching(/could not record 1 fee quote/)); + }); + + it("still raises the alarm when the write fails", async () => { + db.feePolicy.createMany.mockRejectedValue(new Error("down")); + await recordFeeQuotes("mainnet", 0.1, [{ fee: percentFee({ amount: "0.02" }), basis: "floor" }]); + expect((await feeQuoteAlarmsTotal.get()).values[0]?.value).toBe(1); + }); + + it("retries a policy write that failed, rather than caching it as known", async () => { + db.feePolicy.createMany.mockRejectedValueOnce(new Error("down")); + const quoted = [{ fee: percentFee(), basis: "percent" as const }]; + await recordFeeQuotes("mainnet", 100, quoted); + await recordFeeQuotes("mainnet", 100, quoted); + expect(db.feePolicy.createMany).toHaveBeenCalledTimes(2); + }); +}); + +// ─── Through the route ────────────────────────────────────────────────────── + +describe("GET /fees records what it quoted", () => { + beforeEach(() => { + process.env.VEIL_FEE_DESTINATION = DESTINATION; + process.env.VEIL_FEE_ASSET_ISSUER = ISSUER; + process.env.VEIL_FEE_PERCENT = "0.5"; + process.env.VEIL_FEE_MIN = "0.02"; + process.env.VEIL_FEE_MAX = "2"; + process.env.VEIL_FEE_AMOUNT_XLM = "0.1"; + }); + + it("records every fee it returned, each with the policy that produced it", async () => { + const res = await request(makeApp()).get("/fees?amount=100").set("x-network", "mainnet"); + await settle(); + + expect(res.status).toBe(200); + const returned = res.body.fees as Record; + const rows = quotesWritten(); + expect(rows.map((r) => r.assetCode).sort()).toEqual(Object.keys(returned).sort()); + + const policies = new Map(policiesWritten().map((p) => [p.id, p.policy])); + for (const row of rows) { + const fee = returned[row.assetCode]; + expect(row.fee).toBe(fee.amount); + expect(row.network).toBe("mainnet"); + expect(row.quotedAgainst).toBe("100"); + // The policy recorded is the one the client was sent, field for field. + expect(policies.get(row.policyId)).toEqual(policyOf(fee)); + } + // VEIL_FEE_PERCENT applies to every asset; XLM's flat amount is its floor. + expect(rows.map((r) => r.basis)).toEqual(["percent", "percent"]); + }); + + it("records a quote with no amount as the floor it told the client", async () => { + await request(makeApp()).get("/fees").set("x-network", "mainnet"); + await settle(); + const usdc = quotesWritten().find((r) => r.assetCode === "USDC"); + expect(usdc).toMatchObject({ basis: "no_amount", quotedAgainst: null, fee: "0.02", alarm: false }); + }); + + it("records nothing when no fee is configured", async () => { + delete process.env.VEIL_FEE_DESTINATION; + const res = await request(makeApp()).get("/fees?amount=100"); + await settle(); + expect(res.body.enabled).toBe(false); + expect(db.feeQuote.createMany).not.toHaveBeenCalled(); + }); + + it("still answers when the ledger is down", async () => { + db.feePolicy.createMany.mockRejectedValue(new Error("down")); + db.feeQuote.createMany.mockRejectedValue(new Error("down")); + const res = await request(makeApp()).get("/fees?amount=100"); + await settle(); + expect(res.status).toBe(200); + expect(res.body.enabled).toBe(true); + }); + + it("makes a policy change visible: a new policy row, and quotes that name it", async () => { + await request(makeApp()).get("/fees?amount=100"); + await settle(); + process.env.VEIL_FEE_PERCENT = "1"; + await request(makeApp()).get("/fees?amount=100"); + await settle(); + + const usdc = quotesWritten().filter((r) => r.assetCode === "USDC"); + expect(usdc).toHaveLength(2); + expect(usdc[0].policyId).not.toBe(usdc[1].policyId); + expect(usdc.map((r) => r.fee)).toEqual(["0.5", "1"]); + + const percents = policiesWritten() + .filter((p) => p.policy.assetCode === "USDC") + .map((p) => p.policy.percent); + expect(percents).toEqual(["0.5", "1"]); + // The percent is shared, so XLM's policy changed too — and is recorded. + expect(policiesWritten().filter((p) => p.policy.assetCode === "XLM")).toHaveLength(2); + }); + + it("writes an unchanged policy once, however many quotes it prices", async () => { + for (const amount of [1, 50, 100, 10_000]) { + await request(makeApp()).get(`/fees?amount=${amount}`); + await settle(); + } + expect(quotesWritten()).toHaveLength(8); + expect(policiesWritten()).toHaveLength(2); + // ...and each basis is the rule that actually decided that fee. + expect(quotesWritten().filter((r) => r.assetCode === "USDC").map((r) => r.basis)).toEqual([ + "floor", + "percent", + "percent", + "cap", + ]); + }); + + it("takes nothing from the request beyond the amount", async () => { + await request(makeApp()) + .get("/fees?amount=100&address=GBUO4RL4RTGRFSUDUMRFMC75EWYCRTX5OE3PBIXJVDCZULOXQ2TKDR4Z") + .set("user-agent", "VeilWallet/1.2 (Pixel 7)") + .set("x-forwarded-for", "203.0.113.7"); + await settle(); + const written = JSON.stringify([quotesWritten(), policiesWritten()]); + expect(written).not.toContain("GBUO4RL4"); + expect(written).not.toContain("Pixel"); + expect(written).not.toContain("203.0.113.7"); + }); +}); + +// ─── Reporting ────────────────────────────────────────────────────────────── + +describe("reportRange", () => { + it("accepts a single day and an inclusive range", () => { + expect(reportRange("2026-10-17", "2026-10-17")).toEqual({ from: "2026-10-17", to: "2026-10-17" }); + expect(reportRange("2026-10-01", "2026-10-17")).toEqual({ from: "2026-10-01", to: "2026-10-17" }); + }); + + it.each([ + ["not a date", "yesterday", "2026-10-17"], + ["an impossible date", "2026-02-30", "2026-03-01"], + ["a reversed range", "2026-10-17", "2026-10-01"], + ["more than 92 days", "2026-01-01", "2026-12-31"], + ])("refuses %s", (_what, from, to) => { + expect(typeof reportRange(from, to)).toBe("string"); + }); +}); + +describe("GET /fees/report", () => { + const auth = (r: request.Test) => r.set("authorization", `Bearer ${TOKEN}`).set("x-network", "mainnet"); + + it("answers 503 when no report token is configured, so it is never open by accident", async () => { + const res = await request(makeApp()).get("/fees/report").set("authorization", "Bearer anything"); + expect(res.status).toBe(503); + expect(db.$queryRaw).not.toHaveBeenCalled(); + }); + + it.each([ + ["no token", undefined], + ["a wrong token", "Bearer nope"], + ["the right token without the scheme", TOKEN], + ])("refuses %s", async (_what, header) => { + process.env.VEIL_FEE_REPORT_TOKEN = TOKEN; + const req = request(makeApp()).get("/fees/report"); + const res = await (header ? req.set("authorization", header) : req); + expect(res.status).toBe(401); + expect(res.headers["www-authenticate"]).toBe("Bearer"); + expect(db.$queryRaw).not.toHaveBeenCalled(); + }); + + it("refuses a bad range with 400", async () => { + process.env.VEIL_FEE_REPORT_TOKEN = TOKEN; + const res = await auth(request(makeApp()).get("/fees/report?from=2026-10-17&to=2026-10-01")); + expect(res.status).toBe(400); + }); + + it("answers 500, not a stack trace, when the ledger cannot be read", async () => { + process.env.VEIL_FEE_REPORT_TOKEN = TOKEN; + db.$queryRaw.mockRejectedValue(new Error("relation does not exist")); + const res = await auth(request(makeApp()).get("/fees/report?from=2026-10-17")); + expect(res.status).toBe(500); + expect(JSON.stringify(res.body)).not.toContain("relation"); + }); + + it("defaults to today, UTC", async () => { + process.env.VEIL_FEE_REPORT_TOKEN = TOKEN; + const today = new Date().toISOString().slice(0, 10); + const res = await auth(request(makeApp()).get("/fees/report")); + expect(res.status).toBe(200); + expect(res.body).toMatchObject({ from: today, to: today, days: [] }); + }); + + it("reports per day and asset, with the policies and what was collected", async () => { + process.env.VEIL_FEE_REPORT_TOKEN = TOKEN; + const usdcPolicy = policyOf(percentFee()); + const xlmPolicy = policyOf(flatFee()); + const usdcId = policyId(usdcPolicy); + const xlmId = policyId(xlmPolicy); + const xlmSac = Asset.native().contractId(Networks.PUBLIC); + // XLM's contract is indexed; USDC's is not. + (resolveSacContractIds as jest.Mock).mockReturnValue([xlmSac]); + + db.$queryRaw + .mockResolvedValueOnce([ + { day: "2026-10-16", assetCode: "USDC", assetIssuer: ISSUER, quotes: 3, feesQuoted: "1.0400000", alarms: 1, policyIds: [usdcId] }, + { day: "2026-10-16", assetCode: "XLM", assetIssuer: null, quotes: 2, feesQuoted: "0.2000000", alarms: 0, policyIds: [xlmId] }, + { day: "2026-10-17", assetCode: "XLM", assetIssuer: null, quotes: 1, feesQuoted: "0.1000000", alarms: 0, policyIds: [xlmId] }, + ]) + .mockResolvedValueOnce([ + { day: "2026-10-16", total: "2000000" }, + { day: "2026-10-15", total: "15000000" }, + ]); + db.feePolicy.findMany.mockResolvedValue([ + { id: usdcId, policy: usdcPolicy, firstSeenAt: new Date("2026-10-01T09:00:00Z") }, + { id: xlmId, policy: xlmPolicy, firstSeenAt: new Date("2026-09-30T12:00:00Z") }, + ]); + + const res = await auth(request(makeApp()).get("/fees/report?from=2026-10-15&to=2026-10-17")); + + expect(res.status).toBe(200); + expect(res.body.network).toBe("mainnet"); + expect(res.body.alarmPercent).toBe(10); + expect(res.body.policies[usdcId]).toEqual({ firstSeenAt: "2026-10-01T09:00:00.000Z", policy: usdcPolicy }); + expect(res.body.days.map((d: { date: string }) => d.date)).toEqual(["2026-10-15", "2026-10-16", "2026-10-17"]); + + const [d15, d16, d17] = res.body.days; + // Received with no quote that day: shown, not dropped. + expect(d15.assets).toEqual([ + expect.objectContaining({ assetCode: "XLM", quotes: 0, feesQuoted: "0", collected: "1.5" }), + ]); + expect(d16.assets).toEqual([ + { + assetCode: "USDC", + assetIssuer: ISSUER, + quotes: 3, + feesQuoted: "1.04", + alarms: 1, + policyIds: [usdcId], + collected: null, + collectedNote: expect.stringMatching(/not indexed/), + }, + { assetCode: "XLM", assetIssuer: null, quotes: 2, feesQuoted: "0.2", alarms: 0, policyIds: [xlmId], collected: "0.2" }, + ]); + // Indexed and quoted, but nothing arrived: zero is a real answer here. + expect(d17.assets[0]).toMatchObject({ assetCode: "XLM", collected: "0" }); + + // Scoped to the network and the range asked for, with the end exclusive. + const [, ...values] = db.$queryRaw.mock.calls[0]; + expect(values).toEqual(["mainnet", new Date("2026-10-15T00:00:00Z"), new Date("2026-10-18T00:00:00Z")]); + // Collection is read only for the destination the policies named. + const collectedValues = db.$queryRaw.mock.calls[1].slice(1); + expect(collectedValues).toEqual( + expect.arrayContaining(["mainnet", xlmSac, [DESTINATION]]), + ); + }); + + it("feeReport reads no policies and no transfers for an empty range", async () => { + const report = await feeReport("mainnet", "2026-10-17", "2026-10-17"); + expect(report.days).toEqual([]); + expect(db.feePolicy.findMany).not.toHaveBeenCalled(); + expect(db.$queryRaw).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/api/feeLedger.ts b/src/api/feeLedger.ts new file mode 100644 index 000000000..9fbcf6c81 --- /dev/null +++ b/src/api/feeLedger.ts @@ -0,0 +1,416 @@ +/** + * The platform fee ledger: what `GET /fees` quoted, under which policy. + * + * ## Why this exists + * + * The fee is collected on-chain, in the same transaction as the user's own + * payment, and the money never passes through wraith. So until this, nothing on + * this side knew it had happened: there was no answer to how much had been + * charged, in which asset, on which day, and nothing tied a payment on the + * treasury account to the request that priced it. And because the fee is now + * policy — percent, floor and cap — rather than a constant, a misconfiguration + * would overcharge every user until somebody read a block explorer. + * + * ## What is recorded, and what deliberately is not + * + * One row per asset per quote: the amount priced against, the asset, the fee + * returned, the rule that decided it, and the policy that was in force. The + * policy is taken from the very entry the client was sent — never by reading + * the environment a second time — so the record cannot disagree with what the + * user was told. + * + * **Nothing recorded identifies a person.** No IP, no user agent, no sender or + * wallet address. The only address anywhere in here is the fee destination, + * which `/fees` publishes to anyone. Do not add a column that changes that: + * the amount and the time of a quote are not personal data on their own, but + * either joined to a requester would be. + * + * ## Failure + * + * Recording never throws and is never awaited by the response. The person is + * in the middle of a payment; a ledger outage must not become a failed send. + * A write that fails is logged and counted in + * `veil_fee_quote_record_failures_total`, so a gap is visible rather than a + * quietly low total. + */ + +import { createHash } from "crypto"; +import { Asset, Networks } from "@stellar/stellar-sdk"; + +import { prisma } from "../db"; +import { toDisplayAmount } from "../amount"; +import { resolveSacContractIds } from "../indexer"; +import { feeQuoteAlarmsTotal, feeQuoteRecordFailuresTotal } from "../metrics"; +import type { Network } from "../network"; +import type { FeeBasis, PlatformFee } from "./fees"; + +// ─── Policy ────────────────────────────────────────────────────────────────── + +/** Everything that decides a fee for one asset, and nothing about who asked. */ +export interface FeePolicyShape { + mode: "flat" | "percent"; + assetCode: string; + assetIssuer: string | null; + destination: string; + /** Percent mode only. */ + percent: string | null; + min: string | null; + max: string | null; + /** Flat mode only: the fixed fee. */ + amount: string | null; +} + +/** + * The policy behind one `/fees` entry. + * + * In percent mode `amount` is the *result* for this quote, not policy, so it is + * left out; otherwise every distinct amount would look like a policy change. + */ +export function policyOf(fee: PlatformFee): FeePolicyShape { + const percentMode = fee.percent !== undefined; + return { + mode: percentMode ? "percent" : "flat", + assetCode: fee.assetCode, + assetIssuer: fee.assetIssuer, + destination: fee.destination, + percent: percentMode ? fee.percent! : null, + min: percentMode ? (fee.min ?? null) : null, + max: percentMode ? (fee.max ?? null) : null, + amount: percentMode ? null : fee.amount, + }; +} + +/** + * A stable id for a policy: the same configuration always hashes the same, + * any change to it does not. Keys are written in a fixed order so the id does + * not depend on how the object happened to be built. + */ +export function policyId(policy: FeePolicyShape): string { + const canonical = JSON.stringify([ + policy.mode, + policy.assetCode, + policy.assetIssuer, + policy.destination, + policy.percent, + policy.min, + policy.max, + policy.amount, + ]); + return createHash("sha256").update(canonical).digest("hex").slice(0, 32); +} + +// ─── Alarm ─────────────────────────────────────────────────────────────────── + +/** + * The default ceiling: a fee above 10% of what it was quoted against. + * + * The fee policy's own rationale calls 10% of a send "stark" — it is the + * number that made the flat fee indefensible. A sane percent is well under a + * tenth of that, so crossing it means either a floor on a very small send or + * a misconfigured policy. Both are worth a human looking. + */ +export const DEFAULT_ALARM_PERCENT = 10; + +/** `VEIL_FEE_ALARM_PERCENT`, or the default when unset or not a sane number. */ +export function alarmPercent(): number { + const raw = Number(process.env.VEIL_FEE_ALARM_PERCENT?.trim() || NaN); + return Number.isFinite(raw) && raw > 0 && raw <= 100 ? raw : DEFAULT_ALARM_PERCENT; +} + +// ─── Recording ─────────────────────────────────────────────────────────────── + +export interface QuotedFee { + fee: PlatformFee; + basis: FeeBasis; +} + +const DP = 7; + +/** A positive decimal, at Stellar's seven places, trailing zeros removed. */ +function decimal(value: number): string { + if (!Number.isFinite(value)) return String(value); + // toFixed switches to exponent notation from 1e21 on; such an amount is not + // sendable anyway, but the record should still say what was asked. + if (Math.abs(value) >= 1e21) return String(value); + return value.toFixed(DP).replace(/0+$/, "").replace(/\.$/, "") || "0"; +} + +/** Strip trailing zeros from a decimal string that Postgres returned. */ +function trimDecimal(value: string): string { + return value.includes(".") ? value.replace(/0+$/, "").replace(/\.$/, "") : value; +} + +/** The row this quote becomes, before anything is written. */ +export function quoteRow( + network: Network, + sent: number | undefined, + { fee, basis }: QuotedFee, + limitPercent = alarmPercent(), +) { + const policy = policyOf(fee); + const charged = Number(fee.amount); + const percentOfAmount = sent !== undefined && sent > 0 ? (charged / sent) * 100 : null; + return { + policy, + data: { + network, + assetCode: fee.assetCode, + assetIssuer: fee.assetIssuer, + quotedAgainst: sent !== undefined ? decimal(sent) : null, + fee: fee.amount, + basis, + feePercent: percentOfAmount !== null ? decimal(percentOfAmount) : null, + alarm: percentOfAmount !== null && percentOfAmount > limitPercent, + policyId: policyId(policy), + }, + }; +} + +/** + * Policies already written by this process, so a steady stream of quotes + * under one configuration does not re-insert the same row every time. + */ +const knownPolicies = new Set(); + +/** Test-only. */ +export function _forgetKnownPolicies(): void { + knownPolicies.clear(); +} + +/** + * Write every fee in one `/fees` answer to the ledger. + * + * Never throws — see the module header. The alarm is raised before the write, + * so a database outage cannot also silence an overcharge. + */ +export async function recordFeeQuotes( + network: Network, + sent: number | undefined, + quoted: QuotedFee[], +): Promise { + if (quoted.length === 0) return; + try { + const limit = alarmPercent(); + const rows = quoted.map((q) => quoteRow(network, sent, q, limit)); + + for (const { data } of rows) { + if (!data.alarm) continue; + feeQuoteAlarmsTotal.inc({ network, asset: data.assetCode }); + // Amount, fee and policy only — none of it identifies anyone. + console.error( + `[fees] ALARM ${network} ${data.assetCode}: fee ${data.fee} is ${data.feePercent}% of ` + + `${data.quotedAgainst}, over the ${limit}% limit (policy ${data.policyId}, basis ${data.basis})`, + ); + } + + const newPolicies = new Map(); + for (const { policy, data } of rows) { + if (!knownPolicies.has(data.policyId)) newPolicies.set(data.policyId, policy); + } + if (newPolicies.size > 0) { + // skipDuplicates: two instances seeing a new policy at once is a race we + // expect, and the first writer's `firstSeenAt` is the true one. + await prisma.feePolicy.createMany({ + data: [...newPolicies].map(([id, policy]) => ({ id, policy: { ...policy } })), + skipDuplicates: true, + }); + for (const id of newPolicies.keys()) knownPolicies.add(id); + } + + await prisma.feeQuote.createMany({ data: rows.map((r) => r.data) }); + } catch (err) { + feeQuoteRecordFailuresTotal.inc({ network }); + console.error( + `[fees] could not record ${quoted.length} fee quote(s): ${err instanceof Error ? err.message : String(err)}`, + ); + } +} + +// ─── Reporting ─────────────────────────────────────────────────────────────── + +interface QuotedDayRow { + day: string; + assetCode: string; + assetIssuer: string | null; + quotes: number; + feesQuoted: string; + alarms: number; + policyIds: string[]; +} + +export interface FeeReportAsset { + assetCode: string; + assetIssuer: string | null; + /** Quotes that returned a fee in this asset. */ + quotes: number; + /** The sum of every fee quoted — what clients were told to charge. */ + feesQuoted: string; + /** Quotes over the alarm limit. */ + alarms: number; + /** The policies that priced them. Look each up in `policies`. */ + policyIds: string[]; + /** + * What the fee destination actually received in this asset that day, + * according to the index — or null when this asset is not indexed, which is + * not the same thing as zero. + */ + collected: string | null; + collectedNote?: string; +} + +export interface FeeReport { + network: Network; + from: string; + to: string; + alarmPercent: number; + days: Array<{ date: string; assets: FeeReportAsset[] }>; + policies: Record; +} + +/** The asset's Stellar Asset Contract id on `network`, or null if it has none. */ +function sacId(network: Network, code: string, issuer: string | null): string | null { + try { + const asset = issuer ? new Asset(code, issuer) : Asset.native(); + return asset.contractId(network === "mainnet" ? Networks.PUBLIC : Networks.TESTNET); + } catch { + return null; + } +} + +const assetKey = (code: string, issuer: string | null) => `${code}:${issuer ?? ""}`; + +/** + * What was charged on each day from `from` to `to` (inclusive, UTC dates), + * per asset, and under which policies. + * + * "Collected" comes from the indexed transfers into the destinations those + * policies named. It counts everything the destination received in that + * asset, so it is a true collected total only while the destination is used + * for fees alone — which is how it should be configured. + */ +export async function feeReport(network: Network, from: string, to: string): Promise { + const start = new Date(`${from}T00:00:00.000Z`); + const end = new Date(`${to}T00:00:00.000Z`); + end.setUTCDate(end.getUTCDate() + 1); + + const quoted = await prisma.$queryRaw` + SELECT to_char(date_trunc('day', "quotedAt"), 'YYYY-MM-DD') AS "day", + "assetCode", + "assetIssuer", + count(*)::int AS "quotes", + COALESCE(sum("fee"), 0)::text AS "feesQuoted", + (count(*) FILTER (WHERE "alarm"))::int AS "alarms", + array_agg(DISTINCT "policyId") AS "policyIds" + FROM "wraith"."FeeQuote" + WHERE "network" = ${network} + AND "quotedAt" >= ${start} + AND "quotedAt" < ${end} + GROUP BY 1, 2, 3 + ORDER BY 1, 2, 3`; + + const ids = [...new Set(quoted.flatMap((r) => r.policyIds))]; + const policyRows = ids.length + ? await prisma.feePolicy.findMany({ where: { id: { in: ids } } }) + : []; + const policies: FeeReport["policies"] = {}; + for (const row of policyRows) { + policies[row.id] = { + firstSeenAt: row.firstSeenAt.toISOString(), + policy: row.policy as unknown as FeePolicyShape, + }; + } + + // Collected, per asset that was quoted in the range. + const indexed = new Set(resolveSacContractIds(network)); + const collected = new Map>(); + const notes = new Map(); + const assets = new Map(); + for (const r of quoted) assets.set(assetKey(r.assetCode, r.assetIssuer), { code: r.assetCode, issuer: r.assetIssuer }); + + for (const [key, { code, issuer }] of assets) { + const contractId = sacId(network, code, issuer); + if (!contractId || !indexed.has(contractId)) { + notes.set(key, "This asset's contract is not indexed by this deployment, so collection cannot be read."); + continue; + } + const destinations = [ + ...new Set( + quoted + .filter((r) => assetKey(r.assetCode, r.assetIssuer) === key) + .flatMap((r) => r.policyIds) + .map((id) => policies[id]?.policy.destination) + .filter((d): d is string => typeof d === "string"), + ), + ]; + if (destinations.length === 0) { + notes.set(key, "The policies behind these quotes could not be read, so their destination is unknown."); + continue; + } + const received = await prisma.$queryRaw>` + SELECT to_char(date_trunc('day', "ledgerClosedAt"), 'YYYY-MM-DD') AS "day", + COALESCE(sum("amount"::numeric), 0)::text AS "total" + FROM "wraith"."TokenTransfer" + WHERE "network" = ${network} + AND "contractId" = ${contractId} + AND "eventType" = 'transfer' + AND "toAddress" = ANY(${destinations}) + AND "ledgerClosedAt" >= ${start} + AND "ledgerClosedAt" < ${end} + GROUP BY 1`; + const byDay = new Map(); + for (const row of received) { + // SAC amounts are integer stroops at seven decimals. + byDay.set(row.day, trimDecimal(toDisplayAmount(row.total.split(".")[0], DP))); + } + collected.set(key, byDay); + } + + // One entry per (day, asset): every quoted pair, plus any day the + // destination received funds in a quoted asset without a quote that day. + const days = new Map>(); + const slot = (day: string, code: string, issuer: string | null): FeeReportAsset => { + if (!days.has(day)) days.set(day, new Map()); + const perDay = days.get(day)!; + const key = assetKey(code, issuer); + if (!perDay.has(key)) { + const note = notes.get(key); + perDay.set(key, { + assetCode: code, + assetIssuer: issuer, + quotes: 0, + feesQuoted: "0", + alarms: 0, + policyIds: [], + collected: collected.has(key) ? (collected.get(key)!.get(day) ?? "0") : null, + ...(note ? { collectedNote: note } : {}), + }); + } + return perDay.get(key)!; + }; + + for (const r of quoted) { + const entry = slot(r.day, r.assetCode, r.assetIssuer); + entry.quotes = r.quotes; + entry.feesQuoted = trimDecimal(r.feesQuoted); + entry.alarms = r.alarms; + entry.policyIds = [...r.policyIds].sort(); + } + for (const [key, byDay] of collected) { + const { code, issuer } = assets.get(key)!; + for (const day of byDay.keys()) slot(day, code, issuer); + } + + return { + network, + from, + to, + alarmPercent: alarmPercent(), + days: [...days.keys()].sort().map((date) => ({ + date, + assets: [...days.get(date)!.values()].sort((a, b) => + assetKey(a.assetCode, a.assetIssuer).localeCompare(assetKey(b.assetCode, b.assetIssuer)), + ), + })), + policies, + }; +} diff --git a/src/api/fees.ts b/src/api/fees.ts index 23de81c1b..631705c20 100644 --- a/src/api/fees.ts +++ b/src/api/fees.ts @@ -1,5 +1,9 @@ +import { createHash, timingSafeEqual } from "crypto"; import { Router, type Request, type Response } from "express"; +import { requestNetwork } from "../middleware/network"; +import { feeReport, recordFeeQuotes } from "./feeLedger"; + /** * What Veil charges, answered by the server so the number is never in an APK. * @@ -74,6 +78,27 @@ export function feeForAmount( return fee.toFixed(STROOP_DP).replace(/0+$/, '').replace(/\.$/, ''); } +/** + * Which rule decided a fee — what the ledger records so a charge can be + * explained later without re-deriving it from a policy that may have changed. + * + * `no_amount` is the floor told to a client that sent no amount; `floor` is + * the floor overriding a percent that came out smaller. + */ +export type FeeBasis = "percent" | "floor" | "cap" | "flat" | "no_amount"; + +export function feeBasis(fee: PlatformFee, sent?: number): FeeBasis { + if (fee.percent === undefined) return "flat"; + if (sent === undefined) return "no_amount"; + const charged = Number(fee.amount); + const uncapped = Number( + feeForAmount(sent, { percent: Number(fee.percent), min: null, max: null }), + ); + if (fee.max !== undefined && charged === Number(fee.max) && uncapped > charged) return "cap"; + if (fee.min !== undefined && charged === Number(fee.min) && uncapped < charged) return "floor"; + return "percent"; +} + function validAmount(value: string | undefined): string | null { const amount = value?.trim(); if (!amount) return null; @@ -162,6 +187,38 @@ export function configuredFees(sentAmount?: number): Record return fees; } +/** The longest range one report may cover, so a request cannot scan the whole ledger. */ +export const MAX_REPORT_DAYS = 92; + +/** `YYYY-MM-DD` that is also a real calendar date. */ +function isUtcDate(value: string): boolean { + if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) return false; + const parsed = new Date(`${value}T00:00:00.000Z`); + return !Number.isNaN(parsed.getTime()) && parsed.toISOString().slice(0, 10) === value; +} + +/** A validated inclusive date range, or the reason it is not one. */ +export function reportRange(from: string, to: string): { from: string; to: string } | string { + if (!isUtcDate(from) || !isUtcDate(to)) return "from and to must be dates in YYYY-MM-DD form (UTC)"; + const days = (Date.parse(to) - Date.parse(from)) / 86_400_000 + 1; + if (days < 1) return "from must not be after to"; + if (days > MAX_REPORT_DAYS) return `A report covers at most ${MAX_REPORT_DAYS} days`; + return { from, to }; +} + +function bearer(req: Request): string { + const match = /^Bearer +(\S+)$/i.exec(String(req.headers.authorization ?? "").trim()); + return match ? match[1] : ""; +} + +/** Constant-time comparison; hashing first makes the lengths equal. */ +function sameSecret(given: string, expected: string): boolean { + if (!given) return false; + const a = createHash("sha256").update(given).digest(); + const b = createHash("sha256").update(expected).digest(); + return timingSafeEqual(a, b); +} + export function feesRouter(): Router { const router = Router(); @@ -182,6 +239,53 @@ export function feesRouter(): Router { // from "this backend is too old to know about fees" and does not read a // deliberate waiver as a failure. res.json(assets.length > 0 ? { enabled: true, fees } : { enabled: false, fees: {} }); + + // Recorded after the answer is sent, and never awaited: the person is in + // the middle of a payment, and a ledger outage must not become a failed + // send. `recordFeeQuotes` cannot throw. See `./feeLedger`. + void recordFeeQuotes( + requestNetwork(req), + sent, + Object.values(fees).map((fee) => ({ fee, basis: feeBasis(fee, sent) })), + ); + }); + + /** + * What was charged, per UTC day and asset, and under which policy. + * + * Behind a bearer token, `VEIL_FEE_REPORT_TOKEN`: nothing in it identifies a + * person, but collected totals are the operator's business volume, not + * something to publish. Without the token configured the route answers 503, + * so a deployment never serves it open by accident. + */ + router.get("/report", async (req: Request, res: Response) => { + res.setHeader("Cache-Control", "no-store"); + const expected = process.env.VEIL_FEE_REPORT_TOKEN?.trim(); + if (!expected) { + res.status(503).json({ error: "Fee reporting is not configured on this deployment" }); + return; + } + if (!sameSecret(bearer(req), expected)) { + res.setHeader("WWW-Authenticate", "Bearer"); + res.status(401).json({ error: "A valid report token is required" }); + return; + } + + const today = new Date().toISOString().slice(0, 10); + const from = req.query["from"] === undefined ? today : String(req.query["from"]); + const to = req.query["to"] === undefined ? from : String(req.query["to"]); + const range = reportRange(from, to); + if (typeof range === "string") { + res.status(400).json({ error: range }); + return; + } + + try { + res.json(await feeReport(requestNetwork(req), range.from, range.to)); + } catch (err) { + console.error(`[fees] report failed: ${err instanceof Error ? err.message : String(err)}`); + res.status(500).json({ error: "Could not read the fee ledger" }); + } }); return router; diff --git a/src/metrics.ts b/src/metrics.ts index 89f44268e..f1a8d76ab 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -185,6 +185,35 @@ export function recordSkippedLedger(network: Network): void { ledgersSkippedTotal.inc({ network }); } +/** + * Fee quotes whose fee exceeded `VEIL_FEE_ALARM_PERCENT` of the amount priced. + * + * The fee is policy — percent, floor and cap — so a misconfiguration + * overcharges every user at once, and until now the only way to notice was + * reading the treasury's history on a block explorer. Alert on any increase. + * Labelled by asset and network only: never an amount, never an address. + */ +export const feeQuoteAlarmsTotal = new Counter({ + name: "veil_fee_quote_alarms_total", + help: "Fee quotes whose fee exceeded VEIL_FEE_ALARM_PERCENT of the amount priced", + labelNames: ["network", "asset"] as const, + registers: [registry], +}); + +/** + * Fee quotes that could not be written to the ledger. + * + * The quote itself still succeeds — the user is mid-payment — so a failed + * write is otherwise silent, and a ledger with gaps reports totals that are + * quietly too low. Alert on any increase. + */ +export const feeQuoteRecordFailuresTotal = new Counter({ + name: "veil_fee_quote_record_failures_total", + help: "Fee quotes that could not be written to the fee ledger", + labelNames: ["network"] as const, + registers: [registry], +}); + /** Serialize the registry in Prometheus text exposition format. */ export function renderMetrics(): Promise { return registry.metrics(); diff --git a/src/openapi/build.ts b/src/openapi/build.ts index 7486c938c..e776f95fd 100644 --- a/src/openapi/build.ts +++ b/src/openapi/build.ts @@ -52,6 +52,8 @@ import { offrampErrorResponseSchema, ngnCustomerStatusQuerySchema, ngnCustomerStatusResponseSchema, + feeReportQuerySchema, + feeReportResponseSchema, } from "./schemas"; const registry = new OpenAPIRegistry(); @@ -537,6 +539,27 @@ registry.registerPath({ }, }); +registry.registerComponent("securitySchemes", "bearerAuth", { type: "http", scheme: "bearer" }); + +registry.registerPath({ + method: "get", + path: "/fees/report", + summary: "Platform fees charged, per UTC day and asset, with the policy behind them", + description: + "Internal endpoint. Not part of the public data API: an operator report over the fee " + + "ledger, behind the VEIL_FEE_REPORT_TOKEN bearer token. Nothing in it identifies a person.", + "x-internal": true, + security: [{ bearerAuth: [] }], + request: { query: feeReportQuerySchema }, + responses: { + 200: { description: "OK", content: { "application/json": { schema: feeReportResponseSchema } } }, + 400: { description: "Bad date range", content: { "application/json": { schema: errorResponseSchema } } }, + 401: { description: "Missing or wrong report token", content: { "application/json": { schema: errorResponseSchema } } }, + 500: { description: "The fee ledger could not be read", content: { "application/json": { schema: errorResponseSchema } } }, + 503: { description: "Fee reporting is not configured on this deployment", content: { "application/json": { schema: errorResponseSchema } } }, + }, +}); + const generator = new OpenApiGeneratorV3(registry.definitions); const document = generator.generateDocument({ openapi: "3.0.3", diff --git a/src/openapi/schemas.ts b/src/openapi/schemas.ts index 6e7bdc178..9dc0d47d4 100644 --- a/src/openapi/schemas.ts +++ b/src/openapi/schemas.ts @@ -755,3 +755,51 @@ export const ngnCustomerStatusResponseSchema = z.object({ status: z.string().optional(), }); + +export const feeReportQuerySchema = z.object({ + from: optionalQueryString("First UTC date to report, YYYY-MM-DD. Defaults to today."), + to: optionalQueryString("Last UTC date to report, inclusive, YYYY-MM-DD. Defaults to `from`. At most 92 days after it."), +}); + +const feePolicySchema = z.object({ + mode: z.enum(["flat", "percent"]), + assetCode: z.string(), + assetIssuer: z.string().nullable(), + destination: z.string().openapi({ description: "The fee destination — the address `/fees` already publishes." }), + percent: z.string().nullable(), + min: z.string().nullable(), + max: z.string().nullable(), + amount: z.string().nullable().openapi({ description: "Flat mode only: the fixed fee." }), +}); + +export const feeReportResponseSchema = z.object({ + network: z.string(), + from: z.string(), + to: z.string(), + alarmPercent: z.number().openapi({ description: "VEIL_FEE_ALARM_PERCENT in force for this report." }), + days: z.array( + z.object({ + date: z.string(), + assets: z.array( + z.object({ + assetCode: z.string(), + assetIssuer: z.string().nullable(), + quotes: z.number().int(), + feesQuoted: z.string().openapi({ description: "Sum of every fee quoted — what clients were told to charge." }), + alarms: z.number().int(), + policyIds: z.array(z.string()), + collected: z.string().nullable().openapi({ + description: + "What the fee destination received in this asset that day, per the index. " + + "Null when the asset is not indexed — which is not zero.", + }), + collectedNote: z.string().optional(), + }), + ), + }), + ), + policies: z.record( + z.string(), + z.object({ firstSeenAt: z.string(), policy: feePolicySchema }), + ), +});