diff --git a/packages/api-sync/source/defaults.test.ts b/packages/api-sync/source/defaults.test.ts new file mode 100644 index 000000000..0e871e67f --- /dev/null +++ b/packages/api-sync/source/defaults.test.ts @@ -0,0 +1,86 @@ +import { describe } from "@mainsail/test-runner"; + +import { defaults } from "./defaults.js"; + +const ENV_KEYS = [ + "MAINSAIL_API_SYNC_ENABLED", + "MAINSAIL_API_SYNC_INTERVAL", + "MAINSAIL_API_SYNC_TOKEN_CACHE_SIZE", + "MAINSAIL_API_SYNC_TOKEN_WHITELIST_SYNC_INTERVAL", + "MAINSAIL_API_SYNC_TOKEN_WHITELIST_REMOTE_URL", + "MAINSAIL_API_SYNC_RESTORE_BLOCKS_BATCH_SIZE", +]; + +// The module reads the environment at import time, so a fresh copy is imported +// with a cache-busting query once the environment has been prepared. +const importFresh = async (): Promise => { + const { defaults: freshDefaults } = await import(`./defaults.js?bust=${Math.random()}`); + return freshDefaults; +}; + +describe<{ + previousEnv: Record; +}>("defaults", ({ it, beforeEach, afterEach, assert }) => { + beforeEach((context) => { + context.previousEnv = {}; + for (const key of ENV_KEYS) { + context.previousEnv[key] = process.env[key]; + delete process.env[key]; + } + }); + + afterEach(({ previousEnv }) => { + for (const key of ENV_KEYS) { + if (previousEnv[key] === undefined) { + delete process.env[key]; + } else { + process.env[key] = previousEnv[key]; + } + } + }); + + it("uses the documented defaults when no environment variables are set", async () => { + const fresh = await importFresh(); + + assert.false(fresh.enabled); + assert.equal(fresh.restore.blocks.batchSize, 1000); + assert.equal(fresh.syncInterval, 8000); + assert.equal(fresh.tokenCacheSize, 256); + assert.equal(fresh.tokenWhitelistRefreshInterval, 60_000); + assert.equal(fresh.tokenWhitelistRemoteUrl, ""); + }); + + it("honors the environment variable overrides", async () => { + process.env.MAINSAIL_API_SYNC_ENABLED = "true"; + process.env.MAINSAIL_API_SYNC_INTERVAL = "5000"; + process.env.MAINSAIL_API_SYNC_TOKEN_CACHE_SIZE = "16"; + process.env.MAINSAIL_API_SYNC_TOKEN_WHITELIST_SYNC_INTERVAL = "30000"; + process.env.MAINSAIL_API_SYNC_TOKEN_WHITELIST_REMOTE_URL = "https://tokens.example.org/whitelist.json"; + process.env.MAINSAIL_API_SYNC_RESTORE_BLOCKS_BATCH_SIZE = "50"; + + const fresh = await importFresh(); + + assert.true(fresh.enabled); + assert.equal(fresh.restore.blocks.batchSize, "50"); + assert.equal(fresh.syncInterval, "5000"); + assert.equal(fresh.tokenCacheSize, "16"); + assert.equal(fresh.tokenWhitelistRefreshInterval, "30000"); + assert.equal(fresh.tokenWhitelistRemoteUrl, "https://tokens.example.org/whitelist.json"); + }); + + it("treats '1' as enabled", async () => { + process.env.MAINSAIL_API_SYNC_ENABLED = "1"; + + const fresh = await importFresh(); + + assert.true(fresh.enabled); + }); + + it("treats any other value as disabled", async () => { + process.env.MAINSAIL_API_SYNC_ENABLED = "yes"; + + const fresh = await importFresh(); + + assert.false(fresh.enabled); + }); +}); diff --git a/packages/api-sync/source/listeners.test.ts b/packages/api-sync/source/listeners.test.ts new file mode 100644 index 000000000..80493e862 --- /dev/null +++ b/packages/api-sync/source/listeners.test.ts @@ -0,0 +1,130 @@ +import { Identifiers as ApiDatabaseIdentifiers } from "@mainsail/api-database"; +import { Events, Identifiers } from "@mainsail/constants"; +import { Application } from "@mainsail/kernel"; +import { describe } from "@mainsail/test-runner"; + +import { Listeners } from "./listeners.js"; + +const makeFakeRepo = (primaryColumn: string, tableName: string) => ({ + clear: () => {}, + delete: () => {}, + metadata: { + primaryColumns: [{ propertyName: primaryColumn }], + tableNameWithoutPrefix: tableName, + }, + upsert: () => {}, +}); + +describe<{ + app: Application; + listeners: Listeners; + events: { listen: any; forget: any }; + apiNodeRepo: ReturnType; + contractRepo: ReturnType; + peerRepo: ReturnType; + pluginRepo: ReturnType; + peerFactoryArgs: unknown[]; +}>("Listeners", ({ it, beforeEach, assert, spy, clock }) => { + beforeEach((context) => { + context.events = { forget: () => {}, listen: () => {} }; + context.apiNodeRepo = makeFakeRepo("url", "api_nodes"); + context.contractRepo = makeFakeRepo("name", "contracts"); + context.peerRepo = makeFakeRepo("ip", "peers"); + context.pluginRepo = makeFakeRepo("name", "plugins"); + + context.app = new Application(); + context.app + .bind(Identifiers.ServiceProvider.Configuration) + .toConstantValue({ getRequired: () => 1000 }) + .whenTagged("plugin", "api-sync"); + context.app.bind(ApiDatabaseIdentifiers.DataSource).toConstantValue({}); + context.app.bind(Identifiers.Services.EventDispatcher.Service).toConstantValue(context.events); + context.app.bind(Identifiers.ApiSync.Logger).toConstantValue({ debug: () => {}, error: () => {} }); + context.app.bind(ApiDatabaseIdentifiers.ApiNodeRepositoryFactory).toConstantValue(() => context.apiNodeRepo); + context.app.bind(ApiDatabaseIdentifiers.ContractRepositoryFactory).toConstantValue(() => context.contractRepo); + context.peerFactoryArgs = []; + context.app.bind(ApiDatabaseIdentifiers.PeerRepositoryFactory).toConstantValue((dataSource: unknown) => { + context.peerFactoryArgs.push(dataSource); + return context.peerRepo; + }); + context.app.bind(ApiDatabaseIdentifiers.PluginRepositoryFactory).toConstantValue(() => context.pluginRepo); + context.app.rebind(Identifiers.ServiceProvider.Repository).toConstantValue({ + get: () => ({ config: () => ({ all: () => ({}) }) }), + }); + + context.listeners = context.app.resolve(Listeners); + }); + + it("register: registers every concrete listener on the event dispatcher", async ({ listeners, events }) => { + const listen = spy(events, "listen"); + + await listeners.register(); + + // ApiNodes (2) + DeployerContracts (1) + Peers (3) + Plugins (1) + listen.calledTimes(7); + listen.calledWith(Events.ApiNodeEvent.Added); + listen.calledWith(Events.DeployerEvent.ContractCreated); + listen.calledWith(Events.PeerEvent.Updated); + listen.calledWith(Events.KernelEvent.ServiceProviderBooted); + }); + + it("bootstrap: boots each registered listener (truncating its table)", async ({ + listeners, + apiNodeRepo, + contractRepo, + peerRepo, + pluginRepo, + }) => { + const clk = clock(); + const clears = [apiNodeRepo, contractRepo, peerRepo, pluginRepo].map((repo) => spy(repo, "clear")); + + await listeners.register(); + await listeners.bootstrap(); + await clk.nextAsync(); + + for (const clear of clears) { + clear.calledOnce(); + } + + await listeners.dispose(); + }); + + it("flush: flushes each listener through the given entity manager", async (context) => { + const { listeners, events, peerRepo } = context; + const captured: Record = {}; + events.listen = (name: string, listener: unknown) => { + captured[name] = listener; + }; + + await listeners.register(); + + // Feed one peer event through the captured Peers listener, then flush through the aggregate. + const peers = captured[Events.PeerEvent.Added] as any; + await peers.handle({ + data: { header: { blockNumber: 1 }, ip: "127.0.0.1", latency: 1, plugins: {}, port: 4002, ports: {} }, + name: Events.PeerEvent.Added, + }); + + const upsert = spy(peerRepo, "upsert"); + const entityManager = { marker: "entity-manager" }; + + await listeners.flush(entityManager as any); + + upsert.calledOnce(); + // The repository is created against the flushed entity manager, not the base data source. + assert.equal(context.peerFactoryArgs, [entityManager]); + }); + + it("dispose: forgets all events and clears the listener list", async ({ listeners, events }) => { + const forget = spy(events, "forget"); + + await listeners.register(); + await listeners.dispose(); + + forget.calledTimes(7); + + // A second dispose is a no-op because the listener list was cleared. + await listeners.dispose(); + forget.calledTimes(7); + }); +}); diff --git a/packages/api-sync/source/listeners/abstract-listener.test.ts b/packages/api-sync/source/listeners/abstract-listener.test.ts new file mode 100644 index 000000000..d3c4f5de6 --- /dev/null +++ b/packages/api-sync/source/listeners/abstract-listener.test.ts @@ -0,0 +1,239 @@ +import { Identifiers as ApiDatabaseIdentifiers } from "@mainsail/api-database"; +import { Identifiers } from "@mainsail/constants"; +import { injectable } from "@mainsail/container"; +import { NotImplemented } from "@mainsail/exceptions"; +import { Application } from "@mainsail/kernel"; +import { describe } from "@mainsail/test-runner"; + +import { AbstractListener, ListenerEvent, ListenerEventMapping } from "./abstract-listener.js"; + +type Item = { key: string; payload?: string }; + +const makeFakeRepo = () => ({ + clear: async () => {}, + delete: async (_ids: string[]) => {}, + metadata: { + primaryColumns: [{ propertyName: "key" }], + tableNameWithoutPrefix: "items", + }, + upsert: async (_entities: object[], _conflictPaths: string[]) => {}, +}); + +@injectable() +class ItemListener extends AbstractListener { + public repo = makeFakeRepo(); + + protected getEventMapping(): ListenerEventMapping { + return { + "item.created": ListenerEvent.OnAdded, + "item.deleted": ListenerEvent.OnRemoved, + }; + } + + protected getEventId(event: Item): string { + return event.key; + } + + protected mapEventToEntity(event: Item): Item { + return { key: event.key, payload: event.payload }; + } + + protected makeEntityRepository(): any { + return this.repo; + } +} + +describe<{ + app: Application; + listener: ItemListener; + events: { listen: any; forget: any }; + dataSource: { transaction: any }; + logger: { debug: any; error: any }; +}>("AbstractListener", ({ it, beforeEach, assert, spy, clock }) => { + beforeEach((context) => { + context.events = { forget: () => {}, listen: () => {} }; + context.dataSource = { + transaction: async (_isolation: string, callback: any) => callback({}), + }; + context.logger = { debug: () => {}, error: () => {} }; + + context.app = new Application(); + context.app + .bind(Identifiers.ServiceProvider.Configuration) + .toConstantValue({ getRequired: () => 250 }) + .whenTagged("plugin", "api-sync"); + context.app.bind(ApiDatabaseIdentifiers.DataSource).toConstantValue(context.dataSource); + context.app.bind(Identifiers.Services.EventDispatcher.Service).toConstantValue(context.events); + context.app.bind(Identifiers.ApiSync.Logger).toConstantValue(context.logger); + + context.listener = context.app.resolve(ItemListener); + }); + + it("register: subscribes to every event of the mapping", async ({ listener, events }) => { + const listen = spy(events, "listen"); + + await listener.register(); + + listen.calledTimes(2); + listen.calledWith("item.created", listener); + listen.calledWith("item.deleted", listener); + }); + + it("handle: rejects events outside of the mapping", async ({ listener }) => { + await assert.rejects(() => listener.handle({ data: { key: "k" }, name: "item.renamed" }), NotImplemented); + }); + + it("flush: is a no-op while nothing was collected", async ({ listener }) => { + const upsert = spy(listener.repo, "upsert"); + const remove = spy(listener.repo, "delete"); + + await listener.flush({} as any); + + upsert.neverCalled(); + remove.neverCalled(); + }); + + it("flush: upserts collected additions once and clears them", async ({ listener }) => { + const upsert = spy(listener.repo, "upsert"); + + await listener.handle({ data: { key: "a", payload: "1" }, name: "item.created" }); + await listener.handle({ data: { key: "b", payload: "2" }, name: "item.created" }); + await listener.flush({} as any); + + upsert.calledOnce(); + assert.equal(upsert.getCallArgs(0), [ + [ + { key: "a", payload: "1" }, + { key: "b", payload: "2" }, + ], + ["key"], + ]); + + await listener.flush({} as any); + upsert.calledOnce(); + }); + + it("flush: a repeated addition for the same id keeps the latest payload", async ({ listener }) => { + const upsert = spy(listener.repo, "upsert"); + + await listener.handle({ data: { key: "a", payload: "old" }, name: "item.created" }); + await listener.handle({ data: { key: "a", payload: "new" }, name: "item.created" }); + await listener.flush({} as any); + + assert.equal(upsert.getCallArgs(0)[0], [{ key: "a", payload: "new" }]); + }); + + it("flush: deletes collected removals once and clears them", async ({ listener }) => { + const remove = spy(listener.repo, "delete"); + + await listener.handle({ data: { key: "a" }, name: "item.deleted" }); + await listener.flush({} as any); + + remove.calledOnce(); + assert.equal(remove.getCallArgs(0), [["a"]]); + + await listener.flush({} as any); + remove.calledOnce(); + }); + + it("an addition followed by a removal only removes", async ({ listener }) => { + const upsert = spy(listener.repo, "upsert"); + const remove = spy(listener.repo, "delete"); + + await listener.handle({ data: { key: "a" }, name: "item.created" }); + await listener.handle({ data: { key: "a" }, name: "item.deleted" }); + await listener.flush({} as any); + + upsert.neverCalled(); + remove.calledOnce(); + }); + + it("a removal followed by an addition only adds", async ({ listener }) => { + const upsert = spy(listener.repo, "upsert"); + const remove = spy(listener.repo, "delete"); + + await listener.handle({ data: { key: "a" }, name: "item.deleted" }); + await listener.handle({ data: { key: "a" }, name: "item.created" }); + await listener.flush({} as any); + + remove.neverCalled(); + upsert.calledOnce(); + }); + + it("boot: truncates the table and periodically flushes through a transaction", async ({ listener, dataSource }) => { + const clk = clock(); + const truncate = spy(listener.repo, "clear"); + const transaction = spy(dataSource, "transaction"); + const upsert = spy(listener.repo, "upsert"); + + await listener.handle({ data: { key: "a" }, name: "item.created" }); + await listener.boot(); + await clk.nextAsync(); + + truncate.calledOnce(); + transaction.calledOnce(); + transaction.calledWith("REPEATABLE READ"); + upsert.calledOnce(); + + await listener.dispose(); + }); + + it("boot: the sync loop skips the transaction while nothing was collected", async ({ listener, dataSource }) => { + const clk = clock(); + const transaction = spy(dataSource, "transaction"); + + await listener.boot(); + await clk.nextAsync(); + + transaction.neverCalled(); + + await listener.dispose(); + }); + + it("boot: a failing sync transaction is logged and does not stop the loop", async ({ + listener, + dataSource, + logger, + }) => { + const clk = clock(); + const error = spy(logger, "error"); + + dataSource.transaction = async () => { + throw new Error("connection lost"); + }; + + await listener.handle({ data: { key: "a" }, name: "item.created" }); + await listener.boot(); + // The loop runs once immediately and once through the rescheduled timer. + await clk.nextAsync(); + + error.calledTimes(2); + assert.true(error.getCallArgs(0)[0].includes("connection lost")); + + // The loop rescheduled itself despite the failures. + await clk.nextAsync(); + error.calledTimes(3); + + await listener.dispose(); + }); + + it("dispose: unsubscribes from the mapping and stops the sync loop", async ({ listener, events }) => { + const clk = clock(); + const forget = spy(events, "forget"); + + await listener.boot(); + await clk.nextAsync(); + + // The loop keeps exactly one pending reschedule while running. + assert.equal(clk.countTimers(), 1); + + await listener.dispose(); + + forget.calledTimes(2); + forget.calledWith("item.created", listener); + forget.calledWith("item.deleted", listener); + + // The pending reschedule was cleared; nothing runs after dispose. + assert.equal(clk.countTimers(), 0); + }); +}); diff --git a/packages/api-sync/source/listeners/api-nodes.test.ts b/packages/api-sync/source/listeners/api-nodes.test.ts new file mode 100644 index 000000000..96c3dbfa5 --- /dev/null +++ b/packages/api-sync/source/listeners/api-nodes.test.ts @@ -0,0 +1,77 @@ +import { Identifiers as ApiDatabaseIdentifiers } from "@mainsail/api-database"; +import { Events, Identifiers } from "@mainsail/constants"; +import { Application } from "@mainsail/kernel"; +import { describe } from "@mainsail/test-runner"; + +import { ApiNodes } from "./api-nodes.js"; + +const makeFakeRepo = () => ({ + clear: () => {}, + delete: () => {}, + metadata: { + primaryColumns: [{ propertyName: "url" }], + tableNameWithoutPrefix: "api_nodes", + }, + upsert: () => {}, +}); + +describe<{ + app: Application; + listener: ApiNodes; + events: { listen: any; forget: any }; + repo: ReturnType; + repositoryFactory: any; +}>("ApiNodes", ({ it, beforeEach, assert, spy }) => { + beforeEach((context) => { + context.events = { forget: () => {}, listen: () => {} }; + context.repo = makeFakeRepo(); + context.repositoryFactory = (dataSource: unknown) => context.repo; + + context.app = new Application(); + context.app + .bind(Identifiers.ServiceProvider.Configuration) + .toConstantValue({ getRequired: () => 1000 }) + .whenTagged("plugin", "api-sync"); + context.app.bind(ApiDatabaseIdentifiers.DataSource).toConstantValue({}); + context.app.bind(Identifiers.Services.EventDispatcher.Service).toConstantValue(context.events); + context.app.bind(Identifiers.ApiSync.Logger).toConstantValue({ debug: () => {}, error: () => {} }); + context.app.bind(ApiDatabaseIdentifiers.ApiNodeRepositoryFactory).toConstantValue(context.repositoryFactory); + + context.listener = context.app.resolve(ApiNodes); + }); + + const apiNode = { height: 42, latency: 5, url: "http://1.1.1.1:4003", version: "1.0.0" } as any; + + it("register: listens on the added and removed events", async ({ listener, events }) => { + const listen = spy(events, "listen"); + + await listener.register(); + + listen.calledWith(Events.ApiNodeEvent.Added, listener); + listen.calledWith(Events.ApiNodeEvent.Removed, listener); + listen.calledTimes(2); + }); + + it("added event: flush upserts the entity keyed by url", async ({ listener, repo }) => { + const upsert = spy(repo, "upsert"); + + await listener.handle({ data: apiNode, name: Events.ApiNodeEvent.Added }); + await listener.flush({} as any); + + upsert.calledOnce(); + assert.equal(upsert.getCallArgs(0), [ + [{ height: 42, latency: 5, url: "http://1.1.1.1:4003", version: "1.0.0" }], + ["url"], + ]); + }); + + it("removed event: flush deletes by url", async ({ listener, repo }) => { + const del = spy(repo, "delete"); + + await listener.handle({ data: apiNode, name: Events.ApiNodeEvent.Removed }); + await listener.flush({} as any); + + del.calledOnce(); + assert.equal(del.getCallArgs(0), [["http://1.1.1.1:4003"]]); + }); +}); diff --git a/packages/api-sync/source/listeners/contracts.test.ts b/packages/api-sync/source/listeners/contracts.test.ts new file mode 100644 index 000000000..18d12dd0f --- /dev/null +++ b/packages/api-sync/source/listeners/contracts.test.ts @@ -0,0 +1,94 @@ +import { Identifiers as ApiDatabaseIdentifiers } from "@mainsail/api-database"; +import { Events, Identifiers } from "@mainsail/constants"; +import { Application } from "@mainsail/kernel"; +import { describe } from "@mainsail/test-runner"; + +import { DeployerContracts } from "./contracts.js"; + +const makeFakeRepo = () => ({ + clear: () => {}, + delete: () => {}, + metadata: { + primaryColumns: [{ propertyName: "name" }], + tableNameWithoutPrefix: "contracts", + }, + upsert: () => {}, +}); + +describe<{ + app: Application; + listener: DeployerContracts; + events: { listen: any; forget: any }; + repo: ReturnType; +}>("DeployerContracts", ({ it, beforeEach, assert, spy }) => { + beforeEach((context) => { + context.events = { forget: () => {}, listen: () => {} }; + context.repo = makeFakeRepo(); + + context.app = new Application(); + context.app + .bind(Identifiers.ServiceProvider.Configuration) + .toConstantValue({ getRequired: () => 1000 }) + .whenTagged("plugin", "api-sync"); + context.app.bind(ApiDatabaseIdentifiers.DataSource).toConstantValue({}); + context.app.bind(Identifiers.Services.EventDispatcher.Service).toConstantValue(context.events); + context.app.bind(Identifiers.ApiSync.Logger).toConstantValue({ debug: () => {}, error: () => {} }); + context.app.bind(ApiDatabaseIdentifiers.ContractRepositoryFactory).toConstantValue(() => context.repo); + + context.listener = context.app.resolve(DeployerContracts); + }); + + const contractEvent = { + activeImplementation: "0ximpl", + address: "0xproxy", + implementations: [{ abi: {}, address: "0ximpl" }], + name: "ConsensusV1", + proxy: "UUPS", + } as any; + + it("register: listens on the contract created event", async ({ listener, events }) => { + const listen = spy(events, "listen"); + + await listener.register(); + + listen.calledWith(Events.DeployerEvent.ContractCreated, listener); + listen.calledTimes(1); + }); + + it("created event: flush upserts the entity keyed by name", async ({ listener, repo }) => { + const upsert = spy(repo, "upsert"); + + await listener.handle({ data: contractEvent, name: Events.DeployerEvent.ContractCreated }); + await listener.flush({} as any); + + upsert.calledOnce(); + assert.equal(upsert.getCallArgs(0), [ + [ + { + activeImplementation: "0ximpl", + address: "0xproxy", + implementations: contractEvent.implementations, + name: "ConsensusV1", + proxy: "UUPS", + }, + ], + ["name"], + ]); + }); + + it("created event: falls back to the contract address when no active implementation is set", async ({ + listener, + repo, + }) => { + const upsert = spy(repo, "upsert"); + + await listener.handle({ + data: { ...contractEvent, activeImplementation: undefined }, + name: Events.DeployerEvent.ContractCreated, + }); + await listener.flush({} as any); + + upsert.calledOnce(); + assert.equal((upsert.getCallArgs(0)[0] as any[])[0].activeImplementation, "0xproxy"); + }); +}); diff --git a/packages/api-sync/source/listeners/peers.test.ts b/packages/api-sync/source/listeners/peers.test.ts new file mode 100644 index 000000000..99ac85a96 --- /dev/null +++ b/packages/api-sync/source/listeners/peers.test.ts @@ -0,0 +1,109 @@ +import { Identifiers as ApiDatabaseIdentifiers } from "@mainsail/api-database"; +import { Events, Identifiers } from "@mainsail/constants"; +import { Application } from "@mainsail/kernel"; +import { describe } from "@mainsail/test-runner"; + +import { Peers } from "./peers.js"; + +const makeFakeRepo = () => ({ + clear: () => {}, + delete: () => {}, + metadata: { + primaryColumns: [{ propertyName: "ip" }], + tableNameWithoutPrefix: "peers", + }, + upsert: () => {}, +}); + +describe<{ + app: Application; + listener: Peers; + events: { listen: any; forget: any }; + repo: ReturnType; +}>("Peers", ({ it, beforeEach, assert, spy }) => { + beforeEach((context) => { + context.events = { forget: () => {}, listen: () => {} }; + context.repo = makeFakeRepo(); + + context.app = new Application(); + context.app + .bind(Identifiers.ServiceProvider.Configuration) + .toConstantValue({ getRequired: () => 1000 }) + .whenTagged("plugin", "api-sync"); + context.app.bind(ApiDatabaseIdentifiers.DataSource).toConstantValue({}); + context.app.bind(Identifiers.Services.EventDispatcher.Service).toConstantValue(context.events); + context.app.bind(Identifiers.ApiSync.Logger).toConstantValue({ debug: () => {}, error: () => {} }); + context.app.bind(ApiDatabaseIdentifiers.PeerRepositoryFactory).toConstantValue(() => context.repo); + + context.listener = context.app.resolve(Peers); + }); + + const peer = { + header: { blockNumber: 7 }, + ip: "127.0.0.1", + latency: 12, + plugins: { "api-http": { enabled: true } }, + port: 4002, + ports: { "api-http": 4003 }, + version: "1.2.3", + } as any; + + it("register: listens on the added, removed and updated events", async ({ listener, events }) => { + const listen = spy(events, "listen"); + + await listener.register(); + + listen.calledWith(Events.PeerEvent.Added, listener); + listen.calledWith(Events.PeerEvent.Removed, listener); + listen.calledWith(Events.PeerEvent.Updated, listener); + listen.calledTimes(3); + }); + + it("added event: flush upserts the mapped peer entity keyed by ip", async ({ listener, repo }) => { + const upsert = spy(repo, "upsert"); + + await listener.handle({ data: peer, name: Events.PeerEvent.Added }); + await listener.flush({} as any); + + upsert.calledOnce(); + assert.equal(upsert.getCallArgs(0), [ + [ + { + blockNumber: 7, + ip: "127.0.0.1", + latency: 12, + plugins: peer.plugins, + port: 4002, + ports: peer.ports, + version: "1.2.3", + }, + ], + ["ip"], + ]); + }); + + it("updated event: is treated as an upsert", async ({ listener, repo }) => { + const upsert = spy(repo, "upsert"); + const del = spy(repo, "delete"); + + await listener.handle({ data: peer, name: Events.PeerEvent.Updated }); + await listener.flush({} as any); + + del.neverCalled(); + upsert.calledOnce(); + }); + + it("removed event: flush deletes by ip", async ({ listener, repo }) => { + const del = spy(repo, "delete"); + + await listener.handle({ data: peer, name: Events.PeerEvent.Removed }); + await listener.flush({} as any); + + del.calledOnce(); + assert.equal(del.getCallArgs(0), [["127.0.0.1"]]); + }); + + it("rejects a peer without a string ip", async ({ listener }) => { + await assert.rejects(() => listener.handle({ data: { ...peer, ip: undefined }, name: Events.PeerEvent.Added })); + }); +}); diff --git a/packages/api-sync/source/listeners/plugins.test.ts b/packages/api-sync/source/listeners/plugins.test.ts new file mode 100644 index 000000000..9cf45656a --- /dev/null +++ b/packages/api-sync/source/listeners/plugins.test.ts @@ -0,0 +1,111 @@ +import { Identifiers as ApiDatabaseIdentifiers } from "@mainsail/api-database"; +import { Events, Identifiers } from "@mainsail/constants"; +import { Application } from "@mainsail/kernel"; +import { describe } from "@mainsail/test-runner"; + +import { Plugins } from "./plugins.js"; + +const makeFakeRepo = () => ({ + clear: () => {}, + delete: () => {}, + metadata: { + primaryColumns: [{ propertyName: "name" }], + tableNameWithoutPrefix: "plugins", + }, + upsert: () => {}, +}); + +describe<{ + app: Application; + listener: Plugins; + events: { listen: any; forget: any }; + repo: ReturnType; + pluginConfig: Record; +}>("Plugins", ({ it, beforeEach, assert, spy }) => { + beforeEach((context) => { + context.events = { forget: () => {}, listen: () => {} }; + context.repo = makeFakeRepo(); + context.pluginConfig = {}; + + context.app = new Application(); + context.app + .bind(Identifiers.ServiceProvider.Configuration) + .toConstantValue({ getRequired: () => 1000 }) + .whenTagged("plugin", "api-sync"); + context.app.bind(ApiDatabaseIdentifiers.DataSource).toConstantValue({}); + context.app.bind(Identifiers.Services.EventDispatcher.Service).toConstantValue(context.events); + context.app.bind(Identifiers.ApiSync.Logger).toConstantValue({ debug: () => {}, error: () => {} }); + context.app.bind(ApiDatabaseIdentifiers.PluginRepositoryFactory).toConstantValue(() => context.repo); + context.app.rebind(Identifiers.ServiceProvider.Repository).toConstantValue({ + get: () => ({ config: () => ({ all: () => context.pluginConfig }) }), + }); + + context.listener = context.app.resolve(Plugins); + }); + + it("register: listens on the service provider booted event", async ({ listener, events }) => { + const listen = spy(events, "listen"); + + await listener.register(); + + listen.calledWith(Events.KernelEvent.ServiceProviderBooted, listener); + listen.calledTimes(1); + }); + + it("booted event: flush upserts the plugin with its configuration", async (context) => { + const { listener, repo } = context; + context.pluginConfig = { syncInterval: 8000 }; + const upsert = spy(repo, "upsert"); + + await listener.handle({ data: { name: "api-sync" }, name: Events.KernelEvent.ServiceProviderBooted }); + await listener.flush({} as any); + + upsert.calledOnce(); + assert.equal(upsert.getCallArgs(0), [[{ configuration: { syncInterval: 8000 }, name: "api-sync" }], ["name"]]); + }); + + it("masks password values, including nested ones", async (context) => { + const { listener, repo } = context; + context.pluginConfig = { + database: { host: "localhost", password: "super-secret" }, + password: "top-level-secret", + username: "postgres", + }; + const upsert = spy(repo, "upsert"); + + await listener.handle({ data: { name: "api-database" }, name: Events.KernelEvent.ServiceProviderBooted }); + await listener.flush({} as any); + + const [[entity]] = upsert.getCallArgs(0) as [any[]]; + assert.equal(entity.configuration, { + database: { host: "localhost", password: "-" }, + password: "-", + username: "postgres", + }); + }); + + it("masks password keys regardless of casing", async (context) => { + const { listener, repo } = context; + context.pluginConfig = { PASSWORD: "secret", Password: "secret" }; + const upsert = spy(repo, "upsert"); + + await listener.handle({ data: { name: "p2p" }, name: Events.KernelEvent.ServiceProviderBooted }); + await listener.flush({} as any); + + const [[entity]] = upsert.getCallArgs(0) as [any[]]; + assert.equal(entity.configuration, { PASSWORD: "-", Password: "-" }); + }); + + it("keeps null and array values intact while sanitizing", async (context) => { + const { listener, repo } = context; + // `typeof null === "object"` sends both into the recursive sanitizer. + context.pluginConfig = { database: null, servers: [{ password: "secret" }, "plain"] }; + const upsert = spy(repo, "upsert"); + + await listener.handle({ data: { name: "api-http" }, name: Events.KernelEvent.ServiceProviderBooted }); + await listener.flush({} as any); + + const [[entity]] = upsert.getCallArgs(0) as [any[]]; + assert.equal(entity.configuration, { database: null, servers: [{ password: "-" }, "plain"] }); + }); +}); diff --git a/packages/api-sync/source/logger.test.ts b/packages/api-sync/source/logger.test.ts new file mode 100644 index 000000000..079fc2d79 --- /dev/null +++ b/packages/api-sync/source/logger.test.ts @@ -0,0 +1,73 @@ +import { Identifiers } from "@mainsail/constants"; +import { Application } from "@mainsail/kernel"; +import { describe } from "@mainsail/test-runner"; + +import { Logger } from "./logger.js"; + +describe<{ + app: Application; + logger: Logger; + kernelLogger: Record void>; + previousLogExtra: string | undefined; +}>("Logger", ({ it, beforeEach, afterEach, spy }) => { + beforeEach((context) => { + context.previousLogExtra = process.env.MAINSAIL_API_SYNC_LOG_EXTRA; + delete process.env.MAINSAIL_API_SYNC_LOG_EXTRA; + + context.kernelLogger = { + alert: () => {}, + debug: () => {}, + error: () => {}, + info: () => {}, + notice: () => {}, + warn: () => {}, + }; + + context.app = new Application(); + context.app.bind(Identifiers.Services.Log.Service).toConstantValue(context.kernelLogger); + + context.logger = context.app.resolve(Logger); + }); + + afterEach(({ previousLogExtra }) => { + if (previousLogExtra === undefined) { + delete process.env.MAINSAIL_API_SYNC_LOG_EXTRA; + } else { + process.env.MAINSAIL_API_SYNC_LOG_EXTRA = previousLogExtra; + } + }); + + for (const method of ["alert", "error", "warn", "notice", "info", "debug"] as const) { + it(`${method}: delegates to the kernel logger`, ({ logger, kernelLogger }) => { + const delegate = spy(kernelLogger, method); + + logger[method]("message", { tag: "context" }); + + delegate.calledOnce(); + delegate.calledWith("message", { tag: "context" }); + }); + } + + for (const [method, delegated] of [ + ["warnExtra", "warn"], + ["debugExtra", "debug"], + ] as const) { + it(`${method}: is silent by default`, ({ logger, kernelLogger }) => { + const delegate = spy(kernelLogger, delegated); + + logger[method]("message"); + + delegate.neverCalled(); + }); + + it(`${method}: delegates when extra logging is enabled`, ({ logger, kernelLogger }) => { + process.env.MAINSAIL_API_SYNC_LOG_EXTRA = "true"; + const delegate = spy(kernelLogger, delegated); + + logger[method]("message"); + + delegate.calledOnce(); + delegate.calledWith("message", undefined); + }); + } +}); diff --git a/packages/api-sync/source/parsers/multi-payment.test.ts b/packages/api-sync/source/parsers/multi-payment.test.ts new file mode 100644 index 000000000..f0dad0c12 --- /dev/null +++ b/packages/api-sync/source/parsers/multi-payment.test.ts @@ -0,0 +1,70 @@ +import { describe } from "@mainsail/test-runner"; +import { encodeAbiParameters, encodeEventTopics, parseAbi } from "viem"; + +import { parseMultiPayments } from "./multi-payment.js"; + +const paymentAbi = parseAbi(["event Payment(address indexed recipient, uint256 amount, bool success)"] as const); + +const MULTIPAYMENT_CONTRACT = "0x1000000000000000000000000000000000000000"; +const RECIPIENT_ONE = "0x1111111111111111111111111111111111111111"; +const RECIPIENT_TWO = "0x2222222222222222222222222222222222222222"; + +const paymentLog = (recipient: string, amount: bigint, success: boolean) => ({ + address: MULTIPAYMENT_CONTRACT, + data: encodeAbiParameters( + [ + { name: "amount", type: "uint256" }, + { name: "success", type: "bool" }, + ], + [amount, success], + ), + topics: encodeEventTopics({ abi: paymentAbi, args: { recipient }, eventName: "Payment" }), +}); + +const otherLog = () => ({ + address: MULTIPAYMENT_CONTRACT, + data: "0x" as const, + topics: ["0xdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef"] as [`0x${string}`], +}); + +const transaction = (to: string | undefined): any => ({ hash: "0xtransaction", to }); + +describe("parseMultiPayments", ({ assert, it }) => { + it("ignores transactions that do not target the multi payment contract", () => { + const result = parseMultiPayments(MULTIPAYMENT_CONTRACT, transaction(RECIPIENT_ONE), { + logs: [paymentLog(RECIPIENT_ONE, 10n, true)], + } as any); + + assert.equal(result, []); + }); + + it("ignores contract deployments (transaction without recipient)", () => { + const result = parseMultiPayments(MULTIPAYMENT_CONTRACT, transaction(undefined), { + logs: [paymentLog(RECIPIENT_ONE, 10n, true)], + } as any); + + assert.equal(result, []); + }); + + it("returns nothing without logs", () => { + assert.equal( + parseMultiPayments(MULTIPAYMENT_CONTRACT, transaction(MULTIPAYMENT_CONTRACT), { logs: [] } as any), + [], + ); + }); + + it("tolerates a receipt without a log field", () => { + assert.equal(parseMultiPayments(MULTIPAYMENT_CONTRACT, transaction(MULTIPAYMENT_CONTRACT), {} as any), []); + }); + + it("maps every Payment event to a model with a running log index", () => { + const result = parseMultiPayments(MULTIPAYMENT_CONTRACT, transaction(MULTIPAYMENT_CONTRACT), { + logs: [paymentLog(RECIPIENT_ONE, 10n, true), otherLog(), paymentLog(RECIPIENT_TWO, 20n, false)], + } as any); + + assert.equal(result, [ + { amount: "10", hash: "0xtransaction", logIndex: 0, success: true, to: RECIPIENT_ONE }, + { amount: "20", hash: "0xtransaction", logIndex: 1, success: false, to: RECIPIENT_TWO }, + ]); + }); +}); diff --git a/packages/api-sync/source/parsers/multi-payment.ts b/packages/api-sync/source/parsers/multi-payment.ts index df07bf759..e3a6a581f 100644 --- a/packages/api-sync/source/parsers/multi-payment.ts +++ b/packages/api-sync/source/parsers/multi-payment.ts @@ -17,7 +17,7 @@ export function parseMultiPayments( const payments = parseEventLogs({ abi: paymentAbi, eventName: "Payment", - logs: receipt.logs, + logs: receipt.logs ?? [], }); return payments.map((payment, logIndex) => { diff --git a/packages/api-sync/source/parsers/tokens.test.ts b/packages/api-sync/source/parsers/tokens.test.ts new file mode 100644 index 000000000..e8dd0e0e5 --- /dev/null +++ b/packages/api-sync/source/parsers/tokens.test.ts @@ -0,0 +1,332 @@ +import { Identifiers as ApiDatabaseIdentifiers } from "@mainsail/api-database"; +import { Identifiers } from "@mainsail/constants"; +import { Application } from "@mainsail/kernel"; +import { describe } from "@mainsail/test-runner"; +import { encodeAbiParameters, encodeEventTopics, encodeFunctionData, encodeFunctionResult, parseAbi } from "viem"; + +import { TokenParserService } from "./tokens.js"; + +const erc20Abi = parseAbi([ + "function totalSupply() view returns (uint256)", + "function balanceOf(address account) view returns (uint256)", + "function name() view returns (string)", + "function symbol() view returns (string)", + "function decimals() view returns (uint8)", + "event Transfer(address indexed from, address indexed to, uint256 value)", + "event Approval(address indexed owner, address indexed spender, uint256 value)", +] as const); + +const TOKEN = "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const OTHER_TOKEN = "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +const HOLDER_ONE = "0x1111111111111111111111111111111111111111"; +const HOLDER_TWO = "0x2222222222222222222222222222222222222222"; + +const transferLog = (token: string, from: string, to: string, value: bigint) => ({ + address: token, + data: encodeAbiParameters([{ type: "uint256" }], [value]), + topics: encodeEventTopics({ abi: erc20Abi, args: { from, to }, eventName: "Transfer" }), +}); + +const approvalLog = (token: string, owner: string, spender: string, value: bigint) => ({ + address: token, + data: encodeAbiParameters([{ type: "uint256" }], [value]), + topics: encodeEventTopics({ abi: erc20Abi, args: { owner, spender }, eventName: "Approval" }), +}); + +const calldataOf = (functionName: "totalSupply" | "name" | "symbol" | "decimals"): string => + encodeFunctionData({ abi: erc20Abi, functionName }).slice(2); + +const balanceOfCalldata = (account: `0x${string}`): string => + encodeFunctionData({ abi: erc20Abi, args: [account], functionName: "balanceOf" }).slice(2); + +const asBuffer = (hex: `0x${string}`): Buffer => Buffer.from(hex.slice(2), "hex"); + +const ok = (hex: `0x${string}`) => ({ output: asBuffer(hex), success: true }); + +// evm.view responses for a well-behaved ERC20 contract, keyed by calldata. +const erc20Responses = (): Map => + new Map([ + [ + calldataOf("totalSupply"), + ok(encodeFunctionResult({ abi: erc20Abi, functionName: "totalSupply", result: 5000n })), + ], + [calldataOf("name"), ok(encodeFunctionResult({ abi: erc20Abi, functionName: "name", result: "Test Token" }))], + [calldataOf("symbol"), ok(encodeFunctionResult({ abi: erc20Abi, functionName: "symbol", result: "TEST" }))], + [calldataOf("decimals"), ok(encodeFunctionResult({ abi: erc20Abi, functionName: "decimals", result: 8 }))], + ]); + +type ViewResponse = { success: boolean; output: Buffer }; + +type Ctx = { + app: Application; + service: TokenParserService; + evm: { view: (options: { data: Buffer }) => Promise }; + logger: any; + databaseToken: unknown; + respond: (impl: (calldata: string) => ViewResponse) => void; +}; + +describe("TokenParserService", ({ it, beforeEach, assert, spy }) => { + beforeEach((context) => { + context.databaseToken = undefined; + + const queryBuilder = { + getOne: async () => context.databaseToken, + where: () => queryBuilder, + }; + + context.evm = { view: async () => ({ output: Buffer.alloc(0), success: false }) }; + context.respond = (impl) => { + context.evm.view = async ({ data }) => impl(data.toString("hex")); + }; + context.logger = { debug: () => {}, debugExtra: () => {}, info: () => {}, warn: () => {} }; + + context.app = new Application(); + context.app.bind(Identifiers.Evm.Instance).toConstantValue(context.evm).whenTagged("instance", "evm"); + context.app.bind(Identifiers.ApiSync.Logger).toConstantValue(context.logger); + context.app + .bind(Identifiers.Cryptography.Configuration) + .toConstantValue({ getMilestone: () => ({ evmSpec: "shanghai" }) }); + context.app + .bind(ApiDatabaseIdentifiers.TokenRepositoryFactory) + .toConstantValue(() => ({ createQueryBuilder: () => queryBuilder })); + context.app + .bind(Identifiers.ServiceProvider.Configuration) + .toConstantValue({ getRequired: () => 32 }) + .whenTagged("plugin", "api-sync"); + + context.service = context.app.resolve(TokenParserService); + }); + + const header = (number: number): any => ({ number }); + const transaction = (to: string | undefined = HOLDER_ONE): any => ({ hash: "0xtransaction", to }); + + // Responses for a valid token plus per-holder balances; every probe succeeds. + const respondAsErc20 = (context: Ctx, balances: Record = {}) => { + const known = erc20Responses(); + context.respond((calldata) => { + if (known.has(calldata)) { + return known.get(calldata)!; + } + + for (const [account, balance] of Object.entries(balances)) { + if (calldata === balanceOfCalldata(account as `0x${string}`)) { + return ok(encodeFunctionResult({ abi: erc20Abi, functionName: "balanceOf", result: balance })); + } + } + + return ok("0x01"); + }); + }; + + it("collects a token action per Transfer event", async ({ service }) => { + const { tokenActions, tokenHolders, tokens } = await service.parseReceipt(header(5), transaction(), { + logs: [transferLog(TOKEN, HOLDER_ONE, HOLDER_TWO, 123n)], + } as any); + + assert.equal(tokenActions, [ + { + action: "Transfer", + address: TOKEN, + blockNumber: "5", + from: HOLDER_ONE, + index: 0, + to: HOLDER_TWO, + transactionHash: "0xtransaction", + value: "123", + }, + ]); + // The contract does not respond to the ERC20 probing, so no token metadata is stored. + assert.equal(tokens, []); + assert.equal(tokenHolders, []); + }); + + it("maps Approval events onto the same from/to shape", async ({ service }) => { + const { tokenActions } = await service.parseReceipt(header(5), transaction(), { + logs: [approvalLog(TOKEN, HOLDER_ONE, HOLDER_TWO, 9n)], + } as any); + + assert.equal(tokenActions.length, 1); + assert.equal(tokenActions[0].action, "Approval"); + assert.equal(tokenActions[0].from, HOLDER_ONE); + assert.equal(tokenActions[0].to, HOLDER_TWO); + assert.equal(tokenActions[0].value, "9"); + }); + + it("tolerates a receipt without a log field", async ({ service }) => { + const { tokenActions, tokenHolders, tokens } = await service.parseReceipt(header(5), transaction(), {} as any); + + assert.equal(tokenActions, []); + assert.equal(tokenHolders, []); + assert.equal(tokens, []); + }); + + it("numbers actions per contract and resets the numbering on the next block", async ({ service }) => { + const receiptOf = (token: string) => ({ logs: [transferLog(token, HOLDER_ONE, HOLDER_TWO, 1n)] }) as any; + + const first = await service.parseReceipt(header(5), transaction(), { + logs: [ + transferLog(TOKEN, HOLDER_ONE, HOLDER_TWO, 1n), + transferLog(TOKEN, HOLDER_TWO, HOLDER_ONE, 2n), + transferLog(OTHER_TOKEN, HOLDER_ONE, HOLDER_TWO, 3n), + ], + } as any); + assert.equal( + first.tokenActions.map((action) => [action.address, action.index]), + [ + [TOKEN, 0], + [TOKEN, 1], + [OTHER_TOKEN, 0], + ], + ); + + // A later receipt in the same block continues the numbering ... + const sameBlock = await service.parseReceipt(header(5), transaction(), receiptOf(TOKEN)); + assert.equal(sameBlock.tokenActions[0].index, 2); + + // ... and a new block starts over. + const nextBlock = await service.parseReceipt(header(6), transaction(), receiptOf(TOKEN)); + assert.equal(nextBlock.tokenActions[0].index, 0); + }); + + it("stores a detected ERC20 token including holder balances", async (context) => { + respondAsErc20(context, { [HOLDER_ONE]: 70n, [HOLDER_TWO]: 30n }); + + const { tokenActions, tokenHolders, tokens } = await context.service.parseReceipt(header(5), transaction(), { + logs: [transferLog(TOKEN, HOLDER_ONE, HOLDER_TWO, 30n)], + } as any); + + assert.equal(tokenActions.length, 1); + assert.equal(tokens, [ + { + address: TOKEN, + decimals: 8, + deploymentHash: undefined, + name: "Test Token", + symbol: "TEST", + // normalized from the decoded bigint + totalSupply: "5000", + }, + ]); + + const balancesByAddress = new Map(tokenHolders.map((holder) => [holder.address, holder.balance])); + assert.equal(balancesByAddress.get(HOLDER_ONE), "70"); + assert.equal(balancesByAddress.get(HOLDER_TWO), "30"); + assert.equal( + tokenHolders.map((holder) => holder.tokenAddress), + [TOKEN, TOKEN], + ); + }); + + it("links a deployment transaction to the token", async (context) => { + respondAsErc20(context); + + const deployment: any = { hash: "0xtransaction", to: undefined }; + const { tokens } = await context.service.parseReceipt(header(5), deployment, { + logs: [transferLog(TOKEN, HOLDER_ONE, HOLDER_TWO, 1n)], + } as any); + + assert.equal(tokens[0].deploymentHash, "0xtransaction"); + }); + + it("does not treat a contract with empty call results as a token", async (context) => { + // Default view responds unsuccessful + empty, which means "function does not exist". + const { tokenHolders, tokens } = await context.service.parseReceipt(header(5), transaction(), { + logs: [transferLog(TOKEN, HOLDER_ONE, HOLDER_TWO, 1n)], + } as any); + + assert.equal(tokens, []); + assert.equal(tokenHolders, []); + }); + + it("does not treat a contract with incomplete metadata as a token", async (context) => { + const known = erc20Responses(); + context.respond((calldata) => { + // symbol() reverts with data; everything else looks fine. + if (calldata === calldataOf("symbol")) { + return { output: asBuffer("0xff"), success: false }; + } + return known.get(calldata) ?? ok("0x01"); + }); + + const { tokens } = await context.service.parseReceipt(header(5), transaction(), { + logs: [transferLog(TOKEN, HOLDER_ONE, HOLDER_TWO, 1n)], + } as any); + + assert.equal(tokens, []); + }); + + it("swallows exceptions thrown while probing a contract", async (context) => { + const known = erc20Responses(); + context.respond((calldata) => { + if (calldata === calldataOf("name")) { + throw new Error("view blew up"); + } + return known.get(calldata) ?? ok("0x01"); + }); + + const { tokenActions, tokens } = await context.service.parseReceipt(header(5), transaction(), { + logs: [transferLog(TOKEN, HOLDER_ONE, HOLDER_TWO, 1n)], + } as any); + + // The action is still recorded even though the metadata could not be resolved. + assert.equal(tokenActions.length, 1); + assert.equal(tokens, []); + }); + + it("falls back to a zero balance when balanceOf fails or throws", async (context) => { + const known = erc20Responses(); + context.respond((calldata) => { + if (calldata === balanceOfCalldata(HOLDER_ONE)) { + return { output: Buffer.alloc(0), success: false }; + } + if (calldata === balanceOfCalldata(HOLDER_TWO)) { + throw new Error("balanceOf blew up"); + } + return known.get(calldata) ?? ok("0x01"); + }); + const warn = spy(context.logger, "warn"); + + const { tokenHolders, tokens } = await context.service.parseReceipt(header(5), transaction(), { + logs: [transferLog(TOKEN, HOLDER_ONE, HOLDER_TWO, 1n)], + } as any); + + assert.equal(tokens.length, 1); + assert.equal( + tokenHolders.map((holder) => holder.balance), + ["0", "0"], + ); + warn.calledTimes(2); + }); + + it("serves repeated contracts from the cache without re-emitting the token", async (context) => { + respondAsErc20(context, { [HOLDER_ONE]: 1n, [HOLDER_TWO]: 1n }); + const view = spy(context.evm, "view"); + + const receipt = () => ({ logs: [transferLog(TOKEN, HOLDER_ONE, HOLDER_TWO, 1n)] }) as any; + + const first = await context.service.parseReceipt(header(5), transaction(), receipt()); + assert.equal(first.tokens.length, 1); + view.reset(); + + const second = await context.service.parseReceipt(header(6), transaction(), receipt()); + assert.equal(second.tokens, []); + assert.equal(second.tokenHolders.length, 2); + + // Only the two balance lookups remain; the metadata probing is not repeated. + view.calledTimes(2); + }); + + it("treats a token known to the database as existing", async (context) => { + context.databaseToken = { address: TOKEN, decimals: 8, name: "Db Token", symbol: "DBT", totalSupply: "1" }; + respondAsErc20(context, { [HOLDER_ONE]: 4n, [HOLDER_TWO]: 4n }); + + const { tokenHolders, tokens } = await context.service.parseReceipt(header(5), transaction(), { + logs: [transferLog(TOKEN, HOLDER_ONE, HOLDER_TWO, 1n)], + } as any); + + // Nothing new to insert, but the holders are still tracked. + assert.equal(tokens, []); + assert.equal(tokenHolders.length, 2); + }); +}); diff --git a/packages/api-sync/source/parsers/tokens.ts b/packages/api-sync/source/parsers/tokens.ts index ddf77e783..363e23960 100644 --- a/packages/api-sync/source/parsers/tokens.ts +++ b/packages/api-sync/source/parsers/tokens.ts @@ -164,7 +164,7 @@ export class TokenParserService implements TokenParser { const eventLogs = parseEventLogs({ abi: erc20AbiEvents, eventName: ["Transfer", "Approval"], - logs: receipt.logs, + logs: receipt.logs ?? [], }); const dirtyAccounts = new Map<`0x${string}`, Set>(); @@ -232,7 +232,7 @@ export class TokenParserService implements TokenParser { // Skip anything if not deemed a token. if (!foundTokens.has(contract)) { this.logger.debugExtra( - `Ignoring tx to contract '${receipt.contractAddress}' because it does not implemented expected ERC20 ABI.`, + `Ignoring tx to contract '${contract}' because it does not implement the expected ERC20 ABI.`, ); continue; } else { @@ -343,7 +343,11 @@ export class TokenParserService implements TokenParser { functionName: call.functionName, }); - tokenMetadata[call.functionName] = decoded as unknown as undefined; + // totalSupply decodes to a bigint; normalize to string to match the + // declared metadata/model contract (as balanceOf does below). + tokenMetadata[call.functionName] = (typeof decoded === "bigint" + ? decoded.toString() + : decoded) as unknown as undefined; } continue; @@ -398,7 +402,7 @@ export class TokenParserService implements TokenParser { return decoded.toString(); } catch (rawError) { const error = ensureError(rawError); - console.log(error.message); + this.logger.warn(`Failed to call balanceOf on '${contract}' for ${account}: ${error.message}`); } return "0"; diff --git a/packages/api-sync/source/parsers/username.test.ts b/packages/api-sync/source/parsers/username.test.ts new file mode 100644 index 000000000..f0e7b3ac0 --- /dev/null +++ b/packages/api-sync/source/parsers/username.test.ts @@ -0,0 +1,91 @@ +import { describe } from "@mainsail/test-runner"; +import { encodeAbiParameters, encodeEventTopics, parseAbi } from "viem"; + +import { parseUsernames } from "./username.js"; + +const usernamesAbi = parseAbi([ + "event UsernameRegistered(address addr, string username, string previousUsername)", + "event UsernameResigned(address addr, string username)", +] as const); + +const USERNAMES_CONTRACT = "0x1000000000000000000000000000000000000000"; +// Digit-only address, unaffected by checksumming. +const WALLET = "0x1111111111111111111111111111111111111111"; + +const registeredLog = (addr: string, username: string) => ({ + address: USERNAMES_CONTRACT, + data: encodeAbiParameters( + [ + { name: "addr", type: "address" }, + { name: "username", type: "string" }, + { name: "previousUsername", type: "string" }, + ], + [addr, username, ""], + ), + topics: encodeEventTopics({ abi: usernamesAbi, eventName: "UsernameRegistered" }), +}); + +const resignedLog = (addr: string, username: string) => ({ + address: USERNAMES_CONTRACT, + data: encodeAbiParameters( + [ + { name: "addr", type: "address" }, + { name: "username", type: "string" }, + ], + [addr, username], + ), + topics: encodeEventTopics({ abi: usernamesAbi, eventName: "UsernameResigned" }), +}); + +const otherLog = () => ({ + address: USERNAMES_CONTRACT, + data: "0x" as const, + topics: ["0xdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef"] as [`0x${string}`], +}); + +const transaction = (to: string): any => ({ hash: "0xtransaction", to }); + +describe("parseUsernames", ({ assert, it }) => { + it("ignores transactions that do not target the usernames contract", () => { + const result = parseUsernames(USERNAMES_CONTRACT, transaction(WALLET), { + logs: [registeredLog(WALLET, "alice")], + } as any); + + assert.equal(result, []); + }); + + it("returns nothing without logs", () => { + assert.equal(parseUsernames(USERNAMES_CONTRACT, transaction(USERNAMES_CONTRACT), { logs: [] } as any), []); + }); + + it("tolerates a receipt without a log field", () => { + assert.equal(parseUsernames(USERNAMES_CONTRACT, transaction(USERNAMES_CONTRACT), {} as any), []); + }); + + it("maps a registration to the registered username", () => { + const result = parseUsernames(USERNAMES_CONTRACT, transaction(USERNAMES_CONTRACT), { + logs: [registeredLog(WALLET, "alice")], + } as any); + + assert.equal(result, [{ address: WALLET, username: "alice" }]); + }); + + it("maps a resignation to an undefined username", () => { + const result = parseUsernames(USERNAMES_CONTRACT, transaction(USERNAMES_CONTRACT), { + logs: [resignedLog(WALLET, "alice")], + } as any); + + assert.equal(result, [{ address: WALLET, username: undefined }]); + }); + + it("keeps the log order and skips unrelated events", () => { + const result = parseUsernames(USERNAMES_CONTRACT, transaction(USERNAMES_CONTRACT), { + logs: [registeredLog(WALLET, "alice"), otherLog(), resignedLog(WALLET, "alice")], + } as any); + + assert.equal(result, [ + { address: WALLET, username: "alice" }, + { address: WALLET, username: undefined }, + ]); + }); +}); diff --git a/packages/api-sync/source/parsers/username.ts b/packages/api-sync/source/parsers/username.ts index 49a79da0f..4a5e31409 100644 --- a/packages/api-sync/source/parsers/username.ts +++ b/packages/api-sync/source/parsers/username.ts @@ -19,7 +19,7 @@ export function parseUsernames( const logs = parseEventLogs({ abi: paymentAbi, //eventName: "UsernameRegistered", - logs: receipt.logs, + logs: receipt.logs ?? [], }); return logs.map((l) => { diff --git a/packages/api-sync/source/restore.test.ts b/packages/api-sync/source/restore.test.ts new file mode 100644 index 000000000..b298ab8a8 --- /dev/null +++ b/packages/api-sync/source/restore.test.ts @@ -0,0 +1,653 @@ +import { Identifiers as ApiDatabaseIdentifiers } from "@mainsail/api-database"; +import { Identifiers } from "@mainsail/constants"; +import { Application } from "@mainsail/kernel"; +import { describe } from "@mainsail/test-runner"; +import { encodeAbiParameters, encodeEventTopics, parseAbi } from "viem"; + +import { Restore } from "./restore.js"; + +const GENESIS_PROPOSER = "0xgenesis"; +const PROPOSER = "0xvalidator"; +// Digit-only addresses survive viem checksumming unchanged. +const NAMED_USER = "0x1111111111111111111111111111111111111111"; +const PAYMENT_RECIPIENT = "0x3333333333333333333333333333333333333333"; +const USERNAME_CONTRACT = "0xusernames"; +const MULTIPAYMENT_CONTRACT = "0xmultipayment"; + +const usernameAbi = parseAbi([ + "event UsernameRegistered(address addr, string username, string previousUsername)", +] as const); + +const usernameRegisteredLog = (addr: string, username: string) => ({ + address: "0x0000000000000000000000000000000000000001", + data: encodeAbiParameters( + [ + { name: "addr", type: "address" }, + { name: "username", type: "string" }, + { name: "previousUsername", type: "string" }, + ], + [addr, username, ""], + ), + topics: encodeEventTopics({ abi: usernameAbi, eventName: "UsernameRegistered" }), +}); + +const paymentAbi = parseAbi(["event Payment(address indexed recipient, uint256 amount, bool success)"] as const); + +const paymentLog = (recipient: string, amount: bigint) => ({ + address: "0x0000000000000000000000000000000000000001", + data: encodeAbiParameters( + [ + { name: "amount", type: "uint256" }, + { name: "success", type: "bool" }, + ], + [amount, true], + ), + topics: encodeEventTopics({ abi: paymentAbi, args: { recipient }, eventName: "Payment" }), +}); + +// Chainable TypeORM query-builder fake that records every call. +const makeQb = () => { + const calls: Record = {}; + const record = (method: string, args: any[]) => { + (calls[method] ??= []).push(args); + }; + + const qb: any = { calls }; + for (const method of ["insert", "orIgnore", "orUpdate", "update", "set", "values", "where", "andWhere"]) { + qb[method] = (...args: any[]) => { + // Copy array arguments as the source reuses (and clears) its batch arrays. + record( + method, + args.map((argument) => (Array.isArray(argument) ? [...argument] : argument)), + ); + return qb; + }; + } + qb.execute = async () => { + record("execute", []); + }; + + return qb; +}; + +const makeRepo = (tableName: string) => { + const qb = makeQb(); + const repo: any = { + createQueryBuilder: () => qb, + metadata: { tableName }, + qb, + queries: [] as any[][], + }; + repo.query = async (...args: any[]) => { + repo.queries.push(args); + }; + return repo; +}; + +const makeBlock = (overrides: Record = {}) => ({ + fee: 100n, + gasUsed: 21_000, + hash: "0xblock1", + number: 1, + parentHash: "0xblock0", + payloadSize: 10, + proposer: PROPOSER, + reward: 200n, + round: 0, + stateRoot: "0xstateroot", + timestamp: 1_720_000_000, + transactions: [] as any[], + transactionsCount: 0, + transactionsRoot: "0xtxroot", + version: 1, + ...overrides, +}); + +const makeTransaction = (hash: string, to: string) => ({ + data: "0x", + from: "0xsender", + gasLimit: 21_000, + gasPrice: 5, + hash, + legacySecondSignature: undefined, + nonce: 1n, + r: "aa", + s: "bb", + senderPublicKey: "pk-sender", + to, + transactionIndex: 0, + v: 27, + value: 0n, +}); + +const makeReceipt = (blockNumber: number, txHash: string, logs: any[] = []) => ({ + blockNumber: BigInt(blockNumber), + contractAddress: undefined, + cumulativeGasUsed: 21_000n, + gasRefunded: 0n, + gasUsed: 21_000n, + logs, + output: undefined, + status: 1, + txHash, +}); + +type Ctx = { + app: Application; + restore: Restore; + repos: Record; + systemRepo: any; + entityManager: any; + dataSource: any; + databaseService: any; + stateStore: any; + migrations: any; + evm: any; + logger: any; + consensusContractService: any; + tokenParser: any; + listeners: any; + snapshotImporter: any; + configuration: any; + commits: any[]; + receipts: any[]; + accountPages: any[]; + legacyColdWalletPages: any[]; + validatorRounds: any[]; + milestone: Record; + batchSize: number; +}; + +describe("Restore", ({ it, beforeEach, assert, spy }) => { + beforeEach((context) => { + context.entityManager = { + queries: [] as any[][], + query: async (...args: any[]) => { + context.entityManager.queries.push(args); + }, + }; + + context.repos = { + block: makeRepo("blocks"), + configuration: makeRepo("configuration"), + contract: makeRepo("contracts"), + legacyColdWallet: makeRepo("legacy_cold_wallets"), + multiPayment: makeRepo("multi_payments"), + state: makeRepo("state"), + token: makeRepo("tokens"), + tokenAction: makeRepo("token_actions"), + tokenHolder: makeRepo("token_holders"), + transaction: makeRepo("transactions"), + validatorRound: makeRepo("validator_rounds"), + wallet: makeRepo("wallets"), + }; + + context.systemRepo = { + maintenance: [] as boolean[], + setMaintenance: async (flag: boolean) => { + context.systemRepo.maintenance.push(flag); + }, + }; + + context.dataSource = { + transaction: async (levelOrCallback: any, callback?: any) => + typeof levelOrCallback === "function" + ? levelOrCallback(context.entityManager) + : callback(context.entityManager), + }; + + const genesisBlock = makeBlock({ + hash: "0xblock0", + number: 0, + parentHash: "0x0", + proposer: GENESIS_PROPOSER, + }); + const block = makeBlock({ + transactions: [ + makeTransaction("0xtx1", USERNAME_CONTRACT), + makeTransaction("0xtx2", MULTIPAYMENT_CONTRACT), + ], + transactionsCount: 2, + }); + + context.commits = [ + { block: genesisBlock, proof: { round: 0, signature: "0xsig0", validators: [true] } }, + { block, proof: { round: 0, signature: "0xsig1", validators: [true] } }, + ]; + + context.receipts = [ + makeReceipt(1, "0xtx1", [usernameRegisteredLog(NAMED_USER, "alice")]), + makeReceipt(1, "0xtx2", [paymentLog(PAYMENT_RECIPIENT, 25n)]), + ]; + + context.databaseService = { + getLastCommit: async () => context.commits.at(-1), + isEmpty: async () => false, + readCommits: async function* (from: number, to: number) { + for (const commit of context.commits) { + if (commit.block.number >= from && commit.block.number <= to) { + yield commit; + } + } + }, + }; + + context.stateStore = { getGenesisCommit: () => context.commits[0] }; + context.migrations = { runMigrations: async () => {} }; + + context.accountPages = [ + { + accounts: [ + { address: PROPOSER, balance: 1000n, legacyAttributes: {}, nonce: 1n }, + { address: NAMED_USER, balance: 10n, legacyAttributes: {}, nonce: 2n }, + ], + nextOffset: 2n, + }, + { + accounts: [ + { + address: "0xlegacyuser", + balance: 20n, + legacyAttributes: { legacyNonce: 3n, multiSignature: { min: 2 }, secondPublicKey: "0xspk" }, + nonce: 3n, + }, + { address: "0xmergetarget", balance: 30n, legacyAttributes: {}, nonce: 4n }, + { address: "0xsnapvalidator", balance: 40n, legacyAttributes: {}, nonce: 5n }, + { address: "0xlegacynonceonly", balance: 1n, legacyAttributes: { legacyNonce: 9n }, nonce: 6n }, + { + address: "0xmultisigonly", + balance: 2n, + legacyAttributes: { multiSignature: { min: 1 } }, + nonce: 7n, + }, + ], + nextOffset: undefined, + }, + ]; + + context.legacyColdWalletPages = [ + { + nextOffset: undefined, + wallets: [ + { + address: "LEGACY_MERGED", + balance: 50n, + legacyAttributes: { legacyNonce: 2n }, + mergeInfo: { address: "0xmergetarget", txHash: "0xmergetx" }, + }, + { address: "LEGACY_UNMERGED", balance: 60n, legacyAttributes: {}, mergeInfo: undefined }, + ], + }, + ]; + + let accountCall = 0; + let legacyCall = 0; + context.evm = { + getAccounts: async () => context.accountPages[accountCall++], + getLegacyColdWallets: async () => context.legacyColdWalletPages[legacyCall++], + getReceiptsByBlockRange: async () => ({ receipts: context.receipts }), + }; + + context.logger = { debug: () => {}, debugExtra: () => {}, error: () => {}, info: () => {}, warn: () => {} }; + + context.validatorRounds = [ + { round: 1, roundHeight: 1, validators: [{ address: PROPOSER, voteBalance: 100n }] }, + ]; + + context.consensusContractService = { + getAllValidators: async () => [ + { + address: PROPOSER, + blsPublicKey: "0xbls", + fee: 1n, + isResigned: false, + voteBalance: 100n, + votersCount: 1, + }, + ], + getValidatorRounds: async function* () { + yield* context.validatorRounds; + }, + getVotes: async function* () { + yield { validatorAddress: PROPOSER, voterAddress: NAMED_USER }; + }, + }; + + context.tokenParser = { + parseReceipt: async () => ({ + tokenActions: [ + { + action: "Transfer", + address: "0xtoken", + blockNumber: "1", + from: "0xa", + index: 0, + to: "0xb", + transactionHash: "0xtx1", + value: "5", + }, + ], + tokenHolders: [ + { address: "0xa", balance: "5", tokenAddress: "0xtoken" }, + { address: "0xb", balance: "0", tokenAddress: "0xtoken" }, + ], + tokens: [{ address: "0xtoken", decimals: 18, name: "Token", symbol: "TKN", totalSupply: "100" }], + }), + }; + + context.listeners = { flush: async () => {} }; + context.milestone = { evmSpec: "shanghai", snapshot: true }; + + context.snapshotImporter = { + drain: () => [ + { + ethAddress: "0xlegacyuser", + legacyAttributes: { legacyNonce: 7n }, + publicKey: "pk-legacy", + }, + { ethAddress: undefined, legacyAttributes: {}, publicKey: undefined }, + ], + prepareRestore: async () => {}, + validators: [ + // Already named via the usernames contract -> snapshot username must be skipped. + { ethAddress: NAMED_USER, username: "snapshot-name" }, + { ethAddress: "0xsnapvalidator", username: "snapval" }, + ], + }; + + context.app = new Application(); + context.app.bind(Identifiers.Application.Version).toConstantValue("0.0.1-test"); + context.app + .bind(Identifiers.Cryptography.Identity.Address.Factory) + .toConstantValue({ fromPublicKey: async () => "0xsender" }); + context.configuration = { + all: () => ({ network: "testnet" }), + getGenesisHeight: () => 0, + getMilestone: () => context.milestone, + }; + context.app.bind(Identifiers.Cryptography.Configuration).toConstantValue(context.configuration); + context.app.bind(ApiDatabaseIdentifiers.DataSource).toConstantValue(context.dataSource); + context.app.bind(ApiDatabaseIdentifiers.Migrations).toConstantValue(context.migrations); + context.app.bind(Identifiers.Evm.Instance).toConstantValue(context.evm).whenTagged("instance", "evm"); + context.app.bind(Identifiers.ApiSync.Logger).toConstantValue(context.logger); + context.app.bind(Identifiers.Database.Service).toConstantValue(context.databaseService); + context.app.bind(Identifiers.State.Store).toConstantValue(context.stateStore); + context.app.bind(Identifiers.BlockchainUtils.RoundCalculator).toConstantValue({ + calculateRound: (number: number) => ({ maxValidators: 1, round: number, roundHeight: number }), + isNewRound: () => true, + }); + context.app.bind(Identifiers.BlockchainUtils.ProposerCalculator).toConstantValue({ + getValidatorIndexFrom: (_maxValidators: number, _totalRound: number, index: number) => index, + }); + context.app.bind(ApiDatabaseIdentifiers.BlockRepositoryFactory).toConstantValue(() => context.repos.block); + context.app + .bind(ApiDatabaseIdentifiers.ConfigurationRepositoryFactory) + .toConstantValue(() => context.repos.configuration); + context.app + .bind(ApiDatabaseIdentifiers.ContractRepositoryFactory) + .toConstantValue(() => context.repos.contract); + context.app.bind(ApiDatabaseIdentifiers.StateRepositoryFactory).toConstantValue(() => context.repos.state); + context.app.bind(ApiDatabaseIdentifiers.SystemRepositoryFactory).toConstantValue(() => context.systemRepo); + context.app + .bind(ApiDatabaseIdentifiers.TransactionRepositoryFactory) + .toConstantValue(() => context.repos.transaction); + context.app + .bind(ApiDatabaseIdentifiers.MultiPaymentRepositoryFactory) + .toConstantValue(() => context.repos.multiPayment); + context.app.bind(ApiDatabaseIdentifiers.TokenRepositoryFactory).toConstantValue(() => context.repos.token); + context.app + .bind(ApiDatabaseIdentifiers.TokenHolderRepositoryFactory) + .toConstantValue(() => context.repos.tokenHolder); + context.app + .bind(ApiDatabaseIdentifiers.TokenActionRepositoryFactory) + .toConstantValue(() => context.repos.tokenAction); + context.app + .bind(ApiDatabaseIdentifiers.ValidatorRoundRepositoryFactory) + .toConstantValue(() => context.repos.validatorRound); + context.app.bind(ApiDatabaseIdentifiers.WalletRepositoryFactory).toConstantValue(() => context.repos.wallet); + context.app + .bind(ApiDatabaseIdentifiers.LegacyColdWalletRepositoryFactory) + .toConstantValue(() => context.repos.legacyColdWallet); + context.app.bind(Identifiers.Evm.ContractService.Consensus).toConstantValue(context.consensusContractService); + context.app.bind(Identifiers.EvmConsensus.Contracts.MultiPayment).toConstantValue(MULTIPAYMENT_CONTRACT); + context.app.bind(Identifiers.EvmConsensus.Contracts.Usernames).toConstantValue(USERNAME_CONTRACT); + context.app.bind(Identifiers.ApiSync.TokenParser).toConstantValue(context.tokenParser); + context.app.bind(Identifiers.ApiSync.Listener).toConstantValue(context.listeners); + context.app.bind(Identifiers.Snapshot.Legacy.Importer).toConstantValue(context.snapshotImporter); + context.batchSize = 100; + context.app + .bind(Identifiers.ServiceProvider.Configuration) + .toConstantValue({ getRequired: () => context.batchSize }) + .whenTagged("plugin", "api-sync"); + + context.restore = context.app.resolve(Restore); + }); + + it("restores blocks, transactions, wallets, rounds and state", async (context) => { + const { repos, restore, snapshotImporter, systemRepo, entityManager, listeners, migrations } = context; + + const prepareRestore = spy(snapshotImporter, "prepareRestore"); + const flush = spy(listeners, "flush"); + const runMigrations = spy(migrations, "runMigrations"); + + await restore.restore(); + + prepareRestore.calledOnce(); + + // Both blocks are inserted with their proof data. + const blocks = repos.block.qb.calls.values[0][0]; + assert.equal(blocks.length, 2); + assert.equal(blocks[0].number, "0"); + assert.equal(blocks[1].number, "1"); + assert.equal(blocks[1].signature, "0xsig1"); + assert.equal(blocks[1].fee, "100"); + assert.equal(blocks[1].reward, "200"); + + // Both transactions land in the transactions table. + const transactions = repos.transaction.qb.calls.values[0][0]; + assert.equal( + transactions.map((transaction: any) => transaction.hash), + ["0xtx1", "0xtx2"], + ); + // formatEcdsaSignature concatenates r || s || v (hex) + assert.equal(transactions[0].signature, "aabb1b"); + + // The multi payment on tx2 is recorded. + const multiPayments = repos.multiPayment.qb.calls.values[0][0]; + assert.equal(multiPayments.length, 1); + assert.equal(multiPayments[0].hash, "0xtx2"); + assert.equal(multiPayments[0].amount, "25"); + + // Tokens/holders/actions from the parser; the zero-balance holder is skipped. + assert.equal(repos.token.qb.calls.values[0][0].length, 1); + assert.equal(repos.tokenAction.qb.calls.values[0][0].length, 2); + const holders = repos.tokenHolder.qb.calls.values[0][0]; + assert.equal(holders, [{ address: "0xa", balance: "5", tokenAddress: "0xtoken" }]); + + // Wallets are ingested across both pages. + const walletRows = repos.wallet.qb.calls.values.flatMap(([batch]: [any[]]) => batch); + const byAddress = new Map(walletRows.map((row: any) => [row.address, row])); + assert.equal(walletRows.length, 7); + + const validatorWallet = byAddress.get(PROPOSER)!; + assert.equal(validatorWallet.attributes.validatorPublicKey, "0xbls"); + assert.equal(validatorWallet.attributes.validatorVoteBalance, "100"); + assert.equal(validatorWallet.attributes.validatorForgedFees, "100"); + assert.equal(validatorWallet.attributes.validatorForgedRewards, "200"); + assert.equal(validatorWallet.attributes.validatorForgedTotal, "300"); + assert.equal(validatorWallet.attributes.validatorProducedBlocks, 1); + assert.equal(validatorWallet.attributes.validatorLastBlock.hash, "0xblock1"); + + // The username registered on-chain wins over the snapshot username. + const namedWallet = byAddress.get(NAMED_USER)!; + assert.equal(namedWallet.attributes.username, "alice"); + assert.equal(namedWallet.attributes.vote, PROPOSER); + + // The snapshot-only username is applied. + assert.equal(byAddress.get("0xsnapvalidator")!.attributes.username, "snapval"); + + // Legacy wallet attributes from the snapshot and the EVM are merged. + const legacyWallet = byAddress.get("0xlegacyuser")!; + assert.true(legacyWallet.attributes.isLegacy); + assert.equal(legacyWallet.attributes.legacyNonce, "3"); + assert.equal(legacyWallet.attributes.secondPublicKey, "0xspk"); + assert.equal(legacyWallet.attributes.multiSignature, { min: 2 }); + assert.equal(legacyWallet.publicKey, "pk-legacy"); + + // The merged cold wallet target is flagged with the merge info. + const mergeTarget = byAddress.get("0xmergetarget")!; + assert.true(mergeTarget.attributes.isLegacy); + assert.equal(mergeTarget.attributes.legacyMerge, { address: "LEGACY_MERGED", txHash: "0xmergetx" }); + + // Legacy cold wallets are inserted. + const coldWallets = repos.legacyColdWallet.qb.calls.values[0][0]; + assert.equal(coldWallets.length, 2); + assert.equal(coldWallets[0].mergeInfoTransactionHash, "0xmergetx"); + assert.equal(coldWallets[0].attributes.legacyNonce, "2"); + + // Validator rounds honor the proposer ordering. + const rounds = repos.validatorRound.qb.calls.values[0][0]; + assert.equal(rounds, [{ round: 1, roundHeight: 1, validators: [PROPOSER], votes: ["100"] }]); + + // Configuration and state are written. + assert.equal(repos.configuration.qb.calls.values[0][0].version, "0.0.1-test"); + assert.equal(repos.configuration.qb.calls.values[0][0].cryptoConfiguration, { network: "testnet" }); + // Partial legacy attributes only keep what is present. + assert.equal(byAddress.get("0xlegacynonceonly")!.attributes, { legacyNonce: "9" }); + assert.equal(byAddress.get("0xmultisigonly")!.attributes, { multiSignature: { min: 1 } }); + + const state = repos.state.qb.calls.values[0][0]; + assert.equal(state.blockNumber, "1"); + // account balances (1000+10+20+30+40+1+2) + unmerged legacy balance (60) + assert.equal(state.supply, "1163"); + + // Listener data is flushed inside the restore transaction and migrations run afterwards. + flush.calledOnce(); + runMigrations.calledOnce(); + + // Maintenance mode is toggled around the restore. + assert.equal(systemRepo.maintenance, [true, false]); + + // Post-restore housekeeping: analyze + rank/token count refresh. + const analyzed = Object.values(repos).filter((repo: any) => + repo.queries.some(([sql]: [string]) => sql.startsWith("ANALYZE")), + ); + assert.equal(analyzed.length, 12); + assert.equal( + entityManager.queries.map(([sql]: [string]) => sql), + [ + "SET LOCAL statement_timeout = 0;", + "SELECT update_validator_ranks();", + "SELECT update_wallet_token_counts();", + ], + ); + }); + + it("restores from the genesis commit when the consensus database is empty", async (context) => { + const { repos, restore, databaseService, configuration } = context; + + databaseService.isEmpty = async () => true; + context.commits = context.commits.slice(0, 1); + context.receipts = []; + configuration.all = () => undefined; + + await restore.restore(); + + const blocks = repos.block.qb.calls.values[0][0]; + assert.equal(blocks.length, 1); + assert.equal(blocks[0].number, "0"); + + // Missing crypto configuration falls back to an empty object. + assert.equal(repos.configuration.qb.calls.values[0][0].cryptoConfiguration, {}); + }); + + it("ingests across multiple batches when the batch size is smaller than the chain", async (context) => { + const { repos, restore } = context; + + context.batchSize = 1; + + await restore.restore(); + + // One block insert per batch. + assert.equal( + repos.block.qb.calls.values.map(([batch]: [any[]]) => batch.map((block: any) => block.number)), + [["0"], ["1"]], + ); + + // The chunk size follows the batch size, so every transaction is flushed individually ... + assert.equal( + repos.transaction.qb.calls.values.map(([batch]: [any[]]) => batch.map((t: any) => t.hash)), + [["0xtx1"], ["0xtx2"]], + ); + + // ... and the token data is flushed as soon as it exceeds the chunk size. + assert.equal(repos.token.qb.calls.values.length, 2); + + const multiPayments = repos.multiPayment.qb.calls.values.flatMap(([batch]: [any[]]) => batch); + assert.equal(multiPayments.length, 1); + assert.equal(multiPayments[0].hash, "0xtx2"); + }); + + it("restores without a snapshot importer", async (context) => { + const { repos, snapshotImporter } = context; + + // The importer is an optional dependency; simulate a network without a legacy snapshot. + context.app.rebind(Identifiers.Snapshot.Legacy.Importer).toConstantValue(undefined); + const restore = context.app.resolve(Restore); + + const prepareRestore = spy(snapshotImporter, "prepareRestore"); + + await restore.restore(); + + prepareRestore.neverCalled(); + + const walletRows = repos.wallet.qb.calls.values.flatMap(([batch]: [any[]]) => batch); + const byAddress = new Map(walletRows.map((row: any) => [row.address, row])); + + // No snapshot: no public key mapping, no isLegacy flag and no snapshot username. + const legacyWallet = byAddress.get("0xlegacyuser")!; + assert.null(legacyWallet.publicKey); + assert.undefined(legacyWallet.attributes.isLegacy); + assert.undefined(byAddress.get("0xsnapvalidator")!.attributes.username); + + // EVM-provided legacy attributes and on-chain usernames are unaffected. + assert.equal(legacyWallet.attributes.legacyNonce, "3"); + assert.equal(byAddress.get(NAMED_USER)!.attributes.username, "alice"); + }); + + it("skips the snapshot preparation when the milestone has no snapshot", async (context) => { + const { restore, snapshotImporter } = context; + + context.milestone.snapshot = undefined; + const prepareRestore = spy(snapshotImporter, "prepareRestore"); + + await restore.restore(); + + prepareRestore.neverCalled(); + }); + + it("rejects when a non-genesis block was proposed by an unknown validator", async (context) => { + const { restore } = context; + + context.commits[1].block.proposer = "0xunknown"; + + await assert.rejects(() => restore.restore(), "unexpected validator"); + }); + + it("rejects when the on-chain validator round size does not match the expected one", async (context) => { + const { restore } = context; + + context.validatorRounds = [ + { + round: 1, + roundHeight: 1, + validators: [ + { address: PROPOSER, voteBalance: 100n }, + { address: "0xother", voteBalance: 50n }, + ], + }, + ]; + + await assert.rejects(() => restore.restore(), /mismatch in expected/); + }); +}); diff --git a/packages/api-sync/source/restore.ts b/packages/api-sync/source/restore.ts index 7b6677a1b..cd47d8a53 100644 --- a/packages/api-sync/source/restore.ts +++ b/packages/api-sync/source/restore.ts @@ -944,6 +944,7 @@ export class Restore { legacyColdWalletRepository, multiPaymentRepository, stateRepository, + tokenActionRepository, tokenHolderRepository, tokenRepository, transactionRepository, @@ -960,6 +961,7 @@ export class Restore { legacyColdWalletRepository, multiPaymentRepository, tokenRepository, + tokenActionRepository, tokenHolderRepository, configurationRepository, ]) { diff --git a/packages/api-sync/source/service-provider.test.ts b/packages/api-sync/source/service-provider.test.ts new file mode 100644 index 000000000..0d60544fd --- /dev/null +++ b/packages/api-sync/source/service-provider.test.ts @@ -0,0 +1,151 @@ +import { Identifiers } from "@mainsail/constants"; +import { Application, Providers } from "@mainsail/kernel"; +import { describe } from "@mainsail/test-runner"; +import Joi from "joi"; + +import { Listeners } from "./listeners.js"; +import { ServiceProvider } from "./service-provider.js"; + +const baseConfig = { + enabled: true, + restore: { blocks: { batchSize: 500 } }, + syncInterval: 8000, + tokenCacheSize: 256, + tokenWhitelistRefreshInterval: 60_000, + tokenWhitelistRemoteUrl: "", +}; + +describe<{ + app: Application; + serviceProvider: ServiceProvider; + configure: (config: Record) => void; + validate: (config: Record) => Joi.ValidationResult; +}>("ServiceProvider", ({ it, beforeEach, assert, stub, spy }) => { + beforeEach((context) => { + context.app = new Application(); + context.serviceProvider = context.app.resolve(ServiceProvider); + + context.configure = (config) => { + context.serviceProvider.setConfig( + context.app.resolve(Providers.PluginConfiguration).from("api-sync", config), + ); + }; + + context.validate = (config) => context.serviceProvider.configSchema().validate(config); + }); + + it("register: binds all api-sync services and hooks up the listeners", async ({ + app, + serviceProvider, + configure, + }) => { + configure(baseConfig); + + // The concrete listeners can only resolve against a live database, so their + // registration is stubbed out; the wiring itself is under test here. + const register = stub(Listeners.prototype, "register").resolvedValue(undefined); + + await serviceProvider.register(); + + assert.true(app.isBound(Identifiers.ApiSync.Listener)); + assert.true(app.isBound(Identifiers.ApiSync.Logger)); + assert.true(app.isBound(Identifiers.ApiSync.TokenParser)); + assert.true(app.isBound(Identifiers.ApiSync.TokenWhitelist)); + assert.true(app.isBound(Identifiers.ApiSync.Service)); + register.calledOnce(); + + register.restore(); + }); + + it("register: does nothing when the plugin is disabled", async ({ app, serviceProvider, configure }) => { + configure({ ...baseConfig, enabled: false }); + + await serviceProvider.register(); + + assert.false(app.isBound(Identifiers.ApiSync.Listener)); + assert.false(app.isBound(Identifiers.ApiSync.Service)); + }); + + it("register: does nothing inside a worker thread", async ({ app, serviceProvider, configure }) => { + configure(baseConfig); + const isWorker = stub(app, "isWorker").returnValue(true); + + await serviceProvider.register(); + + assert.false(app.isBound(Identifiers.ApiSync.Listener)); + assert.false(app.isBound(Identifiers.ApiSync.Service)); + + isWorker.restore(); + }); + + it("dispose: flushes the service and disposes listeners and whitelist", async ({ + app, + serviceProvider, + configure, + }) => { + configure(baseConfig); + + const service = { flush: async () => {} }; + const listeners = { dispose: async () => {} }; + const whitelist = { dispose: async () => {} }; + + app.bind(Identifiers.ApiSync.Service).toConstantValue(service); + app.bind(Identifiers.ApiSync.Listener).toConstantValue(listeners); + app.bind(Identifiers.ApiSync.TokenWhitelist).toConstantValue(whitelist); + + const flush = spy(service, "flush"); + const disposeListeners = spy(listeners, "dispose"); + const disposeWhitelist = spy(whitelist, "dispose"); + + await serviceProvider.dispose(); + + flush.calledOnce(); + disposeListeners.calledOnce(); + disposeWhitelist.calledOnce(); + }); + + it("dispose: does nothing when the plugin is disabled", async ({ serviceProvider, configure }) => { + configure({ ...baseConfig, enabled: false }); + + // No services are bound; a non-guarded dispose would fail to resolve them. + await assert.resolves(() => serviceProvider.dispose()); + }); + + it("configSchema: accepts a valid configuration and unknown keys", ({ validate }) => { + const result = validate({ ...baseConfig, customFlag: true }); + + assert.undefined(result.error); + assert.equal(result.value.restore.blocks.batchSize, 500); + assert.true(result.value.customFlag); + }); + + it("configSchema: requires syncInterval", ({ validate }) => { + const { syncInterval, ...config } = baseConfig; + + const result = validate(config); + + assert.defined(result.error); + assert.true(result.error!.message.includes("syncInterval")); + }); + + it("configSchema: rejects non-positive intervals and sizes", ({ validate }) => { + for (const override of [ + { syncInterval: 0 }, + { tokenCacheSize: -1 }, + { tokenWhitelistRefreshInterval: 0 }, + { restore: { blocks: { batchSize: 0 } } }, + ]) { + const result = validate({ ...baseConfig, ...override }); + + assert.defined(result.error); + } + }); + + it("configSchema: rejects non-integer values", ({ validate }) => { + for (const override of [{ syncInterval: 1.5 }, { tokenCacheSize: 2.5 }]) { + const result = validate({ ...baseConfig, ...override }); + + assert.defined(result.error); + } + }); +}); diff --git a/packages/api-sync/source/service.test.ts b/packages/api-sync/source/service.test.ts new file mode 100644 index 000000000..9788157e5 --- /dev/null +++ b/packages/api-sync/source/service.test.ts @@ -0,0 +1,736 @@ +import { Identifiers as ApiDatabaseIdentifiers } from "@mainsail/api-database"; +import { Identifiers } from "@mainsail/constants"; +import { Application } from "@mainsail/kernel"; +import { describe } from "@mainsail/test-runner"; + +import { Sync } from "./service.js"; + +const PROPOSER = "0xproposer"; +const SENDER_PUBLIC_KEY = "02aabbcc"; +const SENDER = "0xsender"; + +// Chainable TypeORM query-builder fake that records every call. +const makeQb = () => { + const calls: Record = {}; + const record = (method: string, args: any[]) => { + (calls[method] ??= []).push(args); + }; + + const qb: any = { calls }; + for (const method of ["insert", "orIgnore", "orUpdate", "update", "set", "values", "where", "andWhere"]) { + qb[method] = (...args: any[]) => { + record(method, args); + return qb; + }; + } + qb.execute = async () => { + record("execute", []); + }; + + return qb; +}; + +const makeRepo = () => { + const qb = makeQb(); + const repo: any = { + createQueryBuilder: () => qb, + qb, + queries: [] as any[][], + removed: [] as any[], + }; + repo.query = async (...args: any[]) => { + repo.queries.push(args); + }; + repo.remove = async (entities: any[]) => { + repo.removed.push(entities); + }; + return repo; +}; + +const makeHeader = () => ({ + fee: 100n, + gasUsed: 21_000, + hash: "0xblockhash", + number: 1, + parentHash: "0xparent", + payloadSize: 10, + proposer: PROPOSER, + reward: 200n, + round: 0, + stateRoot: "0xstateroot", + timestamp: 1_720_000_000, + transactionsCount: 1, + transactionsRoot: "0xtxroot", + version: 1, +}); + +const makeTransaction = () => ({ + data: "0x", + from: SENDER, + gasLimit: 21_000, + gasPrice: 5, + hash: "0xtx1", + legacySecondSignature: undefined, + nonce: 1n, + r: "aa", + s: "bb", + senderPublicKey: SENDER_PUBLIC_KEY, + to: "0xrecipient", + transactionIndex: 0, + v: 27, + value: 0n, +}); + +const makeReceipt = () => ({ + contractAddress: undefined, + cumulativeGasUsed: 21_000n, + gasRefunded: 0n, + gasUsed: 21_000n, + logs: [], + output: undefined, + status: 1, +}); + +type Ctx = { + app: Application; + sync: Sync; + addressFactory: any; + configuration: any; + roundCalculator: any; + dataSource: any; + databaseService: any; + migrations: any; + entityManager: any; + repos: Record; + systemRepo: any; + evm: any; + state: any; + validatorSet: any; + proposerCalculator: any; + logger: any; + pluginConfiguration: any; + queue: any; + listeners: any; + tokenParser: any; + tokenWhitelist: any; + queryResults: Record; + makeUnit: (overrides?: Partial>) => any; +}; + +describe("Sync", ({ it, beforeEach, assert, stub, spy, clock }) => { + beforeEach((context) => { + context.entityManager = { + queries: [] as any[][], + query: async (...args: any[]) => { + context.entityManager.queries.push(args); + }, + }; + + context.repos = { + block: makeRepo(), + configuration: makeRepo(), + legacyColdWallet: makeRepo(), + multiPayment: makeRepo(), + state: makeRepo(), + token: makeRepo(), + tokenAction: makeRepo(), + tokenHolder: makeRepo(), + transaction: makeRepo(), + validatorRound: makeRepo(), + wallet: makeRepo(), + }; + context.repos.block.getLatestHeight = async () => 42; + + context.queryResults = { + blocksCount: [{ count: "0" }], + maxHeight: [{ count: "0", max_height: "0" }], + }; + + context.dataSource = { + createQueryRunner: () => ({ query: async () => {} }), + query: async (sql: string) => + sql.startsWith("select count(1)") ? context.queryResults.blocksCount : context.queryResults.maxHeight, + synchronize: async () => {}, + transaction: async (_level: string, callback: any) => callback(context.entityManager), + }; + + context.databaseService = { + getLastCommit: async () => ({ block: { number: 0 } }), + isEmpty: async () => true, + }; + + context.migrations = { runMigrations: async () => {}, synchronizeEntities: async () => {} }; + context.systemRepo = { inMaintenance: async () => false }; + context.evm = { getAccountInfo: async () => ({ balance: 1000n, nonce: 3n }) }; + context.state = { isBootstrap: () => false }; + context.validatorSet = { getDirtyValidators: () => [], getRoundValidators: () => [] }; + context.proposerCalculator = { getValidatorIndex: (index: number) => index }; + context.addressFactory = { fromPublicKey: async () => SENDER }; + context.configuration = { + getGenesisHeight: () => 0, + getMilestone: () => ({ evmSpec: "shanghai" }), + isNewMilestone: () => false, + }; + context.roundCalculator = { + calculateRound: (number: number) => ({ maxValidators: 2, round: number, roundHeight: number }), + isNewRound: () => false, + }; + context.logger = { debug: () => {}, debugExtra: () => {}, error: () => {}, info: () => {}, warn: () => {} }; + context.pluginConfiguration = { + getOptional: (key: string, defaultValue: unknown) => defaultValue, + getRequired: () => 1000, + }; + context.queue = { + drain: async () => {}, + jobs: [] as any[], + push(job: any) { + context.queue.jobs.push(job); + }, + start: async () => {}, + }; + context.listeners = { bootstrap: async () => {} }; + context.tokenParser = { parseReceipt: async () => ({ tokenActions: [], tokenHolders: [], tokens: [] }) }; + context.tokenWhitelist = { bootstrap: async () => {} }; + + context.app = new Application(); + context.app.bind(Identifiers.Application.Version).toConstantValue("0.0.1-test"); + context.app.bind(Identifiers.Cryptography.Identity.Address.Factory).toConstantValue(context.addressFactory); + context.app.bind(Identifiers.Cryptography.Configuration).toConstantValue(context.configuration); + context.app.bind(Identifiers.BlockchainUtils.RoundCalculator).toConstantValue(context.roundCalculator); + context.app.bind(ApiDatabaseIdentifiers.DataSource).toConstantValue(context.dataSource); + context.app.bind(Identifiers.Database.Service).toConstantValue(context.databaseService); + context.app.bind(ApiDatabaseIdentifiers.Migrations).toConstantValue(context.migrations); + context.app.bind(ApiDatabaseIdentifiers.BlockRepositoryFactory).toConstantValue(() => context.repos.block); + context.app + .bind(ApiDatabaseIdentifiers.ConfigurationRepositoryFactory) + .toConstantValue(() => context.repos.configuration); + context.app + .bind(ApiDatabaseIdentifiers.MultiPaymentRepositoryFactory) + .toConstantValue(() => context.repos.multiPayment); + context.app.bind(ApiDatabaseIdentifiers.StateRepositoryFactory).toConstantValue(() => context.repos.state); + context.app.bind(ApiDatabaseIdentifiers.SystemRepositoryFactory).toConstantValue(() => context.systemRepo); + context.app + .bind(ApiDatabaseIdentifiers.TransactionRepositoryFactory) + .toConstantValue(() => context.repos.transaction); + context.app.bind(ApiDatabaseIdentifiers.TokenRepositoryFactory).toConstantValue(() => context.repos.token); + context.app + .bind(ApiDatabaseIdentifiers.TokenHolderRepositoryFactory) + .toConstantValue(() => context.repos.tokenHolder); + context.app + .bind(ApiDatabaseIdentifiers.TokenActionRepositoryFactory) + .toConstantValue(() => context.repos.tokenAction); + context.app + .bind(ApiDatabaseIdentifiers.ValidatorRoundRepositoryFactory) + .toConstantValue(() => context.repos.validatorRound); + context.app.bind(ApiDatabaseIdentifiers.WalletRepositoryFactory).toConstantValue(() => context.repos.wallet); + context.app + .bind(ApiDatabaseIdentifiers.LegacyColdWalletRepositoryFactory) + .toConstantValue(() => context.repos.legacyColdWallet); + context.app.bind(Identifiers.Evm.Instance).toConstantValue(context.evm).whenTagged("instance", "evm"); + context.app.bind(Identifiers.EvmConsensus.Contracts.MultiPayment).toConstantValue("0xmultipayment"); + context.app.bind(Identifiers.State.State).toConstantValue(context.state); + context.app.bind(Identifiers.ValidatorSet.Service).toConstantValue(context.validatorSet); + context.app.bind(Identifiers.BlockchainUtils.ProposerCalculator).toConstantValue(context.proposerCalculator); + context.app.bind(Identifiers.ApiSync.Logger).toConstantValue(context.logger); + context.app + .bind(Identifiers.ServiceProvider.Configuration) + .toConstantValue(context.pluginConfiguration) + .whenTagged("plugin", "api-sync"); + context.app.bind(Identifiers.Services.Queue.Factory).toConstantValue(async () => context.queue); + context.app.bind(Identifiers.ApiSync.Listener).toConstantValue(context.listeners); + context.app.bind(Identifiers.ApiSync.TokenParser).toConstantValue(context.tokenParser); + context.app.bind(Identifiers.ApiSync.TokenWhitelist).toConstantValue(context.tokenWhitelist); + + context.app.config("crypto.genesisBlock", { block: { number: 0, proposer: PROPOSER } }); + + context.sync = context.app.resolve(Sync); + + context.makeUnit = (overrides = {}) => { + const header = { ...makeHeader(), ...(overrides.header ?? {}) }; + const transactions = overrides.transactions ?? [makeTransaction()]; + const receipts = + overrides.receipts ?? + new Map(transactions.map((transaction: any) => [transaction.hash, makeReceipt()])); + + return { + blockNumber: header.number, + getAccountUpdates: () => overrides.accountUpdates ?? [], + getCommit: async () => ({ + block: { ...header, transactions }, + proof: { round: 0, signature: "0xproof", validators: [true, false] }, + }), + getProcessorResult: () => ({ receipts }), + }; + }; + }); + + const runQueuedJob = async (context: Ctx) => { + assert.equal(context.queue.jobs.length, 1); + await context.queue.jobs[0].handle(); + }; + + it("bootstrap: restores from scratch when the api database is empty", async (context) => { + const { app, migrations, listeners, tokenWhitelist, queue, sync } = context; + + const restore = { restore: async () => {} }; + const restoreSpy = spy(restore, "restore"); + const appResolve = stub(app, "resolve").returnValue(restore); + const synchronizeEntities = spy(migrations, "synchronizeEntities"); + const runMigrations = spy(migrations, "runMigrations"); + const listenersBootstrap = spy(listeners, "bootstrap"); + const whitelistBootstrap = spy(tokenWhitelist, "bootstrap"); + const queueStart = spy(queue, "start"); + + await sync.bootstrap(); + + synchronizeEntities.calledOnce(); + restoreSpy.calledOnce(); + runMigrations.neverCalled(); + listenersBootstrap.calledOnce(); + whitelistBootstrap.calledOnce(); + queueStart.calledOnce(); + + appResolve.restore(); + }); + + it("bootstrap: runs migrations instead of a restore when blocks are present and consistent", async (context) => { + const { dataSource, databaseService, migrations, sync } = context; + + databaseService.isEmpty = async () => false; + databaseService.getLastCommit = async () => ({ block: { number: 5 } }); + context.queryResults.blocksCount = [{ count: "5" }]; + context.queryResults.maxHeight = [{ count: "5", max_height: "5" }]; + + const synchronize = spy(dataSource, "synchronize"); + const runMigrations = spy(migrations, "runMigrations"); + + await sync.bootstrap(); + + synchronize.neverCalled(); + runMigrations.calledOnce(); + }); + + it("bootstrap: clears the database when storage and api database heights mismatch", async (context) => { + const { dataSource, databaseService, logger, sync } = context; + + databaseService.isEmpty = async () => false; + databaseService.getLastCommit = async () => ({ block: { number: 5 } }); + context.queryResults.blocksCount = [{ count: "3" }]; + context.queryResults.maxHeight = [{ count: "3", max_height: "3" }]; + + const restore = { restore: async () => {} }; + const appResolve = stub(context.app, "resolve").returnValue(restore); + const synchronize = spy(dataSource, "synchronize"); + const warn = spy(logger, "warn"); + + await sync.bootstrap(); + + warn.calledWith("Clearing API database for full restore."); + synchronize.calledWith(true); + + appResolve.restore(); + }); + + it("bootstrap: clears the database when a truncate is forced", async (context) => { + const { dataSource, pluginConfiguration, sync } = context; + + pluginConfiguration.getOptional = (key: string, defaultValue: unknown) => + key === "truncateDatabase" ? true : defaultValue; + + const restore = { restore: async () => {} }; + const appResolve = stub(context.app, "resolve").returnValue(restore); + const synchronize = spy(dataSource, "synchronize"); + + await sync.bootstrap(); + + synchronize.calledWith(true); + + appResolve.restore(); + }); + + it("bootstrap: clears the database when a previous restore left maintenance mode on", async (context) => { + const { dataSource, databaseService, systemRepo, logger, sync } = context; + + // Heights are consistent, but the maintenance flag signals an interrupted restore. + databaseService.isEmpty = async () => false; + databaseService.getLastCommit = async () => ({ block: { number: 5 } }); + context.queryResults.blocksCount = [{ count: "5" }]; + context.queryResults.maxHeight = [{ count: "5", max_height: "5" }]; + systemRepo.inMaintenance = async () => true; + + const restore = { restore: async () => {} }; + const appResolve = stub(context.app, "resolve").returnValue(restore); + const synchronize = spy(dataSource, "synchronize"); + const warn = spy(logger, "warn"); + + await sync.bootstrap(); + + warn.calledWith("Clearing API database for full restore."); + synchronize.calledWith(true); + + appResolve.restore(); + }); + + it("bootstrap: terminates the application when the database reset fails", async (context) => { + const { app, dataSource, sync } = context; + + dataSource.synchronize = async () => { + throw new Error("synchronize failed"); + }; + const restore = { restore: async () => {} }; + const appResolve = stub(app, "resolve").returnValue(restore); + const terminate = stub(app, "terminate").resolvedValue(undefined); + + await sync.bootstrap(); + + terminate.calledOnce(); + assert.equal(terminate.getCallArgs(0)[0], "failed to reset database"); + + appResolve.restore(); + terminate.restore(); + }); + + it("flush: drains the queue", async (context) => { + const { queue, sync } = context; + const restore = { restore: async () => {} }; + const appResolve = stub(context.app, "resolve").returnValue(restore); + await sync.bootstrap(); + appResolve.restore(); + + const drain = spy(queue, "drain"); + + await sync.flush(); + + drain.calledOnce(); + }); + + it("getLastSyncedBlockHeight: returns the latest height from the block repository", async ({ sync }) => { + assert.equal(await sync.getLastSyncedBlockHeight(), 42); + }); + + it("getLastSyncedBlockHeight: falls back to the genesis height", async (context) => { + context.repos.block.getLatestHeight = async () => undefined; + + assert.equal(await context.sync.getLastSyncedBlockHeight(), 0); + }); + + it("onCommit: persists the block, transaction, state and wallets", async (context) => { + const { repos, entityManager, logger, sync } = context; + const restore = { restore: async () => {} }; + const appResolve = stub(context.app, "resolve").returnValue(restore); + await sync.bootstrap(); + appResolve.restore(); + + const debug = spy(logger, "debug"); + + await sync.onCommit(context.makeUnit()); + await runQueuedJob(context); + + // Block insert + assert.equal(repos.block.qb.calls.values.length, 1); + const block = repos.block.qb.calls.values[0][0]; + assert.equal(block.number, "1"); + assert.equal(block.hash, "0xblockhash"); + assert.equal(block.fee, "100"); + assert.equal(block.reward, "200"); + assert.equal(block.proposer, PROPOSER); + assert.equal(block.signature, "0xproof"); + assert.equal(block.validatorRound, 1); + + // State supply update anchored on the previous block number + assert.equal(repos.state.qb.calls.set[0][0].blockNumber, "1"); + assert.equal(repos.state.qb.calls.andWhere[0][1], { previousBlockNumber: "0" }); + + // Transaction insert + const [transactionBatch] = repos.transaction.qb.calls.values[0]; + assert.equal(transactionBatch.length, 1); + assert.equal(transactionBatch[0].hash, "0xtx1"); + assert.equal(transactionBatch[0].blockHash, "0xblockhash"); + assert.equal(transactionBatch[0].senderPublicKey, SENDER_PUBLIC_KEY); + // formatEcdsaSignature concatenates r || s || v (hex) + assert.equal(transactionBatch[0].signature, "aabb1b"); + assert.equal(transactionBatch[0].nonce, "1"); + assert.undefined(transactionBatch[0].decodedError); + + // The genesis proposer account is force-created on the first non-genesis block, + // and the block proposer (not part of any account update) is manually inserted. + assert.equal(repos.wallet.queries.length, 1); + const [sql, parameters] = repos.wallet.queries[0]; + assert.true(sql.includes("INSERT INTO wallets")); + // genesis proposer row + no separate proposer row (same address) + assert.equal(parameters.length, 6); + assert.equal(parameters[0], PROPOSER); + assert.equal(parameters[2], "1000"); // balance from evm.getAccountInfo + + // Configuration version update without new milestones + assert.equal(repos.configuration.qb.calls.set[0][0].version, "0.0.1-test"); + assert.undefined(repos.configuration.qb.calls.set[0][0].activeMilestones); + + // Validator ranks are refreshed at the end of the transaction + assert.equal(entityManager.queries[0][0], "SELECT update_validator_ranks();"); + + debug.calledOnce(); + }); + + it("onCommit: stores the decoded error of a failed transaction", async (context) => { + const { repos, sync } = context; + const restore = { restore: async () => {} }; + const appResolve = stub(context.app, "resolve").returnValue(restore); + await sync.bootstrap(); + appResolve.restore(); + + const transaction = makeTransaction(); + // Reverted without output while consuming the full gas limit -> "out of gas". + const receipt = { ...makeReceipt(), gasUsed: 21_000n, status: 0 }; + + await sync.onCommit( + context.makeUnit({ receipts: new Map([[transaction.hash, receipt]]), transactions: [transaction] }), + ); + await runQueuedJob(context); + + const [transactionBatch] = repos.transaction.qb.calls.values[0]; + assert.equal(transactionBatch[0].status, 0); + assert.equal(transactionBatch[0].decodedError, "out of gas"); + }); + + it("onCommit: does not log sync progress during bootstrap", async (context) => { + const { logger, state, sync } = context; + const restore = { restore: async () => {} }; + const appResolve = stub(context.app, "resolve").returnValue(restore); + await sync.bootstrap(); + appResolve.restore(); + + state.isBootstrap = () => true; + const debug = spy(logger, "debug"); + + await sync.onCommit(context.makeUnit()); + await runQueuedJob(context); + + debug.neverCalled(); + }); + + it("onCommit: maps account updates including votes, usernames and legacy merges", async (context) => { + const { repos, sync } = context; + const restore = { restore: async () => {} }; + const appResolve = stub(context.app, "resolve").returnValue(restore); + await sync.bootstrap(); + appResolve.restore(); + + const accountUpdates = [ + { address: PROPOSER, balance: 50n, nonce: 1n, vote: "0xvalidator" }, + { address: "0xunvoter", balance: 10n, nonce: 2n, unvote: "0xvalidator" }, + { address: "0xnamed", balance: 20n, nonce: 3n, username: "alice" }, + { address: "0xresigned", balance: 30n, nonce: 4n, usernameResigned: true }, + { + address: "0xmerged", + balance: 40n, + legacyMergeInfo: { address: "LEGACY_ADDR", txHash: "0xmerge" }, + nonce: 5n, + }, + ]; + + await sync.onCommit(context.makeUnit({ accountUpdates })); + await runQueuedJob(context); + + const [, parameters] = repos.wallet.queries[0]; + const rows: any[][] = []; + for (let index = 0; index < parameters.length; index += 6) { + rows.push(parameters.slice(index, index + 6)); + } + + const byAddress = new Map(rows.map((row) => [row[0], row])); + assert.equal(byAddress.get(PROPOSER)![4].vote, "0xvalidator"); + assert.equal(byAddress.get("0xunvoter")![4].unvote, "0xvalidator"); + assert.equal(byAddress.get("0xnamed")![4].username, "alice"); + assert.true(byAddress.get("0xresigned")![4].usernameResigned); + assert.equal(byAddress.get("0xmerged")![4].legacyMerge, { address: "LEGACY_ADDR", txHash: "0xmerge" }); + + // The proposer receives forged attributes because it produced the block. + assert.equal(byAddress.get(PROPOSER)![4].validatorForgedTotal, "300"); + assert.equal(byAddress.get(PROPOSER)![4].validatorProducedBlocks, 1); + + // The merged legacy cold wallet is updated with the merge info. + assert.equal(repos.legacyColdWallet.qb.calls.set[0][0], { + mergeInfoTransactionHash: "0xmerge", + mergeInfoWalletAddress: "0xmerged", + }); + assert.equal(repos.legacyColdWallet.qb.calls.where[0][1], { legacyAddress: "LEGACY_ADDR" }); + }); + + it("onCommit: inserts dirty validators that are not part of the account updates", async (context) => { + const { repos, validatorSet, sync } = context; + const restore = { restore: async () => {} }; + const appResolve = stub(context.app, "resolve").returnValue(restore); + await sync.bootstrap(); + appResolve.restore(); + + validatorSet.getDirtyValidators = () => [ + { + address: "0xdirty", + blsPublicKey: "0xbls", + fee: 7n, + isResigned: false, + voteBalance: 999n, + votersCount: 3, + }, + ]; + + await sync.onCommit(context.makeUnit()); + await runQueuedJob(context); + + const [, parameters] = repos.wallet.queries[0]; + const rows: any[][] = []; + for (let index = 0; index < parameters.length; index += 6) { + rows.push(parameters.slice(index, index + 6)); + } + const dirtyRow = rows.find((row) => row[0] === "0xdirty"); + assert.defined(dirtyRow); + // Sentinel balance/nonce so postgres keeps the existing values. + assert.equal(dirtyRow![2], "-1"); + assert.equal(dirtyRow![3], "-1"); + assert.equal(dirtyRow![4].validatorPublicKey, "0xbls"); + assert.equal(dirtyRow![4].validatorVoteBalance, "999"); + assert.equal(dirtyRow![4].validatorVotersCount, 3); + assert.equal(dirtyRow![4].validatorFee, "7"); + assert.false(dirtyRow![4].validatorResigned); + }); + + it("onCommit: stores the upcoming validator round on round boundaries", async (context) => { + const { repos, roundCalculator, validatorSet, proposerCalculator, sync } = context; + const restore = { restore: async () => {} }; + const appResolve = stub(context.app, "resolve").returnValue(restore); + await sync.bootstrap(); + appResolve.restore(); + + roundCalculator.isNewRound = (number: number) => number === 2; + validatorSet.getRoundValidators = () => [ + { address: "0xval-a", voteBalance: 10n }, + { address: "0xval-b", voteBalance: 20n }, + ]; + // Reverse the proposal order to prove the proposer calculator is honored. + proposerCalculator.getValidatorIndex = (index: number) => 1 - index; + + await sync.onCommit(context.makeUnit()); + await runQueuedJob(context); + + assert.equal(repos.validatorRound.qb.calls.values.length, 1); + const round = repos.validatorRound.qb.calls.values[0][0]; + assert.equal(round.validators, ["0xval-b", "0xval-a"]); + assert.equal(round.votes, ["20", "10"]); + }); + + it("onCommit: stores new milestones when the next block starts one", async (context) => { + const { repos, configuration, sync } = context; + const restore = { restore: async () => {} }; + const appResolve = stub(context.app, "resolve").returnValue(restore); + await sync.bootstrap(); + appResolve.restore(); + + configuration.isNewMilestone = (number: number) => number === 2; + configuration.getMilestone = () => ({ evmSpec: "shanghai", height: 2 }); + + await sync.onCommit(context.makeUnit()); + await runQueuedJob(context); + + assert.equal(repos.configuration.qb.calls.set[0][0].activeMilestones, { evmSpec: "shanghai", height: 2 }); + }); + + it("onCommit: upserts token entities and removes zeroed holders", async (context) => { + const { repos, tokenParser, sync } = context; + const restore = { restore: async () => {} }; + const appResolve = stub(context.app, "resolve").returnValue(restore); + await sync.bootstrap(); + appResolve.restore(); + + tokenParser.parseReceipt = async () => ({ + tokenActions: [ + { + action: "Transfer", + address: "0xtoken", + blockNumber: "1", + from: "0xa", + index: 0, + to: "0xb", + transactionHash: "0xtx1", + value: "5", + }, + ], + tokenHolders: [ + { address: "0xa", balance: "0", tokenAddress: "0xtoken" }, + { address: "0xb", balance: "5", tokenAddress: "0xtoken" }, + ], + tokens: [{ address: "0xtoken", decimals: 18, name: "Token", symbol: "TKN", totalSupply: "100" }], + }); + + await sync.onCommit(context.makeUnit()); + await runQueuedJob(context); + + assert.equal(repos.token.qb.calls.values[0][0].length, 1); + assert.equal(repos.tokenAction.qb.calls.values[0][0].length, 1); + + // Holder with balance 0 is deleted, the other upserted. + assert.equal(repos.tokenHolder.removed[0], [{ address: "0xa", balance: "0", tokenAddress: "0xtoken" }]); + assert.equal(repos.tokenHolder.qb.calls.values[0][0], [ + { address: "0xb", balance: "5", tokenAddress: "0xtoken" }, + ]); + }); + + it("onCommit: retries a failed sync until it succeeds", async (context) => { + const { dataSource, entityManager, logger, sync } = context; + const restore = { restore: async () => {} }; + const appResolve = stub(context.app, "resolve").returnValue(restore); + await sync.bootstrap(); + appResolve.restore(); + + const clk = clock(); + const warn = spy(logger, "warn"); + + let failures = 1; + const transaction = dataSource.transaction.bind(dataSource); + dataSource.transaction = async (level: string, callback: any) => { + if (failures > 0) { + failures--; + throw new Error("deadlock"); + } + return transaction(level, callback); + }; + + await sync.onCommit(context.makeUnit()); + + const pending = context.queue.jobs[0].handle(); + await clk.runAllAsync(); + await pending; + + warn.calledOnce(); + assert.true(warn.getCallArgs(0)[0].includes("deadlock")); + // The retry ultimately succeeded and refreshed the validator ranks. + assert.equal(entityManager.queries[0][0], "SELECT update_validator_ranks();"); + }); + + it("onCommit: stops retrying once the configured attempts are exhausted", async (context) => { + const { dataSource, entityManager, logger, pluginConfiguration, sync } = context; + const restore = { restore: async () => {} }; + const appResolve = stub(context.app, "resolve").returnValue(restore); + await sync.bootstrap(); + appResolve.restore(); + + pluginConfiguration.getOptional = (key: string, defaultValue: unknown) => + key === "maxSyncAttempts" ? 1 : defaultValue; + + const clk = clock(); + const warn = spy(logger, "warn"); + + dataSource.transaction = async () => { + throw new Error("permanent failure"); + }; + + await sync.onCommit(context.makeUnit()); + + const pending = context.queue.jobs[0].handle(); + await clk.runAllAsync(); + await pending; + + warn.calledOnce(); + assert.equal(entityManager.queries.length, 0); + }); +}); diff --git a/packages/api-sync/source/service.ts b/packages/api-sync/source/service.ts index 04d804218..b3f59b4d6 100644 --- a/packages/api-sync/source/service.ts +++ b/packages/api-sync/source/service.ts @@ -449,11 +449,15 @@ export class Sync implements Contracts.ApiSync.Service { await this.#syncToDatabase(deferredSync); success = true; } catch (rawError) { + attempts++; + const error = ensureError(rawError); const nextAttemptDelay = Math.min(baseDelay + attempts * 500, maxDelay); - attempts++; + const { query } = error as { query?: string }; + const querySuffix = query ? ` (query: ${query})` : ""; + this.logger.warn( - `sync encountered exception: ${error.message} (query: ${(error as { query?: string }).query}). retry #${attempts} in ... ${nextAttemptDelay}ms`, + `sync encountered exception: ${error.message}${querySuffix}. retry #${attempts} in ${nextAttemptDelay}ms`, ); await sleep(nextAttemptDelay); } diff --git a/packages/api-sync/source/tokens/whitelist.test.ts b/packages/api-sync/source/tokens/whitelist.test.ts new file mode 100644 index 000000000..1e05ecbe9 --- /dev/null +++ b/packages/api-sync/source/tokens/whitelist.test.ts @@ -0,0 +1,225 @@ +import { Identifiers as ApiDatabaseIdentifiers, Models } from "@mainsail/api-database"; +import { Identifiers } from "@mainsail/constants"; +import { Application } from "@mainsail/kernel"; +import { describe } from "@mainsail/test-runner"; +import { http } from "@mainsail/utils"; + +import { TokenWhitelist } from "./whitelist.js"; + +const REMOTE_URL = "https://tokens.example.org/whitelist.json"; + +const TOKEN_ADDRESS = "0x1111111111111111111111111111111111111111"; +const VALID_TIMESTAMP = "2026-02-11T14:25:00.000Z"; + +const response = (data: unknown) => ({ + data, + headers: [], + method: "GET", + statusCode: 200, + statusMessage: "OK", +}); + +type Ctx = { + app: Application; + whitelist: TokenWhitelist; + entityManager: { clear: any; save: any }; + dataSource: { transaction: any }; + logger: Record; + addressFactory: { validate: any }; + remoteUrl: string; +}; + +describe("TokenWhitelist", ({ it, beforeEach, afterEach, assert, spy, stub }) => { + beforeEach((context) => { + context.remoteUrl = REMOTE_URL; + context.entityManager = { clear: async () => {}, save: async () => {} }; + context.dataSource = { + transaction: async (callback: any) => callback(context.entityManager), + }; + context.logger = { debug: () => {}, debugExtra: () => {}, error: () => {}, info: () => {} }; + context.addressFactory = { validate: async () => true }; + + context.app = new Application(); + context.app + .bind(Identifiers.ServiceProvider.Configuration) + .toConstantValue({ + getRequired: (key: string) => (key === "tokenWhitelistRemoteUrl" ? context.remoteUrl : 3_600_000), + }) + .whenTagged("plugin", "api-sync"); + context.app.bind(ApiDatabaseIdentifiers.DataSource).toConstantValue(context.dataSource); + context.app.bind(Identifiers.ApiSync.Logger).toConstantValue(context.logger); + context.app.bind(Identifiers.Cryptography.Identity.Address.Factory).toConstantValue(context.addressFactory); + + context.whitelist = context.app.resolve(TokenWhitelist); + }); + + afterEach(async ({ whitelist }) => { + await whitelist.dispose(); + }); + + const tick = async (): Promise => + new Promise((resolve) => { + setImmediate(resolve); + }); + + // bootstrap() fires the sync loop without awaiting it; wait for its observable effect. + const waitFor = async (predicate: () => boolean): Promise => { + for (let attempt = 0; attempt < 100 && !predicate(); attempt++) { + await tick(); + } + + assert.true(predicate()); + + // Let the loop reach its `finally` and schedule the follow-up timer, so the + // dispose() in afterEach reliably clears it and the process can exit. + await tick(); + await tick(); + }; + + it("replaces the stored whitelist with the fetched one", async ({ whitelist, entityManager, logger }) => { + const get = stub(http, "get").resolvedValue( + response([{ address: TOKEN_ADDRESS, comment: " a comment ", createdAt: VALID_TIMESTAMP }]), + ); + + const info = spy(logger, "info"); + const clear = spy(entityManager, "clear"); + const save = spy(entityManager, "save"); + + await whitelist.bootstrap(); + await waitFor(() => save.toFunction().callCount > 0); + + info.calledWith(`Starting TokenWhitelist using remote: ${REMOTE_URL}`); + get.calledWith(REMOTE_URL, { maxContentLength: 16 * 1024, timeout: 2500 }); + clear.calledWith(Models.TokenWhitelist); + save.calledOnce(); + assert.equal(save.getCallArgs(0), [ + Models.TokenWhitelist, + [{ address: TOKEN_ADDRESS, comment: "a comment", createdAt: VALID_TIMESTAMP }], + { chunk: 1000 }, + ]); + }); + + it("parses a whitelist served as a raw string", async ({ whitelist, entityManager }) => { + stub(http, "get").resolvedValue( + response(JSON.stringify([{ address: TOKEN_ADDRESS, createdAt: VALID_TIMESTAMP }])), + ); + + const save = spy(entityManager, "save"); + + await whitelist.bootstrap(); + await waitFor(() => save.toFunction().callCount > 0); + + assert.equal(save.getCallArgs(0)[1], [{ address: TOKEN_ADDRESS, createdAt: VALID_TIMESTAMP }]); + }); + + it("drops tokens with a malformed address", async ({ whitelist, entityManager, addressFactory, logger }) => { + stub(http, "get").resolvedValue( + response([ + { address: "not-an-address", createdAt: VALID_TIMESTAMP }, + { address: TOKEN_ADDRESS, createdAt: VALID_TIMESTAMP }, + ]), + ); + + addressFactory.validate = async (address: string) => address === TOKEN_ADDRESS; + const debugExtra = spy(logger, "debugExtra"); + const save = spy(entityManager, "save"); + + await whitelist.bootstrap(); + await waitFor(() => save.toFunction().callCount > 0); + + assert.equal(save.getCallArgs(0)[1], [{ address: TOKEN_ADDRESS, createdAt: VALID_TIMESTAMP }]); + debugExtra.calledOnce(); + }); + + it("drops tokens with a malformed timestamp", async ({ whitelist, entityManager, logger }) => { + stub(http, "get").resolvedValue( + response([ + { address: TOKEN_ADDRESS, createdAt: "yesterday" }, + { address: TOKEN_ADDRESS, createdAt: "2026-02-11T14:25:00Z" }, // missing milliseconds + { address: TOKEN_ADDRESS, createdAt: VALID_TIMESTAMP }, + ]), + ); + + const debugExtra = spy(logger, "debugExtra"); + const save = spy(entityManager, "save"); + + await whitelist.bootstrap(); + await waitFor(() => save.toFunction().callCount > 0); + + assert.equal(save.getCallArgs(0)[1], [{ address: TOKEN_ADDRESS, createdAt: VALID_TIMESTAMP }]); + debugExtra.calledTimes(2); + }); + + it("drops tokens whose validation throws", async ({ whitelist, entityManager, addressFactory, logger }) => { + stub(http, "get").resolvedValue( + response([ + { address: "0xboom", createdAt: VALID_TIMESTAMP }, + { address: TOKEN_ADDRESS, createdAt: VALID_TIMESTAMP }, + ]), + ); + + addressFactory.validate = async (address: string) => { + if (address === "0xboom") { + throw new Error("validator crashed"); + } + return true; + }; + const debugExtra = spy(logger, "debugExtra"); + const save = spy(entityManager, "save"); + + await whitelist.bootstrap(); + await waitFor(() => save.toFunction().callCount > 0); + + assert.equal(save.getCallArgs(0)[1], [{ address: TOKEN_ADDRESS, createdAt: VALID_TIMESTAMP }]); + debugExtra.calledOnce(); + }); + + it("does nothing without a configured remote", async (context) => { + context.remoteUrl = ""; + + const get = stub(http, "get").resolvedValue(response([])); + const transaction = spy(context.dataSource, "transaction"); + const error = spy(context.logger, "error"); + + await context.whitelist.bootstrap(); + // Allow the fire-and-forget run to settle and schedule its follow-up timer. + await tick(); + await tick(); + + get.neverCalled(); + transaction.neverCalled(); + error.neverCalled(); + }); + + it("logs fetch failures without updating the whitelist", async ({ whitelist, dataSource, logger }) => { + stub(http, "get").rejectedValue(new Error("connection refused")); + + const transaction = spy(dataSource, "transaction"); + const error = spy(logger, "error"); + + await whitelist.bootstrap(); + await waitFor(() => error.toFunction().callCount > 0); + + assert.true(error.getCallArgs(0)[0].startsWith("fetchWhitelist failed")); + assert.true(error.getCallArgs(0)[0].includes("connection refused")); + transaction.neverCalled(); + }); + + it("logs sync failures raised by the database transaction", async ({ whitelist, dataSource, logger }) => { + stub(http, "get").resolvedValue(response([{ address: TOKEN_ADDRESS, createdAt: VALID_TIMESTAMP }])); + + dataSource.transaction = async () => { + throw new Error("database offline"); + }; + const error = spy(logger, "error"); + + await whitelist.bootstrap(); + await waitFor(() => error.toFunction().callCount > 0); + + assert.true(error.getCallArgs(0)[0].startsWith("#syncWhitelist failed")); + }); + + it("dispose: can be called before bootstrap", async ({ whitelist }) => { + await assert.resolves(() => whitelist.dispose()); + }); +}); diff --git a/packages/api-sync/source/tokens/whitelist.ts b/packages/api-sync/source/tokens/whitelist.ts index 7388f3d16..6f83231bf 100644 --- a/packages/api-sync/source/tokens/whitelist.ts +++ b/packages/api-sync/source/tokens/whitelist.ts @@ -84,11 +84,13 @@ export class TokenWhitelist { } try { - const { data } = await http.get(this.#getTokenWhitelistRemoteUrl(), { + const { data } = await http.get(this.#getTokenWhitelistRemoteUrl(), { maxContentLength: 16 * Units.KILOBYTE, timeout: 2500, }); - return JSON.parse(data) as WhitelistedToken[]; + // http.get already JSON-parses responses served as application/json, so only + // parse when the body came back as a raw string (e.g. text/plain). + return (typeof data === "string" ? JSON.parse(data) : data) as WhitelistedToken[]; } catch (rawError) { const error = ensureError(rawError); this.logger.error(`fetchWhitelist failed: ${error}`);