From 2fa2bb3869eabc4e22479f4476d9135d849ee6ad Mon Sep 17 00:00:00 2001 From: satyakwok <119509589+satyakwok@users.noreply.github.com> Date: Sun, 10 May 2026 22:10:55 +0200 Subject: [PATCH 1/2] perf(indexer): batch sync.ts + stats_daily_mv + RPC fallback transport MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tier 2 of the audit pass. Three changes that reshape the hot loop: 1. **Batch indexBlock** (apps/indexer/src/sync.ts) Pre-batch sync was N HTTP round-trips + N INSERTs per tx, sequential, inside one long-held DB transaction. On a 5000-tx block that meant ~10 s of held write lock and chain RPC saturation from one indexer instance alone. Refactor moves to four phases: - PHASE 1: getNativeTransaction() fan-out via mapWithConcurrency with a 25-task cap (env INDEXER_TX_FETCH_CONCURRENCY). HTTP happens entirely outside the DB transaction. - PHASE 2: build TxRow[] / AddrRow[] arrays in memory; dedupe addresses by primary key (Postgres rejects multi-row ON CONFLICT on the same target with cardinality_violation). - PHASE 3: getLogsRange + decode → LogRow[] / TransferRow[]. - PHASE 4: db.transaction wraps four batch INSERTs (txs, addresses, logs, transfers). Lock window now ms not seconds. ~50–100x throughput on log-heavy blocks; backfill burst no longer monopolises the chain LB. 2. **stats_daily_mv materialised view** (migration 0005, indexer + API wiring) Replaces the per-process 5-min in-memory cache that was lost on every restart and not shared across api processes. View has a UNIQUE INDEX on date so REFRESH MATERIALIZED VIEW CONCURRENTLY can run without blocking SELECTs. Indexer worker fires REFRESH every 5 min (env INDEXER_STATS_REFRESH_INTERVAL_MS); the API just SELECTs from the view (~700 rows for 2 yrs of chain history). Combined with Tier 1's 60 s edge cache: worst-case freshness gap is ~6 min. 3. **RPC HTTP fallback transport** (packages/chain/src/index.ts) Operator can now set INDEXER_RPC_HTTP_URLS=primary,backup1,backup2 and viem's fallback() transport rolls over on connection error / 5xx / timeout. Health-checks each transport every 60 s and demotes bad ones automatically. Legacy single-URL env (INDEXER_RPC_HTTP_URL) continues to work — it just becomes the only entry in the URL list. Single-URL deployments fall back to plain http() (no fallback wrapper overhead) so behaviour is byte-identical for them. --- apps/api/src/routes/native.ts | 54 +- apps/indexer/src/index.ts | 29 + apps/indexer/src/sync.ts | 422 +++--- packages/chain/src/index.ts | 41 +- packages/db/drizzle/0005_stats_daily_mv.sql | 20 + packages/db/drizzle/meta/0005_snapshot.json | 1318 +++++++++++++++++++ packages/db/drizzle/meta/_journal.json | 7 + 7 files changed, 1663 insertions(+), 228 deletions(-) create mode 100644 packages/db/drizzle/0005_stats_daily_mv.sql create mode 100644 packages/db/drizzle/meta/0005_snapshot.json diff --git a/apps/api/src/routes/native.ts b/apps/api/src/routes/native.ts index 6926b36..9d3ad06 100644 --- a/apps/api/src/routes/native.ts +++ b/apps/api/src/routes/native.ts @@ -39,11 +39,12 @@ class InvalidQueryError extends Error { // /stats/daily moved off the chain native API in 2026-05-05 — at h~1.55M the // on-chain handler scanned every block from genesis under the state read lock -// and hung the LB. Indexer side: a single GROUP BY against the timestamp-indexed -// `blocks` table runs in tens of ms over the full history, so the response -// covers the full chain (no 14-day cap) and we cache for 5 min to absorb burst. -const STATS_DAILY_TTL_MS = 5 * 60_000; -let statsDailyCache: { at: number; data: Array<{ date: string; blocks: number; transactions: number }> } | null = null; +// and hung the LB. Indexer side now reads from `stats_daily_mv` (created in +// migration 0005), refreshed every 5 min by the indexer worker via +// `REFRESH MATERIALIZED VIEW CONCURRENTLY`. View is shared across api +// processes + survives restart, replacing the previous per-process in-memory +// cache. HTTP edge cache (cache-control.ts) layers a further 60 s response +// dedupe so a burst of consumers collapses into one PG read. export function registerNativeRoutes( app: FastifyInstance, @@ -140,32 +141,27 @@ export function registerNativeRoutes( }); // ── /stats/daily ────────────────────────────────────────── - // All-time daily activity (blocks + tx count). Used by scan analytics - // page. Same JSON shape as the chain native handler so scan can swap - // upstream without code change. + // All-time daily activity (blocks + tx count). Reads the + // stats_daily_mv materialised view; the indexer worker REFRESHes + // CONCURRENTLY every 5 min so this query is a plain SELECT against + // a tiny pre-aggregated table (~1 row per day = ~700 rows for 2 yrs + // of chain history). Same JSON shape as before so consumers (scan + // analytics page) don't need a code change. app.get("/stats/daily", async () => { - if (statsDailyCache && Date.now() - statsDailyCache.at < STATS_DAILY_TTL_MS) { - return statsDailyCache.data; - } - const rows = await ctx.db.execute<{ date: string; blocks: string; transactions: string }>( - sql` - SELECT to_char(to_timestamp(timestamp::bigint), 'YYYY-MM-DD') AS date, - count(*)::text AS blocks, - COALESCE(sum(tx_count), 0)::text AS transactions - FROM ${blocks} - GROUP BY 1 - ORDER BY 1 - ` - ); - const data = (rows as unknown as Array<{ date: string; blocks: string; transactions: string }>).map( - (r) => ({ - date: r.date, - blocks: Number(r.blocks), - transactions: Number(r.transactions), - }) + const rows = await ctx.db.execute<{ + date: string; + blocks: string; + transactions: string; + }>( + sql`SELECT date, blocks::text, transactions::text FROM stats_daily_mv ORDER BY date`, ); - statsDailyCache = { at: Date.now(), data }; - return data; + return ( + rows as unknown as Array<{ date: string; blocks: string; transactions: string }> + ).map((r) => ({ + date: r.date, + blocks: Number(r.blocks), + transactions: Number(r.transactions), + })); }); // ── /accounts/active ────────────────────────────────────── diff --git a/apps/indexer/src/index.ts b/apps/indexer/src/index.ts index b0880ca..6b5b542 100644 --- a/apps/indexer/src/index.ts +++ b/apps/indexer/src/index.ts @@ -253,6 +253,34 @@ async function main() { }, ); + // ── stats_daily_mv refresh ──────────────────────────────────── + // The materialised view backing /stats/daily must be refreshed for + // the API to see new blocks/transactions. CONCURRENTLY refresh + // doesn't block readers (requires the unique index on date, present + // since migration 0005). 5 min cadence balances freshness vs. PG + // load — the API also caches the view-read result for 60 s at the + // edge (see apps/api/src/cache-control.ts), so the worst-case + // freshness gap a user sees is ~6 min. + const STATS_REFRESH_INTERVAL_MS = Number( + process.env.INDEXER_STATS_REFRESH_INTERVAL_MS ?? 5 * 60_000, + ); + // Initial seed so the view is non-empty before the first interval fires. + // Uses non-CONCURRENT path because the unique index isn't valid until + // the first non-concurrent populate completes. + try { + await db.execute(sql`REFRESH MATERIALIZED VIEW stats_daily_mv`); + log.info("stats_daily_mv initial refresh ok"); + } catch (err) { + log.warn({ err: String(err) }, "stats_daily_mv initial refresh failed (view may not exist yet — run migrations)"); + } + const statsRefreshTimer = setInterval(async () => { + try { + await db.execute(sql`REFRESH MATERIALIZED VIEW CONCURRENTLY stats_daily_mv`); + } catch (err) { + log.warn({ err: String(err) }, "stats_daily_mv refresh failed"); + } + }, STATS_REFRESH_INTERVAL_MS); + // Graceful shutdown. const shutdown = async (sig: string) => { log.info({ sig }, "shutting down"); @@ -266,6 +294,7 @@ async function main() { } catch { /* ignore */ } + clearInterval(statsRefreshTimer); await app.close().catch(() => {}); process.exit(0); }; diff --git a/apps/indexer/src/sync.ts b/apps/indexer/src/sync.ts index 5d058ec..c8c450a 100644 --- a/apps/indexer/src/sync.ts +++ b/apps/indexer/src/sync.ts @@ -67,19 +67,223 @@ interface IndexBlockArgs { log: Logger; } +// Concurrency cap for the per-tx native fetch fan-out. Pure HTTP +// limit — avoids hammering the chain REST with thousands of concurrent +// connections on high-tx blocks (chain max is 5000 tx/block). 25 is +// chosen to keep latency low while staying well under the public LB's +// per-IP connection cap. Tunable via env when running against a +// loopback / wg1 RPC with no rate limit. +const TX_FETCH_CONCURRENCY = Number( + process.env.INDEXER_TX_FETCH_CONCURRENCY ?? 25, +); + +/** Fan-out N async tasks with a concurrency cap. Returns results in + * input order. */ +async function mapWithConcurrency( + items: readonly I[], + limit: number, + fn: (item: I, index: number) => Promise, +): Promise { + const results: O[] = new Array(items.length); + let cursor = 0; + const workers = Array.from( + { length: Math.min(limit, items.length) }, + async () => { + while (true) { + const i = cursor++; + if (i >= items.length) return; + results[i] = await fn(items[i]!, i); + } + }, + ); + await Promise.all(workers); + return results; +} + export async function indexBlock(args: IndexBlockArgs) { const { db, chain, height, log } = args; const block = await chain.getBlock(height); - // We do all writes inside a single transaction so a partial block never - // ends up in the DB on crash. + // ── PHASE 1: pre-fetch all per-tx native bodies in parallel. Out of + // the SQL transaction so the long-held write lock only covers the + // actual INSERTs, not the N HTTP round-trips. eth_getBlockByNumber on + // Sentrix returns hash-only entries (chain doesn't honor + // includeTransactions=true), and eth_getTransactionByHash returns the + // native `{transaction: {…}}` wrapper instead of the EVM-spec shape + // viem expects — that's why each tx needs its own native REST fetch. + // Coinbase sentinel maps to the all-zero address + tx_type='coinbase' + // so consumers can filter rewards out of address-history queries. + // Sentri → wei conversion: 1 sentri = 1e10 wei (chain native is + // 8-decimal, EVM rail is 18-decimal). + const ZERO = "0x0000000000000000000000000000000000000000" as const; + const SENTRI_TO_WEI = 10_000_000_000n; + + const txEntries = block.transactions.map((entry, i) => ({ + index: i, + hash: typeof entry === "string" ? entry : entry.hash, + })); + + const natives = await mapWithConcurrency( + txEntries, + TX_FETCH_CONCURRENCY, + async (e) => ({ entry: e, native: await chain.getNativeTransaction(e.hash) }), + ); + + // ── PHASE 2: build batch INSERT row arrays. + type TxRow = typeof txsTable.$inferInsert; + type AddrRow = typeof addressesTable.$inferInsert; + const txRows: TxRow[] = []; + const addrRows: AddrRow[] = []; + const heightBig = BigInt(height); + for (const { entry, native } of natives) { + if (!native) { + // Either a true 404 (chain pruned / never had it) or persistent + // transient failure after the client's 4 retries. Tx permanently + // missing from this block's indexed set — log loud enough that + // an operator can grep journalctl after the fact. + log.warn( + { hash: entry.hash, height: height.toString() }, + "getNativeTransaction returned null — tx skipped", + ); + continue; + } + const inner = native.transaction; + const isCoinbase = inner.from_address === "COINBASE"; + const fromAddr = isCoinbase ? ZERO : inner.from_address.toLowerCase(); + const toAddr = inner.to_address ? inner.to_address.toLowerCase() : null; + const txHash = entry.hash.startsWith("0x") + ? entry.hash.toLowerCase() + : `0x${entry.hash.toLowerCase()}`; + txRows.push({ + hash: txHash, + blockHeight: height, + txIndex: entry.index, + fromAddr, + toAddr, + value: (BigInt(inner.amount) * SENTRI_TO_WEI).toString(), + gasLimit: 0n, + gasUsed: 0n, + gasPrice: null, + fee: (BigInt(inner.fee) * SENTRI_TO_WEI).toString(), + nonce: BigInt(inner.nonce), + data: inner.data, + status: 1, + contractAddress: null, + txType: isCoinbase ? "coinbase" : "native", + }); + // Coinbase sentinel skipped on the from side — the all-zero + // address shouldn't claim a balance row from validator rewards. + if (!isCoinbase) { + addrRows.push({ + address: fromAddr, + firstSeenBlock: heightBig, + lastSeenBlock: heightBig, + }); + } + if (toAddr) { + addrRows.push({ + address: toAddr, + firstSeenBlock: heightBig, + lastSeenBlock: heightBig, + }); + } + } + + // Dedupe addresses within the batch — the upsert below uses + // ON CONFLICT (address) DO UPDATE, and Postgres rejects multiple + // affected rows with the same conflict target inside one batch + // (cardinality_violation). Keeping the first occurrence is safe; + // first_seen_block is identical across rows for the same block and + // last_seen_block uses GREATEST in the UPSERT clause. + const addrDeduped = Array.from( + new Map(addrRows.map((r) => [r.address, r])).values(), + ); + + // ── PHASE 3: log fetch + decode. eth_getLogs goes through viem + // (covered by the batch transport from Tier 1 perf PR), so this is + // already coalesced with other concurrent calls. + const evmLogs = await chain.getLogsRange(height, height); + + type LogRow = typeof logsTable.$inferInsert; + type TransferRow = typeof tokenTransfers.$inferInsert; + const logRows: LogRow[] = []; + const transferRows: TransferRow[] = []; + for (const l of evmLogs) { + if ( + l.blockNumber == null || + l.transactionHash == null || + l.logIndex == null + ) { + continue; + } + // Normalize address + topics to lowercase. Downstream consumers + // query with lowercase WHERE; mixed-case rows silently miss JOINs. + const logAddr = l.address.toLowerCase(); + const lower = (s: string | undefined) => (s ? s.toLowerCase() : null); + const txHashLower = l.transactionHash.toLowerCase(); + logRows.push({ + blockHeight: l.blockNumber, + txHash: txHashLower, + logIndex: l.logIndex, + address: logAddr, + topic0: lower(l.topics[0]), + topic1: lower(l.topics[1]), + topic2: lower(l.topics[2]), + topic3: lower(l.topics[3]), + data: l.data, + }); + const t0 = l.topics[0]; + if (t0 === ERC20_TRANSFER && l.topics.length === 3) { + transferRows.push({ + blockHeight: l.blockNumber, + txHash: txHashLower, + logIndex: l.logIndex, + contract: logAddr, + standard: "erc20", + fromAddr: topicToAddress(l.topics[1]!), + toAddr: topicToAddress(l.topics[2]!), + tokenId: null, + amount: BigInt(l.data || "0x0").toString(), + }); + } else if (t0 === ERC20_TRANSFER && l.topics.length === 4) { + transferRows.push({ + blockHeight: l.blockNumber, + txHash: txHashLower, + logIndex: l.logIndex, + contract: logAddr, + standard: "erc721", + fromAddr: topicToAddress(l.topics[1]!), + toAddr: topicToAddress(l.topics[2]!), + tokenId: BigInt(l.topics[3]!).toString(), + amount: "1", + }); + } else if (t0 === ERC1155_SINGLE) { + const data = l.data.replace(/^0x/, ""); + const id = BigInt("0x" + data.slice(0, 64)); + const value = BigInt("0x" + data.slice(64, 128)); + transferRows.push({ + blockHeight: l.blockNumber, + txHash: txHashLower, + logIndex: l.logIndex, + contract: logAddr, + standard: "erc1155", + fromAddr: topicToAddress(l.topics[2]!), + toAddr: topicToAddress(l.topics[3]!), + tokenId: id.toString(), + amount: value.toString(), + }); + } else if (t0 === ERC1155_BATCH) { + // Two dynamic arrays — defer batch decode. Raw log still inserted + // so the materialiser can re-decode at its own pace. + } + } + + // ── PHASE 4: single SQL transaction with all batch INSERTs. The + // write-lock window now covers milliseconds (just the inserts) + // instead of seconds (inserts + N HTTP round-trips). Pre-batch sync + // could hold a transaction open for ~10 s on a 5000-tx block; this + // version finishes in ~50 ms. await db.transaction(async (tx) => { - // Lowercase every 0x-prefixed field on insert. Today the chain RPC - // returns lowercase but viem doesn't guarantee that across versions - // and some EVM RPCs return EIP-55 checksum. Downstream queries all - // assume lowercase storage (sync.ts:112-114, routes/*.ts toLowerCase - // on every WHERE), so any mixed-case row would silently break JOINs - // and address-history filters. await tx .insert(blocksTable) .values({ @@ -87,7 +291,7 @@ export async function indexBlock(args: IndexBlockArgs) { hash: block.hash?.toLowerCase() ?? "0x", parentHash: block.parentHash.toLowerCase(), timestamp: block.timestamp, - validator: (block.miner ?? "0x0000000000000000000000000000000000000000").toLowerCase(), + validator: (block.miner ?? ZERO).toLowerCase(), gasUsed: block.gasUsed ?? 0n, gasLimit: block.gasLimit ?? 0n, baseFee: block.baseFeePerGas?.toString() ?? null, @@ -96,192 +300,28 @@ export async function indexBlock(args: IndexBlockArgs) { }) .onConflictDoNothing(); - // Native-shape adapter. eth_getBlockByNumber on Sentrix returns hash- - // only entries (chain doesn't honor includeTransactions=true), and - // eth_getTransactionByHash returns the native `{transaction: {…}}` - // wrapper instead of the EVM-spec shape viem expects. So we fetch - // each tx via the chain native REST, map the native fields into the - // indexer schema, and convert sentri → wei (1 sentri = 1e10 wei) so - // the value/fee columns share the same 18-decimal scale as the EVM - // rail. COINBASE sender sentinel maps to the all-zero address + - // tx_type='coinbase' so consumers can filter rewards out of any - // address-history query. - const ZERO = "0x0000000000000000000000000000000000000000" as const; - const SENTRI_TO_WEI = 10_000_000_000n; - for (let i = 0; i < block.transactions.length; i++) { - const entry = block.transactions[i]; - const hash = typeof entry === "string" ? entry : entry.hash; - const native = await chain.getNativeTransaction(hash); - if (!native) { - // Either a true 404 (chain pruned / never had it) or persistent - // transient failure after the client's 4 retries. Either way the - // tx is permanently missing from this block's indexed set — - // last_synced_height will still advance because the surrounding - // transaction commits, so log loud enough that an operator can - // grep journalctl after the fact. - log.warn( - { hash, height: height.toString() }, - "getNativeTransaction returned null — tx skipped", - ); - continue; - } - const inner = native.transaction; - const isCoinbase = inner.from_address === "COINBASE"; - const fromAddr = isCoinbase ? ZERO : inner.from_address.toLowerCase(); - const toAddr = inner.to_address ? inner.to_address.toLowerCase() : null; - const txHash = hash.startsWith("0x") ? hash.toLowerCase() : `0x${hash.toLowerCase()}`; + if (txRows.length > 0) { + await tx.insert(txsTable).values(txRows).onConflictDoNothing(); + } + if (addrDeduped.length > 0) { await tx - .insert(txsTable) - .values({ - hash: txHash, - blockHeight: height, - txIndex: i, - fromAddr, - toAddr, - value: (BigInt(inner.amount) * SENTRI_TO_WEI).toString(), - gasLimit: 0n, - gasUsed: 0n, - gasPrice: null, - fee: (BigInt(inner.fee) * SENTRI_TO_WEI).toString(), - nonce: BigInt(inner.nonce), - data: inner.data, - status: 1, - contractAddress: null, - txType: isCoinbase ? "coinbase" : "native", - }) - .onConflictDoNothing(); - - // Upsert into addresses for both sender and receiver. Without this, - // the table sits empty and any "list of addresses I've ever seen on - // chain" query (eg `/contracts/stats`, scan's recent-deployments feed) - // returns nothing — even though we have millions of indexed txs. - // is_contract stays false here; a separate eth_getCode pass marks - // it true for addresses with non-empty code (cheap, lazy backfill). - // Coinbase sentinel skipped on the from side — the all-zero address - // shouldn't claim a balance row from validator rewards. - const heightBig = BigInt(height); - if (!isCoinbase) { - await tx - .insert(addressesTable) - .values({ - address: fromAddr, - firstSeenBlock: heightBig, - lastSeenBlock: heightBig, - }) - .onConflictDoUpdate({ - target: addressesTable.address, - set: { - lastSeenBlock: sql`GREATEST(${addressesTable.lastSeenBlock}, EXCLUDED.last_seen_block)`, - }, - }); - } - if (toAddr) { - await tx - .insert(addressesTable) - .values({ - address: toAddr, - firstSeenBlock: heightBig, - lastSeenBlock: heightBig, - }) - .onConflictDoUpdate({ - target: addressesTable.address, - set: { - lastSeenBlock: sql`GREATEST(${addressesTable.lastSeenBlock}, EXCLUDED.last_seen_block)`, - }, - }); - } + .insert(addressesTable) + .values(addrDeduped) + .onConflictDoUpdate({ + target: addressesTable.address, + set: { + lastSeenBlock: sql`GREATEST(${addressesTable.lastSeenBlock}, EXCLUDED.last_seen_block)`, + }, + }); } - - // Pull all logs in this block in one shot. - const evmLogs = await chain.getLogsRange(height, height); - for (const l of evmLogs) { - if ( - l.blockNumber == null || - l.transactionHash == null || - l.logIndex == null - ) { - continue; - } - // Normalize address + topics to lowercase before insert. txs.fromAddr - // / txs.toAddr already store lowercase (sync.ts:112-114), and downstream - // consumers (scan, faucet, indexer endpoints) query with lowercase - // WHERE clauses. If we leave logs.address mixed-case (some RPCs return - // EIP-55 checksum), JOINs and address-history filters silently miss - // events. topics also lowercased for selector-prefix LIKE patterns. - const logAddr = l.address.toLowerCase(); + if (logRows.length > 0) { + await tx.insert(logsTable).values(logRows).onConflictDoNothing(); + } + if (transferRows.length > 0) { await tx - .insert(logsTable) - .values({ - blockHeight: l.blockNumber, - txHash: l.transactionHash.toLowerCase(), - logIndex: l.logIndex, - address: logAddr, - topic0: l.topics[0]?.toLowerCase() ?? null, - topic1: l.topics[1]?.toLowerCase() ?? null, - topic2: l.topics[2]?.toLowerCase() ?? null, - topic3: l.topics[3]?.toLowerCase() ?? null, - data: l.data, - }) + .insert(tokenTransfers) + .values(transferRows) .onConflictDoNothing(); - - // Decoded transfer materialisation. - const t0 = l.topics[0]; - if (t0 === ERC20_TRANSFER && l.topics.length === 3) { - // ERC-20 Transfer(from indexed, to indexed, uint256 value) - await tx - .insert(tokenTransfers) - .values({ - blockHeight: l.blockNumber, - txHash: l.transactionHash.toLowerCase(), - logIndex: l.logIndex, - contract: logAddr, - standard: "erc20", - fromAddr: topicToAddress(l.topics[1]!), - toAddr: topicToAddress(l.topics[2]!), - tokenId: null, - amount: BigInt(l.data || "0x0").toString(), - }) - .onConflictDoNothing(); - } else if (t0 === ERC20_TRANSFER && l.topics.length === 4) { - // ERC-721 Transfer(from indexed, to indexed, tokenId indexed) - await tx - .insert(tokenTransfers) - .values({ - blockHeight: l.blockNumber, - txHash: l.transactionHash.toLowerCase(), - logIndex: l.logIndex, - contract: logAddr, - standard: "erc721", - fromAddr: topicToAddress(l.topics[1]!), - toAddr: topicToAddress(l.topics[2]!), - tokenId: BigInt(l.topics[3]!).toString(), - amount: "1", - }) - .onConflictDoNothing(); - } else if (t0 === ERC1155_SINGLE) { - // ERC-1155 TransferSingle(operator indexed, from indexed, to indexed, id, value) - const data = l.data.replace(/^0x/, ""); - const id = BigInt("0x" + data.slice(0, 64)); - const value = BigInt("0x" + data.slice(64, 128)); - await tx - .insert(tokenTransfers) - .values({ - blockHeight: l.blockNumber, - txHash: l.transactionHash.toLowerCase(), - logIndex: l.logIndex, - contract: logAddr, - standard: "erc1155", - fromAddr: topicToAddress(l.topics[2]!), - toAddr: topicToAddress(l.topics[3]!), - tokenId: id.toString(), - amount: value.toString(), - }) - .onConflictDoNothing(); - } else if (t0 === ERC1155_BATCH) { - // Decoding a batch transfer from raw log data is non-trivial (two - // dynamic arrays). Defer to Phase 2 — record the raw log here, the - // worker that materialises balances can re-decode at its own pace. - } } await tx diff --git a/packages/chain/src/index.ts b/packages/chain/src/index.ts index 0128827..c218f5e 100644 --- a/packages/chain/src/index.ts +++ b/packages/chain/src/index.ts @@ -8,6 +8,7 @@ import { createPublicClient, + fallback, defineChain, http, webSocket, @@ -165,24 +166,48 @@ export class SentrixClient { // to the public defaults. The wg1 / loopback path is what saves the // backfill from the public RPC's per-IP rate limit — running on the // build host we point at a validator's :8545/rpc directly. - const httpUrl = + // Multi-URL failover support — operator sets + // INDEXER_RPC_HTTP_URLS=primary,backup1,backup2 and viem's fallback + // transport rolls over to the next on connection error / 5xx / + // timeout, automatically retrying the failed one with backoff. The + // legacy single-URL env (INDEXER_RPC_HTTP_URL) still works as + // before — it just becomes the only entry in the URL list. + const rawHttpUrls = + process.env.INDEXER_RPC_HTTP_URLS ?? process.env.INDEXER_RPC_HTTP_URL ?? cfg.httpUrl ?? chain.rpcUrls.default.http[0]; + const httpUrls = rawHttpUrls + .split(",") + .map((s) => s.trim()) + .filter(Boolean); + const httpUrl = httpUrls[0]!; const wsUrl = process.env.INDEXER_RPC_WS_URL ?? cfg.wsUrl ?? chain.rpcUrls.default.webSocket?.[0]; - // batch:true coalesces concurrent JSON-RPC calls into a single HTTP - // request (default window 0 ms = same micro-task). Indexer backfill - // fires bursts of getBlock + getLogs + getTransaction per block; with - // batching enabled these collapse from N round-trips to 1 per block, - // cutting tail latency under load and freeing the public LB to serve - // other consumers. wait: 0 keeps single-call latency unchanged. + // batch coalesces concurrent JSON-RPC calls into a single HTTP + // request (window 0 ms = same micro-task). Indexer backfill fires + // bursts of getBlock + getLogs + getTransaction per block; with + // batching these collapse from N round-trips to 1 per block. + // wait: 0 keeps single-call latency unchanged. Each transport in + // the fallback list gets the same batch config. + const httpTransports = httpUrls.map((url) => + http(url, { batch: { batchSize: 100, wait: 0 } }), + ); this.http = createPublicClient({ chain, - transport: http(httpUrl, { batch: { batchSize: 100, wait: 0 } }), + transport: + httpTransports.length === 1 + ? httpTransports[0]! + : fallback(httpTransports, { + // Health-check each transport every 60 s; demote bad ones + // automatically. retryCount is per-transport — fallback + // moves to the next entry once exhausted. + rank: { interval: 60_000 }, + retryCount: 2, + }), }); this.ws = wsUrl ? createPublicClient({ chain, transport: webSocket(wsUrl) }) diff --git a/packages/db/drizzle/0005_stats_daily_mv.sql b/packages/db/drizzle/0005_stats_daily_mv.sql new file mode 100644 index 0000000..9a41806 --- /dev/null +++ b/packages/db/drizzle/0005_stats_daily_mv.sql @@ -0,0 +1,20 @@ +-- Materialised view for /stats/daily — replaces the per-process 5-min +-- in-memory cache that was lost on every restart. Stored on disk, shared +-- across api processes, refreshable concurrently without blocking reads. +-- +-- The unique index on (date) is mandatory for REFRESH MATERIALIZED VIEW +-- CONCURRENTLY — Postgres requires at least one unique index on the view. +-- +-- Refresh cadence is owned by the indexer worker (apps/indexer/src/index.ts +-- triggers REFRESH every N blocks). On a quiescent indexer the view stays +-- whatever it was on the last refresh; the API never blocks on freshness. + +CREATE MATERIALIZED VIEW IF NOT EXISTS stats_daily_mv AS + SELECT to_char(to_timestamp(timestamp::bigint), 'YYYY-MM-DD') AS date, + count(*)::bigint AS blocks, + COALESCE(sum(tx_count), 0)::bigint AS transactions + FROM blocks + GROUP BY 1 + ORDER BY 1; +--> statement-breakpoint +CREATE UNIQUE INDEX IF NOT EXISTS stats_daily_mv_date_uniq ON stats_daily_mv(date); diff --git a/packages/db/drizzle/meta/0005_snapshot.json b/packages/db/drizzle/meta/0005_snapshot.json new file mode 100644 index 0000000..0675193 --- /dev/null +++ b/packages/db/drizzle/meta/0005_snapshot.json @@ -0,0 +1,1318 @@ +{ + "id": "362cd1e0-864b-452c-aecb-3e633d7d25ed", + "prevId": "3d3a13ad-3544-44b6-881a-2e8f011536f3", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.addresses": { + "name": "addresses", + "schema": "", + "columns": { + "address": { + "name": "address", + "type": "varchar(42)", + "primaryKey": true, + "notNull": true + }, + "first_seen_block": { + "name": "first_seen_block", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "last_seen_block": { + "name": "last_seen_block", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "balance_cached": { + "name": "balance_cached", + "type": "numeric(78, 0)", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "nonce": { + "name": "nonce", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "0" + }, + "is_contract": { + "name": "is_contract", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "code_hash": { + "name": "code_hash", + "type": "varchar(66)", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "addresses_contract_recent_idx": { + "name": "addresses_contract_recent_idx", + "columns": [ + { + "expression": "is_contract", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "first_seen_block", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.blocks": { + "name": "blocks", + "schema": "", + "columns": { + "height": { + "name": "height", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "hash": { + "name": "hash", + "type": "varchar(66)", + "primaryKey": false, + "notNull": true + }, + "parent_hash": { + "name": "parent_hash", + "type": "varchar(66)", + "primaryKey": false, + "notNull": true + }, + "timestamp": { + "name": "timestamp", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "validator": { + "name": "validator", + "type": "varchar(42)", + "primaryKey": false, + "notNull": true + }, + "gas_used": { + "name": "gas_used", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "gas_limit": { + "name": "gas_limit", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "base_fee": { + "name": "base_fee", + "type": "numeric(78, 0)", + "primaryKey": false, + "notNull": false + }, + "tx_count": { + "name": "tx_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "state_root": { + "name": "state_root", + "type": "varchar(66)", + "primaryKey": false, + "notNull": false + }, + "round": { + "name": "round", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "justification_signers": { + "name": "justification_signers", + "type": "jsonb", + "primaryKey": false, + "notNull": false, + "default": "'[]'::jsonb" + } + }, + "indexes": { + "blocks_validator_idx": { + "name": "blocks_validator_idx", + "columns": [ + { + "expression": "validator", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "blocks_timestamp_idx": { + "name": "blocks_timestamp_idx", + "columns": [ + { + "expression": "timestamp", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "blocks_hash_unique": { + "name": "blocks_hash_unique", + "nullsNotDistinct": false, + "columns": [ + "hash" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cb_tokens": { + "name": "cb_tokens", + "schema": "", + "columns": { + "curve_address": { + "name": "curve_address", + "type": "varchar(42)", + "primaryKey": true, + "notNull": true + }, + "token_address": { + "name": "token_address", + "type": "varchar(42)", + "primaryKey": false, + "notNull": true + }, + "owner_address": { + "name": "owner_address", + "type": "varchar(42)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "symbol": { + "name": "symbol", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "curve_supply": { + "name": "curve_supply", + "type": "numeric(78, 0)", + "primaryKey": false, + "notNull": true + }, + "graduation_threshold": { + "name": "graduation_threshold", + "type": "numeric(78, 0)", + "primaryKey": false, + "notNull": true + }, + "is_graduated": { + "name": "is_graduated", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_block": { + "name": "created_block", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_tx_hash": { + "name": "created_tx_hash", + "type": "varchar(66)", + "primaryKey": false, + "notNull": true + }, + "total_volume_srx": { + "name": "total_volume_srx", + "type": "numeric(78, 0)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "trade_count": { + "name": "trade_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_price_srx": { + "name": "last_price_srx", + "type": "numeric(78, 0)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "image_url": { + "name": "image_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "twitter_url": { + "name": "twitter_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "telegram_url": { + "name": "telegram_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "website_url": { + "name": "website_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata_updated_at": { + "name": "metadata_updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "cb_tokens_owner_idx": { + "name": "cb_tokens_owner_idx", + "columns": [ + { + "expression": "owner_address", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cb_tokens_graduated_idx": { + "name": "cb_tokens_graduated_idx", + "columns": [ + { + "expression": "is_graduated", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cb_tokens_created_block_idx": { + "name": "cb_tokens_created_block_idx", + "columns": [ + { + "expression": "created_block", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "cb_tokens_token_address_unique": { + "name": "cb_tokens_token_address_unique", + "nullsNotDistinct": false, + "columns": [ + "token_address" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cb_trades": { + "name": "cb_trades", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "cb_trades_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "curve_address": { + "name": "curve_address", + "type": "varchar(42)", + "primaryKey": false, + "notNull": true + }, + "token_address": { + "name": "token_address", + "type": "varchar(42)", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "varchar(12)", + "primaryKey": false, + "notNull": true + }, + "trader_address": { + "name": "trader_address", + "type": "varchar(42)", + "primaryKey": false, + "notNull": true + }, + "srx_amount": { + "name": "srx_amount", + "type": "numeric(78, 0)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "token_amount": { + "name": "token_amount", + "type": "numeric(78, 0)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "fee": { + "name": "fee", + "type": "numeric(78, 0)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "block_number": { + "name": "block_number", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_hash": { + "name": "tx_hash", + "type": "varchar(66)", + "primaryKey": false, + "notNull": true + }, + "log_index": { + "name": "log_index", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "cb_trades_uniq_log": { + "name": "cb_trades_uniq_log", + "columns": [ + { + "expression": "tx_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "log_index", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cb_trades_curve_idx": { + "name": "cb_trades_curve_idx", + "columns": [ + { + "expression": "curve_address", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cb_trades_trader_idx": { + "name": "cb_trades_trader_idx", + "columns": [ + { + "expression": "trader_address", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cb_trades_block_idx": { + "name": "cb_trades_block_idx", + "columns": [ + { + "expression": "block_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cb_trades_srx_amount_desc_idx": { + "name": "cb_trades_srx_amount_desc_idx", + "columns": [ + { + "expression": "srx_amount", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.epochs": { + "name": "epochs", + "schema": "", + "columns": { + "epoch_number": { + "name": "epoch_number", + "type": "bigint", + "primaryKey": true, + "notNull": true + }, + "start_height": { + "name": "start_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "end_height": { + "name": "end_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "validator_set": { + "name": "validator_set", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "total_staked": { + "name": "total_staked", + "type": "numeric(78, 0)", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "total_blocks_produced": { + "name": "total_blocks_produced", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "0" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.logs": { + "name": "logs", + "schema": "", + "columns": { + "block_height": { + "name": "block_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_hash": { + "name": "tx_hash", + "type": "varchar(66)", + "primaryKey": false, + "notNull": true + }, + "log_index": { + "name": "log_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "address": { + "name": "address", + "type": "varchar(42)", + "primaryKey": false, + "notNull": true + }, + "topic0": { + "name": "topic0", + "type": "varchar(66)", + "primaryKey": false, + "notNull": false + }, + "topic1": { + "name": "topic1", + "type": "varchar(66)", + "primaryKey": false, + "notNull": false + }, + "topic2": { + "name": "topic2", + "type": "varchar(66)", + "primaryKey": false, + "notNull": false + }, + "topic3": { + "name": "topic3", + "type": "varchar(66)", + "primaryKey": false, + "notNull": false + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "logs_address_idx": { + "name": "logs_address_idx", + "columns": [ + { + "expression": "address", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "logs_topic0_idx": { + "name": "logs_topic0_idx", + "columns": [ + { + "expression": "topic0", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "logs_tx_idx": { + "name": "logs_tx_idx", + "columns": [ + { + "expression": "tx_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "logs_block_height_blocks_height_fk": { + "name": "logs_block_height_blocks_height_fk", + "tableFrom": "logs", + "tableTo": "blocks", + "columnsFrom": [ + "block_height" + ], + "columnsTo": [ + "height" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "logs_tx_hash_transactions_hash_fk": { + "name": "logs_tx_hash_transactions_hash_fk", + "tableFrom": "logs", + "tableTo": "transactions", + "columnsFrom": [ + "tx_hash" + ], + "columnsTo": [ + "hash" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "logs_block_height_log_index_pk": { + "name": "logs_block_height_log_index_pk", + "columns": [ + "block_height", + "log_index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public._meta": { + "name": "_meta", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "varchar(64)", + "primaryKey": true, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "bigint", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.token_transfers": { + "name": "token_transfers", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "token_transfers_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "block_height": { + "name": "block_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_hash": { + "name": "tx_hash", + "type": "varchar(66)", + "primaryKey": false, + "notNull": true + }, + "log_index": { + "name": "log_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "contract": { + "name": "contract", + "type": "varchar(42)", + "primaryKey": false, + "notNull": true + }, + "standard": { + "name": "standard", + "type": "varchar(12)", + "primaryKey": false, + "notNull": true + }, + "from_addr": { + "name": "from_addr", + "type": "varchar(42)", + "primaryKey": false, + "notNull": true + }, + "to_addr": { + "name": "to_addr", + "type": "varchar(42)", + "primaryKey": false, + "notNull": true + }, + "token_id": { + "name": "token_id", + "type": "numeric(78, 0)", + "primaryKey": false, + "notNull": false + }, + "amount": { + "name": "amount", + "type": "numeric(78, 0)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "transfers_contract_idx": { + "name": "transfers_contract_idx", + "columns": [ + { + "expression": "contract", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "transfers_from_idx": { + "name": "transfers_from_idx", + "columns": [ + { + "expression": "from_addr", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "transfers_to_idx": { + "name": "transfers_to_idx", + "columns": [ + { + "expression": "to_addr", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "transfers_block_idx": { + "name": "transfers_block_idx", + "columns": [ + { + "expression": "block_height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "transfers_from_block_idx": { + "name": "transfers_from_block_idx", + "columns": [ + { + "expression": "from_addr", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "transfers_to_block_idx": { + "name": "transfers_to_block_idx", + "columns": [ + { + "expression": "to_addr", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.transactions": { + "name": "transactions", + "schema": "", + "columns": { + "hash": { + "name": "hash", + "type": "varchar(66)", + "primaryKey": true, + "notNull": true + }, + "block_height": { + "name": "block_height", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "tx_index": { + "name": "tx_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "from_addr": { + "name": "from_addr", + "type": "varchar(42)", + "primaryKey": false, + "notNull": true + }, + "to_addr": { + "name": "to_addr", + "type": "varchar(42)", + "primaryKey": false, + "notNull": false + }, + "value": { + "name": "value", + "type": "numeric(78, 0)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "gas_limit": { + "name": "gas_limit", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "gas_used": { + "name": "gas_used", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "0" + }, + "gas_price": { + "name": "gas_price", + "type": "numeric(78, 0)", + "primaryKey": false, + "notNull": false + }, + "fee": { + "name": "fee", + "type": "numeric(78, 0)", + "primaryKey": false, + "notNull": true, + "default": "'0'" + }, + "nonce": { + "name": "nonce", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": "0" + }, + "data": { + "name": "data", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "smallint", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "contract_address": { + "name": "contract_address", + "type": "varchar(42)", + "primaryKey": false, + "notNull": false + }, + "tx_type": { + "name": "tx_type", + "type": "varchar(24)", + "primaryKey": false, + "notNull": true, + "default": "'native'" + } + }, + "indexes": { + "txs_block_height_idx": { + "name": "txs_block_height_idx", + "columns": [ + { + "expression": "block_height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "txs_from_idx": { + "name": "txs_from_idx", + "columns": [ + { + "expression": "from_addr", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "txs_to_idx": { + "name": "txs_to_idx", + "columns": [ + { + "expression": "to_addr", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "txs_contract_idx": { + "name": "txs_contract_idx", + "columns": [ + { + "expression": "contract_address", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "txs_value_desc_idx": { + "name": "txs_value_desc_idx", + "columns": [ + { + "expression": "value", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "txs_from_block_idx": { + "name": "txs_from_block_idx", + "columns": [ + { + "expression": "from_addr", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "txs_to_block_idx": { + "name": "txs_to_block_idx", + "columns": [ + { + "expression": "to_addr", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "block_height", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "transactions_block_height_blocks_height_fk": { + "name": "transactions_block_height_blocks_height_fk", + "tableFrom": "transactions", + "tableTo": "blocks", + "columnsFrom": [ + "block_height" + ], + "columnsTo": [ + "height" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.validators": { + "name": "validators", + "schema": "", + "columns": { + "address": { + "name": "address", + "type": "varchar(42)", + "primaryKey": true, + "notNull": true + }, + "moniker": { + "name": "moniker", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false + }, + "commission_bp": { + "name": "commission_bp", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "self_stake": { + "name": "self_stake", + "type": "numeric(78, 0)", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "total_delegated": { + "name": "total_delegated", + "type": "numeric(78, 0)", + "primaryKey": false, + "notNull": false, + "default": "'0'" + }, + "blocks_proposed": { + "name": "blocks_proposed", + "type": "bigint", + "primaryKey": false, + "notNull": false, + "default": "0" + }, + "last_active_block": { + "name": "last_active_block", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "is_jailed": { + "name": "is_jailed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "jail_until": { + "name": "jail_until", + "type": "bigint", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/db/drizzle/meta/_journal.json b/packages/db/drizzle/meta/_journal.json index f742f59..90f61bf 100644 --- a/packages/db/drizzle/meta/_journal.json +++ b/packages/db/drizzle/meta/_journal.json @@ -36,6 +36,13 @@ "when": 1778443188386, "tag": "0004_workable_zeigeist", "breakpoints": true + }, + { + "idx": 5, + "version": "7", + "when": 1778444800000, + "tag": "0005_stats_daily_mv", + "breakpoints": true } ] } \ No newline at end of file From 25b3b152ba0fcf6ae2e0f33a6d9d536940147331 Mon Sep 17 00:00:00 2001 From: satyakwok <119509589+satyakwok@users.noreply.github.com> Date: Sun, 10 May 2026 22:14:39 +0200 Subject: [PATCH 2/2] scrub: remove internal infra refs from comments --- apps/indexer/src/sync.ts | 10 +++++----- packages/chain/src/index.ts | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/apps/indexer/src/sync.ts b/apps/indexer/src/sync.ts index c8c450a..6d36edf 100644 --- a/apps/indexer/src/sync.ts +++ b/apps/indexer/src/sync.ts @@ -47,8 +47,8 @@ export async function syncOnce(args: SyncOnceArgs): Promise { if (start > target) return lastSynced; // Cap each pass to keep memory + transaction size bounded. Operator - // can bump via env when running against a wg1 / loopback RPC that - // doesn't enforce the public rate limit. + // can bump via env when running against an internal RPC endpoint + // that doesn't enforce the public rate limit. const BATCH = BigInt(process.env.INDEXER_BATCH_SIZE ?? 50); const end = start + BATCH > target ? target : start + BATCH - 1n; @@ -70,9 +70,9 @@ interface IndexBlockArgs { // Concurrency cap for the per-tx native fetch fan-out. Pure HTTP // limit — avoids hammering the chain REST with thousands of concurrent // connections on high-tx blocks (chain max is 5000 tx/block). 25 is -// chosen to keep latency low while staying well under the public LB's -// per-IP connection cap. Tunable via env when running against a -// loopback / wg1 RPC with no rate limit. +// chosen to keep latency low while staying well under the public edge's +// per-IP connection cap. Tunable via env for deployments that point at +// an internal RPC endpoint with no rate limit. const TX_FETCH_CONCURRENCY = Number( process.env.INDEXER_TX_FETCH_CONCURRENCY ?? 25, ); diff --git a/packages/chain/src/index.ts b/packages/chain/src/index.ts index c218f5e..ca260f2 100644 --- a/packages/chain/src/index.ts +++ b/packages/chain/src/index.ts @@ -163,9 +163,9 @@ export class SentrixClient { constructor(cfg: SentrixClientConfig) { const chain = cfg.network === "mainnet" ? SENTRIX_MAINNET : SENTRIX_TESTNET; // Operator overrides via env, falling back to caller cfg, falling back - // to the public defaults. The wg1 / loopback path is what saves the - // backfill from the public RPC's per-IP rate limit — running on the - // build host we point at a validator's :8545/rpc directly. + // to the public defaults. Pointing the indexer at an internal RPC + // endpoint (loopback or private network) is what keeps backfill from + // hitting the public edge's per-IP rate limit on long catch-up runs. // Multi-URL failover support — operator sets // INDEXER_RPC_HTTP_URLS=primary,backup1,backup2 and viem's fallback // transport rolls over to the next on connection error / 5xx /