diff --git a/cli.ts b/cli.ts index 825a33b..d4831ac 100644 --- a/cli.ts +++ b/cli.ts @@ -48,12 +48,7 @@ const SERVICES = { hyperliquid: { path: './services/hyperliquid/index.ts', description: - 'Fetch and store Hyperliquid spot pair name lookups from the Info API spotMeta endpoint', - }, - 'hyperliquid-outcomes': { - path: './services/hyperliquid-outcomes/index.ts', - description: - 'Fetch HIP-4 outcome + question metadata (outcomeMeta + per-id settledOutcome recovery) into state_outcome_meta and state_question_meta', + 'Fetch Hyperliquid spot pair names + HIP-4 outcome / question metadata in one cycle (spotMeta + outcomeMeta + per-id settledOutcome recovery) into state_spot_pair_names, state_outcome_meta, and state_question_meta', }, 'kalshi-live': { path: './services/kalshi/live.ts', @@ -102,14 +97,12 @@ const SETUP_ACTIONS = { 'Deploy polymarket tables (polymarket_markets, polymarket_assets)', }, hyperliquid: { - files: ['./sql.schemas/schema.hyperliquid.sql'], + files: [ + './sql.schemas/schema.hyperliquid.sql', + './sql.schemas/schema.hyperliquid_outcomes.sql', + ], description: - 'Deploy hyperliquid spot pair name lookup table (state_spot_pair_names)', - }, - 'hyperliquid-outcomes': { - files: ['./sql.schemas/schema.hyperliquid_outcomes.sql'], - description: - 'Deploy hyperliquid HIP-4 outcome metadata tables (state_outcome_meta, state_question_meta)', + 'Deploy hyperliquid scraper-managed tables (state_spot_pair_names + HIP-4 state_outcome_meta + state_question_meta)', }, kalshi: { files: ['./sql.schemas/schema.kalshi.sql'], @@ -426,7 +419,6 @@ Services: metadata-balances ${SERVICES['metadata-balances'].description} polymarket ${SERVICES['polymarket'].description} hyperliquid ${SERVICES['hyperliquid'].description} - hyperliquid-outcomes ${SERVICES['hyperliquid-outcomes'].description} kalshi-live ${SERVICES['kalshi-live'].description} kalshi-backfill ${SERVICES['kalshi-backfill'].description} metadata-solana-rpc ${SERVICES['metadata-solana-rpc'].description} @@ -439,7 +431,6 @@ Examples: $ npm run cli run metadata-balances $ npm run cli run polymarket $ npm run cli run hyperliquid - $ npm run cli run hyperliquid-outcomes $ npm run cli run metadata-solana-rpc $ npm run cli run metadata-solana-extras-rpc $ npm run cli run metadata-solana-clickhouse @@ -661,11 +652,13 @@ const setupHyperliquid = setupCommand .addHelpText( 'after', ` -This command deploys the Hyperliquid spot pair name lookup table. -It only needs to be run once per database to initialize the table. +This command deploys the Hyperliquid scraper-managed tables. It only +needs to be run once per database to initialize the tables. Tables created: - state_spot_pair_names: Resolves \`@N\` and canonical pair coin values to BASE/QUOTE strings + - state_outcome_meta: HIP-4 per-outcome metadata (name, description, side specs, settlement) + - state_question_meta: HIP-4 multi-outcome question groupings (namedOutcomes, fallbackOutcome) Example: $ npm run cli setup hyperliquid @@ -673,7 +666,7 @@ Example: `, ) .action(async (options: any) => { - log.info('Setting up hyperliquid spot pair name lookup table'); + log.info('Setting up hyperliquid scraper-managed tables'); const files = SETUP_ACTIONS.hyperliquid.files.map((f) => resolve(__dirname, f), ); @@ -681,34 +674,6 @@ Example: }); addClickhouseOptions(setupHyperliquid); -// ---- setup hyperliquid-outcomes ---- -const setupHyperliquidOutcomes = setupCommand - .command('hyperliquid-outcomes') - .description(SETUP_ACTIONS['hyperliquid-outcomes'].description) - .addHelpText( - 'after', - ` -This command deploys the Hyperliquid HIP-4 outcome metadata tables. -It only needs to be run once per database to initialize the tables. - -Tables created: - - state_outcome_meta: Per-outcome metadata (name, description, side specs, settlement) - - state_question_meta: Multi-outcome question groupings (namedOutcomes, fallbackOutcome) - -Example: - $ npm run cli setup hyperliquid-outcomes - $ npm run cli setup hyperliquid-outcomes --cluster my_cluster -`, - ) - .action(async (options: any) => { - log.info('Setting up hyperliquid outcome metadata tables'); - const files = SETUP_ACTIONS['hyperliquid-outcomes'].files.map((f) => - resolve(__dirname, f), - ); - await handleSetupCommand(files, options); - }); -addClickhouseOptions(setupHyperliquidOutcomes); - // ---- setup kalshi ---- const setupKalshi = setupCommand .command('kalshi') diff --git a/lib/setup.test.ts b/lib/setup.test.ts index c0567bc..d716674 100644 --- a/lib/setup.test.ts +++ b/lib/setup.test.ts @@ -320,7 +320,7 @@ describe('schema files', () => { expect(plainMergeTree).toBeNull(); }); - test('should parse hyperliquid-outcomes schema and transform for cluster', async () => { + test('should parse hyperliquid outcomes schema and transform for cluster', async () => { const sql = await Bun.file( './sql.schemas/schema.hyperliquid_outcomes.sql', ).text(); diff --git a/scripts/check-hyperliquid-outcomes.ts b/scripts/check-hyperliquid-outcomes.ts index 4444d2b..0ed508b 100644 --- a/scripts/check-hyperliquid-outcomes.ts +++ b/scripts/check-hyperliquid-outcomes.ts @@ -1,7 +1,7 @@ #!/usr/bin/env bun /** - * Validate the `hyperliquid-outcomes` scraper deployment against whichever - * ClickHouse the env points at. + * Validate the outcome-metadata side of the consolidated `hyperliquid` + * scraper deployment against whichever ClickHouse the env points at. * * Reports: * - schema presence + column shape for `state_outcome_meta` / diff --git a/services/hyperliquid/index.test.ts b/services/hyperliquid/index.test.ts index 7b30e53..5ec408b 100644 --- a/services/hyperliquid/index.test.ts +++ b/services/hyperliquid/index.test.ts @@ -1,82 +1,38 @@ import { afterEach, beforeEach, describe, expect, mock, test } from 'bun:test'; -const mockInsert = mock(() => Promise.resolve()); -const mockQuery = mock(() => - Promise.resolve({ - data: [], - metrics: { httpRequestTimeMs: 0, dataFetchTimeMs: 0, totalTimeMs: 0 }, - }), -); +const mockRunSpot = mock((_url: string) => Promise.resolve()); +const mockRunOutcomes = mock((_url: string) => Promise.resolve()); +const mockInitService = mock(() => {}); +const mockMarkServiceAlive = mock(() => {}); const mockIncrementSuccess = mock(() => {}); const mockIncrementError = mock(() => {}); -const mockInitService = mock(() => {}); -// `mock.module` applies process-wide for the test session. Mirror the full set -// of exports other test files mock so a later `mock.module` call in another -// suite (e.g. polymarket) can override cleanly without leaving dangling -// undefined imports. -mock.module('../../lib/clickhouse', () => ({ - insertClient: { insert: mockInsert }, - query: mockQuery, +mock.module('./spot', () => ({ runSpotCycle: mockRunSpot })); +mock.module('./outcomes', () => ({ runOutcomesCycle: mockRunOutcomes })); +mock.module('../../lib/service-init', () => ({ + initService: mockInitService, + markServiceAlive: mockMarkServiceAlive, })); - mock.module('../../lib/prometheus', () => ({ incrementSuccess: mockIncrementSuccess, incrementError: mockIncrementError, })); -mock.module('../../lib/service-init', () => ({ - initService: mockInitService, -})); - -const sampleMeta = { - tokens: [ - { - name: 'USDC', - fullName: null, - index: 0, - tokenId: '0x00', - szDecimals: 8, - weiDecimals: 8, - isCanonical: true, - evmContract: null, - deployerTradingFeeShare: '0.0', - }, - { - name: 'HYPE', - fullName: null, - index: 150, - tokenId: '0x96', - szDecimals: 2, - weiDecimals: 8, - isCanonical: false, - evmContract: null, - deployerTradingFeeShare: '0.0', - }, - ], - universe: [ - { - tokens: [150, 0], - name: '@107', - index: 107, - isCanonical: false, - }, - ], -}; - -describe('hyperliquid run()', () => { - const originalFetch = globalThis.fetch; +describe('hyperliquid run() — combined orchestrator', () => { const originalUrl = process.env.HYPERLIQUID_INFO_URL; beforeEach(() => { - mockInsert.mockClear(); + mockRunSpot.mockClear(); + mockRunOutcomes.mockClear(); + mockInitService.mockClear(); + mockMarkServiceAlive.mockClear(); mockIncrementSuccess.mockClear(); mockIncrementError.mockClear(); - mockInitService.mockClear(); + mockRunSpot.mockImplementation(() => Promise.resolve()); + mockRunOutcomes.mockImplementation(() => Promise.resolve()); }); afterEach(() => { - globalThis.fetch = originalFetch; if (originalUrl === undefined) { delete process.env.HYPERLIQUID_INFO_URL; } else { @@ -88,54 +44,91 @@ describe('hyperliquid run()', () => { delete process.env.HYPERLIQUID_INFO_URL; const { run } = await import('./index'); await expect(run()).rejects.toThrow(/HYPERLIQUID_INFO_URL/); + expect(mockRunSpot).not.toHaveBeenCalled(); + expect(mockRunOutcomes).not.toHaveBeenCalled(); + expect(mockMarkServiceAlive).not.toHaveBeenCalled(); + expect(mockIncrementSuccess).not.toHaveBeenCalled(); }); - test('inserts resolved spot pair names with refresh_time', async () => { + test('runs spot + outcomes cycles in parallel with the configured info URL', async () => { process.env.HYPERLIQUID_INFO_URL = 'http://example/info'; - globalThis.fetch = mock(() => - Promise.resolve( - new Response(JSON.stringify(sampleMeta), { status: 200 }), - ), - ) as unknown as typeof fetch; const { run } = await import('./index'); await run(); - expect(mockInsert).toHaveBeenCalledTimes(1); - const arg = mockInsert.mock.calls[0]![0] as { - table: string; - values: Array<{ - coin: string; - market_name: string; - base_token: string; - quote_token: string; - refresh_time: string; - }>; - format: string; - }; - expect(arg.table).toBe('state_spot_pair_names'); - expect(arg.format).toBe('JSONEachRow'); - expect(arg.values).toHaveLength(1); - expect(arg.values[0]!.coin).toBe('@107'); - expect(arg.values[0]!.market_name).toBe('HYPE/USDC'); - expect(arg.values[0]!.base_token).toBe('HYPE'); - expect(arg.values[0]!.quote_token).toBe('USDC'); - expect(arg.values[0]!.refresh_time).toMatch( - /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}$/, - ); + expect(mockInitService).toHaveBeenCalledTimes(1); + expect(mockRunSpot).toHaveBeenCalledTimes(1); + expect(mockRunOutcomes).toHaveBeenCalledTimes(1); + expect(mockRunSpot.mock.calls[0]![0]).toBe('http://example/info'); + expect(mockRunOutcomes.mock.calls[0]![0]).toBe('http://example/info'); + }); + + test('advances heartbeat + success metric only when both sub-cycles succeed', async () => { + process.env.HYPERLIQUID_INFO_URL = 'http://example/info'; + + const { run } = await import('./index'); + await run(); + + expect(mockMarkServiceAlive).toHaveBeenCalledTimes(1); expect(mockIncrementSuccess).toHaveBeenCalledTimes(1); - expect(mockIncrementError).not.toHaveBeenCalled(); }); - test('records an error metric and rethrows when fetch fails', async () => { + test('does NOT advance heartbeat when outcomes sub-cycle fails', async () => { process.env.HYPERLIQUID_INFO_URL = 'http://example/info'; - globalThis.fetch = mock(() => - Promise.resolve(new Response('boom', { status: 502 })), - ) as unknown as typeof fetch; + mockRunOutcomes.mockImplementation(() => + Promise.reject(new Error('outcomes boom')), + ); + + const { run } = await import('./index'); + await expect(run()).rejects.toThrow('outcomes boom'); + + // Both sub-cycles still attempted — failure in one must not short-circuit + // the other (`Promise.allSettled` semantics). + expect(mockRunSpot).toHaveBeenCalledTimes(1); + expect(mockRunOutcomes).toHaveBeenCalledTimes(1); + // Liveness contract: partial success is not success. /live must reflect + // that the service is degraded even though spot completed. + expect(mockMarkServiceAlive).not.toHaveBeenCalled(); + expect(mockIncrementSuccess).not.toHaveBeenCalled(); + }); + + test('does NOT advance heartbeat when spot sub-cycle fails', async () => { + process.env.HYPERLIQUID_INFO_URL = 'http://example/info'; + mockRunSpot.mockImplementation(() => + Promise.reject(new Error('spot boom')), + ); const { run } = await import('./index'); - await expect(run()).rejects.toThrow(); - expect(mockIncrementError).toHaveBeenCalledTimes(1); - expect(mockInsert).not.toHaveBeenCalled(); + await expect(run()).rejects.toThrow('spot boom'); + + expect(mockMarkServiceAlive).not.toHaveBeenCalled(); + expect(mockIncrementSuccess).not.toHaveBeenCalled(); + }); + + test('both failures surface together via AggregateError, no heartbeat', async () => { + process.env.HYPERLIQUID_INFO_URL = 'http://example/info'; + mockRunSpot.mockImplementation(() => + Promise.reject(new Error('spot boom')), + ); + mockRunOutcomes.mockImplementation(() => + Promise.reject(new Error('outcomes boom')), + ); + + const { run } = await import('./index'); + const err = await run().then( + () => null, + (e) => e, + ); + expect(err).toBeInstanceOf(AggregateError); + // Both sub-cycle errors preserved — operator sees both reasons in the + // supervisor's stack instead of losing the second to a silent drop. + const reasons = (err as AggregateError).errors.map( + (e: unknown) => (e as Error).message, + ); + expect(reasons).toContain('spot boom'); + expect(reasons).toContain('outcomes boom'); + + expect(mockMarkServiceAlive).not.toHaveBeenCalled(); + expect(mockIncrementSuccess).not.toHaveBeenCalled(); }); }); diff --git a/services/hyperliquid/index.ts b/services/hyperliquid/index.ts index 6a1b45f..1107f92 100644 --- a/services/hyperliquid/index.ts +++ b/services/hyperliquid/index.ts @@ -1,21 +1,30 @@ -import { insertClient } from '../../lib/clickhouse'; import { createLogger } from '../../lib/logger'; -import { incrementError, incrementSuccess } from '../../lib/prometheus'; -import { initService } from '../../lib/service-init'; -import { - fetchSpotMeta, - type HyperliquidSpotMeta, - resolvePairNames, -} from './info'; +import { incrementSuccess } from '../../lib/prometheus'; +import { initService, markServiceAlive } from '../../lib/service-init'; +import { runOutcomesCycle } from './outcomes'; +import { runSpotCycle } from './spot'; const serviceName = 'hyperliquid'; const log = createLogger(serviceName); /** - * Fetch the latest spot universe + tokens, resolve `@N` pair names into their - * `BASE/QUOTE` market names with split base/quote token symbols, and snapshot - * all rows into `state_spot_pair_names` with a fresh `refresh_time`. The CLI - * runner loops with `AUTO_RESTART_DELAY` between iterations, so a single + * Combined Hyperliquid scraper. One cycle runs both the spot-pair-names + * snapshot (`state_spot_pair_names`) and the HIP-4 outcome / question + * metadata snapshot (`state_outcome_meta` + `state_question_meta`) against + * the same HL Info endpoint and ClickHouse database. + * + * Both sub-cycles share `HYPERLIQUID_INFO_URL` + the global CH client. They + * run in parallel — they hit independent Info endpoints and disjoint tables, + * so there's no ordering constraint. + * + * Liveness contract: `markServiceAlive()` + `incrementSuccess()` only fire + * after BOTH sub-cycles complete without throwing. A partial success — e.g. + * spot snapshot lands but outcomes fetch fails — must not advance the + * heartbeat, otherwise `/live` would report healthy while the service is + * silently flapping. Per-cycle error metrics still fire from inside each + * sub-cycle's catch block before the throw propagates. + * + * The CLI runner loops with `AUTO_RESTART_DELAY` between cycles, so a single * `run()` invocation maps to one poll cycle. */ export async function run(): Promise { @@ -28,55 +37,40 @@ export async function run(): Promise { ); } - log.info('Fetching spot metadata'); - const startTime = performance.now(); + const results = await Promise.allSettled([ + runSpotCycle(infoUrl), + runOutcomesCycle(infoUrl), + ]); - let meta: HyperliquidSpotMeta; - try { - meta = await fetchSpotMeta(infoUrl); - } catch (error) { - log.error('Failed to fetch spot metadata', { error }); - incrementError(serviceName); - throw error; - } - - const fetchTimeMs = Math.round(performance.now() - startTime); - const rows = resolvePairNames(meta); - - log.info('Resolved spot pair names', { - pairs: meta.universe.length, - tokens: meta.tokens.length, - rows: rows.length, - fetchTimeMs, - }); - - if (rows.length === 0) { - log.warn('Empty spot universe — skipping insert'); - return; - } - - // ClickHouse DateTime64(3, 'UTC') rejects the trailing `Z`, so trim it - // but keep the millisecond precision so closely-spaced polls produce - // distinct `refresh_time` values (deterministic RMT merges). - const refresh_time = new Date() - .toISOString() - .slice(0, 23) - .replace('T', ' '); - const values = rows.map((r) => ({ ...r, refresh_time })); + const errors = results + .filter((r): r is PromiseRejectedResult => r.status === 'rejected') + .map((r) => r.reason); - try { - await insertClient.insert({ - table: 'state_spot_pair_names', - values, - format: 'JSONEachRow', + if (errors.length > 0) { + log.error('Cycle completed with failures', { + spot: results[0].status, + outcomes: results[1].status, + errors: errors.map((e) => + e instanceof Error ? e.message : String(e), + ), }); - } catch (error) { - log.error('Failed to insert spot pair names', { error }); - incrementError(serviceName); - throw error; + // Heartbeat must NOT advance on a partial- or full-failure cycle: + // a successful sub-cycle alone is not enough to claim the service + // is healthy. Surface every failure so the supervisor's stack + // trace doesn't silently drop the second error when both + // sub-cycles reject — per-cycle error metrics already fired from + // inside each cycle's catch block. + if (errors.length === 1) throw errors[0]; + throw new AggregateError(errors, 'hyperliquid cycle failed'); } - log.info('Inserted spot pair names', { count: values.length }); + // Both sub-cycles completed (either inserted or early-returned on + // empty/cold-cluster). Mark the cycle a success: bump the wall-clock + // heartbeat so `/live` reflects progress (we insert directly via + // `insertClient` rather than the batch-insert queue, so the queue's + // `getLastSuccessfulFlushAt()` never advances on its own) and increment + // the per-service success metric. + markServiceAlive(); incrementSuccess(serviceName); } diff --git a/services/hyperliquid-outcomes/info.test.ts b/services/hyperliquid/outcomes-info.test.ts similarity index 99% rename from services/hyperliquid-outcomes/info.test.ts rename to services/hyperliquid/outcomes-info.test.ts index afdcdc0..9bb0f1a 100644 --- a/services/hyperliquid-outcomes/info.test.ts +++ b/services/hyperliquid/outcomes-info.test.ts @@ -10,7 +10,7 @@ import { type HyperliquidOutcomeSpec, type HyperliquidQuestion, type HyperliquidSettledOutcome, -} from './info'; +} from './outcomes-info'; const REFRESH = '2026-06-11 12:00:00.000'; diff --git a/services/hyperliquid-outcomes/info.ts b/services/hyperliquid/outcomes-info.ts similarity index 99% rename from services/hyperliquid-outcomes/info.ts rename to services/hyperliquid/outcomes-info.ts index 0554ea1..4213eae 100644 --- a/services/hyperliquid-outcomes/info.ts +++ b/services/hyperliquid/outcomes-info.ts @@ -13,7 +13,7 @@ const FETCH_TIMEOUT_MS = (() => { return Number.isFinite(parsed) && parsed > 0 ? parsed : 30000; })(); -const log = createLogger('hyperliquid-outcomes'); +const log = createLogger('hyperliquid:outcomes'); /** * One outcome entry as returned by `POST /info {type: outcomeMeta}` and (under diff --git a/services/hyperliquid-outcomes/index.test.ts b/services/hyperliquid/outcomes.test.ts similarity index 69% rename from services/hyperliquid-outcomes/index.test.ts rename to services/hyperliquid/outcomes.test.ts index 5c4854a..58ae275 100644 --- a/services/hyperliquid-outcomes/index.test.ts +++ b/services/hyperliquid/outcomes.test.ts @@ -9,13 +9,8 @@ const mockQuery = mock(() => ); const mockIncrementSuccess = mock(() => {}); const mockIncrementError = mock(() => {}); -const mockInitService = mock(() => {}); const mockMarkServiceAlive = mock(() => {}); -// Mock only the `lib/clickhouse` exports this service imports -// (`insertClient` + `query`). `mock.module` is process-wide, so providing -// fields beyond what's needed would risk shadowing real exports for other -// suites. mock.module('../../lib/clickhouse', () => ({ insertClient: { insert: mockInsert }, query: mockQuery, @@ -26,8 +21,11 @@ mock.module('../../lib/prometheus', () => ({ incrementError: mockIncrementError, })); +// `mock.module` applies process-wide for the test session. Mirror the full set +// of exports other test files mock so a later `mock.module` call in another +// suite can't leave dangling undefined imports for whichever test runs last. mock.module('../../lib/service-init', () => ({ - initService: mockInitService, + initService: mock(() => {}), markServiceAlive: mockMarkServiceAlive, })); @@ -78,14 +76,6 @@ interface InsertCallArg { format: string; } -/** - * Build a `mockQuery` implementation that routes to different result sets - * depending on which CH table the SQL references. Tests need this because - * the service fires two reads per cycle — `outcome_fills` for the known - * universe and `state_outcome_meta` for already-settled ids — and feeding - * both the same rows would mark every known id as already-settled and - * suppress the settled-lookup probes the test is verifying. - */ function mockQueryRouter(routes: { knownIds?: string[]; alreadySettled?: string[]; @@ -110,16 +100,14 @@ function mockQueryRouter(routes: { }; } -describe('hyperliquid-outcomes run()', () => { +describe('hyperliquid runOutcomesCycle()', () => { const originalFetch = globalThis.fetch; - const originalUrl = process.env.HYPERLIQUID_INFO_URL; beforeEach(() => { mockInsert.mockClear(); mockQuery.mockClear(); mockIncrementSuccess.mockClear(); mockIncrementError.mockClear(); - mockInitService.mockClear(); mockMarkServiceAlive.mockClear(); mockQuery.mockImplementation(() => Promise.resolve({ @@ -135,24 +123,9 @@ describe('hyperliquid-outcomes run()', () => { afterEach(() => { globalThis.fetch = originalFetch; - if (originalUrl === undefined) { - delete process.env.HYPERLIQUID_INFO_URL; - } else { - process.env.HYPERLIQUID_INFO_URL = originalUrl; - } - }); - - test('throws when HYPERLIQUID_INFO_URL is unset', async () => { - delete process.env.HYPERLIQUID_INFO_URL; - const { run } = await import('./index'); - await expect(run()).rejects.toThrow(/HYPERLIQUID_INFO_URL/); }); test('inserts live + settled outcomes and questions in one cycle', async () => { - process.env.HYPERLIQUID_INFO_URL = 'http://example/info'; - // outcome_fills knows 104, 172, and a settled 0 — 0 is missing from - // the live snapshot and not in already-settled, so the cycle must - // probe settledOutcome for it. mockQuery.mockImplementation( mockQueryRouter({ knownIds: ['104', '172', '0'], @@ -175,8 +148,8 @@ describe('hyperliquid-outcomes run()', () => { return Promise.resolve(new Response('null', { status: 200 })); }) as unknown as typeof fetch; - const { run } = await import('./index'); - await run(); + const { runOutcomesCycle } = await import('./outcomes'); + await runOutcomesCycle('http://example/info'); expect(mockInsert).toHaveBeenCalledTimes(2); const calls = mockInsert.mock.calls.map((c) => c[0] as InsertCallArg); @@ -189,7 +162,6 @@ describe('hyperliquid-outcomes run()', () => { expect(questionCall).toBeDefined(); expect(outcomeCall!.format).toBe('JSONEachRow'); - // 2 live + 1 settled expect(outcomeCall!.values).toHaveLength(3); const live104 = outcomeCall!.values.find((r) => r.outcome_id === 104); const settled0 = outcomeCall!.values.find((r) => r.outcome_id === 0); @@ -198,23 +170,21 @@ describe('hyperliquid-outcomes run()', () => { expect(settled0?.status).toBe('settled'); expect(settled0?.settle_fraction).toBe(0); expect(settled0?.settle_details).toBe('price:78212.4'); - // 172 belongs to question 32 via namedOutcomes — reverse map must apply. expect(live172?.question_id).toBe(32); - // 104 is standalone. expect(live104?.question_id).toBeNull(); expect(questionCall!.values).toHaveLength(1); expect(questionCall!.values[0]!.question_id).toBe(32); - expect(mockIncrementSuccess).toHaveBeenCalledTimes(1); + // The orchestrator (index.ts run()) is responsible for the success + // metric + heartbeat once BOTH sub-cycles complete; this sub-cycle + // must not advance them on its own. + expect(mockIncrementSuccess).not.toHaveBeenCalled(); expect(mockIncrementError).not.toHaveBeenCalled(); - // Direct-insert services bump the wall-clock heartbeat so the - // liveness probe reflects progress (batch queue is unused here). - expect(mockMarkServiceAlive).toHaveBeenCalledTimes(1); + expect(mockMarkServiceAlive).not.toHaveBeenCalled(); }); test('proceeds when known-ids query fails (cold cluster)', async () => { - process.env.HYPERLIQUID_INFO_URL = 'http://example/info'; mockQuery.mockImplementation(() => Promise.reject(new Error('table not found')), ); @@ -224,17 +194,15 @@ describe('hyperliquid-outcomes run()', () => { ), ) as unknown as typeof fetch; - const { run } = await import('./index'); - await run(); + const { runOutcomesCycle } = await import('./outcomes'); + await runOutcomesCycle('http://example/info'); - // Still inserted live outcomes + questions. expect(mockInsert).toHaveBeenCalledTimes(2); - expect(mockIncrementSuccess).toHaveBeenCalledTimes(1); + expect(mockIncrementSuccess).not.toHaveBeenCalled(); expect(mockIncrementError).not.toHaveBeenCalled(); }); test('continues past per-id settledOutcome failures without aborting the cycle', async () => { - process.env.HYPERLIQUID_INFO_URL = 'http://example/info'; mockQuery.mockImplementation( mockQueryRouter({ knownIds: ['999'], @@ -248,28 +216,21 @@ describe('hyperliquid-outcomes run()', () => { new Response(JSON.stringify(liveBody), { status: 200 }), ); } - // settledOutcome lookups fail — cycle should still complete with - // the live snapshot. return Promise.resolve(new Response('boom', { status: 502 })); }) as unknown as typeof fetch; - const { run } = await import('./index'); - await run(); + const { runOutcomesCycle } = await import('./outcomes'); + await runOutcomesCycle('http://example/info'); expect(mockInsert).toHaveBeenCalledTimes(2); const outcomeCall = mockInsert.mock.calls .map((c) => c[0] as InsertCallArg) .find((c) => c.table === 'state_outcome_meta'); - // Only the 2 live outcomes — settled lookup failed and was skipped. expect(outcomeCall!.values).toHaveLength(2); - expect(mockIncrementSuccess).toHaveBeenCalledTimes(1); + expect(mockIncrementSuccess).not.toHaveBeenCalled(); }); test('skips settledOutcome probes for ids already captured as settled', async () => { - process.env.HYPERLIQUID_INFO_URL = 'http://example/info'; - // outcome_fills has 104, 172, and a historically-settled 0; but our - // own `state_outcome_meta` already contains 0 with status='settled', - // so the cycle must NOT re-probe HL for it. mockQuery.mockImplementation( mockQueryRouter({ knownIds: ['104', '172', '0'], @@ -290,22 +251,19 @@ describe('hyperliquid-outcomes run()', () => { return Promise.resolve(new Response('null', { status: 200 })); }) as unknown as typeof fetch; - const { run } = await import('./index'); - await run(); + const { runOutcomesCycle } = await import('./outcomes'); + await runOutcomesCycle('http://example/info'); expect(settledProbed).toEqual([]); const outcomeCall = mockInsert.mock.calls .map((c) => c[0] as InsertCallArg) .find((c) => c.table === 'state_outcome_meta'); - // Only the 2 live outcomes inserted this cycle — settled 0 stayed in - // the table from the prior cycle's insert (RMT keeps the older row). expect(outcomeCall!.values).toHaveLength(2); - expect(mockIncrementSuccess).toHaveBeenCalledTimes(1); - expect(mockMarkServiceAlive).toHaveBeenCalledTimes(1); + expect(mockIncrementSuccess).not.toHaveBeenCalled(); + expect(mockMarkServiceAlive).not.toHaveBeenCalled(); }); - test('returns early without success metric when outcomeMeta is empty', async () => { - process.env.HYPERLIQUID_INFO_URL = 'http://example/info'; + test('returns early without inserting when outcomeMeta is empty', async () => { globalThis.fetch = mock(() => Promise.resolve( new Response(JSON.stringify({ outcomes: [], questions: [] }), { @@ -314,11 +272,9 @@ describe('hyperliquid-outcomes run()', () => { ), ) as unknown as typeof fetch; - const { run } = await import('./index'); - await run(); + const { runOutcomesCycle } = await import('./outcomes'); + await runOutcomesCycle('http://example/info'); - // Empty universe is a soft anomaly — don't insert, don't claim - // success, but also don't throw. Next cycle retries. expect(mockInsert).not.toHaveBeenCalled(); expect(mockIncrementSuccess).not.toHaveBeenCalled(); expect(mockIncrementError).not.toHaveBeenCalled(); @@ -326,17 +282,14 @@ describe('hyperliquid-outcomes run()', () => { }); test('records an error metric and rethrows when outcomeMeta fetch fails', async () => { - process.env.HYPERLIQUID_INFO_URL = 'http://example/info'; globalThis.fetch = mock(() => Promise.resolve(new Response('nope', { status: 502 })), ) as unknown as typeof fetch; - const { run } = await import('./index'); - await expect(run()).rejects.toThrow(); + const { runOutcomesCycle } = await import('./outcomes'); + await expect(runOutcomesCycle('http://example/info')).rejects.toThrow(); expect(mockIncrementError).toHaveBeenCalledTimes(1); expect(mockInsert).not.toHaveBeenCalled(); - // Liveness heartbeat must NOT advance on a failed cycle, so silent - // upstream failures still surface via the `/live` probe. expect(mockMarkServiceAlive).not.toHaveBeenCalled(); }); }); diff --git a/services/hyperliquid-outcomes/index.ts b/services/hyperliquid/outcomes.ts similarity index 83% rename from services/hyperliquid-outcomes/index.ts rename to services/hyperliquid/outcomes.ts index 1515eb9..8d27b25 100644 --- a/services/hyperliquid-outcomes/index.ts +++ b/services/hyperliquid/outcomes.ts @@ -1,8 +1,7 @@ import PQueue from 'p-queue'; import { insertClient, query } from '../../lib/clickhouse'; import { createLogger } from '../../lib/logger'; -import { incrementError, incrementSuccess } from '../../lib/prometheus'; -import { initService, markServiceAlive } from '../../lib/service-init'; +import { incrementError } from '../../lib/prometheus'; import { buildLiveOutcomeRow, buildOutcomeToQuestion, @@ -12,10 +11,11 @@ import { fetchSettledOutcome, type OutcomeMetaRow, type QuestionMetaRow, -} from './info'; +} from './outcomes-info'; +import { nowRefreshTime } from './refresh-time'; -const serviceName = 'hyperliquid-outcomes'; -const log = createLogger(serviceName); +const serviceName = 'hyperliquid'; +const log = createLogger(`${serviceName}:outcomes`); /** * Max in-flight `settledOutcome` lookups per cycle. Cold-start probes ~200 @@ -65,17 +65,7 @@ function parseUint64Set(rows: { outcome_id: string }[]): Set { } /** - * Format a `DateTime64(3, 'UTC')`-compatible timestamp. CH rejects the - * trailing `Z` but accepts the millisecond fraction, and we preserve ms so - * closely-spaced polls remain deterministic for RMT merges (same convention - * as the sibling `hyperliquid` service). - */ -function nowRefreshTime(): string { - return new Date().toISOString().slice(0, 23).replace('T', ' '); -} - -/** - * One poll cycle: + * One outcome-meta poll cycle: * 1. Pull `outcomeMeta` (live outcomes + question groupings). * 2. Read distinct `outcome_id` from `outcome_fills` to discover settled * outcomes not in the live snapshot. @@ -83,19 +73,10 @@ function nowRefreshTime(): string { * 4. Insert all rows with a single `refresh_time`. RMT collapses repeated * rows on subsequent polls. * - * The CLI runner loops with `AUTO_RESTART_DELAY` between cycles, so one - * `run()` call = one snapshot. + * Caller (the combined `hyperliquid` service in index.ts) orchestrates with + * the spot poller in one cycle. */ -export async function run(): Promise { - initService({ serviceName }); - - const infoUrl = process.env.HYPERLIQUID_INFO_URL; - if (!infoUrl) { - throw new Error( - 'HYPERLIQUID_INFO_URL is required (set to a Hyperliquid /info endpoint)', - ); - } - +export async function runOutcomesCycle(infoUrl: string): Promise { log.info('Fetching outcome metadata'); const startTime = performance.now(); @@ -233,14 +214,4 @@ export async function run(): Promise { settledErrored, cycleMs, }); - // We insert directly via `insertClient` rather than the batch-insert - // queue, so the queue's `getLastSuccessfulFlushAt()` never advances. - // Bump the wall-clock heartbeat here so `/live` reflects real progress - // after the startup grace window. - markServiceAlive(); - incrementSuccess(serviceName); -} - -if (import.meta.main) { - await run(); } diff --git a/services/hyperliquid/refresh-time.ts b/services/hyperliquid/refresh-time.ts new file mode 100644 index 0000000..ee8037d --- /dev/null +++ b/services/hyperliquid/refresh-time.ts @@ -0,0 +1,15 @@ +/** + * Format a `DateTime64(3, 'UTC')`-compatible timestamp shared by every + * scraper-managed table in the hyperliquid service (`state_spot_pair_names`, + * `state_outcome_meta`, `state_question_meta`). + * + * ClickHouse rejects the trailing `Z` on `DateTime64` literals but accepts + * the millisecond fraction; we preserve ms so closely-spaced polls produce + * distinct `refresh_time` values for deterministic ReplacingMergeTree merges. + * + * Both sub-cycles call this once per insert pass, and the format must stay + * in lockstep across the three tables — keep the single source of truth here. + */ +export function nowRefreshTime(): string { + return new Date().toISOString().slice(0, 23).replace('T', ' '); +} diff --git a/services/hyperliquid/info.test.ts b/services/hyperliquid/spot-info.test.ts similarity index 99% rename from services/hyperliquid/info.test.ts rename to services/hyperliquid/spot-info.test.ts index cafdbe3..9f01594 100644 --- a/services/hyperliquid/info.test.ts +++ b/services/hyperliquid/spot-info.test.ts @@ -3,7 +3,7 @@ import { fetchSpotMeta, type HyperliquidSpotMeta, resolvePairNames, -} from './info'; +} from './spot-info'; describe('resolvePairNames', () => { test('passes canonical pair name through unchanged', () => { diff --git a/services/hyperliquid/info.ts b/services/hyperliquid/spot-info.ts similarity index 100% rename from services/hyperliquid/info.ts rename to services/hyperliquid/spot-info.ts diff --git a/services/hyperliquid/spot.test.ts b/services/hyperliquid/spot.test.ts new file mode 100644 index 0000000..cdfe4ec --- /dev/null +++ b/services/hyperliquid/spot.test.ts @@ -0,0 +1,128 @@ +import { afterEach, beforeEach, describe, expect, mock, test } from 'bun:test'; + +const mockInsert = mock(() => Promise.resolve()); +const mockQuery = mock(() => + Promise.resolve({ + data: [], + metrics: { httpRequestTimeMs: 0, dataFetchTimeMs: 0, totalTimeMs: 0 }, + }), +); +const mockIncrementSuccess = mock(() => {}); +const mockIncrementError = mock(() => {}); + +mock.module('../../lib/clickhouse', () => ({ + insertClient: { insert: mockInsert }, + query: mockQuery, +})); + +mock.module('../../lib/prometheus', () => ({ + incrementSuccess: mockIncrementSuccess, + incrementError: mockIncrementError, +})); + +// `mock.module` is process-wide; ensure other tests importing the orchestrator +// or outcomes via this module name don't see missing exports if this suite +// happens to run last. +mock.module('../../lib/service-init', () => ({ + initService: mock(() => {}), + markServiceAlive: mock(() => {}), +})); + +const sampleMeta = { + tokens: [ + { + name: 'USDC', + fullName: null, + index: 0, + tokenId: '0x00', + szDecimals: 8, + weiDecimals: 8, + isCanonical: true, + evmContract: null, + deployerTradingFeeShare: '0.0', + }, + { + name: 'HYPE', + fullName: null, + index: 150, + tokenId: '0x96', + szDecimals: 2, + weiDecimals: 8, + isCanonical: false, + evmContract: null, + deployerTradingFeeShare: '0.0', + }, + ], + universe: [ + { + tokens: [150, 0], + name: '@107', + index: 107, + isCanonical: false, + }, + ], +}; + +describe('hyperliquid runSpotCycle()', () => { + const originalFetch = globalThis.fetch; + + beforeEach(() => { + mockInsert.mockClear(); + mockIncrementSuccess.mockClear(); + mockIncrementError.mockClear(); + }); + + afterEach(() => { + globalThis.fetch = originalFetch; + }); + + test('inserts resolved spot pair names with refresh_time', async () => { + globalThis.fetch = mock(() => + Promise.resolve( + new Response(JSON.stringify(sampleMeta), { status: 200 }), + ), + ) as unknown as typeof fetch; + + const { runSpotCycle } = await import('./spot'); + await runSpotCycle('http://example/info'); + + expect(mockInsert).toHaveBeenCalledTimes(1); + const arg = mockInsert.mock.calls[0]![0] as { + table: string; + values: Array<{ + coin: string; + market_name: string; + base_token: string; + quote_token: string; + refresh_time: string; + }>; + format: string; + }; + expect(arg.table).toBe('state_spot_pair_names'); + expect(arg.format).toBe('JSONEachRow'); + expect(arg.values).toHaveLength(1); + expect(arg.values[0]!.coin).toBe('@107'); + expect(arg.values[0]!.market_name).toBe('HYPE/USDC'); + expect(arg.values[0]!.base_token).toBe('HYPE'); + expect(arg.values[0]!.quote_token).toBe('USDC'); + expect(arg.values[0]!.refresh_time).toMatch( + /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}$/, + ); + // The orchestrator (index.ts run()) is responsible for the success + // metric + heartbeat once BOTH sub-cycles complete; this sub-cycle + // must not advance them on its own. + expect(mockIncrementSuccess).not.toHaveBeenCalled(); + expect(mockIncrementError).not.toHaveBeenCalled(); + }); + + test('records an error metric and rethrows when fetch fails', async () => { + globalThis.fetch = mock(() => + Promise.resolve(new Response('boom', { status: 502 })), + ) as unknown as typeof fetch; + + const { runSpotCycle } = await import('./spot'); + await expect(runSpotCycle('http://example/info')).rejects.toThrow(); + expect(mockIncrementError).toHaveBeenCalledTimes(1); + expect(mockInsert).not.toHaveBeenCalled(); + }); +}); diff --git a/services/hyperliquid/spot.ts b/services/hyperliquid/spot.ts new file mode 100644 index 0000000..b802948 --- /dev/null +++ b/services/hyperliquid/spot.ts @@ -0,0 +1,65 @@ +import { insertClient } from '../../lib/clickhouse'; +import { createLogger } from '../../lib/logger'; +import { incrementError } from '../../lib/prometheus'; +import { nowRefreshTime } from './refresh-time'; +import { + fetchSpotMeta, + type HyperliquidSpotMeta, + resolvePairNames, +} from './spot-info'; + +const serviceName = 'hyperliquid'; +const log = createLogger(`${serviceName}:spot`); + +/** + * Fetch the latest spot universe + tokens, resolve `@N` pair names into their + * `BASE/QUOTE` market names with split base/quote token symbols, and snapshot + * all rows into `state_spot_pair_names` with a fresh `refresh_time`. Caller + * (the combined `hyperliquid` service in index.ts) orchestrates with the + * outcomes poller in one cycle. + */ +export async function runSpotCycle(infoUrl: string): Promise { + log.info('Fetching spot metadata'); + const startTime = performance.now(); + + let meta: HyperliquidSpotMeta; + try { + meta = await fetchSpotMeta(infoUrl); + } catch (error) { + log.error('Failed to fetch spot metadata', { error }); + incrementError(serviceName); + throw error; + } + + const fetchTimeMs = Math.round(performance.now() - startTime); + const rows = resolvePairNames(meta); + + log.info('Resolved spot pair names', { + pairs: meta.universe.length, + tokens: meta.tokens.length, + rows: rows.length, + fetchTimeMs, + }); + + if (rows.length === 0) { + log.warn('Empty spot universe — skipping insert'); + return; + } + + const refresh_time = nowRefreshTime(); + const values = rows.map((r) => ({ ...r, refresh_time })); + + try { + await insertClient.insert({ + table: 'state_spot_pair_names', + values, + format: 'JSONEachRow', + }); + } catch (error) { + log.error('Failed to insert spot pair names', { error }); + incrementError(serviceName); + throw error; + } + + log.info('Inserted spot pair names', { count: values.length }); +}