diff --git a/cli.ts b/cli.ts index e0465d9..825a33b 100644 --- a/cli.ts +++ b/cli.ts @@ -50,6 +50,11 @@ const SERVICES = { description: 'Fetch and store Hyperliquid spot pair name lookups from the Info API spotMeta endpoint', }, + 'hyperliquid-outcomes': { + path: './services/hyperliquid-outcomes/index.ts', + description: + 'Fetch HIP-4 outcome + question metadata (outcomeMeta + per-id settledOutcome recovery) into state_outcome_meta and state_question_meta', + }, 'kalshi-live': { path: './services/kalshi/live.ts', description: @@ -101,6 +106,11 @@ const SETUP_ACTIONS = { description: 'Deploy hyperliquid spot pair name lookup table (state_spot_pair_names)', }, + 'hyperliquid-outcomes': { + files: ['./sql.schemas/schema.hyperliquid_outcomes.sql'], + description: + 'Deploy hyperliquid HIP-4 outcome metadata tables (state_outcome_meta, state_question_meta)', + }, kalshi: { files: ['./sql.schemas/schema.kalshi.sql'], description: @@ -416,6 +426,7 @@ Services: metadata-balances ${SERVICES['metadata-balances'].description} polymarket ${SERVICES['polymarket'].description} hyperliquid ${SERVICES['hyperliquid'].description} + hyperliquid-outcomes ${SERVICES['hyperliquid-outcomes'].description} kalshi-live ${SERVICES['kalshi-live'].description} kalshi-backfill ${SERVICES['kalshi-backfill'].description} metadata-solana-rpc ${SERVICES['metadata-solana-rpc'].description} @@ -428,6 +439,7 @@ Examples: $ npm run cli run metadata-balances $ npm run cli run polymarket $ npm run cli run hyperliquid + $ npm run cli run hyperliquid-outcomes $ npm run cli run metadata-solana-rpc $ npm run cli run metadata-solana-extras-rpc $ npm run cli run metadata-solana-clickhouse @@ -669,6 +681,34 @@ Example: }); addClickhouseOptions(setupHyperliquid); +// ---- setup hyperliquid-outcomes ---- +const setupHyperliquidOutcomes = setupCommand + .command('hyperliquid-outcomes') + .description(SETUP_ACTIONS['hyperliquid-outcomes'].description) + .addHelpText( + 'after', + ` +This command deploys the Hyperliquid HIP-4 outcome metadata tables. +It only needs to be run once per database to initialize the tables. + +Tables created: + - state_outcome_meta: Per-outcome metadata (name, description, side specs, settlement) + - state_question_meta: Multi-outcome question groupings (namedOutcomes, fallbackOutcome) + +Example: + $ npm run cli setup hyperliquid-outcomes + $ npm run cli setup hyperliquid-outcomes --cluster my_cluster +`, + ) + .action(async (options: any) => { + log.info('Setting up hyperliquid outcome metadata tables'); + const files = SETUP_ACTIONS['hyperliquid-outcomes'].files.map((f) => + resolve(__dirname, f), + ); + await handleSetupCommand(files, options); + }); +addClickhouseOptions(setupHyperliquidOutcomes); + // ---- setup kalshi ---- const setupKalshi = setupCommand .command('kalshi') diff --git a/lib/setup.test.ts b/lib/setup.test.ts index c257c46..c0567bc 100644 --- a/lib/setup.test.ts +++ b/lib/setup.test.ts @@ -319,6 +319,24 @@ describe('schema files', () => { ); expect(plainMergeTree).toBeNull(); }); + + test('should parse hyperliquid-outcomes schema and transform for cluster', async () => { + const sql = await Bun.file( + './sql.schemas/schema.hyperliquid_outcomes.sql', + ).text(); + const statements = splitSqlStatements(sql); + expect(statements.length).toBe(2); + + const transformed = transformSqlForCluster(sql, 'test_cluster'); + const tableMatches = transformed.match(/CREATE\s+TABLE/gi); + const tableClusterMatches = transformed.match( + /CREATE\s+TABLE.*ON\s+CLUSTER/gi, + ); + expect(tableMatches?.length).toBe(tableClusterMatches?.length); + expect(transformed).toContain('ReplicatedReplacingMergeTree'); + expect(transformed).toContain('state_outcome_meta'); + expect(transformed).toContain('state_question_meta'); + }); }); describe('error handling', () => { diff --git a/scripts/check-hyperliquid-outcomes.ts b/scripts/check-hyperliquid-outcomes.ts new file mode 100644 index 0000000..4444d2b --- /dev/null +++ b/scripts/check-hyperliquid-outcomes.ts @@ -0,0 +1,220 @@ +#!/usr/bin/env bun +/** + * Validate the `hyperliquid-outcomes` scraper deployment against whichever + * ClickHouse the env points at. + * + * Reports: + * - schema presence + column shape for `state_outcome_meta` / + * `state_question_meta` + * - row counts by `status` and total + * - coverage gap: distinct `outcome_id` values in `outcome_fills` that have + * no row in `state_outcome_meta` + * - refresh-time freshness (max `refresh_time`, lag from now()) + * - question reverse-map sanity: every `named_outcome_ids` member is present + * in `state_outcome_meta` and points back at its question + * + * Env (same vars the scraper reads): + * - CLICKHOUSE_URL, CLICKHOUSE_USERNAME, CLICKHOUSE_PASSWORD + * - CLICKHOUSE_DATABASE (must be the DB containing `outcome_fills`) + * + * Exits 0 on PASS, 1 on FAIL. PASS = both tables present + coverage gap == 0. + */ +import { query } from '../lib/clickhouse'; + +interface ColumnRow { + name: string; + type: string; +} + +interface CountByStatusRow { + status: string; + rows: string; +} + +interface OutcomeRow { + outcome_id: string; + question_id: string | null; +} + +interface QuestionRow { + question_id: string; + named_outcome_ids: number[]; +} + +interface FreshnessRow { + max_refresh: string; + lag_s: string; +} + +let failures = 0; + +function pass(msg: string) { + console.log(` ✓ ${msg}`); +} + +function fail(msg: string) { + console.log(` ✗ ${msg}`); + failures++; +} + +async function checkSchema(table: string, expected: Record) { + console.log(`\n[schema] ${table}`); + const { data } = await query( + `SELECT name, type FROM system.columns WHERE database = currentDatabase() AND table = {table:String}`, + { table }, + ); + if (data.length === 0) { + fail(`${table} missing — schema not deployed?`); + return; + } + const got = new Map(data.map((r) => [r.name, r.type])); + for (const [col, type] of Object.entries(expected)) { + const actual = got.get(col); + if (!actual) fail(`column ${col} missing`); + else if (!actual.includes(type)) + fail( + `column ${col}: expected type containing "${type}", got "${actual}"`, + ); + else pass(`${col} :: ${actual}`); + } +} + +async function checkRowCounts() { + console.log('\n[rows] state_outcome_meta'); + const { data } = await query( + `SELECT status, toString(count()) AS rows + FROM state_outcome_meta FINAL + GROUP BY status + ORDER BY status`, + ); + if (data.length === 0) { + fail('state_outcome_meta is empty — scraper has not run yet'); + return; + } + let live = 0; + let settled = 0; + for (const r of data) { + const n = Number.parseInt(r.rows, 10); + if (!Number.isFinite(n)) { + fail(`non-numeric row count "${r.rows}" for status "${r.status}"`); + continue; + } + if (r.status === 'live') live = n; + else if (r.status === 'settled') settled = n; + else fail(`unexpected status "${r.status}" (vocab is live | settled)`); + } + pass(`live=${live}, settled=${settled}, total=${live + settled}`); + + const { data: qData } = await query<{ rows: string }>( + `SELECT toString(count()) AS rows FROM state_question_meta FINAL`, + ); + pass(`state_question_meta rows=${qData[0]?.rows ?? '0'}`); +} + +async function checkCoverage() { + console.log('\n[coverage] outcome_fills → state_outcome_meta'); + const { data } = await query<{ missing: string }>( + `SELECT toString(count()) AS missing FROM ( + SELECT DISTINCT outcome_id FROM outcome_fills + ) f + LEFT JOIN (SELECT outcome_id FROM state_outcome_meta FINAL) m USING (outcome_id) + WHERE m.outcome_id IS NULL`, + ); + const missing = Number.parseInt(data[0]?.missing ?? '0', 10); + if (!Number.isFinite(missing)) { + fail(`non-numeric coverage gap "${data[0]?.missing}"`); + return; + } + if (missing === 0) + pass('every outcome_id in outcome_fills has a state_outcome_meta row'); + else fail(`${missing} outcome_ids in outcome_fills lack a meta row`); +} + +async function checkFreshness() { + console.log('\n[freshness] refresh_time lag'); + const { data } = await query( + `SELECT toString(max(refresh_time)) AS max_refresh, + toString(toUInt32(dateDiff('second', max(refresh_time), now()))) AS lag_s + FROM state_outcome_meta`, + ); + const lag = Number.parseInt(data[0]?.lag_s ?? '999999', 10); + if (!Number.isFinite(lag)) { + fail(`non-numeric freshness lag "${data[0]?.lag_s}"`); + return; + } + if (lag <= 900) + pass(`max(refresh_time)=${data[0]?.max_refresh} (lag ${lag}s)`); + else + fail( + `max(refresh_time)=${data[0]?.max_refresh} is ${lag}s old — scraper may be down`, + ); +} + +async function checkQuestionRoundtrip() { + console.log('\n[questions] reverse-map consistency'); + const { data: qs } = await query( + `SELECT toString(question_id) AS question_id, named_outcome_ids + FROM state_question_meta FINAL`, + ); + if (qs.length === 0) { + pass('no questions yet (vacuous)'); + return; + } + const { data: os } = await query( + `SELECT toString(outcome_id) AS outcome_id, + if(isNull(question_id), NULL, toString(question_id)) AS question_id + FROM state_outcome_meta FINAL`, + ); + const outcomeQ = new Map(os.map((r) => [r.outcome_id, r.question_id])); + let mismatches = 0; + for (const q of qs) { + for (const named of q.named_outcome_ids) { + const got = outcomeQ.get(String(named)); + if (got === undefined) { + fail( + `question ${q.question_id} references outcome ${named} which has no meta row`, + ); + mismatches++; + } else if (got !== q.question_id) { + fail( + `outcome ${named}.question_id=${got ?? 'null'} but should be ${q.question_id}`, + ); + mismatches++; + } + } + } + if (mismatches === 0) + pass(`${qs.length} questions × namedOutcomes all back-resolved`); +} + +await checkSchema('state_outcome_meta', { + outcome_id: 'UInt64', + question_id: 'Nullable(UInt64)', + name: 'String', + description: 'String', + side_specs: 'Array(String)', + quote_token: 'LowCardinality(String)', + status: 'LowCardinality(String)', + settle_fraction: 'Nullable(Float64)', + settle_details: 'Nullable(String)', + refresh_time: "DateTime64(3, 'UTC')", +}); +await checkSchema('state_question_meta', { + question_id: 'UInt64', + name: 'String', + description: 'String', + fallback_outcome_id: 'Nullable(UInt64)', + named_outcome_ids: 'Array(UInt64)', + settled_outcome_ids: 'Array(UInt64)', + refresh_time: "DateTime64(3, 'UTC')", +}); + +if (failures === 0) { + await checkRowCounts(); + await checkCoverage(); + await checkFreshness(); + await checkQuestionRoundtrip(); +} + +console.log(`\n${failures === 0 ? 'PASS' : `FAIL (${failures})`}`); +process.exit(failures === 0 ? 0 : 1); diff --git a/services/hyperliquid-outcomes/index.test.ts b/services/hyperliquid-outcomes/index.test.ts new file mode 100644 index 0000000..5c4854a --- /dev/null +++ b/services/hyperliquid-outcomes/index.test.ts @@ -0,0 +1,342 @@ +import { afterEach, beforeEach, describe, expect, mock, test } from 'bun:test'; + +const mockInsert = mock(() => Promise.resolve()); +const mockQuery = mock(() => + Promise.resolve({ + data: [] as { outcome_id: string }[], + metrics: { httpRequestTimeMs: 0, dataFetchTimeMs: 0, totalTimeMs: 0 }, + }), +); +const mockIncrementSuccess = mock(() => {}); +const mockIncrementError = mock(() => {}); +const mockInitService = mock(() => {}); +const mockMarkServiceAlive = mock(() => {}); + +// Mock only the `lib/clickhouse` exports this service imports +// (`insertClient` + `query`). `mock.module` is process-wide, so providing +// fields beyond what's needed would risk shadowing real exports for other +// suites. +mock.module('../../lib/clickhouse', () => ({ + insertClient: { insert: mockInsert }, + query: mockQuery, +})); + +mock.module('../../lib/prometheus', () => ({ + incrementSuccess: mockIncrementSuccess, + incrementError: mockIncrementError, +})); + +mock.module('../../lib/service-init', () => ({ + initService: mockInitService, + markServiceAlive: mockMarkServiceAlive, +})); + +const liveBody = { + outcomes: [ + { + outcome: 104, + name: 'June Fed rate change', + description: 'Resolves to ...', + sideSpecs: [{ name: 'Change' }, { name: 'No Change' }], + quoteToken: 'USDC', + }, + { + outcome: 172, + name: 'Algeria', + description: 'Resolves Yes if ...', + sideSpecs: [{ name: 'Yes' }, { name: 'No' }], + quoteToken: 'USDC', + }, + ], + questions: [ + { + question: 32, + name: '2026 World Cup Champion', + description: 'Each ...', + fallbackOutcome: 171, + namedOutcomes: [172], + settledNamedOutcomes: [], + }, + ], +}; + +const settledBody = { + spec: { + outcome: 0, + name: 'Recurring', + description: 'class:priceBinary|underlying:BTC', + sideSpecs: [{ name: 'Yes' }, { name: 'No' }], + quoteToken: 'USDH', + }, + settleFraction: '0.0', + details: 'price:78212.4', +}; + +interface InsertCallArg { + table: string; + values: Array>; + format: string; +} + +/** + * Build a `mockQuery` implementation that routes to different result sets + * depending on which CH table the SQL references. Tests need this because + * the service fires two reads per cycle — `outcome_fills` for the known + * universe and `state_outcome_meta` for already-settled ids — and feeding + * both the same rows would mark every known id as already-settled and + * suppress the settled-lookup probes the test is verifying. + */ +function mockQueryRouter(routes: { + knownIds?: string[]; + alreadySettled?: string[]; +}) { + return (sql: string) => { + let rows: { outcome_id: string }[] = []; + if (sql.includes('outcome_fills')) { + rows = (routes.knownIds ?? []).map((id) => ({ outcome_id: id })); + } else if (sql.includes('state_outcome_meta')) { + rows = (routes.alreadySettled ?? []).map((id) => ({ + outcome_id: id, + })); + } + return Promise.resolve({ + data: rows, + metrics: { + httpRequestTimeMs: 0, + dataFetchTimeMs: 0, + totalTimeMs: 0, + }, + }); + }; +} + +describe('hyperliquid-outcomes run()', () => { + const originalFetch = globalThis.fetch; + const originalUrl = process.env.HYPERLIQUID_INFO_URL; + + beforeEach(() => { + mockInsert.mockClear(); + mockQuery.mockClear(); + mockIncrementSuccess.mockClear(); + mockIncrementError.mockClear(); + mockInitService.mockClear(); + mockMarkServiceAlive.mockClear(); + mockQuery.mockImplementation(() => + Promise.resolve({ + data: [] as { outcome_id: string }[], + metrics: { + httpRequestTimeMs: 0, + dataFetchTimeMs: 0, + totalTimeMs: 0, + }, + }), + ); + }); + + afterEach(() => { + globalThis.fetch = originalFetch; + if (originalUrl === undefined) { + delete process.env.HYPERLIQUID_INFO_URL; + } else { + process.env.HYPERLIQUID_INFO_URL = originalUrl; + } + }); + + test('throws when HYPERLIQUID_INFO_URL is unset', async () => { + delete process.env.HYPERLIQUID_INFO_URL; + const { run } = await import('./index'); + await expect(run()).rejects.toThrow(/HYPERLIQUID_INFO_URL/); + }); + + test('inserts live + settled outcomes and questions in one cycle', async () => { + process.env.HYPERLIQUID_INFO_URL = 'http://example/info'; + // outcome_fills knows 104, 172, and a settled 0 — 0 is missing from + // the live snapshot and not in already-settled, so the cycle must + // probe settledOutcome for it. + mockQuery.mockImplementation( + mockQueryRouter({ + knownIds: ['104', '172', '0'], + alreadySettled: [], + }), + ); + + globalThis.fetch = mock((_url: string, init: RequestInit) => { + const body = JSON.parse(init.body as string); + if (body.type === 'outcomeMeta') { + return Promise.resolve( + new Response(JSON.stringify(liveBody), { status: 200 }), + ); + } + if (body.type === 'settledOutcome' && body.outcome === 0) { + return Promise.resolve( + new Response(JSON.stringify(settledBody), { status: 200 }), + ); + } + return Promise.resolve(new Response('null', { status: 200 })); + }) as unknown as typeof fetch; + + const { run } = await import('./index'); + await run(); + + expect(mockInsert).toHaveBeenCalledTimes(2); + const calls = mockInsert.mock.calls.map((c) => c[0] as InsertCallArg); + + const outcomeCall = calls.find((c) => c.table === 'state_outcome_meta'); + const questionCall = calls.find( + (c) => c.table === 'state_question_meta', + ); + expect(outcomeCall).toBeDefined(); + expect(questionCall).toBeDefined(); + expect(outcomeCall!.format).toBe('JSONEachRow'); + + // 2 live + 1 settled + expect(outcomeCall!.values).toHaveLength(3); + const live104 = outcomeCall!.values.find((r) => r.outcome_id === 104); + const settled0 = outcomeCall!.values.find((r) => r.outcome_id === 0); + const live172 = outcomeCall!.values.find((r) => r.outcome_id === 172); + expect(live104?.status).toBe('live'); + expect(settled0?.status).toBe('settled'); + expect(settled0?.settle_fraction).toBe(0); + expect(settled0?.settle_details).toBe('price:78212.4'); + // 172 belongs to question 32 via namedOutcomes — reverse map must apply. + expect(live172?.question_id).toBe(32); + // 104 is standalone. + expect(live104?.question_id).toBeNull(); + + expect(questionCall!.values).toHaveLength(1); + expect(questionCall!.values[0]!.question_id).toBe(32); + + expect(mockIncrementSuccess).toHaveBeenCalledTimes(1); + expect(mockIncrementError).not.toHaveBeenCalled(); + // Direct-insert services bump the wall-clock heartbeat so the + // liveness probe reflects progress (batch queue is unused here). + expect(mockMarkServiceAlive).toHaveBeenCalledTimes(1); + }); + + test('proceeds when known-ids query fails (cold cluster)', async () => { + process.env.HYPERLIQUID_INFO_URL = 'http://example/info'; + mockQuery.mockImplementation(() => + Promise.reject(new Error('table not found')), + ); + globalThis.fetch = mock(() => + Promise.resolve( + new Response(JSON.stringify(liveBody), { status: 200 }), + ), + ) as unknown as typeof fetch; + + const { run } = await import('./index'); + await run(); + + // Still inserted live outcomes + questions. + expect(mockInsert).toHaveBeenCalledTimes(2); + expect(mockIncrementSuccess).toHaveBeenCalledTimes(1); + expect(mockIncrementError).not.toHaveBeenCalled(); + }); + + test('continues past per-id settledOutcome failures without aborting the cycle', async () => { + process.env.HYPERLIQUID_INFO_URL = 'http://example/info'; + mockQuery.mockImplementation( + mockQueryRouter({ + knownIds: ['999'], + alreadySettled: [], + }), + ); + globalThis.fetch = mock((_url: string, init: RequestInit) => { + const body = JSON.parse(init.body as string); + if (body.type === 'outcomeMeta') { + return Promise.resolve( + new Response(JSON.stringify(liveBody), { status: 200 }), + ); + } + // settledOutcome lookups fail — cycle should still complete with + // the live snapshot. + return Promise.resolve(new Response('boom', { status: 502 })); + }) as unknown as typeof fetch; + + const { run } = await import('./index'); + await run(); + + expect(mockInsert).toHaveBeenCalledTimes(2); + const outcomeCall = mockInsert.mock.calls + .map((c) => c[0] as InsertCallArg) + .find((c) => c.table === 'state_outcome_meta'); + // Only the 2 live outcomes — settled lookup failed and was skipped. + expect(outcomeCall!.values).toHaveLength(2); + expect(mockIncrementSuccess).toHaveBeenCalledTimes(1); + }); + + test('skips settledOutcome probes for ids already captured as settled', async () => { + process.env.HYPERLIQUID_INFO_URL = 'http://example/info'; + // outcome_fills has 104, 172, and a historically-settled 0; but our + // own `state_outcome_meta` already contains 0 with status='settled', + // so the cycle must NOT re-probe HL for it. + mockQuery.mockImplementation( + mockQueryRouter({ + knownIds: ['104', '172', '0'], + alreadySettled: ['0'], + }), + ); + const settledProbed: number[] = []; + globalThis.fetch = mock((_url: string, init: RequestInit) => { + const body = JSON.parse(init.body as string); + if (body.type === 'outcomeMeta') { + return Promise.resolve( + new Response(JSON.stringify(liveBody), { status: 200 }), + ); + } + if (body.type === 'settledOutcome') { + settledProbed.push(body.outcome); + } + return Promise.resolve(new Response('null', { status: 200 })); + }) as unknown as typeof fetch; + + const { run } = await import('./index'); + await run(); + + expect(settledProbed).toEqual([]); + const outcomeCall = mockInsert.mock.calls + .map((c) => c[0] as InsertCallArg) + .find((c) => c.table === 'state_outcome_meta'); + // Only the 2 live outcomes inserted this cycle — settled 0 stayed in + // the table from the prior cycle's insert (RMT keeps the older row). + expect(outcomeCall!.values).toHaveLength(2); + expect(mockIncrementSuccess).toHaveBeenCalledTimes(1); + expect(mockMarkServiceAlive).toHaveBeenCalledTimes(1); + }); + + test('returns early without success metric when outcomeMeta is empty', async () => { + process.env.HYPERLIQUID_INFO_URL = 'http://example/info'; + globalThis.fetch = mock(() => + Promise.resolve( + new Response(JSON.stringify({ outcomes: [], questions: [] }), { + status: 200, + }), + ), + ) as unknown as typeof fetch; + + const { run } = await import('./index'); + await run(); + + // Empty universe is a soft anomaly — don't insert, don't claim + // success, but also don't throw. Next cycle retries. + expect(mockInsert).not.toHaveBeenCalled(); + expect(mockIncrementSuccess).not.toHaveBeenCalled(); + expect(mockIncrementError).not.toHaveBeenCalled(); + expect(mockMarkServiceAlive).not.toHaveBeenCalled(); + }); + + test('records an error metric and rethrows when outcomeMeta fetch fails', async () => { + process.env.HYPERLIQUID_INFO_URL = 'http://example/info'; + globalThis.fetch = mock(() => + Promise.resolve(new Response('nope', { status: 502 })), + ) as unknown as typeof fetch; + + const { run } = await import('./index'); + await expect(run()).rejects.toThrow(); + expect(mockIncrementError).toHaveBeenCalledTimes(1); + expect(mockInsert).not.toHaveBeenCalled(); + // Liveness heartbeat must NOT advance on a failed cycle, so silent + // upstream failures still surface via the `/live` probe. + expect(mockMarkServiceAlive).not.toHaveBeenCalled(); + }); +}); diff --git a/services/hyperliquid-outcomes/index.ts b/services/hyperliquid-outcomes/index.ts new file mode 100644 index 0000000..1515eb9 --- /dev/null +++ b/services/hyperliquid-outcomes/index.ts @@ -0,0 +1,246 @@ +import PQueue from 'p-queue'; +import { insertClient, query } from '../../lib/clickhouse'; +import { createLogger } from '../../lib/logger'; +import { incrementError, incrementSuccess } from '../../lib/prometheus'; +import { initService, markServiceAlive } from '../../lib/service-init'; +import { + buildLiveOutcomeRow, + buildOutcomeToQuestion, + buildQuestionRow, + buildSettledOutcomeRow, + fetchOutcomeMeta, + fetchSettledOutcome, + type OutcomeMetaRow, + type QuestionMetaRow, +} from './info'; + +const serviceName = 'hyperliquid-outcomes'; +const log = createLogger(serviceName); + +/** + * Max in-flight `settledOutcome` lookups per cycle. Cold-start probes ~200 + * settled ids against api.hyperliquid.xyz; default 4 keeps us well under + * HL's per-IP allowance while finishing the cold-start sweep in ~50s. + */ +const SETTLED_CONCURRENCY = (() => { + const parsed = Number.parseInt( + process.env.HL_OUTCOMES_SETTLED_CONCURRENCY ?? '', + 10, + ); + return Number.isFinite(parsed) && parsed > 0 ? parsed : 4; +})(); + +/** + * Discover all `outcome_id`s the substream has ever written into the live DB. + * Settled outcomes drop out of `outcomeMeta`, so this is the only source of + * truth for which ids need a `settledOutcome` probe. + */ +async function fetchKnownOutcomeIds(): Promise> { + const { data } = await query<{ outcome_id: string }>( + 'SELECT DISTINCT toString(outcome_id) AS outcome_id FROM outcome_fills', + ); + return parseUint64Set(data); +} + +/** + * Discover outcome_ids we've already captured as `status='settled'`. Settled + * payloads are immutable on the HL side, so once we have one we never need to + * re-probe — skipping them turns the steady-state cycle into a no-op on the + * Info API even as the cumulative settled universe grows. + */ +async function fetchAlreadySettledIds(): Promise> { + const { data } = await query<{ outcome_id: string }>( + "SELECT toString(outcome_id) AS outcome_id FROM state_outcome_meta FINAL WHERE status = 'settled'", + ); + return parseUint64Set(data); +} + +function parseUint64Set(rows: { outcome_id: string }[]): Set { + const ids = new Set(); + for (const row of rows) { + const n = Number.parseInt(row.outcome_id, 10); + if (Number.isFinite(n)) ids.add(n); + } + return ids; +} + +/** + * Format a `DateTime64(3, 'UTC')`-compatible timestamp. CH rejects the + * trailing `Z` but accepts the millisecond fraction, and we preserve ms so + * closely-spaced polls remain deterministic for RMT merges (same convention + * as the sibling `hyperliquid` service). + */ +function nowRefreshTime(): string { + return new Date().toISOString().slice(0, 23).replace('T', ' '); +} + +/** + * One poll cycle: + * 1. Pull `outcomeMeta` (live outcomes + question groupings). + * 2. Read distinct `outcome_id` from `outcome_fills` to discover settled + * outcomes not in the live snapshot. + * 3. For each settled id, `settledOutcome` lookup (concurrency-bounded). + * 4. Insert all rows with a single `refresh_time`. RMT collapses repeated + * rows on subsequent polls. + * + * The CLI runner loops with `AUTO_RESTART_DELAY` between cycles, so one + * `run()` call = one snapshot. + */ +export async function run(): Promise { + initService({ serviceName }); + + const infoUrl = process.env.HYPERLIQUID_INFO_URL; + if (!infoUrl) { + throw new Error( + 'HYPERLIQUID_INFO_URL is required (set to a Hyperliquid /info endpoint)', + ); + } + + log.info('Fetching outcome metadata'); + const startTime = performance.now(); + + let meta: Awaited>; + try { + meta = await fetchOutcomeMeta(infoUrl); + } catch (error) { + log.error('Failed to fetch outcomeMeta', { error }); + incrementError(serviceName); + throw error; + } + + const liveIds = new Set(meta.outcomes.map((o) => o.outcome)); + log.info('Fetched live outcomeMeta', { + outcomes: meta.outcomes.length, + questions: meta.questions.length, + }); + + if (meta.outcomes.length === 0 && meta.questions.length === 0) { + // Empty universe is unusual (HL has had standing outcomes for months). + // Treat as a soft failure: don't bump the success counter, don't + // overwrite live rows on RMT, but also don't error-throw — next + // cycle will retry. + log.warn('outcomeMeta returned empty outcomes and questions'); + return; + } + + // Cold cluster (`outcome_fills` empty/absent) is non-fatal — proceed + // with just the live snapshot so the first cycle still records the + // current universe. + const knownIds = await fetchKnownOutcomeIds().catch((error) => { + log.warn( + 'Failed to read known outcome_ids — proceeding with live only', + { error }, + ); + return new Set(); + }); + const alreadySettled = await fetchAlreadySettledIds().catch((error) => { + log.warn('Failed to read already-settled outcome_ids — will re-probe', { + error, + }); + return new Set(); + }); + + const settledIds = [...knownIds].filter( + (id) => !liveIds.has(id) && !alreadySettled.has(id), + ); + log.info('Resolved outcome universe', { + known: knownIds.size, + live: liveIds.size, + alreadySettled: alreadySettled.size, + settledLookups: settledIds.length, + settledConcurrency: SETTLED_CONCURRENCY, + }); + + const refresh_time = nowRefreshTime(); + const outcomeToQuestion = buildOutcomeToQuestion(meta.questions); + + const outcomeRows: OutcomeMetaRow[] = meta.outcomes.map((o) => + buildLiveOutcomeRow( + o, + outcomeToQuestion.get(o.outcome) ?? null, + refresh_time, + ), + ); + + const queue = new PQueue({ concurrency: SETTLED_CONCURRENCY }); + let settledFetched = 0; + let settledStillLive = 0; + let settledErrored = 0; + await Promise.all( + settledIds.map((id) => + queue.add(async () => { + try { + const s = await fetchSettledOutcome(infoUrl, id); + if (s === null) { + // HL returned `null` — outcome must have re-opened or + // is in a transient between-state. Skip; next cycle + // will retry naturally. + settledStillLive++; + return; + } + outcomeRows.push( + buildSettledOutcomeRow( + s, + outcomeToQuestion.get(s.spec.outcome) ?? null, + refresh_time, + ), + ); + settledFetched++; + } catch (error) { + // Per-id failures should not abort the whole cycle. + // Transient HL outages will retry on the next poll. + settledErrored++; + log.warn('settledOutcome lookup failed', { + outcomeId: id, + error, + }); + } + }), + ), + ); + + const questionRows: QuestionMetaRow[] = meta.questions.map((q) => + buildQuestionRow(q, refresh_time), + ); + + try { + if (outcomeRows.length > 0) { + await insertClient.insert({ + table: 'state_outcome_meta', + values: outcomeRows, + format: 'JSONEachRow', + }); + } + if (questionRows.length > 0) { + await insertClient.insert({ + table: 'state_question_meta', + values: questionRows, + format: 'JSONEachRow', + }); + } + } catch (error) { + log.error('Failed to insert outcome metadata', { error }); + incrementError(serviceName); + throw error; + } + + const cycleMs = Math.round(performance.now() - startTime); + log.info('Inserted outcome metadata', { + outcomeRows: outcomeRows.length, + questionRows: questionRows.length, + settledFetched, + settledStillLive, + settledErrored, + cycleMs, + }); + // We insert directly via `insertClient` rather than the batch-insert + // queue, so the queue's `getLastSuccessfulFlushAt()` never advances. + // Bump the wall-clock heartbeat here so `/live` reflects real progress + // after the startup grace window. + markServiceAlive(); + incrementSuccess(serviceName); +} + +if (import.meta.main) { + await run(); +} diff --git a/services/hyperliquid-outcomes/info.test.ts b/services/hyperliquid-outcomes/info.test.ts new file mode 100644 index 0000000..afdcdc0 --- /dev/null +++ b/services/hyperliquid-outcomes/info.test.ts @@ -0,0 +1,251 @@ +import { afterEach, describe, expect, mock, test } from 'bun:test'; +import { + buildLiveOutcomeRow, + buildOutcomeToQuestion, + buildQuestionRow, + buildSettledOutcomeRow, + fetchOutcomeMeta, + fetchSettledOutcome, + type HyperliquidOutcomeMeta, + type HyperliquidOutcomeSpec, + type HyperliquidQuestion, + type HyperliquidSettledOutcome, +} from './info'; + +const REFRESH = '2026-06-11 12:00:00.000'; + +const liveOutcome: HyperliquidOutcomeSpec = { + outcome: 104, + name: 'June Fed rate change', + description: 'Resolves to Change if ...', + sideSpecs: [{ name: 'Change' }, { name: 'No Change' }], + quoteToken: 'USDC', +}; + +const worldCupQuestion: HyperliquidQuestion = { + question: 32, + name: '2026 World Cup Champion', + description: 'Each associated outcome ...', + fallbackOutcome: 171, + namedOutcomes: [172, 173, 174], + settledNamedOutcomes: [], +}; + +describe('buildOutcomeToQuestion', () => { + test('maps namedOutcomes and fallbackOutcome to their parent question', () => { + const map = buildOutcomeToQuestion([worldCupQuestion]); + expect(map.get(171)).toBe(32); + expect(map.get(172)).toBe(32); + expect(map.get(173)).toBe(32); + expect(map.get(174)).toBe(32); + expect(map.has(999)).toBe(false); + }); + + test('handles a question with a null fallbackOutcome', () => { + const q: HyperliquidQuestion = { + ...worldCupQuestion, + fallbackOutcome: null, + namedOutcomes: [200, 201], + }; + const map = buildOutcomeToQuestion([q]); + expect(map.get(200)).toBe(32); + expect(map.get(201)).toBe(32); + expect(map.size).toBe(2); + }); + + test('returns an empty map for zero questions', () => { + expect(buildOutcomeToQuestion([]).size).toBe(0); + }); +}); + +describe('buildLiveOutcomeRow', () => { + test('flattens sideSpecs and marks status=live', () => { + const row = buildLiveOutcomeRow(liveOutcome, null, REFRESH); + expect(row).toEqual({ + outcome_id: 104, + question_id: null, + name: 'June Fed rate change', + description: 'Resolves to Change if ...', + side_specs: ['Change', 'No Change'], + quote_token: 'USDC', + status: 'live', + settle_fraction: null, + settle_details: null, + refresh_time: REFRESH, + }); + }); + + test('carries through a parent question_id when supplied', () => { + const row = buildLiveOutcomeRow( + { ...liveOutcome, outcome: 172 }, + 32, + REFRESH, + ); + expect(row.question_id).toBe(32); + }); +}); + +describe('buildSettledOutcomeRow', () => { + const settled: HyperliquidSettledOutcome = { + spec: { + outcome: 0, + name: 'Recurring', + description: + 'class:priceBinary|underlying:BTC|expiry:20260503-0600|targetPrice:78213|period:1d', + sideSpecs: [{ name: 'Yes' }, { name: 'No' }], + quoteToken: 'USDH', + }, + settleFraction: '0.0', + details: 'price:78212.4', + }; + + test('parses settleFraction to float and marks status=settled', () => { + const row = buildSettledOutcomeRow(settled, null, REFRESH); + expect(row.status).toBe('settled'); + expect(row.settle_fraction).toBe(0); + expect(row.settle_details).toBe('price:78212.4'); + expect(row.outcome_id).toBe(0); + expect(row.name).toBe('Recurring'); + expect(row.quote_token).toBe('USDH'); + expect(row.side_specs).toEqual(['Yes', 'No']); + }); + + test('preserves scalar settleFraction values', () => { + const row = buildSettledOutcomeRow( + { ...settled, settleFraction: '0.42' }, + null, + REFRESH, + ); + expect(row.settle_fraction).toBe(0.42); + }); + + test('coerces a non-numeric settleFraction to null instead of NaN', () => { + const row = buildSettledOutcomeRow( + { ...settled, settleFraction: 'bogus' }, + null, + REFRESH, + ); + expect(row.settle_fraction).toBeNull(); + }); +}); + +describe('buildQuestionRow', () => { + test('flattens question metadata into the row shape', () => { + expect(buildQuestionRow(worldCupQuestion, REFRESH)).toEqual({ + question_id: 32, + name: '2026 World Cup Champion', + description: 'Each associated outcome ...', + fallback_outcome_id: 171, + named_outcome_ids: [172, 173, 174], + settled_outcome_ids: [], + refresh_time: REFRESH, + }); + }); + + test('preserves a null fallbackOutcome', () => { + const row = buildQuestionRow( + { ...worldCupQuestion, fallbackOutcome: null }, + REFRESH, + ); + expect(row.fallback_outcome_id).toBeNull(); + }); +}); + +describe('fetchOutcomeMeta', () => { + const originalFetch = globalThis.fetch; + afterEach(() => { + globalThis.fetch = originalFetch; + }); + + test('parses a well-formed outcomeMeta response', async () => { + const body: HyperliquidOutcomeMeta = { + outcomes: [liveOutcome], + questions: [worldCupQuestion], + }; + globalThis.fetch = mock(() => + Promise.resolve( + new Response(JSON.stringify(body), { status: 200 }), + ), + ) as unknown as typeof fetch; + const got = await fetchOutcomeMeta('http://example/info'); + expect(got).toEqual(body); + }); + + test('throws on non-2xx response', async () => { + globalThis.fetch = mock(() => + Promise.resolve(new Response('nope', { status: 500 })), + ) as unknown as typeof fetch; + await expect(fetchOutcomeMeta('http://example/info')).rejects.toThrow( + /HTTP 500/, + ); + }); + + test('throws when outcomes or questions are missing', async () => { + globalThis.fetch = mock(() => + Promise.resolve( + new Response(JSON.stringify({ outcomes: [] }), { status: 200 }), + ), + ) as unknown as typeof fetch; + await expect(fetchOutcomeMeta('http://example/info')).rejects.toThrow( + /missing outcomes\/questions/, + ); + }); +}); + +describe('fetchSettledOutcome', () => { + const originalFetch = globalThis.fetch; + afterEach(() => { + globalThis.fetch = originalFetch; + }); + + test('returns null when HL responds with top-level null (still live)', async () => { + globalThis.fetch = mock(() => + Promise.resolve(new Response('null', { status: 200 })), + ) as unknown as typeof fetch; + expect( + await fetchSettledOutcome('http://example/info', 104), + ).toBeNull(); + }); + + test('returns the spec + settlement when present', async () => { + const body: HyperliquidSettledOutcome = { + spec: { + outcome: 0, + name: 'Recurring', + description: 'class:priceBinary|underlying:BTC', + sideSpecs: [{ name: 'Yes' }, { name: 'No' }], + quoteToken: 'USDH', + }, + settleFraction: '1.0', + details: 'price:80000.0', + }; + globalThis.fetch = mock(() => + Promise.resolve( + new Response(JSON.stringify(body), { status: 200 }), + ), + ) as unknown as typeof fetch; + expect(await fetchSettledOutcome('http://example/info', 0)).toEqual( + body, + ); + }); + + test('returns null on malformed payload (missing settleFraction)', async () => { + globalThis.fetch = mock(() => + Promise.resolve( + new Response(JSON.stringify({ spec: liveOutcome }), { + status: 200, + }), + ), + ) as unknown as typeof fetch; + expect(await fetchSettledOutcome('http://example/info', 1)).toBeNull(); + }); + + test('throws on non-2xx response', async () => { + globalThis.fetch = mock(() => + Promise.resolve(new Response('boom', { status: 503 })), + ) as unknown as typeof fetch; + await expect( + fetchSettledOutcome('http://example/info', 1), + ).rejects.toThrow(/HTTP 503/); + }); +}); diff --git a/services/hyperliquid-outcomes/info.ts b/services/hyperliquid-outcomes/info.ts new file mode 100644 index 0000000..0554ea1 --- /dev/null +++ b/services/hyperliquid-outcomes/info.ts @@ -0,0 +1,243 @@ +import { createLogger } from '../../lib/logger'; + +/** + * Per-request timeout for Hyperliquid Info API calls. Shared with the spotMeta + * service via the same env var so both pollers tolerate the same upstream + * slowness budget. Falls back to 30s if the override is missing or invalid. + */ +const FETCH_TIMEOUT_MS = (() => { + const parsed = Number.parseInt( + process.env.HYPERLIQUID_FETCH_TIMEOUT_MS ?? '', + 10, + ); + return Number.isFinite(parsed) && parsed > 0 ? parsed : 30000; +})(); + +const log = createLogger('hyperliquid-outcomes'); + +/** + * One outcome entry as returned by `POST /info {type: outcomeMeta}` and (under + * `spec`) by `POST /info {type: settledOutcome, outcome: N}`. + * + * `sideSpecs` is positional — `side_index` on `outcome_fills` indexes directly + * into this array. Typical shapes: `[{name:'Yes'},{name:'No'}]` for binary + * outcomes, or domain-specific labels like `[{name:'Change'},{name:'No Change'}]`. + */ +export interface HyperliquidOutcomeSpec { + outcome: number; + name: string; + description: string; + sideSpecs: Array<{ name: string }>; + quoteToken: string; +} + +/** + * One question entry — multi-outcome grouping like a "World Cup Champion" + * question that aggregates per-team outcomes. + * + * `namedOutcomes` lists the outcome ids that compose the question; + * `fallbackOutcome` is the outcome id used when none of the named ones resolve + * YES; `settledNamedOutcomes` is the subset already resolved. + */ +export interface HyperliquidQuestion { + question: number; + name: string; + description: string; + fallbackOutcome: number | null; + namedOutcomes: number[]; + settledNamedOutcomes: number[]; +} + +export interface HyperliquidOutcomeMeta { + outcomes: HyperliquidOutcomeSpec[]; + questions: HyperliquidQuestion[]; +} + +/** + * Settlement payload returned by `POST /info {type: settledOutcome, outcome: N}` + * when the outcome has resolved. Returns `null` (top-level) for outcomes still + * live — callers must distinguish that case before parsing. + */ +export interface HyperliquidSettledOutcome { + spec: HyperliquidOutcomeSpec; + settleFraction: string; + details: string; +} + +/** Row shape inserted into `state_outcome_meta`. */ +export interface OutcomeMetaRow { + outcome_id: number; + question_id: number | null; + name: string; + description: string; + side_specs: string[]; + quote_token: string; + status: 'live' | 'settled'; + settle_fraction: number | null; + settle_details: string | null; + refresh_time: string; +} + +/** Row shape inserted into `state_question_meta`. */ +export interface QuestionMetaRow { + question_id: number; + name: string; + description: string; + fallback_outcome_id: number | null; + named_outcome_ids: number[]; + settled_outcome_ids: number[]; + refresh_time: string; +} + +async function postInfo( + infoUrl: string, + body: Record, +): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); + try { + const response = await fetch(infoUrl, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + signal: controller.signal, + }); + if (!response.ok) { + throw new Error( + `Hyperliquid /info returned HTTP ${response.status}`, + ); + } + return (await response.json()) as T; + } finally { + clearTimeout(timeout); + } +} + +/** + * Fetch the live outcome universe + question groupings. `outcomes` only + * contains outcomes whose markets are still open — settled outcomes drop off + * and must be recovered via `fetchSettledOutcome`. + */ +export async function fetchOutcomeMeta( + infoUrl: string, +): Promise { + const body = await postInfo(infoUrl, { + type: 'outcomeMeta', + }); + if (!Array.isArray(body.outcomes) || !Array.isArray(body.questions)) { + throw new Error( + 'Hyperliquid /info outcomeMeta response missing outcomes/questions arrays', + ); + } + return body; +} + +/** + * Recover a settled outcome's spec + resolution payload. Returns `null` when + * the outcome is still live (HL returns top-level `null`). + */ +export async function fetchSettledOutcome( + infoUrl: string, + outcomeId: number, +): Promise { + const body = await postInfo(infoUrl, { + type: 'settledOutcome', + outcome: outcomeId, + }); + if (body === null) return null; + if ( + typeof body !== 'object' || + body.spec === undefined || + typeof body.settleFraction !== 'string' + ) { + log.warn('settledOutcome response missing expected fields', { + outcomeId, + }); + return null; + } + return body; +} + +/** + * Build the reverse index from outcome_id → parent question_id. Walks each + * question's `namedOutcomes` + `fallbackOutcome` and records the question id + * for every referenced outcome. An outcome can only belong to one question on + * the HL side, so last-write-wins is fine (and not exercised in practice). + */ +export function buildOutcomeToQuestion( + questions: HyperliquidQuestion[], +): Map { + const map = new Map(); + for (const q of questions) { + for (const id of q.namedOutcomes) map.set(id, q.question); + if (q.fallbackOutcome !== null) { + map.set(q.fallbackOutcome, q.question); + } + } + return map; +} + +/** Project a live `outcomeMeta` outcome into a `state_outcome_meta` row. */ +export function buildLiveOutcomeRow( + outcome: HyperliquidOutcomeSpec, + questionId: number | null, + refreshTime: string, +): OutcomeMetaRow { + return { + outcome_id: outcome.outcome, + question_id: questionId, + name: outcome.name, + description: outcome.description, + side_specs: outcome.sideSpecs.map((s) => s.name), + quote_token: outcome.quoteToken, + status: 'live', + settle_fraction: null, + settle_details: null, + refresh_time: refreshTime, + }; +} + +/** + * Project a `settledOutcome` response into a `state_outcome_meta` row. + * Re-uses the spec embedded in the settlement payload so name/description + * survive after the outcome drops out of the live universe. + * + * `settleFraction` comes off the wire as a string ("0.0" / "1.0" / scalar); + * we coerce to Float64. Non-numeric strings produce `null` so a malformed + * payload doesn't poison the row. + */ +export function buildSettledOutcomeRow( + settled: HyperliquidSettledOutcome, + questionId: number | null, + refreshTime: string, +): OutcomeMetaRow { + const fraction = Number.parseFloat(settled.settleFraction); + return { + outcome_id: settled.spec.outcome, + question_id: questionId, + name: settled.spec.name, + description: settled.spec.description, + side_specs: settled.spec.sideSpecs.map((s) => s.name), + quote_token: settled.spec.quoteToken, + status: 'settled', + settle_fraction: Number.isFinite(fraction) ? fraction : null, + settle_details: settled.details, + refresh_time: refreshTime, + }; +} + +/** Project a question into a `state_question_meta` row. */ +export function buildQuestionRow( + question: HyperliquidQuestion, + refreshTime: string, +): QuestionMetaRow { + return { + question_id: question.question, + name: question.name, + description: question.description, + fallback_outcome_id: question.fallbackOutcome, + named_outcome_ids: question.namedOutcomes, + settled_outcome_ids: question.settledNamedOutcomes, + refresh_time: refreshTime, + }; +} diff --git a/sql.schemas/schema.hyperliquid_outcomes.sql b/sql.schemas/schema.hyperliquid_outcomes.sql new file mode 100644 index 0000000..0ee2dea --- /dev/null +++ b/sql.schemas/schema.hyperliquid_outcomes.sql @@ -0,0 +1,43 @@ +-- Hyperliquid outcome (HIP-4) metadata — outcome + question lookups. +-- +-- Populated by the `hyperliquid-outcomes` scraper service polling the +-- Hyperliquid Info API: +-- * `POST /info {type: outcomeMeta}` → currently-live outcomes + questions +-- * `POST /info {type: settledOutcome, outcome: N}` → settled outcomes (per-id recovery) +-- +-- `outcomeMeta` only returns currently-active outcomes; settled outcomes drop +-- out of the response. The scraper discovers unknown ids by reading +-- `outcome_fills.outcome_id` (substreams-written) and probes `settledOutcome` +-- for any not in the live snapshot. ReplacingMergeTree on `refresh_time` +-- collapses repeated upserts of the same id. +-- +-- Token API joins these tables on `outcome_id` / `question_id` to expose +-- human-readable labels on the `/v1/hyperliquid/outcomes/*` family. +CREATE TABLE IF NOT EXISTS state_outcome_meta ( + outcome_id UInt64 COMMENT 'matches outcome_fills.outcome_id', + question_id Nullable(UInt64) COMMENT 'parent question if grouped via questions[].namedOutcomes or fallbackOutcome; NULL for standalone outcomes', + name String COMMENT 'human-readable outcome name (e.g. "June Fed rate change") or "Recurring" for daily price binaries', + description String COMMENT 'full resolution description; for "Recurring" outcomes encodes class:|underlying:|expiry:|targetPrice:|period: spec — kept raw, parsed at query time', + side_specs Array(String) COMMENT 'positional side labels; outcome_fills.side_index indexes directly into this array (typically [Yes, No] or domain-specific labels)', + quote_token LowCardinality(String) COMMENT 'settlement token symbol (USDC, USDH, ...)', + status LowCardinality(String) COMMENT 'live | settled — live = present in current outcomeMeta; settled = recovered via settledOutcome', + settle_fraction Nullable(Float64) COMMENT 'NULL for live; 0.0/1.0/scalar for settled (resolution payout fraction per share)', + settle_details Nullable(String) COMMENT 'NULL for live; raw HL details string for settled (e.g. "price:78212.4")', + refresh_time DateTime64(3, 'UTC') COMMENT 'snapshot time for this row (ms precision so closely-spaced polls remain deterministic for ReplacingMergeTree merges)' +) +ENGINE = ReplacingMergeTree(refresh_time) +ORDER BY (outcome_id) +COMMENT 'Hyperliquid HIP-4 outcome metadata populated by token-api-scraper'; + +CREATE TABLE IF NOT EXISTS state_question_meta ( + question_id UInt64 COMMENT 'matches questions[].question on outcomeMeta', + name String COMMENT 'human-readable question name (e.g. "2026 World Cup Champion")', + description String COMMENT 'full resolution description; kept raw', + fallback_outcome_id Nullable(UInt64) COMMENT 'questions[].fallbackOutcome — outcome id used when none of the named outcomes resolve YES', + named_outcome_ids Array(UInt64) COMMENT 'questions[].namedOutcomes — outcome ids that compose this question', + settled_outcome_ids Array(UInt64) COMMENT 'questions[].settledNamedOutcomes — subset of named_outcome_ids that have already resolved', + refresh_time DateTime64(3, 'UTC') COMMENT 'snapshot time for this row (ms precision so closely-spaced polls remain deterministic for ReplacingMergeTree merges)' +) +ENGINE = ReplacingMergeTree(refresh_time) +ORDER BY (question_id) +COMMENT 'Hyperliquid HIP-4 question grouping populated by token-api-scraper';