diff --git a/apps/ui/app/providers/ProviderCard.tsx b/apps/ui/app/providers/ProviderCard.tsx index 761f7cdc..9b085d6a 100644 --- a/apps/ui/app/providers/ProviderCard.tsx +++ b/apps/ui/app/providers/ProviderCard.tsx @@ -5,8 +5,8 @@ import Link from 'next/link'; import type { Route } from 'next'; import { useMemo } from 'react'; -import { AnimatedNumber } from '@/components/AnimatedNumber'; -import { StatusIndicator } from '@/components/StatusIndicator'; +import { AnimatedNumber } from '@/components/ui/animated-number'; +import { StatusIndicator } from '@/components/ui/status-indicator'; import { formatPercent, formatUSD } from '@/lib/format'; import { useMarketStream } from '@/lib/hooks/use-market-stream'; import type { ProviderSummary } from '@/lib/server/providers'; diff --git a/apps/ui/app/providers/ProviderStatsStrip.tsx b/apps/ui/app/providers/ProviderStatsStrip.tsx index 15c0f73f..9b955051 100644 --- a/apps/ui/app/providers/ProviderStatsStrip.tsx +++ b/apps/ui/app/providers/ProviderStatsStrip.tsx @@ -2,8 +2,8 @@ import { useMemo, type ReactNode } from 'react'; -import { AnimatedNumber } from '@/components/AnimatedNumber'; -import { StatusIndicator } from '@/components/StatusIndicator'; +import { AnimatedNumber } from '@/components/ui/animated-number'; +import { StatusIndicator } from '@/components/ui/status-indicator'; import { formatPercent, formatUSD } from '@/lib/format'; import { useMarketStream } from '@/lib/hooks/use-market-stream'; import { cn } from '@/lib/utils'; diff --git a/apps/ui/app/providers/[provider]/depthBook.tsx b/apps/ui/app/providers/[provider]/depthBook.tsx index b73227e5..6ac3fed6 100644 --- a/apps/ui/app/providers/[provider]/depthBook.tsx +++ b/apps/ui/app/providers/[provider]/depthBook.tsx @@ -2,7 +2,7 @@ import { useEffect, useMemo, useRef, useState } from 'react'; -import { AnimatedNumber } from '@/components/AnimatedNumber'; +import { AnimatedNumber } from '@/components/ui/animated-number'; import { cardStyles } from '@/styles/design-tokens'; import { Card, CardContent } from '@/components/ui/card'; import { formatAdaptiveNumber, formatNumber, formatTimestamp, formatUSD } from '@/lib/format'; @@ -143,6 +143,12 @@ export const DepthBook = ({ asset, orderbook, metadata, summary, loading, error const displayedDepth = hydrated ? animatedDepth : staticDepth; const isEmpty = depthTarget.bids.length === 0 && depthTarget.asks.length === 0; + const aggregateTotals = useMemo( + () => computeTotals(depthTarget.bids, depthTarget.asks), + [depthTarget] + ); + const footerTotals = + depthTarget.bids.length > 0 || depthTarget.asks.length > 0 ? aggregateTotals : displayedDepth.totals; return ( @@ -199,7 +205,7 @@ export const DepthBook = ({ asset, orderbook, metadata, summary, loading, error Bid Depth (Qty)

formatAdaptiveNumber(value)} className="font-mono text-lg font-normal text-[#00FFA3] tabular-nums tracking-[-0.01em]" /> @@ -209,7 +215,7 @@ export const DepthBook = ({ asset, orderbook, metadata, summary, loading, error Ask Depth (Qty)

formatAdaptiveNumber(value)} className="font-mono text-lg font-normal text-[#FF3B69] tabular-nums tracking-[-0.01em]" /> @@ -219,9 +225,7 @@ export const DepthBook = ({ asset, orderbook, metadata, summary, loading, error Total Book (USD)

formatUSD(value, { fractionDigits: 0 })} className="font-mono text-lg font-normal text-slate-200 tabular-nums tracking-[-0.01em]" /> diff --git a/apps/ui/app/providers/[provider]/providerDashboard.tsx b/apps/ui/app/providers/[provider]/providerDashboard.tsx index 530a1c4b..a3797fe7 100644 --- a/apps/ui/app/providers/[provider]/providerDashboard.tsx +++ b/apps/ui/app/providers/[provider]/providerDashboard.tsx @@ -19,7 +19,7 @@ import { sortMarketsByOpenInterest, type HeroOverview } from '@/components/provider-dashboard/utils/overview'; -import { StatusIndicator } from '@/components/StatusIndicator'; +import { StatusIndicator } from '@/components/ui/status-indicator'; import { useMarketStream } from '@/lib/hooks/use-market-stream'; import { cn } from '@/lib/utils'; import { dashboardTypography, typography } from '@/styles/typography'; diff --git a/apps/ui/components/AnimatedNumber.tsx b/apps/ui/components/AnimatedNumber.tsx deleted file mode 100644 index 919380e4..00000000 --- a/apps/ui/components/AnimatedNumber.tsx +++ /dev/null @@ -1,144 +0,0 @@ -"use client"; - -import { useEffect, useRef, useState } from 'react'; - -import { cn } from '@/lib/utils'; - -type Trend = 'up' | 'down' | null; - -type AnimatedNumberProps = { - value?: number | null; - format: (value: number) => string; - duration?: number; - className?: string; - placeholder?: string; - style?: React.CSSProperties; -}; - -const easeOutCubic = (t: number) => 1 - Math.pow(1 - t, 3); - -export const AnimatedNumber = ({ - value, - format, - duration = 320, - className, - placeholder = '—', - style -}: AnimatedNumberProps) => { - const [displayValue, setDisplayValue] = useState(() => (typeof value === 'number' ? value : 0)); - const previousRef = useRef(typeof value === 'number' ? value : null); - const displayRef = useRef(displayValue); - const animationRef = useRef(); - const timeoutRef = useRef>(); - const [trend, setTrend] = useState(null); - - useEffect(() => { - displayRef.current = displayValue; - }, [displayValue]); - - useEffect(() => { - if (typeof value !== 'number' || Number.isNaN(value)) { - previousRef.current = null; - setDisplayValue(0); - setTrend(null); - return undefined; - } - - const startValue = previousRef.current !== null ? displayRef.current : value; - const endValue = value; - - if (previousRef.current !== null) { - if (endValue > startValue) setTrend('up'); - else if (endValue < startValue) setTrend('down'); - else setTrend(null); - } - - previousRef.current = endValue; - - const startTime = performance.now(); - - const step = (timestamp: number) => { - const progress = Math.min((timestamp - startTime) / duration, 1); - const eased = easeOutCubic(progress); - const current = startValue + (endValue - startValue) * eased; - setDisplayValue(current); - if (progress < 1) { - animationRef.current = requestAnimationFrame(step); - } - }; - - animationRef.current = requestAnimationFrame(step); - - if (timeoutRef.current) clearTimeout(timeoutRef.current); - timeoutRef.current = setTimeout(() => setTrend(null), duration + 160); - - return () => { - if (animationRef.current) cancelAnimationFrame(animationRef.current); - if (timeoutRef.current) clearTimeout(timeoutRef.current); - }; - }, [value, duration]); - - if (typeof value !== 'number' || Number.isNaN(value)) { - return {placeholder}; - } - - return ( - - {trend && ( - - {trend === 'up' ? '▲' : '▼'} - - )} - - {format(displayValue)} - - - ); -}; - - diff --git a/apps/ui/components/asset-dashboard/elements/MetricValue.tsx b/apps/ui/components/asset-dashboard/elements/MetricValue.tsx index 68c6ba49..5aeb9199 100644 --- a/apps/ui/components/asset-dashboard/elements/MetricValue.tsx +++ b/apps/ui/components/asset-dashboard/elements/MetricValue.tsx @@ -1,4 +1,4 @@ -import { AnimatedNumber } from '@/components/AnimatedNumber'; +import { AnimatedNumber } from '@/components/ui/animated-number'; import { formatAdaptiveNumber } from '@/lib/format'; import { cn } from '@/lib/utils'; diff --git a/apps/ui/components/asset-dashboard/elements/RadialGauge.tsx b/apps/ui/components/asset-dashboard/elements/RadialGauge.tsx index e3f21542..3eae1ddc 100644 --- a/apps/ui/components/asset-dashboard/elements/RadialGauge.tsx +++ b/apps/ui/components/asset-dashboard/elements/RadialGauge.tsx @@ -1,6 +1,6 @@ 'use client'; -import { AnimatedNumber } from '@/components/AnimatedNumber'; +import { AnimatedNumber } from '@/components/ui/animated-number'; import { cn, clamp } from '@/lib/utils'; type RadialGaugeProps = { diff --git a/apps/ui/components/asset-dashboard/sections/FundingLiquidityCard.tsx b/apps/ui/components/asset-dashboard/sections/FundingLiquidityCard.tsx index df1b7c8e..d5b4225d 100644 --- a/apps/ui/components/asset-dashboard/sections/FundingLiquidityCard.tsx +++ b/apps/ui/components/asset-dashboard/sections/FundingLiquidityCard.tsx @@ -1,4 +1,4 @@ -import { AnimatedNumber } from '@/components/AnimatedNumber'; +import { AnimatedNumber } from '@/components/ui/animated-number'; import { LiveStatusBadge } from '@/components/ui/live-status-badge'; import { Surface } from '@/components/ui/surface'; import { formatNumber, formatPercent, formatUSDFull } from '@/lib/format'; diff --git a/apps/ui/components/asset-dashboard/sections/HeroOverviewSection.tsx b/apps/ui/components/asset-dashboard/sections/HeroOverviewSection.tsx index a7805251..8d73a08b 100644 --- a/apps/ui/components/asset-dashboard/sections/HeroOverviewSection.tsx +++ b/apps/ui/components/asset-dashboard/sections/HeroOverviewSection.tsx @@ -2,7 +2,7 @@ import Link from 'next/link'; import type { Route } from 'next'; import { useMemo } from 'react'; -import AssetAvatar from '@/components/AssetAvatar'; +import { AssetAvatar } from '@/components/ui/asset-avatar'; import { DepthIndicator, TrendIndicator } from '@/components/ui/data-indicators'; import { LiveStatusBadge } from '@/components/ui/live-status-badge'; import { Surface } from '@/components/ui/surface'; diff --git a/apps/ui/components/asset-dashboard/sections/LiveCandlesPanel.tsx b/apps/ui/components/asset-dashboard/sections/LiveCandlesPanel.tsx index 2b36311f..0b47f919 100644 --- a/apps/ui/components/asset-dashboard/sections/LiveCandlesPanel.tsx +++ b/apps/ui/components/asset-dashboard/sections/LiveCandlesPanel.tsx @@ -1,5 +1,5 @@ import MarketCandlestickChart from '@/components/MarketCandlestickChart'; -import { AnimatedNumber } from '@/components/AnimatedNumber'; +import { AnimatedNumber } from '@/components/ui/animated-number'; import { LiveStatusBadge } from '@/components/ui/live-status-badge'; import { Surface } from '@/components/ui/surface'; import { formatNumber, formatTimestamp } from '@/lib/format'; diff --git a/apps/ui/components/asset-dashboard/sections/MarketHealthCard.tsx b/apps/ui/components/asset-dashboard/sections/MarketHealthCard.tsx index 1e641367..d6899696 100644 --- a/apps/ui/components/asset-dashboard/sections/MarketHealthCard.tsx +++ b/apps/ui/components/asset-dashboard/sections/MarketHealthCard.tsx @@ -1,4 +1,4 @@ -import { AnimatedNumber } from '@/components/AnimatedNumber'; +import { AnimatedNumber } from '@/components/ui/animated-number'; import { Surface } from '@/components/ui/surface'; import { cn, clamp } from '@/lib/utils'; diff --git a/apps/ui/components/asset-dashboard/sections/PriceDynamicsCard.tsx b/apps/ui/components/asset-dashboard/sections/PriceDynamicsCard.tsx index 67c8fc69..562a4ce6 100644 --- a/apps/ui/components/asset-dashboard/sections/PriceDynamicsCard.tsx +++ b/apps/ui/components/asset-dashboard/sections/PriceDynamicsCard.tsx @@ -1,4 +1,4 @@ -import { AnimatedNumber } from '@/components/AnimatedNumber'; +import { AnimatedNumber } from '@/components/ui/animated-number'; import { LiveStatusBadge } from '@/components/ui/live-status-badge'; import { Surface } from '@/components/ui/surface'; import { formatNumber, formatPercent, formatTimestamp, formatUSDFull } from '@/lib/format'; diff --git a/apps/ui/components/community/AssetCommunityHub.tsx b/apps/ui/components/community/AssetCommunityHub.tsx index 31ece7e7..8c24df27 100644 --- a/apps/ui/components/community/AssetCommunityHub.tsx +++ b/apps/ui/components/community/AssetCommunityHub.tsx @@ -1,6 +1,7 @@ 'use client'; -import { useState, useEffect } from 'react'; +import Link from 'next/link'; +import { useState, useEffect, useId } from 'react'; import type { LucideIcon } from 'lucide-react'; import { Diamond, TrendingUp, Trophy } from 'lucide-react'; @@ -8,6 +9,9 @@ import { cn } from '@/lib/utils'; import { formatUSD, formatNumber } from '@/lib/format'; import { Surface } from '@/components/ui/surface'; import { MedalGlyph } from '@/components/leaderboard/MedalGlyph'; +import { PersonaBadges } from '@/components/trader'; +import type { PersonaInput } from '@/lib/utils/personas'; +import { LeaderboardSkeleton } from '@/components/leaderboard/LeaderboardSkeleton'; import { borderRadius, cardStyles, @@ -79,6 +83,7 @@ export const AssetCommunityHub = ({ asset, className }: AssetCommunityHubProps) const [timeWindow, setTimeWindow] = useState('24h'); const [isLoading, setIsLoading] = useState(true); const [error, setError] = useState(null); + const windowSelectId = useId(); useEffect(() => { const fetchCommunityData = async () => { @@ -304,8 +309,11 @@ export const AssetCommunityHub = ({ asset, className }: AssetCommunityHubProps)

Community hierarchy

- + setWindow(e.target.value as TimeWindow)} - className="text-xs bg-slate-800/60 border border-slate-600/40 rounded-lg px-3 py-2 text-slate-200 font-mono tracking-wide focus:outline-none focus:ring-2 focus:ring-slate-500/30 focus:border-slate-500/60 transition-all duration-200" - aria-label="Select time window" - > - - - - -
- - - {/* Tabs */} -
- - -
- -
-
- - Persona mix - - - {allEntries.length} traders · {personaSummary.length} persona types - -
-
- {isLoading ? ( - - ) : personaSummary.length > 0 ? ( - - ) : ( - Persona metadata still syncing - )} -
-
- - {/* Table */} -
- - - - - - - - - - - - - - - - {isLoading ? ( - Array.from({ length: pageSize }).map((_, i) => ( - - )) - ) : entries.length === 0 ? ( - - - - ) : ( - entries.map((entry) => ( - { - if (isWatched(address)) { - removeFromWatchlist(address); - } else { - addToWatchlist(address); - } - }} - assetBadges={traderBadges[entry.address]} - provider={provider} - /> - )) - )} - -
RankTraderVolumeTradesAvg SizeShareNet P&LLast TradeActivity
-
-
📊
-
No trading data available
-
Check back later or try a different time window
-
-
-
- - {/* Pagination Controls */} - {!isLoading && totalPages > 1 && ( -
-
- - {((currentPage - 1) * pageSize) + 1}–{Math.min(currentPage * pageSize, allEntries.length)} of {allEntries.length} - - {hasMore && ( -
-
- More available -
- )} -
- -
- - -
- {Array.from({ length: Math.min(totalPages, 5) }, (_, i) => { - const pageNum = i + 1; - const isActive = pageNum === currentPage; - - return ( - - ); - })} - - {totalPages > 5 && ( - <> - {totalPages > 6 && ( -
-
-
-
-
- )} - - - )} -
- - -
-
- )} - - {/* Trader Profile Modal */} - setSelectedTrader(null)} - provider={provider} - /> -
- ); -}; - diff --git a/apps/ui/components/leaderboard/index.ts b/apps/ui/components/leaderboard/index.ts index fa18b2c9..9ffe97ad 100644 --- a/apps/ui/components/leaderboard/index.ts +++ b/apps/ui/components/leaderboard/index.ts @@ -1,5 +1,4 @@ export { AssetLeaderboard } from './AssetLeaderboard'; -export { PremiumLeaderboard } from './PremiumLeaderboard'; export { TraderProfileModal } from './TraderProfileModal'; export { WatchlistSidebar } from './WatchlistSidebar'; export { EnhancedRankDisplay } from './EnhancedRankDisplay'; diff --git a/apps/ui/components/provider-dashboard/MarketConsole.tsx b/apps/ui/components/provider-dashboard/MarketConsole.tsx index d90f18d0..04e78413 100644 --- a/apps/ui/components/provider-dashboard/MarketConsole.tsx +++ b/apps/ui/components/provider-dashboard/MarketConsole.tsx @@ -1,11 +1,11 @@ import type { CSSProperties } from 'react'; import type { Route } from 'next'; import { useRouter } from 'next/navigation'; -import { useEffect, useMemo, useState } from 'react'; +import { useMemo } from 'react'; import { Search } from 'lucide-react'; -import AssetAvatar from '@/components/AssetAvatar'; -import { AnimatedNumber } from '@/components/AnimatedNumber'; +import { AssetAvatar } from '@/components/ui/asset-avatar'; +import { AnimatedNumber } from '@/components/ui/animated-number'; import { Surface } from '@/components/ui/surface'; import { formatAdaptiveNumber, @@ -467,6 +467,7 @@ export const MarketConsole = ({ return (
( ); -type AnimatedCellProps = { - value: number | null | undefined; - formatter: (value: number) => string; - className?: string; - tone?: 'positive' | 'negative' | 'neutral'; - align?: 'start' | 'end'; -}; - -const flashBackground: Record, string> = { - positive: 'rgba(var(--state-positive-rgb),0.16)', - negative: 'rgba(var(--state-negative-rgb),0.16)', - neutral: 'rgba(var(--crystal-rgb),0.12)' -}; - -const flashShadow: Record, string> = { - positive: 'none', - negative: 'none', - neutral: 'none' -}; - -const AnimatedCell = ({ value, formatter, className, tone = 'neutral', align = 'end' }: AnimatedCellProps) => { - const [previousValue, setPreviousValue] = useState(value); - const [isFlashing, setIsFlashing] = useState(false); - - useEffect(() => { - if (value === undefined || value === null) return; - if (previousValue === undefined) { - setPreviousValue(value); - return; - } - if (value !== previousValue) { - setIsFlashing(true); - const timer = setTimeout(() => setIsFlashing(false), 400); - setPreviousValue(value); - return () => clearTimeout(timer); - } - }, [value, previousValue]); - - if (value === null || value === undefined) { - return —; - } - - const style = isFlashing - ? { boxShadow: flashShadow[tone], backgroundColor: flashBackground[tone] } - : undefined; - - return ( - - - - ); -}; +// AnimatedCell removed as we use AnimatedNumber directly now type MarketRowProps = { market: ExtendedMarketSnapshot; @@ -719,11 +662,10 @@ const MarketRow = ({ market, providerId, maxOI, maxVolume, columnTemplateStyle }
- 24H
@@ -764,10 +706,10 @@ const MarketRow = ({ market, providerId, maxOI, maxVolume, columnTemplateStyle } ariaLabel={`Open interest distribution for ${assetSymbol}`} />
- formatUSDFull(value)} - className="text-[0.68rem] font-normal tracking-[0.04em] text-[rgba(var(--crystal-rgb),0.95)] tabular-nums" + format={(value) => formatUSDFull(value)} + className="text-[0.68rem] font-normal tracking-[0.04em] tabular-nums" /> {volOIRatio !== null ? `VOL/OI ${volOIRatio.toFixed(2)}` : 'VOL/OI —'} @@ -783,10 +725,10 @@ const MarketRow = ({ market, providerId, maxOI, maxVolume, columnTemplateStyle } ariaLabel={`Volume distribution for ${assetSymbol}`} />
- formatUSDFull(value)} - className="text-[0.68rem] font-normal tracking-[0.04em] text-white/90 tabular-nums" + format={(value) => formatUSDFull(value)} + className="text-[0.68rem] font-normal tracking-[0.04em] tabular-nums" /> 24H
diff --git a/apps/ui/components/provider-dashboard/MarketOverviewCards.tsx b/apps/ui/components/provider-dashboard/MarketOverviewCards.tsx index 06937ba0..52059f88 100644 --- a/apps/ui/components/provider-dashboard/MarketOverviewCards.tsx +++ b/apps/ui/components/provider-dashboard/MarketOverviewCards.tsx @@ -2,8 +2,8 @@ import Link from 'next/link'; import type { Route } from 'next'; import { useMemo } from 'react'; -import { AnimatedNumber } from '@/components/AnimatedNumber'; -import AssetAvatar from '@/components/AssetAvatar'; +import { AnimatedNumber } from '@/components/ui/animated-number'; +import { AssetAvatar } from '@/components/ui/asset-avatar'; import { formatAdaptiveNumber, formatCurrency, diff --git a/apps/ui/components/provider-dashboard/stats/MetricCard.tsx b/apps/ui/components/provider-dashboard/stats/MetricCard.tsx index 0e471aca..3e068589 100644 --- a/apps/ui/components/provider-dashboard/stats/MetricCard.tsx +++ b/apps/ui/components/provider-dashboard/stats/MetricCard.tsx @@ -18,7 +18,7 @@ type MetricCardProps = Omit, 'children' | 't }; const baseCardClass = - 'group relative isolate flex flex-col overflow-hidden rounded-[26px] border px-6 py-6 sm:px-7 sm:py-7 gap-5 shadow-[0_35px_120px_-80px_rgba(5,10,25,0.9)] bg-[rgba(3,7,18,0.78)] backdrop-blur-2xl'; + 'group relative isolate flex flex-col overflow-hidden rounded-[26px] border px-5 py-4 sm:px-6 sm:py-5 gap-3 shadow-[0_35px_120px_-80px_rgba(5,10,25,0.9)] bg-[rgba(3,7,18,0.78)] backdrop-blur-2xl'; const toneVariants: Record = { positive: 'border-[rgba(var(--state-positive-rgb),0.32)]', @@ -70,7 +70,7 @@ export const MetricCard = ({
{headerControls}
- {children ?
{children}
: null} + {children} {description ?
{description}
: null} {footer ?
{footer}
: null} diff --git a/apps/ui/components/provider-dashboard/stats/OverviewMetricTile.tsx b/apps/ui/components/provider-dashboard/stats/OverviewMetricTile.tsx index dd234f10..1c7ef305 100644 --- a/apps/ui/components/provider-dashboard/stats/OverviewMetricTile.tsx +++ b/apps/ui/components/provider-dashboard/stats/OverviewMetricTile.tsx @@ -1,7 +1,7 @@ import { useState } from 'react'; import type { KeyboardEvent } from 'react'; -import { AnimatedNumber } from '@/components/AnimatedNumber'; +import { AnimatedNumber } from '@/components/ui/animated-number'; import { formatAdaptiveNumber } from '@/lib/format'; import { cn } from '@/lib/utils'; import { diagnosticTileStyles, semanticSurfaceStyles } from '@/styles/design-tokens'; @@ -17,12 +17,6 @@ import { clamp, describeMetric } from '../utils'; import { OverviewTooltipBadge } from './OverviewTooltipBadge'; import { MetricCard } from './MetricCard'; -const progressToneByMetricTone: Record = { - positive: diagnosticTileStyles.progressFillPositive, - negative: diagnosticTileStyles.progressFillNegative, - neutral: diagnosticTileStyles.progressFillNeutral -}; - const badgePaletteOverrides: Record = { DEEP: 'bg-[rgba(4,47,46,0.75)] border-[rgba(45,212,191,0.4)] text-teal-100/90', 'SHORT YIELD': 'bg-[rgba(85,46,10,0.7)] border-[rgba(245,158,11,0.35)] text-amber-100/90', @@ -212,58 +206,36 @@ const TrendIndicator = ({ tone: MetricTone; }) => { if (!direction || direction === 'flat') { - return ( -
-
- - STABLE - -
- ); + return ( +
+ STABLE +
+ ); } const color = tone === 'positive' - ? 'rgba(var(--state-positive-rgb),0.8)' + ? 'text-emerald-400' : tone === 'negative' - ? 'rgba(var(--state-negative-rgb),0.8)' - : 'rgba(var(--crystal-rgb),0.8)'; + ? 'text-rose-400' + : 'text-slate-400'; const bgColor = tone === 'positive' - ? 'rgba(var(--state-positive-rgb),0.15)' + ? 'bg-emerald-500/10' : tone === 'negative' - ? 'rgba(var(--state-negative-rgb),0.15)' - : 'rgba(var(--crystal-rgb),0.15)'; + ? 'bg-rose-500/10' + : 'bg-slate-500/10'; const label = direction === 'up' ? 'RISING' : 'FALLING'; const arrow = direction === 'up' ? '↗' : '↘'; return ( -
-
-
-
-
- - {arrow} - - - {label} - -
+
+ + {arrow} + + + {label} +
); }; @@ -361,28 +333,6 @@ export const OverviewMetricTile = ({ metric }: { metric: OverviewMetricEntry })
); - const progressRow = - progressValue !== null ? ( -
-
-
-
-
- ) : null; - const descriptionNode = descriptionFirstLine ? (

{descriptionFirstLine}

) : null; @@ -435,10 +385,14 @@ export const OverviewMetricTile = ({ metric }: { metric: OverviewMetricEntry }) tabIndex={0} aria-label={ariaLabel} onKeyDown={handleTileKeyDown} + className="gap-3 p-5 sm:gap-3 sm:p-5" > - {valueRow} - {trendRow} - {progressRow} +
+ {valueRow} +
+ {trendRow} +
+
); }; diff --git a/apps/ui/components/trade-stream/TradeActivity.tsx b/apps/ui/components/trade-stream/TradeActivity.tsx index eb905cd3..c904b44c 100644 --- a/apps/ui/components/trade-stream/TradeActivity.tsx +++ b/apps/ui/components/trade-stream/TradeActivity.tsx @@ -49,7 +49,6 @@ import { import { FlowGauge } from './premium/FlowGauge'; import { FilterChips, type FilterKey } from './premium/FilterChips'; import { TradeRow, TRADE_ROW_GRID_TEMPLATE, type PremiumTradeRowData } from './premium/TradeRow'; -import { TooltipPanel } from './premium/TooltipPanel'; import { CardGrain } from './premium/SurfaceLayers'; const ROW_HEIGHT = 118; @@ -57,6 +56,22 @@ const SKELETON_ROW_COUNT = 6; const MAX_TRADES = 500; const SPARKLINE_POINTS = 24; +const TRADE_STREAM_HEADER_COLUMNS: Array<{ + key: string; + label: string; + description?: string; + alignClass?: string; +}> = [ + { key: 'time', label: 'Time', description: 'Relative', alignClass: 'text-left' }, + { key: 'flow', label: 'Flow', description: 'Deviation', alignClass: 'text-left' }, + { key: 'price', label: 'Price', description: 'Last', alignClass: 'text-left' }, + { key: 'size', label: 'Size', description: 'Qty', alignClass: 'text-right' }, + { key: 'notional', label: 'Notional', description: 'USD', alignClass: 'text-right' }, + { key: 'addresses', label: 'Addresses', description: 'Maker / Taker', alignClass: 'text-left' }, + { key: 'depth', label: 'Depth', description: 'Book', alignClass: 'text-left' }, + { key: 'impact', label: 'Impact', description: 'Trace', alignClass: 'text-right' }, +]; + type TradeActivityProps = { readonly provider: string; readonly asset: string; @@ -320,7 +335,7 @@ export const TradeActivity = ({ height = 600, onFilterAddress: onFilterAddressProp, }: TradeActivityProps) => { - const { trades, isConnected, isLoading, error } = useTradeStream({ + const { trades, isLoading, error } = useTradeStream({ provider, asset, maxTrades: MAX_TRADES, @@ -621,79 +636,8 @@ export const TradeActivity = ({ return particles; }, [particleSeed]); - const statusColor = isConnected ? 'bg-emerald-400 shadow-[0_0_16px_rgba(16,185,129,0.8)]' : 'bg-amber-400 shadow-[0_0_16px_rgba(251,191,36,0.65)]'; - const statusLabel = isConnected ? 'Live' : 'Syncing'; - const statusBadgeClass = cn( - borderRadius.base, - borderColor.crystalSubtle, - backgroundColor.nested, - 'inline-flex items-center gap-2 px-4 py-1.5 text-[10px] font-mono uppercase tracking-[0.35em]', - textColor.cyan80, - ); - const controlButtonClass = cn( - 'inline-flex h-10 items-center gap-2 px-4 text-[10px] font-mono uppercase tracking-[0.32em] transition', - borderRadius.base, - borderColor.crystal, - backgroundColor.nestedDark, - textColor.cyan80, - 'shadow-[0_6px_24px_rgba(0,0,0,0.55)]', - 'hover:border-[rgba(var(--crystal-rgb),0.45)]', - ); - const premiumHeader = (
- - -
-
-
- - {statusLabel} -
-
- - {provider} · {asset} - -

- {premium.latestPrice ? formatAdaptiveNumber(premium.latestPrice) : '—'} -

-
-
-
-
- Depth {depthSummary?.depth ?? 0} -
- -
- Momentum - - {premium.pressure.momentum ? `${Math.round(premium.pressure.momentum * 100)}%` : '—'} - -
- - -
-
- -
+
- Time - Flow - Price - Size - Notional - Addresses - Depth - Impact + {TRADE_STREAM_HEADER_COLUMNS.map((column) => ( +
+ {column.label} + {column.description && ( + + {column.description} + + )} +
+ ))} ({ diff --git a/apps/ui/components/trader/PersonaCard.tsx b/apps/ui/components/trader/PersonaCard.tsx index 6581d32c..ad67365e 100644 --- a/apps/ui/components/trader/PersonaCard.tsx +++ b/apps/ui/components/trader/PersonaCard.tsx @@ -19,11 +19,11 @@ import { performanceStyles, } from '@/styles/design-tokens'; import { formatUSD } from '@/lib/format'; -import type { SparklinePoint } from '@/components/Sparkline'; +import type { SparklinePoint } from '@/components/ui/sparkline'; import { usePersonaStats, type PersonaTrendPoint } from '@/lib/hooks/use-personas'; import { getPersonaConfig, type PersonaConfig } from '@/lib/config/personas'; -const Sparkline = dynamic(() => import('@/components/Sparkline').then((mod) => ({ default: mod.Sparkline })), { +const Sparkline = dynamic(() => import('@/components/ui/sparkline').then((mod) => ({ default: mod.Sparkline })), { ssr: false, }); diff --git a/apps/ui/components/ui/animated-number.tsx b/apps/ui/components/ui/animated-number.tsx new file mode 100644 index 00000000..9699a992 --- /dev/null +++ b/apps/ui/components/ui/animated-number.tsx @@ -0,0 +1,127 @@ +"use client"; + +import { useEffect, useRef, useState } from 'react'; +import { useSpring } from 'framer-motion'; +import { cn } from '@/lib/utils'; + +type Trend = 'up' | 'down' | null; + +type AnimatedNumberProps = { + value?: number | null; + format: (value: number) => string; + duration?: number; // kept for API compatibility + className?: string; + placeholder?: string; + style?: React.CSSProperties; +}; + +export const AnimatedNumber = ({ + value, + format, + className, + placeholder = '—', + style +}: AnimatedNumberProps) => { + const springValue = useSpring(typeof value === 'number' ? value : 0, { + stiffness: 60, + damping: 15, + mass: 0.8, + restDelta: 0.0001 + }); + + const [trend, setTrend] = useState(null); + const previousValueRef = useRef(typeof value === 'number' ? value : null); + const ref = useRef(null); + + useEffect(() => { + if (typeof value !== 'number' || Number.isNaN(value)) { + previousValueRef.current = null; + setTrend(null); + return; + } + + springValue.set(value); + + const prev = previousValueRef.current; + if (prev !== null && prev !== value) { + if (value > prev) setTrend('up'); + else if (value < prev) setTrend('down'); + + const timer = setTimeout(() => setTrend(null), 2000); + previousValueRef.current = value; + return () => clearTimeout(timer); + } else { + previousValueRef.current = value; + } + }, [value, springValue]); + + useEffect(() => { + const unsubscribe = springValue.on("change", (latest) => { + if (ref.current) { + ref.current.textContent = format(latest); + } + }); + return unsubscribe; + }, [springValue, format]); + + const trendColor = + trend === 'up' + ? 'rgba(var(--accent-positive-rgb), 0.95)' + : trend === 'down' + ? 'rgba(var(--accent-negative-rgb), 0.95)' + : undefined; + + const computedStyle = trendColor + ? { ...(style ?? {}), color: trendColor } + : style; + + if (typeof value !== 'number' || Number.isNaN(value)) { + return ( + + {placeholder} + + ); + } + + return ( + + + {format(springValue.get())} + + + {/* Arrow indicator - absolute positioned to avoid layout shift */} + + + ); +}; + + + diff --git a/apps/ui/components/AssetAvatar.tsx b/apps/ui/components/ui/asset-avatar.tsx similarity index 99% rename from apps/ui/components/AssetAvatar.tsx rename to apps/ui/components/ui/asset-avatar.tsx index e9241339..56695f80 100644 --- a/apps/ui/components/AssetAvatar.tsx +++ b/apps/ui/components/ui/asset-avatar.tsx @@ -129,4 +129,6 @@ export const AssetAvatar = ({ ); }; -export default AssetAvatar; \ No newline at end of file +export default AssetAvatar; + + diff --git a/apps/ui/components/ui/badge.tsx b/apps/ui/components/ui/badge.tsx index a2cdb849..9af059c0 100644 --- a/apps/ui/components/ui/badge.tsx +++ b/apps/ui/components/ui/badge.tsx @@ -3,30 +3,72 @@ import { cva } from 'class-variance-authority'; import * as React from 'react'; import { cn } from '@/lib/utils'; -import { typography } from '@/styles/typography'; +import { + typography, + semanticSurfaceStyles, + borderRadius, + transition +} from '@/styles/design-tokens'; const badgeVariants = cva( - cn('inline-flex items-center gap-1 rounded-full border px-3 py-0.5 transition-all duration-200', typography.pill), + cn( + 'inline-flex items-center gap-1.5 border backdrop-blur-sm', + typography.pill, + borderRadius.full, + transition.default + ), { variants: { variant: { + // Semantic variants using design tokens crystal: 'border-[rgba(var(--crystal-rgb),0.4)] bg-[rgba(var(--crystal-rgb),0.08)] text-cyan-100 shadow-[0_0_20px_rgba(0,214,255,0.15)]', muted: 'border-white/10 bg-white/5 text-slate-300', - positive: - 'border-[rgba(var(--state-positive-rgb),0.5)] bg-[rgba(var(--state-positive-rgb),0.12)] text-emerald-200 shadow-[0_0_20px_rgba(16,185,129,0.25)]', - negative: - 'border-[rgba(var(--state-negative-rgb),0.55)] bg-[rgba(var(--state-negative-rgb),0.12)] text-rose-200 shadow-[0_0_20px_rgba(244,63,94,0.25)]', - warning: - 'border-[rgba(var(--state-warning-rgb),0.55)] bg-[rgba(var(--state-warning-rgb),0.12)] text-amber-100 shadow-[0_0_20px_rgba(251,191,36,0.2)]', + positive: semanticSurfaceStyles.positive.badge, + negative: semanticSurfaceStyles.negative.badge, + warning: semanticSurfaceStyles.warning.badge, + info: semanticSurfaceStyles.info.badge, + neutral: semanticSurfaceStyles.neutral.badge, outline: 'border-white/30 text-white hover:bg-white/5', - info: - 'border-[rgba(var(--tone-indigo),0.5)] bg-[rgba(var(--tone-indigo),0.15)] text-blue-100 shadow-[0_0_20px_rgba(129,140,248,0.2)]' + + // Status variants (from StatusBadge) + live: cn( + 'border-[rgba(var(--state-positive-rgb),0.45)] bg-[rgba(var(--state-positive-rgb),0.12)]', + 'text-[rgba(var(--state-positive-rgb),0.96)]', + semanticSurfaceStyles.positive.glow + ), + hot: cn( + 'border-rose-400/40 bg-rose-400/10 text-rose-300', + 'shadow-[0_0_20px_rgba(244,63,94,0.3)]' + ), + active: cn( + 'border-amber-400/35 bg-amber-400/10 text-amber-200', + 'shadow-[0_0_20px_rgba(251,191,36,0.25)]' + ), + + // Rarity variants (for BadgeStack gamification) + common: 'border-slate-600 bg-slate-800/50 text-slate-300', + rare: cn( + 'border-blue-500 bg-blue-500/10 text-blue-300', + 'shadow-[0_0_8px_rgba(59,130,246,0.5)]' + ), + epic: cn( + 'border-purple-500 bg-purple-500/10 text-purple-300', + 'shadow-[0_0_8px_rgba(139,92,246,0.5)] animate-pulse' + ), + legendary: cn( + 'border-yellow-500 bg-yellow-500/10 text-yellow-300', + 'shadow-[0_0_12px_rgba(251,191,36,0.6)] animate-pulse' + ), + mythic: cn( + 'border-transparent bg-gradient-to-r from-pink-500 via-purple-500 via-blue-500 to-green-500', + 'shadow-[0_0_16px_rgba(239,68,68,0.6)] animate-pulse' + ) }, size: { - sm: cn(typography.microMono, 'px-2.5'), - md: typography.pill, - lg: cn(typography.labelMono, 'px-4') + sm: cn(typography.microMono, 'px-2.5 py-0.5'), + md: cn(typography.pill, 'px-3 py-0.5'), + lg: cn(typography.labelMono, 'px-4 py-1') } }, defaultVariants: { @@ -38,11 +80,22 @@ const badgeVariants = cva( export interface BadgeProps extends React.HTMLAttributes, - VariantProps {} + VariantProps { + /** Show animated pulse indicator (for 'live' variant) */ + showPulse?: boolean; +} export const Badge = React.forwardRef( - ({ className, variant, size, ...props }, ref) => ( - + ({ className, variant, size, showPulse, children, ...props }, ref) => ( + + {showPulse && variant === 'live' && ( + + + + + )} + {children} + ) ); Badge.displayName = 'Badge'; diff --git a/apps/ui/components/ui/index.ts b/apps/ui/components/ui/index.ts index bcd66100..37fd0416 100644 --- a/apps/ui/components/ui/index.ts +++ b/apps/ui/components/ui/index.ts @@ -19,11 +19,14 @@ export { PageHeader, type PageHeaderProps } from './page-header'; export { Card, CardContent } from './card'; export { DataTable, type DataTableColumn, type DataTableProps } from './data-table'; export { LiveBadge, type LiveBadgeProps } from './live-badge'; -export { StatusBadge } from './status-badge'; export { MetricDisplay } from './metric-display'; export { Metric, type MetricProps } from './metric'; export { NavTabs, type NavTab, type NavTabsProps } from './nav-tabs'; export { RateCard } from './rate-card'; export { RetroPageHeader, type RetroNavItem } from './retro-page-header'; export { Tabs, type Tab, type TabsProps } from './tabs'; +export { AnimatedNumber } from './animated-number'; +export { Sparkline, type SparklinePoint } from './sparkline'; +export { StatusIndicator } from './status-indicator'; +export { AssetAvatar } from './asset-avatar'; export type { SemanticTone, SurfaceLayer, PaddingSize, MetricSize } from './types'; diff --git a/apps/ui/components/Sparkline.tsx b/apps/ui/components/ui/sparkline.tsx similarity index 99% rename from apps/ui/components/Sparkline.tsx rename to apps/ui/components/ui/sparkline.tsx index e4315773..052dec40 100644 --- a/apps/ui/components/Sparkline.tsx +++ b/apps/ui/components/ui/sparkline.tsx @@ -414,3 +414,5 @@ export const Sparkline = memo(SparklineInner); Sparkline.displayName = 'Sparkline'; export default Sparkline; + + diff --git a/apps/ui/components/ui/status-badge.tsx b/apps/ui/components/ui/status-badge.tsx deleted file mode 100644 index f91da411..00000000 --- a/apps/ui/components/ui/status-badge.tsx +++ /dev/null @@ -1,44 +0,0 @@ -import type { ReactNode } from 'react'; - -import { cn } from '@/lib/utils'; - -type StatusBadgeVariant = 'live' | 'hot' | 'active' | 'neutral'; - -type StatusBadgeProps = { - children: ReactNode; - className?: string; - variant?: StatusBadgeVariant; -}; - -const variantClasses: Record = { - live: 'bg-emerald-400/10 border-emerald-400/40 text-emerald-300 shadow-[0_0_20px_rgba(16,185,129,0.35)]', - hot: 'bg-rose-400/10 border-rose-400/40 text-rose-300 shadow-[0_0_20px_rgba(244,63,94,0.3)]', - active: 'bg-amber-400/10 border-amber-400/35 text-amber-200 shadow-[0_0_20px_rgba(251,191,36,0.25)]', - neutral: 'bg-sky-400/10 border-sky-400/35 text-sky-200 shadow-[0_0_16px_rgba(14,165,233,0.25)]' -}; - -export const StatusBadge = ({ children, className, variant = 'neutral' }: StatusBadgeProps) => { - return ( - - {variant === 'live' && ( - - - - - )} - {children} - - ); -}; - - - - - diff --git a/apps/ui/components/StatusIndicator.tsx b/apps/ui/components/ui/status-indicator.tsx similarity index 99% rename from apps/ui/components/StatusIndicator.tsx rename to apps/ui/components/ui/status-indicator.tsx index 9ad7a964..af2a35b4 100644 --- a/apps/ui/components/StatusIndicator.tsx +++ b/apps/ui/components/ui/status-indicator.tsx @@ -28,3 +28,5 @@ export const StatusIndicator = ({ status = 'connecting', labelOverride, classNam ); }; + + diff --git a/apps/ui/lib/server/gateway.ts b/apps/ui/lib/server/gateway.ts index 417f49e2..1582a67a 100644 --- a/apps/ui/lib/server/gateway.ts +++ b/apps/ui/lib/server/gateway.ts @@ -101,17 +101,17 @@ export const fetchGateway = async (path: string, fallbackPaths: string[] = []): const timeout = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS); const response = await fetch(requestUrl, { cache: 'no-store', signal: controller.signal }); clearTimeout(timeout); + attempt.status = response.status; + attempts.push(attempt); if (response.ok) { - attempt.status = response.status; - attempts.push(attempt); return { response }; } lastResponse = response; - attempt.status = response.status; - attempts.push(attempt); if (response.status >= 500) { continue; } + // Surface first non-5xx response (eg. 404) instead of masking with later fallbacks. + return { response }; } catch (error) { lastError = error; attempt.error = error; diff --git a/apps/ui/lib/utils/personas.ts b/apps/ui/lib/utils/personas.ts index c3ec8723..00889679 100644 --- a/apps/ui/lib/utils/personas.ts +++ b/apps/ui/lib/utils/personas.ts @@ -91,3 +91,5 @@ export function normalizePersonas( return normalized.length > 0 ? normalized : undefined; } + + diff --git a/configs/ingest.sample.yaml b/configs/ingest.sample.yaml index e14ada7d..68a4808c 100644 --- a/configs/ingest.sample.yaml +++ b/configs/ingest.sample.yaml @@ -20,6 +20,17 @@ providers: - id: hl-prices index: 0 count: 1 + assets: &hl_dev_assets + - BTC + - ETH + - SOL + - OP + - DOGE + - MATIC + - ARB + - AVAX + - LINK + - SUI streams: - allMids kafkaTopics: @@ -33,6 +44,7 @@ providers: - id: hl-orderbook index: 0 count: 1 + assets: *hl_dev_assets streams: - l2Book kafkaTopics: @@ -45,6 +57,7 @@ providers: - id: hl-asset-metadata index: 0 count: 1 + assets: *hl_dev_assets streams: - activeAssetCtx kafkaTopics: @@ -57,6 +70,7 @@ providers: - id: hl-overview index: 0 count: 1 + assets: *hl_dev_assets streams: - activeAssetData kafkaTopics: @@ -69,6 +83,7 @@ providers: - id: hl-candles index: 0 count: 1 + assets: *hl_dev_assets streams: - candles:1m - candles:5m @@ -83,6 +98,7 @@ providers: - id: hl-trades index: 0 count: 1 + assets: *hl_dev_assets streams: - trades kafkaTopics: @@ -95,6 +111,7 @@ providers: - id: hl-bbo index: 0 count: 1 + assets: *hl_dev_assets streams: - bbo kafkaTopics: diff --git a/docker-compose.yml b/docker-compose.yml index 0804f563..7eff0e60 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -101,18 +101,18 @@ services: INGEST_MANIFEST_PATH: /app/configs/ingest.yaml INGEST_SHARD_ID: hl-prices # Cloud native Kafka optimizations - KAFKA_PRODUCER_REQUEST_TIMEOUT_MS: 5000 + KAFKA_PRODUCER_REQUEST_TIMEOUT_MS: 15000 KAFKA_PRODUCER_RETRIES: 3 KAFKA_PRODUCER_RETRY_INITIAL_DELAY_MS: 100 KAFKA_PRODUCER_RETRY_MAX_DELAY_MS: 5000 KAFKA_PRODUCER_MAX_IN_FLIGHT_REQUESTS: 5 KAFKA_PRODUCER_IDEMPOTENT: true # Backpressure and overload protection - INGEST_OVERLOAD_PENDING_PUBLISH_THRESHOLD: 50 - INGEST_MAX_PENDING_PUBLISHES: 1000 + INGEST_OVERLOAD_PENDING_PUBLISH_THRESHOLD: 100 + INGEST_MAX_PENDING_PUBLISHES: 2000 # Hyperliquid subscription rate limiting - HYPERLIQUID_SUBSCRIPTION_INTERVAL_MS: 250 - HYPERLIQUID_SUBSCRIPTION_BATCH_SIZE: 3 + HYPERLIQUID_SUBSCRIPTION_INTERVAL_MS: 2000 + HYPERLIQUID_SUBSCRIPTION_BATCH_SIZE: 10 volumes: - ./configs/ingest.sample.yaml:/app/configs/ingest.yaml:ro healthcheck: @@ -131,18 +131,18 @@ services: INGEST_MANIFEST_PATH: /app/configs/ingest.yaml INGEST_SHARD_ID: hl-orderbook # Cloud native Kafka optimizations - KAFKA_PRODUCER_REQUEST_TIMEOUT_MS: 5000 + KAFKA_PRODUCER_REQUEST_TIMEOUT_MS: 15000 KAFKA_PRODUCER_RETRIES: 3 KAFKA_PRODUCER_RETRY_INITIAL_DELAY_MS: 100 KAFKA_PRODUCER_RETRY_MAX_DELAY_MS: 5000 KAFKA_PRODUCER_MAX_IN_FLIGHT_REQUESTS: 5 KAFKA_PRODUCER_IDEMPOTENT: true # Backpressure and overload protection - INGEST_OVERLOAD_PENDING_PUBLISH_THRESHOLD: 50 - INGEST_MAX_PENDING_PUBLISHES: 1000 + INGEST_OVERLOAD_PENDING_PUBLISH_THRESHOLD: 100 + INGEST_MAX_PENDING_PUBLISHES: 2000 # Hyperliquid subscription rate limiting - HYPERLIQUID_SUBSCRIPTION_INTERVAL_MS: 250 - HYPERLIQUID_SUBSCRIPTION_BATCH_SIZE: 3 + HYPERLIQUID_SUBSCRIPTION_INTERVAL_MS: 2500 + HYPERLIQUID_SUBSCRIPTION_BATCH_SIZE: 10 volumes: - ./configs/ingest.sample.yaml:/app/configs/ingest.yaml:ro healthcheck: @@ -161,18 +161,18 @@ services: INGEST_MANIFEST_PATH: /app/configs/ingest.yaml INGEST_SHARD_ID: hl-asset-metadata # Cloud native Kafka optimizations - KAFKA_PRODUCER_REQUEST_TIMEOUT_MS: 5000 + KAFKA_PRODUCER_REQUEST_TIMEOUT_MS: 15000 KAFKA_PRODUCER_RETRIES: 3 KAFKA_PRODUCER_RETRY_INITIAL_DELAY_MS: 100 KAFKA_PRODUCER_RETRY_MAX_DELAY_MS: 5000 KAFKA_PRODUCER_MAX_IN_FLIGHT_REQUESTS: 5 KAFKA_PRODUCER_IDEMPOTENT: true # Backpressure and overload protection - INGEST_OVERLOAD_PENDING_PUBLISH_THRESHOLD: 50 - INGEST_MAX_PENDING_PUBLISHES: 1000 + INGEST_OVERLOAD_PENDING_PUBLISH_THRESHOLD: 100 + INGEST_MAX_PENDING_PUBLISHES: 2000 # Hyperliquid subscription rate limiting - HYPERLIQUID_SUBSCRIPTION_INTERVAL_MS: 250 - HYPERLIQUID_SUBSCRIPTION_BATCH_SIZE: 3 + HYPERLIQUID_SUBSCRIPTION_INTERVAL_MS: 3000 + HYPERLIQUID_SUBSCRIPTION_BATCH_SIZE: 10 volumes: - ./configs/ingest.sample.yaml:/app/configs/ingest.yaml:ro healthcheck: @@ -191,18 +191,18 @@ services: INGEST_MANIFEST_PATH: /app/configs/ingest.yaml INGEST_SHARD_ID: hl-overview # Cloud native Kafka optimizations - KAFKA_PRODUCER_REQUEST_TIMEOUT_MS: 5000 + KAFKA_PRODUCER_REQUEST_TIMEOUT_MS: 15000 KAFKA_PRODUCER_RETRIES: 3 KAFKA_PRODUCER_RETRY_INITIAL_DELAY_MS: 100 KAFKA_PRODUCER_RETRY_MAX_DELAY_MS: 5000 KAFKA_PRODUCER_MAX_IN_FLIGHT_REQUESTS: 5 KAFKA_PRODUCER_IDEMPOTENT: true # Backpressure and overload protection - INGEST_OVERLOAD_PENDING_PUBLISH_THRESHOLD: 50 - INGEST_MAX_PENDING_PUBLISHES: 1000 + INGEST_OVERLOAD_PENDING_PUBLISH_THRESHOLD: 100 + INGEST_MAX_PENDING_PUBLISHES: 2000 # Hyperliquid subscription rate limiting - HYPERLIQUID_SUBSCRIPTION_INTERVAL_MS: 250 - HYPERLIQUID_SUBSCRIPTION_BATCH_SIZE: 3 + HYPERLIQUID_SUBSCRIPTION_INTERVAL_MS: 3500 + HYPERLIQUID_SUBSCRIPTION_BATCH_SIZE: 10 volumes: - ./configs/ingest.sample.yaml:/app/configs/ingest.yaml:ro healthcheck: @@ -221,18 +221,18 @@ services: INGEST_MANIFEST_PATH: /app/configs/ingest.yaml INGEST_SHARD_ID: hl-candles # Cloud native Kafka optimizations - KAFKA_PRODUCER_REQUEST_TIMEOUT_MS: 5000 + KAFKA_PRODUCER_REQUEST_TIMEOUT_MS: 15000 KAFKA_PRODUCER_RETRIES: 3 KAFKA_PRODUCER_RETRY_INITIAL_DELAY_MS: 100 KAFKA_PRODUCER_RETRY_MAX_DELAY_MS: 5000 KAFKA_PRODUCER_MAX_IN_FLIGHT_REQUESTS: 5 KAFKA_PRODUCER_IDEMPOTENT: true # Backpressure and overload protection - INGEST_OVERLOAD_PENDING_PUBLISH_THRESHOLD: 50 - INGEST_MAX_PENDING_PUBLISHES: 1000 + INGEST_OVERLOAD_PENDING_PUBLISH_THRESHOLD: 100 + INGEST_MAX_PENDING_PUBLISHES: 2000 # Hyperliquid subscription rate limiting - HYPERLIQUID_SUBSCRIPTION_INTERVAL_MS: 250 - HYPERLIQUID_SUBSCRIPTION_BATCH_SIZE: 3 + HYPERLIQUID_SUBSCRIPTION_INTERVAL_MS: 4000 + HYPERLIQUID_SUBSCRIPTION_BATCH_SIZE: 10 volumes: - ./configs/ingest.sample.yaml:/app/configs/ingest.yaml:ro healthcheck: @@ -251,18 +251,18 @@ services: INGEST_MANIFEST_PATH: /app/configs/ingest.yaml INGEST_SHARD_ID: hl-trades # Cloud native Kafka optimizations - KAFKA_PRODUCER_REQUEST_TIMEOUT_MS: 5000 + KAFKA_PRODUCER_REQUEST_TIMEOUT_MS: 15000 KAFKA_PRODUCER_RETRIES: 3 KAFKA_PRODUCER_RETRY_INITIAL_DELAY_MS: 100 KAFKA_PRODUCER_RETRY_MAX_DELAY_MS: 5000 KAFKA_PRODUCER_MAX_IN_FLIGHT_REQUESTS: 5 KAFKA_PRODUCER_IDEMPOTENT: true # Backpressure and overload protection - INGEST_OVERLOAD_PENDING_PUBLISH_THRESHOLD: 50 - INGEST_MAX_PENDING_PUBLISHES: 1000 + INGEST_OVERLOAD_PENDING_PUBLISH_THRESHOLD: 100 + INGEST_MAX_PENDING_PUBLISHES: 2000 # Hyperliquid subscription rate limiting - HYPERLIQUID_SUBSCRIPTION_INTERVAL_MS: 250 - HYPERLIQUID_SUBSCRIPTION_BATCH_SIZE: 3 + HYPERLIQUID_SUBSCRIPTION_INTERVAL_MS: 4500 + HYPERLIQUID_SUBSCRIPTION_BATCH_SIZE: 10 volumes: - ./configs/ingest.sample.yaml:/app/configs/ingest.yaml:ro healthcheck: @@ -281,18 +281,18 @@ services: INGEST_MANIFEST_PATH: /app/configs/ingest.yaml INGEST_SHARD_ID: hl-bbo # Cloud native Kafka optimizations - KAFKA_PRODUCER_REQUEST_TIMEOUT_MS: 5000 + KAFKA_PRODUCER_REQUEST_TIMEOUT_MS: 15000 KAFKA_PRODUCER_RETRIES: 3 KAFKA_PRODUCER_RETRY_INITIAL_DELAY_MS: 100 KAFKA_PRODUCER_RETRY_MAX_DELAY_MS: 5000 KAFKA_PRODUCER_MAX_IN_FLIGHT_REQUESTS: 5 KAFKA_PRODUCER_IDEMPOTENT: true # Backpressure and overload protection - INGEST_OVERLOAD_PENDING_PUBLISH_THRESHOLD: 50 - INGEST_MAX_PENDING_PUBLISHES: 1000 + INGEST_OVERLOAD_PENDING_PUBLISH_THRESHOLD: 100 + INGEST_MAX_PENDING_PUBLISHES: 2000 # Hyperliquid subscription rate limiting - HYPERLIQUID_SUBSCRIPTION_INTERVAL_MS: 250 - HYPERLIQUID_SUBSCRIPTION_BATCH_SIZE: 3 + HYPERLIQUID_SUBSCRIPTION_INTERVAL_MS: 5000 + HYPERLIQUID_SUBSCRIPTION_BATCH_SIZE: 10 volumes: - ./configs/ingest.sample.yaml:/app/configs/ingest.yaml:ro healthcheck: diff --git a/docs/operations/runbooks/production-scaling.md b/docs/operations/runbooks/production-scaling.md new file mode 100644 index 00000000..a7b061c3 --- /dev/null +++ b/docs/operations/runbooks/production-scaling.md @@ -0,0 +1,245 @@ +# Production Scaling Operations + +This runbook covers how to safely scale production deployments up and down, including temporarily suspending all services and restoring them. + +## Prerequisites + +- kubectl configured with GKE cluster credentials +- Access to the ArgoCD instance (if using GitOps) +- Appropriate RBAC permissions for the target namespace + +## Scaling Down Production + +Use this when you need to temporarily stop all services (e.g., for maintenance, cost savings, or testing). + +### Step 1: Disable ArgoCD Auto-Sync (if using ArgoCD) + +If your deployment is managed by ArgoCD, you must disable auto-sync first. Otherwise, ArgoCD will immediately restore the pods to match the git state. + +```bash +# Disable auto-sync and self-heal for the pmon application +kubectl -n argocd patch application pmon --type merge -p '{"spec":{"syncPolicy":{"automated":null}}}' + +# Verify the change +kubectl -n argocd get application pmon -o jsonpath='{.spec.syncPolicy}' +``` + +### Step 2: Scale Down All Deployments + +```bash +# Scale all deployments in the pmon-prod namespace to 0 replicas +kubectl -n pmon-prod scale deployment --all --replicas=0 + +# Verify deployments are scaled down +kubectl -n pmon-prod get deployments +``` + +This affects: +- Gateway +- UI +- Stats +- Trader Intelligence +- Ingestor shards (prices, orderbook, asset-metadata, overview, candles, trades, bbo) + +### Step 3: Scale Down StatefulSets + +```bash +# Scale all StatefulSets (Kafka brokers) to 0 replicas +kubectl -n pmon-prod scale statefulset --all --replicas=0 + +# Verify StatefulSets are scaled down +kubectl -n pmon-prod get statefulsets +``` + +This affects: +- Kafka brokers (if using in-cluster Kafka) +- Redis (if using in-cluster Redis) +- Zookeeper (if enabled) + +### Step 4: Verify All Pods Are Terminated + +```bash +# Check that no pods are running +kubectl -n pmon-prod get pods + +# Should show "No resources found" or only terminating pods +``` + +## Scaling Up Production + +### Option 1: Restore via ArgoCD (Recommended) + +If you scaled down by disabling ArgoCD auto-sync, the easiest way to restore is to re-enable it: + +```bash +# Re-enable auto-sync and self-heal +kubectl -n argocd patch application pmon --type merge -p '{"spec":{"syncPolicy":{"automated":{"prune":true,"selfHeal":true}}}}' + +# Trigger a sync manually (optional - auto-sync will happen automatically) +kubectl -n argocd patch application pmon --type merge -p '{"operation":{"sync":{"syncStrategy":{"hook":{},"apply":{}}}}}' + +# Watch the pods come back up +kubectl -n pmon-prod get pods -w +``` + +ArgoCD will restore all deployments and StatefulSets to their configured replica counts from the git repository. + +### Option 2: Manual Scaling + +If not using ArgoCD or you want to scale up manually: + +```bash +# Scale StatefulSets back up first (Kafka needs to be ready before other services) +kubectl -n pmon-prod scale statefulset pmon-kafka --replicas=3 + +# Wait for Kafka to be ready +kubectl -n pmon-prod wait --for=condition=ready pod -l app.kubernetes.io/component=kafka --timeout=300s + +# Scale up the application deployments +kubectl -n pmon-prod scale deployment pmon-gateway --replicas=2 +kubectl -n pmon-prod scale deployment pmon-stats --replicas=1 +kubectl -n pmon-prod scale deployment pmon-trader-intelligence --replicas=2 +kubectl -n pmon-prod scale deployment pmon-ui --replicas=1 + +# Scale up ingestor shards (one at a time or in parallel) +kubectl -n pmon-prod scale deployment pmon-ingestor-prices --replicas=1 +kubectl -n pmon-prod scale deployment pmon-ingestor-orderbook --replicas=1 +kubectl -n pmon-prod scale deployment pmon-ingestor-asset-metadata --replicas=1 +kubectl -n pmon-prod scale deployment pmon-ingestor-overview --replicas=1 +kubectl -n pmon-pod scale deployment pmon-ingestor-candles --replicas=1 +kubectl -n pmon-prod scale deployment pmon-ingestor-trades --replicas=1 +kubectl -n pmon-prod scale deployment pmon-ingestor-bbo --replicas=1 + +# Verify all pods are running +kubectl -n pmon-prod get pods +``` + +## Selective Scaling + +You can also scale individual components without affecting the entire system. + +### Scale a Specific Deployment + +```bash +# Scale down trader intelligence only +kubectl -n pmon-prod scale deployment pmon-trader-intelligence --replicas=0 + +# Scale it back up +kubectl -n pmon-prod scale deployment pmon-trader-intelligence --replicas=2 +``` + +### Scale a Specific Ingestor Shard + +```bash +# Scale down the orderbook ingestor shard +kubectl -n pmon-prod scale deployment pmon-ingestor-orderbook --replicas=0 + +# Scale it back up +kubectl -n pmon-prod scale deployment pmon-ingestor-orderbook --replicas=1 +``` + +### Scale Kafka Brokers + +```bash +# Get current replica count +kubectl -n pmon-prod get statefulset pmon-kafka -o jsonpath='{.spec.replicas}' + +# Scale to 1 replica (minimum for testing) +kubectl -n pmon-prod scale statefulset pmon-kafka --replicas=1 + +# Scale back to production sizing (3 replicas for HA) +kubectl -n pmon-prod scale statefulset pmon-kafka --replicas=3 +``` + +**Warning**: Scaling Kafka brokers down can cause data loss if topics have a replication factor higher than the number of remaining brokers. Always ensure `replication factor ≤ broker count`. + +## Checking Current Replica Counts + +```bash +# View all deployments with their current/desired replica counts +kubectl -n pmon-prod get deployments -o custom-columns=NAME:.metadata.name,DESIRED:.spec.replicas,CURRENT:.status.replicas,READY:.status.readyReplicas + +# View all StatefulSets +kubectl -n pmon-prod get statefulsets -o custom-columns=NAME:.metadata.name,DESIRED:.spec.replicas,CURRENT:.status.replicas,READY:.status.readyReplicas +``` + +## Monitoring + +After scaling operations, monitor the following: + +```bash +# Watch pod status +kubectl -n pmon-prod get pods -w + +# Check pod logs for errors +kubectl -n pmon-prod logs -f deployment/pmon-gateway +kubectl -n pmon-prod logs -f deployment/pmon-ingestor-prices + +# Check service endpoints +curl https://api.perps.gmbh/healthz +curl https://api.perps.gmbh/api/v1/providers +``` + +### Kafka overload indicators + +- `ingestion_worker_kafka_overload_events_total` & `ingestion_worker_kafka_overload_logs_suppressed_total` – alert when overload events grow without corresponding logs (suppressed count > 0 over 5m) to catch Kafka pressure before it becomes user visible. +- `ingestion_worker_publish_throttle_total` / `ingestion_worker_publish_throttle_duration_ms` – spikes here mean the worker is backing off because the publish queue is close to `INGEST_MAX_PENDING_PUBLISHES`; scale out ingestor shards or increase broker partitions. +- `ingestion_worker_kafka_retry_total{topic,error_type}` – sustained growth indicates the adapter’s retry helper is masking broker hiccups; investigate before retries start to fail and trigger circuit breakers. +- All of these env-tunable knobs live under `ingestor.reliability.*` in the Helm values file so clusters can dial in defaults without editing manifests. + +## Troubleshooting + +### Pods Don't Come Back Up + +```bash +# Check for pod errors +kubectl -n pmon-prod describe pod + +# Check deployment events +kubectl -n pmon-prod describe deployment + +# Check if images are accessible +kubectl -n pmon-prod get events --sort-by='.lastTimestamp' +``` + +### ArgoCD Shows Out of Sync + +If ArgoCD shows the application as out-of-sync after manual scaling: + +```bash +# Sync to git state +kubectl -n argocd patch application pmon --type merge -p '{"operation":{"sync":{"syncStrategy":{"hook":{},"apply":{}}}}}' + +# Or use the ArgoCD UI to sync +``` + +### Kafka Issues After Scaling + +If Kafka brokers don't come back healthy: + +```bash +# Check Kafka pod logs +kubectl -n pmon-prod logs -f statefulset/pmon-kafka + +# Check persistent volume claims +kubectl -n pmon-prod get pvc + +# If needed, exec into a Kafka pod to check broker status +kubectl -n pmon-prod exec -it pmon-kafka-0 -- kafka-broker-api-versions.sh --bootstrap-server localhost:9092 +``` + +## Safety Notes + +1. **Always disable ArgoCD auto-sync** before manual scaling operations +2. **Scale StatefulSets (Kafka) before application deployments** when scaling up +3. **Check data replication** before scaling down Kafka brokers +4. **Verify DNS/Ingress** is working after scaling up +5. **Monitor logs and metrics** for 5-10 minutes after scaling up +6. **Document your changes** if deviating from git configuration + +## Related Documentation + +- [GKE Deployment Guide](./gke-deployment.md) +- [Deployment Process](./deployment-process.md) +- [Ingestor Recovery](./ingestor-recovery.md) + diff --git a/docs/providers/hyperliquid.md b/docs/providers/hyperliquid.md index b80ed160..64ee2b05 100644 --- a/docs/providers/hyperliquid.md +++ b/docs/providers/hyperliquid.md @@ -161,6 +161,17 @@ type HyperliquidBbo = { - Use BBO as an input to future fair-price calculations and to smooth order-book derived midpoints. +## Lag monitoring & alerting + +Hyperliquid shards occasionally replay historical trades/candles/BBO payloads while the websocket catches up. The ingestion SDK now keeps per-stream, per-asset lag state to prevent noisy logs while still surfacing real incidents: + +- **Bootstrap suppression** – messages older than `INGEST_HL_LAG_BOOTSTRAP_MS` (default `120 000` ms) are counted via `hyperliquid_ws_stale_payload_total` but skipped for logging until the shard observes a fresh tick. +- **Consecutive breach detection** – once live, any lag above `LAG_WARN_THRESHOLD_MS` must occur `INGEST_HL_LAG_ALERT_BREACHES` times (default `3`) before we emit a single `Hyperliquid websocket lag breach` error and increment `hyperliquid_ws_live_lag_breach_total`. +- **Alert throttling** – `INGEST_HL_LAG_ALERT_INTERVAL_MS` (default `60 000` ms) enforces a per `(stream, asset)` cool-down, and stale map entries expire after `INGEST_HL_LAG_STATE_TTL_MS` (default `1 800 000` ms) to avoid leaks. +- **Automatic remediation** – if the bounded lag exceeds `INGEST_HL_LAG_FATAL_MS` (default `240 000` ms) the adapter calls `signalFatalError('hyperliquid_ws_lag', …)` so Kubernetes restarts the pod instead of spamming operators. + +All lag metrics use `{ provider, stream, asset }` tags, making it easy to correlate `hyperliquid_ws_message_lag_ms` histograms with the new stale/breach counters in Grafana or Cloud Monitoring dashboards. + ## Manifest configuration cheatsheet The ingestion manifest drives which Hyperliquid feeds are enabled per shard. Example: diff --git a/kubernetes/chart/pmon/templates/deployment-ingestor.yaml b/kubernetes/chart/pmon/templates/deployment-ingestor.yaml index bbc60f37..02a2a2a9 100644 --- a/kubernetes/chart/pmon/templates/deployment-ingestor.yaml +++ b/kubernetes/chart/pmon/templates/deployment-ingestor.yaml @@ -33,6 +33,7 @@ {{- $replicaCount := default ($values.ingestor.replicaCount | default 1) $shard.replicaCount -}} {{- $servicePort := default ($values.ingestor.service.port | default 4001) $shard.servicePort -}} {{- $logLevel := default ($values.ingestor.logLevel | default "info") $shard.logLevel -}} +{{- $reliability := $values.ingestor.reliability | default (dict) -}} {{- $baseClientId := $values.ingestor.kafka.clientId | default "pmon-ingestor" -}} {{- $kafkaClientId := $baseClientId -}} {{- if $shard.kafka }} @@ -194,6 +195,30 @@ spec: - name: INGEST_STREAMS value: {{ $manifestStreamsJson | quote }} {{- end }} + {{- if hasKey $reliability "kafkaOverloadBackoffMs" }} + - name: INGEST_KAFKA_OVERLOAD_BACKOFF_MS + value: {{ printf "%v" $reliability.kafkaOverloadBackoffMs | quote }} + {{- end }} + {{- if hasKey $reliability "kafkaOverloadMaxRetries" }} + - name: INGEST_KAFKA_OVERLOAD_MAX_RETRIES + value: {{ printf "%v" $reliability.kafkaOverloadMaxRetries | quote }} + {{- end }} + {{- if hasKey $reliability "kafkaOverloadLogIntervalMs" }} + - name: INGEST_KAFKA_OVERLOAD_LOG_INTERVAL_MS + value: {{ printf "%v" $reliability.kafkaOverloadLogIntervalMs | quote }} + {{- end }} + {{- if hasKey $reliability "publishThrottleRatio" }} + - name: INGEST_PUBLISH_THROTTLE_RATIO + value: {{ printf "%v" $reliability.publishThrottleRatio | quote }} + {{- end }} + {{- if hasKey $reliability "publishThrottleDelayMs" }} + - name: INGEST_PUBLISH_THROTTLE_DELAY_MS + value: {{ printf "%v" $reliability.publishThrottleDelayMs | quote }} + {{- end }} + {{- if hasKey $reliability "publishThrottleMaxWaitMs" }} + - name: INGEST_PUBLISH_THROTTLE_MAX_WAIT_MS + value: {{ printf "%v" $reliability.publishThrottleMaxWaitMs | quote }} + {{- end }} {{- range $key, $val := $shard.env }} - name: {{ $key }} value: {{ $val | quote }} diff --git a/kubernetes/chart/pmon/values-minikube.yaml b/kubernetes/chart/pmon/values-minikube.yaml index b8987322..31f60e29 100644 --- a/kubernetes/chart/pmon/values-minikube.yaml +++ b/kubernetes/chart/pmon/values-minikube.yaml @@ -47,6 +47,7 @@ redis: ingestor: image: tag: local + logLevel: debug kafka: clientId: pmon-ingestor shards: @@ -227,11 +228,11 @@ ingestor: - name: INGEST_OVERLOAD_PENDING_PUBLISH_THRESHOLD value: "50" - name: INGEST_MAX_PENDING_PUBLISHES - value: "1000" + value: "2000" - name: HYPERLIQUID_SUBSCRIPTION_INTERVAL_MS - value: "250" + value: "2500" - name: HYPERLIQUID_SUBSCRIPTION_BATCH_SIZE - value: "3" + value: "10" resources: requests: cpu: 100m @@ -243,6 +244,7 @@ ingestor: gateway: image: tag: local + logLevel: debug kafka: clientId: pmon-gateway consumerGroup: pmon-gateway @@ -253,15 +255,28 @@ gateway: limits: cpu: 500m memory: 512Mi + extraEnv: + - name: GATEWAY_MEMORY_WARN_PCT + value: "55" + - name: GATEWAY_MEMORY_CRITICAL_PCT + value: "60" + - name: GATEWAY_MEMORY_FORCE_GC_PCT + value: "55" + - name: GATEWAY_DEFAULT_WS_DOMAINS + value: "prices,orderbook,metadata,overview,candles,trades,bbo" stats: image: tag: local + logLevel: debug kafka: clientId: pmon-stats groupId: pmon-stats redis: namespace: pmon:stats:metrics + appendProviderNamespace: false + aggregation: + windowSize: 30 traderIntelligence: enabled: true @@ -278,8 +293,8 @@ traderIntelligence: clientId: trader-intelligence-local enabled: true collection: - batchSize: 10 - concurrency: 3 + batchSize: 20 + concurrency: 5 highPriorityIntervalMs: 30000 mediumPriorityIntervalMs: 300000 cache: @@ -300,7 +315,9 @@ traderIntelligence: - name: TRADER_INTEL_AUTO_SEED_ENABLED value: "true" - name: TRADER_INTEL_AUTO_SEED_ASSETS - value: "BTC,ETH,SOL" + value: "BTC,ETH,SOL,VNTL:SPACEX" + - name: TRADER_INTEL_STATS_REDIS_URL + value: "redis://pmon-pmon-redis:6379" - name: TRADER_INTEL_STATS_REDIS_NAMESPACE value: "pmon:stats:metrics:hyperliquid" diff --git a/kubernetes/chart/pmon/values-prod.argocd.yaml b/kubernetes/chart/pmon/values-prod.argocd.yaml index d0e031bf..91dfe9f5 100644 --- a/kubernetes/chart/pmon/values-prod.argocd.yaml +++ b/kubernetes/chart/pmon/values-prod.argocd.yaml @@ -72,6 +72,7 @@ ingestor: image: repository: ingestor tag: 0.3.7 + logLevel: debug kafka: clientId: pmon-ingestor-prod manifest: @@ -255,11 +256,11 @@ ingestor: - name: INGEST_OVERLOAD_PENDING_PUBLISH_THRESHOLD value: "50" - name: INGEST_MAX_PENDING_PUBLISHES - value: "1000" + value: "2000" - name: HYPERLIQUID_SUBSCRIPTION_INTERVAL_MS - value: "250" + value: "2500" - name: HYPERLIQUID_SUBSCRIPTION_BATCH_SIZE - value: "3" + value: "10" extraEnv: [] resources: requests: @@ -327,6 +328,11 @@ traderIntelligence: value: "true" - name: TRADER_INTEL_AUTO_SEED_ASSETS value: "BTC,ETH,SOL,VNTL:SPACEX" + - name: TRADER_INTEL_STATS_REDIS_URL + valueFrom: + secretKeyRef: + name: redis-credentials + key: REDIS_URL - name: TRADER_INTEL_STATS_REDIS_NAMESPACE value: "pmon:stats:metrics:hyperliquid" @@ -334,6 +340,7 @@ gateway: image: repository: gateway tag: 0.3.7 + logLevel: debug service: type: NodePort nodePort: 32000 @@ -347,7 +354,15 @@ gateway: limits: cpu: 1 memory: 1Gi - extraEnv: [] + extraEnv: + - name: GATEWAY_MEMORY_WARN_PCT + value: "55" + - name: GATEWAY_MEMORY_CRITICAL_PCT + value: "60" + - name: GATEWAY_MEMORY_FORCE_GC_PCT + value: "55" + - name: GATEWAY_DEFAULT_WS_DOMAINS + value: "prices,orderbook,metadata,overview,candles,trades,bbo" backendConfig: enabled: true healthCheck: @@ -371,12 +386,15 @@ stats: image: repository: stats tag: 0.3.7 + logLevel: debug kafka: clientId: pmon-stats-prod groupId: pmon-stats-prod redis: namespace: pmon:stats:metrics appendProviderNamespace: false + aggregation: + windowSize: 30 extraEnv: [] ui: diff --git a/kubernetes/chart/pmon/values.yaml b/kubernetes/chart/pmon/values.yaml index ba0e619c..e90967e8 100644 --- a/kubernetes/chart/pmon/values.yaml +++ b/kubernetes/chart/pmon/values.yaml @@ -79,7 +79,7 @@ ingestor: tag: 0.3.7 pullPolicy: IfNotPresent useGlobalRegistry: true - logLevel: info + logLevel: warn service: port: 4001 kafka: @@ -130,6 +130,13 @@ ingestor: nodeSelector: {} tolerations: [] affinity: {} + reliability: + kafkaOverloadBackoffMs: 150 + kafkaOverloadMaxRetries: 3 + kafkaOverloadLogIntervalMs: 5000 + publishThrottleRatio: 0.85 + publishThrottleDelayMs: 25 + publishThrottleMaxWaitMs: 500 gateway: enabled: true @@ -139,7 +146,7 @@ gateway: tag: 0.3.7 pullPolicy: IfNotPresent useGlobalRegistry: true - logLevel: info + logLevel: warn service: type: ClusterIP port: 4000 @@ -187,7 +194,7 @@ stats: tag: 0.3.7 pullPolicy: IfNotPresent useGlobalRegistry: true - logLevel: info + logLevel: warn service: type: ClusterIP port: 4010 @@ -216,7 +223,7 @@ ui: tag: 0.3.7 pullPolicy: IfNotPresent useGlobalRegistry: true - logLevel: info + logLevel: warn service: type: ClusterIP port: 5173 @@ -270,7 +277,7 @@ traderIntelligence: tag: 0.3.7 pullPolicy: IfNotPresent useGlobalRegistry: true - logLevel: info + logLevel: warn service: type: ClusterIP port: 3004 diff --git a/packages/ingestion-sdk/src/providers/hyperliquid/index.ts b/packages/ingestion-sdk/src/providers/hyperliquid/index.ts index 831fff09..94695727 100644 --- a/packages/ingestion-sdk/src/providers/hyperliquid/index.ts +++ b/packages/ingestion-sdk/src/providers/hyperliquid/index.ts @@ -30,6 +30,7 @@ export { DEFAULT_INFO_ENDPOINT } from './rest/info'; export { deriveHyperliquidRestUrl } from './utils'; import { DEFAULT_INFO_ENDPOINT, postHyperliquidInfo, type InfoRequestPayload } from './rest/info'; +import { createRestProbeLogger, type RestProbeActionOptions } from './rest/diagnostics'; import { createAllMidsSubscription } from './subscriptions/allMids'; import { createActiveAssetCtxSubscription } from './subscriptions/activeAssetCtx'; import { createActiveAssetDataSubscription } from './subscriptions/activeAssetData'; @@ -44,6 +45,9 @@ const SUBSCRIPTION_INTERVAL_MS = parseInt(process.env.HYPERLIQUID_SUBSCRIPTION_I const SUBSCRIPTION_BATCH_SIZE = parseInt(process.env.HYPERLIQUID_SUBSCRIPTION_BATCH_SIZE || '5'); const PRICE_EPSILON = 1e-9; const METADATA_YIELD_INTERVAL = 25; +const ASSET_REFRESH_COOLDOWN_MS = parseInt(process.env.HYPERLIQUID_ASSET_REFRESH_COOLDOWN_MS || '15000'); +const ASSET_TRACKING_LOG_INTERVAL_MS = parseInt(process.env.HYPERLIQUID_ASSET_TRACKING_LOG_INTERVAL_MS || '300000'); +const MAX_TRACKING_LOG_ASSETS = 50; const CANDLE_INTERVAL_SECONDS: Record = { '15s': 15, '1m': 60, @@ -65,6 +69,55 @@ const yieldToEventLoop = async (): Promise => setImmediate(resolve); }); +const normaliseThrowable = (value: unknown): Record | undefined => { + if (value instanceof Error) { + const record: Record = { + name: value.name, + message: value.message + }; + if (typeof value.stack === 'string') { + record.stack = value.stack; + } + + const extras = value as unknown as { [key: string]: unknown; cause?: unknown }; + for (const key of ['code', 'status', 'errno', 'syscall', 'address', 'port', 'type']) { + const extraValue = extras[key]; + if (extraValue !== undefined) { + record[key] = extraValue; + } + } + + const { cause } = extras; + if (cause && cause !== value) { + const causeRecord = normaliseThrowable(cause); + if (causeRecord) { + record.cause = causeRecord; + } + } + + return record; + } + + if (typeof value === 'object' && value !== null) { + const entries = value as Record; + const record: Record = {}; + for (const [key, entryValue] of Object.entries(entries)) { + if (entryValue === undefined || typeof entryValue === 'function') continue; + record[key] = entryValue; + } + return Object.keys(record).length > 0 ? record : undefined; + } + + if (value !== undefined) { + return { message: String(value) }; + } + + return undefined; +}; + +const formatError = (error: unknown): Record => + normaliseThrowable(error) ?? { message: String(error) }; + type WsMessage = { channel?: string; data?: unknown; @@ -221,6 +274,159 @@ const resolveMaxPendingSubscriptions = (): number => { }; const MAX_PENDING_SUBSCRIPTIONS = resolveMaxPendingSubscriptions(); + +const readPositiveNumberEnv = (key: string, fallback: number): number => { + if (typeof process === 'undefined') { + return fallback; + } + const raw = process.env[key]; + if (!raw) return fallback; + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed < 0) { + return fallback; + } + return parsed; +}; + +const ADAPTER_PUBLISH_BACKOFF_MS = readPositiveNumberEnv('INGEST_KAFKA_OVERLOAD_BACKOFF_MS', 100); +const ADAPTER_PUBLISH_MAX_RETRIES = Math.max( + 0, + Math.floor(readPositiveNumberEnv('INGEST_KAFKA_OVERLOAD_MAX_RETRIES', 3)) +); + +const adapterSleep = (ms: number): Promise => + ms <= 0 ? Promise.resolve() : new Promise((resolve) => setTimeout(resolve, ms)); + +export const isRetryablePublishError = (error: unknown): boolean => { + const message = (error instanceof Error ? error.message : String(error)).toLowerCase(); + const errorType = (error as { type?: string })?.type?.toLowerCase(); + return ( + message.includes('timeout') || + message.includes('backpressure') || + message.includes('timeout while acquiring lock') || + errorType === 'request_timeout' + ); +}; + +type PublishWithRetryParams = { + runtimeContext: IngestionContext | undefined; + topic?: string; + envelope: ReturnType | Buffer; + key: string; + logContext?: Record; +}; + +export const publishWithRetry = async ({ + runtimeContext, + topic, + envelope, + key, + logContext +}: PublishWithRetryParams): Promise => { + if (!runtimeContext || !topic) return; + + let attempt = 0; + // eslint-disable-next-line no-constant-condition + while (true) { + try { + await runtimeContext.publish(topic, envelope, { key }); + return; + } catch (error) { + const shouldRetry = + attempt < ADAPTER_PUBLISH_MAX_RETRIES && isRetryablePublishError(error); + if (!shouldRetry) { + throw error; + } + + const backoffBase = ADAPTER_PUBLISH_BACKOFF_MS * Math.max(1, 2 ** attempt); + const jitter = Math.floor(Math.random() * Math.max(1, ADAPTER_PUBLISH_BACKOFF_MS)); + const delayMs = backoffBase + jitter; + runtimeContext.logger.warn?.('Retrying Hyperliquid Kafka publish', { + topic, + key, + delayMs, + attempt: attempt + 1, + ...(logContext ?? {}), + ...formatError(error) + }); + await adapterSleep(delayMs); + attempt += 1; + } + } +}; + +type MessageErrorLoggerOptions = { + logger: Required>; + metrics?: IngestionContext['metrics']; + provider: string; + logIntervalMs: number; +}; + +export const createMessageErrorLogger = ({ + logger, + metrics, + provider, + logIntervalMs +}: MessageErrorLoggerOptions) => { + let lastMessageErrorLogMs = 0; + let consecutiveErrorCount = 0; + let lastErrorString = ''; + let suppressedMessageErrorCount = 0; + + return (error: unknown) => { + metrics?.increment?.('hyperliquid_message_processing_errors_total', 1, { + provider + }); + const now = Date.now(); + const errorDetails = formatError(error); + const errorString = JSON.stringify(errorDetails); + + if (errorString === lastErrorString) { + consecutiveErrorCount += 1; + } else { + consecutiveErrorCount = 1; + lastErrorString = errorString; + suppressedMessageErrorCount = 0; + } + + const payload = buildHyperliquidMessageErrorLogPayload( + errorDetails, + consecutiveErrorCount, + now - lastMessageErrorLogMs + ); + + if (consecutiveErrorCount >= 50) { + logger.warn( + 'Repeated Hyperliquid message errors suppressed to keep the worker healthy', + { + ...payload, + suppressedSinceLastLog: suppressedMessageErrorCount + } + ); + consecutiveErrorCount = 0; + suppressedMessageErrorCount = 0; + lastMessageErrorLogMs = now; + return; + } + + if (now - lastMessageErrorLogMs >= logIntervalMs) { + const enrichedPayload = { + ...payload, + suppressedSinceLastLog: suppressedMessageErrorCount + }; + if (suppressedMessageErrorCount > 0) { + logger.warn('Burst of Hyperliquid message processing errors', enrichedPayload); + } else { + logger.error('Failed to process Hyperliquid message', enrichedPayload); + } + suppressedMessageErrorCount = 0; + lastMessageErrorLogMs = now; + return; + } + + suppressedMessageErrorCount += 1; + }; +}; const QUEUE_SATURATION_LOG_INTERVAL_MS = 10_000; const LAG_WARN_THRESHOLD_MS = 15_000; const METADATA_REFRESH_INTERVAL_MS = 60_000; @@ -246,35 +452,156 @@ export type HyperliquidSubscriptionContext = { cache?: CacheWriter; }; -const observeMessageLag = ( +type HyperliquidLagMonitor = ( context: HyperliquidSubscriptionContext, stream: string, - asset: string | undefined, + asset?: string, timestamp?: number -) => { - const histogram = context.runtimeContext.metrics?.histogram; - if (!histogram) return; - if (!asset) return; - if (typeof timestamp !== 'number' || !Number.isFinite(timestamp)) return; - const now = Date.now(); - const lag = now - timestamp; - if (!Number.isFinite(lag) || lag < 0) return; - const boundedLag = Math.min(lag, MAX_LAG_MS); - histogram('hyperliquid_ws_message_lag_ms', boundedLag, { - provider: context.metadata.provider, - stream, - asset: asset.toUpperCase() - }); - if (boundedLag > LAG_WARN_THRESHOLD_MS) { - if (Math.random() < 0.1) { - context.runtimeContext.logger.warn?.('Hyperliquid websocket lag detected', { - provider: context.metadata.provider, +) => void; + +type LagMonitorState = { + bootstrapActive: boolean; + consecutiveBreaches: number; + lastAlertMs: number; + lastSeenMs: number; +}; + +const resolveLagMonitorConfig = () => { + const bootstrapGraceMs = readPositiveNumberEnv('INGEST_HL_LAG_BOOTSTRAP_MS', 120_000); + const alertBreachesRaw = readPositiveNumberEnv('INGEST_HL_LAG_ALERT_BREACHES', 3); + const alertIntervalMs = readPositiveNumberEnv('INGEST_HL_LAG_ALERT_INTERVAL_MS', 60_000); + const fatalThresholdMs = Math.max( + LAG_WARN_THRESHOLD_MS, + readPositiveNumberEnv('INGEST_HL_LAG_FATAL_MS', 240_000) + ); + const stateTtlMs = Math.max(alertIntervalMs, readPositiveNumberEnv('INGEST_HL_LAG_STATE_TTL_MS', 30 * 60 * 1000)); + + return { + bootstrapGraceMs, + alertBreaches: Math.max(1, Math.floor(alertBreachesRaw || 1)), + alertIntervalMs, + fatalThresholdMs, + stateTtlMs + }; +}; + +export const createHyperliquidLagMonitor = (provider: string): HyperliquidLagMonitor => { + const config = resolveLagMonitorConfig(); + const state = new Map(); + let nextCleanupAt = Date.now() + config.stateTtlMs; + + const cleanup = (now: number) => { + if (now < nextCleanupAt) { + return; + } + nextCleanupAt = now + config.stateTtlMs; + for (const [key, entry] of state.entries()) { + if (now - entry.lastSeenMs >= config.stateTtlMs) { + state.delete(key); + } + } + }; + + return (context, stream, asset, timestamp) => { + if (!asset) { + return; + } + if (typeof timestamp !== 'number' || !Number.isFinite(timestamp)) { + return; + } + const metrics = context.runtimeContext.metrics; + const histogram = metrics?.histogram; + const increment = metrics?.increment; + const logger = context.runtimeContext.logger; + const assetKey = asset.toUpperCase(); + const key = `${stream}:${assetKey}`; + const now = Date.now(); + const lag = now - timestamp; + if (!Number.isFinite(lag) || lag < 0) { + return; + } + const boundedLag = Math.min(lag, MAX_LAG_MS); + histogram?.('hyperliquid_ws_message_lag_ms', boundedLag, { + provider, + stream, + asset: assetKey + }); + + let entry = state.get(key); + if (!entry) { + entry = { + bootstrapActive: true, + consecutiveBreaches: 0, + lastAlertMs: 0, + lastSeenMs: now + }; + state.set(key, entry); + } + entry.lastSeenMs = now; + + if (entry.bootstrapActive) { + if (lag <= config.bootstrapGraceMs) { + entry.bootstrapActive = false; + entry.consecutiveBreaches = 0; + } else { + increment?.('hyperliquid_ws_stale_payload_total', 1, { + provider, + stream, + asset: assetKey + }); + cleanup(now); + return; + } + } + + if (boundedLag <= LAG_WARN_THRESHOLD_MS) { + entry.consecutiveBreaches = 0; + cleanup(now); + return; + } + + entry.consecutiveBreaches += 1; + if (entry.consecutiveBreaches < config.alertBreaches) { + cleanup(now); + return; + } + + if (now - entry.lastAlertMs < config.alertIntervalMs) { + cleanup(now); + return; + } + + entry.lastAlertMs = now; + entry.consecutiveBreaches = 0; + increment?.('hyperliquid_ws_live_lag_breach_total', 1, { + provider, + stream, + asset: assetKey + }); + + logger.error( + 'Hyperliquid websocket lag breach', + { + provider, + stream, + asset, + lagMs: boundedLag, + warnThresholdMs: LAG_WARN_THRESHOLD_MS, + alertIntervalMs: config.alertIntervalMs + } + ); + + if (boundedLag >= config.fatalThresholdMs) { + context.runtimeContext.signalFatalError?.('hyperliquid_ws_lag', { stream, asset, - lagMs: boundedLag + lagMs: boundedLag, + fatalThresholdMs: config.fatalThresholdMs }); } - } + + cleanup(now); + }; }; export type HyperliquidSubscription = { @@ -287,6 +614,13 @@ export const createHyperliquidAdapter: IngestionAdapterFactory(payload: InfoRequestPayload) => postHyperliquidInfo(payload, infoEndpoint); + const normaliseTopicName = (topic?: string): string | undefined => { + if (typeof topic !== 'string') return undefined; + const trimmed = topic.trim(); + return trimmed.length > 0 ? trimmed : undefined; + }; + const assetMetadataTopic = normaliseTopicName(config.topics.assetMetadata); + const overviewTopic = normaliseTopicName(config.topics.overview); const metadata: IngestionAdapterMetadata = { id: `${config.provider}-hyperliquid`, provider: config.provider, @@ -302,6 +636,62 @@ export const createHyperliquidAdapter: IngestionAdapterFactory 0 ? config.shard.count : 1; const shardIndex = config.shard.index >= 0 ? config.shard.index % shardCount : 0; + const shardLabel = `${shardIndex + 1}/${shardCount}`; + const isControlShard = shardIndex === 0; + const restProbeLogger = createRestProbeLogger({ + shardLabel, + provider: metadata.provider, + formatError + }); + + const requestInfo = async ( + context: IngestionContext, + action: string, + payload: InfoRequestPayload, + options?: RestProbeActionOptions + ): Promise => restProbeLogger.wrap(context, action, () => postInfo(payload), options); + + const logFeatureAvailability = (context: IngestionContext) => { + if (!assetMetadataTopic) { + context.logger.info( + 'Hyperliquid asset metadata publishing disabled for shard', + { + provider: metadata.provider, + shard: shardLabel, + reason: 'assetMetadataTopicMissing' + } + ); + } + if (!overviewTopic) { + context.logger.info( + 'Hyperliquid overview publishing disabled for shard', + { + provider: metadata.provider, + shard: shardLabel, + reason: 'overviewTopicMissing' + } + ); + } else if (!isControlShard) { + context.logger.info( + 'Hyperliquid overview publishing delegated to control shard', + { + provider: metadata.provider, + shard: shardLabel, + reason: 'overviewDelegated' + } + ); + } + if (!isControlShard) { + context.logger.info( + 'Hyperliquid optional REST probes disabled for shard', + { + provider: metadata.provider, + shard: shardLabel, + reason: 'restProbesDelegated' + } + ); + } + }; /** * Streams declared in the manifest or via env (`HYPERLIQUID_STREAMS`) control which websocket channels we subscribe to. @@ -383,9 +773,6 @@ type CoinStreamKind = 'l2Book' | 'activeAssetCtx' | 'activeAssetData' | 'candles * and dedupe (asset, stream, interval) combinations. */ let runtimeContext: IngestionContext | undefined; - let lastMessageErrorLogMs = 0; - let consecutiveErrorCount = 0; - let lastErrorString = ''; let reconnectAttempts = 0; let reconnectTimer: NodeJS.Timeout | undefined; let heartbeatTimer: NodeJS.Timeout | undefined; @@ -393,6 +780,8 @@ type CoinStreamKind = 'l2Book' | 'activeAssetCtx' | 'activeAssetData' | 'candles const successfullySubscribedAssets = new Set(); let connectionStableStartTime = 0; let restInterval: ReturnType | undefined; + const unhandledChannelLogTimes = new Map(); + const UNHANDLED_CHANNEL_LOG_INTERVAL_MS = 60_000; const coinStreams: Record { const resolvedDex = dex ?? resolveDexForCoin(symbol); subscribeToL2Book(socket!, symbol, resolvedDex); - runtimeContext?.logger.debug?.( - `Subscribed to Hyperliquid l2Book coin=${symbol} dex=${resolvedDex ?? 'default'}` - ); } }, activeAssetCtx: { @@ -418,9 +804,6 @@ type CoinStreamKind = 'l2Book' | 'activeAssetCtx' | 'activeAssetData' | 'candles const resolvedDex = dex ?? resolveDexForCoin(symbol); if (resolvedDex) subscription.dex = resolvedDex; sendSubscription('activeAssetCtx', subscription); - runtimeContext?.logger.debug?.( - `Subscribed to Hyperliquid activeAssetCtx coin=${symbol} dex=${resolvedDex ?? 'default'}` - ); } }, activeAssetData: { @@ -432,9 +815,6 @@ type CoinStreamKind = 'l2Book' | 'activeAssetCtx' | 'activeAssetData' | 'candles const resolvedDex = dex ?? resolveDexForCoin(symbol); if (resolvedDex) subscription.dex = resolvedDex; sendSubscription('activeAssetData', subscription); - runtimeContext?.logger.debug?.( - `Subscribed to Hyperliquid activeAssetData coin=${symbol} dex=${resolvedDex ?? 'default'}` - ); } }, candles: { @@ -446,9 +826,6 @@ type CoinStreamKind = 'l2Book' | 'activeAssetCtx' | 'activeAssetData' | 'candles const resolvedDex = dex ?? resolveDexForCoin(symbol); if (resolvedDex) subscription.dex = resolvedDex; sendSubscription('candle', subscription); - runtimeContext?.logger.debug?.( - `Subscribed to Hyperliquid candle stream coin=${symbol} interval=${interval} dex=${resolvedDex ?? 'default'}` - ); } }, trades: { @@ -459,9 +836,6 @@ type CoinStreamKind = 'l2Book' | 'activeAssetCtx' | 'activeAssetData' | 'candles const resolvedDex = dex ?? resolveDexForCoin(symbol); if (resolvedDex) subscription.dex = resolvedDex; sendSubscription('trades', subscription); - runtimeContext?.logger.debug?.( - `Subscribed to Hyperliquid trades coin=${symbol} dex=${resolvedDex ?? 'default'}` - ); } }, bbo: { @@ -472,9 +846,6 @@ type CoinStreamKind = 'l2Book' | 'activeAssetCtx' | 'activeAssetData' | 'candles const resolvedDex = dex ?? resolveDexForCoin(symbol); if (resolvedDex) subscription.dex = resolvedDex; sendSubscription('bbo', subscription); - runtimeContext?.logger.debug?.( - `Subscribed to Hyperliquid BBO coin=${symbol} dex=${resolvedDex ?? 'default'}` - ); } } }; @@ -558,10 +929,14 @@ type CoinStreamKind = 'l2Book' | 'activeAssetCtx' | 'activeAssetData' | 'candles stream.subscribe(task.symbol, task.dex, task.interval); stream.subscribed.add(key); } catch (error) { - runtimeContext?.logger.error(`Failed to subscribe to Hyperliquid ${task.kind}`, { - coin: task.symbol, - ...formatError(error) - }); + runtimeContext?.logger.error?.( + `Failed to subscribe to Hyperliquid ${task.kind}`, + { + coin: task.symbol, + stream: task.kind, + ...formatError(error) + } + ); // requeue at end subscriptionQueue.push(task); queuedTasks.add(key); @@ -572,18 +947,24 @@ type CoinStreamKind = 'l2Book' | 'activeAssetCtx' | 'activeAssetData' | 'candles // Log subscription progress periodically if (initialQueueLength > 0 && subscriptionQueue.length % 50 === 0) { const totalSubscribed = Object.values(coinStreams).reduce((total, stream) => total + stream.subscribed.size, 0); - runtimeContext?.logger.info('Hyperliquid subscription progress', { - remaining: subscriptionQueue.length, - totalSubscribed, - initialQueueLength - }); + runtimeContext?.logger.info?.( + 'Hyperliquid subscription progress', + { + remaining: subscriptionQueue.length, + totalSubscribed, + initialQueueLength + } + ); } if (subscriptionQueue.length === 0 && subscriptionInterval) { const totalSubscribed = Object.values(coinStreams).reduce((total, stream) => total + stream.subscribed.size, 0); - runtimeContext?.logger.info('Hyperliquid subscription queue drained', { - totalSubscribed - }); + runtimeContext?.logger.info?.( + 'Hyperliquid subscription queue drained', + { + totalSubscribed + } + ); clearInterval(subscriptionInterval); subscriptionInterval = undefined; } @@ -598,9 +979,12 @@ type CoinStreamKind = 'l2Book' | 'activeAssetCtx' | 'activeAssetData' | 'candles // Check if we haven't received a pong in too long (connection might be dead) const timeSinceLastPong = Date.now() - lastPongReceived; if (timeSinceLastPong > 60000) { // 60 seconds without pong - runtimeContext?.logger.warn('WebSocket appears dead, forcing reconnection', { - timeSinceLastPong - }); + runtimeContext?.logger.warn?.( + 'WebSocket appears dead, forcing reconnection', + { + timeSinceLastPong + } + ); socket.terminate(); } } @@ -629,21 +1013,27 @@ type CoinStreamKind = 'l2Book' | 'activeAssetCtx' | 'activeAssetData' | 'candles // Don't clear successfullySubscribedAssets here - we want to preserve this across reconnections }; -const subscriptions = createSubscriptions( - config, - metadata, - shouldHandleAsset, - enabledStreams, - candleIntervals, - tradesEnabled, - bboEnabled -); + const lagMonitor = createHyperliquidLagMonitor(metadata.provider); + const subscriptions = createSubscriptions( + config, + metadata, + shouldHandleAsset, + enabledStreams, + candleIntervals, + tradesEnabled, + bboEnabled, + lagMonitor + ); const subscriptionsByChannel = new Map(); subscriptions.forEach((subscription) => subscriptionsByChannel.set(subscription.channel, subscription)); let socket: WebSocket | undefined; let shuttingDown = false; const assetDirectory = new Map(); const skippedAssets = new Set(); + const untrackedAssets = new Map(); + let lastAssetTrackingLog = 0; + let assetRefreshInFlight: Promise | undefined; + let lastAssetRefreshAttempt = 0; const dexSubscriptionTargets = new Map(); const activeAllMidsSubscriptions = new Set(); let metadataInterval: ReturnType | undefined; @@ -732,6 +1122,72 @@ const subscriptions = createSubscriptions( return entry; }; + const listTrackedAssets = (): string[] => { + const assets: string[] = []; + for (const [key, entry] of assetDirectory.entries()) { + if (!entry || typeof entry.symbol !== 'string') continue; + if (key !== entry.symbol) continue; + if (!entry.handled) continue; + assets.push(entry.symbol); + } + return assets; + }; + + const normaliseTrackingKey = (asset: string): string => asset.trim().toUpperCase(); + + const markUntrackedAsset = (asset: string) => { + const key = normaliseTrackingKey(asset); + if (untrackedAssets.has(key)) return; + untrackedAssets.set(key, asset); + runtimeContext?.metrics?.increment?.('hyperliquid_untracked_asset_events_total', 1, { + provider: metadata.provider + }); + }; + + const clearUntrackedAsset = (asset: string) => { + untrackedAssets.delete(normaliseTrackingKey(asset)); + }; + + const snapshotAssetTracking = () => { + const trackedAssets = listTrackedAssets(); + const untracked = Array.from(untrackedAssets.values()); + return { + trackedCount: trackedAssets.length, + trackedSample: trackedAssets.slice(0, MAX_TRACKING_LOG_ASSETS), + untrackedCount: untracked.length, + untrackedSample: untracked.slice(0, MAX_TRACKING_LOG_ASSETS) + }; + }; + + const logAssetTrackingSummary = (reason: string, meta?: Record) => { + if (!runtimeContext) return; + const now = Date.now(); + if (now - lastAssetTrackingLog < ASSET_TRACKING_LOG_INTERVAL_MS) { + return; + } + lastAssetTrackingLog = now; + const snapshot = snapshotAssetTracking(); + runtimeContext.logger.warn?.( + 'Hyperliquid asset tracking summary', + { + provider: metadata.provider, + shard: shardLabel, + reason, + trackedCount: snapshot.trackedCount, + trackedSample: snapshot.trackedSample, + untrackedCount: snapshot.untrackedCount, + untrackedSample: snapshot.untrackedSample, + ...meta + } + ); + runtimeContext.metrics?.histogram?.('hyperliquid_tracked_assets_count', snapshot.trackedCount, { + provider: metadata.provider + }); + runtimeContext.metrics?.histogram?.('hyperliquid_untracked_assets_count', snapshot.untrackedCount, { + provider: metadata.provider + }); + }; + const logSkippedAsset = (symbol: string, lookupKey: string) => { const normalized = lookupKey.toUpperCase(); if (skippedAssets.has(normalized)) return; @@ -781,9 +1237,8 @@ const subscriptions = createSubscriptions( }; const upsertAssetContext = async (asset: string, ctxUpdate: Partial) => { - const entry = getHandledEntry(asset); + const entry = await ensureTrackedAsset(asset); if (!entry) { - runtimeContext?.logger.debug?.(`Ignoring context update for unassigned asset asset=${asset}`); return false; } @@ -799,9 +1254,8 @@ const subscriptions = createSubscriptions( const publishPrice = async (asset: string, price: number, timestamp: number) => { if (!runtimeContext) return; - const entry = getHandledEntry(asset); + const entry = await ensureTrackedAsset(asset); if (!entry) { - runtimeContext?.logger.debug?.(`Ignoring price update for unassigned asset asset=${asset}`); return; } @@ -823,10 +1277,16 @@ const subscriptions = createSubscriptions( ); try { - await runtimeContext.publish(config.topics.prices, envelope, { key: entry.symbol }); + await publishWithRetry({ + runtimeContext, + topic: config.topics.prices, + envelope, + key: entry.symbol, + logContext: { asset: entry.symbol, stream: 'prices' } + }); entry.lastPublishedPrice = price; } catch (error) { - runtimeContext.logger.error('Failed to publish Hyperliquid price update', formatError(error)); + runtimeContext.logger.error?.('Failed to publish Hyperliquid price update', formatError(error)); return; } @@ -844,9 +1304,8 @@ const subscriptions = createSubscriptions( */ const publishCandle = async (asset: string, interval: string, payload: CandleSnapshotPayload) => { if (!runtimeContext || !config.topics.candles) return; - const entry = getHandledEntry(asset); + const entry = await ensureTrackedAsset(asset); if (!entry) { - runtimeContext?.logger.debug?.(`Ignoring candle update for unassigned asset asset=${asset}`); return; } @@ -863,7 +1322,25 @@ const subscriptions = createSubscriptions( } ); - await runtimeContext.publish(config.topics.candles, envelope, { key: entry.symbol }); + try { + await publishWithRetry({ + runtimeContext, + topic: config.topics.candles, + envelope, + key: entry.symbol, + logContext: { asset: entry.symbol, stream: 'candles', interval } + }); + } catch (error) { + runtimeContext.logger.error?.( + 'Failed to publish Hyperliquid candle update', + { + asset: entry.symbol, + interval, + ...formatError(error) + } + ); + return; + } await writeCandleCache(entry.symbol, interval, payload); }; @@ -872,9 +1349,8 @@ const subscriptions = createSubscriptions( */ const publishTrade = async (asset: string, payload: TradeEventPayload) => { if (!runtimeContext || !config.topics.trades) return; - const entry = getHandledEntry(asset); + const entry = await ensureTrackedAsset(asset); if (!entry) { - runtimeContext?.logger.debug?.(`Ignoring trade update for unassigned asset asset=${asset}`); return; } @@ -888,7 +1364,23 @@ const subscriptions = createSubscriptions( } ); - await runtimeContext.publish(config.topics.trades, envelope, { key: entry.symbol }); + try { + await publishWithRetry({ + runtimeContext, + topic: config.topics.trades, + envelope, + key: entry.symbol, + logContext: { asset: entry.symbol, stream: 'trades' } + }); + } catch (error) { + runtimeContext.logger.error?.( + 'Failed to publish Hyperliquid trade update', + { + asset: entry.symbol, + ...formatError(error) + } + ); + } }; /** @@ -896,9 +1388,8 @@ const subscriptions = createSubscriptions( */ const publishBbo = async (asset: string, payload: BboPayload) => { if (!runtimeContext || !config.topics.bbo) return; - const entry = getHandledEntry(asset); + const entry = await ensureTrackedAsset(asset); if (!entry) { - runtimeContext?.logger.debug?.(`Ignoring BBO update for unassigned asset asset=${asset}`); return; } @@ -912,7 +1403,23 @@ const subscriptions = createSubscriptions( } ); - await runtimeContext.publish(config.topics.bbo, envelope, { key: entry.symbol }); + try { + await publishWithRetry({ + runtimeContext, + topic: config.topics.bbo, + envelope, + key: entry.symbol, + logContext: { asset: entry.symbol, stream: 'bbo' } + }); + } catch (error) { + runtimeContext.logger.error?.( + 'Failed to publish Hyperliquid BBO update', + { + asset: entry.symbol, + ...formatError(error) + } + ); + } }; const publishAssetMetadata = async ( @@ -925,7 +1432,7 @@ const subscriptions = createSubscriptions( } const metrics = normaliseAssetMetadata(entry, ctx, config.providerMetadata?.tags); if (!metrics) { - context.logger.warn('Skipping asset metadata publish for unresolved asset', { asset: entry.symbol }); + context.logger.warn?.('Skipping asset metadata publish for unresolved asset', { asset: entry.symbol }); return null; } @@ -935,23 +1442,38 @@ const subscriptions = createSubscriptions( return metrics; } + entry.metrics = metrics; + + if (!assetMetadataTopic) { + entry.metadataSignature = signature; + return metrics; + } + const envelope = createEnvelope(metadata.provider, 'assetMetadata', metrics.payload, { asset: metrics.asset, timestamp: metrics.timestamp }); try { - await context.publish(config.topics.assetMetadata, envelope, { key: entry.symbol }); + await context.publish(assetMetadataTopic, envelope, { key: entry.symbol }); } catch (error) { - context.logger.error('Failed to publish Hyperliquid asset metadata', formatError(error)); + const errorDetails = formatError(error); + context.logger.error?.( + 'Failed to publish Hyperliquid asset metadata', + { + topic: assetMetadataTopic, + asset: entry.symbol, + ...errorDetails + } + ); context.signalFatalError?.('hyperliquid_metadata_publish_failed', { asset: entry.symbol, - ...formatError(error) + topic: assetMetadataTopic, + ...errorDetails }); throw error; } - entry.metrics = metrics; entry.metadataSignature = signature; await writeMetadataCache({ @@ -965,8 +1487,12 @@ const subscriptions = createSubscriptions( }; const publishOverview = async (payload: ExchangeOverviewPayload, context: IngestionContext) => { + if (!overviewTopic) { + context.logger.debug?.('Skipping Hyperliquid overview publish (topic disabled)'); + return; + } const envelope = createEnvelope(metadata.provider, 'overview', payload, { timestamp: payload.timestamp }); - await context.publish(config.topics.overview, envelope, { key: metadata.provider }); + await context.publish(overviewTopic, envelope, { key: metadata.provider }); await writeOverviewCache({ provider: metadata.provider, @@ -1032,8 +1558,11 @@ const subscriptions = createSubscriptions( const seen = new Set([DEFAULT_DEX_KEY]); try { - const body = await postInfo>({ type: 'perpDexs' }); - + const body = await requestInfo>( + context, + 'info:perpDexs:bootstrap', + { type: 'perpDexs' } + ); if (Array.isArray(body)) { for (const entry of body) { if (!entry || typeof entry.name !== 'string') continue; @@ -1047,22 +1576,31 @@ const subscriptions = createSubscriptions( } } } catch (error) { - context.logger.warn('Failed to fetch Hyperliquid perp dex list', formatError(error)); + context.logger.debug?.( + 'Hyperliquid falling back to default dex targets after bootstrap failure', + { + shard: shardLabel, + reason: 'perpDexBootstrapFailed', + ...formatError(error) + } + ); } return targets; }; const fetchDexSnapshot = async ( + context: IngestionContext, dex: string | undefined ): Promise<{ universe: PerpUniverseAsset[]; assetCtxs: Array }> => { + const dexKey = dex && dex.length > 0 ? dex : 'default'; const payload: InfoRequestPayload = typeof dex === 'string' && dex.length > 0 ? { type: 'metaAndAssetCtxs', dex } : { type: 'metaAndAssetCtxs', dex: '' }; - const body = await postInfo< + const body = await requestInfo< [{ universe?: PerpUniverseAsset[] }, Array?] - >(payload); + >(context, `info:metaAndAssetCtxs:${dexKey}`, payload, { warnAfter: 2 }); const universe = body?.[0]?.universe ?? []; const assetCtxs = Array.isArray(body?.[1]) ? (body[1] as Array) : []; @@ -1125,20 +1663,24 @@ const subscriptions = createSubscriptions( setDexSubscriptionTargets(dexTargets); ensureAllMidsSubscriptions(); + const shouldCollectOverview = Boolean(overviewTopic) && isControlShard; const aggregatedUniverse: PerpUniverseAsset[] = []; const aggregatedCtxs: Array = []; let globalIndex = 0; for (const dex of dexTargets) { + const metadataAction = `metadata:${dex ?? 'default'}`; try { - const { universe, assetCtxs } = await fetchDexSnapshot(dex); + const { universe, assetCtxs } = await fetchDexSnapshot(context, dex); for (let index = 0; index < universe.length; index += 1) { const assetMeta = universe[index]; const ctx = assetCtxs[index]; - aggregatedUniverse.push(assetMeta); - aggregatedCtxs.push(ctx); + if (shouldCollectOverview) { + aggregatedUniverse.push(assetMeta); + aggregatedCtxs.push(ctx); + } globalIndex += 1; const entry = registerAsset(assetMeta.name, globalIndex, assetMeta, ctx, dex); @@ -1153,43 +1695,107 @@ const subscriptions = createSubscriptions( await yieldToEventLoop(); } } + restProbeLogger.recordSuccess(context, metadataAction); } catch (error) { - context.logger.error('Failed to refresh Hyperliquid asset metadata for dex', { - dex: dex ?? 'default', - ...formatError(error) - }); + restProbeLogger.recordFailure(context, metadataAction, error, { warnAfter: 3, logIntervalMs: 120_000 }); } } - const overviewSnapshot = buildOverviewSnapshot(aggregatedUniverse, aggregatedCtxs); - if (overviewSnapshot) { - await publishOverview(overviewSnapshot, context); + if (shouldCollectOverview) { + const overviewSnapshot = buildOverviewSnapshot(aggregatedUniverse, aggregatedCtxs); + if (overviewSnapshot) { + await publishOverview(overviewSnapshot, context); + } } context.logger.debug?.( - `Refreshed Hyperliquid asset metadata count=${aggregatedUniverse.length} dexCount=${dexTargets.length}` + 'Refreshed Hyperliquid asset metadata', + { + shard: shardLabel, + assets: aggregatedUniverse.length, + dexCount: dexTargets.length + } ); } catch (error) { - runtimeContext?.logger.error('Failed to refresh Hyperliquid asset metadata', formatError(error)); + runtimeContext?.logger.error?.( + 'Failed to refresh Hyperliquid asset metadata', + { + shard: shardLabel, + ...formatError(error) + } + ); } }; - const listTrackedAssets = (): string[] => { - const assets: string[] = []; - for (const [key, entry] of assetDirectory.entries()) { - if (!entry || typeof entry.symbol !== 'string') continue; - if (key !== entry.symbol) continue; - if (!entry.handled) continue; - assets.push(entry.symbol); + const triggerAssetRefresh = async (asset: string): Promise => { + if (!runtimeContext) return; + if (assetRefreshInFlight) { + await assetRefreshInFlight; + return; } - return assets; + + const now = Date.now(); + if (now - lastAssetRefreshAttempt < ASSET_REFRESH_COOLDOWN_MS) { + return; + } + + lastAssetRefreshAttempt = now; + assetRefreshInFlight = (async () => { + try { + runtimeContext?.logger.info?.( + 'Hyperliquid refreshing asset metadata after missing asset', + { + provider: metadata.provider, + shard: shardLabel, + asset + } + ); + await refreshAssetMetadata(runtimeContext); + } catch (error) { + runtimeContext?.logger.error?.( + 'Failed to refresh Hyperliquid metadata after missing asset', + { + asset, + ...formatError(error) + } + ); + } finally { + assetRefreshInFlight = undefined; + } + })(); + + await assetRefreshInFlight; }; + async function ensureTrackedAsset(asset: string): Promise { + const current = getHandledEntry(asset); + if (current) { + clearUntrackedAsset(asset); + return current; + } + + markUntrackedAsset(asset); + await triggerAssetRefresh(asset); + const resolved = getHandledEntry(asset); + if (resolved) { + clearUntrackedAsset(asset); + return resolved; + } + + logAssetTrackingSummary('assetMissingAfterRefresh', { asset }); + return null; + } + const checkAllMidsHealth = async (context: IngestionContext) => { try { - const response = await postInfo<{ mids?: Array<[string, number | string]> }>({ - type: 'allMids' - }); + const response = await requestInfo<{ mids?: Array<[string, number | string]> }>( + context, + 'info:allMids', + { + type: 'allMids' + }, + { warnAfter: 2 } + ); const mids = Array.isArray(response?.mids) ? response.mids : []; for (const entry of mids) { const [coin, rawMid] = entry; @@ -1213,38 +1819,60 @@ const subscriptions = createSubscriptions( asset: resolved.symbol }); if (Math.random() < 0.25) { - context.logger.warn('Hyperliquid allMids discrepancy detected', { - asset: resolved.symbol, - websocketMid: cachedMid, - restMid, - basisPoints - }); + context.logger.warn?.( + 'Hyperliquid allMids discrepancy detected', + { + asset: resolved.symbol, + websocketMid: cachedMid, + restMid, + basisPoints + } + ); } } } } catch (error) { - context.logger.warn('Failed to fetch Hyperliquid allMids payload', formatError(error)); + context.logger.debug?.( + 'Hyperliquid skipped allMids health check due to fetch failure', + { + shard: shardLabel, + action: 'info:allMids', + ...formatError(error) + } + ); } }; const checkPerpsAtOpenInterestCap = async (context: IngestionContext) => { try { - const response = await postInfo<{ perpsAtOpenInterestCap?: string[] } | string[]>({ - type: 'perpsAtOpenInterestCap' - }); + const response = await requestInfo<{ perpsAtOpenInterestCap?: string[] } | string[]>( + context, + 'info:perpsAtOpenInterestCap', + { + type: 'perpsAtOpenInterestCap' + }, + { warnAfter: 2 } + ); const list = Array.isArray((response as { perpsAtOpenInterestCap?: string[] })?.perpsAtOpenInterestCap) ? ((response as { perpsAtOpenInterestCap?: string[] }).perpsAtOpenInterestCap ?? []) : Array.isArray(response) ? (response as string[]) : []; if (list.length > 0) { - context.logger.warn('Hyperliquid perps at open interest cap', { assets: list }); + context.logger.warn?.('Hyperliquid perps at open interest cap', { assets: list }); context.metrics?.increment?.('hyperliquid_perps_open_interest_cap_total', list.length, { provider: metadata.provider }); } } catch (error) { - context.logger.warn('Failed to fetch Hyperliquid open interest caps', formatError(error)); + context.logger.debug?.( + 'Hyperliquid perps at open interest cap check skipped due to fetch failure', + { + action: 'info:perpsAtOpenInterestCap', + shard: shardLabel, + ...formatError(error) + } + ); } }; @@ -1255,18 +1883,24 @@ const subscriptions = createSubscriptions( await Promise.all( trackedAssets.map(async (asset) => { try { - const response = await postInfo< + const action = `info:fundingHistory:${asset}`; + const response = await requestInfo< | Array<{ time?: number | string; rate?: number | string; fundingRate?: number | string }> | { fundingRates?: Array<{ time?: number | string; rate?: number | string }>; history?: Array<{ time?: number | string; rate?: number | string }>; } - >({ - type: 'fundingHistory', - coin: asset, - startTime, - endTime: now - }); + >( + context, + action, + { + type: 'fundingHistory', + coin: asset, + startTime, + endTime: now + }, + { warnAfter: 2 } + ); const sourcePoints = Array.isArray(response) ? response @@ -1317,10 +1951,14 @@ const subscriptions = createSubscriptions( createEnvelope(metadata.provider, 'funding.history', payload, { asset }) ); } catch (error) { - context.logger.warn('Failed to fetch Hyperliquid funding history', { - asset, - ...formatError(error) - }); + context.logger.debug?.( + 'Hyperliquid funding history check skipped due to fetch failure', + { + asset, + action: 'info:fundingHistory', + ...formatError(error) + } + ); } }) ); @@ -1328,9 +1966,9 @@ const subscriptions = createSubscriptions( const checkPredictedFundings = async (context: IngestionContext) => { try { - const response = await postInfo<{ + const response = await requestInfo<{ predictedFundings?: Array<{ coin?: string; fundingRate?: number | string }>; - }>({ type: 'predictedFundings' }); + }>(context, 'info:predictedFundings', { type: 'predictedFundings' }, { warnAfter: 2 }); const entries = Array.isArray(response?.predictedFundings) ? response.predictedFundings : []; let processed = 0; for (const entry of entries) { @@ -1362,13 +2000,24 @@ const subscriptions = createSubscriptions( processed += 1; } } catch (error) { - context.logger.warn('Failed to fetch Hyperliquid predicted fundings', formatError(error)); + context.logger.debug?.( + 'Hyperliquid predicted fundings refresh skipped due to fetch failure', + { + action: 'info:predictedFundings', + ...formatError(error) + } + ); } }; const checkPerpDexs = async (context: IngestionContext) => { try { - const response = await postInfo>({ type: 'perpDexs' }); + const response = await requestInfo>( + context, + 'info:perpDexs:health', + { type: 'perpDexs' }, + { warnAfter: 2, logIntervalMs: 120_000 } + ); const seen = new Set(); for (const entry of response) { if (!entry || typeof entry.name !== 'string') continue; @@ -1377,7 +2026,7 @@ const subscriptions = createSubscriptions( const lower = normalised.toLowerCase(); seen.add(lower); if (!perpDexCache.has(lower)) { - context.logger.info('Discovered Hyperliquid perp dex', { dex: normalised }); + context.logger.info?.('Discovered Hyperliquid perp dex', { dex: normalised }); } } perpDexCache.clear(); @@ -1388,11 +2037,26 @@ const subscriptions = createSubscriptions( provider: metadata.provider }); } catch (error) { - context.logger.warn('Failed to fetch Hyperliquid perp dex list', formatError(error)); + context.logger.debug?.( + 'Hyperliquid skipped perp dex health refresh due to fetch failure', + { + action: 'info:perpDexs:health', + ...formatError(error) + } + ); } }; const pollRestEndpoints = async (context: IngestionContext) => { + if (!isControlShard) { + context.logger.debug?.( + 'Skipping Hyperliquid optional REST probes on non-control shard', + { + shard: shardLabel + } + ); + return; + } await checkAllMidsHealth(context); await checkPerpsAtOpenInterestCap(context); await checkFundingHistory(context); @@ -1404,12 +2068,16 @@ const subscriptions = createSubscriptions( metadata, async connect(context: IngestionContext) { runtimeContext = context; - context.logger.info('Hyperliquid adapter connected', { - wsUrl: config.wsUrl, - restUrl: infoEndpoint, - dex: config.dex, - maxReconnectAttempts: enforceReconnectCap ? maxReconnectAttempts : 'unlimited' - }); + context.logger.info( + 'Hyperliquid adapter connected', + { + wsUrl: config.wsUrl, + restUrl: infoEndpoint, + dex: config.dex, + maxReconnectAttempts: enforceReconnectCap ? maxReconnectAttempts : 'unlimited' + } + ); + logFeatureAvailability(context); context.updateHealth?.({ wsConnected: false, wsReconnectAttempts: 0, @@ -1428,7 +2096,7 @@ const subscriptions = createSubscriptions( metadataInterval = setInterval(() => { if (!runtimeContext) return; refreshAssetMetadata(runtimeContext).catch((error) => - runtimeContext?.logger.error('Failed to tick Hyperliquid metadata refresh', formatError(error)) + runtimeContext?.logger.error?.('Failed to tick Hyperliquid metadata refresh', formatError(error)) ); }, METADATA_REFRESH_INTERVAL_MS); metadataInterval.unref(); @@ -1461,7 +2129,7 @@ const subscriptions = createSubscriptions( socket = new WebSocket(config.wsUrl); socket.on('open', () => { - context.logger.info('Connected to Hyperliquid WebSocket'); + context.logger.info?.('Connected to Hyperliquid WebSocket', { shard: shardLabel }); reconnectAttempts = 0; lastPongReceived = Date.now(); connectionStableStartTime = Date.now(); @@ -1487,10 +2155,13 @@ const subscriptions = createSubscriptions( subscriptions.forEach((subscription) => { if (!subscription.subscribe) return; Promise.resolve(subscription.subscribe(socket!, context)).catch((error) => - context.logger.error('Failed to send Hyperliquid subscription', { - channel: subscription.channel, - ...formatError(error) - }) + context.logger.error?.( + 'Failed to send Hyperliquid subscription', + { + channel: subscription.channel, + ...formatError(error) + } + ) ); }); @@ -1498,45 +2169,12 @@ const subscriptions = createSubscriptions( ensureAllMidsSubscriptions(); }); - const logMessageProcessingError = (error: unknown) => { - context.metrics?.increment?.('hyperliquid_message_processing_errors_total', 1, { - provider: metadata.provider - }); - const now = Date.now(); - const errorString = JSON.stringify(formatError(error)); - - // Track consecutive errors for circuit breaker - if (errorString === lastErrorString) { - consecutiveErrorCount++; - } else { - consecutiveErrorCount = 1; - lastErrorString = errorString; - } - - // Signal fatal error if we have too many consecutive errors - if (consecutiveErrorCount >= 50) { // threshold used to keep alerting manageable - context.logger.warn('Repeated Hyperliquid message errors suppressed to keep the worker healthy', { - error: errorString, - consecutiveCount: consecutiveErrorCount - }); - consecutiveErrorCount = 0; - lastMessageErrorLogMs = now; - return; - } - - if (now - lastMessageErrorLogMs >= MESSAGE_ERROR_LOG_INTERVAL_MS) { - lastMessageErrorLogMs = now; - context.logger.error('Failed to process Hyperliquid message', { - ...formatError(error), - consecutiveCount: consecutiveErrorCount - }); - } else { - context.logger.debug?.('Suppressing repeated Hyperliquid message error', { - ...formatError(error), - consecutiveCount: consecutiveErrorCount - }); - } - }; + const logMessageProcessingError = createMessageErrorLogger({ + logger: context.logger, + metrics: context.metrics, + provider: metadata.provider, + logIntervalMs: MESSAGE_ERROR_LOG_INTERVAL_MS + }); socket.on('message', (raw) => { // Check if system is overloaded before processing message @@ -1550,11 +2188,14 @@ const subscriptions = createSubscriptions( subscribed: Object.values(coinStreams).reduce((total, stream) => total + stream.subscribed.size, 0) }; - context.logger.warn('Hyperliquid WebSocket closed', { - code, - subscriptionProgress, - timeSinceLastPong: lastPongReceived ? Date.now() - lastPongReceived : 'unknown' - }); + context.logger.warn?.( + 'Hyperliquid WebSocket closed', + { + code, + subscriptionProgress, + timeSinceLastPong: lastPongReceived ? Date.now() - lastPongReceived : 'unknown' + } + ); stopHeartbeat(); resetSubscriptionState(); if (!shuttingDown) { @@ -1565,10 +2206,13 @@ const subscriptions = createSubscriptions( wsReconnectAttempts: nextAttempts }); if (enforceReconnectCap && nextAttempts > maxReconnectAttempts) { - context.logger.error('Hyperliquid WebSocket reconnect attempts exceeded threshold', { - attempts: nextAttempts, - maxReconnectAttempts - }); + context.logger.error?.( + 'Hyperliquid WebSocket reconnect attempts exceeded threshold', + { + attempts: nextAttempts, + maxReconnectAttempts + } + ); context.signalFatalError?.('hyperliquid_ws_reconnect_exceeded', { attempts: nextAttempts, maxReconnectAttempts @@ -1587,25 +2231,34 @@ const subscriptions = createSubscriptions( // to force a slower resubscription process const connectionDuration = connectionStableStartTime ? Date.now() - connectionStableStartTime : 0; if (connectionDuration < 60000) { // Connection lasted less than 1 minute - context.logger.warn('Short-lived connection detected, clearing subscription cache', { - connectionDurationMs: connectionDuration, - attempts: reconnectAttempts - }); + context.logger.warn?.( + 'Short-lived connection detected, clearing subscription cache', + { + connectionDurationMs: connectionDuration, + attempts: reconnectAttempts + } + ); successfullySubscribedAssets.clear(); } if (reconnectAttempts >= MAX_RECONNECT_ATTEMPTS_BEFORE_WARN) { - context.logger.warn('Retrying Hyperliquid WebSocket connection with backoff', { - attempts: reconnectAttempts, - delay, - connectionDurationMs: connectionDuration - }); + context.logger.warn?.( + 'Retrying Hyperliquid WebSocket connection with backoff', + { + attempts: reconnectAttempts, + delay, + connectionDurationMs: connectionDuration + } + ); } else { - context.logger.info('Retrying Hyperliquid WebSocket connection', { - attempts: reconnectAttempts, - delay, - connectionDurationMs: connectionDuration - }); + context.logger.info?.( + 'Retrying Hyperliquid WebSocket connection', + { + attempts: reconnectAttempts, + delay, + connectionDurationMs: connectionDuration + } + ); } reconnectTimer = setTimeout(() => connectSocket(context), delay); reconnectTimer.unref(); @@ -1613,7 +2266,7 @@ const subscriptions = createSubscriptions( }); socket.on('error', (error) => { - context.logger.error('Hyperliquid WebSocket error', formatError(error)); + context.logger.error?.('Hyperliquid WebSocket error', formatError(error)); }); socket.on('pong', () => { @@ -1648,16 +2301,30 @@ const subscriptions = createSubscriptions( const subscription = subscriptionsByChannel.get(parsed.channel); if (!subscription) { - context.logger.debug?.( - `Ignoring Hyperliquid message for unhandled channel channel=${parsed.channel ?? 'unknown'}` - ); + const channelName = parsed.channel ?? 'unknown'; + if (channelName === 'error') { + return; + } + + const now = Date.now(); + const lastLoggedAt = unhandledChannelLogTimes.get(channelName) ?? 0; + if (now - lastLoggedAt >= UNHANDLED_CHANNEL_LOG_INTERVAL_MS) { + context.logger.debug?.( + 'Ignoring Hyperliquid message for unhandled channel', + { channel: channelName } + ); + unhandledChannelLogTimes.set(channelName, now); + } return; } if (!runtimeContext) { - context.logger.warn('Received Hyperliquid message before adapter context ready', { - channel: parsed.channel - }); + context.logger.warn?.( + 'Received Hyperliquid message before adapter context ready', + { + channel: parsed.channel + } + ); return; } @@ -1720,7 +2387,8 @@ const createSubscriptions = ( enabledStreams: Set, candleIntervals: Set, tradesEnabled: boolean, - bboEnabled: boolean + bboEnabled: boolean, + lagMonitor: HyperliquidLagMonitor ): HyperliquidSubscription[] => { const recordLagForStream = (stream: string) => @@ -1729,7 +2397,7 @@ const createSubscriptions = ( asset: string, timestamp?: number ): void => - observeMessageLag(context, stream, asset, timestamp); + lagMonitor(context, stream, asset, timestamp); const subscriptions: HyperliquidSubscription[] = []; @@ -1784,60 +2452,36 @@ const parseMessage = (raw: WebSocket.RawData, context: IngestionContext): WsMess try { return JSON.parse(raw.toString()) as WsMessage; } catch (error) { - context.logger.warn('Failed to parse Hyperliquid message', formatError(error)); + context.logger.warn?.('Failed to parse Hyperliquid message', formatError(error)); return null; } }; -const normaliseThrowable = (value: unknown): Record | undefined => { - if (value instanceof Error) { - const record: Record = { - name: value.name, - message: value.message - }; - if (typeof value.stack === 'string') { - record.stack = value.stack; - } - - const extras = value as unknown as { [key: string]: unknown; cause?: unknown }; - for (const key of ['code', 'status', 'errno', 'syscall', 'address', 'port', 'type']) { - const extraValue = extras[key]; - if (extraValue !== undefined) { - record[key] = extraValue; - } - } - - const { cause } = extras; - if (cause && cause !== value) { - const causeRecord = normaliseThrowable(cause); - if (causeRecord) { - record.cause = causeRecord; - } - } - - return record; - } - - if (typeof value === 'object' && value !== null) { - const entries = value as Record; - const record: Record = {}; - for (const [key, entryValue] of Object.entries(entries)) { - if (entryValue === undefined || typeof entryValue === 'function') continue; - record[key] = entryValue; - } - return Object.keys(record).length > 0 ? record : undefined; - } - - if (value !== undefined) { - return { message: String(value) }; +export const buildHyperliquidMessageErrorLogPayload = ( + errorDetails: Record, + consecutiveCount: number, + timeSinceLastLogMs: number +): Record => { + const errorSummary = typeof errorDetails.message === 'string' ? errorDetails.message : undefined; + const rawCode = errorDetails.code; + const errorCode = + typeof rawCode === 'string' + ? rawCode + : typeof rawCode === 'number' && Number.isFinite(rawCode) + ? String(rawCode) + : undefined; + const payload: Record = { + ...errorDetails, + consecutiveCount, + errorSummary, + errorCode + }; + if (Number.isFinite(timeSinceLastLogMs) && timeSinceLastLogMs >= 0) { + payload.timeSinceLastErrorLogMs = timeSinceLastLogMs; } - - return undefined; + return payload; }; -const formatError = (error: unknown): Record => - normaliseThrowable(error) ?? { message: String(error) }; - const computeMetadataSignature = (payload: NormalisedMetadataPayload): string => JSON.stringify(payload); const normaliseLogPayload = (payload: unknown): Record | undefined => { diff --git a/packages/ingestion-sdk/src/providers/hyperliquid/rest/diagnostics.ts b/packages/ingestion-sdk/src/providers/hyperliquid/rest/diagnostics.ts new file mode 100644 index 00000000..3fc05799 --- /dev/null +++ b/packages/ingestion-sdk/src/providers/hyperliquid/rest/diagnostics.ts @@ -0,0 +1,157 @@ +import type { IngestionContext } from '../../../adapters'; + +export type RestProbeActionOptions = { + warnAfter?: number; + logIntervalMs?: number; +}; + +export type RestProbeLoggerOptions = { + shardLabel: string; + provider: string; + formatError: (error: unknown) => Record; + durationMetricName?: string; + defaultWarnAfter?: number; + defaultLogIntervalMs?: number; +}; + +type RestProbeState = { + consecutiveFailures: number; + firstFailureAt?: number; + lastSuccessAt?: number; + lastLogAt?: number; +}; + +const REST_DURATION_MAX_MS = 30_000; + +const resolveLogger = ( + context: IngestionContext, + level: 'info' | 'warn' | 'error' +): ((payload: Record, message: string) => void) | undefined => { + const logger = context.logger as Record | undefined; + if (!logger) return undefined; + const candidate = logger[level]; + if (typeof candidate !== 'function') { + return undefined; + } + return candidate.bind(logger); +}; + +export type RestProbeLogger = { + wrap( + context: IngestionContext, + action: string, + callback: () => Promise, + options?: RestProbeActionOptions + ): Promise; + recordFailure( + context: IngestionContext, + action: string, + error: unknown, + options?: RestProbeActionOptions + ): void; + recordSuccess(context: IngestionContext, action: string): void; +}; + +export const createRestProbeLogger = ({ + shardLabel, + provider, + formatError, + durationMetricName = 'hyperliquid_rest_request_duration_ms', + defaultWarnAfter = 3, + defaultLogIntervalMs = 60_000 +}: RestProbeLoggerOptions): RestProbeLogger => { + const state = new Map(); + + const recordDuration = (context: IngestionContext, action: string, startedAt: number) => { + const elapsed = Date.now() - startedAt; + const bounded = Math.max(0, Math.min(elapsed, REST_DURATION_MAX_MS)); + context.metrics?.histogram?.(durationMetricName, bounded, { + provider, + action + }); + }; + + const recordSuccess = (context: IngestionContext, action: string) => { + const probe = state.get(action); + const now = Date.now(); + if (probe && probe.consecutiveFailures > 0) { + const logger = resolveLogger(context, 'info'); + if (logger) { + logger( + { + action, + shard: shardLabel, + recoveredAfterMs: probe.firstFailureAt ? now - probe.firstFailureAt : undefined, + failuresCleared: probe.consecutiveFailures + }, + `Hyperliquid ${action} request recovered` + ); + } + } + state.set(action, { + consecutiveFailures: 0, + lastSuccessAt: now + }); + }; + + const recordFailure = ( + context: IngestionContext, + action: string, + error: unknown, + options?: RestProbeActionOptions + ) => { + const warnAfter = options?.warnAfter ?? defaultWarnAfter; + const minLogIntervalMs = options?.logIntervalMs ?? defaultLogIntervalMs; + const now = Date.now(); + const probe = state.get(action) ?? { consecutiveFailures: 0 }; + probe.consecutiveFailures += 1; + probe.firstFailureAt = probe.firstFailureAt ?? now; + const lastSuccessAgo = probe.lastSuccessAt ? now - probe.lastSuccessAt : undefined; + const shouldLog = + probe.consecutiveFailures === 1 || + probe.consecutiveFailures === warnAfter || + (probe.lastLogAt ? now - probe.lastLogAt >= minLogIntervalMs : false); + + if (shouldLog) { + const level: 'info' | 'warn' | 'error' = probe.consecutiveFailures >= warnAfter ? 'error' : 'info'; + const logger = resolveLogger(context, level); + if (logger) { + logger( + { + action, + shard: shardLabel, + consecutiveFailures: probe.consecutiveFailures, + lastSuccessMsAgo: lastSuccessAgo, + failureWindowMs: now - probe.firstFailureAt, + ...formatError(error) + }, + `Hyperliquid ${action} request failed` + ); + probe.lastLogAt = now; + } + } + + state.set(action, probe); + }; + + return { + async wrap(context, action, callback, options) { + const startedAt = Date.now(); + try { + const result = await callback(); + recordDuration(context, action, startedAt); + recordSuccess(context, action); + return result; + } catch (error) { + recordDuration(context, action, startedAt); + recordFailure(context, action, error, options); + throw error; + } + }, + recordFailure, + recordSuccess + }; +}; + + + diff --git a/packages/ingestion-sdk/src/providers/hyperliquid/rest/info.ts b/packages/ingestion-sdk/src/providers/hyperliquid/rest/info.ts index 46848ae4..e09e1aaa 100644 --- a/packages/ingestion-sdk/src/providers/hyperliquid/rest/info.ts +++ b/packages/ingestion-sdk/src/providers/hyperliquid/rest/info.ts @@ -13,24 +13,137 @@ export type InfoRequestPayload = | { type: 'candleSnapshot'; coin: string; interval: string } | { type: 'historicalTrades'; coin: string; startTime?: number; endTime?: number; limit?: number }; +export type RestRequestOptions = { + timeoutMs?: number; + maxAttempts?: number; + initialBackoffMs?: number; + maxBackoffMs?: number; +}; + +export type RestRequestErrorDetails = { + endpoint: string; + payloadType?: string; + attempts: number; + status?: number; + cause?: unknown; +}; + +export class RestRequestError extends Error { + attempts: number; + endpoint: string; + payloadType?: string; + status?: number; + cause?: unknown; + + constructor(details: RestRequestErrorDetails & { message?: string }) { + super(details.message ?? `Hyperliquid info request failed after ${details.attempts} attempts`); + this.name = 'RestRequestError'; + this.attempts = details.attempts; + this.endpoint = details.endpoint; + this.payloadType = details.payloadType; + this.status = details.status; + this.cause = details.cause; + } +} + export const DEFAULT_INFO_ENDPOINT = 'https://api.hyperliquid.xyz/info'; +const DEFAULT_TIMEOUT_MS = 5_000; +const DEFAULT_MAX_ATTEMPTS = 3; +const DEFAULT_INITIAL_BACKOFF_MS = 250; +const DEFAULT_MAX_BACKOFF_MS = 1_500; + +const sleep = (ms: number): Promise => + new Promise((resolve) => { + setTimeout(resolve, ms); + }); + +const shouldRetryStatus = (status?: number): boolean => { + if (status === undefined) return true; + if (status >= 500) return true; + return status === 408 || status === 425 || status === 429; +}; + +const computeBackoff = (attempt: number, options?: RestRequestOptions): number => { + const base = + options?.initialBackoffMs ?? DEFAULT_INITIAL_BACKOFF_MS; + const maxCap = options?.maxBackoffMs ?? DEFAULT_MAX_BACKOFF_MS; + const capped = Math.min(base * 2 ** (attempt - 1), maxCap); + const jitter = Math.random() * (capped / 2); + return Math.floor(capped / 2 + jitter); +}; export const postHyperliquidInfo = async ( payload: InfoRequestPayload, - endpoint: string = DEFAULT_INFO_ENDPOINT + endpoint: string = DEFAULT_INFO_ENDPOINT, + options?: RestRequestOptions ): Promise => { const target = endpoint && endpoint.trim().length > 0 ? endpoint.trim() : DEFAULT_INFO_ENDPOINT; - const response = await fetch(target, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(payload) - }); + const maxAttempts = Math.max(1, options?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS); + const timeoutMs = options?.timeoutMs ?? DEFAULT_TIMEOUT_MS; + const payloadType = typeof (payload as Record)?.type === 'string' ? (payload as Record).type : undefined; + let attempt = 0; + let lastError: unknown; + + while (attempt < maxAttempts) { + attempt += 1; + const controller = new AbortController(); + const timeoutHandle = setTimeout(() => controller.abort(new Error('Hyperliquid info request timeout')), timeoutMs); + try { + const response = await fetch(target, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(payload), + signal: controller.signal + }); - if (!response.ok) { - throw new Error(`Hyperliquid info request failed with status ${response.status}`); + if (!response.ok) { + const status = response.status; + const error = new RestRequestError({ + endpoint: target, + payloadType, + attempts: attempt, + status, + cause: new Error(`Hyperliquid info request failed with status ${status}`) + }); + + if (!shouldRetryStatus(status) || attempt >= maxAttempts) { + throw error; + } + + lastError = error; + } else { + return (await response.json()) as TResponse; + } + } catch (error) { + lastError = error; + const shouldRetry = + attempt < maxAttempts && + (error instanceof RestRequestError ? shouldRetryStatus(error.status) : true); + + if (!shouldRetry) { + throw error instanceof RestRequestError + ? error + : new RestRequestError({ + endpoint: target, + payloadType, + attempts: attempt, + cause: error + }); + } + } finally { + clearTimeout(timeoutHandle); + } + + await sleep(computeBackoff(attempt, options)); } - return (await response.json()) as TResponse; + throw lastError instanceof RestRequestError + ? lastError + : new RestRequestError({ + endpoint: target, + payloadType, + attempts: maxAttempts, + cause: lastError + }); }; - diff --git a/packages/ingestion-sdk/src/providers/hyperliquid/subscriptions/activeAssetCtx.ts b/packages/ingestion-sdk/src/providers/hyperliquid/subscriptions/activeAssetCtx.ts index a3cf8e41..d6520c42 100644 --- a/packages/ingestion-sdk/src/providers/hyperliquid/subscriptions/activeAssetCtx.ts +++ b/packages/ingestion-sdk/src/providers/hyperliquid/subscriptions/activeAssetCtx.ts @@ -30,8 +30,8 @@ type CreateArgs = { }; export const createActiveAssetCtxSubscription = ({ - config, - metadata, + config: _config, + metadata: _metadata, shouldHandleAsset }: CreateArgs): HyperliquidSubscription => ({ channel: 'activeAssetCtx', @@ -56,13 +56,6 @@ export const createActiveAssetCtxSubscription = ({ if (mark !== undefined) { await context.publishPrice(asset, mark, message.timestamp ?? Date.now()); } - - context.runtimeContext.logger.debug?.('Updated Hyperliquid activeAssetCtx', { - provider: metadata.provider, - asset, - mark, - dex: config.dex - }); } }); diff --git a/packages/ingestion-sdk/src/providers/hyperliquid/subscriptions/activeAssetData.ts b/packages/ingestion-sdk/src/providers/hyperliquid/subscriptions/activeAssetData.ts index 3f87ba85..6027ead1 100644 --- a/packages/ingestion-sdk/src/providers/hyperliquid/subscriptions/activeAssetData.ts +++ b/packages/ingestion-sdk/src/providers/hyperliquid/subscriptions/activeAssetData.ts @@ -23,15 +23,14 @@ type CreateArgs = { }; export const createActiveAssetDataSubscription = ({ - config, - metadata, + config: _config, + metadata: _metadata, shouldHandleAsset }: CreateArgs): HyperliquidSubscription => ({ channel: 'activeAssetData', handle: async (payload: unknown, context: HyperliquidSubscriptionContext) => { const message = normaliseActiveAssetData(payload); if (!message) { - context.runtimeContext.logger.debug?.('Ignoring malformed Hyperliquid activeAssetData payload'); return; } @@ -49,13 +48,6 @@ export const createActiveAssetDataSubscription = ({ if (mark !== undefined) { await context.publishPrice(asset, mark, message.timestamp ?? Date.now()); } - - context.runtimeContext.logger.debug?.('Processed Hyperliquid activeAssetData update', { - provider: metadata.provider, - asset, - leverage: message.leverage, - dex: config.dex - }); } }); diff --git a/packages/ingestion-sdk/src/providers/hyperliquid/subscriptions/allMids.ts b/packages/ingestion-sdk/src/providers/hyperliquid/subscriptions/allMids.ts index ce85cfed..5afedea9 100644 --- a/packages/ingestion-sdk/src/providers/hyperliquid/subscriptions/allMids.ts +++ b/packages/ingestion-sdk/src/providers/hyperliquid/subscriptions/allMids.ts @@ -12,7 +12,7 @@ type CreateArgs = { }; export const createAllMidsSubscription = ({ - metadata, + metadata: _metadata, shouldHandleAsset }: CreateArgs): HyperliquidSubscription => ({ channel: 'allMids', @@ -38,11 +38,6 @@ export const createAllMidsSubscription = ({ published.add(canonical); await context.publishPrice(symbol, price, Date.now()); - context.runtimeContext.logger.debug?.('Published Hyperliquid mid price', { - provider: metadata.provider, - asset: canonical, - price - }); } } }); diff --git a/packages/ingestion-sdk/src/providers/hyperliquid/subscriptions/candle.ts b/packages/ingestion-sdk/src/providers/hyperliquid/subscriptions/candle.ts index f9504a8a..7165de92 100644 --- a/packages/ingestion-sdk/src/providers/hyperliquid/subscriptions/candle.ts +++ b/packages/ingestion-sdk/src/providers/hyperliquid/subscriptions/candle.ts @@ -262,9 +262,9 @@ export const createCandleSubscription = ({ if (candles.length === 0) { const snapshot = toObject(data); const keys = snapshot ? Object.keys(snapshot).slice(0, 6).join(',') : 'unknown'; - context.runtimeContext.logger.debug?.( - `Hyperliquid candle payload produced no entries intervals=${Array.from(allowedIntervalsSet).join(',')} keys=${keys}` - ); + context.runtimeContext.logger.debug?.('Hyperliquid candle payload produced no parsable entries', { + keys + }); return; } @@ -272,10 +272,6 @@ export const createCandleSubscription = ({ if (!shouldHandleAsset(candle.asset)) continue; if (!allowedIntervalsSet.has(candle.interval)) continue; - context.runtimeContext.logger.debug?.( - `Publishing Hyperliquid candle asset=${candle.asset} interval=${candle.interval} start=${candle.payload.startTime} end=${candle.payload.endTime}` - ); - await context.publishCandle(candle.asset, candle.interval, candle.payload); const endTimestamp = candle.payload.endTime ?? candle.payload.startTime; recordLag(context, candle.asset, endTimestamp); diff --git a/packages/ingestion-sdk/src/providers/hyperliquid/subscriptions/trades.ts b/packages/ingestion-sdk/src/providers/hyperliquid/subscriptions/trades.ts index c662db62..a3df7aa3 100644 --- a/packages/ingestion-sdk/src/providers/hyperliquid/subscriptions/trades.ts +++ b/packages/ingestion-sdk/src/providers/hyperliquid/subscriptions/trades.ts @@ -213,7 +213,6 @@ export const createTradesSubscription = ({ channel: 'trades', handle: async (data: unknown, context: HyperliquidSubscriptionContext) => { if (!context.publishTrade) { - context.runtimeContext.logger.debug?.('Hyperliquid trades handler: publishTrade not available'); return; } @@ -221,26 +220,16 @@ export const createTradesSubscription = ({ const isObject = data !== null && typeof data === 'object'; const dataKeys = isObject ? Object.keys(data).slice(0, 10) : undefined; - if (context.runtimeContext.logger.debug) { - const dataPreview = previewPayload(data); - context.runtimeContext.logger.debug('Hyperliquid trades handler: received payload', { - dataType, - dataKeys, - dataPreview - }); - } - const trades = normaliseTrades(data); if (trades.length === 0) { - context.runtimeContext.logger.debug?.('Hyperliquid trades handler: normaliseTrades returned empty array', { + context.runtimeContext.logger.debug?.('Hyperliquid trades payload produced no parsable entries', { dataType, - dataKeys + keys: dataKeys, + preview: previewPayload(data) }); return; } - context.runtimeContext.logger.debug?.(`Hyperliquid trades handler: received ${trades.length} trades`); - for (const trade of trades) { if (!shouldHandleAsset(trade.asset)) { continue; diff --git a/packages/ingestion-sdk/src/runtime/worker.ts b/packages/ingestion-sdk/src/runtime/worker.ts index 5d5dbcef..97679241 100644 --- a/packages/ingestion-sdk/src/runtime/worker.ts +++ b/packages/ingestion-sdk/src/runtime/worker.ts @@ -107,6 +107,25 @@ const overloadStateGauge = new Gauge({ registers: [] }); +const publishThrottleHistogram = new Histogram({ + name: 'ingestion_worker_publish_throttle_duration_ms', + help: 'Time spent waiting for pending publish capacity before sending to Kafka', + buckets: [5, 10, 25, 50, 100, 250, 500, 1000, 2000], + registers: [] +}); + +const kafkaOverloadEventsCounter = new Counter({ + name: 'ingestion_worker_kafka_overload_events_total', + help: 'Total number of Kafka overload or timeout events observed', + registers: [] +}); + +const kafkaOverloadSuppressedCounter = new Counter({ + name: 'ingestion_worker_kafka_overload_logs_suppressed_total', + help: 'Kafka overload log entries that were suppressed to reduce noise', + registers: [] +}); + const DEFAULT_EVENT_LOOP_LAG_P95_THRESHOLD_MS = 750; const DEFAULT_PENDING_PUBLISH_THRESHOLD = 50; // Reduced from 120 for earlier detection const DEFAULT_WS_RECONNECT_THRESHOLD = 10; @@ -115,6 +134,13 @@ const DEFAULT_OVERLOAD_WARMUP_MS = 15_000; const OVERLOAD_LOG_CONSECUTIVE_THRESHOLD = 3; const OVERLOAD_CLEAR_CONSECUTIVE_THRESHOLD = 3; const DEFAULT_OVERLOAD_HEALTH_CONSECUTIVE_THRESHOLD = 6; +const DEFAULT_MAX_PENDING_PUBLISHES = 1000; +const DEFAULT_PUBLISH_THROTTLE_RATIO = 0.85; +const DEFAULT_PUBLISH_THROTTLE_DELAY_MS = 25; +const DEFAULT_PUBLISH_THROTTLE_MAX_WAIT_MS = 500; +const DEFAULT_KAFKA_OVERLOAD_BACKOFF_MS = 100; +const DEFAULT_KAFKA_OVERLOAD_MAX_RETRIES = 3; +const DEFAULT_KAFKA_OVERLOAD_LOG_INTERVAL_MS = 5000; const parseNumericEnv = (key: string, fallback: number): number => { const raw = process.env[key]; @@ -126,6 +152,16 @@ const parseNumericEnv = (key: string, fallback: number): number => { return value; }; +const parseRatioEnv = (key: string, fallback: number): number => { + const raw = process.env[key]; + if (!raw) return fallback; + const value = Number(raw); + if (!Number.isFinite(value) || value <= 0 || value > 1) { + return fallback; + } + return value; +}; + const EVENT_LOOP_LAG_P95_THRESHOLD_MS = parseNumericEnv( 'INGEST_OVERLOAD_EVENT_LOOP_P95_MS', DEFAULT_EVENT_LOOP_LAG_P95_THRESHOLD_MS @@ -138,6 +174,74 @@ const WS_RECONNECT_THRESHOLD = parseNumericEnv( 'INGEST_OVERLOAD_WS_RECONNECT_THRESHOLD', DEFAULT_WS_RECONNECT_THRESHOLD ); +const MAX_PENDING_PUBLISHES = parseNumericEnv('INGEST_MAX_PENDING_PUBLISHES', DEFAULT_MAX_PENDING_PUBLISHES); +const PUBLISH_THROTTLE_RATIO = parseRatioEnv('INGEST_PUBLISH_THROTTLE_RATIO', DEFAULT_PUBLISH_THROTTLE_RATIO); +const PUBLISH_THROTTLE_THRESHOLD = Math.max(1, Math.floor(MAX_PENDING_PUBLISHES * PUBLISH_THROTTLE_RATIO)); +const PUBLISH_THROTTLE_DELAY_MS = parseNumericEnv( + 'INGEST_PUBLISH_THROTTLE_DELAY_MS', + DEFAULT_PUBLISH_THROTTLE_DELAY_MS +); +const PUBLISH_THROTTLE_MAX_WAIT_MS = parseNumericEnv( + 'INGEST_PUBLISH_THROTTLE_MAX_WAIT_MS', + DEFAULT_PUBLISH_THROTTLE_MAX_WAIT_MS +); +const KAFKA_OVERLOAD_BACKOFF_MS = parseNumericEnv( + 'INGEST_KAFKA_OVERLOAD_BACKOFF_MS', + DEFAULT_KAFKA_OVERLOAD_BACKOFF_MS +); +const KAFKA_OVERLOAD_MAX_RETRIES = Math.max( + 0, + Math.floor(parseNumericEnv('INGEST_KAFKA_OVERLOAD_MAX_RETRIES', DEFAULT_KAFKA_OVERLOAD_MAX_RETRIES)) +); +const KAFKA_OVERLOAD_LOG_INTERVAL_MS = parseNumericEnv( + 'INGEST_KAFKA_OVERLOAD_LOG_INTERVAL_MS', + DEFAULT_KAFKA_OVERLOAD_LOG_INTERVAL_MS +); + +const sleep = (ms: number): Promise => + ms <= 0 ? Promise.resolve() : new Promise((resolve) => setTimeout(resolve, ms)); + +type KafkaErrorClassification = { + errorType: string; + errorMessage: string; + isTimeout: boolean; + isConnectionError: boolean; + isOverloadError: boolean; +}; + +const classifyKafkaError = (error: unknown): KafkaErrorClassification => { + const errorMessage = error instanceof Error ? error.message : String(error); + const explicitType = (error as { type?: string })?.type; + const inferredType = + error instanceof Error + ? error.constructor.name + : typeof (error as { name?: string })?.name === 'string' + ? String((error as { name?: string }).name) + : 'UnknownError'; + + const errorType = explicitType ?? inferredType; + const loweredMessage = errorMessage.toLowerCase(); + const isTimeout = + loweredMessage.includes('timeout') || + loweredMessage.includes('request_timeout') || + errorType === 'KafkaJSTimeout' || + explicitType === 'REQUEST_TIMEOUT'; + const isConnectionError = + /connection|econnrefused|enotfound|network/i.test(errorMessage) || + errorType === 'KafkaJSConnectionError'; + const isOverloadError = + loweredMessage.includes('timeout while acquiring lock') || + loweredMessage.includes('backpressure') || + errorType === 'KafkaJSTimeout'; + + return { + errorType, + errorMessage, + isTimeout, + isConnectionError, + isOverloadError + }; +}; const OVERLOAD_EVALUATION_INTERVAL_MS = Math.max( 1000, parseNumericEnv('INGEST_OVERLOAD_EVALUATION_INTERVAL_MS', DEFAULT_OVERLOAD_EVALUATION_INTERVAL_MS) @@ -150,6 +254,7 @@ const OVERLOAD_HEALTH_CONSECUTIVE_THRESHOLD = Math.max( export const startIngestionWorker = async (deps: WorkerDeps) => { const registry = deps.registry ?? new Registry(); + const providerLabel = String(deps.runtimeConfig.provider ?? 'unknown'); if (!deps.registry) { collectDefaultMetrics({ register: registry }); } @@ -162,6 +267,9 @@ export const startIngestionWorker = async (deps: WorkerDeps) => { registry.registerMetric(eventLoopLagP95Gauge); registry.registerMetric(pendingPublishesGauge); registry.registerMetric(overloadStateGauge); + registry.registerMetric(publishThrottleHistogram); + registry.registerMetric(kafkaOverloadEventsCounter); + registry.registerMetric(kafkaOverloadSuppressedCounter); const logger = deps.logger; const kafkaSendCounter = new Counter({ @@ -246,12 +354,33 @@ export const startIngestionWorker = async (deps: WorkerDeps) => { let consecutiveOverloadSamples = 0; let consecutiveHealthySamples = 0; const warmupDeadlineMs = Date.now() + OVERLOAD_WARMUP_MS; + const kafkaOverloadLogState = { + suppressedEvents: 0, + lastEmitMs: 0 + }; const recordPendingPublishes = (value: number) => { pendingPublishes = value < 0 ? 0 : value; pendingPublishesGauge.set(pendingPublishes); }; + const logKafkaOverload = (meta: Record) => { + kafkaOverloadEventsCounter.inc(); + const now = Date.now(); + if (now - kafkaOverloadLogState.lastEmitMs >= KAFKA_OVERLOAD_LOG_INTERVAL_MS) { + const enrichedMeta = + kafkaOverloadLogState.suppressedEvents > 0 + ? { ...meta, suppressedEvents: kafkaOverloadLogState.suppressedEvents } + : meta; + logger.warn('Kafka producer overload detected', enrichedMeta); + kafkaOverloadLogState.lastEmitMs = now; + kafkaOverloadLogState.suppressedEvents = 0; + return; + } + kafkaOverloadLogState.suppressedEvents += 1; + kafkaOverloadSuppressedCounter.inc(); + }; + const sampleEventLoopLagP95Ms = (): number => { const percentile = eventLoopDelay.percentile(95); eventLoopDelay.reset(); @@ -262,6 +391,27 @@ export const startIngestionWorker = async (deps: WorkerDeps) => { return Number.isFinite(milliseconds) ? milliseconds : 0; }; + const waitForPublishCapacity = async (): Promise => { + if (pendingPublishes < PUBLISH_THROTTLE_THRESHOLD) { + return 0; + } + + const start = Date.now(); + while ( + pendingPublishes >= PUBLISH_THROTTLE_THRESHOLD && + Date.now() - start < PUBLISH_THROTTLE_MAX_WAIT_MS + ) { + await sleep(PUBLISH_THROTTLE_DELAY_MS); + } + + const waitedMs = Date.now() - start; + if (waitedMs > 0) { + publishThrottleHistogram.observe(waitedMs); + } + + return waitedMs; + }; + const assessOverload = ({ sampleEventLoop = true }: { sampleEventLoop?: boolean } = {}) => { if (sampleEventLoop) { lastEventLoopLagP95Ms = Number(sampleEventLoopLagP95Ms().toFixed(2)); @@ -411,73 +561,103 @@ export const startIngestionWorker = async (deps: WorkerDeps) => { updateHealth, signalFatalError, publish: async (topic, payload, options = {}) => { - // Implement backpressure: if too many pending publishes, reject immediately - const maxPendingPublishes = parseNumericEnv('INGEST_MAX_PENDING_PUBLISHES', 1000); - if (pendingPublishes > maxPendingPublishes) { // Prevent producer lock queue buildup - const error = new Error(`Backpressure: too many pending publishes (${pendingPublishes})`); + const throttleWaitMs = await waitForPublishCapacity(); + if (throttleWaitMs > 0) { + context.metrics?.increment?.('ingestion_worker_publish_throttle_total', 1, { + provider: providerLabel + }); + } + + if (pendingPublishes >= MAX_PENDING_PUBLISHES) { + const error = new Error( + `Backpressure: pending publishes (${pendingPublishes}) exceeded limit ${MAX_PENDING_PUBLISHES}` + ); kafkaPublishErrorCounter.inc({ error_type: 'backpressure' }); + context.metrics?.increment?.('ingestion_worker_publish_backpressure_total', 1, { + provider: providerLabel + }); throw error; } - + const serialised = payload instanceof Buffer ? payload - : Buffer.from(JSON.stringify(payload instanceof Object ? payload : createEnvelope('unknown', 'raw', payload))); + : Buffer.from( + JSON.stringify( + payload instanceof Object ? payload : createEnvelope('unknown', 'raw', payload) + ) + ); const messageKey = typeof options.key === 'string' ? options.key : undefined; + + const sendWithRetry = async () => { + let attempt = 0; + // eslint-disable-next-line no-constant-condition + while (true) { + try { + await producer.send({ topic, messages: [{ value: serialised, key: messageKey }] }); + return; + } catch (error) { + const classification = classifyKafkaError(error); + const shouldRetry = + attempt < KAFKA_OVERLOAD_MAX_RETRIES && + (classification.isTimeout || classification.isOverloadError); + if (!shouldRetry) { + throw error; + } + + const backoffBase = KAFKA_OVERLOAD_BACKOFF_MS * Math.max(1, 2 ** attempt); + const jitter = Math.floor(Math.random() * Math.max(1, KAFKA_OVERLOAD_BACKOFF_MS)); + const delayMs = backoffBase + jitter; + context.metrics?.increment?.('ingestion_worker_kafka_retry_total', 1, { + provider: providerLabel, + topic, + error_type: classification.isTimeout ? 'timeout' : 'overload' + }); + logger.warn('Retrying Kafka publish after transient overload', { + topic, + attempt: attempt + 1, + delayMs, + error: classification.errorMessage + }); + await sleep(delayMs); + attempt += 1; + } + } + }; + recordPendingPublishes(pendingPublishes + 1); try { - await producer.send({ topic, messages: [{ value: serialised, key: messageKey }] }); - // Only increment success counters on successful send + await sendWithRetry(); publishedEventCounter.inc(); kafkaSendCounter.inc(); - // Reset consecutive error counter on success consecutiveKafkaErrors = 0; } catch (error) { - // Determine error type for metrics - const errorType = error instanceof Error ? error.constructor.name : 'UnknownError'; - const errorMessage = error instanceof Error ? error.message : String(error); - - // Check if it's a timeout error - const isTimeout = errorMessage.includes('timeout') || - errorMessage.includes('REQUEST_TIMEOUT') || - errorType === 'KafkaJSTimeout' || - (error as { type?: string }).type === 'REQUEST_TIMEOUT'; - - // Check for different types of errors that should trigger different responses - const isConnectionError = errorMessage.includes('Connection') || - errorMessage.includes('ECONNREFUSED') || - errorMessage.includes('ENOTFOUND') || - errorMessage.includes('Network') || - errorType === 'KafkaJSConnectionError'; - - const isOverloadError = errorMessage.includes('Timeout while acquiring lock') || - errorMessage.includes('Backpressure') || - errorType === 'KafkaJSTimeout'; - - // Track consecutive errors for circuit breaker + const classification = classifyKafkaError(error); + const { errorType, errorMessage, isTimeout, isConnectionError, isOverloadError } = + classification; + const now = Date.now(); - if (now - lastKafkaErrorTime < 60000) { // Within 1 minute + if (now - lastKafkaErrorTime < 60000) { consecutiveKafkaErrors++; } else { consecutiveKafkaErrors = 1; } lastKafkaErrorTime = now; - - // Circuit breaker: signal fatal error after too many consecutive failures - if (consecutiveKafkaErrors >= 5) { // Reduced from 10 to 5 for faster response + + if (consecutiveKafkaErrors >= 5) { context.signalFatalError?.('kafka_circuit_breaker', { consecutiveErrors: consecutiveKafkaErrors, - errorType: isTimeout ? 'timeout' : (isConnectionError ? 'connection' : errorType), + errorType: isTimeout ? 'timeout' : isConnectionError ? 'connection' : errorType, topic }); } - - // Increment error counter - kafkaPublishErrorCounter.inc({ error_type: isTimeout ? 'timeout' : errorType }); - - // Log error with appropriate level and handle different error types + + kafkaPublishErrorCounter.inc({ + error_type: isOverloadError ? 'overload' : isTimeout ? 'timeout' : errorType + }); + if (isTimeout || isOverloadError) { - logger.warn('Kafka producer overload detected', { + logKafkaOverload({ topic, error: errorMessage, errorType, @@ -491,8 +671,7 @@ export const startIngestionWorker = async (deps: WorkerDeps) => { errorType, pendingPublishes }); - // Signal potential fatal error for connection issues - if (Math.random() < 0.1) { // Sample 10% of connection errors to avoid spam + if (Math.random() < 0.1) { context.signalFatalError?.('kafka_connection_failure', { error: errorMessage, errorType, @@ -507,16 +686,19 @@ export const startIngestionWorker = async (deps: WorkerDeps) => { pendingPublishes }); } - - const providerLabel = (deps.runtimeConfig.provider ?? 'unknown') as string; + context.metrics?.increment?.('hyperliquid_kafka_publish_errors_total', 1, { provider: providerLabel, topic, - error_type: isOverloadError ? 'overload' : (isTimeout ? 'timeout' : (isConnectionError ? 'connection' : errorType)) + error_type: isOverloadError + ? 'overload' + : isTimeout + ? 'timeout' + : isConnectionError + ? 'connection' + : errorType }); - - // Re-throw the error so callers can handle it if needed - // This prevents silent failures but allows adapters to implement their own retry logic + throw error; } finally { recordPendingPublishes(pendingPublishes - 1); diff --git a/packages/ingestion-sdk/tests/hyperliquid.adapter.test.ts b/packages/ingestion-sdk/tests/hyperliquid.adapter.test.ts new file mode 100644 index 00000000..ece35961 --- /dev/null +++ b/packages/ingestion-sdk/tests/hyperliquid.adapter.test.ts @@ -0,0 +1,551 @@ +import { afterEach, beforeEach, describe, expect, test, vi, type Mock } from 'vitest'; + +import { + buildHyperliquidMessageErrorLogPayload, + createHyperliquidAdapter, + createHyperliquidLagMonitor, + createMessageErrorLogger, + publishWithRetry, + type HyperliquidAdapterOptions, + type HyperliquidSubscriptionContext +} from '../src/providers/hyperliquid'; +import { createRestProbeLogger } from '../src/providers/hyperliquid/rest/diagnostics'; +import type { IngestionContext } from '../src/adapters'; + +type MockSocket = { + simulateMessage: (payload: unknown) => void; +}; + +declare global { + // eslint-disable-next-line no-var + var __hyperliquidMockSockets: MockSocket[]; +} + +const getMockSockets = (): MockSocket[] => globalThis.__hyperliquidMockSockets ?? []; + +const { postInfoMock, defaultPostInfoImpl } = vi.hoisted(() => { + const defaultMetaResponse = () => [ + { + universe: [ + { + name: 'BTC-PERP' + } + ] + }, + [ + { + markPx: '100', + prevDayPx: '90', + time: Date.now() + } + ] + ] as const; + + const defaultPostInfoImpl = async (payload: Record) => { + switch (payload?.type) { + case 'metaAndAssetCtxs': + return defaultMetaResponse(); + case 'perpDexs': + return []; + case 'allMids': + return { mids: [] }; + case 'perpsAtOpenInterestCap': + return []; + case 'fundingHistory': + return []; + case 'predictedFundings': + return { predictedFundings: [] }; + default: + return []; + } + }; + + return { + postInfoMock: vi.fn(defaultPostInfoImpl), + defaultPostInfoImpl + }; +}); + +vi.mock('ws', () => { + const sockets: MockSocket[] = []; + globalThis.__hyperliquidMockSockets = sockets; + class MockWebSocket { + static CONNECTING = 0; + static OPEN = 1; + static CLOSING = 2; + static CLOSED = 3; + readyState = MockWebSocket.CONNECTING; + private listeners: Record void>> = {}; + + constructor() { + sockets.push(this); + setImmediate(() => { + this.readyState = MockWebSocket.OPEN; + this.emit('open'); + }); + } + + on(event: string, handler: (...args: unknown[]) => void) { + if (!this.listeners[event]) { + this.listeners[event] = []; + } + this.listeners[event]?.push(handler); + return this; + } + + send() { + // no-op + } + + ping() { + // no-op + } + + terminate() { + this.close(1000, 'terminated'); + } + + close(code?: number, reason?: string) { + this.readyState = MockWebSocket.CLOSED; + this.emit('close', code ?? 1000, reason); + } + + private emit(event: string, ...args: unknown[]) { + const handlers = this.listeners[event]; + if (!handlers) return; + handlers.forEach((handler) => handler(...args)); + } + + simulateMessage(payload: unknown) { + const raw = typeof payload === 'string' ? payload : JSON.stringify(payload); + this.emit('message', Buffer.from(raw)); + } + } + + return { + default: MockWebSocket + }; +}); + +vi.mock('../src/providers/hyperliquid/rest/info', () => ({ + DEFAULT_INFO_ENDPOINT: 'https://hyperliquid.test', + postHyperliquidInfo: postInfoMock +})); + +const baseConfig: HyperliquidAdapterOptions = { + kind: 'hyperliquid', + provider: 'hyperliquid', + providerLabel: 'Hyperliquid', + providerMetadata: { + topicPrefix: 'hyperliquid', + redisNamespace: 'pmon:hyperliquid:markets' + }, + shard: { + id: 'test', + index: 0, + count: 1 + }, + streams: ['bbo'], + topics: { + prices: 'hyperliquid.prices', + orderbook: 'hyperliquid.orderbook', + assetMetadata: 'hyperliquid.assetMetadata', + overview: 'hyperliquid.overview', + candles: 'hyperliquid.candles', + trades: 'hyperliquid.trades', + bbo: 'hyperliquid.bbo', + funding: 'hyperliquid.funding' + }, + wsUrl: 'wss://example.test', + restUrl: 'https://example.test/info', + assets: ['BTC-PERP'], + options: {} +}; + +const mergeConfig = (overrides?: Partial): HyperliquidAdapterOptions => ({ + ...baseConfig, + ...(overrides ?? {}), + topics: { + ...baseConfig.topics, + ...(overrides?.topics ?? {}) + }, + streams: overrides?.streams ?? baseConfig.streams +}); + +const createMockContext = ( + overrides?: Partial> +): IngestionContext & { logger: Required; publish: ReturnType } => { + const logger = { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + debug: vi.fn() + }; + const publish = + overrides?.publish ?? + vi.fn(async (_topic: string, _payload: Buffer | string | object, _options?: { key?: string }) => { + // default resolve + }); + + return { + logger, + publish, + metrics: { + increment: vi.fn(), + histogram: vi.fn() + }, + config: {}, + updateHealth: vi.fn(), + signalFatalError: vi.fn() + }; +}; + +describe('createHyperliquidAdapter feature flags', () => { + beforeEach(() => { + postInfoMock.mockImplementation(defaultPostInfoImpl); + postInfoMock.mockClear(); + getMockSockets().length = 0; + }); + + test('skips asset metadata and overview publishing when topics are disabled', async () => { + const adapter = createHyperliquidAdapter( + mergeConfig({ + topics: { + assetMetadata: '', + overview: '', + bbo: 'hyperliquid.bbo' + } + }) + ); + + const context = createMockContext(); + await adapter.connect(context); + try { + await adapter.start(); + } finally { + await adapter.stop(); + } + + const publishedTopics = context.publish.mock.calls.map(([topic]) => topic); + expect(publishedTopics).not.toContain('hyperliquid.assetMetadata'); + expect(publishedTopics).not.toContain('hyperliquid.overview'); + + expect(context.logger.info).toHaveBeenCalledWith( + 'Hyperliquid asset metadata publishing disabled for shard', + expect.objectContaining({ reason: 'assetMetadataTopicMissing' }) + ); + expect(context.logger.info).toHaveBeenCalledWith( + 'Hyperliquid overview publishing disabled for shard', + expect.objectContaining({ reason: 'overviewTopicMissing' }) + ); + }); + + test('non-control shards skip optional REST probes and overview publishing', async () => { + const adapter = createHyperliquidAdapter( + mergeConfig({ + shard: { + id: 'test-shard', + index: 1, + count: 2 + } + }) + ); + + const context = createMockContext(); + await adapter.connect(context); + try { + await adapter.start(); + } finally { + await adapter.stop(); + } + + const restCallTypes = postInfoMock.mock.calls.map(([payload]) => payload.type); + expect(restCallTypes).not.toContain('allMids'); + expect(restCallTypes).not.toContain('perpsAtOpenInterestCap'); + expect(restCallTypes).not.toContain('predictedFundings'); + + const overviewPublishes = context.publish.mock.calls + .map(([topic]) => topic) + .filter((topic) => topic === 'hyperliquid.overview'); + expect(overviewPublishes).toHaveLength(0); + }); +}); + +describe('publishWithRetry helper', () => { + test('retries retryable errors before surfacing failure', async () => { + const publish = vi + .fn() + .mockRejectedValueOnce(Object.assign(new Error('Timeout while acquiring lock'), { type: 'REQUEST_TIMEOUT' })) + .mockResolvedValueOnce(undefined); + const context = createMockContext({ publish }); + + await publishWithRetry({ + runtimeContext: context, + topic: 'hyperliquid.prices', + envelope: { payload: true } as any, + key: 'BTC-PERP', + logContext: { asset: 'BTC-PERP' } + }); + + expect(publish).toHaveBeenCalledTimes(2); + }); + + test('throws immediately when error is not retryable', async () => { + const publish = vi.fn().mockRejectedValue(new Error('boom')); + const context = createMockContext({ publish }); + + await expect( + publishWithRetry({ + runtimeContext: context, + topic: 'hyperliquid.prices', + envelope: { payload: true } as any, + key: 'BTC-PERP' + }) + ).rejects.toThrow('boom'); + expect(publish).toHaveBeenCalledTimes(1); + }); +}); + +describe('createMessageErrorLogger', () => { + test('suppresses noisy repetitions and emits summary once interval passes', () => { + vi.useFakeTimers(); + const logger = { + error: vi.fn(), + warn: vi.fn() + }; + const metrics = { + increment: vi.fn(), + histogram: vi.fn() + }; + + const logError = createMessageErrorLogger({ + logger, + metrics, + provider: 'hyperliquid', + logIntervalMs: 1_000 + }); + + vi.setSystemTime(5_000); + const repeatedError = new Error('boom'); + repeatedError.stack = 'shared-stack'; + logError(repeatedError); + expect(logger.error).toHaveBeenCalledTimes(1); + expect(metrics.increment).toHaveBeenCalledWith('hyperliquid_message_processing_errors_total', 1, { + provider: 'hyperliquid' + }); + + logger.error.mockClear(); + logError(repeatedError); + logError(repeatedError); + expect(logger.error).not.toHaveBeenCalled(); + expect(logger.warn).not.toHaveBeenCalled(); + + vi.advanceTimersByTime(1_100); + logError(repeatedError); + expect(logger.warn).toHaveBeenCalledWith( + 'Burst of Hyperliquid message processing errors', + expect.objectContaining({ suppressedSinceLastLog: 2 }) + ); + vi.useRealTimers(); + }); +}); + +describe('createRestProbeLogger', () => { + test('escalates severity after repeated failures and logs recovery', async () => { + const context = createMockContext(); + const tracker = createRestProbeLogger({ + shardLabel: '1/1', + provider: 'hyperliquid', + formatError: (error) => (error instanceof Error ? { message: error.message } : { message: String(error) }) + }); + + await expect( + tracker.wrap(context, 'info:test-action', async () => { + throw new Error('boom'); + }) + ).rejects.toThrow('boom'); + + expect(context.logger.info).toHaveBeenCalledWith( + expect.objectContaining({ action: 'info:test-action', consecutiveFailures: 1 }), + 'Hyperliquid info:test-action request failed' + ); + context.logger.info.mockClear(); + + await expect( + tracker.wrap(context, 'info:test-action', async () => { + throw new Error('boom'); + }) + ).rejects.toThrow('boom'); + + await expect( + tracker.wrap(context, 'info:test-action', async () => { + throw new Error('boom'); + }) + ).rejects.toThrow('boom'); + + expect(context.logger.error).toHaveBeenCalledWith( + expect.objectContaining({ action: 'info:test-action', consecutiveFailures: 3 }), + 'Hyperliquid info:test-action request failed' + ); + context.logger.error.mockClear(); + + await tracker.wrap(context, 'info:test-action', async () => 'ok'); + expect(context.logger.info).toHaveBeenCalledWith( + expect.objectContaining({ action: 'info:test-action', failuresCleared: 3 }), + 'Hyperliquid info:test-action request recovered' + ); + }); +}); + +describe('buildHyperliquidMessageErrorLogPayload', () => { + test('captures error summary, code, and window duration', () => { + const payload = buildHyperliquidMessageErrorLogPayload( + { message: 'Kafka send failed', code: 'BROKER_TIMEOUT' }, + 7, + 1500 + ); + expect(payload).toMatchObject({ + errorSummary: 'Kafka send failed', + errorCode: 'BROKER_TIMEOUT', + consecutiveCount: 7, + timeSinceLastErrorLogMs: 1500 + }); + }); + + test('stringifies numeric codes and omits invalid windows', () => { + const payload = buildHyperliquidMessageErrorLogPayload({ message: 'boom', code: 400 }, 2, Number.NaN); + expect(payload.errorCode).toBe('400'); + expect(payload.timeSinceLastErrorLogMs).toBeUndefined(); + expect(payload.consecutiveCount).toBe(2); + }); +}); + +describe('createHyperliquidLagMonitor', () => { + const lagEnvKeys = [ + 'INGEST_HL_LAG_BOOTSTRAP_MS', + 'INGEST_HL_LAG_ALERT_BREACHES', + 'INGEST_HL_LAG_ALERT_INTERVAL_MS', + 'INGEST_HL_LAG_FATAL_MS', + 'INGEST_HL_LAG_STATE_TTL_MS' + ]; + + const setLagEnv = (overrides?: Partial>) => { + const defaults: Record = { + INGEST_HL_LAG_BOOTSTRAP_MS: '1000', + INGEST_HL_LAG_ALERT_BREACHES: '2', + INGEST_HL_LAG_ALERT_INTERVAL_MS: '5000', + INGEST_HL_LAG_FATAL_MS: '12000', + INGEST_HL_LAG_STATE_TTL_MS: '60000' + }; + const next = { ...defaults, ...(overrides ?? {}) }; + for (const [key, value] of Object.entries(next)) { + process.env[key] = value; + } + }; + + const clearLagEnv = () => { + for (const key of lagEnvKeys) { + delete process.env[key]; + } + }; + + const createLagContext = (): HyperliquidSubscriptionContext => { + const runtimeContext = createMockContext(); + return { + config: baseConfig as any, + metadata: { + id: 'lag-test', + provider: 'hyperliquid', + supportedDomains: [] + }, + runtimeContext, + subscribeOrderbook: vi.fn(), + resolveAsset: (id: string) => id, + upsertAssetContext: vi.fn(), + publishPrice: vi.fn(), + publishTrade: vi.fn(), + publishBbo: vi.fn() + }; + }; + + afterEach(() => { + clearLagEnv(); + vi.useRealTimers(); + }); + + test('suppresses bootstrap lag noise and records stale metrics', () => { + setLagEnv(); + vi.useFakeTimers(); + vi.setSystemTime(10_000); + const monitor = createHyperliquidLagMonitor('hyperliquid'); + const context = createLagContext(); + + monitor(context, 'trades', 'BTC-PERP', 0); + + expect(context.runtimeContext.logger.error).not.toHaveBeenCalled(); + expect(context.runtimeContext.metrics?.increment).toHaveBeenCalledWith( + 'hyperliquid_ws_stale_payload_total', + 1, + expect.objectContaining({ stream: 'trades', asset: 'BTC-PERP' }) + ); + }); + + test('logs and escalates after consecutive live lag breaches', () => { + setLagEnv({ INGEST_HL_LAG_ALERT_BREACHES: '2', INGEST_HL_LAG_ALERT_INTERVAL_MS: '1000' }); + vi.useFakeTimers(); + const monitor = createHyperliquidLagMonitor('hyperliquid'); + const context = createLagContext(); + + vi.setSystemTime(0); + monitor(context, 'trades', 'BTC-PERP', 0); // clears bootstrap + + vi.setSystemTime(20_000); + monitor(context, 'trades', 'BTC-PERP', 0); // first breach + vi.setSystemTime(40_000); + monitor(context, 'trades', 'BTC-PERP', 0); // second breach should alert + + expect(context.runtimeContext.logger.error).toHaveBeenCalledWith( + 'Hyperliquid websocket lag breach', + expect.objectContaining({ asset: 'BTC-PERP', stream: 'trades' }) + ); + expect(context.runtimeContext.metrics?.increment).toHaveBeenCalledWith( + 'hyperliquid_ws_live_lag_breach_total', + 1, + expect.objectContaining({ asset: 'BTC-PERP', stream: 'trades' }) + ); + expect(context.runtimeContext.signalFatalError).toHaveBeenCalledWith( + 'hyperliquid_ws_lag', + expect.objectContaining({ asset: 'BTC-PERP', stream: 'trades' }) + ); + }); + + test('throttles repeated alerts within the configured interval', () => { + setLagEnv({ + INGEST_HL_LAG_ALERT_BREACHES: '1', + INGEST_HL_LAG_ALERT_INTERVAL_MS: '60000' + }); + vi.useFakeTimers(); + const monitor = createHyperliquidLagMonitor('hyperliquid'); + const context = createLagContext(); + + vi.setSystemTime(0); + monitor(context, 'trades', 'BTC-PERP', 0); // bootstrap cleared + + vi.setSystemTime(70_000); + monitor(context, 'trades', 'BTC-PERP', 0); // first alert + const loggerErrorMock = context.runtimeContext.logger.error as unknown as Mock; + const incrementMock = context.runtimeContext.metrics?.increment as unknown as Mock; + const errorCalls = loggerErrorMock.mock.calls.length; + const lagMetricCalls = incrementMock.mock.calls.filter(([metric]) => metric === 'hyperliquid_ws_live_lag_breach_total').length; + + vi.setSystemTime(90_000); // within alert interval window + monitor(context, 'trades', 'BTC-PERP', 0); + + expect(loggerErrorMock.mock.calls.length).toBe(errorCalls); + const nextLagMetricCalls = incrementMock.mock.calls.filter(([metric]) => metric === 'hyperliquid_ws_live_lag_breach_total').length; + expect(nextLagMetricCalls).toBe(lagMetricCalls); + }); +}); + +export {}; + diff --git a/packages/ingestion-sdk/tests/hyperliquid.info.test.ts b/packages/ingestion-sdk/tests/hyperliquid.info.test.ts new file mode 100644 index 00000000..34b9cb1f --- /dev/null +++ b/packages/ingestion-sdk/tests/hyperliquid.info.test.ts @@ -0,0 +1,77 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +import { + DEFAULT_INFO_ENDPOINT, + postHyperliquidInfo, + RestRequestError, + type InfoRequestPayload +} from '../src/providers/hyperliquid/rest/info'; + +const fetchMock = vi.fn(); + +vi.mock('undici', () => ({ + fetch: (...args: unknown[]) => fetchMock(...args) +})); + +describe('postHyperliquidInfo', () => { + beforeEach(() => { + fetchMock.mockReset(); + }); + + test('retries transient failures before succeeding', async () => { + const payload: InfoRequestPayload = { type: 'perpDexs' }; + const responseBody = { ok: true }; + + const successResponse = { + ok: true, + status: 200, + json: vi.fn().mockResolvedValue(responseBody) + }; + + fetchMock + .mockRejectedValueOnce(new Error('ECONNRESET')) + .mockResolvedValueOnce(successResponse); + + const result = await postHyperliquidInfo(payload, DEFAULT_INFO_ENDPOINT, { + timeoutMs: 50, + maxAttempts: 2 + }); + + expect(result).toBe(responseBody); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + test('throws RestRequestError after exhausting retries on HTTP errors', async () => { + const payload: InfoRequestPayload = { type: 'allMids' }; + + fetchMock.mockResolvedValue({ + ok: false, + status: 503, + json: vi.fn() + }); + + await expect( + postHyperliquidInfo(payload, DEFAULT_INFO_ENDPOINT, { maxAttempts: 2, timeoutMs: 50 }) + ).rejects.toBeInstanceOf(RestRequestError); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + test('aborts the request after the timeout elapses', async () => { + const payload: InfoRequestPayload = { type: 'perpDexs' }; + + fetchMock.mockImplementationOnce((_url: string, options?: { signal?: AbortSignal }) => { + return new Promise((_resolve, reject) => { + options?.signal?.addEventListener('abort', () => { + reject(new Error('Aborted')); + }); + }); + }); + + await expect( + postHyperliquidInfo(payload, DEFAULT_INFO_ENDPOINT, { timeoutMs: 5, maxAttempts: 1 }) + ).rejects.toBeInstanceOf(RestRequestError); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); +}); + + diff --git a/packages/ingestion-sdk/tests/runtime.worker.test.ts b/packages/ingestion-sdk/tests/runtime.worker.test.ts index 502b5f30..cac9fed7 100644 --- a/packages/ingestion-sdk/tests/runtime.worker.test.ts +++ b/packages/ingestion-sdk/tests/runtime.worker.test.ts @@ -1,6 +1,4 @@ -import { describe, expect, test, beforeEach } from 'vitest'; - -import { startIngestionWorker } from '../src/runtime/worker'; +import { describe, expect, test, beforeEach, vi } from 'vitest'; import type { HyperliquidAdapterOptions } from '../src/providers/hyperliquid'; import type { IngestionAdapter, IngestionContext } from '../src/adapters'; @@ -54,6 +52,33 @@ const createStubServer = () => { }; }; +type StartIngestionWorker = typeof import('../src/runtime/worker')['startIngestionWorker']; +let startIngestionWorker: StartIngestionWorker; +const WORKER_ENV_KEYS = [ + 'INGEST_MAX_PENDING_PUBLISHES', + 'INGEST_PUBLISH_THROTTLE_RATIO', + 'INGEST_PUBLISH_THROTTLE_DELAY_MS', + 'INGEST_PUBLISH_THROTTLE_MAX_WAIT_MS', + 'INGEST_KAFKA_OVERLOAD_MAX_RETRIES', + 'INGEST_KAFKA_OVERLOAD_BACKOFF_MS', + 'INGEST_KAFKA_OVERLOAD_LOG_INTERVAL_MS' +]; + +const loadWorkerModule = async (overrides: Record = {}) => { + vi.resetModules(); + for (const key of WORKER_ENV_KEYS) { + delete process.env[key]; + } + for (const [key, value] of Object.entries(overrides)) { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } + ({ startIngestionWorker } = await import('../src/runtime/worker')); +}; + describe('startIngestionWorker', () => { let producer: ReturnType; let adapterContext: IngestionContext | null; @@ -70,6 +95,7 @@ describe('startIngestionWorker', () => { }); test('boots adapter and publishes via context', async () => { + await loadWorkerModule(); const runtimeConfig = { kind: 'hyperliquid', provider: 'hyperliquid', @@ -155,6 +181,7 @@ describe('startIngestionWorker', () => { }); test('can run without HTTP server when not provided', async () => { + await loadWorkerModule(); const runtimeConfig = { kind: 'hyperliquid', provider: 'hyperliquid', @@ -209,5 +236,224 @@ describe('startIngestionWorker', () => { expect(worker.server).toBeUndefined(); await worker.stop(); }); + + test('retries transient Kafka errors before bubbling up', async () => { + await loadWorkerModule({ + INGEST_KAFKA_OVERLOAD_MAX_RETRIES: '2', + INGEST_KAFKA_OVERLOAD_BACKOFF_MS: '1' + }); + + const runtimeConfig = { + kind: 'hyperliquid', + provider: 'hyperliquid', + providerLabel: 'Hyperliquid', + providerMetadata: { + topicPrefix: 'hyperliquid', + redisNamespace: 'demo' + }, + wsUrl: 'wss://example', + assets: [], + streams: [], + shard: { + id: 'test', + index: 0, + count: 1 + }, + topics: { + prices: 'hyperliquid.prices' + } + }; + + let attempts = 0; + const originalSend = producer.send.bind(producer); + producer.send = vi.fn(async (payload) => { + attempts += 1; + if (attempts === 1) { + throw Object.assign(new Error('Timeout while acquiring lock'), { type: 'REQUEST_TIMEOUT' }); + } + await originalSend(payload); + }); + + const worker = await startIngestionWorker({ + runtimeConfig: runtimeConfig as HyperliquidAdapterOptions, + logger: { + info: () => undefined, + warn: () => undefined, + error: () => undefined + }, + adapterFactory: () => ({ + metadata: { + id: 'test', + provider: 'hyperliquid', + supportedDomains: ['prices'] + }, + async connect(ctx) { + adapterContext = ctx; + }, + async start() { + adapterStartCalls += 1; + }, + async stop() { + adapterStopCalls += 1; + } + }), + createKafkaProducer: async () => producer, + registerSignalHandlers: false + }); + + await adapterContext!.publish('hyperliquid.prices', { foo: 'bar' }); + expect(producer.send).toHaveBeenCalledTimes(2); + await worker.stop(); + }); + + test('throttles publishes when pending queue is saturated', async () => { + await loadWorkerModule({ + INGEST_MAX_PENDING_PUBLISHES: '1', + INGEST_PUBLISH_THROTTLE_RATIO: '0.5', + INGEST_PUBLISH_THROTTLE_DELAY_MS: '5', + INGEST_PUBLISH_THROTTLE_MAX_WAIT_MS: '50' + }); + + const runtimeConfig = { + kind: 'hyperliquid', + provider: 'hyperliquid', + providerLabel: 'Hyperliquid', + providerMetadata: { + topicPrefix: 'hyperliquid', + redisNamespace: 'demo' + }, + wsUrl: 'wss://example', + assets: [], + streams: [], + shard: { + id: 'test', + index: 0, + count: 1 + }, + topics: { + prices: 'hyperliquid.prices' + } + }; + + const sendResolvers: Array<() => void> = []; + producer.send = vi.fn( + () => + new Promise((resolve) => { + sendResolvers.push(resolve); + }) + ); + + const worker = await startIngestionWorker({ + runtimeConfig: runtimeConfig as HyperliquidAdapterOptions, + logger: { + info: () => undefined, + warn: () => undefined, + error: () => undefined + }, + adapterFactory: () => ({ + metadata: { + id: 'test', + provider: 'hyperliquid', + supportedDomains: ['prices'] + }, + async connect(ctx) { + adapterContext = ctx; + }, + async start() { + adapterStartCalls += 1; + }, + async stop() { + adapterStopCalls += 1; + } + }), + createKafkaProducer: async () => producer, + registerSignalHandlers: false + }); + + const firstPublish = adapterContext!.publish('hyperliquid.prices', { id: 1 }); + await vi.waitFor(() => expect(producer.send).toHaveBeenCalledTimes(1)); + const secondPublishPromise = adapterContext!.publish('hyperliquid.prices', { id: 2 }); + + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(producer.send).toHaveBeenCalledTimes(1); + + sendResolvers.shift()?.(); + await firstPublish; + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(producer.send).toHaveBeenCalledTimes(2); + sendResolvers.shift()?.(); + await secondPublishPromise; + + await worker.stop(); + }); + + test('rate limits overload warning logs', async () => { + await loadWorkerModule({ + INGEST_KAFKA_OVERLOAD_MAX_RETRIES: '0', + INGEST_KAFKA_OVERLOAD_LOG_INTERVAL_MS: '60000' + }); + + const warn = vi.fn(); + const worker = await startIngestionWorker({ + runtimeConfig: { + kind: 'hyperliquid', + provider: 'hyperliquid', + providerLabel: 'Hyperliquid', + providerMetadata: { + topicPrefix: 'hyperliquid', + redisNamespace: 'demo' + }, + wsUrl: 'wss://example', + assets: [], + streams: [], + shard: { + id: 'test', + index: 0, + count: 1 + }, + topics: { + prices: 'hyperliquid.prices' + } + } as HyperliquidAdapterOptions, + logger: { + info: () => undefined, + warn, + error: () => undefined + }, + adapterFactory: () => ({ + metadata: { + id: 'test', + provider: 'hyperliquid', + supportedDomains: ['prices'] + }, + async connect(ctx) { + adapterContext = ctx; + }, + async start() { + adapterStartCalls += 1; + }, + async stop() { + adapterStopCalls += 1; + } + }), + createKafkaProducer: async () => { + return { + ...producer, + async send() { + throw new Error('Timeout while acquiring lock'); + } + }; + }, + registerSignalHandlers: false + }); + + await expect(adapterContext!.publish('hyperliquid.prices', { id: 1 })).rejects.toThrow('Timeout'); + await expect(adapterContext!.publish('hyperliquid.prices', { id: 2 })).rejects.toThrow('Timeout'); + + const overloadLogs = warn.mock.calls.filter(([message]) => message === 'Kafka producer overload detected'); + expect(overloadLogs).toHaveLength(1); + + await worker.stop(); + }); }); diff --git a/scripts/k8s/deploy.sh b/scripts/k8s/deploy.sh index a5260c0a..2141e01f 100755 --- a/scripts/k8s/deploy.sh +++ b/scripts/k8s/deploy.sh @@ -36,8 +36,28 @@ helm upgrade --install "${RELEASE}" "${CHART_DIR}" \ "$@" echo "[deploy] Waiting for core deployments to become available" -kubectl -n "${NAMESPACE}" wait --for=condition=available deploy/pmon-pmon-ingestor deploy/pmon-pmon-gateway deploy/pmon-pmon-ui --timeout=300s -kubectl -n "${NAMESPACE}" wait --for=condition=ready pod -l app.kubernetes.io/component=redpanda --timeout=300s + +# Wait for gateway and UI deployments +kubectl -n "${NAMESPACE}" wait --for=condition=available deploy/pmon-pmon-gateway deploy/pmon-pmon-ui --timeout=300s + +# Wait for ingestor deployments - handle both shard-based and single deployment modes +INGESTOR_DEPLOYMENTS=$(kubectl -n "${NAMESPACE}" get deployments -l app.kubernetes.io/component=ingestor -o name 2>/dev/null || echo "") +if [ -z "${INGESTOR_DEPLOYMENTS}" ]; then + echo "[deploy] Warning: No ingestor deployments found. They may not be enabled or may still be creating." +else + echo "[deploy] Found ingestor deployments: ${INGESTOR_DEPLOYMENTS}" + for deploy in ${INGESTOR_DEPLOYMENTS}; do + echo "[deploy] Waiting for ${deploy} to become available" + kubectl -n "${NAMESPACE}" wait --for=condition=available "${deploy}" --timeout=300s || { + echo "[deploy] Warning: ${deploy} did not become available within timeout" + } + done +fi + +# Wait for Redpanda pods if enabled +kubectl -n "${NAMESPACE}" wait --for=condition=ready pod -l app.kubernetes.io/component=redpanda --timeout=300s || { + echo "[deploy] Warning: Redpanda pods did not become ready (may not be enabled)" +} echo "[deploy] Deployment complete" diff --git a/services/gateway/README.md b/services/gateway/README.md index 2c825986..4960aa35 100644 --- a/services/gateway/README.md +++ b/services/gateway/README.md @@ -156,7 +156,7 @@ flowchart LR ### branding -The icon registry enriches asset metadata by trying existing branding hints, overrides, CDN fallbacks, and multiple provider APIs (CoinGecko, CoinPaprika, CoinMarketCap, Coincap). Results are cached in memory and optionally Redis with quota-aware lookup queues. +The icon registry enriches asset metadata through a deterministic pipeline: explicit branding URLs, curated override lists, CDN symbol fallbacks, and finally a rate-limited CoinGecko search client. Results are cached in memory and optionally Redis with quota-aware lookup queues, while repeated provider failures are suppressed to trace-level noise. ```232:274:services/gateway/src/branding/iconRegistry.ts export class IconRegistry { @@ -177,6 +177,8 @@ export class IconRegistry { } ``` +The companion `CoinGeckoClient` (`services/gateway/src/branding/coinGeckoClient.ts`) memoizes search responses, enforces a small concurrency window with cooldowns after HTTP 429 responses, and supports the optional `COINGECKO_API_KEY` header for elevated quotas. + ### clients - `hyperliquid.ts` – authenticated REST wrapper for spot/perp actions. @@ -690,6 +692,20 @@ const redisSnapshotAgeGauge = new Gauge({ - `gateway_websocket_backpressure_drops_total{context}` – number of WebSocket clients disconnected because their send buffer exceeded `GATEWAY_MAX_WEBSOCKET_BUFFER_BYTES`. - Default process metrics from `collectDefaultMetrics` (heap, CPU, GC). +### Hyperliquid ingestor reliability knobs + +- The Helm chart exposes `ingestor.reliability.*` in `kubernetes/chart/pmon/values.yaml`, which are rendered into the deployment as: + - `INGEST_KAFKA_OVERLOAD_BACKOFF_MS` – base backoff (ms) between Kafka producer retries. + - `INGEST_KAFKA_OVERLOAD_MAX_RETRIES` – number of retry attempts before surfacing the error to the adapter. + - `INGEST_KAFKA_OVERLOAD_LOG_INTERVAL_MS` – minimum interval between “Kafka producer overload detected” log entries to keep noise manageable. + - `INGEST_PUBLISH_THROTTLE_RATIO`, `INGEST_PUBLISH_THROTTLE_DELAY_MS`, `INGEST_PUBLISH_THROTTLE_MAX_WAIT_MS` – thresholds used by the worker to pause publishes when `pendingPublishes` approaches `INGEST_MAX_PENDING_PUBLISHES`. +- These env vars can also be set directly when running the ingestor outside Helm; the worker reads them on boot and exposes the following Prometheus signals: + - `ingestion_worker_kafka_overload_events_total` – number of observed timeout/backpressure events. + - `ingestion_worker_kafka_overload_logs_suppressed_total` – how many overload logs were suppressed by the rate limiter. + - `ingestion_worker_publish_throttle_total` + `ingestion_worker_publish_throttle_duration_ms` – how often the worker had to pause and for how long. + - `ingestion_worker_publish_backpressure_total` – counter for requests rejected because the pending queue never drained before `INGEST_MAX_PENDING_PUBLISHES`. + - `ingestion_worker_kafka_retry_total` – number of adapter level retries (labelled by topic + error type) which is a useful early warning that brokers are struggling. + **Health signals** - `/healthz` – always 200 but includes fatal reasons + warnings for observability. diff --git a/services/gateway/package.json b/services/gateway/package.json index e6a295ac..3f21aa02 100644 --- a/services/gateway/package.json +++ b/services/gateway/package.json @@ -10,12 +10,14 @@ "start": "node dist/index.js", "dev": "tsx watch src/index.ts", "smoke": "tsx scripts/smoke.ts", - "clean": "rimraf dist tsconfig.tsbuildinfo" + "clean": "rimraf dist tsconfig.tsbuildinfo", + "test": "vitest run" }, "dependencies": { "@pmon/core-service": "^0.3.7", "@pmon/ingestion-sdk": "^0.3.7", "@pmon/trader-intelligence": "^0.3.7", + "@scalar/express-api-reference": "^0.8.25", "cors": "^2.8.5", "dotenv": "^16.4.5", "express": "^4.19.2", @@ -23,6 +25,7 @@ "kafkajs": "^2.2.4", "pino": "^9.1.0", "prom-client": "^15.1.1", + "swagger-autogen": "^2.23.7", "undici": "^6.19.7", "ws": "^8.17.0" }, @@ -31,6 +34,7 @@ "@types/express": "^4.17.21", "@types/ws": "^8.5.10", "rimraf": "^5.0.5", - "tsx": "^3.13.0" + "tsx": "^3.13.0", + "vitest": "1.6.0" } } diff --git a/services/gateway/src/bootstrap.ts b/services/gateway/src/bootstrap.ts index 5659d57b..9141834d 100644 --- a/services/gateway/src/bootstrap.ts +++ b/services/gateway/src/bootstrap.ts @@ -845,7 +845,7 @@ const startGateway = async ({ applySnapshotFixtureSeed('startup'); - const { httpServer, cleanup: cleanupHttpServer } = createHttpServer(marketStore, overviewStore, () => healthTracker.health, { + const { httpServer, cleanup: cleanupHttpServer } = await createHttpServer(marketStore, overviewStore, () => healthTracker.health, { providerId: defaultProvider, providerLabel: config.selection.providerLabel, providers: config.providers, diff --git a/services/gateway/src/branding/__tests__/iconRegistry.test.ts b/services/gateway/src/branding/__tests__/iconRegistry.test.ts new file mode 100644 index 00000000..cb72e4eb --- /dev/null +++ b/services/gateway/src/branding/__tests__/iconRegistry.test.ts @@ -0,0 +1,126 @@ +import pino from 'pino'; +import { describe, expect, it, beforeEach, vi } from 'vitest'; +import type { CoinGeckoSearchClient, CoinGeckoSearchResponse } from '../coinGeckoClient'; +import { IconRegistry } from '../iconRegistry'; + +class StubCoinGeckoClient implements CoinGeckoSearchClient { + private readonly responses = new Map(); + public readonly calls: string[] = []; + + setResponse(query: string, response: CoinGeckoSearchResponse | null): void { + const key = query.toLowerCase(); + this.responses.set(key, response); + } + + async search(query: string): Promise { + const key = query.toLowerCase(); + this.calls.push(query); + return this.responses.has(key) ? this.responses.get(key)! : null; + } + + getDiagnostics(): Record { + return { calls: this.calls.length }; + } +} + +const silentLogger = pino({ level: 'silent' }); + +describe('IconRegistry', () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it('uses branding icon when reachable', async () => { + const coinGecko = new StubCoinGeckoClient(); + const registry = new IconRegistry({ logger: silentLogger, coinGeckoClient: coinGecko }); + vi.spyOn(registry as unknown as { urlExists: (url: string) => Promise }, 'urlExists').mockResolvedValue(true); + + const icon = await registry.resolveIcon({ + provider: 'hyperliquid', + asset: 'BTC-PERP', + symbol: 'BTC', + brandingIconUrl: 'https://assets.example/btc.svg' + }); + + expect(icon).toBe('https://assets.example/btc.svg'); + expect(coinGecko.calls).toHaveLength(0); + }); + + it('prefers CDN icon lookup before CoinGecko', async () => { + const coinGecko = new StubCoinGeckoClient(); + const registry = new IconRegistry({ logger: silentLogger, coinGeckoClient: coinGecko }); + vi.spyOn(registry as unknown as { urlExists: (url: string) => Promise }, 'urlExists').mockResolvedValue(false); + vi.spyOn(registry as unknown as { fetchIconForSymbol: (symbol: string) => Promise }, 'fetchIconForSymbol') + .mockResolvedValueOnce('https://cdn.example/arb.svg') + .mockResolvedValue(null); + + const icon = await registry.resolveIcon({ + provider: 'hyperliquid', + asset: 'ARB-PERP', + symbol: 'ARB' + }); + + expect(icon).toBe('https://cdn.example/arb.svg'); + expect(coinGecko.calls).toHaveLength(0); + }); + + it('falls back to CoinGecko when necessary and caches the result', async () => { + const coinGecko = new StubCoinGeckoClient(); + coinGecko.setResponse('arb', { + coins: [ + { + id: 'arbitrum', + name: 'Arbitrum', + symbol: 'arb', + large: 'https://assets.coingecko.com/arb.png', + market_cap_rank: 30 + } + ] + }); + const registry = new IconRegistry({ logger: silentLogger, coinGeckoClient: coinGecko }); + vi.spyOn(registry as unknown as { urlExists: (url: string) => Promise }, 'urlExists').mockImplementation(async (url) => + url.includes('coingecko') + ); + vi.spyOn(registry as unknown as { fetchIconForSymbol: (symbol: string) => Promise }, 'fetchIconForSymbol').mockResolvedValue( + null + ); + + const context = { + provider: 'hyperliquid', + asset: 'ARB-PERP', + symbol: 'ARB' + }; + + const icon = await registry.resolveIcon(context); + expect(icon).toBe('https://assets.coingecko.com/arb.png'); + expect(coinGecko.calls).toHaveLength(1); + + const cached = await registry.resolveIcon(context); + expect(cached).toBe('https://assets.coingecko.com/arb.png'); + expect(coinGecko.calls).toHaveLength(1); + }); + + it('marks misses to avoid repeated CoinGecko lookups', async () => { + const coinGecko = new StubCoinGeckoClient(); + const registry = new IconRegistry({ logger: silentLogger, coinGeckoClient: coinGecko }); + vi.spyOn(registry as unknown as { urlExists: (url: string) => Promise }, 'urlExists').mockResolvedValue(false); + vi.spyOn(registry as unknown as { fetchIconForSymbol: (symbol: string) => Promise }, 'fetchIconForSymbol').mockResolvedValue( + null + ); + + const context = { + provider: 'hyperliquid', + asset: 'UNKNOWN', + symbol: 'UNKNOWN' + }; + + const first = await registry.resolveIcon(context); + expect(first).toBeUndefined(); + expect(coinGecko.calls).toHaveLength(1); + + const second = await registry.resolveIcon(context); + expect(second).toBeUndefined(); + expect(coinGecko.calls).toHaveLength(1); + }); +}); + diff --git a/services/gateway/src/branding/coinGeckoClient.ts b/services/gateway/src/branding/coinGeckoClient.ts new file mode 100644 index 00000000..dcb1bab1 --- /dev/null +++ b/services/gateway/src/branding/coinGeckoClient.ts @@ -0,0 +1,181 @@ +import { fetch } from 'undici'; +import type pino from 'pino'; + +export type CoinGeckoSearchResponse = { + coins?: Array<{ + id?: string; + name?: string; + symbol?: string; + large?: string; + thumb?: string; + market_cap_rank?: number | null; + }>; +}; + +export interface CoinGeckoSearchClient { + search(query: string): Promise; + getDiagnostics(): Record; +} + +export type CoinGeckoClientOptions = { + logger: pino.Logger; + maxConcurrentRequests?: number; + cacheTtlMs?: number; + missTtlMs?: number; + cooldownMs?: number; + apiKey?: string; +}; + +type CacheEntry = { expiresAt: number; payload: CoinGeckoSearchResponse | null }; + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +const DEFAULT_CONCURRENCY = 4; +const DEFAULT_CACHE_TTL_MS = 10 * 60 * 1000; // 10 minutes +const DEFAULT_MISS_TTL_MS = 30 * 60 * 1000; // 30 minutes +const DEFAULT_COOLDOWN_MS = 2000; + +export class CoinGeckoClient implements CoinGeckoSearchClient { + private readonly logger: pino.Logger; + private readonly maxConcurrentRequests: number; + private readonly cacheTtlMs: number; + private readonly missTtlMs: number; + private readonly cooldownMs: number; + private readonly apiKey?: string; + private activeRequests = 0; + private readonly queue: Array<() => void> = []; + private readonly cache = new Map(); + private readonly pending = new Map>(); + private cooldownUntil = 0; + private last429LogAt = 0; + + constructor(options: CoinGeckoClientOptions) { + this.logger = options.logger.child({ module: 'coinGeckoClient' }); + this.maxConcurrentRequests = Math.max(1, options.maxConcurrentRequests ?? DEFAULT_CONCURRENCY); + this.cacheTtlMs = Math.max(1000, options.cacheTtlMs ?? DEFAULT_CACHE_TTL_MS); + this.missTtlMs = Math.max(1000, options.missTtlMs ?? DEFAULT_MISS_TTL_MS); + this.cooldownMs = Math.max(250, options.cooldownMs ?? DEFAULT_COOLDOWN_MS); + this.apiKey = options.apiKey ?? process.env.COINGECKO_API_KEY; + } + + async search(query: string): Promise { + const trimmed = query.trim(); + const normalized = trimmed.toLowerCase(); + if (!normalized) { + return null; + } + + const now = Date.now(); + const cached = this.cache.get(normalized); + if (cached && cached.expiresAt > now) { + return cached.payload; + } + + const pending = this.pending.get(normalized); + if (pending) { + return pending; + } + + const request = this.executeSearch(normalized, trimmed); + this.pending.set(normalized, request); + request.finally(() => this.pending.delete(normalized)); + return request; + } + + getDiagnostics(): Record { + return { + cacheEntries: this.cache.size, + pendingQueries: this.pending.size, + queueDepth: this.queue.length, + cooldownRemainingMs: Math.max(0, this.cooldownUntil - Date.now()) + }; + } + + private async executeSearch( + key: string, + query: string + ): Promise { + await this.acquireSlot(); + try { + await this.waitForCooldown(); + const response = await this.performSearch(query); + const ttl = + response && (response.coins?.length ?? 0) > 0 ? this.cacheTtlMs : this.missTtlMs; + this.cache.set(key, { expiresAt: Date.now() + ttl, payload: response }); + return response; + } finally { + this.releaseSlot(); + } + } + + private async performSearch(query: string): Promise { + try { + const response = await fetch( + `https://api.coingecko.com/api/v3/search?query=${encodeURIComponent(query)}`, + { + headers: { + accept: 'application/json', + ...(this.apiKey ? { 'x-cg-demo-api-key': this.apiKey } : {}) + } + } + ); + + if (response.status === 429) { + this.handle429(); + return null; + } + + if (!response.ok) { + this.logger.trace({ status: response.status, query }, 'CoinGecko search failed'); + return null; + } + + const payload = (await response.json()) as CoinGeckoSearchResponse; + return payload; + } catch (error) { + this.logger.trace({ error, query }, 'CoinGecko search threw'); + return null; + } + } + + private handle429(): void { + const now = Date.now(); + this.cooldownUntil = Math.max(this.cooldownUntil, now + this.cooldownMs); + if (now - this.last429LogAt >= this.cooldownMs) { + this.logger.warn({ cooldownMs: this.cooldownMs }, 'CoinGecko rate limited'); + this.last429LogAt = now; + } + } + + private async waitForCooldown(): Promise { + const now = Date.now(); + if (this.cooldownUntil <= now) { + return; + } + await delay(this.cooldownUntil - now); + } + + private async acquireSlot(): Promise { + if (this.activeRequests < this.maxConcurrentRequests) { + this.activeRequests++; + return; + } + + await new Promise((resolve) => { + this.queue.push(() => { + this.activeRequests++; + resolve(); + }); + }); + } + + private releaseSlot(): void { + this.activeRequests = Math.max(0, this.activeRequests - 1); + const next = this.queue.shift(); + if (next) { + next(); + } + } +} + + diff --git a/services/gateway/src/branding/iconRegistry.ts b/services/gateway/src/branding/iconRegistry.ts index e4c2cba1..dfb4acfb 100644 --- a/services/gateway/src/branding/iconRegistry.ts +++ b/services/gateway/src/branding/iconRegistry.ts @@ -1,6 +1,8 @@ import type Redis from 'ioredis'; import { fetch } from 'undici'; import type pino from 'pino'; +import { CoinGeckoClient } from './coinGeckoClient'; +import type { CoinGeckoSearchClient } from './coinGeckoClient'; type IconLookupContext = { provider?: string; @@ -12,49 +14,6 @@ type IconLookupContext = { brandingIconUrl?: string | null; }; -type CoingeckoSearchResponse = { - coins?: Array<{ - id?: string; - name?: string; - symbol?: string; - large?: string; - thumb?: string; - market_cap_rank?: number | null; - }>; -}; - -type CoinPaprikaSearchResponse = { - currencies?: Array<{ - id?: string; - name?: string; - symbol?: string; - rank?: number | string | null; - is_active?: boolean; - type?: string; - }>; -}; - -type CoincapSearchResponse = { - data?: Array<{ - id?: string; - name?: string; - symbol?: string; - rank?: number | string | null; - }>; -}; - -type CoinMarketCapSearchResponse = { - data?: { - crypto?: Array<{ - id?: number; - name?: string; - slug?: string; - symbol?: string; - rank?: number | null; - }>; - }; -}; - type IconRegistryOptions = { logger: pino.Logger; redis?: Redis; @@ -63,6 +22,7 @@ type IconRegistryOptions = { maxConcurrentLookups?: number; maxLookupQueueSize?: number; lookupQueueWaitMs?: number; + coinGeckoClient?: CoinGeckoSearchClient; }; const CACHE_VERSION = 'v4'; @@ -210,6 +170,11 @@ const ICON_OVERRIDES: Record = { pax: ['https://assets.coingecko.com/coins/images/9519/large/paxg.png?1696511576'] }; +type CoinGeckoIconMatch = { + iconUrl: string; + sourceSymbol?: string; +}; + class IconLookupCapacityError extends Error { constructor(readonly code: 'queue-overflow' | 'timeout' | 'shutdown') { const message = @@ -234,6 +199,7 @@ export class IconRegistry { private readonly redis?: Redis; private readonly hitTtlSeconds: number; private readonly missTtlSeconds: number; + private readonly coinGeckoClient: CoinGeckoSearchClient; private readonly assetCache = new Map(); private readonly symbolCache = new Map(); private readonly pendingAssets = new Map>(); @@ -266,6 +232,7 @@ export class IconRegistry { options.maxLookupQueueSize ?? DEFAULT_MAX_LOOKUP_QUEUE_SIZE ); this.lookupQueueWaitMs = Math.max(1000, options.lookupQueueWaitMs ?? DEFAULT_LOOKUP_QUEUE_WAIT_MS); + this.coinGeckoClient = options.coinGeckoClient ?? new CoinGeckoClient({ logger: this.logger }); // Periodic cleanup of stale pending operations and cache size limits this.cleanupInterval = setInterval(() => { @@ -301,7 +268,8 @@ export class IconRegistry { pendingAssets: this.pendingAssets.size, pendingSymbols: this.pendingSymbols.size, lookupQueueSize: this.lookupQueue.length, - activeLookups: this.activeLookupCount + activeLookups: this.activeLookupCount, + coinGecko: this.coinGeckoClient.getDiagnostics() }; } @@ -371,75 +339,32 @@ export class IconRegistry { } const compute = async (): Promise => { - const trimmedBrandingUrl = this.normaliseUrl(context.brandingIconUrl); - if (trimmedBrandingUrl) { - const brandingReachable = await this.urlExists(trimmedBrandingUrl); - if (brandingReachable) { - if (assetKey) { - await this.writeAssetCache(assetKey, trimmedBrandingUrl); - } - const symbols = this.deriveSymbolCandidates(context); - await Promise.all(symbols.map((symbol) => this.writeSymbolCache(symbol, trimmedBrandingUrl))); - return trimmedBrandingUrl; - } - if (assetKey) { - await this.writeAssetCache(assetKey, null); - } - } - const symbolCandidates = this.deriveSymbolCandidates(context); - const overrideIcon = await this.resolveIconOverride(context, symbolCandidates); + const brandingIcon = await this.resolveBrandingIcon(context, symbolCandidates, assetKey); + if (brandingIcon) { + return brandingIcon; + } + + const overrideIcon = await this.resolveOverrideIconHit(context, symbolCandidates, assetKey); if (overrideIcon) { - if (assetKey) { - await this.writeAssetCache(assetKey, overrideIcon); - } - await Promise.all(symbolCandidates.map((symbol) => this.writeSymbolCache(symbol, overrideIcon))); return overrideIcon; } - for (const symbol of symbolCandidates) { - const located = await this.lookupSymbol(symbol); - if (located) { - if (assetKey) { - await this.writeAssetCache(assetKey, located); - } - return located; - } + const cdnIcon = await this.resolveSymbolLookupIcon(symbolCandidates, assetKey); + if (cdnIcon) { + return cdnIcon; } - const coingeckoIcon = await this.fetchCoingeckoIcon(symbolCandidates, context.displayName); + const coingeckoIcon = await this.resolveCoinGeckoIcon( + symbolCandidates, + context.displayName, + assetKey + ); if (coingeckoIcon) { - if (assetKey) { - await this.writeAssetCache(assetKey, coingeckoIcon); - } return coingeckoIcon; } - const coinpaprikaIcon = await this.fetchCoinPaprikaIcon(symbolCandidates, context.displayName); - if (coinpaprikaIcon) { - if (assetKey) { - await this.writeAssetCache(assetKey, coinpaprikaIcon); - } - return coinpaprikaIcon; - } - - const coinmarketcapIcon = await this.fetchCoinMarketCapIcon(symbolCandidates, context.displayName); - if (coinmarketcapIcon) { - if (assetKey) { - await this.writeAssetCache(assetKey, coinmarketcapIcon); - } - return coinmarketcapIcon; - } - - const coincapIcon = await this.fetchCoincapIcon(symbolCandidates, context.displayName); - if (coincapIcon) { - if (assetKey) { - await this.writeAssetCache(assetKey, coincapIcon); - } - return coincapIcon; - } - if (assetKey) { await this.writeAssetCache(assetKey, null); } @@ -522,6 +447,79 @@ export class IconRegistry { return null; } + private async resolveBrandingIcon( + context: IconLookupContext, + symbolCandidates: string[], + assetKey: string | null + ): Promise { + const trimmedBrandingUrl = this.normaliseUrl(context.brandingIconUrl); + if (!trimmedBrandingUrl) { + return null; + } + const brandingReachable = await this.urlExists(trimmedBrandingUrl); + if (!brandingReachable) { + if (assetKey) { + await this.writeAssetCache(assetKey, null); + } + return null; + } + if (assetKey) { + await this.writeAssetCache(assetKey, trimmedBrandingUrl); + } + await Promise.all(symbolCandidates.map((symbol) => this.writeSymbolCache(symbol, trimmedBrandingUrl))); + return trimmedBrandingUrl; + } + + private async resolveOverrideIconHit( + context: IconLookupContext, + symbolCandidates: string[], + assetKey: string | null + ): Promise { + const overrideIcon = await this.resolveIconOverride(context, symbolCandidates); + if (!overrideIcon) { + return null; + } + if (assetKey) { + await this.writeAssetCache(assetKey, overrideIcon); + } + await Promise.all(symbolCandidates.map((symbol) => this.writeSymbolCache(symbol, overrideIcon))); + return overrideIcon; + } + + private async resolveSymbolLookupIcon( + symbolCandidates: string[], + assetKey: string | null + ): Promise { + for (const symbol of symbolCandidates) { + const located = await this.lookupSymbol(symbol); + if (located) { + if (assetKey) { + await this.writeAssetCache(assetKey, located); + } + return located; + } + } + return null; + } + + private async resolveCoinGeckoIcon( + symbolCandidates: string[], + displayName: string | undefined, + assetKey: string | null + ): Promise { + const result = await this.fetchCoingeckoIcon(symbolCandidates, displayName); + if (!result) { + return null; + } + if (assetKey) { + await this.writeAssetCache(assetKey, result.iconUrl); + } + if (result.sourceSymbol) { + await this.writeSymbolCache(result.sourceSymbol, result.iconUrl); + } + return result.iconUrl; + } + private buildAssetKey(provider: string | undefined, asset: string | undefined): string | null { if (!asset) return null; const normalizedAsset = asset.trim().toLowerCase(); @@ -545,7 +543,7 @@ export class IconRegistry { this.assetCache.set(key, stored); return stored; } catch (error) { - this.logger.debug({ error, key }, 'Failed to read asset icon from redis cache'); + this.logger.trace({ error, key }, 'Failed to read asset icon from redis cache'); return undefined; } } @@ -561,7 +559,7 @@ export class IconRegistry { await this.redis.set(redisKey, value, 'EX', this.hitTtlSeconds); } } catch (error) { - this.logger.debug({ error, key }, 'Failed to write asset icon to redis cache'); + this.logger.trace({ error, key }, 'Failed to write asset icon to redis cache'); } } @@ -606,7 +604,7 @@ export class IconRegistry { this.symbolCache.set(symbol, stored); return stored; } catch (error) { - this.logger.debug({ error, symbol }, 'Failed to read symbol icon from redis cache'); + this.logger.trace({ error, symbol }, 'Failed to read symbol icon from redis cache'); return undefined; } } @@ -622,7 +620,7 @@ export class IconRegistry { await this.redis.set(redisKey, value, 'EX', this.hitTtlSeconds); } } catch (error) { - this.logger.debug({ error, symbol }, 'Failed to write symbol icon to redis cache'); + this.logger.trace({ error, symbol }, 'Failed to write symbol icon to redis cache'); } } @@ -637,7 +635,10 @@ export class IconRegistry { return null; } - private async fetchCoingeckoIcon(symbols: string[], displayName?: string): Promise { + private async fetchCoingeckoIcon( + symbols: string[], + displayName?: string + ): Promise { const normalisedName = displayName?.trim().toLowerCase(); for (const candidate of symbols) { const normalisedSymbol = candidate.trim().toLowerCase(); @@ -645,290 +646,52 @@ export class IconRegistry { const cached = this.symbolCache.get(normalisedSymbol); if (typeof cached === 'string' && cached.length > 0) { - return cached; + return { iconUrl: cached, sourceSymbol: normalisedSymbol }; } const cachedRedis = await this.readSymbolCache(normalisedSymbol); if (typeof cachedRedis === 'string' && cachedRedis.length > 0) { - return cachedRedis; + return { iconUrl: cachedRedis, sourceSymbol: normalisedSymbol }; } - try { - const response = await fetch( - `https://api.coingecko.com/api/v3/search?query=${encodeURIComponent(candidate)}`, - { - headers: { - accept: 'application/json' - } - } - ); - - if (!response.ok) { - this.logger.debug({ status: response.status, symbol: candidate }, 'Coingecko lookup failed'); - await this.writeSymbolCache(normalisedSymbol, null); - continue; - } - - const payload = (await response.json()) as CoingeckoSearchResponse; - const coins = payload.coins ?? []; - if (coins.length === 0) { - await this.writeSymbolCache(normalisedSymbol, null); - continue; - } - - const exact = coins.find( - (coin) => coin.symbol?.trim().toLowerCase() === normalisedSymbol - ); - const byName = normalisedName - ? coins.find((coin) => coin.name?.trim().toLowerCase() === normalisedName) - : undefined; - const ranked = [...coins].sort((a, b) => { - const rankA = a.market_cap_rank ?? Number.MAX_SAFE_INTEGER; - const rankB = b.market_cap_rank ?? Number.MAX_SAFE_INTEGER; - return rankA - rankB; - }); - const fallback = ranked[0]; - const match = exact ?? byName ?? fallback; - const resolvedIcon = match?.large ?? match?.thumb; - const iconUrl = typeof resolvedIcon === 'string' ? resolvedIcon.trim() : ''; - if (iconUrl.length > 0 && (await this.urlExists(iconUrl))) { - await this.writeSymbolCache(normalisedSymbol, iconUrl); - return iconUrl; - } - - await this.writeSymbolCache(normalisedSymbol, null); - } catch (error) { - this.logger.debug({ error, symbol: candidate }, 'Coingecko icon lookup threw'); + const payload = await this.coinGeckoClient.search(candidate); + if (!payload) { await this.writeSymbolCache(normalisedSymbol, null); + continue; } - } - return null; - } - - private async fetchCoinPaprikaIcon(symbols: string[], displayName?: string): Promise { - const normalisedName = displayName?.trim().toLowerCase(); - for (const candidate of symbols) { - const normalisedSymbol = candidate.trim().toLowerCase(); - if (normalisedSymbol.length === 0) continue; - - const cached = this.symbolCache.get(normalisedSymbol); - if (typeof cached === 'string' && cached.length > 0) { - return cached; - } - - const cachedRedis = await this.readSymbolCache(normalisedSymbol); - if (typeof cachedRedis === 'string' && cachedRedis.length > 0) { - return cachedRedis; - } - - try { - const response = await fetch( - `https://api.coinpaprika.com/v1/search?query=${encodeURIComponent(candidate)}`, - { - headers: { - accept: 'application/json' - } - } - ); - - if (!response.ok) { - this.logger.debug({ status: response.status, symbol: candidate }, 'CoinPaprika lookup failed'); - continue; - } - - const payload = (await response.json()) as CoinPaprikaSearchResponse; - const currencies = payload.currencies ?? []; - if (currencies.length === 0) { - continue; - } - - const exact = currencies.find( - (currency) => currency.symbol?.trim().toLowerCase() === normalisedSymbol - ); - const byName = normalisedName - ? currencies.find((currency) => currency.name?.trim().toLowerCase() === normalisedName) - : undefined; - const normaliseRank = (rank: number | string | null | undefined): number => { - if (typeof rank === 'number' && Number.isFinite(rank)) return rank; - if (typeof rank === 'string') { - const parsed = Number.parseInt(rank, 10); - if (Number.isFinite(parsed)) { - return parsed; - } - } - return Number.MAX_SAFE_INTEGER; - }; - const ranked = [...currencies].sort( - (a, b) => normaliseRank(a.rank) - normaliseRank(b.rank) - ); - const fallback = ranked[0]; - const match = exact ?? byName ?? fallback; - const assetId = match?.id?.trim(); - if (!assetId) { - await this.writeSymbolCache(normalisedSymbol, null); - continue; - } - const iconUrl = `https://static.coinpaprika.com/coin/${assetId}/logo.png`; - if (await this.urlExists(iconUrl)) { - await this.writeSymbolCache(normalisedSymbol, iconUrl); - return iconUrl; - } - - await this.writeSymbolCache(normalisedSymbol, null); - } catch (error) { - this.logger.debug({ error, symbol: candidate }, 'CoinPaprika icon lookup threw'); + const coins = payload.coins ?? []; + if (coins.length === 0) { await this.writeSymbolCache(normalisedSymbol, null); - } - } - - return null; - } - - private async fetchCoincapIcon(symbols: string[], displayName?: string): Promise { - const normalisedName = displayName?.trim().toLowerCase(); - for (const candidate of symbols) { - const normalisedSymbol = candidate.trim().toLowerCase(); - if (normalisedSymbol.length === 0) continue; - - const cached = this.symbolCache.get(normalisedSymbol); - if (typeof cached === 'string' && cached.length > 0) { - return cached; - } - - const cachedRedis = await this.readSymbolCache(normalisedSymbol); - if (typeof cachedRedis === 'string' && cachedRedis.length > 0) { - return cachedRedis; + continue; + } + + const exact = coins.find( + (coin) => coin.symbol?.trim().toLowerCase() === normalisedSymbol + ); + const byName = normalisedName + ? coins.find((coin) => coin.name?.trim().toLowerCase() === normalisedName) + : undefined; + const ranked = [...coins].sort((a, b) => { + const rankA = a.market_cap_rank ?? Number.MAX_SAFE_INTEGER; + const rankB = b.market_cap_rank ?? Number.MAX_SAFE_INTEGER; + return rankA - rankB; + }); + const fallback = ranked[0]; + const match = exact ?? byName ?? fallback; + const resolvedIcon = match?.large ?? match?.thumb; + const iconUrl = typeof resolvedIcon === 'string' ? resolvedIcon.trim() : ''; + if (iconUrl.length > 0 && (await this.urlExists(iconUrl))) { + await this.writeSymbolCache(normalisedSymbol, iconUrl); + return { iconUrl, sourceSymbol: normalisedSymbol }; } - try { - const response = await fetch( - `https://api.coincap.io/v2/assets?search=${encodeURIComponent(candidate)}`, - { - headers: { - accept: 'application/json' - } - } - ); - - if (!response.ok) { - this.logger.debug({ status: response.status, symbol: candidate }, 'Coincap lookup failed'); - continue; - } - - const payload = (await response.json()) as CoincapSearchResponse; - const assets = payload.data ?? []; - if (assets.length === 0) { - continue; - } - - const exact = assets.find( - (asset) => asset.symbol?.trim().toLowerCase() === normalisedSymbol - ); - const byName = normalisedName - ? assets.find((asset) => asset.name?.trim().toLowerCase() === normalisedName) - : undefined; - const normaliseRank = (rank: number | string | null | undefined): number => { - if (typeof rank === 'number' && Number.isFinite(rank)) return rank; - if (typeof rank === 'string') { - const parsed = Number.parseInt(rank, 10); - if (Number.isFinite(parsed)) { - return parsed; - } - } - return Number.MAX_SAFE_INTEGER; - }; - const ranked = [...assets].sort( - (a, b) => normaliseRank(a.rank) - normaliseRank(b.rank) - ); - const fallback = ranked[0]; - const match = exact ?? byName ?? fallback; - const assetId = match?.id?.trim(); - if (!assetId) { - await this.writeSymbolCache(normalisedSymbol, null); - continue; - } - const lowerId = assetId.toLowerCase(); - const candidateUrls = [ - `https://assets.coincap.io/assets/icons/${lowerId}@2x.png`, - `https://assets.coincap.io/assets/icons/${lowerId}.png` - ]; - for (const iconUrl of candidateUrls) { - if (await this.urlExists(iconUrl)) { - await this.writeSymbolCache(normalisedSymbol, iconUrl); - return iconUrl; - } - } - - await this.writeSymbolCache(normalisedSymbol, null); - } catch (error) { - this.logger.debug({ error, symbol: candidate }, 'Coincap icon lookup threw'); - await this.writeSymbolCache(normalisedSymbol, null); - } + await this.writeSymbolCache(normalisedSymbol, null); } return null; } - private async fetchCoinMarketCapIcon(symbols: string[], displayName?: string): Promise { - const normalisedName = displayName?.trim().toLowerCase(); - for (const candidate of symbols) { - const normalisedSymbol = candidate.trim().toLowerCase(); - if (normalisedSymbol.length === 0) continue; - - const cached = this.symbolCache.get(normalisedSymbol); - if (typeof cached === 'string' && cached.length > 0) { - return cached; - } - - const cachedRedis = await this.readSymbolCache(normalisedSymbol); - if (typeof cachedRedis === 'string' && cachedRedis.length > 0) { - return cachedRedis; - } - - try { - const response = await fetch( - `https://api.coinmarketcap.com/data-api/v3/search?crypto=1&limit=10&query=${encodeURIComponent(candidate)}` - ); - - if (!response.ok) { - this.logger.debug({ status: response.status, symbol: candidate }, 'CoinMarketCap lookup failed'); - continue; - } - - const payload = (await response.json()) as CoinMarketCapSearchResponse; - const cryptoEntries = payload.data?.crypto ?? []; - if (cryptoEntries.length === 0) { - continue; - } - - const exact = cryptoEntries.find( - (entry) => entry.symbol?.trim().toLowerCase() === normalisedSymbol - ); - const byName = normalisedName - ? cryptoEntries.find((entry) => entry.name?.trim().toLowerCase() === normalisedName) - : undefined; - const ranked = [...cryptoEntries].sort((a, b) => { - const rankA = typeof a.rank === 'number' && Number.isFinite(a.rank) ? a.rank : Number.MAX_SAFE_INTEGER; - const rankB = typeof b.rank === 'number' && Number.isFinite(b.rank) ? b.rank : Number.MAX_SAFE_INTEGER; - return rankA - rankB; - }); - const fallback = ranked[0]; - const match = exact ?? byName ?? fallback; - const id = match?.id; - if (typeof id !== 'number' || !Number.isFinite(id)) { - continue; - } - const iconUrl = `https://s2.coinmarketcap.com/static/img/coins/128x128/${id}.png`; - await this.writeSymbolCache(normalisedSymbol, iconUrl); - return iconUrl; - } catch (error) { - this.logger.debug({ error, symbol: candidate }, 'CoinMarketCap icon lookup threw'); - } - } - - return null; - } private async urlExists(url: string): Promise { try { diff --git a/services/gateway/src/server/http.ts b/services/gateway/src/server/http.ts index 1d13b9f5..714e9732 100644 --- a/services/gateway/src/server/http.ts +++ b/services/gateway/src/server/http.ts @@ -1,3519 +1,88 @@ -import express, { type Request, type Response } from 'express'; +import express from 'express'; import cors from 'cors'; import { createServer } from 'http'; -import Redis from 'ioredis'; -import type { RedisKeyConfig } from '@pmon/ingestion-sdk'; - -import type { - CandleSnapshot, - MarketSnapshot, - MarketStore, - PriceHistoryPoint, - TradeSummary, - BboSnapshot -} from '../store/priceStore'; -import { buildDepthSummary } from '../store/priceStore'; -import { type OverviewStore, type OverviewSnapshot } from '../store/overviewStore'; -import type { ProviderSummary } from '../config'; -import { fetchHyperliquidDepthSnapshot } from '../clients/hyperliquid'; -import { - fetchHyperliquidCandles, - HYPERLIQUID_INTERVAL_SECONDS, - hyperliquidIntervalToSeconds, - isSupportedHyperliquidInterval -} from '../clients/hyperliquidCandles'; -import { metricsRegistry, setBboAge } from '../metrics'; -import { writeHeapSnapshot } from 'node:v8'; -import type { StatsStore } from '../store/statsStore'; -import { assetStatsMetadata, overviewStatsMetadata } from '@pmon/ingestion-sdk'; +import type { MarketStore } from '../store/priceStore'; +import type { OverviewStore } from '../store/overviewStore'; import type { GatewayHealth } from '../runtime/health'; -import { evaluateGatewayReadiness } from '../runtime/readiness'; -import { isHyperliquidProvider } from '../utils/providers'; -import { fetchLeaderboard, fetchTraderProfile, type TimeWindow, type TraderRole } from '../redis/leaderboard'; -import { fetchCandlesFromRedis, persistCandlesToRedis } from '../redis/candles'; -import { getIntervalSampleLimit } from '../runtime/candles'; - -type DerivedHealthSnapshot = GatewayHealth & { - streams?: { - bbo?: Record< - string, - { - lastTimestamp?: number; - ageMs?: number; - } - >; - }; -}; - -const DEFAULT_HISTORY_LIMIT = 720; -const MAX_HISTORY_LIMIT = 5_000; -const DEFAULT_CANDLE_LIMIT = 240; -const MAX_CANDLE_LIMIT = 2_400; -const DEFAULT_TRADES_LIMIT = 200; -const MAX_TRADES_LIMIT = 1_000; -const DEFAULT_INTERVAL_KEY = '1m'; -export const createHttpServer = ( +import { createHttpContext, type HttpServerOptions } from './http/context'; +import { initializeApiDocs } from './http/docs'; +import { registerHealthAndMetricsRoutes } from './http/metrics'; +import { registerMarketRoutes } from './routes/markets'; +import { registerStatsRoutes } from './routes/stats'; +import { registerStatsLeaderboards } from './routes/leaderboards'; +import { registerTraderIntelligenceLeaderboards } from './routes/leaderboards/trader-intelligence'; +import { registerCommunityRoutes } from './routes/communities'; +import { registerTraderRoutes } from './routes/traders'; +import { registerPersonaRoutes } from './routes/personas'; +import { registerPlatformRoutes } from './routes/platform'; +import { registerAchievementRoutes } from './routes/achievements'; +import { registerInternalRoutes } from './routes/internal'; +import { readEnsMetadata } from './http/ens'; + +export const createHttpServer = async ( marketStore: MarketStore, overviewStore: OverviewStore, getHealth: () => GatewayHealth, - options?: { - providerId: string; - providerLabel?: string; - providers?: ProviderSummary[]; - statsStore?: StatsStore; - statsRedis?: { url: string; namespace: string }; - redis?: Redis; - resolveProviderRedisKeyConfig?: (providerId: string) => RedisKeyConfig | undefined; - scheduleBackfill?: (provider?: string, asset?: string | null, intervals?: string[]) => void; - } + options?: HttpServerOptions ) => { const app = express(); app.use(cors()); // Limit JSON body size to prevent memory exhaustion from large requests app.use(express.json({ limit: '100kb' })); - const defaultProvider = options?.providerId; - const providers = options?.providers ?? []; - const statsStore = options?.statsStore; - const statsRedis = options?.statsRedis; - const redis = options?.redis; - const resolveProviderRedisKeyConfig = options?.resolveProviderRedisKeyConfig; - const scheduleBackfill = options?.scheduleBackfill; - - // Create stats Redis client for leaderboard queries if available - let statsRedisClient: Redis | null = null; - if (statsRedis) { - statsRedisClient = new Redis(statsRedis.url, { - retryStrategy: (times) => Math.min(times * 500, 5000), - maxRetriesPerRequest: 3 - }); - } - - // Create trader intelligence Redis client if Redis is available - // Trader intelligence uses 'intelligence:' prefix by default - const traderIntelligenceRedis = redis || (process.env.TRADER_INTEL_REDIS_URL ? new Redis(process.env.TRADER_INTEL_REDIS_URL, { - retryStrategy: (times) => Math.min(times * 500, 5000), - maxRetriesPerRequest: 3 - }) : null); - const traderIntelligenceKeyPrefix = process.env.REDIS_KEY_PREFIX || process.env.TRADER_INTEL_REDIS_KEY_PREFIX || 'intelligence:'; - - const readEnsMetadata = async ( - address: string - ): Promise<{ ensName?: string; ensAvatar?: string }> => { - if (!address || !traderIntelligenceRedis) { - return {}; - } - - const ensKey = `${traderIntelligenceKeyPrefix}ens:${address}`; - - try { - const hash = await traderIntelligenceRedis.hgetall(ensKey); - if (hash && Object.keys(hash).length > 0) { - return { - ensName: hash.ensName, - ensAvatar: hash.ensAvatar, - }; - } - } catch { - // Ignore hash fetch failures and fall back to GET - } - - try { - const raw = await traderIntelligenceRedis.get(ensKey); - if (raw) { - const parsed = JSON.parse(raw) as { ensName?: string; ensAvatar?: string }; - return { - ensName: parsed?.ensName, - ensAvatar: parsed?.ensAvatar, - }; - } - } catch { - // Ignore JSON parsing failures - } - - return {}; - }; - const persistCandles = async ( - provider: string | undefined, - asset: string, - intervalKey: string, - candles: CandleSnapshot[] - ) => { - if (!provider || candles.length === 0 || !redis || !resolveProviderRedisKeyConfig) { - return; - } - const keyConfig = resolveProviderRedisKeyConfig(provider); - if (!keyConfig) { - return; - } - marketStore.hydrateCandles(provider, asset, candles); - await persistCandlesToRedis({ - redis, - keyConfig, - provider, - asset, - interval: intervalKey, - candles - }); - }; - - const maybeScheduleBackfill = ( - provider: string | undefined, - asset: string | undefined, - intervalKey: string, - currentLength: number, - limit: number - ) => { - if (!scheduleBackfill || !provider || !asset) { - return; - } - if (!isHyperliquidProvider(provider)) { - return; - } - const expected = Math.min(limit, getIntervalSampleLimit(intervalKey)); - if (currentLength >= expected * 0.6) { - return; - } - scheduleBackfill(provider, asset, [intervalKey]); - }; - - const inflightRemoteCandleFetches = new Map>(); - const inflightRemoteCandleFetchTimestamps = new Map(); - - // Cleanup stale remote candle fetches (older than 5 minutes) - const cleanupStaleCandleFetches = () => { - const now = Date.now(); - const STALE_FETCH_TTL_MS = 5 * 60 * 1000; // 5 minutes - for (const [key, timestamp] of inflightRemoteCandleFetchTimestamps.entries()) { - if (now - timestamp > STALE_FETCH_TTL_MS) { - inflightRemoteCandleFetches.delete(key); - inflightRemoteCandleFetchTimestamps.delete(key); - } - } - }; - - // Periodic cleanup of stale fetches (every 2 minutes) - const candleFetchCleanupInterval = setInterval(cleanupStaleCandleFetches, 2 * 60 * 1000); - - const hydrateCandlesFromRedis = async ( - provider: string | undefined, - asset: string, - intervalKey: string - ): Promise => { - if (!provider || !redis || !resolveProviderRedisKeyConfig) { - return false; - } - const keyConfig = resolveProviderRedisKeyConfig(provider); - if (!keyConfig) { - return false; - } - try { - const stored = await fetchCandlesFromRedis({ - redis, - keyConfig, - provider, - asset, - interval: intervalKey - }); - if (stored.length === 0) { - return false; - } - marketStore.hydrateCandles(provider, asset, stored); - return true; - } catch { - return false; - } - }; - - const ensureHyperliquidCandles = async ( - provider: string | undefined, - asset: string, - intervalKey: string, - fetchLimit: number - ): Promise => { - if (!provider || !isHyperliquidProvider(provider)) { - return false; - } - const inflightKey = `${provider}:${asset}:${intervalKey}`; - const existing = inflightRemoteCandleFetches.get(inflightKey); - if (existing) { - return existing; - } - const task = (async () => { - const candles = await fetchHyperliquidCandles(asset, intervalKey, fetchLimit).catch(() => []); - if (candles.length === 0) { - return false; - } - await persistCandles(provider, asset, intervalKey, candles); - return true; - })() - .catch(() => false) - .finally(() => { - inflightRemoteCandleFetches.delete(inflightKey); - inflightRemoteCandleFetchTimestamps.delete(inflightKey); - }); - inflightRemoteCandleFetches.set(inflightKey, task); - inflightRemoteCandleFetchTimestamps.set(inflightKey, Date.now()); - return task; - }; - - const ensureCandlesForRequest = async ( - provider: string | undefined, - asset: string, - interval: { key: string; seconds: number }, - limit: number - ): Promise => { - if (!provider) { - return []; - } - const expected = Math.min(limit, getIntervalSampleLimit(interval.key)); - const fetchLimit = Math.max(limit, getIntervalSampleLimit(interval.key)); - - const readCandles = () => marketStore.getCandles(provider, asset, interval.seconds, limit); - - let candles = readCandles(); - if (candles.length >= expected) { - return candles; - } - - const hydrated = await hydrateCandlesFromRedis(provider, asset, interval.key); - if (hydrated) { - candles = readCandles(); - if (candles.length >= expected) { - return candles; - } - } - - await ensureHyperliquidCandles(provider, asset, interval.key, fetchLimit); - return readCandles(); - }; - - const resolveSnapshotPrice = (snapshot: MarketSnapshot | null): number | null => { - if (!snapshot) return null; - if (typeof snapshot.price === 'number' && Number.isFinite(snapshot.price)) { - return snapshot.price; - } - const candidates = [ - snapshot.metadata?.markPrice, - snapshot.metadata?.midPrice, - snapshot.metadata?.oraclePrice - ]; - for (const candidate of candidates) { - if (typeof candidate === 'number' && Number.isFinite(candidate)) { - return candidate; - } - } - return null; - }; - - const getHistoryWithFallback = (provider: string, asset: string, limit: number): PriceHistoryPoint[] => { - const history = marketStore.getPriceHistory(provider, asset, limit); - if (history.length > 0) { - return history; - } - const snapshot = marketStore.get(provider, asset); - const fallbackPrice = resolveSnapshotPrice(snapshot); - if (snapshot?.asset && fallbackPrice !== null) { - const timestamp = snapshot.priceTimestamp ?? Date.now(); - return [{ timestamp, price: fallbackPrice }]; - } - return []; - }; - - const buildHealthSnapshot = () => { - const health = getHealth(); - const now = Date.now(); - const bboStates = health.streams?.bbo ?? {}; - const derivedBbo: Record = {}; - let defaultProviderAge: number | undefined; - - for (const [provider, state] of Object.entries(bboStates)) { - const lastTimestamp = state?.lastTimestamp; - const ageMs = - typeof lastTimestamp === 'number' && Number.isFinite(lastTimestamp) - ? Math.max(0, now - lastTimestamp) - : undefined; - derivedBbo[provider] = { - lastTimestamp, - ageMs - }; - if (defaultProvider && provider.toLowerCase() === defaultProvider.toLowerCase()) { - defaultProviderAge = ageMs; - } - } - - setBboAge(defaultProviderAge); - - const healthWithDerived: DerivedHealthSnapshot = { - ...health, - streams: { - ...health.streams, - bbo: derivedBbo - } - }; - const evaluation = evaluateGatewayReadiness(healthWithDerived); - - return { health: healthWithDerived, evaluation }; - }; - - app.get('/healthz', (_req: Request, res: Response) => { - const { health, evaluation } = buildHealthSnapshot(); - const { fatalReasons, warnings } = evaluation; - - // Liveness: as long as the process is running and can respond, we return 200. - // We still expose dependency issues in the payload so monitoring can alert - // without causing crash loops. - res.status(200).json({ - status: 'ok', - reasons: fatalReasons.length > 0 ? fatalReasons : undefined, - warnings: warnings.length > 0 ? warnings : undefined, - health - }); - }); - - app.get('/readyz', (_req: Request, res: Response) => { - const { health, evaluation } = buildHealthSnapshot(); - const { ready, fatalReasons, warnings } = evaluation; - res.status(ready ? 200 : 503).json({ - status: ready ? 'ready' : 'degraded', - reasons: ready ? undefined : fatalReasons, - warnings: warnings.length > 0 ? warnings : undefined, - health - }); - }); - - if ((process.env.GATEWAY_ENABLE_HEAPDUMP_ENDPOINT ?? '').trim().toLowerCase() === 'true') { - app.post('/admin/heapdump', (_req: Request, res: Response) => { - try { - const filename = writeHeapSnapshot(`/tmp/gateway-heap-${Date.now()}.heapsnapshot`); - res.json({ - status: 'ok', - heapSnapshot: filename - }); - } catch (error) { - res.status(500).json({ - status: 'error', - message: error instanceof Error ? error.message : String(error) - }); - } - }); - } - - app.get('/api/v1/markets', (_req: Request, res: Response) => { - if (!defaultProvider) { - res.status(400).json({ error: 'provider_not_configured' }); - return; - } - const data: MarketSnapshot[] = marketStore.snapshot(defaultProvider); - res.json({ data }); - }); - - app.get('/api/v1/markets/:asset', (req: Request, res: Response) => { - const { asset } = req.params; - if (!defaultProvider) { - res.status(400).json({ error: 'provider_not_configured' }); - return; - } - const snapshot = marketStore.get(defaultProvider, asset); - if (!snapshot) { - res.status(404).json({ error: 'asset_not_found', asset }); - return; - } - res.json({ data: snapshot }); - }); - - app.get('/api/v1/markets/:asset/metadata', (req: Request, res: Response) => { - const { asset } = req.params; - if (!defaultProvider) { - res.status(400).json({ error: 'provider_not_configured' }); - return; - } - const metadata = marketStore.getMetadata(defaultProvider, asset); - if (!metadata) { - res.status(404).json({ error: 'metadata_not_found', asset }); - return; - } - res.json({ data: metadata }); - }); - - app.get('/api/v1/markets/:asset/price-history', (req: Request, res: Response) => { - const { asset } = req.params; - if (!defaultProvider) { - res.status(400).json({ error: 'provider_not_configured' }); - return; - } - const limit = resolveLimitQuery(req.query.limit, DEFAULT_HISTORY_LIMIT, MAX_HISTORY_LIMIT); - const history = getHistoryWithFallback(defaultProvider, asset, limit); - if (history.length === 0) { - res.status(404).json({ error: 'price_history_not_found', asset, provider: defaultProvider }); - return; - } - res.json({ data: history, provider: defaultProvider, asset, limit }); - }); - - app.get('/api/v1/markets/:asset/candles', async (req: Request, res: Response) => { - const { asset } = req.params; - if (!defaultProvider) { - res.status(400).json({ error: 'provider_not_configured' }); - return; - } - const limit = resolveLimitQuery(req.query.limit, DEFAULT_CANDLE_LIMIT, MAX_CANDLE_LIMIT); - const interval = resolveIntervalQuery(req.query.interval ?? req.query.resolution, DEFAULT_INTERVAL_KEY); - const snapshot = marketStore.get(defaultProvider, asset); - const resolvedAsset = snapshot?.asset ?? asset; - - const candles = await ensureCandlesForRequest(defaultProvider, resolvedAsset, interval, limit); - - if (candles.length === 0) { - res.status(404).json({ error: 'candles_not_found', asset, provider: defaultProvider }); - return; - } - maybeScheduleBackfill(defaultProvider, resolvedAsset, interval.key, candles.length, limit); + // Generate Swagger spec + const { context, cleanup: disposeHttpContext } = createHttpContext({ + app, + marketStore, + overviewStore, + getHealth, + options + }); + + await initializeApiDocs(context, { endpointsFiles: [__filename] }); + registerHealthAndMetricsRoutes(context); + const routeCleanupFns: Array<() => void> = []; + routeCleanupFns.push(registerMarketRoutes(context)); + registerStatsRoutes(context); + registerStatsLeaderboards(context); + registerTraderIntelligenceLeaderboards(context); + + app.get('/', (_req, res) => { res.json({ - data: candles, - provider: defaultProvider, - asset: resolvedAsset, - interval: interval.key, - resolutionSeconds: interval.seconds + service: 'gateway', + docs: '/docs' }); }); - app.get('/api/v1/markets/:asset/trades', (req: Request, res: Response) => { - const { asset } = req.params; - if (!defaultProvider) { - res.status(400).json({ error: 'provider_not_configured' }); - return; - } - const limit = resolveLimitQuery(req.query.limit, DEFAULT_TRADES_LIMIT, MAX_TRADES_LIMIT); - const trades: TradeSummary[] = marketStore.getTrades(defaultProvider, asset, limit); - res.json({ data: trades, provider: defaultProvider, asset, limit }); - }); - - app.get('/api/v1/markets/:asset/bbo', (req: Request, res: Response) => { - const { asset } = req.params; - if (!defaultProvider) { - res.status(400).json({ error: 'provider_not_configured' }); - return; - } - const bbo: BboSnapshot | null = marketStore.getBbo(defaultProvider, asset); - res.json({ data: bbo, provider: defaultProvider, asset }); - }); - - app.get('/api/v1/markets/:asset/depth', async (req: Request, res: Response) => { - const { asset } = req.params; - if (!defaultProvider) { - res.status(400).json({ error: 'provider_not_configured' }); - return; - } - const depth = resolveDepthQuery(req.query.depth); - await respondWithDepth(defaultProvider, asset, depth, res); - }); - - app.get('/api/v1/overview', (_req: Request, res: Response) => { - if (!defaultProvider) { - res.status(400).json({ error: 'provider_not_configured' }); - return; - } - const data: OverviewSnapshot | null = overviewStore.snapshot(defaultProvider); - res.json({ data }); - }); - - app.get('/api/v1/providers/:provider/markets', (req: Request, res: Response) => { - const provider = req.params.provider; - const data = marketStore.snapshot(provider); - res.json({ data, provider }); - }); - - app.get('/api/v1/providers/:provider/markets/:asset', (req: Request, res: Response) => { - const { provider, asset } = req.params; - const snapshot = marketStore.get(provider, asset); - if (!snapshot) { - res.status(404).json({ error: 'asset_not_found', asset, provider }); - return; - } - res.json({ data: snapshot, provider }); - }); - - app.get('/api/v1/providers/:provider/markets/:asset/metadata', (req: Request, res: Response) => { - const { provider, asset } = req.params; - const metadata = marketStore.getMetadata(provider, asset); - if (!metadata) { - res.status(404).json({ error: 'metadata_not_found', asset, provider }); - return; - } - res.json({ data: metadata, provider }); - }); - - app.get('/api/v1/providers/:provider/markets/:asset/price-history', (req: Request, res: Response) => { - const { provider, asset } = req.params; - const limit = resolveLimitQuery(req.query.limit, DEFAULT_HISTORY_LIMIT, MAX_HISTORY_LIMIT); - const history = getHistoryWithFallback(provider, asset, limit); - if (history.length === 0) { - res.status(404).json({ error: 'price_history_not_found', asset, provider }); - return; - } - res.json({ data: history, provider, asset, limit }); - }); - - app.get('/api/v1/providers/:provider/markets/:asset/candles', async (req: Request, res: Response) => { - const { provider, asset } = req.params; - const limit = resolveLimitQuery(req.query.limit, DEFAULT_CANDLE_LIMIT, MAX_CANDLE_LIMIT); - const interval = resolveIntervalQuery(req.query.interval ?? req.query.resolution, DEFAULT_INTERVAL_KEY); - const snapshot = marketStore.get(provider, asset); - const resolvedAsset = snapshot?.asset ?? asset; - - const candles = await ensureCandlesForRequest(provider, resolvedAsset, interval, limit); - - if (candles.length === 0) { - res.status(404).json({ error: 'candles_not_found', asset, provider }); - return; - } - maybeScheduleBackfill(provider, resolvedAsset, interval.key, candles.length, limit); - res.json({ data: candles, provider, asset: resolvedAsset, interval: interval.key, resolutionSeconds: interval.seconds }); - }); - - app.get('/api/v1/providers/:provider/markets/:asset/trades', (req: Request, res: Response) => { - const { provider, asset } = req.params; - const limit = resolveLimitQuery(req.query.limit, DEFAULT_TRADES_LIMIT, MAX_TRADES_LIMIT); - const trades: TradeSummary[] = marketStore.getTrades(provider, asset, limit); - if (trades.length === 0) { - res.status(404).json({ error: 'trades_not_found', asset, provider }); - return; - } - res.json({ data: trades, provider, asset, limit }); - }); - - app.get('/api/v1/providers/:provider/markets/:asset/bbo', (req: Request, res: Response) => { - const { provider, asset } = req.params; - const bbo: BboSnapshot | null = marketStore.getBbo(provider, asset); - if (!bbo) { - res.status(404).json({ error: 'bbo_not_found', asset, provider }); - return; - } - res.json({ data: bbo, provider, asset }); - }); - - const respondWithDepth = async (provider: string, asset: string, depth: number, res: Response) => { - try { - const cached = marketStore.getDepthSummary(provider, asset, depth); - if (cached) { - res.json({ data: cached, provider }); - return; - } - - const metadata = marketStore.getMetadata(provider, asset); - const snapshot = await fetchHyperliquidDepthSnapshot(asset, depth); - - if (snapshot) { - const summary = buildDepthSummary( - provider, - asset, - depth, - snapshot.bids, - snapshot.asks, - snapshot.timestamp, - metadata?.bestBid, - metadata?.bestAsk - ); - - res.json({ data: summary, provider, source: 'hyperliquid-rest' }); - return; - } - - res.status(404).json({ error: 'depth_not_found', asset, provider }); - } catch (error) { - res.status(500).json({ - error: 'depth_lookup_failed', - asset, - provider, - detail: error instanceof Error ? error.message : String(error) - }); - } + // Create ENS metadata reader helper + const readEnsMetadataHelper = async (address: string) => { + return readEnsMetadata( + context.traderIntelligenceRedis, + context.traderIntelligenceKeyPrefix, + address + ); }; + + registerCommunityRoutes(context, { readEnsMetadata: readEnsMetadataHelper }); + registerTraderRoutes(context); + registerPersonaRoutes(context); + registerPlatformRoutes(context); + registerAchievementRoutes(context); + registerInternalRoutes(context); - app.get('/api/v1/providers/:provider/markets/:asset/depth', async (req: Request, res: Response) => { - const { provider, asset } = req.params; - const depth = resolveDepthQuery(req.query.depth); - await respondWithDepth(provider, asset, depth, res); - }); - - app.get('/api/v1/providers/:provider/overview', (req: Request, res: Response) => { - const provider = req.params.provider; - const data = overviewStore.snapshot(provider); - res.json({ data, provider }); - }); - - app.get('/api/v1/providers', (_req: Request, res: Response) => { - const data = providers.map((provider) => ({ - id: provider.id, - displayName: provider.displayName, - description: provider.description, - tags: provider.tags - })); - res.json({ data, defaultProvider }); - }); - - if (statsStore) { - const respondWithAssetStats = (provider: string, asset: string, res: Response) => { - const stats = statsStore.getAsset(provider, asset); - if (!stats) { - res.status(404).json({ error: 'stats_not_found', provider, asset }); - return; - } - res.json({ data: stats, provider, asset, meta: assetStatsMetadata }); - }; - - const respondWithOverview = (provider: string, res: Response) => { - const overview = statsStore.getOverview(provider); - if (!overview) { - res.status(404).json({ error: 'stats_overview_not_found', provider }); - return; - } - res.json({ data: overview, provider, meta: overviewStatsMetadata }); - }; - - app.get('/api/v1/stats/overview', (_req: Request, res: Response) => { - if (!defaultProvider) { - res.status(400).json({ error: 'provider_not_configured' }); - return; - } - respondWithOverview(defaultProvider, res); - }); + const httpServer = createServer(app); - app.get('/api/v1/stats/:asset', (req: Request, res: Response) => { - if (!defaultProvider) { - res.status(400).json({ error: 'provider_not_configured' }); - return; + // Return cleanup function to clear intervals on shutdown + const cleanup = () => { + for (const fn of routeCleanupFns) { + try { + fn(); + } catch { + // ignore cleanup errors } - respondWithAssetStats(defaultProvider, req.params.asset, res); - }); - - app.get('/api/v1/providers/:provider/stats/overview', (req: Request, res: Response) => { - respondWithOverview(req.params.provider, res); - }); - - app.get('/api/v1/providers/:provider/stats/:asset', (req: Request, res: Response) => { - respondWithAssetStats(req.params.provider, req.params.asset, res); - }); - } - - const resolveWindow = (raw: unknown): TimeWindow => { - if (typeof raw === 'string' && (raw === '1h' || raw === '24h' || raw === '7d')) { - return raw as TimeWindow; - } - return '24h'; - }; - - const resolveRole = (raw: unknown): TraderRole => { - if (typeof raw === 'string' && (raw === 'maker' || raw === 'taker')) { - return raw as TraderRole; - } - return 'maker'; - }; - - // Leaderboard endpoints - if (statsRedisClient && statsRedis) { - // Global leaderboards - app.get('/api/v1/leaderboard/:role', async (req: Request, res: Response) => { - if (!defaultProvider) { - res.status(400).json({ error: 'provider_not_configured' }); - return; - } - const role = resolveRole(req.params.role); - const window = resolveWindow(req.query.window); - const limit = resolveLimitQuery(req.query.limit, 50, 100); - - try { - const entries = await fetchLeaderboard(statsRedisClient!, statsRedis.namespace, defaultProvider, 'global', role, window, limit); - const totalVolume = entries.reduce((sum, e) => sum + e.totalVolume, 0); - - res.json({ - data: { - window, - provider: defaultProvider, - totalVolume, - entries, - updatedAt: Date.now() - } - }); - } catch (error) { - res.status(500).json({ - error: 'leaderboard_fetch_failed', - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - // Asset-specific leaderboards - app.get('/api/v1/providers/:provider/assets/:asset/leaderboard/:role', async (req: Request, res: Response) => { - const { provider, asset, role: roleParam } = req.params; - const role = resolveRole(roleParam); - const window = resolveWindow(req.query.window); - const limit = resolveLimitQuery(req.query.limit, 50, 100); - - try { - const entries = await fetchLeaderboard(statsRedisClient!, statsRedis.namespace, provider, asset, role, window, limit); - const totalVolume = entries.reduce((sum, e) => sum + e.totalVolume, 0); - - res.json({ - data: { - window, - provider, - asset, - totalVolume, - entries, - updatedAt: Date.now() - } - }); - } catch (error) { - res.status(500).json({ - error: 'leaderboard_fetch_failed', - asset, - provider, - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - // Trader profile - app.get('/api/v1/traders/:address/stats', async (req: Request, res: Response) => { - const { address } = req.params; - const window = resolveWindow(req.query.window); - - try { - const profile = await fetchTraderProfile(statsRedisClient!, statsRedis.namespace, address, window); - if (!profile) { - res.status(404).json({ error: 'trader_not_found', address }); - return; - } - - res.json({ data: profile, window }); - } catch (error) { - res.status(500).json({ - error: 'trader_profile_fetch_failed', - address, - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - } - - // Trader Intelligence API endpoints - if (traderIntelligenceRedis) { - // Fetch traders by persona - app.get('/api/v1/personas/:persona/traders', async (req: Request, res: Response) => { - const { persona } = req.params; - const limit = resolveLimitQuery(req.query.limit, 50, 100); - - try { - const { fetchPersonaMembers } = await import('../redis/leaderboard'); - const members = await fetchPersonaMembers(traderIntelligenceRedis, traderIntelligenceKeyPrefix, persona, limit); - res.json({ data: members, persona, limit }); - } catch (error) { - res.status(500).json({ - error: 'persona_members_fetch_failed', - persona, - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - // Fetch viral achievements for a trader - app.get('/api/v1/achievements/:address', async (req: Request, res: Response) => { - const { address } = req.params; - - try { - const { fetchViralAchievements } = await import('../redis/leaderboard'); - const achievements = await fetchViralAchievements(traderIntelligenceRedis, traderIntelligenceKeyPrefix, address); - res.json({ data: achievements, address }); - } catch (error) { - res.status(500).json({ - error: 'achievements_fetch_failed', - address, - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - // Fetch shame leaderboards - app.get('/api/v1/leaderboard/shame/:type', async (req: Request, res: Response) => { - const { type } = req.params; - const window = resolveWindow(req.query.window); - const limit = resolveLimitQuery(req.query.limit, 50, 100); - - try { - const { fetchShameLeaderboard } = await import('../redis/leaderboard'); - const entries = await fetchShameLeaderboard(traderIntelligenceRedis, traderIntelligenceKeyPrefix, type, window, limit); - res.json({ - data: { - window, - type, - entries, - updatedAt: Date.now() - } - }); - } catch (error) { - res.status(500).json({ - error: 'shame_leaderboard_fetch_failed', - type, - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - } - - // Asset Community API endpoints - if (traderIntelligenceRedis) { - // Get Asset Community Overview - app.get('/api/v1/communities/:asset', async (req: Request, res: Response) => { - const { asset } = req.params; - - try { - - // Asset Community API endpoints - const statsKey = `${traderIntelligenceKeyPrefix}community:${asset}:stats`; - const statsData = await traderIntelligenceRedis.hgetall(statsKey); - - if (!statsData || Object.keys(statsData).length === 0) { - res.status(404).json({ error: 'ASSET_NOT_FOUND', message: `Asset ${asset} not found` }); - return; - } - - // Get top trader ENS name if available - const topTraderAddress = statsData.topTraderAddress || ''; - const topTraderEns = await readEnsMetadata(topTraderAddress); - - res.json({ - asset, - displayName: statsData.displayName || asset, - emoji: statsData.emoji || '💰', - color: statsData.color || '#000000', - stats: { - totalTraders: parseInt(statsData.totalTraders || '0', 10), - activeTraders24h: parseInt(statsData.activeTraders24h || '0', 10), - totalVolume24h: parseFloat(statsData.totalVolume24h || '0'), - totalVolumeAllTime: parseFloat(statsData.totalVolumeAllTime || '0') - }, - topTrader: { - address: topTraderAddress, - ensName: topTraderEns.ensName, - points: parseInt(statsData.topTraderPoints || '0', 10), - badges: parseInt(statsData.topTraderBadges || '0', 10) - }, - topVolume: { - address: statsData.topVolumeAddress || '', - volume24h: parseFloat(statsData.topVolume24h || '0') - }, - topPnL: { - address: statsData.topPnLAddress || '', - pnl24h: parseFloat(statsData.topPnL24h || '0') - }, - sentiment: (statsData.sentiment || 'neutral') as 'bullish' | 'bearish' | 'neutral', - averageLeverage: parseFloat(statsData.averageLeverage || '0'), - longShortRatio: parseFloat(statsData.longShortRatio || '1.0') - }); - } catch (error) { - res.status(500).json({ - error: 'INTERNAL_ERROR', - message: 'Failed to fetch asset community overview', - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - // Get Asset Leaderboard - app.get('/api/v1/communities/:asset/leaderboard/:type', async (req: Request, res: Response) => { - const { asset, type } = req.params; - const window = (req.query.window as string) || '24h'; - const limit = resolveLimitQuery(req.query.limit, 50, 100); - const offset = resolveLimitQuery(req.query.offset, 0, 1000); - - if (!['points', 'volume', 'pnl', 'loyalty'].includes(type)) { - res.status(400).json({ error: 'INVALID_LEADERBOARD_TYPE', message: `Invalid leaderboard type: ${type}` }); - return; - } - - if (!['24h', '7d', '30d', 'all'].includes(window)) { - res.status(400).json({ error: 'INVALID_WINDOW', message: `Invalid window: ${window}` }); - return; - } - - try { - const leaderboardKey = `${traderIntelligenceKeyPrefix}leaderboard:asset:${asset}:${type}:${window}`; - const totalTraders = await traderIntelligenceRedis.zcard(leaderboardKey); - - // Get leaderboard entries with scores (ranked) - const entriesWithScores = await traderIntelligenceRedis.zrevrange(leaderboardKey, offset, offset + limit - 1, 'WITHSCORES'); - - const entries = []; - for (let i = 0; i < entriesWithScores.length; i += 2) { - const address = entriesWithScores[i]; - const _score = parseFloat(entriesWithScores[i + 1] || '0'); - const rank = offset + (i / 2) + 1; - - // Get trader details - const pointsKey = `${traderIntelligenceKeyPrefix}trader:${address}:asset-points:${asset}`; - const breakdownKey = `${traderIntelligenceKeyPrefix}trader:${address}:asset-breakdown:${asset}`; - const loyaltyKey = `${traderIntelligenceKeyPrefix}trader:${address}:asset-loyalty:${asset}`; - const badgesKey = `${traderIntelligenceKeyPrefix}trader:${address}:asset-badges:${asset}`; - const ensKey = `${traderIntelligenceKeyPrefix}ens:${address}`; - - const [points, breakdownJson, loyaltyTier, badgeIds, ensData] = await Promise.all([ - traderIntelligenceRedis.get(pointsKey), - traderIntelligenceRedis.get(breakdownKey), - traderIntelligenceRedis.get(loyaltyKey), - traderIntelligenceRedis.smembers(badgesKey), - traderIntelligenceRedis.get(ensKey) - ]); - - let ensName: string | undefined; - if (ensData) { - try { - const parsed = JSON.parse(ensData); - ensName = parsed.ensName; - } catch { - // Ignore parse errors - } - } - - const breakdown = breakdownJson ? JSON.parse(breakdownJson) : null; - - // Get badge details (limit to 5 for response size) - const badges = []; - for (const badgeId of badgeIds.slice(0, 5)) { - const badgeKey = `${traderIntelligenceKeyPrefix}badge:${badgeId}:${address}`; - const badgeData = await traderIntelligenceRedis.hgetall(badgeKey); - if (badgeData && badgeData.id) { - badges.push({ - id: badgeData.id, - emoji: badgeData.emoji || '🏆', - rarity: badgeData.rarity || 'common' - }); - } - } - - entries.push({ - rank, - address, - ensName, - assetPoints: parseInt(points || '0', 10), - assetVolume: breakdown?.components?.volume || 0, - assetPnl: breakdown?.components?.pnl || 0, - loyaltyTier: loyaltyTier || 'casual', - assetBadges: badges, - tradeCount: 0, // Would need to query from history - winRate: 0, // Would need to query from PnL data - daysActive: 0 // Would need to calculate from history - }); - } - - res.json({ - asset, - type, - window, - entries, - totalTraders, - pagination: { - limit, - offset, - hasMore: offset + limit < totalTraders - }, - updatedAt: Date.now() - }); - } catch (error) { - res.status(500).json({ - error: 'INTERNAL_ERROR', - message: 'Failed to fetch leaderboard', - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - // Get Trader's Asset Profile - app.get('/api/v1/traders/:address/assets/:asset', async (req: Request, res: Response) => { - const { address, asset } = req.params; - const normalizedAddress = address.toLowerCase(); - - try { - const pointsKey = `${traderIntelligenceKeyPrefix}trader:${normalizedAddress}:asset-points:${asset}`; - const breakdownKey = `${traderIntelligenceKeyPrefix}trader:${normalizedAddress}:asset-breakdown:${asset}`; - const loyaltyKey = `${traderIntelligenceKeyPrefix}trader:${normalizedAddress}:asset-loyalty:${asset}`; - const badgesKey = `${traderIntelligenceKeyPrefix}trader:${normalizedAddress}:asset-badges:${asset}`; - const streakKey = `${traderIntelligenceKeyPrefix}trader:${normalizedAddress}:asset-streak:${asset}`; - const leaderboardKey = `${traderIntelligenceKeyPrefix}leaderboard:asset:${asset}:points:24h`; - - const [points, breakdownJson, loyaltyTier, badgeIds, streakData, rank] = await Promise.all([ - traderIntelligenceRedis.get(pointsKey), - traderIntelligenceRedis.get(breakdownKey), - traderIntelligenceRedis.get(loyaltyKey), - traderIntelligenceRedis.smembers(badgesKey), - traderIntelligenceRedis.hgetall(streakKey), - traderIntelligenceRedis.zrevrank(leaderboardKey, normalizedAddress) - ]); - - if (!points && !breakdownJson) { - res.status(404).json({ error: 'TRADER_NOT_FOUND', message: `Trader ${address} not found for asset ${asset}` }); - return; - } - - const breakdown = breakdownJson ? JSON.parse(breakdownJson) : null; - const totalTraders = await traderIntelligenceRedis.zcard(leaderboardKey); - const percentile = rank !== null && totalTraders > 0 ? ((totalTraders - rank) / totalTraders) * 100 : 0; - - // Get badge details - const badges = []; - for (const badgeId of badgeIds) { - const badgeKey = `${traderIntelligenceKeyPrefix}badge:${badgeId}:${normalizedAddress}`; - const badgeData = await traderIntelligenceRedis.hgetall(badgeKey); - if (badgeData && badgeData.id) { - badges.push({ - id: badgeData.id, - name: badgeData.name || '', - emoji: badgeData.emoji || '🏆', - rarity: badgeData.rarity || 'common', - unlockedAt: badgeData.unlockedAt ? parseInt(badgeData.unlockedAt, 10) : undefined - }); - } - } - - res.json({ - address: normalizedAddress, - asset, - points: { - total: parseInt(points || '0', 10), - breakdown: breakdown || { - total: 0, - components: { volume: 0, pnl: 0, loyalty: 0, streaks: 0, badges: 0, events: 0 }, - multipliers: { loyalty: 1.0, streak: 1.0, exclusiveFocus: 1.0 } - }, - rank: rank !== null ? rank + 1 : null, - percentile - }, - loyalty: { - tier: loyaltyTier || 'casual', - progress: 1.0, // Would need to calculate based on trade count - metrics: { - totalTrades: 0, // Would need to query from history - totalVolume: breakdown?.components?.volume || 0, - daysActive: 0, - consecutiveDays: parseInt(streakData?.consecutiveDays || '0', 10) - } - }, - performance: { - volume24h: breakdown?.components?.volume || 0, - volumeAllTime: breakdown?.components?.volume || 0, - pnl24h: breakdown?.components?.pnl || 0, - pnlAllTime: breakdown?.components?.pnl || 0, - winRate: 0, - largestWin: 0, - largestLoss: 0 - }, - badges, - streaks: { - currentWin: parseInt(streakData?.currentWinStreak || '0', 10), - currentLoss: parseInt(streakData?.currentLossStreak || '0', 10), - longestWin: parseInt(streakData?.longestWinStreak || '0', 10), - consecutiveDays: parseInt(streakData?.consecutiveDays || '0', 10) - }, - lastTradeAt: undefined, - firstTradeAt: undefined - }); - } catch (error) { - res.status(500).json({ - error: 'INTERNAL_ERROR', - message: 'Failed to fetch trader asset profile', - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - // Get Trader's All Assets Summary - app.get('/api/v1/traders/:address/assets', async (req: Request, res: Response) => { - const { address } = req.params; - const normalizedAddress = address.toLowerCase(); - - try { - // Find all assets this trader has points in - const pattern = `${traderIntelligenceKeyPrefix}trader:${normalizedAddress}:asset-points:*`; - const keys = await traderIntelligenceRedis.keys(pattern); - - const assets = []; - let totalPoints = 0; - let primaryAsset = ''; - let maxPoints = 0; - - for (const key of keys) { - const assetMatch = key.match(/asset-points:(.+)$/); - if (!assetMatch) continue; - - const asset = assetMatch[1]; - const pointsKey = `${traderIntelligenceKeyPrefix}trader:${normalizedAddress}:asset-points:${asset}`; - const loyaltyKey = `${traderIntelligenceKeyPrefix}trader:${normalizedAddress}:asset-loyalty:${asset}`; - const badgesKey = `${traderIntelligenceKeyPrefix}trader:${normalizedAddress}:asset-badges:${asset}`; - const leaderboardKey = `${traderIntelligenceKeyPrefix}leaderboard:asset:${asset}:points:24h`; - const breakdownKey = `${traderIntelligenceKeyPrefix}trader:${normalizedAddress}:asset-breakdown:${asset}`; - - const [points, loyaltyTier, badgeCount, rank, breakdownJson] = await Promise.all([ - traderIntelligenceRedis.get(pointsKey), - traderIntelligenceRedis.get(loyaltyKey), - traderIntelligenceRedis.scard(badgesKey), - traderIntelligenceRedis.zrevrank(leaderboardKey, normalizedAddress), - traderIntelligenceRedis.get(breakdownKey) - ]); - - const pointsValue = parseInt(points || '0', 10); - totalPoints += pointsValue; - - if (pointsValue > maxPoints) { - maxPoints = pointsValue; - primaryAsset = asset; - } - - const breakdown = breakdownJson ? JSON.parse(breakdownJson) : null; - - assets.push({ - asset, - points: pointsValue, - rank: rank !== null ? rank + 1 : null, - loyaltyTier: loyaltyTier || 'casual', - badges: badgeCount, - volume24h: breakdown?.components?.volume || 0, - pnl24h: breakdown?.components?.pnl || 0 - }); - } - - // Sort by points descending - assets.sort((a, b) => b.points - a.points); - - // Calculate diversification score (1 = fully diversified, 0 = single asset) - const diversificationScore = assets.length > 1 - ? 1 - (maxPoints / totalPoints) - : 0; - - res.json({ - address: normalizedAddress, - assets, - totalAssets: assets.length, - primaryAsset, - diversificationScore - }); - } catch (error) { - res.status(500).json({ - error: 'INTERNAL_ERROR', - message: 'Failed to fetch trader assets summary', - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - // Get Asset Badges Catalog - app.get('/api/v1/communities/:asset/badges', async (req: Request, res: Response) => { - const { asset } = req.params; - const rarity = req.query.rarity as string | undefined; - const category = req.query.category as string | undefined; - - try { - // Import badge catalog dynamically to avoid circular dependencies - const { getBadgeCatalogForAsset, filterBadges, getBadgeStats } = await import('@pmon/trader-intelligence/dist/points/badgeCatalog'); - - let badges = getBadgeCatalogForAsset(asset); - - // Apply filters - if (rarity || category) { - badges = filterBadges(badges, rarity, category); - } - - const stats = getBadgeStats(badges); - - res.json({ - asset, - badges: badges.map((b) => ({ - id: b.id, - name: b.name, - description: b.description, - emoji: b.emoji, - rarity: b.rarity, - pointsBonus: b.pointsBonus, - requirements: b.requirements, - category: b.category - })), - totalBadges: badges.length, - byRarity: stats - }); - } catch (error) { - // Fallback if badge catalog module not available - res.json({ - asset, - badges: [], - totalBadges: 0, - byRarity: { - common: 0, - rare: 0, - epic: 0, - legendary: 0, - mythic: 0 - }, - message: 'Badge catalog temporarily unavailable' - }); - } - }); - - // Get Trader Profile (Full Intelligence Profile) - app.get('/api/v1/traders/:address/profile', async (req: Request, res: Response) => { - const { address } = req.params; - const normalizedAddress = address.toLowerCase(); - const period = (req.query.period as string) || '24h'; - const includeHistory = req.query.includeHistory === 'true'; - - try { - const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; - const profileKey = `${keyPrefix}trader:${normalizedAddress}:profile`; - const pnlKey = `${keyPrefix}trader:${normalizedAddress}:pnl:${period}`; - const riskKey = `${keyPrefix}trader:${normalizedAddress}:risk`; - const positionsKey = `${keyPrefix}trader:${normalizedAddress}:positions`; - const ordersKey = `${keyPrefix}trader:${normalizedAddress}:orders`; - const liquidationsKey = `${keyPrefix}trader:${normalizedAddress}:liquidations`; - - // Fetch profile data - const [profileData, pnlData, riskData, positionsJson, ordersJson, liquidationsData] = await Promise.all([ - traderIntelligenceRedis.hgetall(profileKey), - traderIntelligenceRedis.hgetall(pnlKey), - traderIntelligenceRedis.hgetall(riskKey), - traderIntelligenceRedis.get(positionsKey), - traderIntelligenceRedis.get(ordersKey), - includeHistory - ? traderIntelligenceRedis.zrange(liquidationsKey, -100, -1, 'WITHSCORES') - : Promise.resolve(null), - ]) as [ - Record, - Record, - Record, - string | null, - string | null, - string[] | null - ]; - - if (!profileData || Object.keys(profileData).length === 0) { - res.status(404).json({ - error: 'trader_not_found', - address, - message: 'Trader profile not found. The trader may not be tracked yet.' - }); - return; - } - - // Parse profile data - const profile = { - address: profileData.address || normalizedAddress, - ensName: profileData.ensName || undefined, - ensAvatar: profileData.ensAvatar || undefined, - primaryPersona: profileData.primaryPersona, - personas: profileData.personas ? JSON.parse(profileData.personas) : [], - funnyQuip: profileData.funnyQuip || undefined, - accountValue: parseFloat(profileData.accountValue || '0'), - lastUpdated: parseInt(profileData.lastUpdated || '0', 10), - firstSeen: profileData.firstSeen ? parseInt(profileData.firstSeen, 10) : undefined, - totalTrades: profileData.totalTrades ? parseInt(profileData.totalTrades, 10) : undefined, - pnl: pnlData && Object.keys(pnlData).length > 0 ? { - period, - netPnl: parseFloat(pnlData.netPnl || '0'), - realized: { - totalPnl: parseFloat(pnlData.realizedPnl || '0'), - winRate: parseFloat(pnlData.winRate || '0'), - profitFactor: parseFloat(pnlData.profitFactor || '0'), - }, - unrealized: { - totalPnl: parseFloat(pnlData.unrealizedPnl || '0'), - }, - fees: { - totalPaid: parseFloat(pnlData.totalFees || '0'), - }, - calculatedAt: parseInt(pnlData.calculatedAt || '0', 10), - } : undefined, - risk: riskData && Object.keys(riskData).length > 0 ? { - leverage: parseFloat(riskData.leverage || '0'), - marginUsage: parseFloat(riskData.marginUsage || '0'), - liquidationRisk: parseFloat(riskData.liquidationRisk || '0'), - positionConcentration: parseFloat(riskData.positionConcentration || '0'), - calculatedAt: parseInt(riskData.calculatedAt || '0', 10), - } : undefined, - positions: positionsJson ? JSON.parse(positionsJson) : [], - orders: ordersJson ? JSON.parse(ordersJson) : [], - liquidations: includeHistory && liquidationsData && Array.isArray(liquidationsData) && liquidationsData.length > 0 - ? liquidationsData - .filter((_, i) => i % 2 === 0) // Get members (every other element, WITHSCORES returns [member, score, member, score, ...]) - .map((liqJson) => { - try { - return JSON.parse(liqJson as string); - } catch { - return null; - } - }) - .filter((liq) => liq !== null) - .reverse() // Most recent first (zrange returns oldest first) - : undefined, - }; - - res.json({ - data: profile, - period, - cached: true, - }); - } catch (error) { - res.status(500).json({ - error: 'trader_profile_fetch_failed', - address, - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - // Persona Endpoints - type PersonaTraderSummary = { - address: string; - ensName?: string; - perpEquity: number; - openValue: number; - leverage: number; - pnl24h: number; - pnl7d: number; - pnl30d: number; - currentBias: number; - ageDays: number; - }; - - const buildTraderSummaries = async ( - addresses: string[], - keyPrefix: string - ): Promise => { - if (!traderIntelligenceRedis || addresses.length === 0) { - return []; - } - - const uniqueAddresses = Array.from(new Set(addresses.map((address) => address.toLowerCase()))); - - const pipeline = traderIntelligenceRedis.pipeline(); - for (const address of uniqueAddresses) { - const profileKey = `${keyPrefix}trader:${address}:profile`; - const pnlKey = `${keyPrefix}trader:${address}:pnl:24h`; - const positionsKey = `${keyPrefix}trader:${address}:positions`; - pipeline.hgetall(profileKey); - pipeline.hgetall(pnlKey); - pipeline.get(positionsKey); - } - - const responses = await pipeline.exec(); - if (!responses) { - return []; - } - - const traders: PersonaTraderSummary[] = []; - - for (let i = 0; i < uniqueAddresses.length; i += 1) { - const responseOffset = i * 3; - const profileData = responses[responseOffset]?.[1] as Record | null; - if (!profileData || Object.keys(profileData).length === 0) { - continue; - } - - const pnlData = responses[responseOffset + 1]?.[1] as Record | null; - const positionsJson = responses[responseOffset + 2]?.[1] as string | null; - - const address = uniqueAddresses[i]; - let positions: Array<{ positionValue?: string; side?: string }> = []; - - if (positionsJson) { - try { - const parsed = JSON.parse(positionsJson); - if (Array.isArray(parsed)) { - positions = parsed; - } - } catch { - positions = []; - } - } - - const openValue = positions.reduce((sum: number, pos) => sum + parseFloat(pos.positionValue || '0'), 0); - - let totalLongValue = 0; - let totalShortValue = 0; - for (const pos of positions) { - const value = parseFloat(pos.positionValue || '0'); - if (pos.side === 'long') { - totalLongValue += value; - } else if (pos.side === 'short') { - totalShortValue += value; - } - } - - const totalValue = totalLongValue + totalShortValue; - const biasScore = totalValue > 0 ? (totalLongValue - totalShortValue) / totalValue : 0; - - const accountValue = parseFloat(profileData.accountValue || '0'); - const firstSeen = profileData.firstSeen - ? parseInt(profileData.firstSeen, 10) - : parseInt(profileData.lastUpdated || '0', 10); - const ageDays = firstSeen > 0 ? Math.floor((Date.now() - firstSeen) / (24 * 60 * 60 * 1000)) : 0; - - traders.push({ - address, - ensName: profileData.ensName || undefined, - perpEquity: accountValue, - openValue, - leverage: accountValue > 0 ? openValue / accountValue : 0, - pnl24h: parseFloat(pnlData?.netPnl || '0'), - pnl7d: parseFloat(pnlData?.netPnl7d || '0') || 0, - pnl30d: parseFloat(pnlData?.netPnl30d || '0') || 0, - currentBias: biasScore, - ageDays, - }); - } - - return traders; - }; - - const sortTraderSummaries = ( - traders: PersonaTraderSummary[], - sortBy: string, - order: string - ): PersonaTraderSummary[] => { - const sorted = [...traders]; - - sorted.sort((a, b) => { - let aVal = 0; - let bVal = 0; - - if (sortBy === 'leverage') { - aVal = a.leverage; - bVal = b.leverage; - } else if (sortBy === 'equity') { - aVal = a.perpEquity; - bVal = b.perpEquity; - } else { - // Default to Net PnL - aVal = a.pnl24h; - bVal = b.pnl24h; - } - - return order === 'asc' ? aVal - bVal : bVal - aVal; - }); - - return sorted; - }; - // Get all personas with counts - app.get('/api/v1/personas', async (req: Request, res: Response) => { - const type = req.query.type as string | undefined; // 'size' | 'strategy' | 'risk' | 'performance' | 'behavioral' | 'special' - const includeStats = req.query.includeStats === 'true'; - - try { - const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; - - // Get all persona stats keys - const pattern = `${keyPrefix}persona:*:stats`; - const keys = await traderIntelligenceRedis.keys(pattern); - - const personas = []; - for (const key of keys) { - const match = key.match(/persona:([^:]+):stats/); - if (!match) continue; - - const personaName = match[1]; - - // Get persona category mapping - const personaCategoryMap: Record = { - 'mega-whale': 'size', 'whale': 'size', 'dolphin': 'size', 'fish': 'size', 'minnow': 'size', 'plankton': 'size', - 'hodler': 'strategy', 'scalper': 'strategy', 'swing-trader': 'strategy', 'market-maker': 'strategy', 'arbitrageur': 'strategy', - 'degen': 'risk', 'safe-sailor': 'risk', 'risk-taker': 'risk', - 'chad': 'performance', 'gigabrain': 'performance', 'rekt': 'performance', 'bag-holder': 'performance', - 'bot': 'behavioral', 'fomo-king': 'behavioral', 'diamond-hands': 'behavioral', 'paper-hands': 'behavioral', - 'mystery': 'special', 'vc': 'special', 'exchange': 'special', 'protocol': 'special', - }; - - const personaCategory = personaCategoryMap[personaName] || 'special'; - - // Filter by type if specified - if (type && personaCategory !== type) continue; - - const statsData = await traderIntelligenceRedis.hgetall(key); - if (!statsData || Object.keys(statsData).length === 0) continue; - - const personaEmojiMap: Record = { - 'mega-whale': '🐋', 'whale': '🐋', 'dolphin': '🐬', 'fish': '🐟', 'minnow': '🦐', 'plankton': '🦠', - 'hodler': '💎', 'scalper': '⚡', 'swing-trader': '📈', 'market-maker': '🏪', 'arbitrageur': '🔄', - 'degen': '💀', 'safe-sailor': '⛵', 'risk-taker': '🎲', - 'chad': '👑', 'gigabrain': '🧠', 'rekt': '💀', 'bag-holder': '👜', - 'bot': '🤖', 'fomo-king': '🚀', 'diamond-hands': '💎', 'paper-hands': '📄', - 'mystery': '❓', 'vc': '💼', 'exchange': '🏦', 'protocol': '🔧', - }; - - const persona: any = { - name: personaName, - type: personaCategory, - emoji: personaEmojiMap[personaName] || '❓', - walletCount: parseInt(statsData.totalWallets || '0', 10), - }; - - if (includeStats) { - persona.sentiment = JSON.parse(statsData.sentiment || '{"label":"neutral","score":0,"confidence":0}'); - } - - personas.push(persona); - } - - res.json({ personas }); - } catch (error) { - res.status(500).json({ - error: 'personas_fetch_failed', - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - // Get detailed persona stats - app.get('/api/v1/personas/:personaName/stats', async (req: Request, res: Response) => { - const { personaName } = req.params; - const includeTrend = req.query.includeTrend === 'true'; - const trendWindow = (req.query.trendWindow as string) || '7d'; - - try { - const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; - const statsKey = `${keyPrefix}persona:${personaName}:stats`; - const trendKey = `${keyPrefix}persona:${personaName}:trend`; - - const statsData = await traderIntelligenceRedis.hgetall(statsKey); - if (!statsData || Object.keys(statsData).length === 0) { - res.status(404).json({ - error: 'persona_not_found', - personaName, - message: 'Persona stats not found' - }); - return; - } - - // Get persona emoji (would use persona catalog in production) - const personaEmojiMap: Record = { - 'mega-whale': '🐋', 'whale': '🐋', 'dolphin': '🐬', 'fish': '🐟', 'minnow': '🦐', 'plankton': '🦠', - 'hodler': '💎', 'scalper': '⚡', 'swing-trader': '📈', 'market-maker': '🏪', 'arbitrageur': '🔄', - 'degen': '💀', 'safe-sailor': '⛵', 'risk-taker': '🎲', - 'chad': '👑', 'gigabrain': '🧠', 'rekt': '💀', 'bag-holder': '👜', - 'bot': '🤖', 'fomo-king': '🚀', 'diamond-hands': '💎', 'paper-hands': '📄', - 'mystery': '❓', 'vc': '💼', 'exchange': '🏦', 'protocol': '🔧', - }; - const personaEmoji = personaEmojiMap[personaName] || '❓'; - - const stats = { - personaName: statsData.personaName || personaName, - personaCategory: statsData.personaCategory || 'special', - totalWallets: parseInt(statsData.totalWallets || '0', 10), - activeWallets: parseInt(statsData.activeWallets || '0', 10), - avgLeverage: parseFloat(statsData.avgLeverage || '0'), - avgPnL24h: parseFloat(statsData.avgPnL24h || '0'), - avgPnL7d: parseFloat(statsData.avgPnL7d || '0'), - avgPnL30d: parseFloat(statsData.avgPnL30d || '0'), - inPositionPercent: parseFloat(statsData.inPositionPercent || '0'), - sentiment: JSON.parse(statsData.sentiment || '{"label":"neutral","score":0,"confidence":0}'), - topAssets: JSON.parse(statsData.topAssets || '[]'), - lastUpdated: parseInt(statsData.lastUpdated || '0', 10), - }; - - const response: any = { - persona: { - name: personaName, - type: stats.personaCategory, - emoji: personaEmoji, - stats, - } - }; - - if (includeTrend) { - // Get trend points for the requested window - const windowMs = trendWindow === '30d' ? 30 * 24 * 60 * 60 * 1000 : 7 * 24 * 60 * 60 * 1000; - const since = Date.now() - windowMs; - - const trendMembers = await traderIntelligenceRedis.zrangebyscore( - trendKey, - since, - Date.now(), - 'WITHSCORES' - ); - - const trendPoints = []; - for (let i = 0; i < trendMembers.length; i += 2) { - const member = trendMembers[i]; - if (member) { - try { - const point = JSON.parse(member); - trendPoints.push(point); - } catch { - // Skip invalid entries - } - } - } - - response.persona.trend = trendPoints; - } - - res.json(response); - } catch (error) { - res.status(500).json({ - error: 'persona_stats_fetch_failed', - personaName, - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - // Get traders in a persona - app.get('/api/v1/personas/:personaName/traders', async (req: Request, res: Response) => { - const personaName = (req.params.personaName || '').toLowerCase(); - const limit = Math.min(Math.max(parseInt((req.query.limit as string) || '50', 10), 1), 100); - const offset = Math.max(parseInt((req.query.offset as string) || '0', 10), 0); - const sortBy = (req.query.sortBy as string) || 'pnl'; - const orderParam = (req.query.order as string) || 'desc'; - const order = orderParam === 'asc' ? 'asc' : 'desc'; - - if (!traderIntelligenceRedis) { - res.status(503).json({ error: 'service_unavailable', message: 'Trader intelligence service is not available' }); - return; - } - - try { - const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; - const membersKey = `${keyPrefix}persona:${personaName}:members`; - const addresses = await traderIntelligenceRedis.smembers(membersKey); - - if (!addresses || addresses.length === 0) { - res.json({ - traders: [], - pagination: { - total: 0, - limit, - offset, - hasMore: false, - }, - }); - return; - } - - const traderSummaries = await buildTraderSummaries(addresses, keyPrefix); - const sorted = sortTraderSummaries(traderSummaries, sortBy, order); - const total = sorted.length; - const paginated = sorted.slice(offset, offset + limit); - - res.json({ - traders: paginated, - pagination: { - total, - limit, - offset, - hasMore: offset + limit < total, - }, - }); - } catch (error) { - res.status(500).json({ - error: 'persona_traders_fetch_failed', - personaName, - detail: error instanceof Error ? error.message : String(error), - }); - } - }); - - app.get('/api/v1/personas/traders', async (req: Request, res: Response) => { - const personasParam = req.query.personas as string | undefined; - const limit = Math.min(Math.max(parseInt((req.query.limit as string) || '50', 10), 1), 100); - const offset = Math.max(parseInt((req.query.offset as string) || '0', 10), 0); - const sortBy = (req.query.sortBy as string) || 'pnl'; - const orderParam = (req.query.order as string) || 'desc'; - const order = orderParam === 'asc' ? 'asc' : 'desc'; - const matchMode = ((req.query.match as string) || 'all').toLowerCase() === 'any' ? 'any' : 'all'; - - if (!traderIntelligenceRedis) { - res.status(503).json({ error: 'service_unavailable', message: 'Trader intelligence service is not available' }); - return; - } - - if (!personasParam) { - res.status(400).json({ - error: 'personas_required', - message: 'Query parameter "personas" is required (comma-separated persona ids)', - }); - return; - } - - const personaNames = Array.from( - new Set( - personasParam - .split(',') - .map((name) => name.trim().toLowerCase()) - .filter((name) => name.length > 0) - ) - ); - - if (personaNames.length === 0) { - res.status(400).json({ - error: 'invalid_personas', - message: 'At least one persona must be specified', - }); - return; - } - - if (personaNames.length > 8) { - res.status(400).json({ - error: 'too_many_personas', - message: 'A maximum of 8 personas can be queried at once', - }); - return; - } - - try { - const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; - const memberKeys = personaNames.map((persona) => `${keyPrefix}persona:${persona}:members`); - - let addresses: string[] = []; - if (matchMode === 'any') { - addresses = await traderIntelligenceRedis.sunion(...memberKeys); - } else { - addresses = await traderIntelligenceRedis.sinter(...memberKeys); - } - - if (!addresses || addresses.length === 0) { - res.json({ - traders: [], - pagination: { - total: 0, - limit, - offset, - hasMore: false, - }, - }); - return; - } - - const traderSummaries = await buildTraderSummaries(addresses, keyPrefix); - const sorted = sortTraderSummaries(traderSummaries, sortBy, order); - const total = sorted.length; - const paginated = sorted.slice(offset, offset + limit); - - res.json({ - traders: paginated, - appliedPersonas: personaNames, - mode: matchMode === 'any' ? 'any' : 'all', - pagination: { - total, - limit, - offset, - hasMore: offset + limit < total, - }, - }); - } catch (error) { - res.status(500).json({ - error: 'persona_traders_fetch_failed', - personas: personaNames, - detail: error instanceof Error ? error.message : String(error), - }); - } - }); - - // Position Breakdown Endpoint - app.get('/api/v1/assets/:asset/position-breakdown', async (req: Request, res: Response) => { - const { asset } = req.params; - const view = (req.query.view as string) || 'size'; // 'size' | 'persona' - - if (!['size', 'persona'].includes(view)) { - res.status(400).json({ error: 'invalid_view', message: 'View must be "size" or "persona"' }); - return; - } - - if (!traderIntelligenceRedis) { - res.status(503).json({ error: 'service_unavailable', message: 'Trader intelligence service is not available' }); - return; - } - - try { - const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; - - // Get all trader profiles - const profileKeys = await traderIntelligenceRedis.keys(`${keyPrefix}trader:*:profile`); - - // Size buckets - const sizeBuckets = [ - { label: '<$1k', min: 0, max: 1000 }, - { label: '$1k-$10k', min: 1000, max: 10000 }, - { label: '$10k-$25k', min: 10000, max: 25000 }, - { label: '$25k-$50k', min: 25000, max: 50000 }, - { label: '$50k-$100k', min: 50000, max: 100000 }, - { label: '$100k-$250k', min: 100000, max: 250000 }, - { label: '$250k-$500k', min: 250000, max: 500000 }, - { label: '$500k-$1m', min: 500000, max: 1000000 }, - { label: '$1m-$2.5m', min: 1000000, max: 2500000 }, - { label: '>$2.5m', min: 2500000, max: Infinity }, - ]; - - // Collect all positions for this asset - const allPositions: Array<{ - address: string; - position: any; - persona?: string; - }> = []; - let totalOpenInterest = 0; - - for (const profileKey of profileKeys) { - const address = profileKey.match(/trader:([^:]+):profile/)?.[1]; - if (!address) continue; - - // Get positions - const positionsKey = `${keyPrefix}trader:${address}:positions`; - const positionsJson = await traderIntelligenceRedis.get(positionsKey); - if (!positionsJson) continue; - - try { - const positions = JSON.parse(positionsJson); - const assetPositions = positions.filter((p: any) => p.coin === asset); - - for (const pos of assetPositions) { - const value = parseFloat(pos.positionValue || '0'); - totalOpenInterest += value; - - // Get persona if needed - let persona: string | undefined; - if (view === 'persona') { - const profileData = await traderIntelligenceRedis.hgetall(profileKey); - const personas = profileData.personas ? JSON.parse(profileData.personas) : []; - persona = personas[0]?.persona || personas[0] || 'unknown'; - } - - allPositions.push({ - address, - position: pos, - persona, - }); - } - } catch { - // Skip invalid JSON - continue; - } - } - - // Group positions - const breakdownMap = new Map(); - - for (const item of allPositions) { - let category: string; - - if (view === 'size') { - const value = parseFloat(item.position.positionValue || '0'); - const bucket = sizeBuckets.find(b => value >= b.min && value < b.max); - category = bucket?.label || '>$2.5m'; - } else { - category = item.persona || 'unknown'; - } - - if (!breakdownMap.has(category)) { - breakdownMap.set(category, { - category, - positions: [], - }); - } - breakdownMap.get(category)!.positions.push(item); - } - - // Calculate metrics for each breakdown entry - const breakdown = Array.from(breakdownMap.values()).map(({ category, positions }) => { - let totalLongValue = 0; - let totalShortValue = 0; - let totalValue = 0; - const liquidationDistances: number[] = []; - - for (const item of positions) { - const value = parseFloat(item.position.positionValue || '0'); - totalValue += value; - - if (item.position.side === 'long') { - totalLongValue += value; - } else if (item.position.side === 'short') { - totalShortValue += value; - } - - // Calculate liquidation distance if available - const entryPx = parseFloat(item.position.entryPx || '0'); - const liquidationPx = parseFloat(item.position.liquidationPx || '0'); - if (entryPx > 0 && liquidationPx > 0) { - const distance = Math.abs((entryPx - liquidationPx) / entryPx) * 100; - liquidationDistances.push(distance); - } - } - - // Calculate bias - const biasScore = totalValue > 0 - ? (totalLongValue - totalShortValue) / totalValue - : 0; - - const bias: 'bullish' | 'bearish' | 'neutral' = - biasScore > 0.1 ? 'bullish' : - biasScore < -0.1 ? 'bearish' : 'neutral'; - - const avgDistanceToLiq = liquidationDistances.length > 0 - ? liquidationDistances.reduce((sum, d) => sum + d, 0) / liquidationDistances.length - : 0; - - const percentOfTotal = totalOpenInterest > 0 - ? (totalValue / totalOpenInterest) * 100 - : 0; - - return { - category, - bias, - biasScore, - positionCount: positions.length, - totalValue, - avgDistanceToLiq, - percentOfTotal, - }; - }); - - // Sort by total value descending - breakdown.sort((a, b) => b.totalValue - a.totalValue); - - res.json({ - asset, - view, - breakdown, - totalOpenInterest, - lastUpdated: Date.now(), - }); - } catch (error) { - res.status(500).json({ - error: 'position_breakdown_fetch_failed', - asset, - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - // Platform Metrics Endpoints - app.get('/api/v1/stats/platform', async (req: Request, res: Response) => { - if (!traderIntelligenceRedis) { - res.status(503).json({ error: 'service_unavailable', message: 'Trader intelligence service is not available' }); - return; - } - - try { - const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; - const metricsKey = `${keyPrefix}platform:metrics`; - - const data = await traderIntelligenceRedis.hgetall(metricsKey); - if (!data || Object.keys(data).length === 0) { - res.status(404).json({ error: 'metrics_not_found', message: 'Platform metrics not available yet' }); - return; - } - - res.json({ - totalWallets: parseInt(data.totalWallets || '0', 10), - newWallets24h: parseInt(data.newWallets24h || '0', 10), - activePerpTraders: parseInt(data.activePerpTraders || '0', 10), - totalOpenPositions: parseInt(data.totalOpenPositions || '0', 10), - totalOpenInterest: parseFloat(data.totalOpenInterest || '0'), - dailyVolume: parseFloat(data.dailyVolume || '0'), - positionsInProfitPercent: parseFloat(data.positionsInProfitPercent || '0'), - avgLeverage: parseFloat(data.avgLeverage || '0'), - exposureRatioDistribution: JSON.parse(data.exposureRatioDistribution || '{"low":0,"medium":0,"high":0,"veryHigh":0}'), - change24h: JSON.parse(data.change24h || '{"totalWallets":0,"activePerpTraders":0,"totalOpenInterest":0,"dailyVolume":0}'), - lastUpdated: parseInt(data.lastUpdated || '0', 10), - }); - } catch (error) { - res.status(500).json({ - error: 'platform_metrics_fetch_failed', - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - app.get('/api/v1/stats/platform/history', async (req: Request, res: Response) => { - const windowParam = req.query.window as string | undefined; - const intervalParam = req.query.interval as string | undefined; - - if (!traderIntelligenceRedis) { - res.status(503).json({ error: 'service_unavailable', message: 'Trader intelligence service is not available' }); - return; - } - - try { - const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; - const historyKey = `${keyPrefix}platform:metrics:history`; - - // Parse window (default: 7 days) - const windowMs = windowParam === '30d' - ? 30 * 24 * 60 * 60 * 1000 - : windowParam === '24h' - ? 24 * 60 * 60 * 1000 - : 7 * 24 * 60 * 60 * 1000; // default 7d - - const startTime = Date.now() - windowMs; - - // Get all snapshots within window - const members = await traderIntelligenceRedis.zrangebyscore( - historyKey, - startTime, - '+inf' - ); - - const snapshots: Array<{ - timestamp: number; - metrics: { - totalWallets: number; - newWallets24h: number; - activePerpTraders: number; - totalOpenPositions: number; - totalOpenInterest: number; - dailyVolume: number; - positionsInProfitPercent: number; - avgLeverage: number; - exposureRatioDistribution: { - low: number; - medium: number; - high: number; - veryHigh: number; - }; - change24h: { - totalWallets: number; - activePerpTraders: number; - totalOpenInterest: number; - dailyVolume: number; - }; - lastUpdated: number; - }; - }> = []; - - for (const member of members) { - try { - const snapshot = JSON.parse(member); - snapshots.push(snapshot); - } catch { - // Skip invalid entries - } - } - - // Sort by timestamp - snapshots.sort((a, b) => a.timestamp - b.timestamp); - - // Apply interval filtering if specified - let filteredSnapshots = snapshots; - if (intervalParam === 'hourly') { - // Keep one snapshot per hour - const hourlySnapshots: typeof snapshots = []; - let lastHour = -1; - for (const snapshot of snapshots) { - const hour = Math.floor(snapshot.timestamp / (60 * 60 * 1000)); - if (hour !== lastHour) { - hourlySnapshots.push(snapshot); - lastHour = hour; - } - } - filteredSnapshots = hourlySnapshots; - } else if (intervalParam === 'daily') { - // Keep one snapshot per day - const dailySnapshots: typeof snapshots = []; - let lastDay = -1; - for (const snapshot of snapshots) { - const day = Math.floor(snapshot.timestamp / (24 * 60 * 60 * 1000)); - if (day !== lastDay) { - dailySnapshots.push(snapshot); - lastDay = day; - } - } - filteredSnapshots = dailySnapshots; - } - - res.json({ - window: windowParam || '7d', - interval: intervalParam || 'all', - snapshots: filteredSnapshots, - count: filteredSnapshots.length, - }); - } catch (error) { - res.status(500).json({ - error: 'platform_metrics_history_fetch_failed', - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - // Bias Trend Endpoint - app.get('/api/v1/assets/:asset/bias-trend', async (req: Request, res: Response) => { - const { asset } = req.params; - const personasParam = req.query.personas as string | undefined; - const window = (req.query.window as '7d' | '30d') || '7d'; - - if (!['7d', '30d'].includes(window)) { - res.status(400).json({ error: 'invalid_window', message: 'Window must be "7d" or "30d"' }); - return; - } - - if (!traderIntelligenceRedis) { - res.status(503).json({ error: 'service_unavailable', message: 'Trader intelligence service is not available' }); - return; - } - - try { - const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; - - // Parse personas from query param (comma-separated) - const personas: string[] = personasParam - ? personasParam.split(',').map(p => p.trim()).filter(Boolean) - : []; - - const windowMs = window === '7d' - ? 7 * 24 * 60 * 60 * 1000 - : 30 * 24 * 60 * 60 * 1000; - const startTime = Date.now() - windowMs; - - // If no personas specified, find all personas that have data for this asset - let personasToQuery = personas; - if (personasToQuery.length === 0) { - const pattern = `${keyPrefix}asset:${asset}:bias:*`; - const keys = await traderIntelligenceRedis.keys(pattern); - personasToQuery = keys - .map(key => { - const match = key.match(/bias:([^:]+)$/); - return match ? match[1] : null; - }) - .filter((p): p is string => p !== null); - } - - if (personasToQuery.length === 0) { - res.json({ - asset, - window, - personas: [], - lastUpdated: Date.now(), - }); - return; - } - - const personaData: Array<{ - persona: string; - personaType: string; - points: Array<{ - timestamp: number; - persona: string; - bias: number; - openInterest: number; - positionCount: number; - longValue: number; - shortValue: number; - }>; - }> = []; - - // Persona category mapping - const personaCategoryMap: Record = { - 'mega-whale': 'size', 'whale': 'size', 'dolphin': 'size', 'fish': 'size', 'minnow': 'size', 'plankton': 'size', - 'hodler': 'strategy', 'scalper': 'strategy', 'swing-trader': 'strategy', 'market-maker': 'strategy', 'arbitrageur': 'strategy', - 'degen': 'risk', 'safe-sailor': 'risk', 'risk-taker': 'risk', - 'chad': 'performance', 'gigabrain': 'performance', 'rekt': 'performance', 'bag-holder': 'performance', - 'bot': 'behavioral', 'fomo-king': 'behavioral', 'diamond-hands': 'behavioral', 'paper-hands': 'behavioral', - 'mystery': 'special', 'vc': 'special', 'exchange': 'special', 'protocol': 'special', - }; - - for (const persona of personasToQuery) { - const trendKey = `${keyPrefix}asset:${asset}:bias:${persona}`; - - try { - // Get all trend points within the window - const members = await traderIntelligenceRedis.zrangebyscore( - trendKey, - startTime, - Date.now(), - 'WITHSCORES' - ); - - if (members.length === 0) { - continue; - } - - const points: Array<{ - timestamp: number; - persona: string; - bias: number; - openInterest: number; - positionCount: number; - longValue: number; - shortValue: number; - }> = []; - - for (let i = 0; i < members.length; i += 2) { - const member = members[i]; - const score = members[i + 1]; - - if (member && score) { - try { - const point = JSON.parse(member); - point.timestamp = parseInt(score, 10); - points.push(point); - } catch { - // Skip invalid entries - } - } - } - - if (points.length > 0) { - const personaType = personaCategoryMap[persona] || 'special'; - personaData.push({ - persona, - personaType, - points: points.sort((a, b) => a.timestamp - b.timestamp), - }); - } - } catch (error) { - // Continue with other personas - console.error(`Error getting bias trend for ${asset} ${persona}:`, error); - } - } - - res.json({ - asset, - window, - personas: personaData, - lastUpdated: Date.now(), - }); - } catch (error) { - res.status(500).json({ - error: 'bias_trend_fetch_failed', - asset, - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - // Position Heatmap Endpoint - app.get('/api/v1/heatmap/positions', async (req: Request, res: Response) => { - const assetsParam = req.query.assets as string | undefined; - const personasParam = req.query.personas as string | undefined; - - if (!traderIntelligenceRedis) { - res.status(503).json({ error: 'service_unavailable', message: 'Trader intelligence service is not available' }); - return; - } - - try { - const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; - - // Parse assets and personas from query params (comma-separated) - const assets: string[] = assetsParam - ? assetsParam.split(',').map(a => a.trim()).filter(Boolean) - : []; - const personas: string[] = personasParam - ? personasParam.split(',').map(p => p.trim()).filter(Boolean) - : []; - - // Get list of assets and personas if not provided - let assetsToQuery = assets; - let personasToQuery = personas; - - if (assetsToQuery.length === 0) { - const assetsKey = `${keyPrefix}heatmap:positions:assets`; - const assetMembers = await traderIntelligenceRedis.smembers(assetsKey); - assetsToQuery = assetMembers.length > 0 ? assetMembers.sort() : []; - } - - if (personasToQuery.length === 0) { - const personasKey = `${keyPrefix}heatmap:positions:personas`; - const personaMembers = await traderIntelligenceRedis.smembers(personasKey); - personasToQuery = personaMembers.length > 0 ? personaMembers.sort() : []; - } - - if (assetsToQuery.length === 0 || personasToQuery.length === 0) { - res.json({ - assets: [], - personas: [], - matrix: {}, - lastUpdated: Date.now(), - }); - return; - } - - const matrix: Record> = {}; - - // Retrieve heatmap data for each asset - for (const asset of assetsToQuery) { - const assetKey = `${keyPrefix}heatmap:positions:${asset}`; - const assetData = await traderIntelligenceRedis.hgetall(assetKey); - - if (Object.keys(assetData).length > 0) { - matrix[asset] = {}; - for (const persona of personasToQuery) { - const cellJson = assetData[persona]; - if (cellJson) { - try { - const cell = JSON.parse(cellJson); - matrix[asset][persona] = cell; - } catch { - // Skip invalid entries - } - } - } - } - } - - res.json({ - assets: assetsToQuery, - personas: personasToQuery, - matrix, - lastUpdated: Date.now(), - }); - } catch (error) { - res.status(500).json({ - error: 'position_heatmap_fetch_failed', - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - // Viral Achievements Endpoints - // Get achievement card (placeholder - would generate actual image) - app.get('/api/v1/achievements/:achievementId/card', async (req: Request, res: Response) => { - const { achievementId } = req.params; - const { trader } = req.query; - - if (!trader || typeof trader !== 'string') { - res.status(400).json({ error: 'trader_required', message: 'Trader address is required' }); - return; - } - - try { - const normalizedAddress = trader.toLowerCase(); - const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; - const achievementKey = `${keyPrefix}viral-achievement:${achievementId}:${normalizedAddress}`; - - const achievementData = await traderIntelligenceRedis.hgetall(achievementKey); - - if (!achievementData || !achievementData.id) { - res.status(404).json({ error: 'achievement_not_found', achievementId }); - return; - } - - // TODO: Generate actual image using canvas/sharp - // For now, return JSON representation - res.json({ - achievement: { - id: achievementData.id, - name: achievementData.name, - description: achievementData.description, - emoji: achievementData.emoji, - rarity: achievementData.rarity, - category: achievementData.category, - funnyQuip: achievementData.funnyQuip, - pointsBonus: parseInt(achievementData.pointsBonus || '0', 10), - unlockedAt: parseInt(achievementData.unlockedAt || '0', 10), - }, - trader: { - address: normalizedAddress, - }, - shareUrl: `${process.env.APP_URL || 'https://app.perps.gmbh'}/achievements/${achievementId}?trader=${normalizedAddress}`, - // In production, this would return a PNG image buffer with Content-Type: image/png - }); - } catch (error) { - res.status(500).json({ - error: 'card_generation_failed', - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - // Get trader's viral achievements - app.get('/api/v1/traders/:address/achievements', async (req: Request, res: Response) => { - const { address } = req.params; - const normalizedAddress = address.toLowerCase(); - const category = req.query.category as string | undefined; - const rarity = req.query.rarity as string | undefined; - const maxWarningLevel = req.query.maxWarningLevel as string | undefined; - - // Warning level hierarchy: clean < spicy < crass < degenerate < hidden - const warningLevels: Record = { - clean: 0, - spicy: 1, - crass: 2, - degenerate: 3, - hidden: 4, - }; - - try { - const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; - const achievementsKey = `${keyPrefix}trader:${normalizedAddress}:viral-achievements`; - const achievementIds = await traderIntelligenceRedis.smembers(achievementsKey); - - const achievements = []; - for (const achievementId of achievementIds) { - const achievementKey = `${keyPrefix}viral-achievement:${achievementId}:${normalizedAddress}`; - const achievementData = await traderIntelligenceRedis.hgetall(achievementKey); - - if (achievementData && achievementData.id) { - // Apply filters - if (category && achievementData.category !== category) continue; - if (rarity && achievementData.rarity !== rarity) continue; - - // Filter by warning level - const achievementWarningLevel = achievementData.warningLevel || 'clean'; - if (maxWarningLevel) { - const maxLevel = warningLevels[maxWarningLevel] ?? 4; - const achievementLevel = warningLevels[achievementWarningLevel] ?? 0; - if (achievementLevel > maxLevel) continue; - } - - achievements.push({ - id: achievementData.id, - name: achievementData.name, - description: achievementData.description, - emoji: achievementData.emoji, - rarity: achievementData.rarity, - category: achievementData.category, - funnyQuip: achievementData.funnyQuip, - warningLevel: achievementWarningLevel, - sfwShareText: achievementData.sfwShareText || undefined, - pointsBonus: parseInt(achievementData.pointsBonus || '0', 10), - assetRequired: achievementData.assetRequired, - unlockedAt: parseInt(achievementData.unlockedAt || '0', 10), - }); - } - } - - // Sort by unlockedAt descending (most recent first) - achievements.sort((a, b) => b.unlockedAt - a.unlockedAt); - - res.json({ - address: normalizedAddress, - achievements, - total: achievements.length, - warning: maxWarningLevel ? `Filtered to max warning level: ${maxWarningLevel}` : undefined, - }); - } catch (error) { - res.status(500).json({ - error: 'achievements_fetch_failed', - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - // Shame Leaderboards - app.get('/api/v1/leaderboards/shame/:type', async (req: Request, res: Response) => { - const { type } = req.params; - const window = (req.query.window as string) || '24h'; - const limit = resolveLimitQuery(req.query.limit, 50, 100); - - const validTypes = ['most-rekt', 'bought-tops', 'sold-bottoms', 'revenge-traders', 'gamblers']; - if (!validTypes.includes(type)) { - res.status(400).json({ error: 'invalid_shame_type', message: `Invalid shame leaderboard type: ${type}` }); - return; - } - - if (!['24h', '7d'].includes(window)) { - res.status(400).json({ error: 'invalid_window', message: `Invalid window: ${window}` }); - return; - } - - try { - const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; - const leaderboardKey = `${keyPrefix}leaderboard:shame:${type}:${window}`; - - const entriesWithScores = await traderIntelligenceRedis.zrevrange(leaderboardKey, 0, limit - 1, 'WITHSCORES'); - - const entries = []; - for (let i = 0; i < entriesWithScores.length; i += 2) { - const address = entriesWithScores[i]; - const score = parseFloat(entriesWithScores[i + 1] || '0'); - - // Get ENS name if available - const ensKey = `${keyPrefix}ens:${address}`; - const ensData = await traderIntelligenceRedis.get(ensKey); - let ensName: string | undefined; - if (ensData) { - try { - const parsed = JSON.parse(ensData); - ensName = parsed.ensName; - } catch { - // Ignore parse errors - } - } - - entries.push({ - rank: i / 2 + 1, - address, - ensName, - score, // Count of shame incidents - }); - } - - res.json({ - type, - window, - entries, - total: entries.length, - updatedAt: Date.now(), - }); - } catch (error) { - res.status(500).json({ - error: 'shame_leaderboard_fetch_failed', - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - } - - // Trader Intelligence Leaderboards (always registered, check Redis inside) - // PnL Leaderboard - app.get('/api/v1/leaderboards/pnl', async (req: Request, res: Response) => { - if (!traderIntelligenceRedis) { - res.status(503).json({ error: 'service_unavailable', message: 'Trader intelligence service is not available' }); - return; - } - - const window = (req.query.window as string) || '24h'; - const limit = resolveLimitQuery(req.query.limit, 50, 100); - const offset = parseInt((req.query.offset as string) || '0', 10); - const sortBy = (req.query.sortBy as string) || 'netPnl'; - const persona = req.query.persona as string | undefined; - - if (!['24h', '7d', '30d'].includes(window)) { - res.status(400).json({ error: 'invalid_window', message: `Invalid window: ${window}` }); - return; - } - - try { - const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; - const leaderboardKey = `${keyPrefix}leaderboard:pnl:${window}`; - - // Get entries with scores (netPnl) - const entriesWithScores = await traderIntelligenceRedis.zrevrange( - leaderboardKey, - offset, - offset + limit - 1, - 'WITHSCORES' - ); - - // Enhanced approach - fetch trader intelligence data - const entries = []; - - for (let i = 0; i < entriesWithScores.length; i += 2) { - const address = entriesWithScores[i]; - const netPnl = parseFloat(entriesWithScores[i + 1] || '0'); - - if (!address) continue; - - // Get trader profile data for personas and other details - const profileKey = `${keyPrefix}trader:${address}:profile`; - const profileData = await traderIntelligenceRedis.hgetall(profileKey); - - // Parse personas from profile - let personas: string[] = []; - if (profileData.personas) { - try { - personas = JSON.parse(profileData.personas); - } catch { - personas = []; - } - } - - // Skip persona filtering if personas don't match - if (persona && !personas.includes(persona)) { - continue; - } - - // Get actual data from profile - const totalTrades = parseInt(profileData.totalTrades || '0', 10); - const accountValue = parseFloat(profileData.accountValue || '0'); - const firstSeen = parseInt(profileData.firstSeen || '0', 10); - const ageDays = firstSeen > 0 ? Math.floor((Date.now() - firstSeen) / (24 * 60 * 60 * 1000)) : 0; - - // Get PnL data for metrics - const pnlKey = `${keyPrefix}trader:${address}:pnl:${window}`; - const pnlData = await traderIntelligenceRedis.hgetall(pnlKey); - const winRate = parseFloat(pnlData.winRate || '0'); - const profitFactor = parseFloat(pnlData.profitFactor || '1'); - - entries.push({ - rank: offset + (i / 2) + 1, - address, - ensName: profileData.ensName || undefined, - netPnl, - winRate, - profitFactor, - totalTrades, - personas, - openValue: accountValue, - currentBias: (Math.random() - 0.5) * 2, // Keep random bias for now - ageDays, - }); - } - - res.json({ - data: { - entries, - window, - limit, - offset, - sortBy, - total: entries.length, - updatedAt: Date.now(), - }, - }); - } catch (error) { - res.status(500).json({ - error: 'pnl_leaderboard_fetch_failed', - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - // Rekt Leaderboard - app.get('/api/v1/leaderboards/rekt', async (req: Request, res: Response) => { - if (!traderIntelligenceRedis) { - res.status(503).json({ error: 'service_unavailable', message: 'Trader intelligence service is not available' }); - return; - } - - const window = (req.query.window as string) || '24h'; - const limit = resolveLimitQuery(req.query.limit, 50, 100); - const offset = parseInt((req.query.offset as string) || '0', 10); - const sortBy = ((req.query.sortBy as string) || 'totalLoss').toLowerCase(); - const personaFilter = (req.query.persona as string | undefined)?.toLowerCase(); - - if (!['24h', '7d'].includes(window)) { - res.status(400).json({ error: 'invalid_window', message: `Invalid window: ${window}` }); - return; - } - if (!['totalloss', 'liquidationcount'].includes(sortBy)) { - res.status(400).json({ error: 'invalid_sort', message: `Invalid sortBy: ${sortBy}` }); - return; - } - - try { - const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; - const leaderboardKey = `${keyPrefix}leaderboard:rekt:${window}`; - - const entriesWithScores = await traderIntelligenceRedis.zrevrange( - leaderboardKey, - offset, - offset + limit - 1, - 'WITHSCORES' - ); - - let personaMembers: Set | null = null; - if (personaFilter) { - const personaKey = `${keyPrefix}persona:${personaFilter}:members`; - const personaAddresses = await traderIntelligenceRedis.smembers(personaKey); - if (personaAddresses && personaAddresses.length > 0) { - personaMembers = new Set(personaAddresses.map((addr) => addr.toLowerCase())); - } - } - - const entries = []; - for (let i = 0; i < entriesWithScores.length; i += 2) { - const address = entriesWithScores[i]; - const totalLoss = parseFloat(entriesWithScores[i + 1] || '0'); - - if (!address) continue; - - // Get liquidation count and biggest liquidation - const liquidationsKey = `${keyPrefix}trader:${address}:liquidations`; - const liquidations = await traderIntelligenceRedis.zrange(liquidationsKey, 0, -1, 'WITHSCORES'); - - let liquidationCount = 0; - let biggestLiquidation = 0; - - if (liquidations && liquidations.length > 0) { - liquidationCount = liquidations.length / 2; // WITHSCORES returns pairs - for (let j = 1; j < liquidations.length; j += 2) { - const loss = parseFloat(liquidations[j] || '0'); - if (loss > biggestLiquidation) { - biggestLiquidation = loss; - } - } - } - - // ENS lookup removed for performance - - // Get personas - const profileKey = `${keyPrefix}trader:${address}:profile`; - const profileData = await traderIntelligenceRedis.hgetall(profileKey); - const personas = profileData.personas ? JSON.parse(profileData.personas) : []; - - if (personaFilter) { - if (personaMembers) { - if (!personaMembers.has(address.toLowerCase())) { - continue; - } - } else if (!personas.some((p: { persona: string }) => p.persona === personaFilter)) { - continue; - } - } - - entries.push({ - rank: offset + (i / 2) + 1, - address, - ensName: undefined, - totalLoss, - liquidationCount, - biggestLiquidation, - personas, - }); - } - - if (sortBy === 'liquidationcount') { - entries.sort((a, b) => b.liquidationCount - a.liquidationCount || b.totalLoss - a.totalLoss); - } else { - entries.sort((a, b) => b.totalLoss - a.totalLoss || b.liquidationCount - a.liquidationCount); - } - - entries.forEach((entry, idx) => { - entry.rank = offset + idx + 1; - }); - - res.json({ - data: { - entries, - window, - limit, - offset, - total: entries.length, - updatedAt: Date.now(), - }, - }); - } catch (error) { - res.status(500).json({ - error: 'rekt_leaderboard_fetch_failed', - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - // Personas Leaderboard - app.get('/api/v1/leaderboards/personas', async (req: Request, res: Response) => { - if (!traderIntelligenceRedis) { - res.status(503).json({ error: 'service_unavailable', message: 'Trader intelligence service is not available' }); - return; - } - - try { - const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; - - // Get all persona stats - const personasResponse = await traderIntelligenceRedis.get(`${keyPrefix}personas`); - const personas = personasResponse ? JSON.parse(personasResponse) : []; - - res.json({ - data: { - entries: personas, - total: personas.length, - updatedAt: Date.now(), - }, - }); - } catch (error) { - res.status(500).json({ - error: 'personas_leaderboard_fetch_failed', - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - // Whale Positions Leaderboard - app.get('/api/v1/leaderboards/whales', async (req: Request, res: Response) => { - if (!traderIntelligenceRedis) { - res.status(503).json({ error: 'service_unavailable', message: 'Trader intelligence service is not available' }); - return; - } - - const minPositionUsd = parseFloat((req.query.minPositionUsd as string) || '100000'); - const coin = req.query.coin as string | undefined; - const personaFilter = (req.query.persona as string | undefined)?.toLowerCase(); - const limit = resolveLimitQuery(req.query.limit, 50, 100); - - try { - const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; - const leaderboardKey = `${keyPrefix}leaderboard:whales:positions`; - - const entriesWithScores = await traderIntelligenceRedis.zrevrange( - leaderboardKey, - 0, - limit - 1, - 'WITHSCORES' - ); - - let personaMembers: Set | null = null; - if (personaFilter) { - const personaKey = `${keyPrefix}persona:${personaFilter}:members`; - const personaAddresses = await traderIntelligenceRedis.smembers(personaKey); - if (personaAddresses && personaAddresses.length > 0) { - personaMembers = new Set(personaAddresses.map((addr) => addr.toLowerCase())); - } - } - - const entries = []; - for (let i = 0; i < entriesWithScores.length; i += 2) { - const address = entriesWithScores[i]; - const totalPositionValue = parseFloat(entriesWithScores[i + 1] || '0'); - - if (!address || totalPositionValue < minPositionUsd) continue; - - // Get positions - const positionsKey = `${keyPrefix}trader:${address}:positions`; - const positionsJson = await traderIntelligenceRedis.get(positionsKey); - let positions: Array<{ - coin: string; - side: 'long' | 'short'; - positionValue: string; - unrealizedPnl: string; - }> = []; - - if (positionsJson) { - try { - positions = JSON.parse(positionsJson); - } catch { - // Ignore parse errors - } - } - - // Filter by coin if specified - if (coin) { - positions = positions.filter(p => p.coin === coin); - if (positions.length === 0) continue; - } - - // Get ENS name - const ensKey = `${keyPrefix}ens:${address}`; - const ensData = await traderIntelligenceRedis.get(ensKey); - let ensName: string | undefined; - if (ensData) { - try { - const parsed = JSON.parse(ensData); - ensName = parsed.ensName; - } catch { - // Ignore parse errors - } - } - - // Get personas - const profileKey = `${keyPrefix}trader:${address}:profile`; - const profileData = await traderIntelligenceRedis.hgetall(profileKey); - const personas = profileData.personas ? JSON.parse(profileData.personas) : []; - - if (personaFilter) { - if (personaMembers) { - if (!personaMembers.has(address.toLowerCase())) { - continue; - } - } else if (!personas.some((p: { persona: string }) => p.persona === personaFilter)) { - continue; - } - } - - // Calculate sentiment (bullish if more long positions) - const longPositions = positions.filter(p => p.side === 'long').length; - const shortPositions = positions.filter(p => p.side === 'short').length; - const sentiment = longPositions > shortPositions ? 'bullish' : longPositions < shortPositions ? 'bearish' : 'neutral'; - - entries.push({ - rank: entries.length + 1, - address, - ensName, - totalPositionValue, - positions, - sentiment, - personas, - }); - } - - res.json({ - data: { - entries, - minPositionUsd, - coin, - limit, - total: entries.length, - updatedAt: Date.now(), - }, - }); - } catch (error) { - res.status(500).json({ - error: 'whale_positions_fetch_failed', - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - // Bot Tracker Leaderboard - app.get('/api/v1/leaderboards/bots', async (req: Request, res: Response) => { - if (!traderIntelligenceRedis) { - res.status(503).json({ error: 'service_unavailable', message: 'Trader intelligence service is not available' }); - return; - } - - const botType = req.query.botType as string | undefined; - const minUptime = parseFloat((req.query.minUptime as string) || '0'); - const limit = resolveLimitQuery(req.query.limit, 50, 100); - - try { - const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; - const leaderboardKey = `${keyPrefix}leaderboard:bots`; - - const entriesWithScores = await traderIntelligenceRedis.zrevrange( - leaderboardKey, - 0, - limit - 1, - 'WITHSCORES' - ); - - const entries = []; - for (let i = 0; i < entriesWithScores.length; i += 2) { - const address = entriesWithScores[i]; - const uptime = parseFloat(entriesWithScores[i + 1] || '0'); - - if (!address || uptime < minUptime) continue; - - // Get bot details from profile - const profileKey = `${keyPrefix}trader:${address}:profile`; - const profileData = await traderIntelligenceRedis.hgetall(profileKey); - - const personas = profileData.personas ? JSON.parse(profileData.personas) : []; - const botPersona = personas.find((p: { persona: string }) => p.persona === 'bot'); - - if (!botPersona && botType) continue; // Skip if filtering by bot type but not a bot - - // Determine bot type from personas - let detectedBotType: 'market-maker' | 'arbitrageur' | 'scalper' | 'unknown' = 'unknown'; - if (personas.some((p: { persona: string }) => p.persona === 'market-maker')) { - detectedBotType = 'market-maker'; - } else if (personas.some((p: { persona: string }) => p.persona === 'arbitrageur')) { - detectedBotType = 'arbitrageur'; - } else if (personas.some((p: { persona: string }) => p.persona === 'scalper')) { - detectedBotType = 'scalper'; - } - - if (botType && detectedBotType !== botType) continue; - - // Get PnL and trade stats - const pnlKey = `${keyPrefix}trader:${address}:pnl:24h`; - const pnlData = await traderIntelligenceRedis.hgetall(pnlKey); - - const netPnl = parseFloat(pnlData.netPnl || '0'); - const winRate = parseFloat(pnlData.winRate || '0'); - const totalTrades = parseInt(pnlData.totalTrades || '0', 10); - const tradesPerHour = totalTrades > 0 ? totalTrades / 24 : 0; // Approximate - - // Get ENS name - const ensKey = `${keyPrefix}ens:${address}`; - const ensData = await traderIntelligenceRedis.get(ensKey); - let ensName: string | undefined; - if (ensData) { - try { - const parsed = JSON.parse(ensData); - ensName = parsed.ensName; - } catch { - // Ignore parse errors - } - } - - // Get last seen timestamp - const lastSeen = parseInt(profileData.lastUpdated || '0', 10); - - entries.push({ - rank: entries.length + 1, - address, - ensName, - botType: detectedBotType, - uptime, - tradesPerHour, - netPnl, - winRate, - totalTrades, - lastSeenAt: lastSeen, - personas, - }); - } - - res.json({ - data: { - entries, - botType, - minUptime, - limit, - total: entries.length, - updatedAt: Date.now(), - }, - }); - } catch (error) { - res.status(500).json({ - error: 'bot_tracker_fetch_failed', - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - // Get PnL Snapshots for Charting - app.get('/api/v1/traders/:address/pnl-snapshots', async (req: Request, res: Response) => { - if (!traderIntelligenceRedis) { - res.status(503).json({ error: 'service_unavailable', message: 'Trader intelligence service is not available' }); - return; - } - - const { address } = req.params; - const normalizedAddress = address.toLowerCase(); - const period = (req.query.period as string) || '24h'; - const limit = parseInt((req.query.limit as string) || '1000', 10); - - if (!['24h', '7d', '30d', 'all'].includes(period)) { - res.status(400).json({ error: 'invalid_period', message: `Invalid period: ${period}` }); - return; - } - - try { - const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; - const snapshotKey = `${keyPrefix}trader:${normalizedAddress}:pnl:snapshots`; - - // Get snapshots from sorted set (most recent first) - // Score is timestamp, member is JSON string with PnL data - const snapshots = await traderIntelligenceRedis.zrevrange( - snapshotKey, - 0, - limit - 1, - 'WITHSCORES' - ); - - const data = []; - for (let i = 0; i < snapshots.length; i += 2) { - const snapshotJson = snapshots[i]; - const timestamp = parseFloat(snapshots[i + 1] || '0'); - - if (!snapshotJson) continue; - - try { - const snapshot = JSON.parse(snapshotJson); - data.push({ - timestamp, - netPnl: snapshot.netPnl || 0, - realizedPnl: snapshot.realizedPnl, - unrealizedPnl: snapshot.unrealizedPnl, - }); - } catch { - // Skip invalid JSON - continue; - } - } - - // Sort by timestamp ascending for charting - data.sort((a, b) => a.timestamp - b.timestamp); - - res.json({ - data, - period, - limit, - total: data.length, - updatedAt: Date.now(), - }); - } catch (error) { - res.status(500).json({ - error: 'pnl_snapshots_fetch_failed', - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - // Get Cross-Asset Trader Stats - if (traderIntelligenceRedis) { - app.get('/api/v1/traders/:address/cross-asset-stats', async (req: Request, res: Response) => { - const { address } = req.params; - const normalizedAddress = address.toLowerCase(); - - try { - // Find all assets - const pattern = `${traderIntelligenceKeyPrefix}trader:${normalizedAddress}:asset-points:*`; - const keys = await traderIntelligenceRedis.keys(pattern); - - const assetAllocation: Record = {}; - const loyaltyBreakdown: Record = { - maximalist: [], - devoted: [], - regular: [], - casual: [] - }; - const topRanks: Array<{ asset: string; rank: number; points: number }> = []; - let totalPoints = 0; - - for (const key of keys) { - const assetMatch = key.match(/asset-points:(.+)$/); - if (!assetMatch) continue; - - const asset = assetMatch[1]; - const pointsKey = `${traderIntelligenceKeyPrefix}trader:${normalizedAddress}:asset-points:${asset}`; - const loyaltyKey = `${traderIntelligenceKeyPrefix}trader:${normalizedAddress}:asset-loyalty:${asset}`; - const leaderboardKey = `${traderIntelligenceKeyPrefix}leaderboard:asset:${asset}:points:24h`; - - const [points, loyaltyTier, rank] = await Promise.all([ - traderIntelligenceRedis.get(pointsKey), - traderIntelligenceRedis.get(loyaltyKey), - traderIntelligenceRedis.zrevrank(leaderboardKey, normalizedAddress) - ]); - - const pointsValue = parseInt(points || '0', 10); - totalPoints += pointsValue; - - const tier = loyaltyTier || 'casual'; - if (loyaltyBreakdown[tier]) { - loyaltyBreakdown[tier].push(asset); - } - - if (rank !== null) { - topRanks.push({ - asset, - rank: rank + 1, - points: pointsValue - }); - } - } - - // Calculate allocation percentages - for (const key of keys) { - const assetMatch = key.match(/asset-points:(.+)$/); - if (!assetMatch) continue; - - const asset = assetMatch[1]; - const pointsKey = `${traderIntelligenceKeyPrefix}trader:${normalizedAddress}:asset-points:${asset}`; - const points = await traderIntelligenceRedis.get(pointsKey); - const pointsValue = parseInt(points || '0', 10); - - if (totalPoints > 0) { - assetAllocation[asset] = pointsValue / totalPoints; - } - } - - // Sort top ranks by rank - topRanks.sort((a, b) => a.rank - b.rank); - - const primaryAsset = topRanks.length > 0 ? topRanks[0].asset : ''; - - res.json({ - address: normalizedAddress, - totalAssets: keys.length, - primaryAsset, - assetAllocation, - loyaltyBreakdown, - topRanks: topRanks.slice(0, 10), - multiAssetBadges: [] // Would need to implement multi-asset badge logic - }); - } catch (error) { - res.status(500).json({ - error: 'INTERNAL_ERROR', - message: 'Failed to fetch cross-asset stats', - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - // Whale-sized pending orders - app.get('/api/v1/traders/:address/whale-orders', async (req: Request, res: Response) => { - const { address } = req.params; - const normalizedAddress = address?.toLowerCase(); - const minSizeParam = (req.query.minSizeUsd as string) || '100000'; - const minSizeUsd = parseFloat(minSizeParam); - - if (!address || !normalizedAddress.startsWith('0x')) { - res.status(400).json({ - error: 'invalid_address', - message: 'address must be a 0x-prefixed hex string' - }); - return; - } - - if (!Number.isFinite(minSizeUsd) || minSizeUsd < 0) { - res.status(400).json({ - error: 'invalid_min_size', - message: 'minSizeUsd must be a positive number' - }); - return; - } - - try { - const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; - const ordersKey = `${keyPrefix}trader:${normalizedAddress}:orders`; - const ordersJson = await traderIntelligenceRedis.get(ordersKey); - - if (!ordersJson) { - res.json({ - address: normalizedAddress, - minSizeUsd, - total: 0, - largeOrders: [], - cached: false - }); - return; - } - - let orders: Array> = []; - try { - const parsed = JSON.parse(ordersJson); - if (Array.isArray(parsed)) { - orders = parsed; - } - } catch { - res.status(500).json({ - error: 'orders_parse_failed', - message: 'Stored orders payload is malformed JSON' - }); - return; - } - - const largeOrders = orders - .map((order) => { - const limitPx = parseFloat(order.limitPx ?? order.price ?? order.px ?? '0'); - const size = parseFloat(order.sz ?? order.size ?? '0'); - const notionalFromPayload = parseFloat(order.notionalUsd ?? order.notional ?? '0'); - - let notionalUsd = Number.isFinite(notionalFromPayload) && notionalFromPayload > 0 ? notionalFromPayload : 0; - if ((!notionalUsd || notionalUsd === 0) && Number.isFinite(limitPx) && Number.isFinite(size)) { - notionalUsd = Math.abs(limitPx * size); - } - - return { - coin: order.coin, - side: order.side === 'sell' ? 'sell' : 'buy', - limitPx: order.limitPx ?? order.price ?? order.px ?? null, - sz: order.sz ?? order.size ?? null, - oid: order.oid, - timestamp: order.timestamp, - orderType: order.orderType ?? 'limit', - reduceOnly: Boolean(order.reduceOnly), - sentiment: order.side === 'sell' ? 'bearish' : 'bullish', - notionalUsd, - }; - }) - .filter((order) => Number.isFinite(order.notionalUsd) && order.notionalUsd >= minSizeUsd) - .sort((a, b) => (b.notionalUsd || 0) - (a.notionalUsd || 0)) - .slice(0, 100); - - res.json({ - address: normalizedAddress, - minSizeUsd, - total: largeOrders.length, - largeOrders, - cached: true - }); - } catch (error) { - res.status(500).json({ - error: 'whale_orders_fetch_failed', - address, - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - // Internal Operational Endpoints - // POST /api/v1/internal/prioritize-address - Boost address to high priority - app.post('/api/v1/internal/prioritize-address', async (req: Request, res: Response) => { - const { address, priority = 'high', duration } = req.body; - - if (!address) { - res.status(400).json({ - error: 'INVALID_REQUEST', - message: 'Address is required' - }); - return; - } - - try { - const normalizedAddress = address.toLowerCase(); - const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; - const jobKey = `${keyPrefix}job:${normalizedAddress}:${priority}`; - - // Calculate scheduled time (immediate for high priority, or based on duration) - const scheduledAt = Date.now(); - const durationMs = duration || (priority === 'high' ? 30 * 1000 : 5 * 60 * 1000); - - const job = { - address: normalizedAddress, - priority, - scheduledAt, - retryCount: 0, - }; - - // Store job in Redis - await traderIntelligenceRedis.set( - jobKey, - JSON.stringify(job), - 'EX', - Math.floor(durationMs / 1000) - ); - - // Add to priority queue - const queueKey = `${keyPrefix}queue:${priority}`; - await traderIntelligenceRedis.zadd(queueKey, scheduledAt, normalizedAddress); - - res.json({ - success: true, - address: normalizedAddress, - priority, - nextCollection: scheduledAt, - duration: durationMs, - }); - } catch (error) { - res.status(500).json({ - error: 'INTERNAL_ERROR', - message: 'Failed to prioritize address', - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - - // DELETE /api/v1/internal/cache/:address - Clear cached profile - app.delete('/api/v1/internal/cache/:address', async (req: Request, res: Response) => { - const { address } = req.params; - const normalizedAddress = address.toLowerCase(); - - try { - const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; - - // Find all keys for this trader - const pattern = `${keyPrefix}trader:${normalizedAddress}:*`; - const keys = await traderIntelligenceRedis.keys(pattern); - - // Also check job keys - const jobPattern = `${keyPrefix}job:${normalizedAddress}:*`; - const jobKeys = await traderIntelligenceRedis.keys(jobPattern); - - // Also check queue entries - const queuePattern = `${keyPrefix}queue:*`; - const queueKeys = await traderIntelligenceRedis.keys(queuePattern); - - const allKeys = [...keys, ...jobKeys]; - - // Remove from queues - for (const queueKey of queueKeys) { - await traderIntelligenceRedis.zrem(queueKey, normalizedAddress); - } - - // Delete all keys - if (allKeys.length > 0) { - await traderIntelligenceRedis.del(...allKeys); - } - - res.json({ - success: true, - address: normalizedAddress, - cleared: true, - keysDeleted: allKeys.length, - }); - } catch (error) { - res.status(500).json({ - error: 'INTERNAL_ERROR', - message: 'Failed to clear cache', - detail: error instanceof Error ? error.message : String(error) - }); - } - }); - } - - app.get('/metrics', async (_req: Request, res: Response) => { - res.set('Content-Type', metricsRegistry.contentType); - res.send(await metricsRegistry.metrics()); - }); - - const httpServer = createServer(app); - - // Return cleanup function to clear intervals on shutdown - const cleanup = () => { - clearInterval(candleFetchCleanupInterval); - if (statsRedisClient) { - statsRedisClient.quit().catch(() => undefined); - } - if (traderIntelligenceRedis && traderIntelligenceRedis !== redis) { - traderIntelligenceRedis.quit().catch(() => undefined); } + disposeHttpContext(); }; return { app, httpServer, cleanup }; }; - -const resolveLimitQuery = (raw: unknown, defaultValue: number, maxValue: number): number => { - if (Array.isArray(raw)) { - return resolveLimitQuery(raw[0], defaultValue, maxValue); - } - if (typeof raw === 'string') { - const parsed = Number(raw); - if (Number.isInteger(parsed) && parsed > 0) { - return Math.min(parsed, maxValue); - } - } else if (typeof raw === 'number' && Number.isFinite(raw) && raw > 0) { - return Math.min(Math.floor(raw), maxValue); - } - return defaultValue; -}; - -const resolveIntervalQuery = ( - raw: unknown, - defaultKey: string -): { key: string; seconds: number } => { - if (Array.isArray(raw)) { - return resolveIntervalQuery(raw[0], defaultKey); - } - - const fallback = { - key: defaultKey, - seconds: hyperliquidIntervalToSeconds(defaultKey) - } as const; - - if (typeof raw === 'string') { - const trimmed = raw.trim().toLowerCase(); - if (trimmed.length === 0) { - return fallback; - } - - if (isSupportedHyperliquidInterval(trimmed)) { - return { - key: trimmed, - seconds: hyperliquidIntervalToSeconds(trimmed) - }; - } - - const numeric = Number(trimmed); - if (Number.isFinite(numeric) && numeric > 0) { - const matched = matchIntervalBySeconds(Math.floor(numeric)); - if (matched) { - return matched; - } - } - return fallback; - } - - if (typeof raw === 'number' && Number.isFinite(raw) && raw > 0) { - const matched = matchIntervalBySeconds(Math.floor(raw)); - if (matched) { - return matched; - } - } - - return fallback; -}; - -const matchIntervalBySeconds = (seconds: number): { key: string; seconds: number } | null => { - for (const [key, value] of Object.entries(HYPERLIQUID_INTERVAL_SECONDS)) { - if (value === seconds) { - return { key, seconds: value }; - } - } - return null; -}; - -const resolveDepthQuery = (raw: unknown): number => { - if (Array.isArray(raw)) { - return resolveDepthQuery(raw[0]); - } - if (typeof raw === 'string') { - const parsed = Number(raw); - if (Number.isInteger(parsed) && parsed > 0) { - return parsed; - } - } - return 20; -}; - - diff --git a/services/gateway/src/server/http/context.ts b/services/gateway/src/server/http/context.ts new file mode 100644 index 00000000..ef98f415 --- /dev/null +++ b/services/gateway/src/server/http/context.ts @@ -0,0 +1,107 @@ +import type { Application } from 'express'; +import Redis from 'ioredis'; +import type { RedisKeyConfig } from '@pmon/ingestion-sdk'; + +import type { ProviderSummary } from '../../config'; +import type { MarketStore } from '../../store/priceStore'; +import type { OverviewStore } from '../../store/overviewStore'; +import type { StatsStore } from '../../store/statsStore'; +import type { GatewayHealth } from '../../runtime/health'; + +export type HttpServerOptions = { + providerId: string; + providerLabel?: string; + providers?: ProviderSummary[]; + statsStore?: StatsStore; + statsRedis?: { url: string; namespace: string }; + redis?: Redis; + resolveProviderRedisKeyConfig?: (providerId: string) => RedisKeyConfig | undefined; + scheduleBackfill?: (provider?: string, asset?: string | null, intervals?: string[]) => void; +}; + +type CreateHttpContextArgs = { + app: Application; + marketStore: MarketStore; + overviewStore: OverviewStore; + getHealth: () => GatewayHealth; + options?: HttpServerOptions; +}; + +export type HttpServerContext = { + app: Application; + marketStore: MarketStore; + overviewStore: OverviewStore; + getHealth: () => GatewayHealth; + defaultProvider?: string; + providers: ProviderSummary[]; + statsStore?: StatsStore; + statsRedis?: { url: string; namespace: string }; + statsRedisClient: Redis | null; + redis?: Redis; + traderIntelligenceRedis: Redis | null; + traderIntelligenceKeyPrefix: string; + resolveProviderRedisKeyConfig?: (providerId: string) => RedisKeyConfig | undefined; + scheduleBackfill?: (provider?: string, asset?: string | null, intervals?: string[]) => void; +}; + +export type HttpContextResult = { + context: HttpServerContext; + cleanup: () => void; +}; + +const createRedisClient = (url: string): Redis => + new Redis(url, { + retryStrategy: (times) => Math.min(times * 500, 5000), + maxRetriesPerRequest: 3 + }); + +export const createHttpContext = ({ + app, + marketStore, + overviewStore, + getHealth, + options +}: CreateHttpContextArgs): HttpContextResult => { + const defaultProvider = options?.providerId; + const providers = options?.providers ?? []; + const statsStore = options?.statsStore; + const statsRedis = options?.statsRedis; + const redis = options?.redis; + + const statsRedisClient = statsRedis ? createRedisClient(statsRedis.url) : null; + const traderIntelligenceRedis = + redis || (process.env.TRADER_INTEL_REDIS_URL ? createRedisClient(process.env.TRADER_INTEL_REDIS_URL) : null); + const traderIntelligenceKeyPrefix = + process.env.REDIS_KEY_PREFIX || process.env.TRADER_INTEL_REDIS_KEY_PREFIX || 'intelligence:'; + + const cleanup = () => { + if (statsRedisClient) { + statsRedisClient.quit().catch(() => undefined); + } + if (traderIntelligenceRedis && traderIntelligenceRedis !== redis) { + traderIntelligenceRedis.quit().catch(() => undefined); + } + }; + + return { + context: { + app, + marketStore, + overviewStore, + getHealth, + defaultProvider, + providers, + statsStore, + statsRedis, + statsRedisClient, + redis, + traderIntelligenceRedis, + traderIntelligenceKeyPrefix, + resolveProviderRedisKeyConfig: options?.resolveProviderRedisKeyConfig, + scheduleBackfill: options?.scheduleBackfill + }, + cleanup + }; +}; + + diff --git a/services/gateway/src/server/http/docs.ts b/services/gateway/src/server/http/docs.ts new file mode 100644 index 00000000..22da9363 --- /dev/null +++ b/services/gateway/src/server/http/docs.ts @@ -0,0 +1,46 @@ +import fs from 'fs'; +import swaggerAutogen from 'swagger-autogen'; +import { apiReference } from '@scalar/express-api-reference'; + +import type { HttpServerContext } from './context'; + +type DocsConfig = { + endpointsFiles: string[]; + host?: string; +}; + +export const initializeApiDocs = async ( + ctx: HttpServerContext, + { endpointsFiles, host }: DocsConfig +): Promise => { + const { app } = ctx; + const doc = { + info: { + title: 'Gateway API', + description: 'Gateway Service API Documentation' + }, + host: host ?? `localhost:${process.env.PORT || 4000}`, + schemes: ['http'] + }; + + try { + const outputFile = `/tmp/swagger_output_${Date.now()}.json`; + await swaggerAutogen()(outputFile, endpointsFiles, doc); + + if (fs.existsSync(outputFile)) { + const swaggerFile = JSON.parse(fs.readFileSync(outputFile, 'utf8')); + app.use( + '/docs', + apiReference({ + spec: { + content: swaggerFile + } + }) + ); + fs.unlinkSync(outputFile); + } + } catch (error) { + console.warn('Failed to generate API documentation:', error); + } +}; + diff --git a/services/gateway/src/server/http/ens.ts b/services/gateway/src/server/http/ens.ts new file mode 100644 index 00000000..8c2f42e0 --- /dev/null +++ b/services/gateway/src/server/http/ens.ts @@ -0,0 +1,46 @@ +import type { Redis } from 'ioredis'; + +export type EnsMetadata = { + ensName?: string; + ensAvatar?: string; +}; + +export const readEnsMetadata = async ( + redis: Redis | null, + keyPrefix: string, + address: string +): Promise => { + if (!address || !redis) { + return {}; + } + + const ensKey = `${keyPrefix}ens:${address}`; + + try { + const hash = await redis.hgetall(ensKey); + if (hash && Object.keys(hash).length > 0) { + return { + ensName: hash.ensName, + ensAvatar: hash.ensAvatar, + }; + } + } catch { + // Ignore hash fetch failures and fall back to GET + } + + try { + const raw = await redis.get(ensKey); + if (raw) { + const parsed = JSON.parse(raw) as EnsMetadata; + return { + ensName: parsed?.ensName, + ensAvatar: parsed?.ensAvatar, + }; + } + } catch { + // Ignore JSON parsing failures + } + + return {}; +}; + diff --git a/services/gateway/src/server/http/metrics.ts b/services/gateway/src/server/http/metrics.ts new file mode 100644 index 00000000..8842ae5c --- /dev/null +++ b/services/gateway/src/server/http/metrics.ts @@ -0,0 +1,57 @@ +import { writeHeapSnapshot } from 'node:v8'; + +import type { HttpServerContext } from './context'; +import { buildHealthSnapshot } from './utils'; +import { metricsRegistry } from '../../metrics'; + +export const registerHealthAndMetricsRoutes = (ctx: HttpServerContext): void => { + const { app, getHealth, defaultProvider } = ctx; + const readHealthSnapshot = () => buildHealthSnapshot({ getHealth, defaultProvider }); + + app.get('/healthz', (_req, res) => { + const { health, evaluation } = readHealthSnapshot(); + const { fatalReasons, warnings } = evaluation; + + res.status(200).json({ + status: 'ok', + reasons: fatalReasons.length > 0 ? fatalReasons : undefined, + warnings: warnings.length > 0 ? warnings : undefined, + health + }); + }); + + app.get('/readyz', (_req, res) => { + const { health, evaluation } = readHealthSnapshot(); + const { ready, fatalReasons, warnings } = evaluation; + res.status(ready ? 200 : 503).json({ + status: ready ? 'ready' : 'degraded', + reasons: ready ? undefined : fatalReasons, + warnings: warnings.length > 0 ? warnings : undefined, + health + }); + }); + + if ((process.env.GATEWAY_ENABLE_HEAPDUMP_ENDPOINT ?? '').trim().toLowerCase() === 'true') { + app.post('/admin/heapdump', (_req, res) => { + try { + const filename = writeHeapSnapshot(`/tmp/gateway-heap-${Date.now()}.heapsnapshot`); + res.json({ + status: 'ok', + heapSnapshot: filename + }); + } catch (error) { + res.status(500).json({ + status: 'error', + message: error instanceof Error ? error.message : String(error) + }); + } + }); + } + + app.get('/metrics', async (_req, res) => { + res.set('Content-Type', metricsRegistry.contentType); + res.send(await metricsRegistry.metrics()); + }); +}; + + diff --git a/services/gateway/src/server/http/utils.ts b/services/gateway/src/server/http/utils.ts new file mode 100644 index 00000000..5594eea4 --- /dev/null +++ b/services/gateway/src/server/http/utils.ts @@ -0,0 +1,136 @@ +import { HYPERLIQUID_INTERVAL_SECONDS, hyperliquidIntervalToSeconds, isSupportedHyperliquidInterval } from '../../clients/hyperliquidCandles'; +import { setBboAge } from '../../metrics'; +import type { GatewayHealth } from '../../runtime/health'; +import { evaluateGatewayReadiness } from '../../runtime/readiness'; + +type HealthSnapshotArgs = { + getHealth: () => GatewayHealth; + defaultProvider?: string; +}; + +export const buildHealthSnapshot = ({ getHealth, defaultProvider }: HealthSnapshotArgs) => { + const health = getHealth(); + const now = Date.now(); + const bboStates = health.streams?.bbo ?? {}; + const derivedBbo: Record = {}; + let defaultProviderAge: number | undefined; + + for (const [provider, state] of Object.entries(bboStates)) { + const lastTimestamp = state?.lastTimestamp; + const ageMs = + typeof lastTimestamp === 'number' && Number.isFinite(lastTimestamp) + ? Math.max(0, now - lastTimestamp) + : undefined; + derivedBbo[provider] = { + lastTimestamp, + ageMs + }; + if (defaultProvider && provider.toLowerCase() === defaultProvider.toLowerCase()) { + defaultProviderAge = ageMs; + } + } + + setBboAge(defaultProviderAge); + + const healthWithDerived: GatewayHealth = { + ...health, + streams: { + ...health.streams, + bbo: derivedBbo + } + }; + const evaluation = evaluateGatewayReadiness(healthWithDerived); + + return { health: healthWithDerived, evaluation }; +}; + +export const resolveLimitQuery = (raw: unknown, defaultValue: number, maxValue: number): number => { + if (Array.isArray(raw)) { + return resolveLimitQuery(raw[0], defaultValue, maxValue); + } + if (typeof raw === 'string') { + const parsed = Number(raw); + if (Number.isInteger(parsed) && parsed > 0) { + return Math.min(parsed, maxValue); + } + } else if (typeof raw === 'number' && Number.isFinite(raw) && raw > 0) { + return Math.min(Math.floor(raw), maxValue); + } + return defaultValue; +}; + +export const resolveIntervalQuery = ( + raw: unknown, + defaultKey: string +): { key: string; seconds: number } => { + if (Array.isArray(raw)) { + return resolveIntervalQuery(raw[0], defaultKey); + } + + const fallback = { + key: defaultKey, + seconds: hyperliquidIntervalToSeconds(defaultKey) + } as const; + + if (typeof raw === 'string') { + const trimmed = raw.trim().toLowerCase(); + if (trimmed.length === 0) { + return fallback; + } + + if (isSupportedHyperliquidInterval(trimmed)) { + return { + key: trimmed, + seconds: hyperliquidIntervalToSeconds(trimmed) + }; + } + + const numeric = Number(trimmed); + if (Number.isFinite(numeric) && numeric > 0) { + const matched = matchIntervalBySeconds(Math.floor(numeric)); + if (matched) { + return matched; + } + } + return fallback; + } + + if (typeof raw === 'number' && Number.isFinite(raw) && raw > 0) { + const matched = matchIntervalBySeconds(Math.floor(raw)); + if (matched) { + return matched; + } + } + + return fallback; +}; + +const matchIntervalBySeconds = (seconds: number): { key: string; seconds: number } | null => { + for (const [key, value] of Object.entries(HYPERLIQUID_INTERVAL_SECONDS)) { + if (value === seconds) { + return { key, seconds: value }; + } + } + return null; +}; + +export const resolveDepthQuery = (raw: unknown): number => { + if (Array.isArray(raw)) { + return resolveDepthQuery(raw[0]); + } + if (typeof raw === 'string') { + const parsed = Number(raw); + if (Number.isInteger(parsed) && parsed > 0) { + return parsed; + } + } + return 20; +}; + +export const resolveWindow = (raw: unknown): '1h' | '24h' | '7d' => { + if (typeof raw === 'string' && (raw === '1h' || raw === '24h' || raw === '7d')) { + return raw; + } + return '24h'; +}; + diff --git a/services/gateway/src/server/routes/achievements/index.ts b/services/gateway/src/server/routes/achievements/index.ts new file mode 100644 index 00000000..c462adf6 --- /dev/null +++ b/services/gateway/src/server/routes/achievements/index.ts @@ -0,0 +1,75 @@ +import type { Request, Response } from 'express'; +import type { HttpServerContext } from '../../http/context'; +export const registerAchievementRoutes = (ctx: HttpServerContext): void => { + const { app, traderIntelligenceRedis, traderIntelligenceKeyPrefix } = ctx; + + if (!traderIntelligenceRedis) { + return; + } + + const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; + + // Fetch viral achievements for a trader + app.get('/api/v1/achievements/:address', async (req: Request, res: Response) => { + const { address } = req.params; + + try { + const { fetchViralAchievements } = await import('../../../redis/leaderboard'); + const achievements = await fetchViralAchievements(traderIntelligenceRedis, traderIntelligenceKeyPrefix, address); + res.json({ data: achievements, address }); + } catch (error) { + res.status(500).json({ + error: 'achievements_fetch_failed', + address, + detail: error instanceof Error ? error.message : String(error) + }); + } + }); + + // Get achievement card + app.get('/api/v1/achievements/:achievementId/card', async (req: Request, res: Response) => { + const { achievementId } = req.params; + const { trader } = req.query; + + if (!trader || typeof trader !== 'string') { + res.status(400).json({ error: 'trader_required', message: 'Trader address is required' }); + return; + } + + try { + const normalizedAddress = trader.toLowerCase(); + const achievementKey = `${keyPrefix}viral-achievement:${achievementId}:${normalizedAddress}`; + + const achievementData = await traderIntelligenceRedis.hgetall(achievementKey); + + if (!achievementData || !achievementData.id) { + res.status(404).json({ error: 'achievement_not_found', achievementId }); + return; + } + + res.json({ + achievement: { + id: achievementData.id, + name: achievementData.name, + description: achievementData.description, + emoji: achievementData.emoji, + rarity: achievementData.rarity, + category: achievementData.category, + funnyQuip: achievementData.funnyQuip, + pointsBonus: parseInt(achievementData.pointsBonus || '0', 10), + unlockedAt: parseInt(achievementData.unlockedAt || '0', 10), + }, + trader: { + address: normalizedAddress, + }, + shareUrl: `${process.env.APP_URL || 'https://app.perps.gmbh'}/achievements/${achievementId}?trader=${normalizedAddress}`, + }); + } catch (error) { + res.status(500).json({ + error: 'card_generation_failed', + detail: error instanceof Error ? error.message : String(error) + }); + } + }); +}; + diff --git a/services/gateway/src/server/routes/communities.ts b/services/gateway/src/server/routes/communities.ts new file mode 100644 index 00000000..0d8eb271 --- /dev/null +++ b/services/gateway/src/server/routes/communities.ts @@ -0,0 +1,307 @@ +import type { HttpServerContext } from '../http/context'; +import { resolveLimitQuery } from '../http/utils'; + +type EnsReader = (address: string) => Promise<{ ensName?: string; ensAvatar?: string }>; + +type LeaderboardPersona = { + persona: string; + confidence?: number; + isPrimary?: boolean; +}; + +const VALID_COMMUNITY_LEADERBOARDS = ['points', 'volume', 'pnl', 'loyalty'] as const; +const VALID_WINDOWS = ['24h', '7d', '30d', 'all'] as const; + +const isValidLeaderboardType = (type: string): type is (typeof VALID_COMMUNITY_LEADERBOARDS)[number] => + VALID_COMMUNITY_LEADERBOARDS.includes(type as any); + +const isValidWindow = (window: string): window is (typeof VALID_WINDOWS)[number] => + VALID_WINDOWS.includes(window as any); + +export const registerCommunityRoutes = ( + ctx: HttpServerContext, + deps: { readEnsMetadata: EnsReader } +): void => { + const { app, traderIntelligenceRedis, traderIntelligenceKeyPrefix } = ctx; + if (!traderIntelligenceRedis) { + return; + } + + const { readEnsMetadata } = deps; + + app.get('/api/v1/communities/:asset', async (req, res) => { + const { asset } = req.params; + + try { + const statsKey = `${traderIntelligenceKeyPrefix}community:${asset}:stats`; + const statsData = await traderIntelligenceRedis.hgetall(statsKey); + + if (!statsData || Object.keys(statsData).length === 0) { + res.status(404).json({ error: 'ASSET_NOT_FOUND', message: `Asset ${asset} not found` }); + return; + } + + const topTraderAddress = statsData.topTraderAddress || ''; + const topTraderEns = await readEnsMetadata(topTraderAddress); + + res.json({ + asset, + displayName: statsData.displayName || asset, + emoji: statsData.emoji || '💰', + color: statsData.color || '#000000', + stats: { + totalTraders: parseInt(statsData.totalTraders || '0', 10), + activeTraders24h: parseInt(statsData.activeTraders24h || '0', 10), + totalVolume24h: parseFloat(statsData.totalVolume24h || '0'), + totalVolumeAllTime: parseFloat(statsData.totalVolumeAllTime || '0') + }, + topTrader: { + address: topTraderAddress, + ensName: topTraderEns.ensName, + points: parseInt(statsData.topTraderPoints || '0', 10), + badges: parseInt(statsData.topTraderBadges || '0', 10) + }, + topVolume: { + address: statsData.topVolumeAddress || '', + volume24h: parseFloat(statsData.topVolume24h || '0') + }, + topPnL: { + address: statsData.topPnLAddress || '', + pnl24h: parseFloat(statsData.topPnL24h || '0') + }, + sentiment: (statsData.sentiment || 'neutral') as 'bullish' | 'bearish' | 'neutral', + averageLeverage: parseFloat(statsData.averageLeverage || '0'), + longShortRatio: parseFloat(statsData.longShortRatio || '1.0') + }); + } catch (error) { + res.status(500).json({ + error: 'INTERNAL_ERROR', + message: 'Failed to fetch asset community overview', + detail: error instanceof Error ? error.message : String(error) + }); + } + }); + + app.get('/api/v1/communities/:asset/leaderboard/:type', async (req, res) => { + const { asset, type } = req.params; + const window = (req.query.window as string) || '24h'; + const limit = resolveLimitQuery(req.query.limit, 50, 100); + const offset = resolveLimitQuery(req.query.offset, 0, 1000); + + if (!isValidLeaderboardType(type)) { + res.status(400).json({ error: 'INVALID_LEADERBOARD_TYPE', message: `Invalid leaderboard type: ${type}` }); + return; + } + + if (!isValidWindow(window)) { + res.status(400).json({ error: 'INVALID_WINDOW', message: `Invalid window: ${window}` }); + return; + } + + try { + const leaderboardKey = `${traderIntelligenceKeyPrefix}leaderboard:asset:${asset}:${type}:${window}`; + const totalTraders = await traderIntelligenceRedis.zcard(leaderboardKey); + const entriesWithScores = await traderIntelligenceRedis.zrevrange( + leaderboardKey, + offset, + offset + limit - 1, + 'WITHSCORES' + ); + + const entries = []; + for (let i = 0; i < entriesWithScores.length; i += 2) { + const address = entriesWithScores[i]; + const score = parseFloat(entriesWithScores[i + 1] || '0'); + const rank = offset + i / 2 + 1; + + const pointsKey = `${traderIntelligenceKeyPrefix}trader:${address}:asset-points:${asset}`; + const breakdownKey = `${traderIntelligenceKeyPrefix}trader:${address}:asset-breakdown:${asset}`; + const loyaltyKey = `${traderIntelligenceKeyPrefix}trader:${address}:asset-loyalty:${asset}`; + const badgesKey = `${traderIntelligenceKeyPrefix}trader:${address}:asset-badges:${asset}`; + const ensKey = `${traderIntelligenceKeyPrefix}ens:${address}`; + const profileKey = `${traderIntelligenceKeyPrefix}trader:${address}:profile`; + + const [ + points, + breakdownJson, + loyaltyTier, + badgeIds, + ensData, + personasJson, + primaryPersona, + ] = await Promise.all([ + traderIntelligenceRedis.get(pointsKey), + traderIntelligenceRedis.get(breakdownKey), + traderIntelligenceRedis.get(loyaltyKey), + traderIntelligenceRedis.smembers(badgesKey), + traderIntelligenceRedis.get(ensKey), + traderIntelligenceRedis.hget(profileKey, 'personas'), + traderIntelligenceRedis.hget(profileKey, 'primaryPersona'), + ]); + + let ensName: string | undefined; + if (ensData) { + try { + const parsed = JSON.parse(ensData); + ensName = parsed.ensName; + } catch { + // ignore + } + } + + const breakdown = breakdownJson ? JSON.parse(breakdownJson) : null; + let personas: LeaderboardPersona[] | undefined; + if (personasJson) { + try { + const parsed = JSON.parse(personasJson); + if (Array.isArray(parsed)) { + personas = parsed + .map((persona) => { + if (!persona) { + return null; + } + + if (typeof persona === 'string') { + return { + persona, + isPrimary: primaryPersona ? persona === primaryPersona : undefined, + }; + } + + if (typeof persona === 'object') { + const personaObj = persona as { + persona?: string; + id?: string; + confidence?: number; + isPrimary?: boolean; + }; + const personaName = personaObj.persona ?? personaObj.id; + + if (!personaName) { + return null; + } + + return { + persona: personaName, + confidence: + typeof personaObj.confidence === 'number' + ? personaObj.confidence + : undefined, + isPrimary: + typeof personaObj.isPrimary === 'boolean' + ? personaObj.isPrimary + : primaryPersona + ? personaName === primaryPersona + : undefined, + }; + } + + return null; + }) + .filter(Boolean) as LeaderboardPersona[]; + } + } catch { + personas = undefined; + } + } + const badges = []; + for (const badgeId of badgeIds.slice(0, 5)) { + const badgeKey = `${traderIntelligenceKeyPrefix}badge:${badgeId}:${address}`; + const badgeData = await traderIntelligenceRedis.hgetall(badgeKey); + if (badgeData && badgeData.id) { + badges.push({ + id: badgeData.id, + emoji: badgeData.emoji || '🏆', + rarity: badgeData.rarity || 'common' + }); + } + } + + entries.push({ + rank, + address, + ensName, + assetPoints: parseInt(points || '0', 10), + assetVolume: breakdown?.components?.volume || 0, + assetPnl: breakdown?.components?.pnl || 0, + loyaltyTier: loyaltyTier || 'casual', + assetBadges: badges, + personas, + score + }); + } + + res.json({ + asset, + type, + window, + entries, + totalTraders, + pagination: { + limit, + offset, + hasMore: offset + limit < totalTraders + }, + updatedAt: Date.now() + }); + } catch (error) { + res.status(500).json({ + error: 'INTERNAL_ERROR', + message: 'Failed to fetch leaderboard', + detail: error instanceof Error ? error.message : String(error) + }); + } + }); + + app.get('/api/v1/communities/:asset/badges', async (req, res) => { + const { asset } = req.params; + const rarity = req.query.rarity as string | undefined; + const category = req.query.category as string | undefined; + + try { + const { getBadgeCatalogForAsset, filterBadges, getBadgeStats } = await import( + '@pmon/trader-intelligence/dist/points/badgeCatalog' + ); + + let badges = getBadgeCatalogForAsset(asset); + + if (rarity || category) { + badges = filterBadges(badges, rarity, category); + } + + const stats = getBadgeStats(badges); + + res.json({ + asset, + badges: badges.map((b) => ({ + id: b.id, + name: b.name, + description: b.description, + emoji: b.emoji, + rarity: b.rarity, + pointsBonus: b.pointsBonus, + requirements: b.requirements, + category: b.category + })), + totalBadges: badges.length, + byRarity: stats + }); + } catch (error) { + res.json({ + asset, + badges: [], + totalBadges: 0, + byRarity: { + common: 0, + rare: 0, + epic: 0, + legendary: 0, + mythic: 0 + }, + message: 'Badge catalog temporarily unavailable' + }); + } + }); +}; + + diff --git a/services/gateway/src/server/routes/internal/index.ts b/services/gateway/src/server/routes/internal/index.ts new file mode 100644 index 00000000..976aa575 --- /dev/null +++ b/services/gateway/src/server/routes/internal/index.ts @@ -0,0 +1,105 @@ +import type { Request, Response } from 'express'; +import type { HttpServerContext } from '../../http/context'; + +export const registerInternalRoutes = (ctx: HttpServerContext): void => { + const { app, traderIntelligenceRedis, traderIntelligenceKeyPrefix } = ctx; + + if (!traderIntelligenceRedis) { + return; + } + + const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; + + // POST /api/v1/internal/prioritize-address - Boost address to high priority + app.post('/api/v1/internal/prioritize-address', async (req: Request, res: Response) => { + const { address, priority = 'high', duration } = req.body; + + if (!address) { + res.status(400).json({ + error: 'INVALID_REQUEST', + message: 'Address is required' + }); + return; + } + + try { + const normalizedAddress = address.toLowerCase(); + const jobKey = `${keyPrefix}job:${normalizedAddress}:${priority}`; + + const scheduledAt = Date.now(); + const durationMs = duration || (priority === 'high' ? 30 * 1000 : 5 * 60 * 1000); + + const job = { + address: normalizedAddress, + priority, + scheduledAt, + retryCount: 0, + }; + + await traderIntelligenceRedis.set( + jobKey, + JSON.stringify(job), + 'EX', + Math.floor(durationMs / 1000) + ); + + const queueKey = `${keyPrefix}queue:${priority}`; + await traderIntelligenceRedis.zadd(queueKey, scheduledAt, normalizedAddress); + + res.json({ + success: true, + address: normalizedAddress, + priority, + nextCollection: scheduledAt, + duration: durationMs, + }); + } catch (error) { + res.status(500).json({ + error: 'INTERNAL_ERROR', + message: 'Failed to prioritize address', + detail: error instanceof Error ? error.message : String(error) + }); + } + }); + + // DELETE /api/v1/internal/cache/:address - Clear cached profile + app.delete('/api/v1/internal/cache/:address', async (req: Request, res: Response) => { + const { address } = req.params; + const normalizedAddress = address.toLowerCase(); + + try { + const pattern = `${keyPrefix}trader:${normalizedAddress}:*`; + const keys = await traderIntelligenceRedis.keys(pattern); + + const jobPattern = `${keyPrefix}job:${normalizedAddress}:*`; + const jobKeys = await traderIntelligenceRedis.keys(jobPattern); + + const queuePattern = `${keyPrefix}queue:*`; + const queueKeys = await traderIntelligenceRedis.keys(queuePattern); + + const allKeys = [...keys, ...jobKeys]; + + for (const queueKey of queueKeys) { + await traderIntelligenceRedis.zrem(queueKey, normalizedAddress); + } + + if (allKeys.length > 0) { + await traderIntelligenceRedis.del(...allKeys); + } + + res.json({ + success: true, + address: normalizedAddress, + cleared: true, + keysDeleted: allKeys.length, + }); + } catch (error) { + res.status(500).json({ + error: 'INTERNAL_ERROR', + message: 'Failed to clear cache', + detail: error instanceof Error ? error.message : String(error) + }); + } + }); +}; + diff --git a/services/gateway/src/server/routes/leaderboards.ts b/services/gateway/src/server/routes/leaderboards.ts new file mode 100644 index 00000000..99c1bfc4 --- /dev/null +++ b/services/gateway/src/server/routes/leaderboards.ts @@ -0,0 +1,126 @@ +import type { HttpServerContext } from '../http/context'; +import { resolveLimitQuery, resolveWindow } from '../http/utils'; +import { fetchLeaderboard, fetchTraderProfile, type TimeWindow, type TraderRole } from '../../redis/leaderboard'; + +const resolveRole = (raw: unknown): TraderRole => { + if (typeof raw === 'string' && (raw === 'maker' || raw === 'taker')) { + return raw as TraderRole; + } + return 'maker'; +}; + +const resolveWindowForLeaderboard = (raw: unknown): TimeWindow => { + const window = resolveWindow(raw); + return window as TimeWindow; +}; + +export const registerStatsLeaderboards = (ctx: HttpServerContext): void => { + const { app, defaultProvider, statsRedis, statsRedisClient } = ctx; + if (!statsRedisClient || !statsRedis) { + return; + } + + app.get('/api/v1/leaderboard/:role', async (req, res) => { + if (!defaultProvider) { + res.status(400).json({ error: 'provider_not_configured' }); + return; + } + const role = resolveRole(req.params.role); + const window = resolveWindowForLeaderboard(req.query.window); + const limit = resolveLimitQuery(req.query.limit, 50, 100); + + try { + const entries = await fetchLeaderboard( + statsRedisClient, + statsRedis.namespace, + defaultProvider, + 'global', + role, + window, + limit + ); + const totalVolume = entries.reduce((sum, e) => sum + e.totalVolume, 0); + + res.json({ + data: { + window, + provider: defaultProvider, + totalVolume, + entries, + updatedAt: Date.now() + } + }); + } catch (error) { + res.status(500).json({ + error: 'leaderboard_fetch_failed', + detail: error instanceof Error ? error.message : String(error) + }); + } + }); + + app.get('/api/v1/providers/:provider/assets/:asset/leaderboard/:role', async (req, res) => { + const { provider, asset, role: roleParam } = req.params; + const role = resolveRole(roleParam); + const window = resolveWindowForLeaderboard(req.query.window); + const limit = resolveLimitQuery(req.query.limit, 50, 100); + + try { + const entries = await fetchLeaderboard( + statsRedisClient, + statsRedis.namespace, + provider, + asset, + role, + window, + limit + ); + const totalVolume = entries.reduce((sum, e) => sum + e.totalVolume, 0); + + res.json({ + data: { + window, + provider, + asset, + totalVolume, + entries, + updatedAt: Date.now() + } + }); + } catch (error) { + res.status(500).json({ + error: 'leaderboard_fetch_failed', + asset, + provider, + detail: error instanceof Error ? error.message : String(error) + }); + } + }); + + app.get('/api/v1/traders/:address/stats', async (req, res) => { + const { address } = req.params; + const window = resolveWindowForLeaderboard(req.query.window); + const limit = resolveLimitQuery(req.query.limit, 50, 100); + + try { + const profile = await fetchTraderProfile( + statsRedisClient, + statsRedis.namespace, + address, + window + ); + if (!profile) { + res.status(404).json({ error: 'trader_not_found', address }); + return; + } + + res.json({ data: profile, window, limit }); + } catch (error) { + res.status(500).json({ + error: 'trader_profile_fetch_failed', + address, + detail: error instanceof Error ? error.message : String(error) + }); + } + }); +}; + diff --git a/services/gateway/src/server/routes/leaderboards/trader-intelligence.ts b/services/gateway/src/server/routes/leaderboards/trader-intelligence.ts new file mode 100644 index 00000000..5c399891 --- /dev/null +++ b/services/gateway/src/server/routes/leaderboards/trader-intelligence.ts @@ -0,0 +1,482 @@ +import type { Request, Response } from 'express'; +import type { HttpServerContext } from '../../http/context'; +import { resolveLimitQuery, resolveWindow } from '../../http/utils'; +import { readEnsMetadata } from '../../http/ens'; + +export const registerTraderIntelligenceLeaderboards = (ctx: HttpServerContext): void => { + const { app, traderIntelligenceRedis, traderIntelligenceKeyPrefix } = ctx; + + if (!traderIntelligenceRedis) { + return; + } + + const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; + + // PnL Leaderboard + app.get('/api/v1/leaderboards/pnl', async (req: Request, res: Response) => { + const window = (req.query.window as string) || '24h'; + const limit = resolveLimitQuery(req.query.limit, 50, 100); + const offset = parseInt((req.query.offset as string) || '0', 10); + const sortBy = (req.query.sortBy as string) || 'netPnl'; + const persona = req.query.persona as string | undefined; + + if (!['24h', '7d', '30d'].includes(window)) { + res.status(400).json({ error: 'invalid_window', message: `Invalid window: ${window}` }); + return; + } + + try { + const leaderboardKey = `${keyPrefix}leaderboard:pnl:${window}`; + + const entriesWithScores = await traderIntelligenceRedis.zrevrange( + leaderboardKey, + offset, + offset + limit - 1, + 'WITHSCORES' + ); + + const entries = []; + + for (let i = 0; i < entriesWithScores.length; i += 2) { + const address = entriesWithScores[i]; + const netPnl = parseFloat(entriesWithScores[i + 1] || '0'); + + if (!address) continue; + + const profileKey = `${keyPrefix}trader:${address}:profile`; + const profileData = await traderIntelligenceRedis.hgetall(profileKey); + + let personas: string[] = []; + if (profileData.personas) { + try { + personas = JSON.parse(profileData.personas); + } catch { + personas = []; + } + } + + if (persona && !personas.includes(persona)) { + continue; + } + + const totalTrades = parseInt(profileData.totalTrades || '0', 10); + const accountValue = parseFloat(profileData.accountValue || '0'); + const firstSeen = parseInt(profileData.firstSeen || '0', 10); + const ageDays = firstSeen > 0 ? Math.floor((Date.now() - firstSeen) / (24 * 60 * 60 * 1000)) : 0; + + const pnlKey = `${keyPrefix}trader:${address}:pnl:${window}`; + const pnlData = await traderIntelligenceRedis.hgetall(pnlKey); + const winRate = parseFloat(pnlData.winRate || '0'); + const profitFactor = parseFloat(pnlData.profitFactor || '1'); + + entries.push({ + rank: offset + (i / 2) + 1, + address, + ensName: profileData.ensName || undefined, + netPnl, + winRate, + profitFactor, + totalTrades, + personas, + openValue: accountValue, + currentBias: (Math.random() - 0.5) * 2, + ageDays, + }); + } + + res.json({ + data: { + entries, + window, + limit, + offset, + sortBy, + total: entries.length, + updatedAt: Date.now(), + }, + }); + } catch (error) { + res.status(500).json({ + error: 'pnl_leaderboard_fetch_failed', + detail: error instanceof Error ? error.message : String(error) + }); + } + }); + + // Rekt Leaderboard + app.get('/api/v1/leaderboards/rekt', async (req: Request, res: Response) => { + const window = (req.query.window as string) || '24h'; + const limit = resolveLimitQuery(req.query.limit, 50, 100); + const offset = parseInt((req.query.offset as string) || '0', 10); + const sortBy = ((req.query.sortBy as string) || 'totalLoss').toLowerCase(); + const personaFilter = (req.query.persona as string | undefined)?.toLowerCase(); + + if (!['24h', '7d'].includes(window)) { + res.status(400).json({ error: 'invalid_window', message: `Invalid window: ${window}` }); + return; + } + if (!['totalloss', 'liquidationcount'].includes(sortBy)) { + res.status(400).json({ error: 'invalid_sort', message: `Invalid sortBy: ${sortBy}` }); + return; + } + + try { + const leaderboardKey = `${keyPrefix}leaderboard:rekt:${window}`; + + const entriesWithScores = await traderIntelligenceRedis.zrevrange( + leaderboardKey, + offset, + offset + limit - 1, + 'WITHSCORES' + ); + + let personaMembers: Set | null = null; + if (personaFilter) { + const personaKey = `${keyPrefix}persona:${personaFilter}:members`; + const personaAddresses = await traderIntelligenceRedis.smembers(personaKey); + if (personaAddresses && personaAddresses.length > 0) { + personaMembers = new Set(personaAddresses.map((addr) => addr.toLowerCase())); + } + } + + const entries = []; + for (let i = 0; i < entriesWithScores.length; i += 2) { + const address = entriesWithScores[i]; + const totalLoss = parseFloat(entriesWithScores[i + 1] || '0'); + + if (!address) continue; + + const liquidationsKey = `${keyPrefix}trader:${address}:liquidations`; + const liquidations = await traderIntelligenceRedis.zrange(liquidationsKey, 0, -1, 'WITHSCORES'); + + let liquidationCount = 0; + let biggestLiquidation = 0; + + if (liquidations && liquidations.length > 0) { + liquidationCount = liquidations.length / 2; + for (let j = 1; j < liquidations.length; j += 2) { + const loss = parseFloat(liquidations[j] || '0'); + if (loss > biggestLiquidation) { + biggestLiquidation = loss; + } + } + } + + const profileKey = `${keyPrefix}trader:${address}:profile`; + const profileData = await traderIntelligenceRedis.hgetall(profileKey); + const personas = profileData.personas ? JSON.parse(profileData.personas) : []; + + if (personaFilter) { + if (personaMembers) { + if (!personaMembers.has(address.toLowerCase())) { + continue; + } + } else if (!personas.some((p: { persona: string }) => p.persona === personaFilter)) { + continue; + } + } + + entries.push({ + rank: offset + (i / 2) + 1, + address, + ensName: undefined, + totalLoss, + liquidationCount, + biggestLiquidation, + personas, + }); + } + + if (sortBy === 'liquidationcount') { + entries.sort((a, b) => b.liquidationCount - a.liquidationCount || b.totalLoss - a.totalLoss); + } else { + entries.sort((a, b) => b.totalLoss - a.totalLoss || b.liquidationCount - a.liquidationCount); + } + + entries.forEach((entry, idx) => { + entry.rank = offset + idx + 1; + }); + + res.json({ + data: { + entries, + window, + limit, + offset, + total: entries.length, + updatedAt: Date.now(), + }, + }); + } catch (error) { + res.status(500).json({ + error: 'rekt_leaderboard_fetch_failed', + detail: error instanceof Error ? error.message : String(error) + }); + } + }); + + // Personas Leaderboard + app.get('/api/v1/leaderboards/personas', async (req: Request, res: Response) => { + try { + const personasResponse = await traderIntelligenceRedis.get(`${keyPrefix}personas`); + const personas = personasResponse ? JSON.parse(personasResponse) : []; + + res.json({ + data: { + entries: personas, + total: personas.length, + updatedAt: Date.now(), + }, + }); + } catch (error) { + res.status(500).json({ + error: 'personas_leaderboard_fetch_failed', + detail: error instanceof Error ? error.message : String(error) + }); + } + }); + + // Whale Positions Leaderboard + app.get('/api/v1/leaderboards/whales', async (req: Request, res: Response) => { + const minPositionUsd = parseFloat((req.query.minPositionUsd as string) || '100000'); + const coin = req.query.coin as string | undefined; + const personaFilter = (req.query.persona as string | undefined)?.toLowerCase(); + const limit = resolveLimitQuery(req.query.limit, 50, 100); + + try { + const leaderboardKey = `${keyPrefix}leaderboard:whales:positions`; + + const entriesWithScores = await traderIntelligenceRedis.zrevrange( + leaderboardKey, + 0, + limit - 1, + 'WITHSCORES' + ); + + let personaMembers: Set | null = null; + if (personaFilter) { + const personaKey = `${keyPrefix}persona:${personaFilter}:members`; + const personaAddresses = await traderIntelligenceRedis.smembers(personaKey); + if (personaAddresses && personaAddresses.length > 0) { + personaMembers = new Set(personaAddresses.map((addr) => addr.toLowerCase())); + } + } + + const entries = []; + for (let i = 0; i < entriesWithScores.length; i += 2) { + const address = entriesWithScores[i]; + const totalPositionValue = parseFloat(entriesWithScores[i + 1] || '0'); + + if (!address || totalPositionValue < minPositionUsd) continue; + + const positionsKey = `${keyPrefix}trader:${address}:positions`; + const positionsJson = await traderIntelligenceRedis.get(positionsKey); + let positions: Array<{ + coin: string; + side: 'long' | 'short'; + positionValue: string; + unrealizedPnl: string; + }> = []; + + if (positionsJson) { + try { + positions = JSON.parse(positionsJson); + } catch { + // Ignore parse errors + } + } + + if (coin) { + positions = positions.filter(p => p.coin === coin); + if (positions.length === 0) continue; + } + + const ensMetadata = await readEnsMetadata(traderIntelligenceRedis, keyPrefix, address); + + const profileKey = `${keyPrefix}trader:${address}:profile`; + const profileData = await traderIntelligenceRedis.hgetall(profileKey); + const personas = profileData.personas ? JSON.parse(profileData.personas) : []; + + if (personaFilter) { + if (personaMembers) { + if (!personaMembers.has(address.toLowerCase())) { + continue; + } + } else if (!personas.some((p: { persona: string }) => p.persona === personaFilter)) { + continue; + } + } + + const longPositions = positions.filter(p => p.side === 'long').length; + const shortPositions = positions.filter(p => p.side === 'short').length; + const sentiment = longPositions > shortPositions ? 'bullish' : longPositions < shortPositions ? 'bearish' : 'neutral'; + + entries.push({ + rank: entries.length + 1, + address, + ensName: ensMetadata.ensName, + totalPositionValue, + positions, + sentiment, + personas, + }); + } + + res.json({ + data: { + entries, + minPositionUsd, + coin, + limit, + total: entries.length, + updatedAt: Date.now(), + }, + }); + } catch (error) { + res.status(500).json({ + error: 'whale_positions_fetch_failed', + detail: error instanceof Error ? error.message : String(error) + }); + } + }); + + // Bot Tracker Leaderboard + app.get('/api/v1/leaderboards/bots', async (req: Request, res: Response) => { + const botType = req.query.botType as string | undefined; + const minUptime = parseFloat((req.query.minUptime as string) || '0'); + const limit = resolveLimitQuery(req.query.limit, 50, 100); + + try { + const leaderboardKey = `${keyPrefix}leaderboard:bots`; + + const entriesWithScores = await traderIntelligenceRedis.zrevrange( + leaderboardKey, + 0, + limit - 1, + 'WITHSCORES' + ); + + const entries = []; + for (let i = 0; i < entriesWithScores.length; i += 2) { + const address = entriesWithScores[i]; + const uptime = parseFloat(entriesWithScores[i + 1] || '0'); + + if (!address || uptime < minUptime) continue; + + const profileKey = `${keyPrefix}trader:${address}:profile`; + const profileData = await traderIntelligenceRedis.hgetall(profileKey); + + const personas = profileData.personas ? JSON.parse(profileData.personas) : []; + const botPersona = personas.find((p: { persona: string }) => p.persona === 'bot'); + + if (!botPersona && botType) continue; + + let detectedBotType: 'market-maker' | 'arbitrageur' | 'scalper' | 'unknown' = 'unknown'; + if (personas.some((p: { persona: string }) => p.persona === 'market-maker')) { + detectedBotType = 'market-maker'; + } else if (personas.some((p: { persona: string }) => p.persona === 'arbitrageur')) { + detectedBotType = 'arbitrageur'; + } else if (personas.some((p: { persona: string }) => p.persona === 'scalper')) { + detectedBotType = 'scalper'; + } + + if (botType && detectedBotType !== botType) continue; + + const pnlKey = `${keyPrefix}trader:${address}:pnl:24h`; + const pnlData = await traderIntelligenceRedis.hgetall(pnlKey); + + const netPnl = parseFloat(pnlData.netPnl || '0'); + const winRate = parseFloat(pnlData.winRate || '0'); + const totalTrades = parseInt(pnlData.totalTrades || '0', 10); + const tradesPerHour = totalTrades > 0 ? totalTrades / 24 : 0; + + const ensMetadata = await readEnsMetadata(traderIntelligenceRedis, keyPrefix, address); + + const lastSeen = parseInt(profileData.lastUpdated || '0', 10); + + entries.push({ + rank: entries.length + 1, + address, + ensName: ensMetadata.ensName, + botType: detectedBotType, + uptime, + tradesPerHour, + netPnl, + winRate, + totalTrades, + lastSeenAt: lastSeen, + personas, + }); + } + + res.json({ + data: { + entries, + botType, + minUptime, + limit, + total: entries.length, + updatedAt: Date.now(), + }, + }); + } catch (error) { + res.status(500).json({ + error: 'bot_tracker_fetch_failed', + detail: error instanceof Error ? error.message : String(error) + }); + } + }); + + // Shame Leaderboards + app.get('/api/v1/leaderboards/shame/:type', async (req: Request, res: Response) => { + const { type } = req.params; + const window = resolveWindow(req.query.window); + const limit = resolveLimitQuery(req.query.limit, 50, 100); + + const validTypes = ['most-rekt', 'bought-tops', 'sold-bottoms', 'revenge-traders', 'gamblers']; + if (!validTypes.includes(type)) { + res.status(400).json({ error: 'invalid_shame_type', message: `Invalid shame leaderboard type: ${type}` }); + return; + } + + if (!['24h', '7d'].includes(window)) { + res.status(400).json({ error: 'invalid_window', message: `Invalid window: ${window}` }); + return; + } + + try { + const leaderboardKey = `${keyPrefix}leaderboard:shame:${type}:${window}`; + + const entriesWithScores = await traderIntelligenceRedis.zrevrange(leaderboardKey, 0, limit - 1, 'WITHSCORES'); + + const entries = []; + for (let i = 0; i < entriesWithScores.length; i += 2) { + const address = entriesWithScores[i]; + const score = parseFloat(entriesWithScores[i + 1] || '0'); + + const ensMetadata = await readEnsMetadata(traderIntelligenceRedis, keyPrefix, address); + + entries.push({ + rank: i / 2 + 1, + address, + ensName: ensMetadata.ensName, + score, + }); + } + + res.json({ + type, + window, + entries, + total: entries.length, + updatedAt: Date.now(), + }); + } catch (error) { + res.status(500).json({ + error: 'shame_leaderboard_fetch_failed', + detail: error instanceof Error ? error.message : String(error) + }); + } + }); +}; + diff --git a/services/gateway/src/server/routes/markets.ts b/services/gateway/src/server/routes/markets.ts new file mode 100644 index 00000000..60a662d7 --- /dev/null +++ b/services/gateway/src/server/routes/markets.ts @@ -0,0 +1,481 @@ +import type { Request, Response } from 'express'; + +import type { HttpServerContext } from '../http/context'; +import type { + CandleSnapshot, + MarketSnapshot, + PriceHistoryPoint, + TradeSummary, + BboSnapshot +} from '../../store/priceStore'; +import { buildDepthSummary } from '../../store/priceStore'; +import { fetchHyperliquidDepthSnapshot } from '../../clients/hyperliquid'; +import { fetchHyperliquidCandles } from '../../clients/hyperliquidCandles'; +import { fetchCandlesFromRedis, persistCandlesToRedis } from '../../redis/candles'; +import { getIntervalSampleLimit } from '../../runtime/candles'; +import { isHyperliquidProvider } from '../../utils/providers'; +import { resolveDepthQuery, resolveIntervalQuery, resolveLimitQuery } from '../http/utils'; + +const DEFAULT_HISTORY_LIMIT = 720; +const MAX_HISTORY_LIMIT = 5_000; +const DEFAULT_CANDLE_LIMIT = 240; +const MAX_CANDLE_LIMIT = 2_400; +const DEFAULT_TRADES_LIMIT = 200; +const MAX_TRADES_LIMIT = 1_000; +const DEFAULT_INTERVAL_KEY = '1m'; + +export const registerMarketRoutes = (ctx: HttpServerContext): (() => void) => { + const { + app, + marketStore, + overviewStore, + defaultProvider, + providers, + redis, + resolveProviderRedisKeyConfig, + scheduleBackfill + } = ctx; + + const persistCandles = async ( + provider: string | undefined, + asset: string, + intervalKey: string, + candles: CandleSnapshot[] + ) => { + if (!provider || candles.length === 0 || !redis || !resolveProviderRedisKeyConfig) { + return; + } + const keyConfig = resolveProviderRedisKeyConfig(provider); + if (!keyConfig) { + return; + } + marketStore.hydrateCandles(provider, asset, candles); + await persistCandlesToRedis({ + redis, + keyConfig, + provider, + asset, + interval: intervalKey, + candles + }); + }; + + const maybeScheduleBackfill = ( + provider: string | undefined, + asset: string | undefined, + intervalKey: string, + currentLength: number, + limit: number + ) => { + if (!scheduleBackfill || !provider || !asset) { + return; + } + if (!isHyperliquidProvider(provider)) { + return; + } + const expected = Math.min(limit, getIntervalSampleLimit(intervalKey)); + if (currentLength >= expected * 0.6) { + return; + } + scheduleBackfill(provider, asset, [intervalKey]); + }; + + const inflightRemoteCandleFetches = new Map>(); + const inflightRemoteCandleFetchTimestamps = new Map(); + + const cleanupStaleCandleFetches = () => { + const now = Date.now(); + const STALE_FETCH_TTL_MS = 5 * 60 * 1000; + for (const [key, timestamp] of inflightRemoteCandleFetchTimestamps.entries()) { + if (now - timestamp > STALE_FETCH_TTL_MS) { + inflightRemoteCandleFetches.delete(key); + inflightRemoteCandleFetchTimestamps.delete(key); + } + } + }; + + const candleFetchCleanupInterval = setInterval(cleanupStaleCandleFetches, 2 * 60 * 1000); + + const hydrateCandlesFromRedis = async ( + provider: string | undefined, + asset: string, + intervalKey: string + ): Promise => { + if (!provider || !redis || !resolveProviderRedisKeyConfig) { + return false; + } + const keyConfig = resolveProviderRedisKeyConfig(provider); + if (!keyConfig) { + return false; + } + try { + const stored = await fetchCandlesFromRedis({ + redis, + keyConfig, + provider, + asset, + interval: intervalKey + }); + if (stored.length === 0) { + return false; + } + marketStore.hydrateCandles(provider, asset, stored); + return true; + } catch { + return false; + } + }; + + const ensureHyperliquidCandles = async ( + provider: string | undefined, + asset: string, + intervalKey: string, + fetchLimit: number + ): Promise => { + if (!provider || !isHyperliquidProvider(provider)) { + return false; + } + const inflightKey = `${provider}:${asset}:${intervalKey}`; + const existing = inflightRemoteCandleFetches.get(inflightKey); + if (existing) { + return existing; + } + const task = (async () => { + const candles = await fetchHyperliquidCandles(asset, intervalKey, fetchLimit).catch(() => []); + if (candles.length === 0) { + return false; + } + await persistCandles(provider, asset, intervalKey, candles); + return true; + })() + .catch(() => false) + .finally(() => { + inflightRemoteCandleFetches.delete(inflightKey); + inflightRemoteCandleFetchTimestamps.delete(inflightKey); + }); + inflightRemoteCandleFetches.set(inflightKey, task); + inflightRemoteCandleFetchTimestamps.set(inflightKey, Date.now()); + return task; + }; + + const ensureCandlesForRequest = async ( + provider: string | undefined, + asset: string, + interval: { key: string; seconds: number }, + limit: number + ): Promise => { + if (!provider) { + return []; + } + const expected = Math.min(limit, getIntervalSampleLimit(interval.key)); + const fetchLimit = Math.max(limit, getIntervalSampleLimit(interval.key)); + + const readCandles = () => marketStore.getCandles(provider, asset, interval.seconds, limit); + + let candles = readCandles(); + if (candles.length >= expected) { + return candles; + } + + const hydrated = await hydrateCandlesFromRedis(provider, asset, interval.key); + if (hydrated) { + candles = readCandles(); + if (candles.length >= expected) { + return candles; + } + } + + await ensureHyperliquidCandles(provider, asset, interval.key, fetchLimit); + return readCandles(); + }; + + const resolveSnapshotPrice = (snapshot: MarketSnapshot | null): number | null => { + if (!snapshot) return null; + if (typeof snapshot.price === 'number' && Number.isFinite(snapshot.price)) { + return snapshot.price; + } + const candidates = [ + snapshot.metadata?.markPrice, + snapshot.metadata?.midPrice, + snapshot.metadata?.oraclePrice + ]; + for (const candidate of candidates) { + if (typeof candidate === 'number' && Number.isFinite(candidate)) { + return candidate; + } + } + return null; + }; + + const getHistoryWithFallback = (provider: string, asset: string, limit: number): PriceHistoryPoint[] => { + const history = marketStore.getPriceHistory(provider, asset, limit); + if (history.length > 0) { + return history; + } + const snapshot = marketStore.get(provider, asset); + const fallbackPrice = resolveSnapshotPrice(snapshot); + if (snapshot?.asset && fallbackPrice !== null) { + const timestamp = snapshot.priceTimestamp ?? Date.now(); + return [{ timestamp, price: fallbackPrice }]; + } + return []; + }; + + const respondWithDepth = async (provider: string, asset: string, depth: number, res: Response) => { + try { + const cached = marketStore.getDepthSummary(provider, asset, depth); + if (cached) { + res.json({ data: cached, provider }); + return; + } + + const metadata = marketStore.getMetadata(provider, asset); + const snapshot = await fetchHyperliquidDepthSnapshot(asset, depth); + + if (snapshot) { + const summary = buildDepthSummary( + provider, + asset, + depth, + snapshot.bids, + snapshot.asks, + snapshot.timestamp, + metadata?.bestBid, + metadata?.bestAsk + ); + + res.json({ data: summary, provider, source: 'hyperliquid-rest' }); + return; + } + + res.status(404).json({ error: 'depth_not_found', asset, provider }); + } catch (error) { + res.status(500).json({ + error: 'depth_lookup_failed', + asset, + provider, + detail: error instanceof Error ? error.message : String(error) + }); + } + }; + + app.get('/api/v1/markets', (_req: Request, res: Response) => { + if (!defaultProvider) { + res.status(400).json({ error: 'provider_not_configured' }); + return; + } + const data: MarketSnapshot[] = marketStore.snapshot(defaultProvider); + res.json({ data }); + }); + + app.get('/api/v1/markets/:asset', (req: Request, res: Response) => { + const { asset } = req.params; + if (!defaultProvider) { + res.status(400).json({ error: 'provider_not_configured' }); + return; + } + const snapshot = marketStore.get(defaultProvider, asset); + if (!snapshot) { + res.status(404).json({ error: 'asset_not_found', asset }); + return; + } + res.json({ data: snapshot }); + }); + + app.get('/api/v1/markets/:asset/metadata', (req: Request, res: Response) => { + const { asset } = req.params; + if (!defaultProvider) { + res.status(400).json({ error: 'provider_not_configured' }); + return; + } + const metadata = marketStore.getMetadata(defaultProvider, asset); + if (!metadata) { + res.status(404).json({ error: 'metadata_not_found', asset }); + return; + } + res.json({ data: metadata }); + }); + + app.get('/api/v1/markets/:asset/price-history', (req: Request, res: Response) => { + const { asset } = req.params; + if (!defaultProvider) { + res.status(400).json({ error: 'provider_not_configured' }); + return; + } + const limit = resolveLimitQuery(req.query.limit, DEFAULT_HISTORY_LIMIT, MAX_HISTORY_LIMIT); + const history = getHistoryWithFallback(defaultProvider, asset, limit); + if (history.length === 0) { + res.status(404).json({ error: 'price_history_not_found', asset, provider: defaultProvider }); + return; + } + res.json({ data: history, provider: defaultProvider, asset, limit }); + }); + + app.get('/api/v1/markets/:asset/candles', async (req: Request, res: Response) => { + const { asset } = req.params; + if (!defaultProvider) { + res.status(400).json({ error: 'provider_not_configured' }); + return; + } + const limit = resolveLimitQuery(req.query.limit, DEFAULT_CANDLE_LIMIT, MAX_CANDLE_LIMIT); + const interval = resolveIntervalQuery(req.query.interval ?? req.query.resolution, DEFAULT_INTERVAL_KEY); + const snapshot = marketStore.get(defaultProvider, asset); + const resolvedAsset = snapshot?.asset ?? asset; + + const candles = await ensureCandlesForRequest(defaultProvider, resolvedAsset, interval, limit); + + if (candles.length === 0) { + res.status(404).json({ error: 'candles_not_found', asset, provider: defaultProvider }); + return; + } + maybeScheduleBackfill(defaultProvider, resolvedAsset, interval.key, candles.length, limit); + res.json({ + data: candles, + provider: defaultProvider, + asset: resolvedAsset, + interval: interval.key, + resolutionSeconds: interval.seconds + }); + }); + + app.get('/api/v1/markets/:asset/trades', (req: Request, res: Response) => { + const { asset } = req.params; + if (!defaultProvider) { + res.status(400).json({ error: 'provider_not_configured' }); + return; + } + const limit = resolveLimitQuery(req.query.limit, DEFAULT_TRADES_LIMIT, MAX_TRADES_LIMIT); + const trades: TradeSummary[] = marketStore.getTrades(defaultProvider, asset, limit); + res.json({ data: trades, provider: defaultProvider, asset, limit }); + }); + + app.get('/api/v1/markets/:asset/bbo', (req: Request, res: Response) => { + const { asset } = req.params; + if (!defaultProvider) { + res.status(400).json({ error: 'provider_not_configured' }); + return; + } + const bbo: BboSnapshot | null = marketStore.getBbo(defaultProvider, asset); + res.json({ data: bbo, provider: defaultProvider, asset }); + }); + + app.get('/api/v1/markets/:asset/depth', async (req: Request, res: Response) => { + const { asset } = req.params; + if (!defaultProvider) { + res.status(400).json({ error: 'provider_not_configured' }); + return; + } + const depth = resolveDepthQuery(req.query.depth); + await respondWithDepth(defaultProvider, asset, depth, res); + }); + + app.get('/api/v1/overview', (_req: Request, res: Response) => { + if (!defaultProvider) { + res.status(400).json({ error: 'provider_not_configured' }); + return; + } + const data = overviewStore.snapshot(defaultProvider); + res.json({ data }); + }); + + app.get('/api/v1/providers/:provider/markets', (req: Request, res: Response) => { + const provider = req.params.provider; + const data = marketStore.snapshot(provider); + res.json({ data, provider }); + }); + + app.get('/api/v1/providers/:provider/markets/:asset', (req: Request, res: Response) => { + const { provider, asset } = req.params; + const snapshot = marketStore.get(provider, asset); + if (!snapshot) { + res.status(404).json({ error: 'asset_not_found', asset, provider }); + return; + } + res.json({ data: snapshot, provider }); + }); + + app.get('/api/v1/providers/:provider/markets/:asset/metadata', (req: Request, res: Response) => { + const { provider, asset } = req.params; + const metadata = marketStore.getMetadata(provider, asset); + if (!metadata) { + res.status(404).json({ error: 'metadata_not_found', asset, provider }); + return; + } + res.json({ data: metadata, provider }); + }); + + app.get('/api/v1/providers/:provider/markets/:asset/price-history', (req: Request, res: Response) => { + const { provider, asset } = req.params; + const limit = resolveLimitQuery(req.query.limit, DEFAULT_HISTORY_LIMIT, MAX_HISTORY_LIMIT); + const history = getHistoryWithFallback(provider, asset, limit); + if (history.length === 0) { + res.status(404).json({ error: 'price_history_not_found', asset, provider }); + return; + } + res.json({ data: history, provider, asset, limit }); + }); + + app.get('/api/v1/providers/:provider/markets/:asset/candles', async (req: Request, res: Response) => { + const { provider, asset } = req.params; + const limit = resolveLimitQuery(req.query.limit, DEFAULT_CANDLE_LIMIT, MAX_CANDLE_LIMIT); + const interval = resolveIntervalQuery(req.query.interval ?? req.query.resolution, DEFAULT_INTERVAL_KEY); + const snapshot = marketStore.get(provider, asset); + const resolvedAsset = snapshot?.asset ?? asset; + + const candles = await ensureCandlesForRequest(provider, resolvedAsset, interval, limit); + + if (candles.length === 0) { + res.status(404).json({ error: 'candles_not_found', asset, provider }); + return; + } + maybeScheduleBackfill(provider, resolvedAsset, interval.key, candles.length, limit); + res.json({ data: candles, provider, asset: resolvedAsset, interval: interval.key, resolutionSeconds: interval.seconds }); + }); + + app.get('/api/v1/providers/:provider/markets/:asset/trades', (req: Request, res: Response) => { + const { provider, asset } = req.params; + const limit = resolveLimitQuery(req.query.limit, DEFAULT_TRADES_LIMIT, MAX_TRADES_LIMIT); + const trades: TradeSummary[] = marketStore.getTrades(provider, asset, limit); + if (trades.length === 0) { + res.status(404).json({ error: 'trades_not_found', asset, provider }); + return; + } + res.json({ data: trades, provider, asset, limit }); + }); + + app.get('/api/v1/providers/:provider/markets/:asset/bbo', (req: Request, res: Response) => { + const { provider, asset } = req.params; + const bbo: BboSnapshot | null = marketStore.getBbo(provider, asset); + if (!bbo) { + res.status(404).json({ error: 'bbo_not_found', asset, provider }); + return; + } + res.json({ data: bbo, provider, asset }); + }); + + app.get('/api/v1/providers/:provider/markets/:asset/depth', async (req: Request, res: Response) => { + const { provider, asset } = req.params; + const depth = resolveDepthQuery(req.query.depth); + await respondWithDepth(provider, asset, depth, res); + }); + + app.get('/api/v1/providers/:provider/overview', (req: Request, res: Response) => { + const provider = req.params.provider; + const data = overviewStore.snapshot(provider); + res.json({ data, provider }); + }); + + app.get('/api/v1/providers', (_req: Request, res: Response) => { + const data = providers.map((provider) => ({ + id: provider.id, + displayName: provider.displayName, + description: provider.description, + tags: provider.tags + })); + res.json({ data, defaultProvider }); + }); + return () => { + clearInterval(candleFetchCleanupInterval); + }; +}; + diff --git a/services/gateway/src/server/routes/personas/constants.ts b/services/gateway/src/server/routes/personas/constants.ts new file mode 100644 index 00000000..85406bb7 --- /dev/null +++ b/services/gateway/src/server/routes/personas/constants.ts @@ -0,0 +1,66 @@ +export const PERSONA_CATEGORY_MAP: Record = { + 'mega-whale': 'size', + 'whale': 'size', + 'dolphin': 'size', + 'fish': 'size', + 'minnow': 'size', + 'plankton': 'size', + 'hodler': 'strategy', + 'scalper': 'strategy', + 'swing-trader': 'strategy', + 'market-maker': 'strategy', + 'arbitrageur': 'strategy', + 'degen': 'risk', + 'safe-sailor': 'risk', + 'risk-taker': 'risk', + 'chad': 'performance', + 'gigabrain': 'performance', + 'rekt': 'performance', + 'bag-holder': 'performance', + 'bot': 'behavioral', + 'fomo-king': 'behavioral', + 'diamond-hands': 'behavioral', + 'paper-hands': 'behavioral', + 'mystery': 'special', + 'vc': 'special', + 'exchange': 'special', + 'protocol': 'special', +}; + +export const PERSONA_EMOJI_MAP: Record = { + 'mega-whale': '🐋', + 'whale': '🐋', + 'dolphin': '🐬', + 'fish': '🐟', + 'minnow': '🦐', + 'plankton': '🦠', + 'hodler': '💎', + 'scalper': '⚡', + 'swing-trader': '📈', + 'market-maker': '🏪', + 'arbitrageur': '🔄', + 'degen': '💀', + 'safe-sailor': '⛵', + 'risk-taker': '🎲', + 'chad': '👑', + 'gigabrain': '🧠', + 'rekt': '💀', + 'bag-holder': '👜', + 'bot': '🤖', + 'fomo-king': '🚀', + 'diamond-hands': '💎', + 'paper-hands': '📄', + 'mystery': '❓', + 'vc': '💼', + 'exchange': '🏦', + 'protocol': '🔧', +}; + +export const WARNING_LEVELS: Record = { + clean: 0, + spicy: 1, + crass: 2, + degenerate: 3, + hidden: 4, +}; + diff --git a/services/gateway/src/server/routes/personas/index.ts b/services/gateway/src/server/routes/personas/index.ts new file mode 100644 index 00000000..a5495552 --- /dev/null +++ b/services/gateway/src/server/routes/personas/index.ts @@ -0,0 +1,389 @@ +import type { Request, Response } from 'express'; +import type { HttpServerContext } from '../../http/context'; +import { resolveLimitQuery } from '../../http/utils'; +import { buildTraderSummaries, sortTraderSummaries } from '../traders/helpers'; +import { PERSONA_CATEGORY_MAP, PERSONA_EMOJI_MAP } from './constants'; + +type RawPersonaStats = Record; +type PersonaStatsResponse = { + personaName: string; + personaCategory: 'size' | 'strategy' | 'risk' | 'performance' | 'behavioral' | 'special'; + totalWallets: number; + activeWallets: number; + avgLeverage: number; + avgPnL24h: number; + avgPnL7d: number; + avgPnL30d: number; + inPositionPercent: number; + sentiment: { + label: string; + score: number; + confidence: number; + calculatedAt: number; + }; + topAssets: Array<{ + asset: string; + exposure: number; + positionCount: number; + }>; + lastUpdated: number; + trend: number[]; +}; + +const toInt = (value?: string): number => { + if (!value) return 0; + const parsed = Number.parseInt(value, 10); + return Number.isFinite(parsed) ? parsed : 0; +}; + +const toFloat = (value?: string): number => { + if (!value) return 0; + const parsed = Number.parseFloat(value); + return Number.isFinite(parsed) ? parsed : 0; +}; + +const parseJson = (value: string | undefined, fallback: T): T => { + if (!value) return fallback; + try { + return JSON.parse(value) as T; + } catch { + return fallback; + } +}; + +const buildPersonaStats = ( + personaName: string, + personaCategory: string, + statsData: RawPersonaStats +): PersonaStatsResponse => { + const defaultSentiment: PersonaStatsResponse['sentiment'] = { + label: 'neutral', + score: 0, + confidence: 0, + calculatedAt: 0 + }; + const defaultTopAssets: PersonaStatsResponse['topAssets'] = []; + const defaultTrend: PersonaStatsResponse['trend'] = []; + + return { + personaName: statsData.personaName || personaName, + personaCategory: (statsData.personaCategory || personaCategory || 'special') as PersonaStatsResponse['personaCategory'], + totalWallets: toInt(statsData.totalWallets), + activeWallets: toInt(statsData.activeWallets), + avgLeverage: toFloat(statsData.avgLeverage), + avgPnL24h: toFloat(statsData.avgPnL24h), + avgPnL7d: toFloat(statsData.avgPnL7d), + avgPnL30d: toFloat(statsData.avgPnL30d), + inPositionPercent: toFloat(statsData.inPositionPercent), + sentiment: parseJson(statsData.sentiment, defaultSentiment), + topAssets: parseJson(statsData.topAssets, defaultTopAssets), + lastUpdated: toInt(statsData.lastUpdated), + trend: parseJson(statsData.trend, defaultTrend) + }; +}; + +export const registerPersonaRoutes = (ctx: HttpServerContext): void => { + const { app, traderIntelligenceRedis, traderIntelligenceKeyPrefix } = ctx; + + if (!traderIntelligenceRedis) { + return; + } + + const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; + + // Fetch traders by persona + app.get('/api/v1/personas/:persona/traders', async (req: Request, res: Response) => { + const { persona } = req.params; + const limit = resolveLimitQuery(req.query.limit, 50, 100); + + try { + const { fetchPersonaMembers } = await import('../../../redis/leaderboard'); + const members = await fetchPersonaMembers(traderIntelligenceRedis, traderIntelligenceKeyPrefix, persona, limit); + res.json({ data: members, persona, limit }); + } catch (error) { + res.status(500).json({ + error: 'persona_members_fetch_failed', + persona, + detail: error instanceof Error ? error.message : String(error) + }); + } + }); + + // Get all personas with counts + app.get('/api/v1/personas', async (req: Request, res: Response) => { + const type = req.query.type as string | undefined; + const includeStats = req.query.includeStats === 'true'; + + try { + const pattern = `${keyPrefix}persona:*:stats`; + const keys = await traderIntelligenceRedis.keys(pattern); + + const personas = []; + for (const key of keys) { + const match = key.match(/persona:([^:]+):stats/); + if (!match) continue; + + const personaName = match[1]; + const personaCategory = PERSONA_CATEGORY_MAP[personaName] || 'special'; + + if (type && personaCategory !== type) continue; + + const statsData = await traderIntelligenceRedis.hgetall(key); + if (!statsData || Object.keys(statsData).length === 0) continue; + + const persona: any = { + name: personaName, + type: personaCategory, + emoji: PERSONA_EMOJI_MAP[personaName] || '❓', + walletCount: toInt(statsData.totalWallets), + }; + + if (includeStats) { + const stats = buildPersonaStats(personaName, personaCategory, statsData); + persona.stats = stats; + persona.sentiment = stats.sentiment; + } else if (statsData.sentiment) { + persona.sentiment = parseJson(statsData.sentiment, { + label: 'neutral', + score: 0, + confidence: 0, + calculatedAt: 0 + }); + } + + personas.push(persona); + } + + res.json({ personas }); + } catch (error) { + res.status(500).json({ + error: 'personas_fetch_failed', + detail: error instanceof Error ? error.message : String(error) + }); + } + }); + + // Get detailed persona stats + app.get('/api/v1/personas/:personaName/stats', async (req: Request, res: Response) => { + const { personaName } = req.params; + const includeTrend = req.query.includeTrend === 'true'; + const trendWindow = (req.query.trendWindow as string) || '7d'; + + try { + const statsKey = `${keyPrefix}persona:${personaName}:stats`; + const trendKey = `${keyPrefix}persona:${personaName}:trend`; + + const statsData = await traderIntelligenceRedis.hgetall(statsKey); + if (!statsData || Object.keys(statsData).length === 0) { + res.status(404).json({ + error: 'persona_not_found', + personaName, + message: 'Persona stats not found' + }); + return; + } + + const personaEmoji = PERSONA_EMOJI_MAP[personaName] || '❓'; + + const stats = { + personaName: statsData.personaName || personaName, + personaCategory: statsData.personaCategory || 'special', + totalWallets: parseInt(statsData.totalWallets || '0', 10), + activeWallets: parseInt(statsData.activeWallets || '0', 10), + avgLeverage: parseFloat(statsData.avgLeverage || '0'), + avgPnL24h: parseFloat(statsData.avgPnL24h || '0'), + avgPnL7d: parseFloat(statsData.avgPnL7d || '0'), + avgPnL30d: parseFloat(statsData.avgPnL30d || '0'), + inPositionPercent: parseFloat(statsData.inPositionPercent || '0'), + sentiment: JSON.parse(statsData.sentiment || '{"label":"neutral","score":0,"confidence":0}'), + topAssets: JSON.parse(statsData.topAssets || '[]'), + lastUpdated: parseInt(statsData.lastUpdated || '0', 10), + }; + + const response: any = { + persona: { + name: personaName, + type: stats.personaCategory, + emoji: personaEmoji, + stats, + } + }; + + if (includeTrend) { + const windowMs = trendWindow === '30d' ? 30 * 24 * 60 * 60 * 1000 : 7 * 24 * 60 * 60 * 1000; + const since = Date.now() - windowMs; + + const trendMembers = await traderIntelligenceRedis.zrangebyscore( + trendKey, + since, + Date.now(), + 'WITHSCORES' + ); + + const trendPoints = []; + for (let i = 0; i < trendMembers.length; i += 2) { + const member = trendMembers[i]; + if (member) { + try { + const point = JSON.parse(member); + trendPoints.push(point); + } catch { + // Skip invalid entries + } + } + } + + response.persona.trend = trendPoints; + } + + res.json(response); + } catch (error) { + res.status(500).json({ + error: 'persona_stats_fetch_failed', + personaName, + detail: error instanceof Error ? error.message : String(error) + }); + } + }); + + // Get traders in a persona + app.get('/api/v1/personas/:personaName/traders', async (req: Request, res: Response) => { + const personaName = (req.params.personaName || '').toLowerCase(); + const limit = Math.min(Math.max(parseInt((req.query.limit as string) || '50', 10), 1), 100); + const offset = Math.max(parseInt((req.query.offset as string) || '0', 10), 0); + const sortBy = (req.query.sortBy as string) || 'pnl'; + const orderParam = (req.query.order as string) || 'desc'; + const order = orderParam === 'asc' ? 'asc' : 'desc'; + + try { + const membersKey = `${keyPrefix}persona:${personaName}:members`; + const addresses = await traderIntelligenceRedis.smembers(membersKey); + + if (!addresses || addresses.length === 0) { + res.json({ + traders: [], + pagination: { + total: 0, + limit, + offset, + hasMore: false, + }, + }); + return; + } + + const traderSummaries = await buildTraderSummaries(traderIntelligenceRedis, keyPrefix, addresses); + const sorted = sortTraderSummaries(traderSummaries, sortBy, order); + const total = sorted.length; + const paginated = sorted.slice(offset, offset + limit); + + res.json({ + traders: paginated, + pagination: { + total, + limit, + offset, + hasMore: offset + limit < total, + }, + }); + } catch (error) { + res.status(500).json({ + error: 'persona_traders_fetch_failed', + personaName, + detail: error instanceof Error ? error.message : String(error), + }); + } + }); + + // Get traders matching multiple personas + app.get('/api/v1/personas/traders', async (req: Request, res: Response) => { + const personasParam = req.query.personas as string | undefined; + const limit = Math.min(Math.max(parseInt((req.query.limit as string) || '50', 10), 1), 100); + const offset = Math.max(parseInt((req.query.offset as string) || '0', 10), 0); + const sortBy = (req.query.sortBy as string) || 'pnl'; + const orderParam = (req.query.order as string) || 'desc'; + const order = orderParam === 'asc' ? 'asc' : 'desc'; + const matchMode = ((req.query.match as string) || 'all').toLowerCase() === 'any' ? 'any' : 'all'; + + if (!personasParam) { + res.status(400).json({ + error: 'personas_required', + message: 'Query parameter "personas" is required (comma-separated persona ids)', + }); + return; + } + + const personaNames = Array.from( + new Set( + personasParam + .split(',') + .map((name) => name.trim().toLowerCase()) + .filter((name) => name.length > 0) + ) + ); + + if (personaNames.length === 0) { + res.status(400).json({ + error: 'invalid_personas', + message: 'At least one persona must be specified', + }); + return; + } + + if (personaNames.length > 8) { + res.status(400).json({ + error: 'too_many_personas', + message: 'A maximum of 8 personas can be queried at once', + }); + return; + } + + try { + const memberKeys = personaNames.map((persona) => `${keyPrefix}persona:${persona}:members`); + + let addresses: string[] = []; + if (matchMode === 'any') { + addresses = await traderIntelligenceRedis.sunion(...memberKeys); + } else { + addresses = await traderIntelligenceRedis.sinter(...memberKeys); + } + + if (!addresses || addresses.length === 0) { + res.json({ + traders: [], + pagination: { + total: 0, + limit, + offset, + hasMore: false, + }, + }); + return; + } + + const traderSummaries = await buildTraderSummaries(traderIntelligenceRedis, keyPrefix, addresses); + const sorted = sortTraderSummaries(traderSummaries, sortBy, order); + const total = sorted.length; + const paginated = sorted.slice(offset, offset + limit); + + res.json({ + traders: paginated, + appliedPersonas: personaNames, + mode: matchMode === 'any' ? 'any' : 'all', + pagination: { + total, + limit, + offset, + hasMore: offset + limit < total, + }, + }); + } catch (error) { + res.status(500).json({ + error: 'persona_traders_fetch_failed', + personas: personaNames, + detail: error instanceof Error ? error.message : String(error) + }); + } + }); +}; + diff --git a/services/gateway/src/server/routes/platform/index.ts b/services/gateway/src/server/routes/platform/index.ts new file mode 100644 index 00000000..2047f1b8 --- /dev/null +++ b/services/gateway/src/server/routes/platform/index.ts @@ -0,0 +1,504 @@ +import type { Request, Response } from 'express'; +import type { HttpServerContext } from '../../http/context'; +import { PERSONA_CATEGORY_MAP } from '../personas/constants'; + +export const registerPlatformRoutes = (ctx: HttpServerContext): void => { + const { app, traderIntelligenceRedis, traderIntelligenceKeyPrefix } = ctx; + + if (!traderIntelligenceRedis) { + return; + } + + const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; + + // Position Breakdown Endpoint + app.get('/api/v1/assets/:asset/position-breakdown', async (req: Request, res: Response) => { + const { asset } = req.params; + const view = (req.query.view as string) || 'size'; + + if (!['size', 'persona'].includes(view)) { + res.status(400).json({ error: 'invalid_view', message: 'View must be "size" or "persona"' }); + return; + } + + try { + const profileKeys = await traderIntelligenceRedis.keys(`${keyPrefix}trader:*:profile`); + + const sizeBuckets = [ + { label: '<$1k', min: 0, max: 1000 }, + { label: '$1k-$10k', min: 1000, max: 10000 }, + { label: '$10k-$25k', min: 10000, max: 25000 }, + { label: '$25k-$50k', min: 25000, max: 50000 }, + { label: '$50k-$100k', min: 50000, max: 100000 }, + { label: '$100k-$250k', min: 100000, max: 250000 }, + { label: '$250k-$500k', min: 250000, max: 500000 }, + { label: '$500k-$1m', min: 500000, max: 1000000 }, + { label: '$1m-$2.5m', min: 1000000, max: 2500000 }, + { label: '>$2.5m', min: 2500000, max: Infinity }, + ]; + + const allPositions: Array<{ + address: string; + position: any; + persona?: string; + }> = []; + let totalOpenInterest = 0; + + for (const profileKey of profileKeys) { + const address = profileKey.match(/trader:([^:]+):profile/)?.[1]; + if (!address) continue; + + const positionsKey = `${keyPrefix}trader:${address}:positions`; + const positionsJson = await traderIntelligenceRedis.get(positionsKey); + if (!positionsJson) continue; + + try { + const positions = JSON.parse(positionsJson); + const assetPositions = positions.filter((p: any) => p.coin === asset); + + for (const pos of assetPositions) { + const value = parseFloat(pos.positionValue || '0'); + totalOpenInterest += value; + + let persona: string | undefined; + if (view === 'persona') { + const profileData = await traderIntelligenceRedis.hgetall(profileKey); + const personas = profileData.personas ? JSON.parse(profileData.personas) : []; + persona = personas[0]?.persona || personas[0] || 'unknown'; + } + + allPositions.push({ + address, + position: pos, + persona, + }); + } + } catch { + continue; + } + } + + const breakdownMap = new Map(); + + for (const item of allPositions) { + let category: string; + + if (view === 'size') { + const value = parseFloat(item.position.positionValue || '0'); + const bucket = sizeBuckets.find(b => value >= b.min && value < b.max); + category = bucket?.label || '>$2.5m'; + } else { + category = item.persona || 'unknown'; + } + + if (!breakdownMap.has(category)) { + breakdownMap.set(category, { + category, + positions: [], + }); + } + breakdownMap.get(category)!.positions.push(item); + } + + const breakdown = Array.from(breakdownMap.values()).map(({ category, positions }) => { + let totalLongValue = 0; + let totalShortValue = 0; + let totalValue = 0; + const liquidationDistances: number[] = []; + + for (const item of positions) { + const value = parseFloat(item.position.positionValue || '0'); + totalValue += value; + + if (item.position.side === 'long') { + totalLongValue += value; + } else if (item.position.side === 'short') { + totalShortValue += value; + } + + const entryPx = parseFloat(item.position.entryPx || '0'); + const liquidationPx = parseFloat(item.position.liquidationPx || '0'); + if (entryPx > 0 && liquidationPx > 0) { + const distance = Math.abs((entryPx - liquidationPx) / entryPx) * 100; + liquidationDistances.push(distance); + } + } + + const biasScore = totalValue > 0 + ? (totalLongValue - totalShortValue) / totalValue + : 0; + + const bias: 'bullish' | 'bearish' | 'neutral' = + biasScore > 0.1 ? 'bullish' : + biasScore < -0.1 ? 'bearish' : 'neutral'; + + const avgDistanceToLiq = liquidationDistances.length > 0 + ? liquidationDistances.reduce((sum, d) => sum + d, 0) / liquidationDistances.length + : 0; + + const percentOfTotal = totalOpenInterest > 0 + ? (totalValue / totalOpenInterest) * 100 + : 0; + + return { + category, + bias, + biasScore, + positionCount: positions.length, + totalValue, + avgDistanceToLiq, + percentOfTotal, + }; + }); + + breakdown.sort((a, b) => b.totalValue - a.totalValue); + + res.json({ + asset, + view, + breakdown, + totalOpenInterest, + lastUpdated: Date.now(), + }); + } catch (error) { + res.status(500).json({ + error: 'position_breakdown_fetch_failed', + asset, + detail: error instanceof Error ? error.message : String(error) + }); + } + }); + + // Platform Metrics Endpoints + app.get('/api/v1/stats/platform', async (req: Request, res: Response) => { + try { + const metricsKey = `${keyPrefix}platform:metrics`; + + const data = await traderIntelligenceRedis.hgetall(metricsKey); + if (!data || Object.keys(data).length === 0) { + res.status(404).json({ error: 'metrics_not_found', message: 'Platform metrics not available yet' }); + return; + } + + res.json({ + totalWallets: parseInt(data.totalWallets || '0', 10), + newWallets24h: parseInt(data.newWallets24h || '0', 10), + activePerpTraders: parseInt(data.activePerpTraders || '0', 10), + totalOpenPositions: parseInt(data.totalOpenPositions || '0', 10), + totalOpenInterest: parseFloat(data.totalOpenInterest || '0'), + dailyVolume: parseFloat(data.dailyVolume || '0'), + positionsInProfitPercent: parseFloat(data.positionsInProfitPercent || '0'), + avgLeverage: parseFloat(data.avgLeverage || '0'), + exposureRatioDistribution: JSON.parse(data.exposureRatioDistribution || '{"low":0,"medium":0,"high":0,"veryHigh":0}'), + change24h: JSON.parse(data.change24h || '{"totalWallets":0,"activePerpTraders":0,"totalOpenInterest":0,"dailyVolume":0}'), + lastUpdated: parseInt(data.lastUpdated || '0', 10), + }); + } catch (error) { + res.status(500).json({ + error: 'platform_metrics_fetch_failed', + detail: error instanceof Error ? error.message : String(error) + }); + } + }); + + app.get('/api/v1/stats/platform/history', async (req: Request, res: Response) => { + const windowParam = req.query.window as string | undefined; + const intervalParam = req.query.interval as string | undefined; + + try { + const historyKey = `${keyPrefix}platform:metrics:history`; + + const windowMs = windowParam === '30d' + ? 30 * 24 * 60 * 60 * 1000 + : windowParam === '24h' + ? 24 * 60 * 60 * 1000 + : 7 * 24 * 60 * 60 * 1000; + + const startTime = Date.now() - windowMs; + + const members = await traderIntelligenceRedis.zrangebyscore( + historyKey, + startTime, + '+inf' + ); + + const snapshots: Array<{ + timestamp: number; + metrics: { + totalWallets: number; + newWallets24h: number; + activePerpTraders: number; + totalOpenPositions: number; + totalOpenInterest: number; + dailyVolume: number; + positionsInProfitPercent: number; + avgLeverage: number; + exposureRatioDistribution: { + low: number; + medium: number; + high: number; + veryHigh: number; + }; + change24h: { + totalWallets: number; + activePerpTraders: number; + totalOpenInterest: number; + dailyVolume: number; + }; + lastUpdated: number; + }; + }> = []; + + for (const member of members) { + try { + const snapshot = JSON.parse(member); + snapshots.push(snapshot); + } catch { + // Skip invalid entries + } + } + + snapshots.sort((a, b) => a.timestamp - b.timestamp); + + let filteredSnapshots = snapshots; + if (intervalParam === 'hourly') { + const hourlySnapshots: typeof snapshots = []; + let lastHour = -1; + for (const snapshot of snapshots) { + const hour = Math.floor(snapshot.timestamp / (60 * 60 * 1000)); + if (hour !== lastHour) { + hourlySnapshots.push(snapshot); + lastHour = hour; + } + } + filteredSnapshots = hourlySnapshots; + } else if (intervalParam === 'daily') { + const dailySnapshots: typeof snapshots = []; + let lastDay = -1; + for (const snapshot of snapshots) { + const day = Math.floor(snapshot.timestamp / (24 * 60 * 60 * 1000)); + if (day !== lastDay) { + dailySnapshots.push(snapshot); + lastDay = day; + } + } + filteredSnapshots = dailySnapshots; + } + + res.json({ + window: windowParam || '7d', + interval: intervalParam || 'all', + snapshots: filteredSnapshots, + count: filteredSnapshots.length, + }); + } catch (error) { + res.status(500).json({ + error: 'platform_metrics_history_fetch_failed', + detail: error instanceof Error ? error.message : String(error) + }); + } + }); + + // Bias Trend Endpoint + app.get('/api/v1/assets/:asset/bias-trend', async (req: Request, res: Response) => { + const { asset } = req.params; + const personasParam = req.query.personas as string | undefined; + const window = (req.query.window as '7d' | '30d') || '7d'; + + if (!['7d', '30d'].includes(window)) { + res.status(400).json({ error: 'invalid_window', message: 'Window must be "7d" or "30d"' }); + return; + } + + try { + const personas: string[] = personasParam + ? personasParam.split(',').map(p => p.trim()).filter(Boolean) + : []; + + const windowMs = window === '7d' + ? 7 * 24 * 60 * 60 * 1000 + : 30 * 24 * 60 * 60 * 1000; + const startTime = Date.now() - windowMs; + + let personasToQuery = personas; + if (personasToQuery.length === 0) { + const pattern = `${keyPrefix}asset:${asset}:bias:*`; + const keys = await traderIntelligenceRedis.keys(pattern); + personasToQuery = keys + .map(key => { + const match = key.match(/bias:([^:]+)$/); + return match ? match[1] : null; + }) + .filter((p): p is string => p !== null); + } + + if (personasToQuery.length === 0) { + res.json({ + asset, + window, + personas: [], + lastUpdated: Date.now(), + }); + return; + } + + const personaData: Array<{ + persona: string; + personaType: string; + points: Array<{ + timestamp: number; + persona: string; + bias: number; + openInterest: number; + positionCount: number; + longValue: number; + shortValue: number; + }>; + }> = []; + + for (const persona of personasToQuery) { + const trendKey = `${keyPrefix}asset:${asset}:bias:${persona}`; + + try { + const members = await traderIntelligenceRedis.zrangebyscore( + trendKey, + startTime, + Date.now(), + 'WITHSCORES' + ); + + if (members.length === 0) { + continue; + } + + const points: Array<{ + timestamp: number; + persona: string; + bias: number; + openInterest: number; + positionCount: number; + longValue: number; + shortValue: number; + }> = []; + + for (let i = 0; i < members.length; i += 2) { + const member = members[i]; + const score = members[i + 1]; + + if (member && score) { + try { + const point = JSON.parse(member); + point.timestamp = parseInt(score, 10); + points.push(point); + } catch { + // Skip invalid entries + } + } + } + + if (points.length > 0) { + const personaType = PERSONA_CATEGORY_MAP[persona] || 'special'; + personaData.push({ + persona, + personaType, + points: points.sort((a, b) => a.timestamp - b.timestamp), + }); + } + } catch (error) { + console.error(`Error getting bias trend for ${asset} ${persona}:`, error); + } + } + + res.json({ + asset, + window, + personas: personaData, + lastUpdated: Date.now(), + }); + } catch (error) { + res.status(500).json({ + error: 'bias_trend_fetch_failed', + asset, + detail: error instanceof Error ? error.message : String(error) + }); + } + }); + + // Position Heatmap Endpoint + app.get('/api/v1/heatmap/positions', async (req: Request, res: Response) => { + const assetsParam = req.query.assets as string | undefined; + const personasParam = req.query.personas as string | undefined; + + try { + const assets: string[] = assetsParam + ? assetsParam.split(',').map(a => a.trim()).filter(Boolean) + : []; + const personas: string[] = personasParam + ? personasParam.split(',').map(p => p.trim()).filter(Boolean) + : []; + + let assetsToQuery = assets; + let personasToQuery = personas; + + if (assetsToQuery.length === 0) { + const assetsKey = `${keyPrefix}heatmap:positions:assets`; + const assetMembers = await traderIntelligenceRedis.smembers(assetsKey); + assetsToQuery = assetMembers.length > 0 ? assetMembers.sort() : []; + } + + if (personasToQuery.length === 0) { + const personasKey = `${keyPrefix}heatmap:positions:personas`; + const personaMembers = await traderIntelligenceRedis.smembers(personasKey); + personasToQuery = personaMembers.length > 0 ? personaMembers.sort() : []; + } + + if (assetsToQuery.length === 0 || personasToQuery.length === 0) { + res.json({ + assets: [], + personas: [], + matrix: {}, + lastUpdated: Date.now(), + }); + return; + } + + const matrix: Record> = {}; + + for (const asset of assetsToQuery) { + const assetKey = `${keyPrefix}heatmap:positions:${asset}`; + const assetData = await traderIntelligenceRedis.hgetall(assetKey); + + if (Object.keys(assetData).length > 0) { + matrix[asset] = {}; + for (const persona of personasToQuery) { + const cellJson = assetData[persona]; + if (cellJson) { + try { + const cell = JSON.parse(cellJson); + matrix[asset][persona] = cell; + } catch { + // Skip invalid entries + } + } + } + } + } + + res.json({ + assets: assetsToQuery, + personas: personasToQuery, + matrix, + lastUpdated: Date.now(), + }); + } catch (error) { + res.status(500).json({ + error: 'position_heatmap_fetch_failed', + detail: error instanceof Error ? error.message : String(error) + }); + } + }); +}; + diff --git a/services/gateway/src/server/routes/stats.ts b/services/gateway/src/server/routes/stats.ts new file mode 100644 index 00000000..c1acce42 --- /dev/null +++ b/services/gateway/src/server/routes/stats.ts @@ -0,0 +1,54 @@ +import type { Response } from 'express'; + +import type { HttpServerContext } from '../http/context'; +import { assetStatsMetadata, overviewStatsMetadata } from '@pmon/ingestion-sdk'; + +export const registerStatsRoutes = (ctx: HttpServerContext): void => { + const { app, statsStore, defaultProvider } = ctx; + if (!statsStore) { + return; + } + + const respondWithAssetStats = (provider: string, asset: string, res: Response) => { + const stats = statsStore.getAsset(provider, asset); + if (!stats) { + res.status(404).json({ error: 'stats_not_found', provider, asset }); + return; + } + res.json({ data: stats, provider, asset, meta: assetStatsMetadata }); + }; + + const respondWithOverview = (provider: string, res: Response) => { + const overview = statsStore.getOverview(provider); + if (!overview) { + res.status(404).json({ error: 'stats_overview_not_found', provider }); + return; + } + res.json({ data: overview, provider, meta: overviewStatsMetadata }); + }; + + app.get('/api/v1/stats/overview', (_req, res) => { + if (!defaultProvider) { + res.status(400).json({ error: 'provider_not_configured' }); + return; + } + respondWithOverview(defaultProvider, res); + }); + + app.get('/api/v1/stats/:asset', (req, res) => { + if (!defaultProvider) { + res.status(400).json({ error: 'provider_not_configured' }); + return; + } + respondWithAssetStats(defaultProvider, req.params.asset, res); + }); + + app.get('/api/v1/providers/:provider/stats/overview', (req, res) => { + respondWithOverview(req.params.provider, res); + }); + + app.get('/api/v1/providers/:provider/stats/:asset', (req, res) => { + respondWithAssetStats(req.params.provider, req.params.asset, res); + }); +}; + diff --git a/services/gateway/src/server/routes/traders/helpers.ts b/services/gateway/src/server/routes/traders/helpers.ts new file mode 100644 index 00000000..43d6a839 --- /dev/null +++ b/services/gateway/src/server/routes/traders/helpers.ts @@ -0,0 +1,123 @@ +import type { Redis } from 'ioredis'; +import type { PersonaTraderSummary } from './types'; + +export const buildTraderSummaries = async ( + redis: Redis | null, + keyPrefix: string, + addresses: string[] +): Promise => { + if (!redis || addresses.length === 0) { + return []; + } + + const uniqueAddresses = Array.from(new Set(addresses.map((address) => address.toLowerCase()))); + + const pipeline = redis.pipeline(); + for (const address of uniqueAddresses) { + const profileKey = `${keyPrefix}trader:${address}:profile`; + const pnlKey = `${keyPrefix}trader:${address}:pnl:24h`; + const positionsKey = `${keyPrefix}trader:${address}:positions`; + pipeline.hgetall(profileKey); + pipeline.hgetall(pnlKey); + pipeline.get(positionsKey); + } + + const responses = await pipeline.exec(); + if (!responses) { + return []; + } + + const traders: PersonaTraderSummary[] = []; + + for (let i = 0; i < uniqueAddresses.length; i += 1) { + const responseOffset = i * 3; + const profileData = responses[responseOffset]?.[1] as Record | null; + if (!profileData || Object.keys(profileData).length === 0) { + continue; + } + + const pnlData = responses[responseOffset + 1]?.[1] as Record | null; + const positionsJson = responses[responseOffset + 2]?.[1] as string | null; + + const address = uniqueAddresses[i]; + let positions: Array<{ positionValue?: string; side?: string }> = []; + + if (positionsJson) { + try { + const parsed = JSON.parse(positionsJson); + if (Array.isArray(parsed)) { + positions = parsed; + } + } catch { + positions = []; + } + } + + const openValue = positions.reduce((sum: number, pos) => sum + parseFloat(pos.positionValue || '0'), 0); + + let totalLongValue = 0; + let totalShortValue = 0; + for (const pos of positions) { + const value = parseFloat(pos.positionValue || '0'); + if (pos.side === 'long') { + totalLongValue += value; + } else if (pos.side === 'short') { + totalShortValue += value; + } + } + + const totalValue = totalLongValue + totalShortValue; + const biasScore = totalValue > 0 ? (totalLongValue - totalShortValue) / totalValue : 0; + + const accountValue = parseFloat(profileData.accountValue || '0'); + const firstSeen = profileData.firstSeen + ? parseInt(profileData.firstSeen, 10) + : parseInt(profileData.lastUpdated || '0', 10); + const ageDays = firstSeen > 0 ? Math.floor((Date.now() - firstSeen) / (24 * 60 * 60 * 1000)) : 0; + + traders.push({ + address, + ensName: profileData.ensName || undefined, + perpEquity: accountValue, + openValue, + leverage: accountValue > 0 ? openValue / accountValue : 0, + pnl24h: parseFloat(pnlData?.netPnl || '0'), + pnl7d: parseFloat(pnlData?.netPnl7d || '0') || 0, + pnl30d: parseFloat(pnlData?.netPnl30d || '0') || 0, + currentBias: biasScore, + ageDays, + }); + } + + return traders; +}; + +export const sortTraderSummaries = ( + traders: PersonaTraderSummary[], + sortBy: string, + order: string +): PersonaTraderSummary[] => { + const sorted = [...traders]; + + sorted.sort((a, b) => { + let aVal = 0; + let bVal = 0; + + if (sortBy === 'leverage') { + aVal = a.leverage; + bVal = b.leverage; + } else if (sortBy === 'equity') { + aVal = a.perpEquity; + bVal = b.perpEquity; + } else { + // Default to Net PnL + aVal = a.pnl24h; + bVal = b.pnl24h; + } + + return order === 'asc' ? aVal - bVal : bVal - aVal; + }); + + return sorted; +}; + diff --git a/services/gateway/src/server/routes/traders/index.ts b/services/gateway/src/server/routes/traders/index.ts new file mode 100644 index 00000000..6d0c9615 --- /dev/null +++ b/services/gateway/src/server/routes/traders/index.ts @@ -0,0 +1,598 @@ +import type { Request, Response } from 'express'; +import type { HttpServerContext } from '../../http/context'; +export const registerTraderRoutes = (ctx: HttpServerContext): void => { + const { app, traderIntelligenceRedis, traderIntelligenceKeyPrefix } = ctx; + + if (!traderIntelligenceRedis) { + return; + } + + const keyPrefix = traderIntelligenceKeyPrefix || 'intelligence:'; + + // Get Trader's Asset Profile + app.get('/api/v1/traders/:address/assets/:asset', async (req: Request, res: Response) => { + const { address, asset } = req.params; + const normalizedAddress = address.toLowerCase(); + + try { + const pointsKey = `${keyPrefix}trader:${normalizedAddress}:asset-points:${asset}`; + const breakdownKey = `${keyPrefix}trader:${normalizedAddress}:asset-breakdown:${asset}`; + const loyaltyKey = `${keyPrefix}trader:${normalizedAddress}:asset-loyalty:${asset}`; + const badgesKey = `${keyPrefix}trader:${normalizedAddress}:asset-badges:${asset}`; + const streakKey = `${keyPrefix}trader:${normalizedAddress}:asset-streak:${asset}`; + const leaderboardKey = `${keyPrefix}leaderboard:asset:${asset}:points:24h`; + + const [points, breakdownJson, loyaltyTier, badgeIds, streakData, rank] = await Promise.all([ + traderIntelligenceRedis.get(pointsKey), + traderIntelligenceRedis.get(breakdownKey), + traderIntelligenceRedis.get(loyaltyKey), + traderIntelligenceRedis.smembers(badgesKey), + traderIntelligenceRedis.hgetall(streakKey), + traderIntelligenceRedis.zrevrank(leaderboardKey, normalizedAddress) + ]); + + if (!points && !breakdownJson) { + res.status(404).json({ error: 'TRADER_NOT_FOUND', message: `Trader ${address} not found for asset ${asset}` }); + return; + } + + const breakdown = breakdownJson ? JSON.parse(breakdownJson) : null; + const totalTraders = await traderIntelligenceRedis.zcard(leaderboardKey); + const percentile = rank !== null && totalTraders > 0 ? ((totalTraders - rank) / totalTraders) * 100 : 0; + + // Get badge details + const badges = []; + for (const badgeId of badgeIds) { + const badgeKey = `${keyPrefix}badge:${badgeId}:${normalizedAddress}`; + const badgeData = await traderIntelligenceRedis.hgetall(badgeKey); + if (badgeData && badgeData.id) { + badges.push({ + id: badgeData.id, + name: badgeData.name || '', + emoji: badgeData.emoji || '🏆', + rarity: badgeData.rarity || 'common', + unlockedAt: badgeData.unlockedAt ? parseInt(badgeData.unlockedAt, 10) : undefined + }); + } + } + + res.json({ + address: normalizedAddress, + asset, + points: { + total: parseInt(points || '0', 10), + breakdown: breakdown || { + total: 0, + components: { volume: 0, pnl: 0, loyalty: 0, streaks: 0, badges: 0, events: 0 }, + multipliers: { loyalty: 1.0, streak: 1.0, exclusiveFocus: 1.0 } + }, + rank: rank !== null ? rank + 1 : null, + percentile + }, + loyalty: { + tier: loyaltyTier || 'casual', + progress: 1.0, + metrics: { + totalTrades: 0, + totalVolume: breakdown?.components?.volume || 0, + daysActive: 0, + consecutiveDays: parseInt(streakData?.consecutiveDays || '0', 10) + } + }, + performance: { + volume24h: breakdown?.components?.volume || 0, + volumeAllTime: breakdown?.components?.volume || 0, + pnl24h: breakdown?.components?.pnl || 0, + pnlAllTime: breakdown?.components?.pnl || 0, + winRate: 0, + largestWin: 0, + largestLoss: 0 + }, + badges, + streaks: { + currentWin: parseInt(streakData?.currentWinStreak || '0', 10), + currentLoss: parseInt(streakData?.currentLossStreak || '0', 10), + longestWin: parseInt(streakData?.longestWinStreak || '0', 10), + consecutiveDays: parseInt(streakData?.consecutiveDays || '0', 10) + }, + lastTradeAt: undefined, + firstTradeAt: undefined + }); + } catch (error) { + res.status(500).json({ + error: 'INTERNAL_ERROR', + message: 'Failed to fetch trader asset profile', + detail: error instanceof Error ? error.message : String(error) + }); + } + }); + + // Get Trader's All Assets Summary + app.get('/api/v1/traders/:address/assets', async (req: Request, res: Response) => { + const { address } = req.params; + const normalizedAddress = address.toLowerCase(); + + try { + const pattern = `${keyPrefix}trader:${normalizedAddress}:asset-points:*`; + const keys = await traderIntelligenceRedis.keys(pattern); + + const assets = []; + let totalPoints = 0; + let primaryAsset = ''; + let maxPoints = 0; + + for (const key of keys) { + const assetMatch = key.match(/asset-points:(.+)$/); + if (!assetMatch) continue; + + const asset = assetMatch[1]; + const pointsKey = `${keyPrefix}trader:${normalizedAddress}:asset-points:${asset}`; + const loyaltyKey = `${keyPrefix}trader:${normalizedAddress}:asset-loyalty:${asset}`; + const badgesKey = `${keyPrefix}trader:${normalizedAddress}:asset-badges:${asset}`; + const leaderboardKey = `${keyPrefix}leaderboard:asset:${asset}:points:24h`; + const breakdownKey = `${keyPrefix}trader:${normalizedAddress}:asset-breakdown:${asset}`; + + const [points, loyaltyTier, badgeCount, rank, breakdownJson] = await Promise.all([ + traderIntelligenceRedis.get(pointsKey), + traderIntelligenceRedis.get(loyaltyKey), + traderIntelligenceRedis.scard(badgesKey), + traderIntelligenceRedis.zrevrank(leaderboardKey, normalizedAddress), + traderIntelligenceRedis.get(breakdownKey) + ]); + + const pointsValue = parseInt(points || '0', 10); + totalPoints += pointsValue; + + if (pointsValue > maxPoints) { + maxPoints = pointsValue; + primaryAsset = asset; + } + + const breakdown = breakdownJson ? JSON.parse(breakdownJson) : null; + + assets.push({ + asset, + points: pointsValue, + rank: rank !== null ? rank + 1 : null, + loyaltyTier: loyaltyTier || 'casual', + badges: badgeCount, + volume24h: breakdown?.components?.volume || 0, + pnl24h: breakdown?.components?.pnl || 0 + }); + } + + assets.sort((a, b) => b.points - a.points); + + const diversificationScore = assets.length > 1 + ? 1 - (maxPoints / totalPoints) + : 0; + + res.json({ + address: normalizedAddress, + assets, + totalAssets: assets.length, + primaryAsset, + diversificationScore + }); + } catch (error) { + res.status(500).json({ + error: 'INTERNAL_ERROR', + message: 'Failed to fetch trader assets summary', + detail: error instanceof Error ? error.message : String(error) + }); + } + }); + + // Get Trader Profile (Full Intelligence Profile) + app.get('/api/v1/traders/:address/profile', async (req: Request, res: Response) => { + const { address } = req.params; + const normalizedAddress = address.toLowerCase(); + const period = (req.query.period as string) || '24h'; + const includeHistory = req.query.includeHistory === 'true'; + + try { + const profileKey = `${keyPrefix}trader:${normalizedAddress}:profile`; + const pnlKey = `${keyPrefix}trader:${normalizedAddress}:pnl:${period}`; + const riskKey = `${keyPrefix}trader:${normalizedAddress}:risk`; + const positionsKey = `${keyPrefix}trader:${normalizedAddress}:positions`; + const ordersKey = `${keyPrefix}trader:${normalizedAddress}:orders`; + const liquidationsKey = `${keyPrefix}trader:${normalizedAddress}:liquidations`; + + const [profileData, pnlData, riskData, positionsJson, ordersJson, liquidationsData] = await Promise.all([ + traderIntelligenceRedis.hgetall(profileKey), + traderIntelligenceRedis.hgetall(pnlKey), + traderIntelligenceRedis.hgetall(riskKey), + traderIntelligenceRedis.get(positionsKey), + traderIntelligenceRedis.get(ordersKey), + includeHistory + ? traderIntelligenceRedis.zrange(liquidationsKey, -100, -1, 'WITHSCORES') + : Promise.resolve(null), + ]) as [ + Record, + Record, + Record, + string | null, + string | null, + string[] | null + ]; + + if (!profileData || Object.keys(profileData).length === 0) { + res.status(404).json({ + error: 'trader_not_found', + address, + message: 'Trader profile not found. The trader may not be tracked yet.' + }); + return; + } + + const profile = { + address: profileData.address || normalizedAddress, + ensName: profileData.ensName || undefined, + ensAvatar: profileData.ensAvatar || undefined, + primaryPersona: profileData.primaryPersona, + personas: profileData.personas ? JSON.parse(profileData.personas) : [], + funnyQuip: profileData.funnyQuip || undefined, + accountValue: parseFloat(profileData.accountValue || '0'), + lastUpdated: parseInt(profileData.lastUpdated || '0', 10), + firstSeen: profileData.firstSeen ? parseInt(profileData.firstSeen, 10) : undefined, + totalTrades: profileData.totalTrades ? parseInt(profileData.totalTrades, 10) : undefined, + pnl: pnlData && Object.keys(pnlData).length > 0 ? { + period, + netPnl: parseFloat(pnlData.netPnl || '0'), + realized: { + totalPnl: parseFloat(pnlData.realizedPnl || '0'), + winRate: parseFloat(pnlData.winRate || '0'), + profitFactor: parseFloat(pnlData.profitFactor || '0'), + }, + unrealized: { + totalPnl: parseFloat(pnlData.unrealizedPnl || '0'), + }, + fees: { + totalPaid: parseFloat(pnlData.totalFees || '0'), + }, + calculatedAt: parseInt(pnlData.calculatedAt || '0', 10), + } : undefined, + risk: riskData && Object.keys(riskData).length > 0 ? { + leverage: parseFloat(riskData.leverage || '0'), + marginUsage: parseFloat(riskData.marginUsage || '0'), + liquidationRisk: parseFloat(riskData.liquidationRisk || '0'), + positionConcentration: parseFloat(riskData.positionConcentration || '0'), + calculatedAt: parseInt(riskData.calculatedAt || '0', 10), + } : undefined, + positions: positionsJson ? JSON.parse(positionsJson) : [], + orders: ordersJson ? JSON.parse(ordersJson) : [], + liquidations: includeHistory && liquidationsData && Array.isArray(liquidationsData) && liquidationsData.length > 0 + ? liquidationsData + .filter((_, i) => i % 2 === 0) + .map((liqJson) => { + try { + return JSON.parse(liqJson as string); + } catch { + return null; + } + }) + .filter((liq) => liq !== null) + .reverse() + : undefined, + }; + + res.json({ + data: profile, + period, + cached: true, + }); + } catch (error) { + res.status(500).json({ + error: 'trader_profile_fetch_failed', + address, + detail: error instanceof Error ? error.message : String(error) + }); + } + }); + + // Get Cross-Asset Trader Stats + app.get('/api/v1/traders/:address/cross-asset-stats', async (req: Request, res: Response) => { + const { address } = req.params; + const normalizedAddress = address.toLowerCase(); + + try { + const pattern = `${keyPrefix}trader:${normalizedAddress}:asset-points:*`; + const keys = await traderIntelligenceRedis.keys(pattern); + + const assetAllocation: Record = {}; + const loyaltyBreakdown: Record = { + maximalist: [], + devoted: [], + regular: [], + casual: [] + }; + const topRanks: Array<{ asset: string; rank: number; points: number }> = []; + let totalPoints = 0; + + for (const key of keys) { + const assetMatch = key.match(/asset-points:(.+)$/); + if (!assetMatch) continue; + + const asset = assetMatch[1]; + const pointsKey = `${keyPrefix}trader:${normalizedAddress}:asset-points:${asset}`; + const loyaltyKey = `${keyPrefix}trader:${normalizedAddress}:asset-loyalty:${asset}`; + const leaderboardKey = `${keyPrefix}leaderboard:asset:${asset}:points:24h`; + + const [points, loyaltyTier, rank] = await Promise.all([ + traderIntelligenceRedis.get(pointsKey), + traderIntelligenceRedis.get(loyaltyKey), + traderIntelligenceRedis.zrevrank(leaderboardKey, normalizedAddress) + ]); + + const pointsValue = parseInt(points || '0', 10); + totalPoints += pointsValue; + + const tier = loyaltyTier || 'casual'; + if (loyaltyBreakdown[tier]) { + loyaltyBreakdown[tier].push(asset); + } + + if (rank !== null) { + topRanks.push({ + asset, + rank: rank + 1, + points: pointsValue + }); + } + } + + for (const key of keys) { + const assetMatch = key.match(/asset-points:(.+)$/); + if (!assetMatch) continue; + + const asset = assetMatch[1]; + const pointsKey = `${keyPrefix}trader:${normalizedAddress}:asset-points:${asset}`; + const points = await traderIntelligenceRedis.get(pointsKey); + const pointsValue = parseInt(points || '0', 10); + + if (totalPoints > 0) { + assetAllocation[asset] = pointsValue / totalPoints; + } + } + + topRanks.sort((a, b) => a.rank - b.rank); + + const primaryAsset = topRanks.length > 0 ? topRanks[0].asset : ''; + + res.json({ + address: normalizedAddress, + totalAssets: keys.length, + primaryAsset, + assetAllocation, + loyaltyBreakdown, + topRanks: topRanks.slice(0, 10), + multiAssetBadges: [] + }); + } catch (error) { + res.status(500).json({ + error: 'INTERNAL_ERROR', + message: 'Failed to fetch cross-asset stats', + detail: error instanceof Error ? error.message : String(error) + }); + } + }); + + // Whale-sized pending orders + app.get('/api/v1/traders/:address/whale-orders', async (req: Request, res: Response) => { + const { address } = req.params; + const normalizedAddress = address?.toLowerCase(); + const minSizeParam = (req.query.minSizeUsd as string) || '100000'; + const minSizeUsd = parseFloat(minSizeParam); + + if (!address || !normalizedAddress.startsWith('0x')) { + res.status(400).json({ + error: 'invalid_address', + message: 'address must be a 0x-prefixed hex string' + }); + return; + } + + if (!Number.isFinite(minSizeUsd) || minSizeUsd < 0) { + res.status(400).json({ + error: 'invalid_min_size', + message: 'minSizeUsd must be a positive number' + }); + return; + } + + try { + const ordersKey = `${keyPrefix}trader:${normalizedAddress}:orders`; + const ordersJson = await traderIntelligenceRedis.get(ordersKey); + + if (!ordersJson) { + res.json({ + address: normalizedAddress, + minSizeUsd, + total: 0, + largeOrders: [], + cached: false + }); + return; + } + + let orders: Array> = []; + try { + const parsed = JSON.parse(ordersJson); + if (Array.isArray(parsed)) { + orders = parsed; + } + } catch { + res.status(500).json({ + error: 'orders_parse_failed', + message: 'Stored orders payload is malformed JSON' + }); + return; + } + + const largeOrders = orders + .map((order) => { + const limitPx = parseFloat(order.limitPx ?? order.price ?? order.px ?? '0'); + const size = parseFloat(order.sz ?? order.size ?? '0'); + const notionalFromPayload = parseFloat(order.notionalUsd ?? order.notional ?? '0'); + + let notionalUsd = Number.isFinite(notionalFromPayload) && notionalFromPayload > 0 ? notionalFromPayload : 0; + if ((!notionalUsd || notionalUsd === 0) && Number.isFinite(limitPx) && Number.isFinite(size)) { + notionalUsd = Math.abs(limitPx * size); + } + + return { + coin: order.coin, + side: order.side === 'sell' ? 'sell' : 'buy', + limitPx: order.limitPx ?? order.price ?? order.px ?? null, + sz: order.sz ?? order.size ?? null, + oid: order.oid, + timestamp: order.timestamp, + orderType: order.orderType ?? 'limit', + reduceOnly: Boolean(order.reduceOnly), + sentiment: order.side === 'sell' ? 'bearish' : 'bullish', + notionalUsd, + }; + }) + .filter((order) => Number.isFinite(order.notionalUsd) && order.notionalUsd >= minSizeUsd) + .sort((a, b) => (b.notionalUsd || 0) - (a.notionalUsd || 0)) + .slice(0, 100); + + res.json({ + address: normalizedAddress, + minSizeUsd, + total: largeOrders.length, + largeOrders, + cached: true + }); + } catch (error) { + res.status(500).json({ + error: 'whale_orders_fetch_failed', + address, + detail: error instanceof Error ? error.message : String(error) + }); + } + }); + + // Get PnL Snapshots for Charting + app.get('/api/v1/traders/:address/pnl-snapshots', async (req: Request, res: Response) => { + const { address } = req.params; + const normalizedAddress = address.toLowerCase(); + const period = (req.query.period as string) || '24h'; + const limit = parseInt((req.query.limit as string) || '1000', 10); + + if (!['24h', '7d', '30d', 'all'].includes(period)) { + res.status(400).json({ error: 'invalid_period', message: `Invalid period: ${period}` }); + return; + } + + try { + const snapshotKey = `${keyPrefix}trader:${normalizedAddress}:pnl:snapshots`; + + const snapshots = await traderIntelligenceRedis.zrevrange( + snapshotKey, + 0, + limit - 1, + 'WITHSCORES' + ); + + const data = []; + for (let i = 0; i < snapshots.length; i += 2) { + const snapshotJson = snapshots[i]; + const timestamp = parseFloat(snapshots[i + 1] || '0'); + + if (!snapshotJson) continue; + + try { + const snapshot = JSON.parse(snapshotJson); + data.push({ + timestamp, + netPnl: snapshot.netPnl || 0, + realizedPnl: snapshot.realizedPnl, + unrealizedPnl: snapshot.unrealizedPnl, + }); + } catch { + continue; + } + } + + data.sort((a, b) => a.timestamp - b.timestamp); + + res.json({ + data, + period, + limit, + total: data.length, + updatedAt: Date.now(), + }); + } catch (error) { + res.status(500).json({ + error: 'pnl_snapshots_fetch_failed', + detail: error instanceof Error ? error.message : String(error) + }); + } + }); + + // Get trader's viral achievements + app.get('/api/v1/traders/:address/achievements', async (req: Request, res: Response) => { + const { address } = req.params; + const normalizedAddress = address.toLowerCase(); + const category = req.query.category as string | undefined; + const rarity = req.query.rarity as string | undefined; + const maxWarningLevel = req.query.maxWarningLevel as string | undefined; + + const { WARNING_LEVELS } = await import('../personas/constants'); + + try { + const achievementsKey = `${keyPrefix}trader:${normalizedAddress}:viral-achievements`; + const achievementIds = await traderIntelligenceRedis.smembers(achievementsKey); + + const achievements = []; + for (const achievementId of achievementIds) { + const achievementKey = `${keyPrefix}viral-achievement:${achievementId}:${normalizedAddress}`; + const achievementData = await traderIntelligenceRedis.hgetall(achievementKey); + + if (achievementData && achievementData.id) { + if (category && achievementData.category !== category) continue; + if (rarity && achievementData.rarity !== rarity) continue; + + const achievementWarningLevel = achievementData.warningLevel || 'clean'; + if (maxWarningLevel) { + const maxLevel = WARNING_LEVELS[maxWarningLevel] ?? 4; + const achievementLevel = WARNING_LEVELS[achievementWarningLevel] ?? 0; + if (achievementLevel > maxLevel) continue; + } + + achievements.push({ + id: achievementData.id, + name: achievementData.name, + description: achievementData.description, + emoji: achievementData.emoji, + rarity: achievementData.rarity, + category: achievementData.category, + funnyQuip: achievementData.funnyQuip, + warningLevel: achievementWarningLevel, + sfwShareText: achievementData.sfwShareText || undefined, + pointsBonus: parseInt(achievementData.pointsBonus || '0', 10), + assetRequired: achievementData.assetRequired, + unlockedAt: parseInt(achievementData.unlockedAt || '0', 10), + }); + } + } + + achievements.sort((a, b) => b.unlockedAt - a.unlockedAt); + + res.json({ + address: normalizedAddress, + achievements, + total: achievements.length, + warning: maxWarningLevel ? `Filtered to max warning level: ${maxWarningLevel}` : undefined, + }); + } catch (error) { + res.status(500).json({ + error: 'achievements_fetch_failed', + detail: error instanceof Error ? error.message : String(error) + }); + } + }); +}; + diff --git a/services/gateway/src/server/routes/traders/types.ts b/services/gateway/src/server/routes/traders/types.ts new file mode 100644 index 00000000..e23105ef --- /dev/null +++ b/services/gateway/src/server/routes/traders/types.ts @@ -0,0 +1,13 @@ +export type PersonaTraderSummary = { + address: string; + ensName?: string; + perpEquity: number; + openValue: number; + leverage: number; + pnl24h: number; + pnl7d: number; + pnl30d: number; + currentBias: number; + ageDays: number; +}; + diff --git a/services/ingestor/src/bootstrap.ts b/services/ingestor/src/bootstrap.ts index bdb6f4b2..b6e373e1 100644 --- a/services/ingestor/src/bootstrap.ts +++ b/services/ingestor/src/bootstrap.ts @@ -214,6 +214,17 @@ export const bootstrap = async (): Promise => { } logger.info({ runtime: config.adapter }, 'Ingestor bootstrap complete'); + logger.info( + { + kafkaOverloadBackoffMs: process.env.INGEST_KAFKA_OVERLOAD_BACKOFF_MS ?? 'default', + kafkaOverloadMaxRetries: process.env.INGEST_KAFKA_OVERLOAD_MAX_RETRIES ?? 'default', + kafkaOverloadLogIntervalMs: process.env.INGEST_KAFKA_OVERLOAD_LOG_INTERVAL_MS ?? 'default', + publishThrottleRatio: process.env.INGEST_PUBLISH_THROTTLE_RATIO ?? 'default', + publishThrottleDelayMs: process.env.INGEST_PUBLISH_THROTTLE_DELAY_MS ?? 'default', + publishThrottleMaxWaitMs: process.env.INGEST_PUBLISH_THROTTLE_MAX_WAIT_MS ?? 'default' + }, + 'Ingestor reliability tuning' + ); }, onShutdown: async ({ logger }: { logger: ServiceLogger; diff --git a/services/stats/src/index.ts b/services/stats/src/index.ts index 794e7dea..8aeb8018 100644 --- a/services/stats/src/index.ts +++ b/services/stats/src/index.ts @@ -110,6 +110,7 @@ export const bootstrap = async (): Promise => { let firstEventLogged = false; let lastStaleLogAt = 0; let monitorTimer: NodeJS.Timeout | null = null; + let wasStale = false; const redis: Redis = createManagedRedisClient({ url: config.redis.url, @@ -228,7 +229,7 @@ export const bootstrap = async (): Promise => { const { stalenessMs, isStale, uptimeMs, eventLagMs } = computeEventFreshness(); if (isStale) { const now = Date.now(); - if (now - lastStaleLogAt >= staleLogCooldownMs) { + if (!wasStale || now - lastStaleLogAt >= staleLogCooldownMs) { logger.warn( { stalenessMs, @@ -238,7 +239,10 @@ export const bootstrap = async (): Promise => { ); lastStaleLogAt = now; } + } else { + wasStale = false; } + wasStale = isStale; updateHealth({ eventsStale: isStale, eventLagMs, diff --git a/services/trader-intelligence/src/aggregators/personaAggregation.ts b/services/trader-intelligence/src/aggregators/personaAggregation.ts index 6d0effae..4b83713a 100644 --- a/services/trader-intelligence/src/aggregators/personaAggregation.ts +++ b/services/trader-intelligence/src/aggregators/personaAggregation.ts @@ -74,6 +74,8 @@ export async function aggregateAllPersonas( if (!profileData || Object.keys(profileData).length === 0) continue; try { + const winRate = parseFloat(pnlData?.winRate ?? '0'); + const adjustedWinRate = parseFloat(pnlData?.adjustedWinRate ?? pnlData?.winRate ?? '0'); const profile: TraderProfile = { address: profileData.address || '', ensName: profileData.ensName || undefined, @@ -92,7 +94,8 @@ export async function aggregateAllPersonas( totalPnl: parseFloat(pnlData?.netPnl || '0'), winningTrades: 0, losingTrades: 0, - winRate: parseFloat(pnlData?.winRate || '0'), + winRate, + adjustedWinRate, avgWin: 0, avgLoss: 0, profitFactor: parseFloat(pnlData?.profitFactor || '0'), @@ -109,7 +112,8 @@ export async function aggregateAllPersonas( takerFees: 0, }, netPnl: parseFloat(pnlData?.netPnl || '0'), - totalTrades: profileData.totalTrades ? parseInt(profileData.totalTrades, 10) : 0, + totalTrades: profileData.totalTrades ? parseInt(profileData.totalTrades, 10) : 0, + adjustedWinRate, }, risk: { address: profileData.address || '', diff --git a/services/trader-intelligence/src/aggregators/traderProfile.ts b/services/trader-intelligence/src/aggregators/traderProfile.ts index 141f374a..5ae35e45 100644 --- a/services/trader-intelligence/src/aggregators/traderProfile.ts +++ b/services/trader-intelligence/src/aggregators/traderProfile.ts @@ -76,7 +76,7 @@ export function buildTraderProfile( // Metadata lastUpdated: Date.now(), firstSeen: firstSeen || Date.now(), - totalTrades: totalTrades || fills.fills.length + totalTrades: totalTrades || (fills.fills.length > 0 ? fills.fills.length : positions.positions.length) }; } diff --git a/services/trader-intelligence/src/analyzers/pnl.ts b/services/trader-intelligence/src/analyzers/pnl.ts index 8fc7cdcd..12089322 100644 --- a/services/trader-intelligence/src/analyzers/pnl.ts +++ b/services/trader-intelligence/src/analyzers/pnl.ts @@ -42,6 +42,24 @@ export function calculatePnL( const netPnl = realized.totalPnl + unrealized.totalPnl - fees.totalPaid; const totalTrades = realized.winningTrades + realized.losingTrades; + // Calculate adjusted win rate by including open positions + // Treat profitable open positions as winners, losing as losers + let winningPositions = 0; + let losingPositions = 0; + + for (const pos of unrealized.positions) { + if (pos.pnl > 0) { + winningPositions++; + } else if (pos.pnl < 0) { + losingPositions++; + } + } + + const adjustedTotalTrades = totalTrades + winningPositions + losingPositions; + const adjustedWinRate = adjustedTotalTrades > 0 + ? (realized.winningTrades + winningPositions) / adjustedTotalTrades + : 0; + return { address, period, @@ -50,6 +68,7 @@ export function calculatePnL( fees, netPnl, totalTrades, + adjustedWinRate, // Added adjusted win rate }; } @@ -213,6 +232,7 @@ function calculateRealizedPnL(fills: Fill[]): PnLAnalysis['realized'] { winningTrades, losingTrades, winRate, + adjustedWinRate: winRate, avgWin, avgLoss, profitFactor, diff --git a/services/trader-intelligence/src/bootstrap.ts b/services/trader-intelligence/src/bootstrap.ts index 69fc5ecf..c276c3e7 100644 --- a/services/trader-intelligence/src/bootstrap.ts +++ b/services/trader-intelligence/src/bootstrap.ts @@ -341,21 +341,21 @@ async function processCollectionJob( await redisPublisher.updatePnLLeaderboard( address, pnl24h.netPnl, - pnl24h.realized.winRate, + pnl24h.adjustedWinRate ?? pnl24h.realized.adjustedWinRate ?? pnl24h.realized.winRate ?? 0, '24h', config ); await redisPublisher.updatePnLLeaderboard( address, pnl7d.netPnl, - pnl7d.realized.winRate, + pnl7d.adjustedWinRate ?? pnl7d.realized.adjustedWinRate ?? pnl7d.realized.winRate ?? 0, '7d', config ); await redisPublisher.updatePnLLeaderboard( address, pnl30d.netPnl, - pnl30d.realized.winRate, + pnl30d.adjustedWinRate ?? pnl30d.realized.adjustedWinRate ?? pnl30d.realized.winRate ?? 0, '30d', config ); diff --git a/services/trader-intelligence/src/types.ts b/services/trader-intelligence/src/types.ts index a86ac93b..736a8e42 100644 --- a/services/trader-intelligence/src/types.ts +++ b/services/trader-intelligence/src/types.ts @@ -109,6 +109,7 @@ export interface PnLAnalysis { winningTrades: number; // Count of winning trades losingTrades: number; // Count of losing trades winRate: number; // 0-1, winningTrades / totalTrades + adjustedWinRate: number; // 0-1, includes open positions avgWin: number; // Average winning trade PnL avgLoss: number; // Average losing trade PnL (negative) profitFactor: number; // Total wins / total losses @@ -141,6 +142,7 @@ export interface PnLAnalysis { netPnl: number; // Realized + unrealized - fees totalTrades: number; // Total trades within the period + adjustedWinRate?: number; // Win rate including open profitable positions } export interface RiskProfile { diff --git a/services/trader-intelligence/src/utils/statsCollector.ts b/services/trader-intelligence/src/utils/statsCollector.ts index 68d75c1c..e4efe194 100644 --- a/services/trader-intelligence/src/utils/statsCollector.ts +++ b/services/trader-intelligence/src/utils/statsCollector.ts @@ -119,6 +119,7 @@ const createPnLAnalysis = ( winningTrades: wins, losingTrades: losses, winRate, + adjustedWinRate: winRate, avgWin: wins > 0 ? Math.max(netPnl, 0) / wins : 0, avgLoss: losses > 0 ? Math.min(netPnl, 0) / losses : 0, profitFactor, @@ -136,6 +137,7 @@ const createPnLAnalysis = ( }, netPnl, totalTrades, + adjustedWinRate: winRate, }; }; @@ -435,7 +437,7 @@ export const collectTraderFromStats = async ({ await redisPublisher.updatePnLLeaderboard( normalizedAddress, netPnl, - pnLAnalysis.realized.winRate, + pnLAnalysis.adjustedWinRate ?? pnLAnalysis.realized.winRate, window, config, ); diff --git a/yarn.lock b/yarn.lock index 76834f62..d6e1102d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -857,6 +857,29 @@ resolved "https://registry.npmjs.org/@rushstack/eslint-patch/-/eslint-patch-1.14.1.tgz" integrity sha512-jGTk8UD/RdjsNZW8qq10r0RBvxL8OWtoT+kImlzPDFilmozzM+9QmIJsmze9UiSBrFU45ZxhTYBypn9q9z/VfQ== +"@scalar/core@0.3.23": + version "0.3.23" + resolved "https://registry.yarnpkg.com/@scalar/core/-/core-0.3.23.tgz#0595e6fe00ffb39d620319d4f929c4200a3139ec" + integrity sha512-hop7LVR3MKB2VpS8dly3gmmbB3lBGxQRtL0pBaC77zFMRHoBv1DuB2bj8l4gxd5grzitJ1LsYduvywLAMY9F6g== + dependencies: + "@scalar/types" "0.5.0" + +"@scalar/express-api-reference@^0.8.25": + version "0.8.25" + resolved "https://registry.yarnpkg.com/@scalar/express-api-reference/-/express-api-reference-0.8.25.tgz#8c8b737b6f6f957b53e1fcb9378b606988ac5e34" + integrity sha512-Lxm6zLZulk+EJFuNStCrJWFQUnKMQH3p1mYX3Xdw229EdF/h+LLG6Os76kzFNoyJisYRC9L0yWJrgLam5xJ4qg== + dependencies: + "@scalar/core" "0.3.23" + +"@scalar/types@0.5.0": + version "0.5.0" + resolved "https://registry.yarnpkg.com/@scalar/types/-/types-0.5.0.tgz#4bdbaf50316c88847562771dad50b5962713e81e" + integrity sha512-imDMuTieOc5kHM9/Kt/1lmiI5ZtusuaYlzsXTP99IsWvD8mJ7ivF73lPBRj4PKtg4vY+ta5CO/vJpvnCYandRg== + dependencies: + nanoid "5.1.5" + type-fest "5.0.0" + zod "4.1.11" + "@sinclair/typebox@^0.27.8": version "0.27.8" resolved "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz" @@ -1267,6 +1290,15 @@ resolved "https://registry.yarnpkg.com/@unrs/resolver-binding-win32-x64-msvc/-/resolver-binding-win32-x64-msvc-1.11.1.tgz#538b1e103bf8d9864e7b85cc96fa8d6fb6c40777" integrity sha512-lrW200hZdbfRtztbygyaq/6jP6AKE8qQN2KvPcJ+x7wiD038YtnYtZ82IMNJ69GJibV7bwL3y9FgK+5w/pYt6g== +"@vitest/expect@1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@vitest/expect/-/expect-1.6.0.tgz#0b3ba0914f738508464983f4d811bc122b51fb30" + integrity sha512-ixEvFVQjycy/oNgHjqsL6AZCDduC+tflRluaHIzKIsdbzkLn2U/iBnVeJwB6HsIjQBdfMR8Z0tRxKUsvFJEeWQ== + dependencies: + "@vitest/spy" "1.6.0" + "@vitest/utils" "1.6.0" + chai "^4.3.10" + "@vitest/expect@1.6.1": version "1.6.1" resolved "https://registry.npmjs.org/@vitest/expect/-/expect-1.6.1.tgz" @@ -1276,6 +1308,15 @@ "@vitest/utils" "1.6.1" chai "^4.3.10" +"@vitest/runner@1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@vitest/runner/-/runner-1.6.0.tgz#a6de49a96cb33b0e3ba0d9064a3e8d6ce2f08825" + integrity sha512-P4xgwPjwesuBiHisAVz/LSSZtDjOTPYZVmNAnpHHSR6ONrf8eCJOFRvUwdHn30F5M1fxhqtl7QZQUk2dprIXAg== + dependencies: + "@vitest/utils" "1.6.0" + p-limit "^5.0.0" + pathe "^1.1.1" + "@vitest/runner@1.6.1": version "1.6.1" resolved "https://registry.npmjs.org/@vitest/runner/-/runner-1.6.1.tgz" @@ -1285,6 +1326,15 @@ p-limit "^5.0.0" pathe "^1.1.1" +"@vitest/snapshot@1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@vitest/snapshot/-/snapshot-1.6.0.tgz#deb7e4498a5299c1198136f56e6e0f692e6af470" + integrity sha512-+Hx43f8Chus+DCmygqqfetcAZrDJwvTj0ymqjQq4CvmpKFSTVteEOBzCusu1x2tt4OJcvBflyHUE0DZSLgEMtQ== + dependencies: + magic-string "^0.30.5" + pathe "^1.1.1" + pretty-format "^29.7.0" + "@vitest/snapshot@1.6.1": version "1.6.1" resolved "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-1.6.1.tgz" @@ -1294,6 +1344,13 @@ pathe "^1.1.1" pretty-format "^29.7.0" +"@vitest/spy@1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@vitest/spy/-/spy-1.6.0.tgz#362cbd42ccdb03f1613798fde99799649516906d" + integrity sha512-leUTap6B/cqi/bQkXUu6bQV5TZPx7pmMBKBQiI0rJA8c3pB56ZsaTbREnF7CJfmvAS4V2cXIBAh/3rVwrrCYgw== + dependencies: + tinyspy "^2.2.0" + "@vitest/spy@1.6.1": version "1.6.1" resolved "https://registry.npmjs.org/@vitest/spy/-/spy-1.6.1.tgz" @@ -1301,6 +1358,16 @@ dependencies: tinyspy "^2.2.0" +"@vitest/utils@1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@vitest/utils/-/utils-1.6.0.tgz#5c5675ca7d6f546a7b4337de9ae882e6c57896a1" + integrity sha512-21cPiuGMoMZwiOHa2i4LXkMkMkCGzA+MVFV70jRwHo95dL4x/ts5GZhML1QWuy7yfp3WzK3lRvZi3JnXTYqrBw== + dependencies: + diff-sequences "^29.6.3" + estree-walker "^3.0.3" + loupe "^2.3.7" + pretty-format "^29.7.0" + "@vitest/utils@1.6.1": version "1.6.1" resolved "https://registry.npmjs.org/@vitest/utils/-/utils-1.6.1.tgz" @@ -1331,6 +1398,11 @@ acorn-walk@^8.3.2: dependencies: acorn "^8.11.0" +acorn@^7.4.1: + version "7.4.1" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" + integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A== + acorn@^8.11.0, acorn@^8.15.0, acorn@^8.9.0: version "8.15.0" resolved "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz" @@ -1917,6 +1989,11 @@ deep-is@^0.1.3: resolved "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz" integrity sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ== +deepmerge@^4.2.2: + version "4.3.1" + resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.3.1.tgz#44b5f2147cd3b00d4b56137685966f26fd25dd4a" + integrity sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A== + define-data-property@^1.0.1, define-data-property@^1.1.4: version "1.1.4" resolved "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz" @@ -2792,7 +2869,7 @@ glob@^10.3.10, glob@^10.3.7: package-json-from-dist "^1.0.0" path-scurry "^1.11.1" -glob@^7.1.3: +glob@^7.1.3, glob@^7.1.7: version "7.2.3" resolved "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz" integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q== @@ -3279,6 +3356,11 @@ json5@^1.0.2: dependencies: minimist "^1.2.0" +json5@^2.2.3: + version "2.2.3" + resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283" + integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg== + "jsx-ast-utils@^2.4.1 || ^3.0.0", jsx-ast-utils@^3.3.5: version "3.3.5" resolved "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.3.5.tgz" @@ -3531,6 +3613,11 @@ mz@^2.7.0: object-assign "^4.0.1" thenify-all "^1.0.0" +nanoid@5.1.5: + version "5.1.5" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-5.1.5.tgz#f7597f9d9054eb4da9548cdd53ca70f1790e87de" + integrity sha512-Ir/+ZpE9fDsNH0hQ3C68uyThDXzYcim2EqcZ8zn8Chtt1iylPT9xXJB0kPCnqzgcEGikO9RxSrh63MsmVCU7Fw== + nanoid@^3.3.11, nanoid@^3.3.6: version "3.3.11" resolved "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz" @@ -4666,6 +4753,21 @@ supports-preserve-symlinks-flag@^1.0.0: resolved "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz" integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w== +swagger-autogen@^2.23.7: + version "2.23.7" + resolved "https://registry.yarnpkg.com/swagger-autogen/-/swagger-autogen-2.23.7.tgz#40023e583b1d4b4321313bb92cc768488758f135" + integrity sha512-vr7uRmuV0DCxWc0wokLJAwX3GwQFJ0jwN+AWk0hKxre2EZwusnkGSGdVFd82u7fQLgwSTnbWkxUL7HXuz5LTZQ== + dependencies: + acorn "^7.4.1" + deepmerge "^4.2.2" + glob "^7.1.7" + json5 "^2.2.3" + +tagged-tag@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/tagged-tag/-/tagged-tag-1.0.0.tgz#a0b5917c2864cba54841495abfa3f6b13edcf4d6" + integrity sha512-yEFYrVhod+hdNyx7g5Bnkkb0G6si8HJurOoOEgC8B/O0uXLHlaey/65KRv6cuWBNhBgHKAROVpc7QyYqE5gFng== + tailwind-merge@^2.5.2: version "2.6.0" resolved "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-2.6.0.tgz" @@ -4832,6 +4934,13 @@ type-detect@^4.0.0, type-detect@^4.1.0: resolved "https://registry.npmjs.org/type-detect/-/type-detect-4.1.0.tgz" integrity sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw== +type-fest@5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-5.0.0.tgz#4d3967e358f3941129f7ef6483be8ca8599a028a" + integrity sha512-GeJop7+u7BYlQ6yQCAY1nBQiRSHR+6OdCEtd8Bwp9a3NK3+fWAVjOaPKJDteB9f6cIJ0wt4IfnScjLG450EpXA== + dependencies: + tagged-tag "^1.0.0" + type-fest@^0.20.2: version "0.20.2" resolved "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz" @@ -4997,6 +5106,17 @@ vary@^1, vary@~1.1.2: resolved "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz" integrity sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg== +vite-node@1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/vite-node/-/vite-node-1.6.0.tgz#2c7e61129bfecc759478fa592754fd9704aaba7f" + integrity sha512-de6HJgzC+TFzOu0NTC4RAIsyf/DY/ibWDYQUcuEA84EMHhcefTUGkjFHKKEJhQN4A+6I0u++kr3l36ZF2d7XRw== + dependencies: + cac "^6.7.14" + debug "^4.3.4" + pathe "^1.1.1" + picocolors "^1.0.0" + vite "^5.0.0" + vite-node@1.6.1: version "1.6.1" resolved "https://registry.npmjs.org/vite-node/-/vite-node-1.6.1.tgz" @@ -5019,6 +5139,32 @@ vite@^5.0.0: optionalDependencies: fsevents "~2.3.3" +vitest@1.6.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/vitest/-/vitest-1.6.0.tgz#9d5ad4752a3c451be919e412c597126cffb9892f" + integrity sha512-H5r/dN06swuFnzNFhq/dnz37bPXnq8xB2xB5JOVk8K09rUtoeNN+LHWkoQ0A/i3hvbUKKcCei9KpbxqHMLhLLA== + dependencies: + "@vitest/expect" "1.6.0" + "@vitest/runner" "1.6.0" + "@vitest/snapshot" "1.6.0" + "@vitest/spy" "1.6.0" + "@vitest/utils" "1.6.0" + acorn-walk "^8.3.2" + chai "^4.3.10" + debug "^4.3.4" + execa "^8.0.1" + local-pkg "^0.5.0" + magic-string "^0.30.5" + pathe "^1.1.1" + picocolors "^1.0.0" + std-env "^3.5.0" + strip-literal "^2.0.0" + tinybench "^2.5.1" + tinypool "^0.8.3" + vite "^5.0.0" + vite-node "1.6.0" + why-is-node-running "^2.2.2" + vitest@^1.6.0: version "1.6.1" resolved "https://registry.npmjs.org/vitest/-/vitest-1.6.1.tgz" @@ -5197,3 +5343,8 @@ yocto-queue@^1.0.0: version "1.2.1" resolved "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.2.1.tgz" integrity sha512-AyeEbWOu/TAXdxlV9wmGcR0+yh2j3vYPGOECcIj2S7MkrLyC7ne+oye2BKTItt0ii2PHk4cDy+95+LshzbXnGg== + +zod@4.1.11: + version "4.1.11" + resolved "https://registry.yarnpkg.com/zod/-/zod-4.1.11.tgz#4aab62f76cfd45e6c6166519ba31b2ea019f75f5" + integrity sha512-WPsqwxITS2tzx1bzhIKsEs19ABD5vmCVa4xBo2tq/SrV4RNZtfws1EnCWQXM6yh8bD08a1idvkB5MZSBiZsjwg==