From c804b76a0c319a445627608a1330b36b96a32fc3 Mon Sep 17 00:00:00 2001 From: Stenvro Date: Thu, 24 Sep 2026 03:30:53 +0000 Subject: [PATCH 1/8] feat: data-driven exchange registry, add Bybit/Gate/Bitget/MEXC/HTX/BingX --- README.md | 14 ++- backend/core/exchange_registry.py | 104 ++++++++++++++---- backend/engine/candle_poller.py | 5 +- backend/routers/keys.py | 46 +++----- frontend/src/api/exchanges.js | 61 ++++++++++ .../src/components/Builder/CustomNodes.jsx | 14 +-- frontend/src/components/DataManager.jsx | 14 +-- frontend/src/components/Settings.jsx | 27 ++--- tests/test_exchange_registry.py | 66 +++++++++++ 9 files changed, 258 insertions(+), 93 deletions(-) create mode 100644 frontend/src/api/exchanges.js create mode 100644 tests/test_exchange_registry.py diff --git a/README.md b/README.md index 8a47e1d..69bbc31 100644 --- a/README.md +++ b/README.md @@ -523,11 +523,17 @@ Click **Duplicate** on any stopped bot card to create a clone with `(copy)` appe | Binance | No | Yes | | | Bitvavo | No | No | EU exchange | | Coinbase | No | No | | -| Crypto.com | No | No | | +| Crypto.com | No | Yes | UAT environment | | Kraken | No | No | | -| KuCoin | Yes | Yes | | - -Adding support for any other CCXT-compatible exchange requires only adding it to the frontend dropdowns and `SUPPORTED_EXCHANGES` in `exchange_registry.py`. +| KuCoin | Yes | No | | +| Bybit | No | Yes | testnet.bybit.com | +| Gate | No | Yes | testnet.gate.io | +| Bitget | Yes | Yes | Demo trading (request header) | +| MEXC | No | No | | +| HTX | No | No | | +| BingX | No | Yes | VST demo trading | + +Adding another CCXT-compatible exchange is one `ExchangeSpec` entry in `backend/core/exchange_registry.py`; the key form, the data manager and the builder read the list from `GET /api/keys/exchanges`. ### Historical data per exchange diff --git a/backend/core/exchange_registry.py b/backend/core/exchange_registry.py index ffa7ea5..88d43fc 100644 --- a/backend/core/exchange_registry.py +++ b/backend/core/exchange_registry.py @@ -1,30 +1,95 @@ import logging import threading import time +from dataclasses import dataclass, field import ccxt logger = logging.getLogger("apexalgo.exchange_registry") -# All exchanges supported by ApexAlgo. Key = ccxt exchange ID, Value = display name. -SUPPORTED_EXCHANGES: dict[str, str] = { - "okx": "OKX", - "binance": "Binance", - "bitvavo": "Bitvavo", - "coinbase": "Coinbase", - "cryptocom": "Crypto.com", - "kraken": "Kraken", - "kucoin": "KuCoin", -} -# Exchanges that authenticate via api_key + api_secret + passphrase. -# All others use only api_key + api_secret. -_PASSPHRASE_EXCHANGES: frozenset[str] = frozenset({"okx", "kucoin"}) +@dataclass(frozen=True) +class ExchangeSpec: + """Everything ApexAlgo knows about one exchange, in one place. -# Per-exchange static configuration applied at instantiation. -_EXCHANGE_CONFIG: dict[str, dict] = { - "okx": {"hostname": "eea.okx.com"}, + Adding an exchange = one entry in `EXCHANGES`; the routers, the key form + and the builder derive their lists from here (via `GET /api/keys/exchanges`). + ccxt itself is id-agnostic, so nothing engine-side needs to change. + """ + name: str # display name + needs_passphrase: bool = False # api_key + secret + passphrase + ccxt_config: dict = field(default_factory=dict) # extra ccxt constructor config + keys_url: str | None = None # where to create API keys + sandbox_note: str | None = None # how to get testnet keys, if any + candle_limit_note: str | None = None # shown when the exchange caps OHLCV history + + +# All exchanges supported by ApexAlgo, keyed by ccxt exchange ID. +EXCHANGES: dict[str, ExchangeSpec] = { + "okx": ExchangeSpec( + "OKX", needs_passphrase=True, ccxt_config={"hostname": "eea.okx.com"}, + keys_url="https://www.okx.com/account/my-api", + sandbox_note="Demo trading keys are created under Trade → Demo trading → API."), + "binance": ExchangeSpec( + "Binance", keys_url="https://www.binance.com/en/my/settings/api-management", + sandbox_note="Spot testnet keys come from testnet.binance.vision (separate account)."), + "bitvavo": ExchangeSpec("Bitvavo", keys_url="https://account.bitvavo.com/user/api"), + "coinbase": ExchangeSpec("Coinbase", keys_url="https://www.coinbase.com/settings/api"), + "cryptocom": ExchangeSpec( + "Crypto.com", keys_url="https://crypto.com/exchange/user/settings/api-management", + sandbox_note="UAT sandbox keys are issued via the Crypto.com Exchange UAT environment."), + "kraken": ExchangeSpec( + "Kraken", keys_url="https://www.kraken.com/u/security/api", + candle_limit_note="Kraken only serves its most recent 720 candles per timeframe"), + "kucoin": ExchangeSpec("KuCoin", needs_passphrase=True, keys_url="https://www.kucoin.com/account/api"), + "bybit": ExchangeSpec( + "Bybit", keys_url="https://www.bybit.com/app/user/api-management", + sandbox_note="Testnet keys are created on testnet.bybit.com (separate account)."), + "gateio": ExchangeSpec( + "Gate", keys_url="https://www.gate.io/myaccount/api_key_manage", + sandbox_note="Testnet keys are created on testnet.gate.io (separate account)."), + "bitget": ExchangeSpec( + "Bitget", needs_passphrase=True, keys_url="https://www.bitget.com/account/newapi", + sandbox_note="Demo-trading keys are created under Demo Trading → API on bitget.com."), + "mexc": ExchangeSpec( + "MEXC", keys_url="https://www.mexc.com/user/openapi", + sandbox_note="MEXC has no public testnet — keys can only be used for real trading."), + "htx": ExchangeSpec( + "HTX", keys_url="https://www.htx.com/en-us/apikey/", + sandbox_note="HTX has no public testnet — keys can only be used for real trading."), + "bingx": ExchangeSpec( + "BingX", keys_url="https://bingx.com/en/account/api/", + sandbox_note="VST demo-trading keys are created under Demo Trading → API on bingx.com."), } +# Backwards-compatible views used by the routers: {id: display name} and the +# set of exchanges that authenticate with a passphrase. +SUPPORTED_EXCHANGES: dict[str, str] = {ex_id: spec.name for ex_id, spec in EXCHANGES.items()} +_PASSPHRASE_EXCHANGES: frozenset[str] = frozenset(ex_id for ex_id, spec in EXCHANGES.items() if spec.needs_passphrase) + + +def exchange_spec(exchange_id: str) -> ExchangeSpec | None: + """Spec for an exchange id (case-insensitive); None when unsupported.""" + return EXCHANGES.get(str(exchange_id or "").lower()) + + +_sandbox_cache: dict[str, bool] = {} + + +def exchange_has_sandbox(exchange_id: str) -> bool: + """Whether ccxt can put this exchange in sandbox/testnet mode. Probed on + a throwaway instance (no network): some exchanges use a test URL, others + a demo-trading header (Bitget), and ccxt raises for the rest.""" + exchange_id = str(exchange_id or "").lower() + if exchange_id not in _sandbox_cache: + ok = False + try: + getattr(ccxt, exchange_id)().set_sandbox_mode(True) + ok = True + except Exception: + pass + _sandbox_cache[exchange_id] = ok + return _sandbox_cache[exchange_id] + def build_exchange( exchange_id: str, @@ -43,7 +108,7 @@ def build_exchange( api_key / api_secret : str, optional Omit for unauthenticated (public market data) connections. passphrase : str, optional - Only applied for exchanges that require it (OKX, KuCoin). + Only applied for exchanges whose spec has `needs_passphrase`. sandbox : bool Enable exchange sandbox / testnet mode when supported. """ @@ -62,14 +127,15 @@ def build_exchange( "Run: pip install --upgrade ccxt" ) + spec = EXCHANGES[exchange_id] config: dict = {"enableRateLimit": True} - config.update(_EXCHANGE_CONFIG.get(exchange_id, {})) + config.update(spec.ccxt_config) if api_key: config["apiKey"] = api_key config["secret"] = api_secret - if passphrase and exchange_id in _PASSPHRASE_EXCHANGES: + if passphrase and spec.needs_passphrase: config["password"] = passphrase exchange = cls(config) diff --git a/backend/engine/candle_poller.py b/backend/engine/candle_poller.py index 3a3a54b..93c224f 100644 --- a/backend/engine/candle_poller.py +++ b/backend/engine/candle_poller.py @@ -9,7 +9,7 @@ from sqlalchemy.orm import Session from backend.core.database import SessionLocal -from backend.core.exchange_registry import build_exchange +from backend.core.exchange_registry import build_exchange, exchange_spec from backend.core.events import event_bus from backend.models.bots import BotConfig from backend.models.candles import Candle @@ -455,7 +455,8 @@ def _backfill_one_symbol( if oldest_have is not None and oldest_have > requested_start + 2 * tf_ms and known_start != oldest_have: self._listing_start[(exchange_name, symbol, timeframe)] = oldest_have available = (now_ms - oldest_have) // tf_ms - cap_note = " (Kraken only serves its most recent 720 candles per timeframe)" if exchange_name == "kraken" else "" + spec = exchange_spec(exchange_name) + cap_note = f" ({spec.candle_limit_note})" if spec and spec.candle_limit_note else "" logger.warning( "Back-fill: %s/%s/%s — exchange has no data before %s%s; %d of the requested %d candles are available. " "Use a larger timeframe or another data exchange for a longer backtest.", diff --git a/backend/routers/keys.py b/backend/routers/keys.py index 58cfdd6..9036d04 100644 --- a/backend/routers/keys.py +++ b/backend/routers/keys.py @@ -14,7 +14,10 @@ from backend.models.bots import BotConfig from backend.core.security import verify_api_key from backend.core.encryption import encrypt_data -from backend.core.exchange_registry import build_exchange, get_authenticated_exchange, invalidate_authenticated_exchange, SUPPORTED_EXCHANGES +from backend.core.exchange_registry import ( + build_exchange, get_authenticated_exchange, invalidate_authenticated_exchange, + exchange_has_sandbox, EXCHANGES, SUPPORTED_EXCHANGES, +) logger = logging.getLogger("apexalgo.keys") @@ -33,37 +36,20 @@ class ExchangeKeyCreate(BaseModel): is_sandbox: bool = True -# Where to create API keys + what the user must switch off. Shown in the UI -# next to the form so nobody has to leave the app to figure this out. -_EXCHANGE_GUIDE = { - "okx": {"keys_url": "https://www.okx.com/account/my-api", "sandbox_note": "Demo trading keys are created under Trade → Demo trading → API."}, - "binance": {"keys_url": "https://www.binance.com/en/my/settings/api-management", "sandbox_note": "Spot testnet keys come from testnet.binance.vision (separate account)."}, - "bitvavo": {"keys_url": "https://account.bitvavo.com/user/api", "sandbox_note": None}, - "coinbase": {"keys_url": "https://www.coinbase.com/settings/api", "sandbox_note": None}, - "cryptocom": {"keys_url": "https://crypto.com/exchange/user/settings/api-management", "sandbox_note": "UAT sandbox keys are issued via the Crypto.com Exchange UAT environment."}, - "kraken": {"keys_url": "https://www.kraken.com/u/security/api", "sandbox_note": None}, - "kucoin": {"keys_url": "https://www.kucoin.com/account/api", "sandbox_note": None}, -} - - @router.get("/exchanges") def list_exchanges(): - """Static capabilities per supported exchange for the connection form.""" + """Capabilities per supported exchange (from the registry spec) for the + connection form, the data manager and the builder. `has_sandbox` is + probed from ccxt so it tracks the installed version.""" out = [] - for ex_id, name in SUPPORTED_EXCHANGES.items(): - has_sandbox = False - try: - has_sandbox = bool(getattr(ccxt, ex_id)().urls.get("test")) - except Exception: - pass - guide = _EXCHANGE_GUIDE.get(ex_id, {}) + for ex_id, spec in EXCHANGES.items(): out.append({ "id": ex_id, - "name": name, - "needs_passphrase": ex_id in ("okx", "kucoin"), - "has_sandbox": has_sandbox, - "keys_url": guide.get("keys_url"), - "sandbox_note": guide.get("sandbox_note"), + "name": spec.name, + "needs_passphrase": spec.needs_passphrase, + "has_sandbox": exchange_has_sandbox(ex_id), + "keys_url": spec.keys_url, + "sandbox_note": spec.sandbox_note, }) return out @@ -304,6 +290,8 @@ def execute_quick_swap(name: str, payload: SwapRequest, db: Session = Depends(ge return JSONResponse(status_code=404, content={"detail": f"API Wallet '{name}' not found"}) exchange = get_authenticated_exchange(key_record) + ex_id = str(key_record.exchange or "").lower() + ex_name = SUPPORTED_EXCHANGES.get(ex_id, ex_id) from_asset = payload.from_asset to_asset = payload.to_asset @@ -349,7 +337,7 @@ def execute_quick_swap(name: str, payload: SwapRequest, db: Session = Depends(ge fetched_order = exchange.fetch_order(order['id'], order['symbol']) if fetched_order['status'] == 'canceled': - return JSONResponse(status_code=400, content={"detail": f"OKX canceled the order. Reason: Zero liquidity for {order['symbol']} on the Testnet."}) + return JSONResponse(status_code=400, content={"detail": f"{ex_name} canceled the order. Reason: Zero liquidity for {order['symbol']} on the testnet."}) if fetched_order['status'] == 'open': exchange.cancel_order(order['id'], order['symbol']) return JSONResponse(status_code=400, content={"detail": f"Order stuck. No volume for {order['symbol']} on the Sandbox. Order auto-canceled to prevent stuck balance."}) @@ -358,7 +346,7 @@ def execute_quick_swap(name: str, payload: SwapRequest, db: Session = Depends(ge except ccxt.ExchangeError as e: error_msg = str(e) - if "51155" in error_msg or "compliance" in error_msg.lower(): + if ex_id == "okx" and ("51155" in error_msg or "compliance" in error_msg.lower()): return JSONResponse(status_code=400, content={"detail": "European Compliance Error (MiCA): You cannot trade USDT on OKX in Europe. Please swap to USDC or EUR instead."}) return JSONResponse(status_code=400, content={"detail": "Exchange rejected the order. Please check your assets and try again."}) except ccxt.InsufficientFunds: diff --git a/frontend/src/api/exchanges.js b/frontend/src/api/exchanges.js new file mode 100644 index 0000000..0541e6b --- /dev/null +++ b/frontend/src/api/exchanges.js @@ -0,0 +1,61 @@ +// Supported exchanges, loaded once from the backend registry +// (GET /api/keys/exchanges). backend/core/exchange_registry.py is the single +// source of truth; the static fallback below only bridges the first render +// and an unreachable backend, so an exchange added there shows up everywhere +// (key form, data manager, builder) without touching the frontend. +// +// Shape of each entry: +// { id, name, needs_passphrase, has_sandbox, keys_url, sandbox_note } +import { useEffect, useSyncExternalStore } from 'react'; +import { apiClient } from './client'; + +export const FALLBACK_EXCHANGES = [ + { id: 'okx', name: 'OKX', needs_passphrase: true, has_sandbox: true }, + { id: 'binance', name: 'Binance', needs_passphrase: false, has_sandbox: true }, + { id: 'bitvavo', name: 'Bitvavo', needs_passphrase: false, has_sandbox: false }, + { id: 'coinbase', name: 'Coinbase', needs_passphrase: false, has_sandbox: false }, + { id: 'cryptocom', name: 'Crypto.com', needs_passphrase: false, has_sandbox: true }, + { id: 'kraken', name: 'Kraken', needs_passphrase: false, has_sandbox: false }, + { id: 'kucoin', name: 'KuCoin', needs_passphrase: true, has_sandbox: false }, + { id: 'bybit', name: 'Bybit', needs_passphrase: false, has_sandbox: true }, + { id: 'gateio', name: 'Gate', needs_passphrase: false, has_sandbox: true }, + { id: 'bitget', name: 'Bitget', needs_passphrase: true, has_sandbox: true }, + { id: 'mexc', name: 'MEXC', needs_passphrase: false, has_sandbox: false }, + { id: 'htx', name: 'HTX', needs_passphrase: false, has_sandbox: false }, + { id: 'bingx', name: 'BingX', needs_passphrase: false, has_sandbox: true }, +]; + +let cache = null; +let inflight = null; +const listeners = new Set(); +const notify = () => listeners.forEach((fn) => fn()); + +/** Fetches the list once; concurrent callers share the request. Failed loads are not cached. */ +export function loadExchanges() { + if (cache) return Promise.resolve(cache); + if (!inflight) { + inflight = apiClient.get('/api/keys/exchanges') + .then((res) => { + if (Array.isArray(res.data) && res.data.length) cache = res.data; + return cache || FALLBACK_EXCHANGES; + }) + .finally(() => { inflight = null; notify(); }); + } + return inflight; +} + +const subscribe = (fn) => { listeners.add(fn); return () => listeners.delete(fn); }; +const snapshot = () => cache || FALLBACK_EXCHANGES; + +/** React hook: the exchange list (fallback until loaded, then the backend registry). */ +export function useExchanges() { + const exchanges = useSyncExternalStore(subscribe, snapshot); + useEffect(() => { if (!cache) loadExchanges().catch(() => {}); }, []); + return exchanges; +} + +/** Display name for an exchange id ('okx' → 'OKX'); falls back to the upper-cased id. */ +export function exchangeName(id) { + const hit = snapshot().find((e) => e.id === id); + return hit ? hit.name : String(id || '').toUpperCase(); +} diff --git a/frontend/src/components/Builder/CustomNodes.jsx b/frontend/src/components/Builder/CustomNodes.jsx index ac6d4af..7d76711 100644 --- a/frontend/src/components/Builder/CustomNodes.jsx +++ b/frontend/src/components/Builder/CustomNodes.jsx @@ -1,6 +1,7 @@ import React from 'react'; import { Handle, Position } from 'reactflow'; import { useIndicators } from './indicatorConfig'; +import { useExchanges } from '../../api/exchanges'; import { DEFAULT_PAIR, parsePairs } from './pairs'; // All known timeframes with display labels @@ -236,17 +237,8 @@ export const BacktestNode = ({ id, data }) => ( ); -const API_KEY_NODE_EXCHANGES = [ - { id: 'okx', name: 'OKX' }, - { id: 'binance', name: 'Binance' }, - { id: 'bitvavo', name: 'Bitvavo' }, - { id: 'coinbase', name: 'Coinbase' }, - { id: 'cryptocom', name: 'Crypto.com' }, - { id: 'kraken', name: 'Kraken' }, - { id: 'kucoin', name: 'KuCoin' }, -]; - export const ApiKeyNode = ({ id, data }) => { + const exchanges = useExchanges(); const selectedKey = data.apiKeyName || ''; const keyRecord = data.availableKeys?.find(k => k.name === selectedKey); const derivedExchange = keyRecord?.exchange || null; @@ -276,7 +268,7 @@ export const ApiKeyNode = ({ id, data }) => {
diff --git a/frontend/src/components/DataManager.jsx b/frontend/src/components/DataManager.jsx index 4385b60..7461f23 100644 --- a/frontend/src/components/DataManager.jsx +++ b/frontend/src/components/DataManager.jsx @@ -14,16 +14,7 @@ import { Input, Select } from './ui/Input'; import { Skeleton } from './ui/Skeleton'; import { toast } from './ui/Toast'; import { confirmDialog } from './ui/ConfirmDialog'; - -const EXCHANGES = [ - { id: 'okx', name: 'OKX' }, - { id: 'binance', name: 'Binance' }, - { id: 'bitvavo', name: 'Bitvavo' }, - { id: 'coinbase', name: 'Coinbase' }, - { id: 'cryptocom', name: 'Crypto.com' }, - { id: 'kraken', name: 'Kraken' }, - { id: 'kucoin', name: 'KuCoin' }, -]; +import { useExchanges } from '../api/exchanges'; /* ── Inline icons (stroke 1.8) ── */ const IconSync = ( @@ -83,6 +74,7 @@ export default function DataManager({ openChart }) { const [syncingSymbol, setSyncingSymbol] = useState(null); const [symbol, setSymbol] = useState('BTC-USDC'); + const exchanges = useExchanges(); const [exchange, setExchange] = useState('okx'); const [timeframe, setTimeframe] = useState('1d'); const [startDate, setStartDate] = useState('2024-01-01T00:00'); @@ -431,7 +423,7 @@ export default function DataManager({ openChart }) {
diff --git a/frontend/src/components/Settings.jsx b/frontend/src/components/Settings.jsx index 082ec5c..0931b26 100644 --- a/frontend/src/components/Settings.jsx +++ b/frontend/src/components/Settings.jsx @@ -11,19 +11,9 @@ import { Input, Select } from './ui/Input'; import { SkeletonCard } from './ui/Skeleton'; import { toast } from './ui/Toast'; import { confirmDialog } from './ui/ConfirmDialog'; +import { useExchanges } from '../api/exchanges'; -/* Static fallback until /api/keys/exchanges answers */ -const FALLBACK_EXCHANGES = [ - { id: 'okx', name: 'OKX', needs_passphrase: true, has_sandbox: true }, - { id: 'binance', name: 'Binance', needs_passphrase: false, has_sandbox: true }, - { id: 'bitvavo', name: 'Bitvavo', needs_passphrase: false, has_sandbox: false }, - { id: 'coinbase', name: 'Coinbase', needs_passphrase: false, has_sandbox: false }, - { id: 'cryptocom', name: 'Crypto.com', needs_passphrase: false, has_sandbox: true }, - { id: 'kraken', name: 'Kraken', needs_passphrase: false, has_sandbox: false }, - { id: 'kucoin', name: 'KuCoin', needs_passphrase: true, has_sandbox: false }, -]; - -/* Deterministic avatar color per exchange (token values) */ +/* Deterministic avatar color per exchange (token values); unknown ids get the neutral class in the avatar */ const AVATAR_COLORS = { okx: 'text-info border-info/30 bg-info/10', binance: 'text-accent border-accent/30 bg-accent/10', @@ -32,6 +22,12 @@ const AVATAR_COLORS = { cryptocom: 'text-purple border-purple/30 bg-purple/10', kraken: 'text-purple border-purple/30 bg-purple/10', kucoin: 'text-success border-success/30 bg-success/10', + bybit: 'text-accent border-accent/30 bg-accent/10', + gateio: 'text-info border-info/30 bg-info/10', + bitget: 'text-info border-info/30 bg-info/10', + mexc: 'text-success border-success/30 bg-success/10', + htx: 'text-purple border-purple/30 bg-purple/10', + bingx: 'text-info border-info/30 bg-info/10', }; const IconKeyEmpty = ( @@ -139,7 +135,7 @@ function WalletPanel({ wallet }) { export default function Settings() { const [keys, setKeys] = useState([]); - const [exchanges, setExchanges] = useState(FALLBACK_EXCHANGES); + const exchanges = useExchanges(); const [initialLoading, setInitialLoading] = useState(true); const [loading, setLoading] = useState(false); const [refreshing, setRefreshing] = useState(false); @@ -156,7 +152,7 @@ export default function Settings() { const [passphrase, setPassphrase] = useState(''); const [isSandbox, setIsSandbox] = useState(true); - const exchangeInfo = exchanges.find(e => e.id === selectedExchange) || FALLBACK_EXCHANGES[0]; + const exchangeInfo = exchanges.find(e => e.id === selectedExchange) || exchanges[0]; const exchangeNames = useMemo(() => Object.fromEntries(exchanges.map(e => [e.id, e.name])), [exchanges]); const needsPassphrase = !!exchangeInfo.needs_passphrase; const hasSandbox = !!exchangeInfo.has_sandbox; @@ -182,9 +178,6 @@ export default function Settings() { useEffect(() => { fetchKeys(); // eslint-disable-line react-hooks/set-state-in-effect -- initial data fetch on mount - apiClient.get('/api/keys/exchanges') - .then(res => { if (Array.isArray(res.data) && res.data.length) setExchanges(res.data); }) - .catch(() => { /* keep fallback list */ }); }, [fetchKeys]); // Exchanges without a testnet can only be added as Live diff --git a/tests/test_exchange_registry.py b/tests/test_exchange_registry.py new file mode 100644 index 0000000..cc17b65 --- /dev/null +++ b/tests/test_exchange_registry.py @@ -0,0 +1,66 @@ +"""The exchange registry is the single source of truth for which exchanges +exist, how they authenticate and what the connection form shows. Nothing +here touches the network: `build_exchange` only instantiates ccxt classes.""" +import os + +import ccxt +import pytest +from fastapi.testclient import TestClient + +from backend.core import exchange_registry as reg +from backend.main import app + +HEADERS = {"X-API-Key": os.environ["MASTER_API_KEY"]} + +NEW_EXCHANGES = ["bybit", "gateio", "bitget", "mexc", "htx", "bingx"] + + +def test_every_spec_is_a_ccxt_exchange_with_consistent_views(): + for ex_id, spec in reg.EXCHANGES.items(): + assert hasattr(ccxt, ex_id), ex_id + assert spec.name and spec.keys_url, ex_id + # ccxt's own credential list must agree with the spec, otherwise the + # key form asks for the wrong fields + needs = bool(getattr(ccxt, ex_id)().requiredCredentials.get("password")) + assert needs == spec.needs_passphrase, ex_id + assert reg.SUPPORTED_EXCHANGES == {k: v.name for k, v in reg.EXCHANGES.items()} + assert reg._PASSPHRASE_EXCHANGES == {"okx", "kucoin", "bitget"} + assert set(NEW_EXCHANGES) <= set(reg.EXCHANGES) + + +def test_build_exchange_applies_spec_and_rejects_unknown(): + for ex_id in NEW_EXCHANGES + ["okx", "kraken"]: + ex = reg.build_exchange(ex_id, api_key="k", api_secret="s", passphrase="p") + assert ex.id == ex_id and ex.apiKey == "k" + assert bool(ex.password) == reg.EXCHANGES[ex_id].needs_passphrase, ex_id + assert reg.build_exchange("okx").hostname == "eea.okx.com" + assert reg.exchange_spec("BYBIT").name == "Bybit" + assert reg.exchange_spec("nope") is None + with pytest.raises(ValueError, match="not supported"): + reg.build_exchange("nope") + + +def test_sandbox_rejected_where_ccxt_has_no_testnet(): + with pytest.raises(ValueError, match="sandbox"): + reg.build_exchange("mexc", api_key="k", api_secret="s", sandbox=True) + assert reg.build_exchange("bybit", api_key="k", api_secret="s", sandbox=True) is not None + # Bitget's demo mode is a request header, not a test URL — must still count + assert reg.build_exchange("bitget", api_key="k", api_secret="s", passphrase="p", sandbox=True).options["sandboxMode"] is True + assert reg.exchange_has_sandbox("bitget") and reg.exchange_has_sandbox("bybit") + assert not reg.exchange_has_sandbox("mexc") and not reg.exchange_has_sandbox("kraken") + + +def test_exchanges_endpoint_mirrors_registry(): + r = TestClient(app).get("/api/keys/exchanges", headers=HEADERS) + assert r.status_code == 200 + rows = {row["id"]: row for row in r.json()} + assert set(rows) == set(reg.EXCHANGES) + for ex_id, spec in reg.EXCHANGES.items(): + row = rows[ex_id] + assert row["name"] == spec.name + assert row["needs_passphrase"] == spec.needs_passphrase + assert row["keys_url"] == spec.keys_url + assert row["sandbox_note"] == spec.sandbox_note + assert isinstance(row["has_sandbox"], bool) + assert rows["bybit"]["has_sandbox"] is True and rows["bitget"]["has_sandbox"] is True + assert rows["mexc"]["has_sandbox"] is False and rows["htx"]["has_sandbox"] is False From 71726dd3c8a367576ec11a32a45a086d51c60831 Mon Sep 17 00:00:00 2001 From: Stenvro Date: Thu, 24 Sep 2026 04:17:49 +0000 Subject: [PATCH 2/8] feat: data-driven exchange registry (+6 exchanges) and perpetual swaps with leverage (long-only) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1 — exchanges - ExchangeSpec registry is the single source of truth (name, passphrase, ccxt config, keys URL, sandbox/candle notes); SUPPORTED_EXCHANGES derived - Added Bybit, Gate, Bitget, MEXC, HTX, BingX (13 exchanges total) - Frontend reads GET /api/keys/exchanges via api/exchanges.js everywhere (Settings, DataManager, builder); no hardcoded lists Phase 2 — market types / perpetual swaps - ExchangeSpec.markets with MarketCaps (max leverage, ccxt id, leverage in order params); build_exchange pins options.defaultType, auth cache keyed on market type, get_exchange_symbols(exchange, market_type) - engine/symbols.py: BASE/QUOTE:SETTLE helpers, market_type_for, leverage_for - Additive migrations: exchange_keys.market_type, positions.market_type/ leverage/contracts, orders.market_type/reduce_only - Backtest/forward: entry locks notional/leverage + fees, PnL on full notional, liquidation at entry*(1-(1-0.005)/lev), summary adds market_type/leverage/liquidations/funding:"ignored" - Live: create_order with reduceOnly/contracts on derivatives (spot path unchanged), ensure_leverage at startup and before the first buy, fetch_positions reconciliation for swaps, force-close reduce-only - Validator: key/bot market-type match, :SETTLE form rules, leverage caps (warn above 3x), margin mode; keys router enforces market type per key - Frontend: Market/Leverage/Margin controls in Exchange Routing, whitelist form hints, go-live checklist line, Perps chips on card/analytics/home, DataManager market select - Tests: test_exchange_registry.py, test_swap.py (13); goldens unchanged Co-Authored-By: Claude Opus 4.8 --- README.md | 39 +- backend/core/database.py | 23 ++ backend/core/exchange_registry.py | 180 ++++++--- backend/engine/backtest.py | 106 ++++- backend/engine/bot_manager.py | 6 + backend/engine/broker.py | 154 +++++++- backend/engine/candle_poller.py | 25 +- backend/engine/live_cycle.py | 167 ++++++-- backend/engine/settings_validator.py | 89 ++++- backend/engine/sizing.py | 54 ++- backend/engine/startup.py | 34 +- backend/engine/symbols.py | 73 ++++ backend/main.py | 4 +- backend/models/exchange_keys.py | 3 + backend/models/orders.py | 2 + backend/models/positions.py | 5 + backend/routers/bots.py | 27 +- backend/routers/data.py | 24 +- backend/routers/keys.py | 38 +- backend/routers/trades.py | 22 +- frontend/src/api/exchanges.js | 44 ++- frontend/src/components/BotManagerUI.jsx | 5 + .../src/components/Builder/BotBuilder.jsx | 49 ++- .../src/components/Builder/CustomNodes.jsx | 91 ++++- frontend/src/components/DataManager.jsx | 22 +- frontend/src/components/Home.jsx | 20 +- frontend/src/components/Settings.jsx | 31 +- frontend/src/components/TradeManager.jsx | 5 +- tests/test_bots_router.py | 16 +- tests/test_exchange_registry.py | 38 ++ tests/test_reproducibility.py | 2 +- tests/test_swap.py | 364 ++++++++++++++++++ 32 files changed, 1522 insertions(+), 240 deletions(-) create mode 100644 backend/engine/symbols.py create mode 100644 tests/test_swap.py diff --git a/README.md b/README.md index 69bbc31..cf00019 100644 --- a/README.md +++ b/README.md @@ -74,9 +74,10 @@ https://github.com/user-attachments/assets/25d03926-1e88-4749-88d4-b97b14ff9c2b - **Async Event-Driven Architecture** — FastAPI backend with concurrent bot management via asyncio - **Multi-Exchange Market Data** — universal REST polling via CCXT; each `(exchange, symbol, timeframe)` gets its own independent polling stream - **CCXT Integration** — exchange-agnostic order execution with market precision handling +- **Perpetual swaps with leverage** — a bot trades spot or USDT/USDC-settled perpetuals (long-only, 1–10×, isolated or cross margin); backtest, forward test and live share the same margin/liquidation model, and spot bots are untouched ### Multi-Exchange Support -- **7 Exchanges out of the box** — OKX, Binance, Bitvavo, Coinbase, Crypto.com, Kraken, KuCoin +- **13 Exchanges out of the box** — OKX, Binance, Bitvavo, Coinbase, Crypto.com, Kraken, KuCoin, Bybit, Gate, Bitget, MEXC, HTX, BingX; perpetual swaps on nine of them - **Isolated data streams** — bots on different exchanges poll independently and store candles separately; no cross-exchange data mixing - **Exchange Registry** — centralized `exchange_registry.py` handles per-exchange config (OKX EU hostname, passphrase exchanges, sandbox modes) - **Automatic migration** — existing databases are upgraded non-destructively on startup; all historical data is preserved @@ -517,24 +518,30 @@ Click **Duplicate** on any stopped bot card to create a clone with `(copy)` appe ## Supported Exchanges -| Exchange | Passphrase | Sandbox | Notes | -| :--- | :--- | :--- | :--- | -| OKX | Yes | Yes | EU hostname (eea.okx.com) | -| Binance | No | Yes | | -| Bitvavo | No | No | EU exchange | -| Coinbase | No | No | | -| Crypto.com | No | Yes | UAT environment | -| Kraken | No | No | | -| KuCoin | Yes | No | | -| Bybit | No | Yes | testnet.bybit.com | -| Gate | No | Yes | testnet.gate.io | -| Bitget | Yes | Yes | Demo trading (request header) | -| MEXC | No | No | | -| HTX | No | No | | -| BingX | No | Yes | VST demo trading | +| Exchange | Passphrase | Spot sandbox | Perpetual swaps | Notes | +| :--- | :--- | :--- | :--- | :--- | +| OKX | Yes | Yes | up to 10× (sandbox) | EU hostname (eea.okx.com); net (one-way) position mode | +| Binance | No | Yes | up to 10× (sandbox) | USDⓈ-M futures, one-way position mode | +| Bitvavo | No | No | — | EU exchange | +| Coinbase | No | No | — | | +| Crypto.com | No | Yes | — | UAT environment | +| Kraken | No | No | up to 5× (demo) | Kraken Futures uses its own keys (futures.kraken.com / demo-futures.kraken.com) | +| KuCoin | Yes | No | up to 10× | KuCoin Futures uses its own keys (Futures → API management) | +| Bybit | No | Yes | up to 10× (testnet) | Unified trading account, one-way position mode | +| Gate | No | Yes | up to 10× (testnet) | | +| Bitget | Yes | Yes | up to 10× (demo) | USDT-M futures, one-way position mode | +| MEXC | No | No | — | Futures API is closed to the public | +| HTX | No | No | up to 10× | | +| BingX | No | Yes | up to 10× (demo) | Perpetual futures, one-way position mode | Adding another CCXT-compatible exchange is one `ExchangeSpec` entry in `backend/core/exchange_registry.py`; the key form, the data manager and the builder read the list from `GET /api/keys/exchanges`. +### Perpetual swaps + +An API key is bound to one market: save a second key with **Market: Perpetual swaps** for derivatives (the same exchange login, with futures/derivatives trade permission; Kraken and KuCoin issue separate futures keys). A bot on a swap key trades the `BASE/QUOTE:SETTLE` symbols (`BTC/USDT:USDT`) — only USDT/USDC-settled linear perpetuals, long-only in this release — with the leverage and margin mode set in the builder's Exchange Routing block (1–10×, isolated by default). The engine confirms leverage and margin mode on the exchange before the first order and refuses to start when that fails; on start-up, open swap positions are reconciled against `fetch_positions` instead of the wallet. + +Economics are identical across backtest, forward test and live: an entry locks `notional / leverage` plus fees, PnL is on the full notional, and a position whose candle low reaches `entry × (1 − (1 − 0.5%) / leverage)` is liquidated for its margin (`liquidation` in the exit list, `liquidations` in the backtest summary). Funding payments are **not** modelled (`funding: "ignored"` in the summary). `Max Order Value` caps the notional, i.e. margin × leverage. Exchanges without a swap testnet (KuCoin, HTX) can only forward test or trade real money. + ### Historical data per exchange Backfill pages through each exchange's OHLCV history, discovers the listing date of young pairs (OKX and Crypto.com return nothing for a `since` before listing instead of clamping), refetches any gaps it finds, and logs a warning when less history is available than the bot's lookback asks for. Limits measured in September 2026: diff --git a/backend/core/database.py b/backend/core/database.py index ed9e362..e8c241e 100644 --- a/backend/core/database.py +++ b/backend/core/database.py @@ -49,6 +49,16 @@ def run_migrations(): if 'exchange' not in pos_cols: conn.execute(text("ALTER TABLE positions ADD COLUMN exchange TEXT DEFAULT 'okx'")) logger.info("Migration: added 'exchange' to positions") + # Derivatives support: existing rows are spot at 1x + if 'market_type' not in pos_cols: + conn.execute(text("ALTER TABLE positions ADD COLUMN market_type TEXT DEFAULT 'spot'")) + logger.info("Migration: added 'market_type' to positions") + if 'leverage' not in pos_cols: + conn.execute(text("ALTER TABLE positions ADD COLUMN leverage REAL DEFAULT 1")) + logger.info("Migration: added 'leverage' to positions") + if 'contracts' not in pos_cols: + conn.execute(text("ALTER TABLE positions ADD COLUMN contracts REAL")) + logger.info("Migration: added 'contracts' to positions") # ── orders ─────────────────────────────────────────────────────────────── if "orders" in existing_tables: @@ -56,6 +66,19 @@ def run_migrations(): if 'exchange' not in ord_cols: conn.execute(text("ALTER TABLE orders ADD COLUMN exchange TEXT DEFAULT 'okx'")) logger.info("Migration: added 'exchange' to orders") + if 'market_type' not in ord_cols: + conn.execute(text("ALTER TABLE orders ADD COLUMN market_type TEXT DEFAULT 'spot'")) + logger.info("Migration: added 'market_type' to orders") + if 'reduce_only' not in ord_cols: + conn.execute(text("ALTER TABLE orders ADD COLUMN reduce_only INTEGER DEFAULT 0")) + logger.info("Migration: added 'reduce_only' to orders") + + # ── exchange_keys ──────────────────────────────────────────────────────── + if "exchange_keys" in existing_tables: + key_cols = {c['name'] for c in inspector.get_columns('exchange_keys')} + if 'market_type' not in key_cols: + conn.execute(text("ALTER TABLE exchange_keys ADD COLUMN market_type TEXT DEFAULT 'spot'")) + logger.info("Migration: added 'market_type' to exchange_keys") # ── candles ────────────────────────────────────────────────────────────── # The unique constraint must include 'exchange'. SQLite cannot alter diff --git a/backend/core/exchange_registry.py b/backend/core/exchange_registry.py index 88d43fc..368f7a2 100644 --- a/backend/core/exchange_registry.py +++ b/backend/core/exchange_registry.py @@ -7,6 +7,22 @@ logger = logging.getLogger("apexalgo.exchange_registry") +@dataclass(frozen=True) +class MarketCaps: + """What ApexAlgo may do on one market type of an exchange.""" + ccxt_id: str | None = None # separate ccxt class for this market type (kucoinfutures) + max_leverage: int = 1 # cap the validator/UI enforce (1 = no leverage) + leverage_in_order: bool = False # exchange takes leverage per order instead of set_leverage + note: str | None = None # shown in the key form / go-live checklist + + +SPOT = MarketCaps() + + +def _swap(**kw) -> MarketCaps: + return MarketCaps(max_leverage=kw.pop("max_leverage", 10), **kw) + + @dataclass(frozen=True) class ExchangeSpec: """Everything ApexAlgo knows about one exchange, in one place. @@ -14,6 +30,9 @@ class ExchangeSpec: Adding an exchange = one entry in `EXCHANGES`; the routers, the key form and the builder derive their lists from here (via `GET /api/keys/exchanges`). ccxt itself is id-agnostic, so nothing engine-side needs to change. + `markets` lists the market types a key/bot may use: every exchange has + "spot"; "swap" (USDT/USDC-margined linear perpetuals, long-only until + shorts land) only where ccxt can place, reduce and list positions. """ name: str # display name needs_passphrase: bool = False # api_key + secret + passphrase @@ -21,6 +40,7 @@ class ExchangeSpec: keys_url: str | None = None # where to create API keys sandbox_note: str | None = None # how to get testnet keys, if any candle_limit_note: str | None = None # shown when the exchange caps OHLCV history + markets: dict = field(default_factory=lambda: {"spot": SPOT}) # market type -> MarketCaps # All exchanges supported by ApexAlgo, keyed by ccxt exchange ID. @@ -28,10 +48,12 @@ class ExchangeSpec: "okx": ExchangeSpec( "OKX", needs_passphrase=True, ccxt_config={"hostname": "eea.okx.com"}, keys_url="https://www.okx.com/account/my-api", - sandbox_note="Demo trading keys are created under Trade → Demo trading → API."), + sandbox_note="Demo trading keys are created under Trade → Demo trading → API.", + markets={"spot": SPOT, "swap": _swap(note="Account must be in net (one-way) position mode.")}), "binance": ExchangeSpec( "Binance", keys_url="https://www.binance.com/en/my/settings/api-management", - sandbox_note="Spot testnet keys come from testnet.binance.vision (separate account)."), + sandbox_note="Spot testnet keys come from testnet.binance.vision, futures testnet keys from testnet.binancefuture.com (separate accounts).", + markets={"spot": SPOT, "swap": _swap(note="USDⓈ-M futures; account must be in one-way position mode.")}), "bitvavo": ExchangeSpec("Bitvavo", keys_url="https://account.bitvavo.com/user/api"), "coinbase": ExchangeSpec("Coinbase", keys_url="https://www.coinbase.com/settings/api"), "cryptocom": ExchangeSpec( @@ -39,26 +61,37 @@ class ExchangeSpec: sandbox_note="UAT sandbox keys are issued via the Crypto.com Exchange UAT environment."), "kraken": ExchangeSpec( "Kraken", keys_url="https://www.kraken.com/u/security/api", - candle_limit_note="Kraken only serves its most recent 720 candles per timeframe"), - "kucoin": ExchangeSpec("KuCoin", needs_passphrase=True, keys_url="https://www.kucoin.com/account/api"), + candle_limit_note="Kraken only serves its most recent 720 candles per timeframe", + markets={"spot": SPOT, "swap": _swap(ccxt_id="krakenfutures", max_leverage=5, + note="Kraken Futures uses its own API keys (futures.kraken.com); demo keys from demo-futures.kraken.com.")}), + "kucoin": ExchangeSpec( + "KuCoin", needs_passphrase=True, keys_url="https://www.kucoin.com/account/api", + markets={"spot": SPOT, "swap": _swap(ccxt_id="kucoinfutures", leverage_in_order=True, + note="KuCoin Futures uses its own API keys (Futures → API management).")}), "bybit": ExchangeSpec( "Bybit", keys_url="https://www.bybit.com/app/user/api-management", - sandbox_note="Testnet keys are created on testnet.bybit.com (separate account)."), + sandbox_note="Testnet keys are created on testnet.bybit.com (separate account).", + markets={"spot": SPOT, "swap": _swap(note="Unified trading account, one-way position mode.")}), "gateio": ExchangeSpec( "Gate", keys_url="https://www.gate.io/myaccount/api_key_manage", - sandbox_note="Testnet keys are created on testnet.gate.io (separate account)."), + sandbox_note="Testnet keys are created on testnet.gate.io (separate account).", + markets={"spot": SPOT, "swap": _swap()}), "bitget": ExchangeSpec( "Bitget", needs_passphrase=True, keys_url="https://www.bitget.com/account/newapi", - sandbox_note="Demo-trading keys are created under Demo Trading → API on bitget.com."), + sandbox_note="Demo-trading keys are created under Demo Trading → API on bitget.com.", + markets={"spot": SPOT, "swap": _swap(note="USDT-M futures, one-way position mode.")}), "mexc": ExchangeSpec( "MEXC", keys_url="https://www.mexc.com/user/openapi", sandbox_note="MEXC has no public testnet — keys can only be used for real trading."), + # spot only: MEXC's futures order API is closed to the public ("under maintenance") "htx": ExchangeSpec( "HTX", keys_url="https://www.htx.com/en-us/apikey/", - sandbox_note="HTX has no public testnet — keys can only be used for real trading."), + sandbox_note="HTX has no public testnet — keys can only be used for real trading.", + markets={"spot": SPOT, "swap": _swap()}), "bingx": ExchangeSpec( "BingX", keys_url="https://bingx.com/en/account/api/", - sandbox_note="VST demo-trading keys are created under Demo Trading → API on bingx.com."), + sandbox_note="VST demo-trading keys are created under Demo Trading → API on bingx.com.", + markets={"spot": SPOT, "swap": _swap(note="Perpetual futures, one-way position mode.")}), } # Backwards-compatible views used by the routers: {id: display name} and the @@ -72,23 +105,42 @@ def exchange_spec(exchange_id: str) -> ExchangeSpec | None: return EXCHANGES.get(str(exchange_id or "").lower()) -_sandbox_cache: dict[str, bool] = {} +def market_caps(exchange_id: str, market_type: str = "spot") -> MarketCaps | None: + """Capabilities of one market type on an exchange; None when the exchange + is unknown or does not offer that market type.""" + spec = exchange_spec(exchange_id) + if spec is None: + return None + return spec.markets.get(str(market_type or "spot").lower()) + + +def ccxt_id_for(exchange_id: str, market_type: str = "spot") -> str: + """The ccxt class that serves this market type (kucoin swap = kucoinfutures).""" + caps = market_caps(exchange_id, market_type) + return (caps.ccxt_id if caps and caps.ccxt_id else str(exchange_id).lower()) + + +_sandbox_cache: dict[tuple[str, str], bool] = {} -def exchange_has_sandbox(exchange_id: str) -> bool: - """Whether ccxt can put this exchange in sandbox/testnet mode. Probed on - a throwaway instance (no network): some exchanges use a test URL, others - a demo-trading header (Bitget), and ccxt raises for the rest.""" +def exchange_has_sandbox(exchange_id: str, market_type: str = "spot") -> bool: + """Whether ccxt can put this exchange (market type) in sandbox/testnet + mode. Probed on a throwaway instance (no network): some exchanges use a + test URL, others a demo-trading header (Bitget), and ccxt raises for the + rest.""" exchange_id = str(exchange_id or "").lower() - if exchange_id not in _sandbox_cache: + market_type = str(market_type or "spot").lower() + key = (exchange_id, market_type) + if key not in _sandbox_cache: ok = False try: - getattr(ccxt, exchange_id)().set_sandbox_mode(True) - ok = True + if market_caps(exchange_id, market_type) is not None: + getattr(ccxt, ccxt_id_for(exchange_id, market_type))().set_sandbox_mode(True) + ok = True except Exception: pass - _sandbox_cache[exchange_id] = ok - return _sandbox_cache[exchange_id] + _sandbox_cache[key] = ok + return _sandbox_cache[key] def build_exchange( @@ -97,6 +149,7 @@ def build_exchange( api_secret: str | None = None, passphrase: str | None = None, sandbox: bool = False, + market_type: str = "spot", ) -> ccxt.Exchange: """ Instantiate and return a configured CCXT exchange. @@ -111,25 +164,39 @@ def build_exchange( Only applied for exchanges whose spec has `needs_passphrase`. sandbox : bool Enable exchange sandbox / testnet mode when supported. + market_type : str + "spot" (default, unchanged behaviour) or "swap": sets ccxt's + `defaultType` and picks the derivatives class where the exchange has + one. Raises ValueError when the spec has no such market type. """ exchange_id = exchange_id.lower() + market_type = str(market_type or "spot").lower() if exchange_id not in SUPPORTED_EXCHANGES: raise ValueError( f"Exchange '{exchange_id}' is not supported. " f"Supported: {', '.join(SUPPORTED_EXCHANGES)}" ) + spec = EXCHANGES[exchange_id] + caps = spec.markets.get(market_type) + if caps is None: + raise ValueError( + f"Exchange '{exchange_id}' has no '{market_type}' market in ApexAlgo. " + f"Available: {', '.join(spec.markets)}" + ) - cls = getattr(ccxt, exchange_id, None) + cls = getattr(ccxt, ccxt_id_for(exchange_id, market_type), None) if cls is None: raise RuntimeError( f"Exchange '{exchange_id}' was not found in the installed CCXT version. " "Run: pip install --upgrade ccxt" ) - spec = EXCHANGES[exchange_id] config: dict = {"enableRateLimit": True} config.update(spec.ccxt_config) + # Pin ccxt's defaultType so balance/position calls address the right + # account: ccxt's own default is not "spot" everywhere (bybit → swap) + config["options"] = {**config.get("options", {}), "defaultType": market_type} if api_key: config["apiKey"] = api_key @@ -152,6 +219,19 @@ def build_exchange( return exchange +def key_market_type(key_record) -> str: + """Market type an ExchangeKey record is bound to (pre-phase-2 rows: spot).""" + return str(getattr(key_record, "market_type", None) or "spot").lower() + + +def build_exchange_for_symbol(exchange_id: str, symbol: str) -> ccxt.Exchange: + """Public (unauthenticated) instance for market data on `symbol`: the + unified symbol says which market it lives on (``BTC/USDT:USDT`` → swap), + so candle/ticker callers need no market-type plumbing of their own.""" + from backend.engine.symbols import market_type_of + return build_exchange(exchange_id, market_type=market_type_of(symbol)) + + # Cache for exchange timeframes (loaded once per exchange, reused) _timeframe_cache: dict[str, dict[str, str]] = {} @@ -174,32 +254,44 @@ def get_exchange_timeframes(exchange_id: str) -> dict[str, str]: return tf_map -# Active spot symbols per exchange, refreshed after _MARKETS_TTL seconds so a -# newly listed pair shows up without a restart -_markets_cache: dict[str, tuple[float, list[str]]] = {} +# Active symbols per (exchange, market type), refreshed after _MARKETS_TTL +# seconds so a newly listed pair shows up without a restart +_markets_cache: dict[tuple[str, str], tuple[float, list[str]]] = {} _MARKETS_TTL = 3600 -def get_exchange_symbols(exchange_id: str) -> list[str]: - """Sorted list of tradeable spot symbols ("BTC/USDT") on the exchange. +def get_exchange_symbols(exchange_id: str, market_type: str = "spot") -> list[str]: + """Sorted list of tradeable symbols on the exchange: spot pairs + ("BTC/USDT") or, for `market_type="swap"`, active linear (stablecoin- + settled) perpetuals ("BTC/USDT:USDT"). Inverse/coin-margined contracts + and dated futures are left out. Empty when the exchange cannot be reached — callers must treat that as "unknown", not "nothing is tradeable".""" exchange_id = exchange_id.lower() - hit = _markets_cache.get(exchange_id) + market_type = str(market_type or "spot").lower() + cache_key = (exchange_id, market_type) + hit = _markets_cache.get(cache_key) if hit and time.monotonic() - hit[0] < _MARKETS_TTL: return hit[1] try: - exchange = build_exchange(exchange_id) + exchange = build_exchange(exchange_id, market_type=market_type) markets = exchange.load_markets() - symbols = sorted( - sym for sym, m in markets.items() - if m.get("spot", True) and m.get("active", True) is not False - ) + if market_type == "spot": + symbols = sorted( + sym for sym, m in markets.items() + if m.get("spot", True) and m.get("active", True) is not False + ) + else: + symbols = sorted( + sym for sym, m in markets.items() + if m.get("type") == market_type and m.get("linear") + and m.get("active", True) is not False + ) except Exception as exc: - logger.warning("Failed to load markets for '%s': %s", exchange_id, exc) + logger.warning("Failed to load %s markets for '%s': %s", market_type, exchange_id, exc) return hit[1] if hit else [] - _markets_cache[exchange_id] = (time.monotonic(), symbols) + _markets_cache[cache_key] = (time.monotonic(), symbols) return symbols @@ -216,23 +308,25 @@ def build_exchange_from_key(key_record) -> ccxt.Exchange: api_secret=decrypt_data(key_record.api_secret), passphrase=decrypt_data(key_record.passphrase) if key_record.passphrase else None, sandbox=key_record.is_sandbox, + market_type=key_market_type(key_record), ) # Authenticated instances shared by the engine and the routers, keyed by -# (exchange, key name, sandbox). Building one per bot per candle re-decrypted -# the credentials and re-downloaded the market list every tick; here markets -# are loaded once and the instance is rebuilt after _AUTH_TTL seconds so a -# stale session or rate-limit bookkeeping never sticks around for good. -# Sync ccxt instances serialize nothing themselves; order paths are already -# serialized per bot, and concurrent REST reads on one instance are safe. -_auth_cache: dict[tuple[str, str, bool], tuple[float, ccxt.Exchange]] = {} +# (exchange, key name, sandbox, market type). Building one per bot per candle +# re-decrypted the credentials and re-downloaded the market list every tick; +# here markets are loaded once and the instance is rebuilt after _AUTH_TTL +# seconds so a stale session or rate-limit bookkeeping never sticks around +# for good. Sync ccxt instances serialize nothing themselves; order paths are +# already serialized per bot, and concurrent REST reads on one instance are safe. +_auth_cache: dict[tuple[str, str, bool, str], tuple[float, ccxt.Exchange]] = {} _auth_lock = threading.Lock() _AUTH_TTL = 3600 -def _auth_key(key_record) -> tuple[str, str, bool]: - return (str(key_record.exchange).lower(), str(key_record.name), bool(key_record.is_sandbox)) +def _auth_key(key_record) -> tuple[str, str, bool, str]: + return (str(key_record.exchange).lower(), str(key_record.name), + bool(key_record.is_sandbox), key_market_type(key_record)) def get_authenticated_exchange(key_record, *, load_markets: bool = True) -> ccxt.Exchange: diff --git a/backend/engine/backtest.py b/backend/engine/backtest.py index c6fb962..c1b315a 100644 --- a/backend/engine/backtest.py +++ b/backend/engine/backtest.py @@ -16,11 +16,22 @@ _num, _int, _naive_utc, sim_frictions, calculate_trade_amount, _record_config_run, backtest_pin, combined_fingerprint, slice_key, ) +from backend.engine.symbols import leverage_for, market_type_for from backend.core import bot_log_buffer as blb logger = logging.getLogger("apexalgo.bot_manager") _BT_COMMIT_EVERY = 500 # timeline steps between backtest commits +# Maintenance-margin approximation for the simulated liquidation of a +# leveraged long: the position is force-closed once the candle low reaches +# entry x (1 - (1 - MMR) / leverage), i.e. when the loss eats the whole +# margin minus the maintenance buffer. Funding payments are ignored. +_MAINTENANCE_MARGIN = 0.005 + + +def liquidation_price(entry_price: float, leverage: float) -> float: + """Approximate long liquidation price at `leverage` (see _MAINTENANCE_MARGIN).""" + return float(entry_price) * (1.0 - (1.0 - _MAINTENANCE_MARGIN) / float(leverage)) @dataclass @@ -35,6 +46,7 @@ class SimResult: block_count: int dd_detail: dict timeline: list = field(default_factory=list) + liquidations: int = 0 def simulate(db, bot, sym_contexts, exchange_name, run_backtest, *, check_exits, position_states, states_lock, on_progress, check_abort): @@ -73,6 +85,21 @@ def simulate(db, bot, sym_contexts, exchange_name, run_backtest, *, check_exits, cooldown_trades = _int(bot.settings.get("cooldown_trades"), 0) cooldown_candles = _int(bot.settings.get("cooldown_candles"), 0) + # Derivatives (phase 2): only the margin (notional / leverage) plus the + # entry fee leaves the pool on open; on close the margin comes back with + # the PnL. Every formula below keeps a literal spot branch so a spot bot + # simulates byte-for-byte as before. + bt_market_type = market_type_for(bot.settings) + bt_derivative = bt_market_type != "spot" + bt_leverage = leverage_for(bot.settings, bt_market_type) if bt_derivative else 1.0 + bt_liquidations = 0 + # Extra Order columns for derivative fills (empty on spot → same rows as before) + _bt_open_extra = {"market_type": bt_market_type} if bt_derivative else {} + _bt_close_extra = {"market_type": bt_market_type, "reduce_only": 1} if bt_derivative else {} + if bt_derivative: + blb.push(bot.name, "INFO", f"Backtest on {bt_market_type} at {bt_leverage:g}x: liquidation approximated at " + f"{100 * (1 - _MAINTENANCE_MARGIN) / bt_leverage:.1f}% adverse move, funding ignored") + # ── Merged chronological execution across all symbols ── timeline = [] for ci, ctx in enumerate(sym_contexts): @@ -136,7 +163,10 @@ def simulate(db, bot, sym_contexts, exchange_name, run_backtest, *, check_exits, just_opened = None # Capital depletion halt / drawdown block: no new entries, exits keep running if is_buy and slot_free and can_buy_cooldown and bt_equity > 0 and not bt_entries_blocked: - trade_amount = calculate_trade_amount(current_price, bot.settings, current_equity=bt_equity) + if bt_derivative: + trade_amount = calculate_trade_amount(current_price, bot.settings, current_equity=bt_equity, leverage=bt_leverage) + else: + trade_amount = calculate_trade_amount(current_price, bot.settings, current_equity=bt_equity) if trade_amount is not None: bt_entry_price = current_price * (1 + bt_entry_slippage) # Percentage sizing spends a share of equity; cap the @@ -144,17 +174,27 @@ def simulate(db, bot, sym_contexts, exchange_name, run_backtest, *, check_exits, # (100% sizing would otherwise always exceed it) _entry_cfg = bot.settings.get("trade_settings", {}).get("entry", {}) if _entry_cfg.get("amount_type", "percentage") != "fixed": - max_affordable = bt_equity / (bt_entry_price * (1 + bt_entry_fee)) + if bt_derivative: + # Margin + fee on the notional must fit the pool + max_affordable = bt_equity / (bt_entry_price * (1 / bt_leverage + bt_entry_fee)) + else: + max_affordable = bt_equity / (bt_entry_price * (1 + bt_entry_fee)) trade_amount = min(trade_amount, max_affordable) investment_cost = bt_entry_price * trade_amount - total_cost = investment_cost * (1 + bt_entry_fee) + if bt_derivative: + total_cost = investment_cost / bt_leverage + investment_cost * bt_entry_fee + else: + total_cost = investment_cost * (1 + bt_entry_fee) if trade_amount > 0 and total_cost <= bt_equity + 1e-9: ctx["trade_entry_indices"].append(index) bt_equity = max(bt_equity - total_cost, 0.0) # Lock capital + entry fee just_opened = Position(exchange=exchange_name, bot_name=bot.name, symbol=symbol, mode="backtest", status="open", side="long", entry_price=bt_entry_price, amount=trade_amount, created_at=_naive_utc(ts)) + if bt_derivative: + just_opened.market_type = bt_market_type + just_opened.leverage = bt_leverage db.add(just_opened) db.flush() - db.add(Order(position_id=just_opened.id, exchange=exchange_name, bot_name=bot.name, mode="backtest", symbol=symbol, side="buy", order_type="market", price=bt_entry_price, amount=trade_amount, timestamp=_naive_utc(ts), status="filled", fee=investment_cost * bt_entry_fee)) + db.add(Order(position_id=just_opened.id, exchange=exchange_name, bot_name=bot.name, mode="backtest", symbol=symbol, side="buy", order_type="market", price=bt_entry_price, amount=trade_amount, timestamp=_naive_utc(ts), status="filled", fee=investment_cost * bt_entry_fee, **_bt_open_extra)) open_list.append(just_opened) ctx["original_amount"][just_opened.id] = trade_amount @@ -165,6 +205,27 @@ def simulate(db, bot, sym_contexts, exchange_name, run_backtest, *, check_exits, for open_bt_pos in list(open_list): if open_bt_pos is just_opened: continue + if bt_derivative and current_low <= liquidation_price(open_bt_pos.entry_price, bt_leverage): + # Liquidated: the whole margin is gone, nothing returns + # to the pool (entry fee was paid on open, no exit fee) + liq_price = liquidation_price(open_bt_pos.entry_price, bt_leverage) + remaining_qty = open_bt_pos.amount + margin_qty = open_bt_pos.entry_price * remaining_qty / bt_leverage + db.add(Order(position_id=open_bt_pos.id, exchange=exchange_name, bot_name=bot.name, mode="backtest", symbol=symbol, side="sell", order_type="market", price=liq_price, amount=remaining_qty, timestamp=_naive_utc(ts), status="filled", fee=0.0, **_bt_close_extra)) + open_bt_pos.profit_abs = (open_bt_pos.profit_abs or 0.0) - margin_qty + original_amount = ctx["original_amount"].get(open_bt_pos.id) + if original_amount and original_amount > 0: + open_bt_pos.profit_pct = (open_bt_pos.profit_pct or 0.0) - 100.0 * (remaining_qty / original_amount) + with states_lock: + st = position_states.pop(open_bt_pos.id, None) + triggered = set((st or {}).get('triggered_exits') or ()) | set(open_bt_pos.triggered_exits or ()) + open_bt_pos.triggered_exits = sorted(str(t) for t in triggered) + ["liquidation"] + open_bt_pos.status = "closed" + open_bt_pos.closed_at = _naive_utc(ts) + open_list.remove(open_bt_pos) + ctx["original_amount"].pop(open_bt_pos.id, None) + bt_liquidations += 1 + continue exit_events = check_exits(open_bt_pos, current_price, current_high, current_low, is_sell, bot.settings, current_atr, row_open=current_open) for ev in exit_events: @@ -181,15 +242,21 @@ def simulate(db, bot, sym_contexts, exchange_name, run_backtest, *, check_exits, if close_qty <= 0: continue actual_price = ev['price'] * (1 - bt_exit_slippage) - db.add(Order(position_id=open_bt_pos.id, exchange=exchange_name, bot_name=bot.name, mode="backtest", symbol=symbol, side="sell", order_type="market", price=actual_price, amount=close_qty, timestamp=_naive_utc(ts), status="filled", fee=actual_price * close_qty * bt_exit_fee)) + db.add(Order(position_id=open_bt_pos.id, exchange=exchange_name, bot_name=bot.name, mode="backtest", symbol=symbol, side="sell", order_type="market", price=actual_price, amount=close_qty, timestamp=_naive_utc(ts), status="filled", fee=actual_price * close_qty * bt_exit_fee, **_bt_close_extra)) entry_cost = open_bt_pos.entry_price * close_qty * (1 + bt_entry_fee) exit_proceeds = actual_price * close_qty * (1 - bt_exit_fee) realized_pnl = exit_proceeds - entry_cost open_bt_pos.profit_abs = (open_bt_pos.profit_abs or 0.0) + realized_pnl - # Return sale proceeds to capital pool - bt_equity += exit_proceeds + if bt_derivative: + # Margin comes back with the price PnL, minus the exit fee + bt_equity += open_bt_pos.entry_price * close_qty / bt_leverage + (actual_price - open_bt_pos.entry_price) * close_qty - actual_price * close_qty * bt_exit_fee + # Return on what was locked (margin + entry fee) + entry_cost = open_bt_pos.entry_price * close_qty * (1 / bt_leverage + bt_entry_fee) + else: + # Return sale proceeds to capital pool + bt_equity += exit_proceeds # Weighted profit_pct: accumulate based on portion of original position closed (fee-adjusted) original_amount = ctx["original_amount"].get(open_bt_pos.id) @@ -224,7 +291,11 @@ def simulate(db, bot, sym_contexts, exchange_name, run_backtest, *, check_exits, open_value = 0.0 for c2 in sym_contexts: if c2["last_close"]: - open_value += sum(p2.amount for p2 in c2["open_positions"]) * c2["last_close"] + if bt_derivative: + open_value += sum(p2.entry_price * p2.amount / bt_leverage + (c2["last_close"] - p2.entry_price) * p2.amount + for p2 in c2["open_positions"]) + else: + open_value += sum(p2.amount for p2 in c2["open_positions"]) * c2["last_close"] equity_now = bt_equity + open_value if equity_now > bt_peak_equity: bt_peak_equity = equity_now @@ -287,6 +358,8 @@ def simulate(db, bot, sym_contexts, exchange_name, run_backtest, *, check_exits, final_pnl = exit_proceeds - entry_cost open_bt_pos.profit_abs = (open_bt_pos.profit_abs or 0.0) + final_pnl + if bt_derivative: + entry_cost = open_bt_pos.entry_price * remaining_qty * (1 / bt_leverage + bt_entry_fee) original_amount = ctx["original_amount"].get(open_bt_pos.id) if original_amount and original_amount > 0: portion_pct = (final_pnl / entry_cost) * 100 if entry_cost > 0 else 0.0 @@ -295,8 +368,11 @@ def simulate(db, bot, sym_contexts, exchange_name, run_backtest, *, check_exits, open_bt_pos.status = "closed" open_bt_pos.closed_at = _naive_utc(last_ts) - bt_equity += exit_proceeds # Return proceeds to capital pool - db.add(Order(position_id=open_bt_pos.id, exchange=exchange_name, bot_name=bot.name, mode="backtest", symbol=ctx["symbol"], side="sell", order_type="market", price=last_price, amount=remaining_qty, timestamp=_naive_utc(last_ts), status="filled", fee=last_price * remaining_qty * bt_exit_fee)) + if bt_derivative: + bt_equity += open_bt_pos.entry_price * remaining_qty / bt_leverage + (last_price - open_bt_pos.entry_price) * remaining_qty - last_price * remaining_qty * bt_exit_fee + else: + bt_equity += exit_proceeds # Return proceeds to capital pool + db.add(Order(position_id=open_bt_pos.id, exchange=exchange_name, bot_name=bot.name, mode="backtest", symbol=ctx["symbol"], side="sell", order_type="market", price=last_price, amount=remaining_qty, timestamp=_naive_utc(last_ts), status="filled", fee=last_price * remaining_qty * bt_exit_fee, **_bt_close_extra)) with states_lock: position_states.pop(open_bt_pos.id, None) @@ -321,7 +397,7 @@ def simulate(db, bot, sym_contexts, exchange_name, run_backtest, *, check_exits, return SimResult( starting_capital=bt_starting_capital, equity=bt_equity, max_dd=bt_max_dd, max_loss=bt_max_loss, dd_action=bt_dd_action, dd_cooldown_secs=bt_dd_cooldown_secs, blocked_secs=bt_blocked_secs, - block_count=bt_block_count, dd_detail=bt_dd_detail, timeline=timeline, + block_count=bt_block_count, dd_detail=bt_dd_detail, timeline=timeline, liquidations=bt_liquidations, ) @@ -394,6 +470,14 @@ def build_summary(db, bot, res: SimResult, sym_contexts, exchange_name=None) -> "data_changed": data_changed, "finished_at": datetime.now(timezone.utc).isoformat(), } + market_type = market_type_for(bot.settings) + if market_type != "spot": + summary.update({ + "market_type": market_type, + "leverage": leverage_for(bot.settings, market_type), + "liquidations": res.liquidations, + "funding": "ignored", + }) return summary diff --git a/backend/engine/bot_manager.py b/backend/engine/bot_manager.py index 4464cf3..65350d6 100644 --- a/backend/engine/bot_manager.py +++ b/backend/engine/bot_manager.py @@ -167,6 +167,12 @@ def _cancel_unfilled_order(self, ccxt_inst, order_id, ccxt_symbol): _wallet_held = staticmethod(broker.wallet_held) _reconcile_positions_with_wallet = staticmethod(broker.reconcile_positions_with_wallet) + # Derivatives (phase 2): position reconciliation via fetch_positions, + # leverage/margin-mode confirmation and contract conversion + _reconcile_positions_with_exchange = staticmethod(broker.reconcile_positions_with_exchange) + _ensure_leverage = staticmethod(broker.ensure_leverage) + _to_contracts = staticmethod(broker.to_contracts) + _from_contracts = staticmethod(broker.from_contracts) def _get_live_capital(self, ccxt_inst, api_key_record, ccxt_symbol, bot_name, ttl=30): return broker.get_live_capital(self._balance_cache, ccxt_inst, api_key_record, ccxt_symbol, bot_name, ttl) diff --git a/backend/engine/broker.py b/backend/engine/broker.py index cec80db..0e585f5 100644 --- a/backend/engine/broker.py +++ b/backend/engine/broker.py @@ -3,11 +3,13 @@ Everything here talks to a ccxt instance and nothing here touches engine state, so tests swap the instance for a fake at `BotManager._get_ccxt_instance`.""" import logging +import threading import time from backend.models.exchange_keys import ExchangeKey from backend.models.positions import Position -from backend.core.exchange_registry import get_authenticated_exchange +from backend.core.exchange_registry import get_authenticated_exchange, market_caps from backend.core import bot_log_buffer as blb +from backend.engine.symbols import base_of, cash_currency, is_derivative, normalize logger = logging.getLogger("apexalgo.bot_manager") @@ -56,16 +58,42 @@ def cancel_unfilled_order(ccxt_inst, order_id, ccxt_symbol): logger.warning("cancel_order %s failed: %s", order_id, exc) return False +def contract_size(ccxt_inst, ccxt_symbol) -> float: + """Base units per contract (1 for spot and for most linear perpetuals; + e.g. 0.001 BTC on kucoinfutures). Unknown metadata counts as 1.""" + if not is_derivative(ccxt_symbol): + return 1.0 + try: + cs = (ccxt_inst.market(ccxt_symbol) or {}).get("contractSize") + return float(cs) if cs else 1.0 + except Exception: + return 1.0 + + +def to_contracts(ccxt_inst, ccxt_symbol, base_amount) -> float: + """Base amount → the `amount` ccxt expects in create_order for a + derivative (contracts). Identity on spot.""" + return float(base_amount) / contract_size(ccxt_inst, ccxt_symbol) + + +def from_contracts(ccxt_inst, ccxt_symbol, contracts) -> float: + return float(contracts) * contract_size(ccxt_inst, ccxt_symbol) + + def below_market_minimum(ccxt_inst, ccxt_symbol, amount, price): """Return a human-readable violation string when an order would fall below the exchange's minimum amount/cost limits, else None. Missing - limit metadata is treated as no restriction.""" + limit metadata is treated as no restriction. `amount` is in base units; + on derivatives the amount limit is in contracts.""" try: limits = (ccxt_inst.market(ccxt_symbol) or {}).get("limits") or {} min_amount = (limits.get("amount") or {}).get("min") min_cost = (limits.get("cost") or {}).get("min") - if min_amount is not None and amount < float(min_amount): - return f"amount {amount} below exchange minimum {float(min_amount)}" + if min_amount is not None: + cs = contract_size(ccxt_inst, ccxt_symbol) + if amount / cs < float(min_amount): + unit = " contracts" if cs != 1.0 else "" + return f"amount {amount / cs:g}{unit} below exchange minimum {float(min_amount)}{unit}" if min_cost is not None and price: order_value = amount * float(price) if order_value < float(min_cost): @@ -84,7 +112,7 @@ def fee_in_quote(fee_info, ccxt_symbol, price): except (TypeError, ValueError): return 0.0 currency = fee_info.get("currency") - base = ccxt_symbol.split('/')[0] if '/' in ccxt_symbol else None + base = base_of(ccxt_symbol) if '/' in ccxt_symbol else None if currency and base and currency.upper() == base.upper() and price: return cost * float(price) return cost @@ -112,8 +140,8 @@ def reconcile_positions_with_wallet(db, bot, ccxt_inst, balance: dict, mode: str ids: dict = {} tol: dict = {} for pos in open_pos: - sym = str(pos.symbol).replace('-', '/').upper() - base = sym.split('/')[0] + sym = normalize(pos.symbol) + base = base_of(sym) expected[base] = expected.get(base, 0.0) + float(pos.amount or 0) ids.setdefault(base, []).append(pos.id) step = 0.0 @@ -132,12 +160,116 @@ def reconcile_positions_with_wallet(db, bot, ccxt_inst, balance: dict, mode: str problems.append(f"Position #{','.join(map(str, ids[base]))}: DB holds {want:g} {base} but exchange holds {held:g} {base}") return problems +def reconcile_positions_with_exchange(db, bot, ccxt_inst, mode: str): + """Derivatives counterpart of `reconcile_positions_with_wallet`: a + perpetual position holds no base coin, so compare the open `mode` + positions in the DB against `fetch_positions()` on the exchange. Returns + mismatch descriptions (empty when consistent): the exchange must hold at + least the DB amount (within two precision steps) on the same side.""" + open_pos = db.query(Position).filter( + Position.bot_name == bot.name, Position.status == "open", Position.mode == mode, + ).order_by(Position.id).all() + if not open_pos: + return [] + expected: dict = {} + ids: dict = {} + tol: dict = {} + for pos in open_pos: + sym = normalize(pos.symbol) + expected[sym] = expected.get(sym, 0.0) + float(pos.amount or 0) + ids.setdefault(sym, []).append(pos.id) + step = 0.0 + try: + prec = (ccxt_inst.market(sym).get("precision") or {}).get("amount") + if prec is not None: + prec = float(prec) + step = prec if prec < 1 else 10.0 ** (-prec) + except Exception: + pass + tol[sym] = max(tol.get(sym, 0.0), 2 * step * contract_size(ccxt_inst, sym)) + held: dict = {} + for p in ccxt_inst.fetch_positions(list(expected)) or []: + sym = normalize(p.get("symbol")) + if sym not in expected: + continue + contracts = float(p.get("contracts") or 0) + if contracts <= 0: + continue + side = str(p.get("side") or "long").lower() + cs = float(p.get("contractSize") or 0) or contract_size(ccxt_inst, sym) + held[sym] = held.get(sym, 0.0) + (contracts * cs if side == "long" else -contracts * cs) + problems = [] + for sym, want in expected.items(): + have = held.get(sym, 0.0) + if have < 0: + problems.append(f"Position #{','.join(map(str, ids[sym]))}: DB holds a long of {want:g} {sym} but the exchange holds a short") + elif have + tol[sym] + 1e-12 < want: + problems.append(f"Position #{','.join(map(str, ids[sym]))}: DB holds {want:g} {sym} but exchange position is {have:g}") + return problems + + +# (key name, symbol, leverage, margin mode) already applied on the exchange +# in this process — set_leverage is a real API call, do it once, not per tick +_leverage_applied: set = set() +_leverage_lock = threading.Lock() + + +def ensure_leverage(ccxt_inst, api_key_record, ccxt_symbol, leverage, margin_mode, bot_name): + """Put the exchange's per-symbol leverage and margin mode in line with the + bot before it trades a derivative. Exchanges without one of the calls + (ccxt NotSupported) are skipped with an INFO line; 'leverage not + modified' style answers are fine. Any other failure raises — a bot must + not trade at an unknown leverage. No-op on spot.""" + if not is_derivative(ccxt_symbol): + return + lev = int(float(leverage or 1)) + mode = str(margin_mode or "isolated").lower() + tag = (str(api_key_record.name), normalize(ccxt_symbol), lev, mode) + with _leverage_lock: + if tag in _leverage_applied: + return + caps = market_caps(api_key_record.exchange, "swap") + import ccxt as _ccxt + if getattr(ccxt_inst, "has", {}).get("setMarginMode"): + try: + ccxt_inst.set_margin_mode(mode, ccxt_symbol, params={"leverage": lev}) + except _ccxt.NotSupported: + blb.push(bot_name, "INFO", f"{ccxt_symbol}: exchange does not support setting margin mode via API — set it to {mode} on the exchange") + except Exception as exc: + if not _benign_leverage_error(exc): + raise RuntimeError(f"set_margin_mode({mode}) failed for {ccxt_symbol}: {exc}") from exc + else: + blb.push(bot_name, "INFO", f"{ccxt_symbol}: exchange does not support setting margin mode via API — set it to {mode} on the exchange") + if caps is not None and caps.leverage_in_order: + blb.push(bot_name, "INFO", f"{ccxt_symbol}: leverage {lev}x is sent with each order") + elif getattr(ccxt_inst, "has", {}).get("setLeverage"): + try: + ccxt_inst.set_leverage(lev, ccxt_symbol, params={"marginMode": mode}) + except _ccxt.NotSupported: + raise RuntimeError(f"Exchange cannot set leverage for {ccxt_symbol} via API") + except Exception as exc: + if not _benign_leverage_error(exc): + raise RuntimeError(f"set_leverage({lev}) failed for {ccxt_symbol}: {exc}") from exc + else: + raise RuntimeError(f"Exchange cannot set leverage for {ccxt_symbol} via API") + blb.push(bot_name, "INFO", f"{ccxt_symbol}: leverage {lev}x ({mode}) confirmed on the exchange") + with _leverage_lock: + _leverage_applied.add(tag) + + +def _benign_leverage_error(exc) -> bool: + """'Already set' answers exchanges return as errors.""" + msg = str(exc).lower() + return any(t in msg for t in ("not modified", "no need to change", "already", "same as", "110043", "59000")) + + def get_live_capital(balance_cache, ccxt_inst, api_key_record, ccxt_symbol, bot_name, ttl=30): - """Free quote-currency balance on the exchange, cached briefly to spare - rate limits. Returns None when the balance cannot be determined so the - caller can fall back to the configured capital. `balance_cache` maps + """Free cash balance on the exchange (quote currency on spot, settle + currency on derivatives), cached briefly to spare rate limits. Returns + None when the balance cannot be determined so the caller can fall back + to the configured capital. `balance_cache` maps (key_name, quote) -> (fetched_at, free).""" - quote = ccxt_symbol.split('/')[-1] + quote = cash_currency(ccxt_symbol) cache_key = (api_key_record.name, quote) now = time.monotonic() cached = balance_cache.get(cache_key) diff --git a/backend/engine/candle_poller.py b/backend/engine/candle_poller.py index 93c224f..ef8a03a 100644 --- a/backend/engine/candle_poller.py +++ b/backend/engine/candle_poller.py @@ -9,7 +9,8 @@ from sqlalchemy.orm import Session from backend.core.database import SessionLocal -from backend.core.exchange_registry import build_exchange, exchange_spec +from backend.core.exchange_registry import build_exchange_for_symbol, exchange_spec +from backend.engine.symbols import market_type_of from backend.core.events import event_bus from backend.models.bots import BotConfig from backend.models.candles import Candle @@ -36,7 +37,7 @@ def __init__(self): self.running = False self.needs_reconnect = False self._poll_tasks: list[asyncio.Task] = [] - self._exchange_cache: dict[str, tuple[float, object, threading.Lock]] = {} # exchange_id → (created_at, ccxt instance, lock) + self._exchange_cache: dict[tuple[str, str], tuple[float, object, threading.Lock]] = {} # (exchange_id, market type) → (created_at, ccxt instance, lock) self._exchange_cache_ttl = 3600 # rebuild exchange instances after 1 hour # Last closed candle ts per (exchange, symbol, timeframe), kept across # reconnects so a restarted poll task does not re-publish the same candle. @@ -261,7 +262,7 @@ def _backfill_one_symbol( lookback_limit: int, ): # Fresh instance per thread to avoid shared rate-limit state - exchange = build_exchange(exchange_name) + exchange = build_exchange_for_symbol(exchange_name, symbol) # Validate timeframe before attempting fetch try: @@ -515,7 +516,7 @@ async def _poll_symbol(self, exchange_name: str, symbol: str, timeframe: str): are not epoch-aligned (1w/1M) fall back to a fixed interval of max(10s, min(60s, tf_seconds / 4)). """ - exchange, ex_lock = self._get_public_exchange(exchange_name) + exchange, ex_lock = self._get_public_exchange(exchange_name, symbol) # Validate timeframe def _load_markets(): @@ -565,7 +566,7 @@ def _next_sleep(found_new: bool) -> float: def _fetch(limit): # Re-resolve from the cache every call so the TTL rebuild takes effect, # and hold the per-exchange lock: sync CCXT instances are not thread-safe. - inst, lock = self._get_public_exchange(exchange_name) + inst, lock = self._get_public_exchange(exchange_name, symbol) with lock: return inst.fetch_ohlcv(symbol, timeframe, None, limit) @@ -688,18 +689,20 @@ async def _listen_for_bot_changes(self): # Internal helpers # ───────────────────────────────────────────────────────────────────────── - def _get_public_exchange(self, exchange_name: str): + def _get_public_exchange(self, exchange_name: str, symbol: str = ""): """Return (instance, lock) for a cached unauthenticated exchange used for - public market data. Callers must hold the lock around every fetch since - sync CCXT instances are not thread-safe.""" + public market data. Spot and swap symbols get separate instances (the + market type is read off the symbol). Callers must hold the lock around + every fetch since sync CCXT instances are not thread-safe.""" + cache_key = (exchange_name, market_type_of(symbol)) now = time.monotonic() - cached = self._exchange_cache.get(exchange_name) + cached = self._exchange_cache.get(cache_key) if cached and (now - cached[0]) < self._exchange_cache_ttl: return cached[1], cached[2] - instance = build_exchange(exchange_name) + instance = build_exchange_for_symbol(exchange_name, symbol) # Keep the existing lock across rebuilds so in-flight fetches stay serialized lock = cached[2] if cached else threading.Lock() - self._exchange_cache[exchange_name] = (now, instance, lock) + self._exchange_cache[cache_key] = (now, instance, lock) return instance, lock diff --git a/backend/engine/live_cycle.py b/backend/engine/live_cycle.py index c901a0b..821c67b 100644 --- a/backend/engine/live_cycle.py +++ b/backend/engine/live_cycle.py @@ -19,11 +19,56 @@ from backend.models.exchange_keys import ExchangeKey from backend.engine.evaluator import NodeEvaluator from backend.engine.sizing import _indicator_fingerprint, _num, _int, _tf_seconds, _naive_utc +from backend.engine.symbols import base_of, cash_currency, is_derivative, leverage_for, market_type_for, normalize +from backend.engine import broker +from backend.core.exchange_registry import market_caps from backend.core import bot_log_buffer as blb logger = logging.getLogger("apexalgo.bot_manager") +# ── Derivative order plumbing ───────────────────────────────────────────── +# Spot orders keep their literal create_market_buy/sell_order calls; the +# helpers below only run for `BASE/QUOTE:SETTLE` symbols, where ccxt wants +# the amount in contracts, closes must be reduce-only, and a few exchanges +# take the leverage per order. + +def _precise_amount(ccxt_inst, ccxt_symbol, base_amount): + """Round a base amount to the market's precision. On derivatives the + precision is defined in contracts, so convert, round, convert back.""" + if not is_derivative(ccxt_symbol): + return float(ccxt_inst.amount_to_precision(ccxt_symbol, base_amount)) + contracts = float(ccxt_inst.amount_to_precision(ccxt_symbol, broker.to_contracts(ccxt_inst, ccxt_symbol, base_amount))) + return broker.from_contracts(ccxt_inst, ccxt_symbol, contracts) + + +def _place_market_order(ccxt_inst, api_key_record, ccxt_symbol, side, base_amount, *, reduce_only, leverage=1): + """Market order in base units. Spot: the pre-existing ccxt shortcuts. + Derivatives: `create_order` in contracts, reduce-only on closes and, + where the exchange wants it, the leverage in the params.""" + if not is_derivative(ccxt_symbol): + if side == "buy": + return ccxt_inst.create_market_buy_order(ccxt_symbol, base_amount) + return ccxt_inst.create_market_sell_order(ccxt_symbol, base_amount) + contracts = broker.to_contracts(ccxt_inst, ccxt_symbol, base_amount) + params = {} + if reduce_only: + params["reduceOnly"] = True + caps = market_caps(getattr(api_key_record, "exchange", ""), "swap") + if caps is not None and caps.leverage_in_order: + params["leverage"] = int(float(leverage or 1)) + return ccxt_inst.create_order(ccxt_symbol, "market", side, contracts, None, params) + + +def _filled_base(ccxt_inst, ccxt_symbol, ex_order) -> float: + """`filled` of a ccxt order in base units (contracts x contract size on + derivatives).""" + filled = float(ex_order.get("filled") or 0) + if filled > 0 and is_derivative(ccxt_symbol): + return broker.from_contracts(ccxt_inst, ccxt_symbol, filled) + return filled + + def close_all_open_positions(engine, bot, db, key_records): """Market-close every open non-backtest position for a bot before it is force-stopped. A stopped bot no longer evaluates SL/TP, so leaving live @@ -61,7 +106,8 @@ def close_all_open_positions(engine, bot, db, key_records): actual_price = close_price actual_fee = 0.0 order_id = f"local_{int(time.time())}_{uuid.uuid4().hex[:8]}" - ccxt_symbol = pos.symbol.replace('-', '/').upper() + ccxt_symbol = normalize(pos.symbol) + _close_cols = {"market_type": "swap", "reduce_only": 1} if is_derivative(ccxt_symbol) else {} if pos.mode in ("paper", "live"): if ccxt_inst is None: @@ -69,14 +115,14 @@ def close_all_open_positions(engine, bot, db, key_records): blb.push(bot.name, "ERROR", f"Could not close {pos.mode} position on {pos.symbol}: exchange unavailable — close it manually!") continue try: - sell_qty = float(ccxt_inst.amount_to_precision(ccxt_symbol, close_qty)) + sell_qty = _precise_amount(ccxt_inst, ccxt_symbol, close_qty) if sell_qty <= 0: continue - ex_order = ccxt_inst.create_market_sell_order(ccxt_symbol, sell_qty) + ex_order = _place_market_order(ccxt_inst, api_key_record, ccxt_symbol, "sell", sell_qty, reduce_only=True, leverage=pos.leverage) ex_order = engine._reconcile_order(ccxt_inst, ex_order, ccxt_symbol) - filled_qty = float(ex_order.get("filled") or 0) + filled_qty = _filled_base(ccxt_inst, ccxt_symbol, ex_order) if filled_qty <= 0 and ex_order.get("status") != "closed": - db.add(Order(position_id=pos.id, exchange=pos.exchange, bot_name=bot.name, mode=pos.mode, symbol=pos.symbol, side="sell", order_type="market", price=close_price, amount=sell_qty, timestamp=now_ts, exchange_order_id=ex_order.get("id"), status="canceled")) + db.add(Order(position_id=pos.id, exchange=pos.exchange, bot_name=bot.name, mode=pos.mode, symbol=pos.symbol, side="sell", order_type="market", price=close_price, amount=sell_qty, timestamp=now_ts, exchange_order_id=ex_order.get("id"), status="canceled", **_close_cols)) db.commit() blb.push(bot.name, "ERROR", f"Forced close on {pos.symbol} did not fill; position left open — close it manually!") continue @@ -94,7 +140,7 @@ def close_all_open_positions(engine, bot, db, key_records): if pos.entry_price and pos.amount: pos.profit_pct = (pos.profit_pct or 0.0) + ((actual_price - pos.entry_price) / pos.entry_price) * 100 * (close_qty / pos.amount) - db.add(Order(position_id=pos.id, exchange=pos.exchange, bot_name=bot.name, mode=pos.mode, symbol=pos.symbol, side="sell", order_type="market", price=actual_price, amount=close_qty, timestamp=now_ts, exchange_order_id=order_id, status="filled", fee=actual_fee)) + db.add(Order(position_id=pos.id, exchange=pos.exchange, bot_name=bot.name, mode=pos.mode, symbol=pos.symbol, side="sell", order_type="market", price=actual_price, amount=close_qty, timestamp=now_ts, exchange_order_id=order_id, status="filled", fee=actual_fee, **_close_cols)) if close_qty >= pos.amount - 0.00001: pos.status = "closed" @@ -119,7 +165,13 @@ def maybe_open_position(engine, db, bot, exchange, symbol, mode, api_key_record, None when no entry was made (no signal, blocked, skipped, or failed). Kept separate from the exit loop on purpose: bailing out of the entry must never skip the SL/TP evaluation of the positions already open.""" - ccxt_symbol = symbol.replace('-', '/').upper() + ccxt_symbol = normalize(symbol) + # Derivatives: leverage from the bot (the linked key fixes the market + # type); every spot bot goes through the unchanged branches below + _market_type = market_type_for(bot.settings, api_key_record) + _derivative = is_derivative(ccxt_symbol) + _leverage = leverage_for(bot.settings, _market_type) if _derivative else 1.0 + _open_cols = {"market_type": _market_type} if _derivative else {} if is_buy and entries_blocked: blb.push(bot.name, "INFO", f"BUY signal on {symbol} skipped — entries blocked by max drawdown") elif is_buy and open_count >= max_pos: @@ -127,7 +179,10 @@ def maybe_open_position(engine, db, bot, exchange, symbol, mode, api_key_record, # global: the whole portfolio) blb.push(bot.name, "INFO", f"BUY signal on {symbol} skipped — max_positions ({max_pos}) reached") elif is_buy and can_buy_cooldown: - trade_amount = engine._calculate_trade_amount(current_price, bot.settings) + if _derivative: + trade_amount = engine._calculate_trade_amount(current_price, bot.settings, leverage=_leverage) + else: + trade_amount = engine._calculate_trade_amount(current_price, bot.settings) if trade_amount is None: logger.warning("Skipping buy for %s: invalid trade amount", symbol) else: @@ -141,22 +196,36 @@ def maybe_open_position(engine, db, bot, exchange, symbol, mode, api_key_record, # pool, pay entry slippage and fee. A forward test that # trades frictionless on the original capital would # confirm a strategy the backtest never ran. - _quote = ccxt_symbol.split('/')[-1] + _quote = cash_currency(ccxt_symbol) pool = engine._forward_pool(db, bot, _quote) if pool <= 0: blb.push(bot.name, "WARN", f"BUY signal on {symbol} skipped — forward-test pool depleted ({pool:,.2f} {_quote})") return None entry_fee_pct, _, entry_slip, _ = engine._sim_frictions(bot.settings) - trade_amount = engine._calculate_trade_amount(current_price, bot.settings, current_equity=pool) - if trade_amount is None: - return None - actual_price = current_price * (1 + entry_slip) - if bot.settings.get("trade_settings", {}).get("entry", {}).get("amount_type", "percentage") != "fixed": - trade_amount = min(trade_amount, pool / (actual_price * (1 + entry_fee_pct))) - if trade_amount <= 0 or actual_price * trade_amount * (1 + entry_fee_pct) > pool + 1e-9: - blb.push(bot.name, "WARN", f"BUY signal on {symbol} skipped — entry does not fit the forward-test pool ({pool:,.2f} {_quote})") - return None - buy_fee = actual_price * trade_amount * entry_fee_pct + if _derivative: + # Margin (notional / leverage) + fee on the notional must fit the pool + trade_amount = engine._calculate_trade_amount(current_price, bot.settings, current_equity=pool, leverage=_leverage) + if trade_amount is None: + return None + actual_price = current_price * (1 + entry_slip) + _unit_cost = actual_price * (1 / _leverage + entry_fee_pct) + if bot.settings.get("trade_settings", {}).get("entry", {}).get("amount_type", "percentage") != "fixed": + trade_amount = min(trade_amount, pool / _unit_cost) + if trade_amount <= 0 or trade_amount * _unit_cost > pool + 1e-9: + blb.push(bot.name, "WARN", f"BUY signal on {symbol} skipped — entry does not fit the forward-test pool ({pool:,.2f} {_quote})") + return None + buy_fee = actual_price * trade_amount * entry_fee_pct + else: + trade_amount = engine._calculate_trade_amount(current_price, bot.settings, current_equity=pool) + if trade_amount is None: + return None + actual_price = current_price * (1 + entry_slip) + if bot.settings.get("trade_settings", {}).get("entry", {}).get("amount_type", "percentage") != "fixed": + trade_amount = min(trade_amount, pool / (actual_price * (1 + entry_fee_pct))) + if trade_amount <= 0 or actual_price * trade_amount * (1 + entry_fee_pct) > pool + 1e-9: + blb.push(bot.name, "WARN", f"BUY signal on {symbol} skipped — entry does not fit the forward-test pool ({pool:,.2f} {_quote})") + return None + buy_fee = actual_price * trade_amount * entry_fee_pct elif mode in ["paper", "live"] and api_key_record: ccxt_inst = get_ccxt() @@ -190,7 +259,7 @@ def maybe_open_position(engine, db, bot, exchange, symbol, mode, api_key_record, sizing_capital = _num(bot.settings.get("backtest_capital"), 1000) logger.info("Bot '%s': sandbox balance unavailable, sizing paper entry from backtest capital $%.2f", bot.name, sizing_capital) else: - _quote = ccxt_symbol.split('/')[-1] + _quote = cash_currency(ccxt_symbol) pool, wallet_total, bot_total = engine._live_allocation(db, bot, _quote, free_balance) if pool <= 0: blb.push(bot.name, "WARN", f"BUY signal on {symbol} skipped — allocation fully deployed ({bot_total:,.0f} {_quote} = {_num(bot.settings.get('live_allocation_pct'), 100):.0f}% of wallet {wallet_total:,.0f})") @@ -198,11 +267,15 @@ def maybe_open_position(engine, db, bot, exchange, symbol, mode, api_key_record, sizing_capital = min(free_balance, pool) logger.info("Bot '%s': sizing %s entry from $%.2f (free=$%.2f, pool remaining=$%.2f of allocation $%.2f, wallet=$%.2f)", bot.name, mode, sizing_capital, free_balance, pool, bot_total, wallet_total) - trade_amount = engine._calculate_trade_amount(current_price, bot.settings, current_equity=sizing_capital) + if _derivative: + # sizing_capital is margin; the order is leverage x bigger + trade_amount = engine._calculate_trade_amount(current_price, bot.settings, current_equity=sizing_capital, leverage=_leverage) + else: + trade_amount = engine._calculate_trade_amount(current_price, bot.settings, current_equity=sizing_capital) if trade_amount is None: logger.warning("Skipping buy for %s: no capital available to size trade", symbol) return None - trade_amount = float(ccxt_inst.amount_to_precision(ccxt_symbol, trade_amount)) + trade_amount = _precise_amount(ccxt_inst, ccxt_symbol, trade_amount) if trade_amount <= 0: logger.warning("Trade amount rounded to zero for %s after precision, skipping", ccxt_symbol) return None @@ -218,24 +291,28 @@ def maybe_open_position(engine, db, bot, exchange, symbol, mode, api_key_record, if max_order_usd > 0: order_value_usd = trade_amount * current_price if order_value_usd > max_order_usd: - capped = float(ccxt_inst.amount_to_precision(ccxt_symbol, max_order_usd / current_price)) + capped = _precise_amount(ccxt_inst, ccxt_symbol, max_order_usd / current_price) logger.warning("SAFETY: BUY order $%.2f exceeds max_order_value $%.2f for %s — capped to %s", order_value_usd, max_order_usd, symbol, capped) - blb.push(bot.name, "WARN", f"BUY on {symbol} capped by max_order_value: {order_value_usd:,.0f} → {max_order_usd:,.0f} {ccxt_symbol.split('/')[-1]} (raise max_order_value or lower the entry amount to match the backtest)") + blb.push(bot.name, "WARN", f"BUY on {symbol} capped by max_order_value: {order_value_usd:,.0f} → {max_order_usd:,.0f} {cash_currency(ccxt_symbol)} (raise max_order_value or lower the entry amount to match the backtest)") trade_amount = capped min_violation = engine._below_market_minimum(ccxt_inst, ccxt_symbol, trade_amount, current_price) if trade_amount <= 0 or min_violation: blb.push(bot.name, "WARN", f"Capped order on {symbol} is below the exchange minimum ({min_violation or 'zero amount'}) — entry skipped") return None - okx_order = ccxt_inst.create_market_buy_order(ccxt_symbol, trade_amount) + if _derivative: + # Leverage/margin mode must be confirmed on the exchange + # before the first contract is bought (raises on failure) + engine._ensure_leverage(ccxt_inst, api_key_record, ccxt_symbol, _leverage, bot.settings.get("margin_mode"), bot.name) + okx_order = _place_market_order(ccxt_inst, api_key_record, ccxt_symbol, "buy", trade_amount, reduce_only=False, leverage=_leverage) logger.info("%s BUY response: id=%s status=%s filled=%s avg=%s fee=%s", mode.upper(), okx_order.get("id"), okx_order.get("status"), okx_order.get("filled"), okx_order.get("average"), okx_order.get("fee")) okx_order = engine._reconcile_order(ccxt_inst, okx_order, ccxt_symbol) - filled_qty = float(okx_order.get("filled") or 0) + filled_qty = _filled_base(ccxt_inst, ccxt_symbol, okx_order) if filled_qty <= 0 and okx_order.get("status") != "closed": if engine._cancel_unfilled_order(ccxt_inst, okx_order.get("id"), ccxt_symbol): logger.warning("%s BUY unfilled (status=%s), canceled on exchange.", mode.upper(), okx_order.get("status")) - db.add(Order(exchange=exchange, bot_name=bot.name, mode=mode, symbol=symbol, side="buy", order_type="market", price=current_price, amount=trade_amount, timestamp=latest_time, exchange_order_id=okx_order.get("id"), status="canceled")) + db.add(Order(exchange=exchange, bot_name=bot.name, mode=mode, symbol=symbol, side="buy", order_type="market", price=current_price, amount=trade_amount, timestamp=latest_time, exchange_order_id=okx_order.get("id"), status="canceled", **_open_cols)) db.commit() return None # Cancel did not go through: the order may have filled @@ -253,8 +330,8 @@ def maybe_open_position(engine, db, bot, exchange, symbol, mode, api_key_record, # A fee charged in base currency comes out of the bought # amount itself; only the net amount is actually held fee_info = okx_order.get("fee") or {} - base_ccy = ccxt_symbol.split('/')[0] - if fee_info.get("currency") and str(fee_info["currency"]).upper() == base_ccy.upper(): + base_ccy = base_of(ccxt_symbol) + if not _derivative and fee_info.get("currency") and str(fee_info["currency"]).upper() == base_ccy.upper(): try: base_fee_cost = float(fee_info.get("cost") or 0) except (TypeError, ValueError): @@ -263,14 +340,19 @@ def maybe_open_position(engine, db, bot, exchange, symbol, mode, api_key_record, net_amount = float(ccxt_inst.amount_to_precision(ccxt_symbol, max(trade_amount - base_fee_cost, 0))) if net_amount > 0: trade_amount = net_amount - engine._balance_cache.pop((api_key_record.name, ccxt_symbol.split('/')[-1]), None) + engine._balance_cache.pop((api_key_record.name, cash_currency(ccxt_symbol)), None) # Position created after successful exchange order open_position = Position(exchange=exchange, bot_name=bot.name, symbol=symbol, mode=mode, status="open", side="long", entry_price=actual_price, amount=trade_amount) + if _derivative: + open_position.market_type = _market_type + open_position.leverage = _leverage + if mode in ["paper", "live"]: + open_position.contracts = broker.to_contracts(get_ccxt(), ccxt_symbol, trade_amount) db.add(open_position) db.flush() - db.add(Order(position_id=open_position.id, exchange=exchange, bot_name=bot.name, mode=mode, symbol=symbol, side="buy", order_type="market", price=actual_price, amount=trade_amount, timestamp=latest_time, exchange_order_id=order_id, status="filled", fee=buy_fee)) + db.add(Order(position_id=open_position.id, exchange=exchange, bot_name=bot.name, mode=mode, symbol=symbol, side="buy", order_type="market", price=actual_price, amount=trade_amount, timestamp=latest_time, exchange_order_id=order_id, status="filled", fee=buy_fee, **_open_cols)) if mode in ["paper", "live"]: # A real exchange fill must be persisted immediately — # a later rollback may not erase the record of it @@ -282,7 +364,7 @@ def maybe_open_position(engine, db, bot, exchange, symbol, mode, api_key_record, except ccxt.InsufficientFunds as e: logger.warning("%s BUY rejected (insufficient funds): %s", mode.upper(), e) blb.push(bot.name, "WARN", f"{mode.upper()} BUY rejected: insufficient funds") - db.add(Order(exchange=exchange, bot_name=bot.name, mode=mode, symbol=symbol, side="buy", order_type="market", price=current_price, amount=trade_amount, timestamp=latest_time, status="rejected")) + db.add(Order(exchange=exchange, bot_name=bot.name, mode=mode, symbol=symbol, side="buy", order_type="market", price=current_price, amount=trade_amount, timestamp=latest_time, status="rejected", **_open_cols)) if mode in ["paper", "live"]: db.commit() except Exception as e: @@ -291,7 +373,7 @@ def maybe_open_position(engine, db, bot, exchange, symbol, mode, api_key_record, # implies an order that existed logger.error("%s BUY failed for %s: %s: %s", mode.upper(), symbol, type(e).__name__, e, exc_info=True) blb.push(bot.name, "ERROR", f"{mode.upper()} BUY {symbol} rejected — {type(e).__name__}: {str(e)[:200]}") - db.add(Order(exchange=exchange, bot_name=bot.name, mode=mode, symbol=symbol, side="buy", order_type="market", price=current_price, amount=trade_amount, timestamp=latest_time, status="rejected")) + db.add(Order(exchange=exchange, bot_name=bot.name, mode=mode, symbol=symbol, side="buy", order_type="market", price=current_price, amount=trade_amount, timestamp=latest_time, status="rejected", **_open_cols)) if mode in ["paper", "live"]: db.commit() return None @@ -592,7 +674,8 @@ def get_ccxt(): # position left open by the backtest must not take a live slot open_count = len(_positions_by_bot_mode.get((bot.name, mode), [])) - ccxt_symbol = symbol.replace('-', '/').upper() + ccxt_symbol = normalize(symbol) + _close_cols = {"market_type": "swap", "reduce_only": 1} if is_derivative(ccxt_symbol) else {} just_opened_ids = set() # Cooldown check using pre-loaded counts @@ -651,7 +734,7 @@ def get_ccxt(): actual_fee = actual_price * close_qty * exit_fee_pct elif mode in ["paper", "live"] and api_key_record: ccxt_inst = get_ccxt() - close_qty = float(ccxt_inst.amount_to_precision(ccxt_symbol, close_qty)) + close_qty = _precise_amount(ccxt_inst, ccxt_symbol, close_qty) if close_qty <= 0: logger.warning("Sell amount rounded to zero for %s after precision, skipping", ccxt_symbol) continue @@ -673,16 +756,16 @@ def get_ccxt(): logger.warning("%s SELL skipped for %s: %s", mode.upper(), symbol, min_violation) blb.push(bot.name, "WARN", f"Sell on {symbol} skipped: {min_violation}") continue - okx_order = ccxt_inst.create_market_sell_order(ccxt_symbol, close_qty) + okx_order = _place_market_order(ccxt_inst, api_key_record, ccxt_symbol, "sell", close_qty, reduce_only=True, leverage=pos.leverage) logger.info("%s SELL response: id=%s status=%s filled=%s avg=%s fee=%s", mode.upper(), okx_order.get("id"), okx_order.get("status"), okx_order.get("filled"), okx_order.get("average"), okx_order.get("fee")) okx_order = engine._reconcile_order(ccxt_inst, okx_order, ccxt_symbol) - filled_qty = float(okx_order.get("filled") or 0) + filled_qty = _filled_base(ccxt_inst, ccxt_symbol, okx_order) if filled_qty <= 0 and okx_order.get("status") != "closed": if engine._cancel_unfilled_order(ccxt_inst, okx_order.get("id"), ccxt_symbol): logger.warning("%s SELL unfilled (status=%s), canceled on exchange.", mode.upper(), okx_order.get("status")) - db.add(Order(position_id=pos.id, exchange=exchange, bot_name=bot.name, mode=mode, symbol=symbol, side="sell", order_type="market", price=ev['price'], amount=close_qty, timestamp=latest_time, exchange_order_id=okx_order.get("id"), status="canceled")) + db.add(Order(position_id=pos.id, exchange=exchange, bot_name=bot.name, mode=mode, symbol=symbol, side="sell", order_type="market", price=ev['price'], amount=close_qty, timestamp=latest_time, exchange_order_id=okx_order.get("id"), status="canceled", **_close_cols)) db.commit() continue # Cancel did not go through: the sell may still fill on @@ -690,7 +773,7 @@ def get_ccxt(): # stop the bot — a second sell here could double-sell. logger.error("%s SELL state unknown for %s (id=%s) — stopping bot '%s'", mode.upper(), symbol, okx_order.get("id"), bot.name) blb.push(bot.name, "ERROR", f"Order state unknown on {symbol} — verify manually on the exchange before restarting") - db.add(Order(position_id=pos.id, exchange=exchange, bot_name=bot.name, mode=mode, symbol=symbol, side="sell", order_type="market", price=ev['price'], amount=close_qty, timestamp=latest_time, exchange_order_id=okx_order.get("id"), status="unknown")) + db.add(Order(position_id=pos.id, exchange=exchange, bot_name=bot.name, mode=mode, symbol=symbol, side="sell", order_type="market", price=ev['price'], amount=close_qty, timestamp=latest_time, exchange_order_id=okx_order.get("id"), status="unknown", **_close_cols)) engine._engine_stop(bot, db, f"Sell order state unknown on {symbol} — verify on the exchange before restarting") db.commit() break @@ -702,9 +785,9 @@ def get_ccxt(): actual_price = okx_order.get("average") or okx_order.get("price") or ev['price'] order_id = okx_order.get("id") actual_fee = engine._fee_in_quote(okx_order.get("fee"), ccxt_symbol, actual_price) - engine._balance_cache.pop((api_key_record.name, ccxt_symbol.split('/')[-1]), None) + engine._balance_cache.pop((api_key_record.name, cash_currency(ccxt_symbol)), None) - db.add(Order(position_id=pos.id, exchange=exchange, bot_name=bot.name, mode=mode, symbol=symbol, side="sell", order_type="market", price=actual_price, amount=close_qty, timestamp=latest_time, exchange_order_id=order_id, status="filled", fee=actual_fee)) + db.add(Order(position_id=pos.id, exchange=exchange, bot_name=bot.name, mode=mode, symbol=symbol, side="sell", order_type="market", price=actual_price, amount=close_qty, timestamp=latest_time, exchange_order_id=order_id, status="filled", fee=actual_fee, **_close_cols)) # Fee-adjusted P&L: subtract proportional entry fee + exit fee total_buy_fees = sum((o.fee or 0.0) for o in (pos.orders or []) if o.side == "buy" and o.status == "filled") @@ -743,7 +826,7 @@ def get_ccxt(): except Exception as e: logger.error("%s SELL failed: %s", mode.upper(), e, exc_info=True) blb.push(bot.name, "ERROR", f"{mode.upper()} SELL failed: {e}") - db.add(Order(position_id=pos.id, exchange=exchange, bot_name=bot.name, mode=mode, symbol=symbol, side="sell", order_type="market", price=current_price, amount=close_qty, timestamp=latest_time, status="rejected")) + db.add(Order(position_id=pos.id, exchange=exchange, bot_name=bot.name, mode=mode, symbol=symbol, side="sell", order_type="market", price=current_price, amount=close_qty, timestamp=latest_time, status="rejected", **_close_cols)) if mode in ["paper", "live"]: db.commit() diff --git a/backend/engine/settings_validator.py b/backend/engine/settings_validator.py index 892131b..fa24f0b 100644 --- a/backend/engine/settings_validator.py +++ b/backend/engine/settings_validator.py @@ -2,11 +2,16 @@ import re from datetime import datetime -from backend.core.exchange_registry import get_exchange_timeframes +from backend.core.exchange_registry import exchange_spec, get_exchange_timeframes, market_caps from backend.engine.indicator_registry import get_spec +from backend.engine.symbols import DEFAULT_MARGIN_MODE, DEFAULT_MARKET_TYPE, MARGIN_MODES, MARKET_TYPES, is_derivative logger = logging.getLogger("apexalgo.settings_validator") -SYMBOL_PATTERN = re.compile(r'^[A-Z0-9]+/[A-Z0-9]+$') +# BASE/QUOTE for spot, BASE/QUOTE:SETTLE for perpetual swaps +SYMBOL_PATTERN = re.compile(r'^[A-Z0-9]+/[A-Z0-9]+(:[A-Z0-9]+)?$') +# Above this the liquidation sits close enough to the entry that a normal +# stop loss barely gets a chance — worth a warning, not a refusal +LEVERAGE_WARN_ABOVE = 3 MAX_PRICE_OFFSET = 500 VALID_EXIT_TYPES = {'percentage', 'trailing', 'atr', 'fixed'} VALID_AMOUNT_TYPES = {'percentage', 'fixed'} @@ -23,7 +28,7 @@ LOOKBACK_MARGIN = 50 -def validate_bot_settings(settings: dict, exchange_id: str | None = None) -> dict: +def validate_bot_settings(settings: dict, exchange_id: str | None = None, key_market_type: str | None = None) -> dict: """Validate bot settings and return errors and warnings. Parameters @@ -33,6 +38,9 @@ def validate_bot_settings(settings: dict, exchange_id: str | None = None) -> dic exchange_id : str, optional Resolved exchange ID for timeframe validation. Falls back to settings['data_exchange'] or 'okx' if not provided. + key_market_type : str, optional + Market type of the linked API key; a key is bound to one market, + so a bot on another market type is refused. Returns dict with 'errors' (list of blocking issues) and 'warnings' (list of non-blocking issues). @@ -40,32 +48,78 @@ def validate_bot_settings(settings: dict, exchange_id: str | None = None) -> dic errors = [] warnings = [] nodes = settings.get("nodes", {}) + eid = exchange_id or settings.get("data_exchange", "okx") + + # Market type (phase 2): spot or perpetual swap; absent = spot for every + # pre-existing bot. Symbol form, leverage and margin mode depend on it. + market_type = str(settings.get("market_type") or DEFAULT_MARKET_TYPE).strip().lower() + if market_type not in MARKET_TYPES: + errors.append(f"Invalid market_type '{settings.get('market_type')}'. Use one of: {', '.join(MARKET_TYPES)}.") + market_type = DEFAULT_MARKET_TYPE + elif "market_type" in settings: + settings["market_type"] = market_type + if key_market_type and str(key_market_type).lower() != market_type: + errors.append(f"API key '{settings.get('api_key_name')}' is a {key_market_type} key but the bot trades {market_type}. " + f"Link a {market_type} key or change the bot's market type.") + if exchange_spec(eid) is not None and market_caps(eid, market_type) is None: + errors.append(f"{exchange_spec(eid).name} has no '{market_type}' market in ApexAlgo.") + derivative = market_type != "spot" + + def _check_symbol(raw) -> str: + norm = str(raw).strip().upper().replace('-', '/') + if not SYMBOL_PATTERN.match(norm): + errors.append(f"Invalid symbol '{raw}'. Use BASE/QUOTE, e.g. BTC/USDC" + (" (BASE/QUOTE:SETTLE for swaps, e.g. BTC/USDT:USDT)." if derivative else ".")) + elif derivative and not is_derivative(norm): + errors.append(f"Symbol '{raw}' is a spot pair; a swap bot needs the BASE/QUOTE:SETTLE form, e.g. {norm}:{norm.split('/')[-1]}.") + elif not derivative and is_derivative(norm): + errors.append(f"Symbol '{raw}' is a perpetual swap; a spot bot needs BASE/QUOTE, e.g. {norm.split(':')[0]} (or set market_type to 'swap').") + return norm # Symbols — normalize and validate BASE/QUOTE format, write back normalized values symbols = settings.get("symbols", []) if symbols: - normalized_symbols = [] - for sym in symbols: - norm = str(sym).strip().upper().replace('-', '/') - if not SYMBOL_PATTERN.match(norm): - errors.append(f"Invalid symbol '{sym}'. Use BASE/QUOTE, e.g. BTC/USDC.") - normalized_symbols.append(norm) - settings["symbols"] = normalized_symbols + settings["symbols"] = [_check_symbol(sym) for sym in symbols] else: if settings.get("symbol"): warnings.append("Using single 'symbol' field; consider using 'symbols' list.") else: errors.append("No trading symbols configured.") if settings.get("symbol"): - norm = str(settings["symbol"]).strip().upper().replace('-', '/') - if not SYMBOL_PATTERN.match(norm): - errors.append(f"Invalid symbol '{settings['symbol']}'. Use BASE/QUOTE, e.g. BTC/USDC.") - settings["symbol"] = norm + settings["symbol"] = _check_symbol(settings["symbol"]) + + # Leverage and margin mode only mean something on a swap + lev_raw = settings.get("leverage") + if lev_raw not in (None, ""): + try: + lev = float(lev_raw) + except (ValueError, TypeError): + lev = None + errors.append(f"leverage '{lev_raw}' is not a valid number.") + if lev is not None: + if lev != int(lev) or lev < 1: + errors.append("leverage must be a whole number >= 1.") + elif derivative: + caps = market_caps(eid, market_type) + if caps is not None and lev > caps.max_leverage: + errors.append(f"leverage {int(lev)}x exceeds the {caps.max_leverage}x ApexAlgo allows on {exchange_spec(eid).name} swaps.") + elif lev > LEVERAGE_WARN_ABOVE: + warnings.append(f"leverage {int(lev)}x: the estimated liquidation sits about {100 / lev:.0f}% below the entry — " + "a stop loss that is not tighter than that never fires.") + elif lev > 1: + errors.append(f"leverage {int(lev)}x has no effect on a spot bot; set it to 1 or switch market_type to 'swap'.") + mm_raw = settings.get("margin_mode") + if mm_raw not in (None, ""): + mm = str(mm_raw).strip().lower() + if mm not in MARGIN_MODES: + errors.append(f"Invalid margin_mode '{mm_raw}'. Use one of: {', '.join(MARGIN_MODES)}.") + else: + settings["margin_mode"] = mm + if derivative and not settings.get("margin_mode"): + settings["margin_mode"] = DEFAULT_MARGIN_MODE # Timeframe — validated against the exchange's supported timeframes tf = settings.get("timeframe") if tf: - eid = exchange_id or settings.get("data_exchange", "okx") supported = get_exchange_timeframes(eid) if supported and tf not in supported: errors.append(f"Exchange '{eid}' does not support timeframe '{tf}'. Supported: {', '.join(sorted(supported.keys()))}") @@ -275,7 +329,7 @@ def validate_bot_settings(settings: dict, exchange_id: str | None = None) -> dic except (ValueError, TypeError): max_order_value = 0 if max_order_value <= 0: - errors.append("Live execution requires max_order_value > 0 as a safety cap.") + errors.append("Live execution requires max_order_value > 0 as a safety cap" + (" (on the notional, i.e. margin x leverage)." if derivative else ".")) else: # The engine clamps every live entry to the cap, so a cap below # the configured size silently turns the strategy into a smaller @@ -285,6 +339,9 @@ def validate_bot_settings(settings: dict, exchange_id: str | None = None) -> dic planned = float(amount_value or 0) if amount_type == "percentage": planned = capital * planned / 100.0 + if derivative: + # The cap is on the notional; the entry is margin x leverage + planned *= max(float(settings.get("leverage") or 1), 1.0) if planned > max_order_value > 0: warnings.append( f"max_order_value ({max_order_value:,.0f}) is below the planned entry size " diff --git a/backend/engine/sizing.py b/backend/engine/sizing.py index 76628ca..be15951 100644 --- a/backend/engine/sizing.py +++ b/backend/engine/sizing.py @@ -7,6 +7,7 @@ from sqlalchemy import text, func from backend.models.bots import BotConfig from backend.models.positions import Position +from backend.engine.symbols import cash_currency, is_derivative logger = logging.getLogger("apexalgo.bot_manager") @@ -69,13 +70,33 @@ def _naive_utc(ts): }) +# Phase-2 (derivatives) settings at their default are dropped from the +# fingerprint: a spot bot saved by a newer UI must hash exactly like the +# same bot saved before these keys existed, or every variant counter bumps. +_DEFAULT_MARKET_KEYS = {"market_type": "spot", "leverage": 1, "margin_mode": "isolated"} + + def _config_fingerprint(settings: dict) -> str: """Stable hash of the strategy-relevant part of a bot's settings, used to count how many distinct variants have been backtested.""" relevant = {k: v for k, v in (settings or {}).items() if k not in _NON_STRATEGY_KEYS} + for k, default in _DEFAULT_MARKET_KEYS.items(): + if k in relevant and _is_default_market_setting(relevant[k], default): + del relevant[k] return md5(json.dumps(relevant, sort_keys=True, default=str).encode()).hexdigest() +def _is_default_market_setting(value, default) -> bool: + if value in (None, ""): + return True + if isinstance(default, (int, float)): + try: + return float(value) == float(default) + except (TypeError, ValueError): + return False + return str(value).lower() == default + + def data_fingerprint(df) -> str: """sha256 of the raw OHLCV rows a backtest walked (sorted by timestamp). `repr` on the floats makes it byte-stable across runs, so two runs on the @@ -158,14 +179,23 @@ def sim_frictions(settings): def deployed_capital(db, bot_names, quote, modes=("paper", "live")): - """Quote-currency cost (entry price x amount) of the open positions of - the given bots in the given modes, in pairs quoted in `quote`.""" + """Cash locked by the open positions of the given bots in the given + modes, in pairs funded in `quote`: entry price x amount on spot, the + margin (notional / leverage) on derivatives.""" if not bot_names: return 0.0 - rows = db.query(Position.entry_price, Position.amount, Position.symbol).filter( + rows = db.query(Position.entry_price, Position.amount, Position.symbol, Position.leverage).filter( Position.bot_name.in_(list(bot_names)), Position.status == "open", Position.mode.in_(list(modes))).all() - return sum((r[0] or 0.0) * (r[1] or 0.0) for r in rows if (r[2] or "").replace('-', '/').upper().endswith('/' + quote)) + total = 0.0 + for entry, amount, sym, lev in rows: + if cash_currency(sym) != quote: + continue + cost = (entry or 0.0) * (amount or 0.0) + if is_derivative(sym) and lev and lev > 1: + cost /= lev + total += cost + return total def forward_pool(db, bot, quote): @@ -198,10 +228,20 @@ def live_allocation(db, bot, quote, free_balance): return max(bot_total - deployed_bot, 0.0), wallet_total, bot_total -def calculate_trade_amount(current_price, bot_settings, current_equity=None): +def calculate_trade_amount(current_price, bot_settings, current_equity=None, leverage=1): + """Base amount to buy. `amount_value` is what the bot puts up (percentage + of its pool, or a fixed cash amount); on derivatives that is the margin, + so the notional — and the returned amount — is `leverage` times bigger. + Spot callers never pass leverage and get the pre-existing sizing.""" if not current_price or current_price <= 0: logger.warning("Invalid current_price (%s), cannot calculate trade amount", current_price) return None + try: + leverage = float(leverage or 1) + except (TypeError, ValueError): + leverage = 1.0 + if leverage < 1: + leverage = 1.0 entry_settings = bot_settings.get("trade_settings", {}).get("entry", {}) amount_type = entry_settings.get("amount_type", "percentage") @@ -214,6 +254,8 @@ def calculate_trade_amount(current_price, bot_settings, current_equity=None): if amount_type == "fixed": trade_amount = amount_value / current_price + if leverage != 1.0: + trade_amount *= leverage return max(trade_amount, 0.0001) else: capital = current_equity if current_equity is not None else _num(bot_settings.get("backtest_capital"), 1000) @@ -221,4 +263,6 @@ def calculate_trade_amount(current_price, bot_settings, current_equity=None): return None investment = capital * (amount_value / 100) trade_amount = investment / current_price + if leverage != 1.0: + trade_amount *= leverage return max(trade_amount, 0.0001) diff --git a/backend/engine/startup.py b/backend/engine/startup.py index 13a0f7e..fff7d79 100644 --- a/backend/engine/startup.py +++ b/backend/engine/startup.py @@ -18,6 +18,7 @@ from backend.engine.evaluator import NodeEvaluator from backend.engine import backtest from backend.engine.sizing import _num, _int, _tf_seconds, _naive_utc, backtest_pin, data_fingerprint +from backend.engine.symbols import DEFAULT_MARGIN_MODE, base_of, cash_currency, is_derivative, leverage_for, market_type_for from backend.core.exchange_registry import get_exchange_timeframes from backend.core import bot_log_buffer as blb @@ -377,14 +378,15 @@ def _latest_candle_ts(): blb.push(bot.name, "WARN", f"Not listed on {api_key.exchange.upper()}{' demo' if api_key.is_sandbox else ''} for key '{api_key.name}': {', '.join(_missing)} — {live_mode} entries on these pairs will be skipped") _tokens = [] for _p in _pairs: - for _t in _p.split('/'): + # Derivatives hold no base coin: only the settle currency matters + for _t in ([cash_currency(_p)] if is_derivative(_p) else [base_of(_p), cash_currency(_p)]): if _t not in _tokens: _tokens.append(_t) def _free(t): v = _bal.get(t) return float((v or {}).get("free") or 0) if isinstance(v, dict) else float((_bal.get("free") or {}).get(t) or 0) _parts = [f"{_free(t):,.4f}".rstrip('0').rstrip('.') + f" {t}" for t in _tokens] - _quote = _pairs[0].split('/')[-1] if _pairs else "USDT" + _quote = cash_currency(_pairs[0]) if _pairs else "USDT" _pool, _wallet_total, _bot_total = engine._live_allocation(db, bot, _quote, _free(_quote)) _pct = _num(bot.settings.get("live_allocation_pct"), 100) blb.push(bot.name, "INFO", f"Wallet '{api_key.name}' ({live_mode}): {', '.join(_parts)} free — this bot: {_pct:.0f}% = {_bot_total:,.2f} {_quote} ({_pool:,.2f} still deployable)") @@ -407,7 +409,12 @@ def _free(t): # coins that are no longer there if _bal is not None: try: - _problems = engine._reconcile_positions_with_wallet(db, bot, _ccxt, _bal, live_mode) + if market_type_for(bot.settings, api_key) != "spot": + # A perp position is not a coin in the wallet: compare + # against the exchange's open positions instead + _problems = engine._reconcile_positions_with_exchange(db, bot, _ccxt, live_mode) + else: + _problems = engine._reconcile_positions_with_wallet(db, bot, _ccxt, _bal, live_mode) except Exception as _exc: logger.warning("Bot '%s': position reconciliation failed: %s", bot.name, _exc) _problems = [] @@ -418,6 +425,27 @@ def _free(t): db.commit() return + # Derivatives: confirm leverage and margin mode on the exchange + # for every pair before the first order — a bot that cannot set + # what it backtested with must not trade + _mt = market_type_for(bot.settings, api_key) + if _mt != "spot": + _lev = leverage_for(bot.settings, _mt) + _mm = bot.settings.get("margin_mode") or DEFAULT_MARGIN_MODE + try: + _ccxt = engine._get_ccxt_instance(api_key) + for _p in [str(s).replace('-', '/').upper() for s in symbols]: + if _ccxt.markets and _p not in _ccxt.markets: + continue + engine._ensure_leverage(_ccxt, api_key, _p, _lev, _mm, bot.name) + except Exception as _exc: + logger.error("Bot '%s': could not apply leverage: %s", bot.name, _exc) + blb.push(bot.name, "ERROR", f"Could not set {_lev:g}x {_mm} on {api_key.exchange.upper()}: {_exc}") + engine._engine_stop(bot, db, f"Could not set {_lev:g}x {_mm} leverage on the exchange: {_exc}") + db.commit() + return + blb.push(bot.name, "INFO", f"Perpetual swap ({live_mode}): {_lev:g}x {_mm} confirmed on {api_key.exchange.upper()} for {len(symbols)} pair(s) — funding payments are not modelled") + # Make the backtest→live handover visible in the console: the next # tick only arrives when the current candle closes on the exchange try: diff --git a/backend/engine/symbols.py b/backend/engine/symbols.py new file mode 100644 index 0000000..d26afc5 --- /dev/null +++ b/backend/engine/symbols.py @@ -0,0 +1,73 @@ +"""Unified-symbol helpers shared by the engine and the routers. + +ccxt symbols are ``BASE/QUOTE`` for spot and ``BASE/QUOTE:SETTLE`` for +derivatives (``BTC/USDT:USDT`` = the USDT-margined BTC perpetual). Every +place that used to split on ``/`` goes through here, so a derivative symbol +never yields ``"USDT:USDT"`` as its quote currency. For spot symbols the +results are exactly what the old ``split('/')`` produced. +""" + +MARKET_TYPES = ("spot", "swap") +DEFAULT_MARKET_TYPE = "spot" +DEFAULT_MARGIN_MODE = "isolated" +MARGIN_MODES = ("isolated", "cross") + + +def normalize(symbol) -> str: + """``btc-usdt`` / ``BTC-USDT:USDT`` → ``BTC/USDT`` / ``BTC/USDT:USDT``.""" + return str(symbol or "").replace('-', '/').upper() + + +def is_derivative(symbol) -> bool: + """True for the ``BASE/QUOTE:SETTLE`` form.""" + return ':' in str(symbol or "") + + +def market_type_of(symbol) -> str: + return "swap" if is_derivative(symbol) else "spot" + + +def base_of(symbol) -> str: + return normalize(symbol).split('/')[0] + + +def quote_of(symbol) -> str: + """Quote currency: ``BTC/USDT`` → ``USDT``, ``BTC/USDT:USDT`` → ``USDT``.""" + sym = normalize(symbol) + rest = sym.split('/')[-1] if '/' in sym else sym + return rest.split(':')[0] + + +def settle_of(symbol): + """Settlement currency of a derivative symbol, None for spot.""" + sym = normalize(symbol) + return sym.split(':')[-1] if ':' in sym else None + + +def cash_currency(symbol) -> str: + """The currency a position is funded in and PnL is paid in: the settle + currency for derivatives, the quote currency for spot.""" + return settle_of(symbol) or quote_of(symbol) + + +def market_type_for(settings: dict, key_record=None) -> str: + """Market type a bot trades on. A linked API key is bound to one market + type and wins (the validator refuses a mismatch on save); otherwise the + bot's own setting, defaulting to spot for every pre-existing bot.""" + if key_record is not None: + kt = getattr(key_record, "market_type", None) + if kt: + return str(kt).lower() + return str((settings or {}).get("market_type") or DEFAULT_MARKET_TYPE).lower() + + +def leverage_for(settings: dict, market_type: str | None = None) -> float: + """Configured leverage; always 1 on spot whatever the setting says.""" + mt = market_type or market_type_for(settings) + if mt == "spot": + return 1.0 + try: + lev = float((settings or {}).get("leverage") or 1) + except (TypeError, ValueError): + lev = 1.0 + return lev if lev >= 1 else 1.0 diff --git a/backend/main.py b/backend/main.py index 12e426c..6b1b57c 100644 --- a/backend/main.py +++ b/backend/main.py @@ -32,7 +32,7 @@ # Import the background services from backend.engine.candle_poller import candle_poller from backend.engine.bot_manager import bot_manager -from backend.core.exchange_registry import build_exchange +from backend.core.exchange_registry import build_exchange_for_symbol # Create database tables and run migrations for existing DBs Base.metadata.create_all(bind=engine) @@ -105,8 +105,8 @@ def read_root(): @app.get("/api/price/{symbol}", dependencies=[Depends(verify_api_key)]) def get_price(symbol: str, exchange: str = Query(default="okx")): try: - exch = build_exchange(exchange.lower()) formatted_symbol = symbol.replace('-', '/').upper() + exch = build_exchange_for_symbol(exchange.lower(), formatted_symbol) ticker = exch.fetch_ticker(formatted_symbol) return { "exchange": exchange.upper(), diff --git a/backend/models/exchange_keys.py b/backend/models/exchange_keys.py index ddd43c3..1933dfa 100644 --- a/backend/models/exchange_keys.py +++ b/backend/models/exchange_keys.py @@ -12,4 +12,7 @@ class ExchangeKey(Base): api_secret = Column(String, nullable=False) passphrase = Column(String, nullable=False) is_sandbox = Column(Boolean, default=True) + # One key = one market type ("spot" | "swap"): spot and perpetual + # accounts are separate ccxt instances (and on some exchanges separate keys) + market_type = Column(String, default="spot") created_at = Column(DateTime, default=lambda: datetime.now(timezone.utc)) diff --git a/backend/models/orders.py b/backend/models/orders.py index 9e4e0ec..cc52c1c 100644 --- a/backend/models/orders.py +++ b/backend/models/orders.py @@ -25,6 +25,8 @@ class Order(Base): fee = Column(Float, nullable=True) status = Column(String, default="open") # "open", "filled", "canceled", "rejected" + market_type = Column(String, default="spot") # "spot" | "swap" + reduce_only = Column(Integer, default=0) # 1 for derivative close orders timestamp = Column(DateTime, default=lambda: datetime.now(timezone.utc)) diff --git a/backend/models/positions.py b/backend/models/positions.py index 938e7c4..9927045 100644 --- a/backend/models/positions.py +++ b/backend/models/positions.py @@ -27,6 +27,11 @@ class Position(Base): closed_at = Column(DateTime, nullable=True) highest_price = Column(Float, nullable=True) + # Derivatives: "swap" positions carry leverage (margin = entry x amount / + # leverage) and the ccxt contract count behind `amount` (base units) + market_type = Column(String, default="spot") + leverage = Column(Float, default=1.0) + contracts = Column(Float, nullable=True) triggered_exits = Column(JSON, nullable=True, default=list) orders = relationship("Order", back_populates="position", cascade="all, delete-orphan") diff --git a/backend/routers/bots.py b/backend/routers/bots.py index db58f5a..db3505c 100644 --- a/backend/routers/bots.py +++ b/backend/routers/bots.py @@ -22,7 +22,7 @@ from backend.engine.bot_manager import bot_manager, _config_fingerprint from backend.engine.sizing import backtest_pin from backend.engine.data_verify import verify_window -from backend.core.exchange_registry import build_exchange +from backend.core.exchange_registry import build_exchange_for_symbol from backend.core import bot_log_buffer as blb from backend.models.bot_logs import BotLog from backend.models.bot_config_runs import BotConfigRun @@ -39,6 +39,17 @@ def _resolve_exchange(settings: dict, db: Session) -> str: return settings.get("data_exchange", "okx") +def _resolve_key_market_type(settings: dict, db: Session) -> str | None: + """Market type of the linked API key (None when no key is linked); a key + is bound to one market, so the validator refuses a bot on another.""" + api_key_name = settings.get("api_key_name") + if api_key_name: + key = db.query(ExchangeKey).filter(ExchangeKey.name == api_key_name).first() + if key: + return getattr(key, "market_type", None) or "spot" + return None + + def _candle_exchange(settings: dict, key_exchange_by_name: dict) -> str: """The exchange this bot's candles (and therefore its signals) come from — the same rule as engine startup: an API key that actually routes orders @@ -150,6 +161,9 @@ def get_bots_summary(db: Session = Depends(get_db)): # Sizing fields for the Analytics capital-allocation panel "max_positions": b.settings.get("max_positions", 1) if b.settings else 1, "max_order_value": b.settings.get("max_order_value") if b.settings else None, + # Phase 2: swap bots carry leverage (chips on card/Analytics/Home) + "market_type": (b.settings.get("market_type") or "spot") if b.settings else "spot", + "leverage": b.settings.get("leverage", 1) if b.settings else 1, "live_allocation_pct": b.settings.get("live_allocation_pct", 100) if b.settings else 100, "live_starting_capital": b.settings.get("live_starting_capital") if b.settings else None, "entry_amount_type": (b.settings.get("trade_settings") or {}).get("entry", {}).get("amount_type", "percentage") if b.settings else "percentage", @@ -218,7 +232,7 @@ def create_bot(bot_in: BotCreate, db: Session = Depends(get_db)): raise HTTPException(status_code=400, detail="A bot with this name already exists.") resolved_exchange = _resolve_exchange(bot_in.settings, db) - validation = validate_bot_settings(bot_in.settings, exchange_id=resolved_exchange) + validation = validate_bot_settings(bot_in.settings, exchange_id=resolved_exchange, key_market_type=_resolve_key_market_type(bot_in.settings, db)) if validation["errors"]: raise HTTPException(status_code=400, detail={"validation_errors": validation["errors"]}) @@ -264,7 +278,7 @@ def import_bot(payload: dict = Body(...), db: Session = Depends(get_db)): if not isinstance(bot_settings, dict): raise HTTPException(status_code=400, detail="Invalid file: bot settings must be an object.") resolved_exchange = _resolve_exchange(bot_settings, db) - validation = validate_bot_settings(bot_settings, exchange_id=resolved_exchange) + validation = validate_bot_settings(bot_settings, exchange_id=resolved_exchange, key_market_type=_resolve_key_market_type(bot_settings, db)) if validation["errors"]: raise HTTPException(status_code=400, detail={"validation_errors": validation["errors"]}) @@ -322,7 +336,7 @@ def update_bot(bot_id: int, background_tasks: BackgroundTasks, bot_data: dict = current_settings[key] = value resolved_exchange = _resolve_exchange(current_settings, db) - validation = validate_bot_settings(current_settings, exchange_id=resolved_exchange) + validation = validate_bot_settings(current_settings, exchange_id=resolved_exchange, key_market_type=_resolve_key_market_type(current_settings, db)) if validation["errors"]: raise HTTPException(status_code=400, detail={"validation_errors": validation["errors"]}) validation_warnings = validation["warnings"] @@ -663,9 +677,8 @@ async def verify_bot_data(bot_id: int, accept: bool = Query(default=False), db: def _run(): _db = SessionLocal() try: - exch = build_exchange(exchange_id) - return [verify_window(_db, exch, exchange_id, str(s).replace('-', '/').upper(), timeframe, start, end, accept=accept) - for s in symbols] + return [verify_window(_db, build_exchange_for_symbol(exchange_id, sym), exchange_id, sym, timeframe, start, end, accept=accept) + for sym in (str(s).replace('-', '/').upper() for s in symbols)] finally: _db.close() try: diff --git a/backend/routers/data.py b/backend/routers/data.py index b2c4eb3..8de0dfd 100644 --- a/backend/routers/data.py +++ b/backend/routers/data.py @@ -12,7 +12,7 @@ from backend.core.database import get_db, SessionLocal from backend.models.candles import Candle from backend.core.security import verify_api_key -from backend.core.exchange_registry import build_exchange, SUPPORTED_EXCHANGES, get_exchange_timeframes, get_exchange_symbols +from backend.core.exchange_registry import build_exchange_for_symbol, SUPPORTED_EXCHANGES, get_exchange_timeframes, get_exchange_symbols, market_caps from backend.engine.data_verify import verify_window logger = logging.getLogger("apexalgo.data") @@ -39,15 +39,19 @@ async def get_timeframes(exchange_id: str): @router.get("/symbols/{exchange_id}") -async def get_symbols(exchange_id: str): - """Tradeable spot symbols on an exchange — the builder validates the - whitelist against this. ``symbols`` is empty (and ``known`` false) when the - exchange could not be reached, so callers do not reject every pair.""" +async def get_symbols(exchange_id: str, market_type: str = Query(default="spot")): + """Tradeable symbols on an exchange (spot pairs, or linear perpetuals with + ``?market_type=swap``) — the builder validates the whitelist against this. + ``symbols`` is empty (and ``known`` false) when the exchange could not be + reached, so callers do not reject every pair.""" exchange_id = exchange_id.lower() + market_type = (market_type or "spot").lower() if exchange_id not in SUPPORTED_EXCHANGES: raise HTTPException(status_code=400, detail=f"Unknown exchange '{exchange_id}'.") - symbols = await asyncio.to_thread(get_exchange_symbols, exchange_id) - return {"exchange": exchange_id, "symbols": symbols, "known": bool(symbols)} + if market_caps(exchange_id, market_type) is None: + raise HTTPException(status_code=400, detail=f"{SUPPORTED_EXCHANGES[exchange_id]} has no '{market_type}' market in ApexAlgo.") + symbols = await asyncio.to_thread(get_exchange_symbols, exchange_id, market_type) + return {"exchange": exchange_id, "market_type": market_type, "symbols": symbols, "known": bool(symbols)} class HistoricalDataFetch(BaseModel): @@ -82,7 +86,7 @@ async def verify_stored_candles(req: VerifyRequest): def _run(): db = SessionLocal() try: - return verify_window(db, build_exchange(exchange_id), exchange_id, symbol, req.timeframe, + return verify_window(db, build_exchange_for_symbol(exchange_id, symbol), exchange_id, symbol, req.timeframe, req.start_date, req.end_date, accept=req.accept) finally: db.close() @@ -127,7 +131,7 @@ async def fetch_historical_data( def _fetch_and_save_data(formatted_symbol: str, exchange_id: str, req: HistoricalDataFetch): - exch = build_exchange(exchange_id) + exch = build_exchange_for_symbol(exchange_id, formatted_symbol) start_ts = int(req.start_date.timestamp() * 1000) end_ts = int(req.end_date.timestamp() * 1000) @@ -368,7 +372,7 @@ def get_market_info(symbol: str, exchange: str = "okx"): if cached and (now - cached[0]) < _TICKER_TTL: return cached[1] - exch = build_exchange(exchange_id) + exch = build_exchange_for_symbol(exchange_id, formatted_symbol) ticker = exch.fetch_ticker(formatted_symbol) result = { "symbol": formatted_symbol, diff --git a/backend/routers/keys.py b/backend/routers/keys.py index 9036d04..c93cb20 100644 --- a/backend/routers/keys.py +++ b/backend/routers/keys.py @@ -16,8 +16,9 @@ from backend.core.encryption import encrypt_data from backend.core.exchange_registry import ( build_exchange, get_authenticated_exchange, invalidate_authenticated_exchange, - exchange_has_sandbox, EXCHANGES, SUPPORTED_EXCHANGES, + exchange_has_sandbox, EXCHANGES, SUPPORTED_EXCHANGES, market_caps, ) +from backend.engine.symbols import DEFAULT_MARKET_TYPE, MARKET_TYPES logger = logging.getLogger("apexalgo.keys") @@ -34,13 +35,16 @@ class ExchangeKeyCreate(BaseModel): api_secret: str passphrase: str = "" is_sandbox: bool = True + # A key is bound to one market: spot keys and swap keys are separate records + market_type: str = DEFAULT_MARKET_TYPE @router.get("/exchanges") def list_exchanges(): """Capabilities per supported exchange (from the registry spec) for the connection form, the data manager and the builder. `has_sandbox` is - probed from ccxt so it tracks the installed version.""" + probed from ccxt so it tracks the installed version. `markets` lists the + market types ApexAlgo can trade on the exchange with their limits.""" out = [] for ex_id, spec in EXCHANGES.items(): out.append({ @@ -50,6 +54,15 @@ def list_exchanges(): "has_sandbox": exchange_has_sandbox(ex_id), "keys_url": spec.keys_url, "sandbox_note": spec.sandbox_note, + "markets": { + mt: { + "has_sandbox": exchange_has_sandbox(ex_id, mt), + "max_leverage": caps.max_leverage, + "leverage_in_order": caps.leverage_in_order, + "note": caps.note, + } + for mt, caps in spec.markets.items() + }, }) return out @@ -59,6 +72,19 @@ def save_exchange_keys(req: ExchangeKeyCreate, db: Session = Depends(get_db)): exchange_id = req.exchange.lower() if exchange_id not in SUPPORTED_EXCHANGES: raise HTTPException(status_code=400, detail=f"Unsupported exchange '{exchange_id}'.") + market_type = (req.market_type or DEFAULT_MARKET_TYPE).strip().lower() + if market_type not in MARKET_TYPES: + raise HTTPException(status_code=400, detail=f"Invalid market_type '{req.market_type}'. Use one of: {', '.join(MARKET_TYPES)}.") + if market_caps(exchange_id, market_type) is None: + raise HTTPException(status_code=400, detail=f"{SUPPORTED_EXCHANGES[exchange_id]} has no '{market_type}' market in ApexAlgo.") + # Bots keep trading the market their key was verified on: re-saving a + # key under another market type is refused while bots still reference it + existing = db.query(ExchangeKey).filter(ExchangeKey.name == req.name).first() + if existing and (getattr(existing, "market_type", None) or DEFAULT_MARKET_TYPE) != market_type: + linked = [b_name for b_name, b_settings in db.query(BotConfig.name, BotConfig.settings).all() + if (b_settings or {}).get("api_key_name") == req.name] + if linked: + raise HTTPException(status_code=409, detail=f"Key '{req.name}' is a {existing.market_type or DEFAULT_MARKET_TYPE} key linked to {', '.join(linked)}; save the {market_type} key under a new name.") try: test_exchange = build_exchange( exchange_id, @@ -66,6 +92,7 @@ def save_exchange_keys(req: ExchangeKeyCreate, db: Session = Depends(get_db)): api_secret=req.api_secret, passphrase=req.passphrase or None, sandbox=req.is_sandbox, + market_type=market_type, ) test_exchange.fetch_balance() except Exception as e: @@ -77,14 +104,13 @@ def save_exchange_keys(req: ExchangeKeyCreate, db: Session = Depends(get_db)): enc_secret = encrypt_data(req.api_secret) enc_passphrase = encrypt_data(req.passphrase) - existing = db.query(ExchangeKey).filter(ExchangeKey.name == req.name).first() - if existing: existing.api_key = enc_key existing.api_secret = enc_secret existing.passphrase = enc_passphrase existing.is_sandbox = req.is_sandbox existing.exchange = req.exchange + existing.market_type = market_type else: new_key = ExchangeKey( name=req.name, @@ -92,7 +118,8 @@ def save_exchange_keys(req: ExchangeKeyCreate, db: Session = Depends(get_db)): api_key=enc_key, api_secret=enc_secret, passphrase=enc_passphrase, - is_sandbox=req.is_sandbox + is_sandbox=req.is_sandbox, + market_type=market_type, ) db.add(new_key) @@ -149,6 +176,7 @@ def get_exchange_keys_status(db: Session = Depends(get_db)): "name": k.name, "exchange": k.exchange, "is_sandbox": k.is_sandbox, + "market_type": getattr(k, "market_type", None) or DEFAULT_MARKET_TYPE, "is_active": is_active, "error_msg": error_msg, "latency_ms": latency_ms, diff --git a/backend/routers/trades.py b/backend/routers/trades.py index 05935be..1b3bc1d 100644 --- a/backend/routers/trades.py +++ b/backend/routers/trades.py @@ -18,6 +18,7 @@ from backend.core.security import verify_api_key from backend.core.exchange_registry import get_authenticated_exchange from backend.engine.bot_manager import bot_manager +from backend.engine.symbols import is_derivative logger = logging.getLogger("apexalgo.trades") @@ -307,13 +308,22 @@ def _execute_live_close(pos: Position, db: Session): ccxt_symbol = pos.symbol.replace('-', '/').upper() close_side = "sell" if pos.side == "long" else "buy" + derivative = is_derivative(ccxt_symbol) try: exchange = get_authenticated_exchange(key_record) - close_qty = float(exchange.amount_to_precision(ccxt_symbol, pos.amount)) + if derivative: + # Derivatives: amount in contracts, reduce-only so a stale record + # can never open the opposite position + close_qty = float(exchange.amount_to_precision(ccxt_symbol, bot_manager._to_contracts(exchange, ccxt_symbol, pos.amount))) + else: + close_qty = float(exchange.amount_to_precision(ccxt_symbol, pos.amount)) if close_qty <= 0: raise HTTPException(status_code=400, detail="Position amount rounds to zero at exchange precision; cannot place a close order.") - exch_order = exchange.create_order(ccxt_symbol, "market", close_side, close_qty) + if derivative: + exch_order = exchange.create_order(ccxt_symbol, "market", close_side, close_qty, None, {"reduceOnly": True}) + else: + exch_order = exchange.create_order(ccxt_symbol, "market", close_side, close_qty) except HTTPException: raise except Exception as e: @@ -326,6 +336,8 @@ def _execute_live_close(pos: Position, db: Session): order_id = exch_order.get("id") order_status = exch_order.get("status") filled_amount = float(exch_order.get("filled") or 0) + if derivative and filled_amount > 0: + filled_amount = bot_manager._from_contracts(exchange, ccxt_symbol, filled_amount) if filled_amount <= 0: if order_status not in ("closed", "canceled", "rejected", "expired") and order_id: @@ -379,6 +391,8 @@ def _record_unfilled_close_order(db: Session, pos: Position, close_side: str, or timestamp=datetime.now(timezone.utc), exchange_order_id=order_id, status=order_status or "unknown", + market_type=pos.market_type or "spot", + reduce_only=1 if is_derivative(pos.symbol) else 0, )) db.commit() except Exception as exc: @@ -443,7 +457,9 @@ def close_position_now(pos: Position, db: Session) -> float: fee=exit_fee, timestamp=datetime.now(timezone.utc), exchange_order_id=exchange_order_id, - status="filled" + status="filled", + market_type=pos.market_type or "spot", + reduce_only=1 if is_derivative(pos.symbol) else 0, ) db.add(close_order) db.commit() diff --git a/frontend/src/api/exchanges.js b/frontend/src/api/exchanges.js index 0541e6b..b7d20f4 100644 --- a/frontend/src/api/exchanges.js +++ b/frontend/src/api/exchanges.js @@ -5,24 +5,32 @@ // (key form, data manager, builder) without touching the frontend. // // Shape of each entry: -// { id, name, needs_passphrase, has_sandbox, keys_url, sandbox_note } +// { id, name, needs_passphrase, has_sandbox, keys_url, sandbox_note, +// markets: { spot: {has_sandbox, max_leverage, leverage_in_order, note}, swap?: {...} } } import { useEffect, useSyncExternalStore } from 'react'; import { apiClient } from './client'; +// Market types per exchange mirror backend ExchangeSpec.markets (spot always; +// swap = USDT-margined perpetuals where ApexAlgo supports them) +const SPOT = { spot: { has_sandbox: false, max_leverage: 1 } }; +const fb = (id, name, needs_passphrase, has_sandbox, swap) => ({ + id, name, needs_passphrase, has_sandbox, + markets: swap ? { spot: { has_sandbox, max_leverage: 1 }, swap: { has_sandbox: swap.has_sandbox, max_leverage: swap.max_leverage || 10 } } : { spot: { ...SPOT.spot, has_sandbox } }, +}); export const FALLBACK_EXCHANGES = [ - { id: 'okx', name: 'OKX', needs_passphrase: true, has_sandbox: true }, - { id: 'binance', name: 'Binance', needs_passphrase: false, has_sandbox: true }, - { id: 'bitvavo', name: 'Bitvavo', needs_passphrase: false, has_sandbox: false }, - { id: 'coinbase', name: 'Coinbase', needs_passphrase: false, has_sandbox: false }, - { id: 'cryptocom', name: 'Crypto.com', needs_passphrase: false, has_sandbox: true }, - { id: 'kraken', name: 'Kraken', needs_passphrase: false, has_sandbox: false }, - { id: 'kucoin', name: 'KuCoin', needs_passphrase: true, has_sandbox: false }, - { id: 'bybit', name: 'Bybit', needs_passphrase: false, has_sandbox: true }, - { id: 'gateio', name: 'Gate', needs_passphrase: false, has_sandbox: true }, - { id: 'bitget', name: 'Bitget', needs_passphrase: true, has_sandbox: true }, - { id: 'mexc', name: 'MEXC', needs_passphrase: false, has_sandbox: false }, - { id: 'htx', name: 'HTX', needs_passphrase: false, has_sandbox: false }, - { id: 'bingx', name: 'BingX', needs_passphrase: false, has_sandbox: true }, + fb('okx', 'OKX', true, true, { has_sandbox: true }), + fb('binance', 'Binance', false, true, { has_sandbox: true }), + fb('bitvavo', 'Bitvavo', false, false), + fb('coinbase', 'Coinbase', false, false), + fb('cryptocom', 'Crypto.com', false, true), + fb('kraken', 'Kraken', false, false, { has_sandbox: true, max_leverage: 5 }), + fb('kucoin', 'KuCoin', true, false, { has_sandbox: false }), + fb('bybit', 'Bybit', false, true, { has_sandbox: true }), + fb('gateio', 'Gate', false, true, { has_sandbox: true }), + fb('bitget', 'Bitget', true, true, { has_sandbox: true }), + fb('mexc', 'MEXC', false, false), + fb('htx', 'HTX', false, false, { has_sandbox: false }), + fb('bingx', 'BingX', false, true, { has_sandbox: true }), ]; let cache = null; @@ -59,3 +67,11 @@ export function exchangeName(id) { const hit = snapshot().find((e) => e.id === id); return hit ? hit.name : String(id || '').toUpperCase(); } + +/** Market capabilities of `exchangeId` for `marketType` ('spot' | 'swap'); null when unsupported. */ +export function marketCaps(exchangeId, marketType = 'spot') { + const hit = snapshot().find((e) => e.id === exchangeId); + if (!hit) return null; + const markets = hit.markets || { spot: { has_sandbox: !!hit.has_sandbox, max_leverage: 1 } }; + return markets[marketType] || null; +} diff --git a/frontend/src/components/BotManagerUI.jsx b/frontend/src/components/BotManagerUI.jsx index 4f350b1..571d829 100644 --- a/frontend/src/components/BotManagerUI.jsx +++ b/frontend/src/components/BotManagerUI.jsx @@ -276,6 +276,11 @@ const BotCard = memo(function BotCard({ bot, index, busyAction, togglingBot, ope : Stopped} {isBacktestOn && + Backtest} + {bot.settings?.market_type === 'swap' && ( + + Perps {Number(bot.settings?.leverage) || 1}× + + )}
{/* Metrics row */} diff --git a/frontend/src/components/Builder/BotBuilder.jsx b/frontend/src/components/Builder/BotBuilder.jsx index 08fd38c..be18e8c 100644 --- a/frontend/src/components/Builder/BotBuilder.jsx +++ b/frontend/src/components/Builder/BotBuilder.jsx @@ -105,7 +105,10 @@ function rebuildLayoutFromSettings(settings, updateNodeData, deleteNode) { nodes.push({ id: 'rebuilt_apikey', type: 'apiKey', position: { x: ctxX, y: ctxY }, data: { onChange: updateNodeData, onDelete: deleteNode, apiKeyName: settings.api_key_name || null, - dataExchange: settings.data_exchange || 'okx' } }); + dataExchange: settings.data_exchange || 'okx', + marketType: settings.market_type || 'spot', + leverage: settings.leverage ?? 1, + marginMode: settings.margin_mode || 'isolated' } }); } // ── AREA 2: Strategy logic ── @@ -314,7 +317,7 @@ const BotBuilderFlow = ({ closeBuilder, editingBot }) => { { id: getId(), type: 'botConfig', position: { x: 50, y: 50 }, data: { onChange: updateNodeData, onDelete: deleteNode, botName: editingBot ? editingBot.name : 'Apex Strategy Alpha', timeframe: editingBot?.settings?.timeframe || '1m', executionMode: 'paper', maxPositions: 1, maxPositionsScope: 'per_pair', cooldownTrades: 0, cooldownCandles: 0 } }, { id: getId(), type: 'whitelist', position: { x: 470, y: 50 }, data: { onChange: updateNodeData, onDelete: deleteNode, pairs: editingBot?.settings?.symbols?.join(', ') || editingBot?.settings?.symbol || DEFAULT_PAIR } }, { id: getId(), type: 'backtest', position: { x: 470, y: 320 }, data: { onChange: updateNodeData, onDelete: deleteNode, runOnStart: true, capital: 1000, lookback: 150 } }, - { id: getId(), type: 'apiKey', position: { x: 470, y: 610 }, data: { onChange: updateNodeData, onDelete: deleteNode, apiKeyName: null, dataExchange: 'okx' } } + { id: getId(), type: 'apiKey', position: { x: 470, y: 610 }, data: { onChange: updateNodeData, onDelete: deleteNode, apiKeyName: null, dataExchange: 'okx', marketType: 'spot', leverage: 1, marginMode: 'isolated' } } ]); initRef.current = true; } @@ -334,6 +337,11 @@ const BotBuilderFlow = ({ closeBuilder, editingBot }) => { const activeApiKeyNode = nodes.find(n => n.type === 'apiKey'); const activeApiKeyName = activeApiKeyNode?.data.apiKeyName; const activeDataExchange = activeApiKeyNode?.data.dataExchange; + // Market type: the linked key decides (a key is bound to one market), + // otherwise the routing block's own choice. Swap symbols are a separate + // market list (BASE/QUOTE:SETTLE). + const activeKeyMarketType = activeApiKeyName ? (availableKeys?.find(k => k.name === activeApiKeyName)?.market_type || null) : null; + const activeMarketType = activeKeyMarketType || activeApiKeyNode?.data.marketType || 'spot'; useEffect(() => { if (!initRef.current) return; let exchange = activeDataExchange || 'okx'; @@ -345,11 +353,11 @@ const BotBuilderFlow = ({ closeBuilder, editingBot }) => { setSupportedTimeframes(res.data.timeframes); }).catch(() => setSupportedTimeframes(null)); let cancelled = false; - apiClient.get(`/api/data/symbols/${exchange}`).then(res => { - if (!cancelled) setKnownSymbols({ exchange, symbols: res.data.symbols || [], known: !!res.data.known }); - }).catch(() => { if (!cancelled) setKnownSymbols({ exchange, symbols: [], known: false }); }); + apiClient.get(`/api/data/symbols/${exchange}`, { params: { market_type: activeMarketType } }).then(res => { + if (!cancelled) setKnownSymbols({ exchange, marketType: activeMarketType, symbols: res.data.symbols || [], known: !!res.data.known }); + }).catch(() => { if (!cancelled) setKnownSymbols({ exchange, marketType: activeMarketType, symbols: [], known: false }); }); return () => { cancelled = true; }; - }, [activeApiKeyName, activeDataExchange, availableKeys]); + }, [activeApiKeyName, activeDataExchange, activeMarketType, availableKeys]); // Pass supported timeframes to config node useEffect(() => { @@ -378,16 +386,20 @@ const BotBuilderFlow = ({ closeBuilder, editingBot }) => { const liveKeySandbox = !!activeKeyRecord?.is_sandbox; const entryActionNode = nodes.find(n => n.type === 'action' && n.data.actionType === 'buy'); const entryFee = entryActionNode ? (entryActionNode.data.fee === '' || entryActionNode.data.fee === undefined ? 0.1 : Number(entryActionNode.data.fee)) : null; + const liveLeverage = activeMarketType === 'swap' ? (Number(activeApiKeyNode?.data.leverage) || 1) : 1; + const liveMarginMode = activeApiKeyNode?.data.marginMode || 'isolated'; useEffect(() => { if (!initRef.current) return; - const liveContext = { keyName: liveKeyName, exchange: liveKeyExchange, isSandbox: liveKeySandbox, entryFee }; + const liveContext = { keyName: liveKeyName, exchange: liveKeyExchange, isSandbox: liveKeySandbox, entryFee, + marketType: activeMarketType, leverage: liveLeverage, marginMode: liveMarginMode }; setNodes(nds => nds.map(n => { if (n.type !== 'botConfig') return n; const cur = n.data.liveContext; - if (cur && cur.keyName === liveContext.keyName && cur.exchange === liveContext.exchange && cur.isSandbox === liveContext.isSandbox && cur.entryFee === liveContext.entryFee) return n; + if (cur && cur.keyName === liveContext.keyName && cur.exchange === liveContext.exchange && cur.isSandbox === liveContext.isSandbox && cur.entryFee === liveContext.entryFee + && cur.marketType === liveContext.marketType && cur.leverage === liveContext.leverage && cur.marginMode === liveContext.marginMode) return n; return { ...n, data: { ...n.data, liveContext } }; })); - }, [liveKeyName, liveKeyExchange, liveKeySandbox, entryFee, setNodes]); + }, [liveKeyName, liveKeyExchange, liveKeySandbox, entryFee, activeMarketType, liveLeverage, liveMarginMode, setNodes]); // Esc asks to close (same dirty-check as the buttons); a browser reload with // unsaved work gets the native "leave page?" prompt. @@ -433,7 +445,7 @@ const BotBuilderFlow = ({ closeBuilder, editingBot }) => { const getDefaultData = useCallback((type) => { const defaultData = { onChange: updateNodeData, onDelete: deleteNode }; - if (type === 'apiKey') defaultData.availableKeys = availableKeys; + if (type === 'apiKey') { defaultData.availableKeys = availableKeys; defaultData.apiKeyName = null; defaultData.dataExchange = 'okx'; defaultData.marketType = 'spot'; defaultData.leverage = 1; defaultData.marginMode = 'isolated'; } if (type === 'indicator') { defaultData.indicator = 'rsi'; defaultData.params = {length: 14}; defaultData.outputIdx = 0; } if (type === 'priceData') { defaultData.priceType = 'close'; defaultData.offset = 0; } if (type === 'condition') { defaultData.operator = '>'; defaultData.rightValue = ''; } @@ -549,13 +561,25 @@ const BotBuilderFlow = ({ closeBuilder, editingBot }) => { const apiKeyName = apiKeyNode?.data.apiKeyName || null; const apiKeyRecord = apiKeyName ? availableKeys?.find(k => k.name === apiKeyName) : null; const dataExchange = apiKeyRecord?.exchange || apiKeyNode?.data.dataExchange || 'okx'; + // The key's market wins; leverage/margin mode only matter on swaps + // (the backend strips the defaults from the config fingerprint) + const marketType = apiKeyRecord?.market_type || apiKeyNode?.data.marketType || 'spot'; + const leverage = marketType === 'swap' ? (parseInt(apiKeyNode?.data.leverage) || 1) : 1; + const marginMode = marketType === 'swap' ? (apiKeyNode?.data.marginMode || 'isolated') : 'isolated'; if (!configNode) return showError("Missing 'Main Configuration' block."); if (!whitelistNode) return showError("Missing 'Asset Whitelist' block."); const symbolsList = parsePairs(whitelistNode.data.pairs); if (symbolsList.length === 0) return showError("Whitelist must contain at least one pair."); - if (knownSymbols?.known && knownSymbols.exchange === dataExchange) { + if (marketType === 'swap') { + const spotPairs = symbolsList.filter(sym => !sym.includes(':')); + if (spotPairs.length) return showError(`Perpetual swaps use the BASE/QUOTE:SETTLE form (e.g. BTC/USDT:USDT); fix ${spotPairs.join(', ')} in the whitelist.`); + } else { + const swapPairs = symbolsList.filter(sym => sym.includes(':')); + if (swapPairs.length) return showError(`${swapPairs.join(', ')} are perpetual swaps; a spot bot needs BASE/QUOTE pairs (or pick a perps market in the Exchange Routing block).`); + } + if (knownSymbols?.known && knownSymbols.exchange === dataExchange && (knownSymbols.marketType || 'spot') === marketType) { const listed = new Set(knownSymbols.symbols); const unknown = symbolsList.filter(sym => !listed.has(sym)); if (unknown.length) return showError(`Not listed on ${dataExchange.toUpperCase()}: ${unknown.join(', ')}. Fix the whitelist before saving.`); @@ -598,6 +622,9 @@ const BotBuilderFlow = ({ closeBuilder, editingBot }) => { backtest_lookback: backtestNode ? (backtestNode.data.lookback || 150) : 150, api_key_name: apiKeyName, data_exchange: dataExchange, + market_type: marketType, + leverage, + margin_mode: marginMode, trade_settings: {}, nodes: {}, ui_layout: { diff --git a/frontend/src/components/Builder/CustomNodes.jsx b/frontend/src/components/Builder/CustomNodes.jsx index 7d76711..9ad2896 100644 --- a/frontend/src/components/Builder/CustomNodes.jsx +++ b/frontend/src/components/Builder/CustomNodes.jsx @@ -1,7 +1,7 @@ import React from 'react'; import { Handle, Position } from 'reactflow'; import { useIndicators } from './indicatorConfig'; -import { useExchanges } from '../../api/exchanges'; +import { useExchanges, marketCaps } from '../../api/exchanges'; import { DEFAULT_PAIR, parsePairs } from './pairs'; // All known timeframes with display labels @@ -115,9 +115,10 @@ export const BotConfigNode = ({ id, data }) => ( ); // Execution mode + go-live checklist. `data.liveContext` is pushed in by the -// builder: { keyName, exchange, isSandbox, entryFee } — null key = no -// exchange orders possible, so the "exchange" option is disabled rather than -// silently saving a bot that would fall back to forward test at start. +// builder: { keyName, exchange, isSandbox, entryFee, marketType, leverage, +// marginMode } — null key = no exchange orders possible, so the "exchange" +// option is disabled rather than silently saving a bot that would fall back +// to forward test at start. const ExecutionModeSection = ({ id, data }) => { const ctx = data.liveContext || {}; const hasKey = !!ctx.keyName; @@ -126,10 +127,16 @@ const ExecutionModeSection = ({ id, data }) => { const isLiveMoney = wantsExchange && hasKey && !ctx.isSandbox; const cap = Number(data.maxOrderValue) || 0; const fee = Number(ctx.entryFee); + const isSwap = ctx.marketType === 'swap'; + const lev = isSwap ? (Number(ctx.leverage) || 1) : 1; + // Isolated-margin liquidation sits roughly one margin's worth of adverse + // move away: −100%/leverage on the notional (maintenance margin ignored) + const liqPct = Math.round(100 / lev); const checks = [ { ok: hasKey, label: hasKey ? `Key "${ctx.keyName}" on ${String(ctx.exchange || '').toUpperCase()} (${ctx.isSandbox ? 'sandbox → paper' : 'real → live'})` : 'Select an API key in the Exchange Routing block' }, - { ok: cap > 0, label: cap > 0 ? `Max order value $${cap} caps every order` : 'Set Max Order Value USD above (required for live orders)' }, + { ok: cap > 0, label: cap > 0 ? `Max order value $${cap} caps every order${isSwap ? ' (notional = margin × leverage)' : ''}` : `Set Max Order Value USD above (required for live orders${isSwap ? '; caps the notional, i.e. margin × leverage' : ''})` }, { ok: fee > 0, label: fee > 0 ? `Entry fee ${fee}% modelled` : 'Entry fee is 0% — the backtest ignores what the exchange will charge' }, + ...(isSwap ? [{ ok: lev <= 3, label: `Perpetuals ${lev}× ${ctx.marginMode || 'isolated'} · liquidation ≈ −${liqPct}% from entry · funding not modelled` }] : []), ]; return (
@@ -171,6 +178,10 @@ export const WhitelistNode = ({ id, data }) => { const listed = market?.known ? new Set(market.symbols) : null; const unknown = listed ? pairs.filter(p => !listed.has(p)) : []; const exch = String(market?.exchange || '').toUpperCase(); + // Perpetual swaps use ccxt's BASE/QUOTE:SETTLE form; a spot pair on a swap + // bot (or vice versa) is flagged before the validator would reject it + const isSwap = market?.marketType === 'swap'; + const wrongForm = market ? pairs.filter(p => p.includes(':') !== isSwap) : []; return (
@@ -181,7 +192,7 @@ export const WhitelistNode = ({ id, data }) => {