diff --git a/backend/core/database.py b/backend/core/database.py index 5fec97d..ed9e362 100644 --- a/backend/core/database.py +++ b/backend/core/database.py @@ -77,6 +77,7 @@ def run_migrations(): close REAL, volume REAL, marketcap REAL, + fetched_at DATETIME, UNIQUE (exchange, symbol, timeframe, timestamp) ) """)) @@ -96,6 +97,9 @@ def run_migrations(): conn.execute(text("CREATE INDEX IF NOT EXISTS ix_candles_timeframe ON candles (timeframe)")) conn.execute(text("CREATE INDEX IF NOT EXISTS ix_candles_timestamp ON candles (timestamp)")) logger.info("Migration: candles table rebuilt successfully.") + elif 'fetched_at' not in can_cols: + conn.execute(text("ALTER TABLE candles ADD COLUMN fetched_at DATETIME")) + logger.info("Migration: added 'fetched_at' to candles") # ── signals — deduplicate and add unique constraint ──────────────────── if "signals" in existing_tables: @@ -141,6 +145,36 @@ def run_migrations(): conn.execute(text("CREATE INDEX IF NOT EXISTS ix_signals_name ON signals (name)")) logger.info("Migration: signals table rebuilt with unique constraint.") + # ── bot_config_runs — from "set of configs" to a run log ─────────────── + # The unique key grew from (bot, config) to (bot, config, slice, data), + # which SQLite cannot alter in place. Old rows keep counting as + # distinct configs (empty slice/data hash), so the total is unchanged. + if "bot_config_runs" in existing_tables: + run_cols = {c['name'] for c in inspector.get_columns('bot_config_runs')} + if 'slice_key' not in run_cols: + logger.info("Migration: rebuilding bot_config_runs table as a run log...") + conn.execute(text(""" + CREATE TABLE bot_config_runs_migration ( + id INTEGER PRIMARY KEY, + bot_name VARCHAR NOT NULL, + config_hash VARCHAR(32) NOT NULL, + slice_key VARCHAR(32) NOT NULL DEFAULT '', + window_from DATETIME, + window_to DATETIME, + data_hash VARCHAR(64) NOT NULL DEFAULT '', + run_at DATETIME NOT NULL, + CONSTRAINT uq_bot_config_runs_run UNIQUE (bot_name, config_hash, slice_key, data_hash) + ) + """)) + conn.execute(text(""" + INSERT INTO bot_config_runs_migration (id, bot_name, config_hash, run_at) + SELECT id, bot_name, config_hash, first_run_at FROM bot_config_runs + """)) + conn.execute(text("DROP TABLE bot_config_runs")) + conn.execute(text("ALTER TABLE bot_config_runs_migration RENAME TO bot_config_runs")) + conn.execute(text("CREATE INDEX IF NOT EXISTS ix_bot_config_runs_bot_name ON bot_config_runs (bot_name)")) + logger.info("Migration: bot_config_runs table rebuilt.") + # Composite indexes for hot query paths (idempotent) conn.execute(text("CREATE INDEX IF NOT EXISTS ix_positions_botname_status ON positions (bot_name, status)")) conn.execute(text("CREATE INDEX IF NOT EXISTS ix_orders_posid_side_status ON orders (position_id, side, status)")) diff --git a/backend/engine/backtest.py b/backend/engine/backtest.py index 380ba22..c6fb962 100644 --- a/backend/engine/backtest.py +++ b/backend/engine/backtest.py @@ -12,7 +12,10 @@ from backend.models.signals import Signal from backend.models.orders import Order from backend.models.positions import Position -from backend.engine.sizing import _num, _int, _naive_utc, sim_frictions, calculate_trade_amount, _record_config_run +from backend.engine.sizing import ( + _num, _int, _naive_utc, sim_frictions, calculate_trade_amount, _record_config_run, + backtest_pin, combined_fingerprint, slice_key, +) from backend.core import bot_log_buffer as blb logger = logging.getLogger("apexalgo.bot_manager") @@ -322,15 +325,34 @@ def simulate(db, bot, sym_contexts, exchange_name, run_backtest, *, check_exits, ) -def build_summary(db, bot, res: SimResult, sym_contexts) -> dict: +def build_summary(db, bot, res: SimResult, sym_contexts, exchange_name=None) -> dict: """`last_backtest_summary`: the numbers the bot card and analytics show, including the drawdown the gate actually enforces (the closed-trade curve - in the UI understates intra-trade dips).""" + in the UI understates intra-trade dips), plus the run's identity — slice, + data hash and both variant counters — for reproducibility.""" closed_bt = db.query(Position.profit_abs).filter( Position.bot_name == bot.name, Position.mode == "backtest", Position.status == "closed" ).all() pnls = [float(p[0] or 0) for p in closed_bt] wins = sum(1 for p in pnls if p > 0) + + data_from = _naive_utc(res.timeline[0][0]) if res.timeline else None + data_to = _naive_utc(res.timeline[-1][0]) if res.timeline else None + # Slice = the candles walked: the pinned window when there is one, else + # the realized range (which moves with every unpinned run) + pin_from, pin_to = backtest_pin(bot.settings) + pinned = pin_from is not None + window_from, window_to = (pin_from, pin_to) if pinned else (data_from, data_to) + hash_by_symbol = {c["symbol"]: c["data_hash"] for c in sym_contexts if c.get("data_hash")} + data_hash = combined_fingerprint(hash_by_symbol) + slice_hash = slice_key(exchange_name or bot.settings.get("data_exchange", "okx"), + [c["symbol"] for c in sym_contexts], bot.settings.get("timeframe"), window_from, window_to) + # Same slice as the previous saved run but different candles underneath + # (re-download, gap repair, exchange restatement) — None when the slice + # moved, because then a differing hash says nothing + prev = (bot.settings or {}).get("last_backtest_summary") or {} + data_changed = (prev["data_hash"] != data_hash) if prev.get("data_hash") and prev.get("slice_key") == slice_hash else None + variants, variants_on_slice = _record_config_run(db, bot.name, bot.settings, slice_hash, data_hash, window_from, window_to) summary = { "trades": len(pnls), "wins": wins, @@ -344,8 +366,8 @@ def build_summary(db, bot, res: SimResult, sym_contexts) -> dict: "candles": sum(len(c["df"]) for c in sym_contexts), # Data range the backtest walked — lets the analytics page # measure flat periods before the first / after the last trade - "data_from": res.timeline[0][0].isoformat() if res.timeline else None, - "data_to": res.timeline[-1][0].isoformat() if res.timeline else None, + "data_from": data_from.isoformat() if data_from else None, + "data_to": data_to.isoformat() if data_to else None, # Buy & hold over the same walked range, per symbol — the # only fair benchmark for the backtest return above "buy_hold": { @@ -357,9 +379,19 @@ def build_summary(db, bot, res: SimResult, sym_contexts) -> dict: } for c in sym_contexts if len(c["df"]) > 0 }, - # Distinct configurations this bot has backtested — plain - # tweak-awareness, no judgement attached - "variants": _record_config_run(db, bot.name, bot.settings), + # Distinct configurations this bot has backtested — ever, and on + # exactly this slice of data — plain tweak-awareness, no judgement + "variants": variants, + "variants_on_slice": variants_on_slice, + # Run identity: pin + slice + raw-candle hash make the run reproducible + # and let the next run on the same slice detect altered data + "pinned": pinned, + "window_from": window_from.isoformat() if window_from else None, + "window_to": window_to.isoformat() if window_to else None, + "slice_key": slice_hash, + "data_hash": data_hash, + "data_hash_by_symbol": hash_by_symbol, + "data_changed": data_changed, "finished_at": datetime.now(timezone.utc).isoformat(), } return summary diff --git a/backend/engine/data_verify.py b/backend/engine/data_verify.py new file mode 100644 index 0000000..094ada8 --- /dev/null +++ b/backend/engine/data_verify.py @@ -0,0 +1,110 @@ +"""Verify stored candles against the exchange. + +Stored candles are never overwritten by the poller or a download (only +missing rows are added), so a backtest on them is reproducible — but an +exchange can silently restate history (Binance most often the volume). This +module re-fetches a window and diffs it against the local rows; with +``accept=True`` the restated rows are overwritten (and stamped `fetched_at`), +which is the one deliberate path that changes a slice's data hash. +""" +import logging +import math +import time +from datetime import datetime, timezone + +from backend.models.candles import Candle +from backend.engine.sizing import _naive_utc + +logger = logging.getLogger("apexalgo.data_verify") + +FIELDS = ("open", "high", "low", "close", "volume") +REL_TOL = 1e-9 +MAX_DETAIL = 50 + + +def _differs(a, b) -> bool: + if a is None or b is None: + return (a is None) != (b is None) + try: + return not math.isclose(float(a), float(b), rel_tol=REL_TOL, abs_tol=0.0) + except (TypeError, ValueError): + return True + + +def fetch_window(exch, symbol: str, timeframe: str, start: datetime, end: datetime, limit: int = 500, pause: float = 0.35): + """Paginated OHLCV between two naive-UTC datetimes (inclusive), as + ``{naive_utc_ts: (o, h, l, c, v)}``.""" + start_ms = int(start.replace(tzinfo=timezone.utc).timestamp() * 1000) + end_ms = int(end.replace(tzinfo=timezone.utc).timestamp() * 1000) + out = {} + since = start_ms + while since <= end_ms: + batch = exch.fetch_ohlcv(symbol, timeframe, since=since, limit=limit) + if not batch: + break + for row in batch: + ts_ms = int(row[0]) + if start_ms <= ts_ms <= end_ms: + ts = datetime.fromtimestamp(ts_ms / 1000.0, tz=timezone.utc).replace(tzinfo=None) + out[ts] = tuple(float(x) if x is not None else None for x in row[1:6]) + last = int(batch[-1][0]) + if last >= end_ms or len(batch) < 2 or last < since: + break + since = last + 1 + time.sleep(pause) + return out + + +def verify_window(db, exch, exchange_id: str, symbol: str, timeframe: str, start, end, accept: bool = False) -> dict: + """Diff the exchange's view of ``[start, end]`` against the stored rows. + + Returns counts plus up to ``MAX_DETAIL`` restated rows with per-field + (local, exchange) pairs. ``accept`` overwrites restated local rows in + place (same ids, so signals keep their candle reference) and commits. + """ + start = _naive_utc(start) + end = _naive_utc(end) + remote = fetch_window(exch, symbol, timeframe, start, end) + local_rows = db.query(Candle).filter( + Candle.exchange == exchange_id, Candle.symbol == symbol, Candle.timeframe == timeframe, + Candle.timestamp >= start, Candle.timestamp <= end, + ).all() + local = {_naive_utc(r.timestamp): r for r in local_rows} + + restated, missing_local = [], 0 + restated_total = accepted = 0 + now = datetime.now(timezone.utc).replace(tzinfo=None) + for ts, values in sorted(remote.items()): + row = local.get(ts) + if row is None: + missing_local += 1 + continue + diffs = {} + for field, new in zip(FIELDS, values): + old = getattr(row, field) + if _differs(old, new): + diffs[field] = {"local": old, "exchange": new} + if not diffs: + continue + restated_total += 1 + if len(restated) < MAX_DETAIL: + restated.append({"timestamp": ts.isoformat(), "fields": diffs}) + if accept: + for field, d in diffs.items(): + setattr(row, field, d["exchange"]) + row.fetched_at = now + accepted += 1 + missing_exchange = sum(1 for ts in local if ts not in remote) + if accept and accepted: + db.commit() + logger.info("Accepted %d restated candle(s) from %s for %s/%s", accepted, exchange_id, symbol, timeframe) + + return { + "exchange": exchange_id, "symbol": symbol, "timeframe": timeframe, + "from": start.isoformat(), "to": end.isoformat(), + "checked": len(remote), "local_rows": len(local), + "restated_count": restated_total, "restated": restated, + "missing_local": missing_local, "missing_exchange": missing_exchange, + "accepted": accepted, + "verified_at": now.isoformat(), + } diff --git a/backend/engine/settings_validator.py b/backend/engine/settings_validator.py index 829a24e..892131b 100644 --- a/backend/engine/settings_validator.py +++ b/backend/engine/settings_validator.py @@ -1,5 +1,6 @@ import logging import re +from datetime import datetime from backend.core.exchange_registry import get_exchange_timeframes from backend.engine.indicator_registry import get_spec @@ -197,6 +198,20 @@ def validate_bot_settings(settings: dict, exchange_id: str | None = None) -> dic warnings.append(f"backtest_lookback ({lookback}) is short for the longest indicator window ({longest}): the first " f"~{longest} candles are warm-up, leaving little to trade on. Use at least {longest + LOOKBACK_MARGIN}.") + # Pinned backtest window: both ends or neither, valid ISO, from < to + pin_raw = {k: settings.get(k) for k in ("backtest_from", "backtest_to")} + if any(pin_raw.values()): + if not all(pin_raw.values()): + errors.append("backtest_from and backtest_to must be set together (or both cleared to rerun against the latest data).") + else: + try: + _pf = datetime.fromisoformat(str(pin_raw["backtest_from"]).replace("Z", "+00:00")) + _pt = datetime.fromisoformat(str(pin_raw["backtest_to"]).replace("Z", "+00:00")) + if _pf >= _pt: + errors.append("backtest_from must be before backtest_to.") + except (ValueError, TypeError): + errors.append("backtest_from / backtest_to must be ISO 8601 timestamps.") + # Trade settings trade_settings = settings.get("trade_settings", {}) entry_ts = trade_settings.get("entry", {}) diff --git a/backend/engine/sizing.py b/backend/engine/sizing.py index 4912ea3..76628ca 100644 --- a/backend/engine/sizing.py +++ b/backend/engine/sizing.py @@ -2,7 +2,7 @@ and the live tick. Pure functions over settings + DB — no engine state.""" import json import logging -from hashlib import md5 +from hashlib import md5, sha256 from datetime import datetime, timezone from sqlalchemy import text, func from backend.models.bots import BotConfig @@ -59,10 +59,12 @@ def _naive_utc(ts): # Settings that do not change what the strategy does on the data: layout, -# order routing, live sizing and the engine's own runtime bookkeeping. +# order routing, live sizing, the pinned backtest window (which data, not +# what the strategy does on it) and the engine's own runtime bookkeeping. _NON_STRATEGY_KEYS = frozenset({ "ui_layout", "api_execution", "api_key_name", "live_allocation_pct", "max_order_value", - "backtest_on_start", "last_backtest_summary", "last_backtest_max_drawdown", + "backtest_on_start", "backtest_from", "backtest_to", + "last_backtest_summary", "last_backtest_max_drawdown", "last_stop_reason", "drawdown_peak_reset_at", "live_starting_capital", }) @@ -74,14 +76,72 @@ def _config_fingerprint(settings: dict) -> str: return md5(json.dumps(relevant, sort_keys=True, default=str).encode()).hexdigest() -def _record_config_run(db, bot_name: str, settings: dict) -> int: - """Register this configuration as backtested and return the number of - distinct configurations the bot has run so far (this one included).""" +def data_fingerprint(df) -> str: + """sha256 of the raw OHLCV rows a backtest walked (sorted by timestamp). + `repr` on the floats makes it byte-stable across runs, so two runs on the + same slice hash equal unless a candle was actually altered — the check + behind the "historical data changed" warning.""" + h = sha256() + if df is None or len(df) == 0: + return h.hexdigest() + cols = ["timestamp", "open", "high", "low", "close", "volume"] + for ts, o, hi, lo, c, v in df.sort_values("timestamp")[cols].itertuples(index=False, name=None): + ts = _naive_utc(ts) + h.update(f"{ts.isoformat() if ts is not None else ''},{o!r},{hi!r},{lo!r},{c!r},{v!r}\n".encode()) + return h.hexdigest() + + +def backtest_pin(settings) -> tuple: + """``(window_from, window_to)`` as naive UTC datetimes when the bot's + backtest window is pinned (``backtest_from``/``backtest_to`` ISO strings), + else ``(None, None)``. A half or malformed pin counts as no pin.""" + out = [] + for key in ("backtest_from", "backtest_to"): + raw = (settings or {}).get(key) + if not raw: + return None, None + try: + out.append(_naive_utc(datetime.fromisoformat(str(raw).replace("Z", "+00:00")))) + except (TypeError, ValueError): + return None, None + if out[0] >= out[1]: + return None, None + return out[0], out[1] + + +def combined_fingerprint(by_symbol: dict) -> str: + """One hash over the per-symbol data hashes (sorted by symbol).""" + return sha256("|".join(f"{s}:{by_symbol[s]}" for s in sorted(by_symbol)).encode()).hexdigest() + + +def slice_key(exchange: str, symbols, timeframe: str, window_from, window_to) -> str: + """Identity of a backtest slice: which candles were walked. Same pairs, + timeframe and window on the same exchange = same slice, whatever the + strategy settings are.""" + f = _naive_utc(window_from) + t = _naive_utc(window_to) + parts = [str(exchange or ""), ",".join(sorted(str(s) for s in (symbols or []))), str(timeframe or ""), + f.isoformat() if f else "", t.isoformat() if t else ""] + return md5("|".join(parts).encode()).hexdigest() + + +def _record_config_run(db, bot_name: str, settings: dict, slice_hash: str = "", data_hash: str = "", + window_from=None, window_to=None) -> tuple: + """Log this backtest run and return ``(variants_total, variants_on_slice)``: + distinct configurations the bot has ever run, and distinct configurations + run on exactly this slice of data (this one included). One row per + (config, slice, data) — a rerun on unchanged data is a no-op.""" db.execute( - text("INSERT OR IGNORE INTO bot_config_runs (bot_name, config_hash, first_run_at) VALUES (:bn, :h, :ts)"), - {"bn": bot_name, "h": _config_fingerprint(settings), "ts": datetime.now(timezone.utc).replace(tzinfo=None)}, + text("INSERT OR IGNORE INTO bot_config_runs (bot_name, config_hash, slice_key, window_from, window_to, data_hash, run_at) " + "VALUES (:bn, :h, :sk, :wf, :wt, :dh, :ts)"), + {"bn": bot_name, "h": _config_fingerprint(settings), "sk": slice_hash or "", "wf": _naive_utc(window_from), + "wt": _naive_utc(window_to), "dh": data_hash or "", "ts": datetime.now(timezone.utc).replace(tzinfo=None)}, ) - return int(db.execute(text("SELECT COUNT(*) FROM bot_config_runs WHERE bot_name = :bn"), {"bn": bot_name}).scalar() or 0) + total = int(db.execute(text("SELECT COUNT(DISTINCT config_hash) FROM bot_config_runs WHERE bot_name = :bn"), + {"bn": bot_name}).scalar() or 0) + on_slice = int(db.execute(text("SELECT COUNT(DISTINCT config_hash) FROM bot_config_runs WHERE bot_name = :bn AND slice_key = :sk"), + {"bn": bot_name, "sk": slice_hash or ""}).scalar() or 0) + return total, on_slice def sim_frictions(settings): diff --git a/backend/engine/startup.py b/backend/engine/startup.py index 71ca218..13a0f7e 100644 --- a/backend/engine/startup.py +++ b/backend/engine/startup.py @@ -17,7 +17,7 @@ from backend.models.exchange_keys import ExchangeKey from backend.engine.evaluator import NodeEvaluator from backend.engine import backtest -from backend.engine.sizing import _num, _int, _tf_seconds, _naive_utc +from backend.engine.sizing import _num, _int, _tf_seconds, _naive_utc, backtest_pin, data_fingerprint from backend.core.exchange_registry import get_exchange_timeframes from backend.core import bot_log_buffer as blb @@ -104,6 +104,11 @@ def _check_abort(): exit_node = bot.settings.get("exit_node") run_backtest = bot.settings.get("backtest_on_start", False) lookback_limit = _int(bot.settings.get("backtest_lookback"), 150) + # A pinned window replays exactly the candles of a saved run instead + # of the newest `lookback` ones, so the result is reproducible until + # the user explicitly reruns against the latest data + pin_from, pin_to = backtest_pin(bot.settings) if run_backtest else (None, None) + pinned = pin_from is not None if run_backtest: # A backtest is deterministic, so always simulate the whole @@ -123,7 +128,8 @@ def _check_abort(): empty_symbols = [] for symbol in symbols: - blb.push(bot.name, "INFO", f"Starting: {symbol} | {timeframe} | {live_mode} | lookback={lookback_limit}") + _window = f"pinned {pin_from:%Y-%m-%d} → {pin_to:%Y-%m-%d}" if pinned else f"lookback={lookback_limit}" + blb.push(bot.name, "INFO", f"Starting: {symbol} | {timeframe} | {live_mode} | {_window}") tf_seconds = 60 if timeframe.endswith('m'): tf_seconds = int(timeframe[:-1]) * 60 elif timeframe.endswith('h'): tf_seconds = int(timeframe[:-1]) * 3600 @@ -232,17 +238,25 @@ def _latest_candle_ts(): try: query = candle_db.query(Candle.id, Candle.timestamp, Candle.open, Candle.high, Candle.low, Candle.close, Candle.volume).filter( Candle.exchange == exchange_name, Candle.symbol == symbol, Candle.timeframe == timeframe - ).order_by(Candle.timestamp.desc()).limit(lookback_limit).statement - df = pd.read_sql(query, candle_db.bind) + ) + if pinned: + query = query.filter(Candle.timestamp >= pin_from, Candle.timestamp <= pin_to).order_by(Candle.timestamp.asc()) + else: + query = query.order_by(Candle.timestamp.desc()).limit(lookback_limit) + df = pd.read_sql(query.statement, candle_db.bind) finally: candle_db.close() if df.empty or len(df) < 20: logger.info("Skipping backfill for %s: insufficient data (%d candles, minimum 20 required).", symbol, len(df)) - blb.push(bot.name, "WARN", f"Skipping backtest: {symbol} — only {len(df)} candles available (minimum 20)") + _hint = " in the pinned window — rerun against latest or re-download the range" if pinned else "" + blb.push(bot.name, "WARN", f"Skipping backtest: {symbol} — only {len(df)} candles available (minimum 20){_hint}") continue df = df.sort_values('timestamp').reset_index(drop=True) + # Hash of the raw candles before any indicator touches them: the + # summary compares it with the previous run on the same slice + data_hash = data_fingerprint(df) if run_backtest else None evaluator = NodeEvaluator(bot.settings) evaluator.df = df.copy() @@ -278,6 +292,7 @@ def _latest_candle_ts(): "atr_arr": evaluator.df['atr'].values if 'atr' in evaluator.df.columns else None, "indicator_cols": [c for c in evaluator.df.columns if c not in _standard_cols], "existing_timestamps": existing_timestamps, + "data_hash": data_hash, "last_bt_ts": last_bt_ts, "open_positions": open_bt_positions, # pyramided up to max_positions "original_amount": {}, # pos.id -> entry size, for weighted profit_pct / partial exits @@ -323,12 +338,20 @@ def _latest_candle_ts(): # Persist the engine's measured drawdown so the analytics UI # can show the number the gate actually enforces try: - summary = backtest.build_summary(db, bot, res, sym_contexts) + summary = backtest.build_summary(db, bot, res, sym_contexts, exchange_name) bot.settings = {**bot.settings, "last_backtest_max_drawdown": round(res.max_dd, 2), "last_backtest_summary": summary} flag_modified(bot, "settings") db.commit() - except Exception: + _sl = summary.get("variants_on_slice") or 0 + _tot = summary.get("variants") or 0 + blb.push(bot.name, "INFO", f"Backtest variant #{_sl} on this slice ({_tot} distinct config{'s' if _tot != 1 else ''} for this bot in total)") + if summary.get("data_changed"): + blb.push(bot.name, "WARN", "Historical data changed since the previous run on this slice — the candles " + "underneath differ (re-download, gap repair or exchange restatement), so the " + "results are not directly comparable") + except Exception as _exc: db.rollback() + logger.warning("Bot '%s': could not persist backtest summary: %s", bot.name, _exc) engine._drawdown_cache.pop((bot.name, "backtest"), None) stop_reason = backtest.gate_stop_reason(bot, res) diff --git a/backend/main.py b/backend/main.py index bedcb02..7d2ff89 100644 --- a/backend/main.py +++ b/backend/main.py @@ -63,7 +63,7 @@ async def lifespan(app: FastAPI): enable_docs = os.getenv("ENABLE_DOCS", "0") == "1" app = FastAPI( title="ApexAlgo Engine API", - version="2.1.1", + version="2.2.0", swagger_ui_init_oauth={"clientId": "test"}, lifespan=lifespan, docs_url="/docs" if enable_docs else None, diff --git a/backend/models/bot_config_runs.py b/backend/models/bot_config_runs.py index f3c3511..e7f5f39 100644 --- a/backend/models/bot_config_runs.py +++ b/backend/models/bot_config_runs.py @@ -3,17 +3,25 @@ class BotConfigRun(Base): - """One row per distinct strategy configuration a bot has backtested. + """Run log of a bot's backtests: one row per (strategy configuration, + data slice, data hash). - Only the hash is kept, not the config itself: the count of rows per bot is - shown as "variant #N" so the user can see how often a strategy was tweaked - and re-run on the same data (overfitting awareness). Cleared by a cache - wipe together with the rest of the bot's derived data. + Only hashes are kept, not the config or the candles. Two counters are + derived from it — distinct configs ever run ("42 configs total") and + distinct configs run on exactly this slice ("variant #3 on this slice") — + so the user can see how often a strategy was tweaked and re-run on the + same data (overfitting awareness). The data hash lets a rerun on the same + slice tell whether the candles underneath changed. Cleared by a cache wipe + together with the rest of the bot's derived data. """ __tablename__ = "bot_config_runs" - __table_args__ = (UniqueConstraint("bot_name", "config_hash", name="uq_bot_config_runs_bot_hash"),) + __table_args__ = (UniqueConstraint("bot_name", "config_hash", "slice_key", "data_hash", name="uq_bot_config_runs_run"),) id = Column(Integer, primary_key=True, autoincrement=True) bot_name = Column(String, nullable=False, index=True) config_hash = Column(String(32), nullable=False) - first_run_at = Column(DateTime, nullable=False) + slice_key = Column(String(32), nullable=False, default="") # md5(exchange, symbols, timeframe, window) + window_from = Column(DateTime, nullable=True) + window_to = Column(DateTime, nullable=True) + data_hash = Column(String(64), nullable=False, default="") # sha256 of the raw candles walked + run_at = Column(DateTime, nullable=False) diff --git a/backend/models/candles.py b/backend/models/candles.py index 077a426..1b980da 100644 --- a/backend/models/candles.py +++ b/backend/models/candles.py @@ -1,7 +1,13 @@ +from datetime import datetime, timezone from sqlalchemy import Column, Integer, String, Float, DateTime, UniqueConstraint from sqlalchemy.orm import relationship from backend.core.database import Base + +def _utcnow(): + return datetime.now(timezone.utc).replace(tzinfo=None) + + class Candle(Base): __tablename__ = "candles" @@ -17,6 +23,11 @@ class Candle(Base): close = Column(Float) volume = Column(Float) marketcap = Column(Float) + # When this row was written (or last overwritten by an accepted exchange + # restatement). NULL on rows that predate the column. Existing rows are + # otherwise never updated, so a backtest on stored candles stays + # reproducible until the user re-downloads or accepts a verification. + fetched_at = Column(DateTime, nullable=True, default=_utcnow) __table_args__ = ( UniqueConstraint('exchange', 'symbol', 'timeframe', 'timestamp', name='_ex_symbol_tf_ts_uc'), diff --git a/backend/routers/bots.py b/backend/routers/bots.py index 0be6878..db58f5a 100644 --- a/backend/routers/bots.py +++ b/backend/routers/bots.py @@ -20,6 +20,9 @@ from backend.core.security import verify_api_key from backend.engine.settings_validator import validate_bot_settings from backend.engine.bot_manager import bot_manager, _config_fingerprint +from backend.engine.sizing import backtest_pin +from backend.engine.data_verify import verify_window +from backend.core.exchange_registry import build_exchange from backend.core import bot_log_buffer as blb from backend.models.bot_logs import BotLog from backend.models.bot_config_runs import BotConfigRun @@ -135,6 +138,8 @@ def get_bots_summary(db: Session = Depends(get_db)): "api_key_name": b.settings.get("api_key_name") if b.settings else None, "backtest_on_start": b.settings.get("backtest_on_start", False) if b.settings else False, "backtest_capital": b.settings.get("backtest_capital", 1000) if b.settings else 1000, + "backtest_from": b.settings.get("backtest_from") if b.settings else None, + "backtest_to": b.settings.get("backtest_to") if b.settings else None, # Needed by chart-open and the Data Vault live-guard: the # same pair on another exchange is a different dataset "data_exchange": b.settings.get("data_exchange", "okx") if b.settings else "okx", @@ -631,6 +636,65 @@ def duplicate_bot(bot_id: int, db: Session = Depends(get_db)): "is_active": new_bot.is_active, "created_at": new_bot.created_at.isoformat() if new_bot.created_at else None, } +@router.post("/{bot_id}/verify-data") +async def verify_bot_data(bot_id: int, accept: bool = Query(default=False), db: Session = Depends(get_db)): + """Check the candles of the bot's last backtest window (pinned window or + the range the last run walked) against the exchange, per whitelist + symbol. Stored candles never change on their own, so this is the only way + to learn about an exchange restatement. ``accept=true`` overwrites the + restated rows — after which the next run on that slice will flag + 'data changed'. The outcome is kept in ``last_backtest_summary``.""" + bot = db.query(BotConfig).filter(BotConfig.id == bot_id).first() + if not bot: + raise HTTPException(status_code=404, detail="Bot not found.") + settings = bot.settings or {} + summary = settings.get("last_backtest_summary") or {} + pin_from, pin_to = backtest_pin(settings) + start = pin_from or summary.get("window_from") or summary.get("data_from") + end = pin_to or summary.get("window_to") or summary.get("data_to") + if not start or not end: + raise HTTPException(status_code=400, detail="No backtest window to verify yet — run a backtest first.") + if isinstance(start, str): + start, end = datetime.fromisoformat(start), datetime.fromisoformat(end) + symbols = settings.get("symbols") or ([settings["symbol"]] if settings.get("symbol") else []) + timeframe = settings.get("timeframe") + exchange_id = _candle_exchange(settings, _key_exchanges(db)) + + def _run(): + _db = SessionLocal() + try: + exch = build_exchange(exchange_id) + return [verify_window(_db, exch, exchange_id, str(s).replace('-', '/').upper(), timeframe, start, end, accept=accept) + for s in symbols] + finally: + _db.close() + try: + results = await asyncio.to_thread(_run) + except Exception as e: + logger.error("Verify failed for bot '%s': %s", bot.name, e, exc_info=True) + raise HTTPException(status_code=502, detail=f"Could not verify against {exchange_id}: {str(e)[:160]}") + + restated = sum(r["restated_count"] for r in results) + accepted = sum(r["accepted"] for r in results) + missing = sum(r["missing_local"] for r in results) + verified_at = datetime.now(timezone.utc).isoformat() + if summary: + bot.settings = {**settings, "last_backtest_summary": { + **summary, "verified_at": verified_at, "restated_candles": restated - accepted, "missing_local": missing, + }} + flag_modified(bot, "settings") + db.commit() + for r in results: + if r["restated_count"]: + blb.push(bot.name, "WARN" if not accept else "INFO", + f"{exchange_id} restated {r['restated_count']} candle(s) of {r['symbol']} {timeframe} in " + f"{r['from'][:10]} → {r['to'][:10]}" + (f" — {r['accepted']} overwritten locally" if accept else " — local snapshot kept")) + else: + blb.push(bot.name, "INFO", f"Verified {r['symbol']} {timeframe}: {r['checked']} candles match {exchange_id}") + return {"exchange": exchange_id, "from": results[0]["from"] if results else None, "to": results[0]["to"] if results else None, + "restated": restated, "accepted": accepted, "missing_local": missing, "verified_at": verified_at, "symbols": results} + + @router.get("/{bot_id}/export") def export_bot(bot_id: int, db: Session = Depends(get_db)): bot = db.query(BotConfig).filter(BotConfig.id == bot_id).first() @@ -643,7 +707,10 @@ def export_bot(bot_id: int, db: Session = Depends(get_db)): "name": bot.name, "is_sandbox": bot.is_sandbox, "strategy": bot.strategy, - "settings": bot.settings or {} + # The pinned backtest window belongs to this machine's candle + # store; on another install it would replay a range that may + # not exist there + "settings": {k: v for k, v in (bot.settings or {}).items() if k not in ("backtest_from", "backtest_to")}, } } safe_name = re.sub(r'[^A-Za-z0-9_-]', '_', bot.name).strip('_') or f"bot_{bot.id}" diff --git a/backend/routers/data.py b/backend/routers/data.py index 186092a..b2c4eb3 100644 --- a/backend/routers/data.py +++ b/backend/routers/data.py @@ -13,6 +13,7 @@ from backend.models.candles import Candle from backend.core.security import verify_api_key from backend.core.exchange_registry import build_exchange, SUPPORTED_EXCHANGES, get_exchange_timeframes, get_exchange_symbols +from backend.engine.data_verify import verify_window logger = logging.getLogger("apexalgo.data") @@ -56,6 +57,42 @@ class HistoricalDataFetch(BaseModel): end_date: datetime +class VerifyRequest(BaseModel): + exchange: str = "okx" + symbol: str + timeframe: str + start_date: datetime + end_date: datetime + accept: bool = False + + +@router.post("/verify") +async def verify_stored_candles(req: VerifyRequest): + """Re-fetch a window from the exchange and diff it against the stored + candles (stored rows are never updated on their own, so an exchange + restatement is invisible until checked). ``accept`` overwrites restated + rows — the deliberate way to change a backtest slice's data.""" + exchange_id = req.exchange.lower() + if exchange_id not in SUPPORTED_EXCHANGES: + raise HTTPException(status_code=400, detail=f"Unknown exchange '{exchange_id}'.") + if req.end_date <= req.start_date: + raise HTTPException(status_code=400, detail="end_date must be after start_date.") + symbol = req.symbol.replace('-', '/').upper() + + def _run(): + db = SessionLocal() + try: + return verify_window(db, build_exchange(exchange_id), exchange_id, symbol, req.timeframe, + req.start_date, req.end_date, accept=req.accept) + finally: + db.close() + try: + return await asyncio.to_thread(_run) + except Exception as e: + logger.error("Verify failed for %s/%s/%s: %s", exchange_id, symbol, req.timeframe, e, exc_info=True) + raise HTTPException(status_code=502, detail=f"Could not verify against {SUPPORTED_EXCHANGES[exchange_id]}: {str(e)[:160]}") + + @router.post("/fetch/{symbol}") async def fetch_historical_data( symbol: str, diff --git a/frontend/package-lock.json b/frontend/package-lock.json index ae60cbe..b03f88c 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "frontend", - "version": "2.1.1", + "version": "2.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "frontend", - "version": "2.1.1", + "version": "2.2.0", "dependencies": { "@fontsource-variable/inter": "^5.3.0", "@fontsource/jetbrains-mono": "^5.3.0", diff --git a/frontend/package.json b/frontend/package.json index c7c9032..e7afa9e 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "frontend", "private": true, - "version": "2.1.1", + "version": "2.2.0", "license": "AGPL-3.0-only", "type": "module", "scripts": { diff --git a/frontend/src/components/BotManagerUI.jsx b/frontend/src/components/BotManagerUI.jsx index 4d8045a..f7f8f97 100644 --- a/frontend/src/components/BotManagerUI.jsx +++ b/frontend/src/components/BotManagerUI.jsx @@ -156,26 +156,80 @@ const fmtDay = (iso) => { }; /* One slim line: how many configurations of this strategy have been - backtested, the range the last one walked, and a jump to Analytics — the - performance numbers themselves live there, not on the card */ -function BacktestResult({ bot }) { + backtested (on this slice of data / ever), the range the last one walked, + whether that range is pinned, and a jump to Analytics — the performance + numbers themselves live there, not on the card */ +function BacktestResult({ bot, updateBotConfig, verifyData, verifying }) { const sm = bot.last_backtest_summary ?? bot.settings?.last_backtest_summary; if (!sm || typeof sm.trades !== 'number') return null; - const variants = Number(sm.variants) || 0; + const restated = Number(sm.restated_candles) || 0; + const verifiedAt = sm.verified_at ? new Date(sm.verified_at) : null; + const canVerify = !!sm.data_from && !!sm.data_to && !verifying; + const total = Number(sm.variants) || 0; + const onSlice = Number(sm.variants_on_slice) || 0; + const pinned = !!(bot.settings?.backtest_from && bot.settings?.backtest_to); + const canPin = !bot.is_active && !!sm.data_from && !!sm.data_to; + const counterTitle = onSlice > 0 + ? `Variant #${onSlice} on this slice of data (same pairs, timeframe and range) — ${total} distinct configuration${total === 1 ? '' : 's'} of this strategy backtested in total. Switching pairs or range starts a new slice; the total keeps counting. Reset the bot to start over.` + : (total > 0 ? `${total} distinct configuration${total === 1 ? '' : 's'} of this strategy have been backtested. Reset the bot to start counting again.` : undefined); + const pin = () => updateBotConfig(bot.id, bot, { settings: { backtest_from: sm.data_from, backtest_to: sm.data_to } }); + const unpin = () => updateBotConfig(bot.id, bot, { settings: { backtest_from: null, backtest_to: null } }); + const action = "text-3xs font-bold uppercase tracking-wider transition-colors"; return ( -
-
0 ? `${variants} distinct configuration${variants === 1 ? '' : 's'} of this strategy have been backtested. Reset the bot to start counting again.` : undefined}> - Backtest{variants > 0 && #{variants}} +
+ {/* Row 1: what ran — counter, range, pin state */} +
+ + Backtest{onSlice > 0 && #{onSlice}} + + {total > onSlice && · {total} total} {sm.data_from && sm.data_to && ( - {fmtDay(sm.data_from)} → {fmtDay(sm.data_to)} + {fmtDay(sm.data_from)} → {fmtDay(sm.data_to)} + )} + {pinned && ( + + Pinned + )}
- + {/* Row 2: actions — wrap instead of squeezing the range on narrow cards */} +
+ {canPin && (pinned + ? + : )} + {canVerify && ( + + )} + {verifying && verifying…} + +
+ {sm.data_changed === true && ( +

+ Data changed + historical candles differ from the previous run on this slice +

+ )} + {verifiedAt && (restated > 0 ? ( +

+ Exchange restated {restated} candle{restated === 1 ? '' : 's'} + local snapshot kept · verified {fmtDay(sm.verified_at)} {verifiedAt.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' })} +

+ ) : ( +

+ Matches exchange · verified {fmtDay(sm.verified_at)} {verifiedAt.toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' })} +

+ ))}
); } @@ -186,7 +240,7 @@ const runtimeKey = (rt) => { return JSON.stringify(rest); }; -const BotCard = memo(function BotCard({ bot, index, busyAction, togglingBot, openConsoles, clearSignals, toggleBotState, restartBot, handleExport, handleDuplicate, handleClearCacheClick, handleDeleteClick, updateBotConfig, toggleConsole }) { +const BotCard = memo(function BotCard({ bot, index, busyAction, togglingBot, openConsoles, clearSignals, toggleBotState, restartBot, handleExport, handleDuplicate, handleClearCacheClick, handleDeleteClick, updateBotConfig, toggleConsole, verifyData, verifyingBot }) { const isBacktestOn = bot.settings?.backtest_on_start === true; const isApiExecutionOn = bot.settings?.api_execution === true; const hasApiKey = !!bot.settings?.api_key_name; @@ -264,7 +318,7 @@ const BotCard = memo(function BotCard({ bot, index, busyAction, togglingBot, ope - + {/* ── Card Body ── */}
@@ -385,6 +439,7 @@ const BotCard = memo(function BotCard({ bot, index, busyAction, togglingBot, ope prev.bot.execution_mode === next.bot.execution_mode && prev.busyAction === next.busyAction && (prev.togglingBot === prev.bot.id) === (next.togglingBot === next.bot.id) && + (prev.verifyingBot === prev.bot.id) === (next.verifyingBot === next.bot.id) && prev.openConsoles[prev.bot.id] === next.openConsoles[next.bot.id] && prev.clearSignals[prev.bot.name] === next.clearSignals[next.bot.name] && // Runtime timestamps tick on every poll but are never rendered — strip @@ -460,6 +515,7 @@ export default function BotManagerUI({ bots = [], refetchBots, backendOk = true const [openConsoles, setOpenConsoles] = useState({}); const [busyAction, setBusyAction] = useState(null); // 'delete:ID' or 'wipe:name' const [togglingBot, setTogglingBot] = useState(null); // bot id being started/stopped + const [verifyingBot, setVerifyingBot] = useState(null); // bot id whose candles are being verified const [clearSignals, setClearSignals] = useState({}); const fileInputRef = useRef(null); @@ -590,6 +646,38 @@ export default function BotManagerUI({ bots = [], refetchBots, backendOk = true } }, [refetchBots]); + /* Verify the last backtest range against the exchange. The local snapshot + is never touched unless the user explicitly accepts the exchange data */ + const verifyData = useCallback(async (bot) => { + setVerifyingBot(bot.id); + try { + const { data } = await apiClient.post(`/api/bots/${bot.id}/verify-data`); + const n = data.restated || 0; + if (n === 0) { + toast.success(`'${bot.name}': stored candles match ${data.exchange}${data.missing_local ? ` (${data.missing_local} candles missing locally)` : ''}`); + refetchBots(); + return; + } + refetchBots(); + const accept = await confirmDialog({ + title: 'Exchange restated candles', + message: `${data.exchange} now reports different values for ${n} candle${n === 1 ? '' : 's'} in ${data.from?.slice(0, 10)} → ${data.to?.slice(0, 10)}. Keep the local snapshot (backtest stays reproducible) or overwrite it with the exchange data (the next run on this slice will be flagged as "data changed")?`, + confirmText: 'Accept exchange data', + cancelText: 'Keep local snapshot', + type: 'warning', + }); + if (!accept) return; + setVerifyingBot(bot.id); + const res = await apiClient.post(`/api/bots/${bot.id}/verify-data`, null, { params: { accept: true } }); + toast.success(`'${bot.name}': ${res.data.accepted} candle${res.data.accepted === 1 ? '' : 's'} overwritten with ${res.data.exchange} data`); + refetchBots(); + } catch (err) { + toast.error(humanizeApiError(err, 'Failed to verify candles against the exchange.')); + } finally { + setVerifyingBot(null); + } + }, [refetchBots]); + const handleExport = useCallback(async (bot) => { try { const res = await apiClient.get(`/api/bots/${bot.id}/export`, { responseType: 'blob' }); @@ -674,6 +762,10 @@ export default function BotManagerUI({ bots = [], refetchBots, backendOk = true
)} + {bots.length > 0 && ( + b.name)} /> + )}
diff --git a/tests/golden/Donchian_Breakout_1d.json b/tests/golden/Donchian_Breakout_1d.json index 71f8efa..9cc2d61 100644 --- a/tests/golden/Donchian_Breakout_1d.json +++ b/tests/golden/Donchian_Breakout_1d.json @@ -256,17 +256,28 @@ } }, "candles": 1200, + "data_changed": null, "data_from": "2023-04-11T00:00:00", + "data_hash": "37d2e7ff1b0c1f180e6f2d04152b7a62bb1aa73ea35371ae8ca29c6cc5407277", + "data_hash_by_symbol": { + "BTC/USDC": "f660b3784b9bf6a7f39627ed7be561abf294d0ae22a47ea4e04e3d379f30a99c", + "ETH/USDC": "85d7df127fae0bdeb2b2e4fd95236edfca10b12a950e14898fc412e2fa0bae2d" + }, "data_to": "2024-11-30T00:00:00", "entries_blocked_count": 0, "entries_blocked_days": 0.0, "max_capital_loss": 12.13, "max_drawdown": 19.9, "net_pnl": 293.49, + "pinned": false, "return_pct": 29.35, + "slice_key": "4bb7e92a956f8b2056235d0451ef0ccf", "trades": 15, "variants": 1, + "variants_on_slice": 1, "win_rate": 60.0, + "window_from": "2023-04-11T00:00:00", + "window_to": "2024-11-30T00:00:00", "wins": 9 } } diff --git a/tests/golden/EMA_Cross_4h.json b/tests/golden/EMA_Cross_4h.json index 2635c3f..e052ccc 100644 --- a/tests/golden/EMA_Cross_4h.json +++ b/tests/golden/EMA_Cross_4h.json @@ -229,17 +229,29 @@ } }, "candles": 1800, + "data_changed": null, "data_from": "2023-01-17T16:00:00", + "data_hash": "029f65508f491446b68415a4a6b52a1e613e5d2ffd4198b65ad6c3946fff68f1", + "data_hash_by_symbol": { + "BTC/USDC": "1937175c21ac086ef4fc2c946ffcef3458fc744dc1382edd55d1dc48c9bfa9fb", + "ETH/USDC": "68f4d0bbed5a5484e8b3c431553d50155be2d6de93a0b3dfc60cb7cc629d4f3d", + "SOL/USDC": "d3d21ada4a1c114fdc56c9026ff97bd48a8be283bc317817394d09bafa76ec8f" + }, "data_to": "2023-04-27T12:00:00", "entries_blocked_count": 0, "entries_blocked_days": 0.0, "max_capital_loss": 6.26, "max_drawdown": 7.76, "net_pnl": 194.09, + "pinned": false, "return_pct": 19.41, + "slice_key": "8e989ab1a778fc89be31d955f7cdcb3d", "trades": 13, "variants": 1, + "variants_on_slice": 1, "win_rate": 46.2, + "window_from": "2023-01-17T16:00:00", + "window_to": "2023-04-27T12:00:00", "wins": 6 } } diff --git a/tests/golden/Supertrend_Trend_1d.json b/tests/golden/Supertrend_Trend_1d.json index 81c197a..08dbb7e 100644 --- a/tests/golden/Supertrend_Trend_1d.json +++ b/tests/golden/Supertrend_Trend_1d.json @@ -240,17 +240,28 @@ } }, "candles": 1200, + "data_changed": null, "data_from": "2023-04-11T00:00:00", + "data_hash": "37d2e7ff1b0c1f180e6f2d04152b7a62bb1aa73ea35371ae8ca29c6cc5407277", + "data_hash_by_symbol": { + "BTC/USDC": "f660b3784b9bf6a7f39627ed7be561abf294d0ae22a47ea4e04e3d379f30a99c", + "ETH/USDC": "85d7df127fae0bdeb2b2e4fd95236edfca10b12a950e14898fc412e2fa0bae2d" + }, "data_to": "2024-11-30T00:00:00", "entries_blocked_count": 0, "entries_blocked_days": 0.0, "max_capital_loss": 18.64, "max_drawdown": 20.2, "net_pnl": -6.77, + "pinned": false, "return_pct": -0.68, + "slice_key": "4bb7e92a956f8b2056235d0451ef0ccf", "trades": 14, "variants": 1, + "variants_on_slice": 1, "win_rate": 42.9, + "window_from": "2023-04-11T00:00:00", + "window_to": "2024-11-30T00:00:00", "wins": 6 } } diff --git a/tests/test_reproducibility.py b/tests/test_reproducibility.py new file mode 100644 index 0000000..a8c3143 --- /dev/null +++ b/tests/test_reproducibility.py @@ -0,0 +1,299 @@ +"""Backtest reproducibility: pinned window, raw-candle data hash and the +two-tier variant counter (distinct configs on this slice / ever).""" +from datetime import datetime, timedelta, timezone + +import pandas as pd +from sqlalchemy import text + +from backend.engine.bot_manager import BotManager +from backend.engine.sizing import backtest_pin, data_fingerprint +from backend.models.bots import BotConfig +from backend.models.candles import Candle +from backend.models.orders import Order +from tests.conftest import _tf_seconds, insert_candles, make_candles + +EXCHANGE, TF, SYMBOL = "binance", "1h", "BTC/USDT" +BOT = "repro-bot" +N_CANDLES = 400 + + +def _settings(**overrides): + base = { + "symbols": [SYMBOL], "timeframe": TF, "data_exchange": EXCHANGE, + "api_execution": False, "api_key_name": None, + "backtest_on_start": True, "backtest_lookback": 300, "backtest_capital": 1000, + "max_positions": 1, "max_positions_scope": "per_pair", + "trade_settings": { + "entry": {"order_type": "market", "amount_type": "percentage", "amount_value": 40, + "fee": 0.1, "slippage": 0, + "stop_losses": [{"type": "percentage", "value": 3, "close_amount_type": "percentage", "close_amount_value": 100}], + "take_profits": [{"type": "percentage", "value": 3, "close_amount_type": "percentage", "close_amount_value": 100}]}, + "exit": {"order_type": "market", "amount_type": "percentage", "amount_value": 100, "fee": 0.1, "slippage": 0}, + }, + "nodes": {"entry": {"class": "condition", "left": "close", "operator": ">", "right": 0}}, + "entry_node": "entry", + "exit_node": None, + } + base.update(overrides) + return base + + +def _seed_rows(db): + return insert_candles(db, make_candles(EXCHANGE, SYMBOL, TF, N_CANDLES, seed=7)) + + +def _seed(db): + _seed_rows(db) + + +def _run(db, bot): + bot.is_active = True + db.commit() + BotManager()._execute_sync_backfill(bot.id) + db.expire_all() + bot = db.get(BotConfig, bot.id) + return bot, bot.settings["last_backtest_summary"] + + +def _orders(db): + return [(o.timestamp, o.side, round(o.price, 8), round(o.amount, 8)) for o in + db.query(Order).filter(Order.bot_name == BOT, Order.mode == "backtest").order_by(Order.timestamp, Order.side, Order.id)] + + +def _set(db, bot, **settings): + bot.settings = {**bot.settings, **settings} + db.commit() + return db.get(BotConfig, bot.id) + + +def _new_bot(db, **overrides): + bot = BotConfig(name=BOT, is_active=False, is_sandbox=True, strategy="node_graph", settings=_settings(**overrides)) + db.add(bot) + db.commit() + return bot + + +# ── data hash ──────────────────────────────────────────────────────────────── + +def test_data_fingerprint_is_deterministic_and_sensitive(): + rows = make_candles(EXCHANGE, SYMBOL, TF, 50, seed=3) + df = pd.DataFrame([{"timestamp": r.timestamp, "open": r.open, "high": r.high, "low": r.low, "close": r.close, "volume": r.volume} for r in rows]) + assert data_fingerprint(df) == data_fingerprint(df.sample(frac=1, random_state=1)), "row order must not matter" + altered = df.copy() + altered.loc[10, "close"] += 0.0001 + assert data_fingerprint(altered) != data_fingerprint(df) + assert data_fingerprint(df.iloc[:-1]) != data_fingerprint(df) + + +def test_backtest_pin_parsing(): + assert backtest_pin({}) == (None, None) + assert backtest_pin({"backtest_from": "2024-01-01T00:00:00"}) == (None, None) + assert backtest_pin({"backtest_from": "2024-02-01T00:00:00", "backtest_to": "2024-01-01T00:00:00"}) == (None, None) + f, t = backtest_pin({"backtest_from": "2024-01-01T00:00:00Z", "backtest_to": "2024-01-02T00:00:00+00:00"}) + assert f.tzinfo is None and t.tzinfo is None and (t - f) == timedelta(days=1) + + +# ── pinned window ──────────────────────────────────────────────────────────── + +def test_pinned_window_replays_the_same_run_after_new_candles_arrive(db): + _seed(db) + bot = _new_bot(db) + bot, first = _run(db, bot) + assert first["pinned"] is False + first_orders = _orders(db) + assert first_orders + + # Pin to the walked range, then let the market move on + bot = _set(db, bot, backtest_from=first["data_from"], backtest_to=first["data_to"]) + last_ts = db.query(Candle.timestamp).filter(Candle.symbol == SYMBOL).order_by(Candle.timestamp.desc()).first()[0] + extra = make_candles(EXCHANGE, SYMBOL, TF, 40, seed=99, start=last_ts + timedelta(seconds=_tf_seconds(TF))) + insert_candles(db, extra) + + bot, second = _run(db, bot) + assert second["pinned"] is True + assert (second["data_from"], second["data_to"]) == (first["data_from"], first["data_to"]) + assert second["data_hash"] == first["data_hash"] + assert second["data_changed"] is False + assert _orders(db) == first_orders + + # Unpinned rerun ("rerun against latest") walks the newest candles again + bot = _set(db, bot, backtest_from=None, backtest_to=None) + bot, third = _run(db, bot) + assert third["pinned"] is False + assert third["data_to"] > first["data_to"] + assert third["data_changed"] is None, "slice moved — hashes are not comparable" + + +def test_altered_candle_on_the_same_slice_is_flagged(db): + _seed(db) + bot = _new_bot(db) + bot, first = _run(db, bot) + bot = _set(db, bot, backtest_from=first["data_from"], backtest_to=first["data_to"]) + + # Simulate a re-download / exchange restatement of one candle + row = db.query(Candle).filter(Candle.symbol == SYMBOL, Candle.timestamp == datetime.fromisoformat(first["data_from"])).one() + row.close = row.close * 1.001 + db.commit() + + bot, second = _run(db, bot) + assert second["data_hash"] != first["data_hash"] + assert second["data_changed"] is True + + +# ── verify against exchange ────────────────────────────────────────────────── + +class ExchangeReplay: + """fetch_ohlcv over an in-memory list of candles, optionally restated.""" + def __init__(self, rows, restate=None, page=50): + self.rows = sorted(rows, key=lambda r: r.timestamp) + self.restate = restate or {} # timestamp -> {field: value} + self.page = page # exchange-side page cap, forces pagination + self.calls = 0 + + def fetch_ohlcv(self, symbol, timeframe, since=None, limit=500): + self.calls += 1 + out = [] + for r in self.rows: + ms = int(r.timestamp.replace(tzinfo=timezone.utc).timestamp() * 1000) + if since is not None and ms < since: + continue + vals = {f: getattr(r, f) for f in ("open", "high", "low", "close", "volume")} + vals.update(self.restate.get(r.timestamp, {})) + out.append([ms, vals["open"], vals["high"], vals["low"], vals["close"], vals["volume"]]) + if len(out) >= min(limit, self.page): + break + return out + + +def _candle_at(db, ts): + return db.query(Candle).filter(Candle.symbol == SYMBOL, Candle.timestamp == ts).one() + + +def test_verify_window_reports_and_optionally_accepts_restated_candles(db, monkeypatch): + import backend.engine.data_verify as dv + monkeypatch.setattr(dv.time, "sleep", lambda *_: None) + rows = insert_candles(db, make_candles(EXCHANGE, SYMBOL, TF, 120, seed=5)) + restate = {rows[10].timestamp: {"volume": rows[10].volume + 1.0}, rows[50].timestamp: {"close": rows[50].close * 1.01}} + exch = ExchangeReplay(rows, restate) + + res = dv.verify_window(db, exch, EXCHANGE, SYMBOL, TF, rows[0].timestamp, rows[-1].timestamp, accept=False) + assert res["checked"] == 120 and res["local_rows"] == 120 + assert res["restated_count"] == 2 and res["accepted"] == 0 + assert [r["timestamp"] for r in res["restated"]] == [rows[10].timestamp.isoformat(), rows[50].timestamp.isoformat()] + assert set(res["restated"][0]["fields"]) == {"volume"} + assert exch.calls > 1, "window must be paginated" + db.expire_all() + assert _candle_at(db, rows[50].timestamp).close == rows[50].close, "keep local snapshot by default" + + res = dv.verify_window(db, exch, EXCHANGE, SYMBOL, TF, rows[0].timestamp, rows[-1].timestamp, accept=True) + assert res["accepted"] == 2 + db.expire_all() + row = _candle_at(db, rows[50].timestamp) + assert row.close == rows[50].close * 1.01 and row.fetched_at is not None + + # Clean now: nothing restated any more + res = dv.verify_window(db, exch, EXCHANGE, SYMBOL, TF, rows[0].timestamp, rows[-1].timestamp) + assert res["restated_count"] == 0 + + +def test_verify_data_route_uses_the_last_window_and_updates_the_summary(db, monkeypatch): + import os + from fastapi.testclient import TestClient + from backend.main import app + from backend.routers import bots as bots_router + import backend.engine.data_verify as dv + monkeypatch.setattr(dv.time, "sleep", lambda *_: None) + rows = _seed_rows(db) + bot = _new_bot(db) + bot, summary = _run(db, bot) + exch = ExchangeReplay(rows, {rows[-5].timestamp: {"high": rows[-5].high * 1.02}}) + monkeypatch.setattr(bots_router, "build_exchange", lambda *_a, **_k: exch) + client = TestClient(app) + headers = {"X-API-Key": os.environ["MASTER_API_KEY"]} + + r = client.post(f"/api/bots/{bot.id}/verify-data", headers=headers) + assert r.status_code == 200, r.text + body = r.json() + assert body["exchange"] == EXCHANGE and body["restated"] == 1 and body["accepted"] == 0 + assert body["from"] == summary["data_from"] and body["to"] == summary["data_to"] + db.expire_all() + sm = db.get(BotConfig, bot.id).settings["last_backtest_summary"] + assert sm["restated_candles"] == 1 and sm["verified_at"] + + r = client.post(f"/api/bots/{bot.id}/verify-data?accept=true", headers=headers) + assert r.status_code == 200 and r.json()["accepted"] == 1 + db.expire_all() + assert db.get(BotConfig, bot.id).settings["last_backtest_summary"]["restated_candles"] == 0 + + # The accepted restatement changes the slice's data hash on the next run + bot = _set(db, db.get(BotConfig, bot.id), backtest_from=summary["data_from"], backtest_to=summary["data_to"]) + bot, again = _run(db, bot) + assert again["data_changed"] is True + + +# ── migration ──────────────────────────────────────────────────────────────── + +def test_bot_config_runs_migration_keeps_old_rows_as_distinct_configs(db): + from sqlalchemy import inspect + from backend.core.database import engine, run_migrations + db.add(BotConfig(name="a", is_active=False, is_sandbox=True, strategy="node_graph", settings={})) + db.commit() # the orphan sweep after migrations drops rows of unknown bots + with engine.begin() as conn: + conn.execute(text("DROP TABLE bot_config_runs")) + conn.execute(text("CREATE TABLE bot_config_runs (id INTEGER PRIMARY KEY, bot_name VARCHAR NOT NULL, " + "config_hash VARCHAR(32) NOT NULL, first_run_at DATETIME NOT NULL, " + "CONSTRAINT uq_bot_config_runs_bot_hash UNIQUE (bot_name, config_hash))")) + conn.execute(text("INSERT INTO bot_config_runs (bot_name, config_hash, first_run_at) VALUES " + "('a', 'h1', '2026-01-01 00:00:00'), ('a', 'h2', '2026-01-02 00:00:00')")) + run_migrations() + cols = {c["name"] for c in inspect(engine).get_columns("bot_config_runs")} + assert {"slice_key", "window_from", "window_to", "data_hash", "run_at"} <= cols + assert "first_run_at" not in cols + with engine.connect() as conn: + rows = conn.execute(text("SELECT bot_name, config_hash, slice_key, data_hash, run_at FROM bot_config_runs ORDER BY id")).fetchall() + assert [tuple(r) for r in rows] == [("a", "h1", "", "", "2026-01-01 00:00:00"), ("a", "h2", "", "", "2026-01-02 00:00:00")] + run_migrations() # idempotent + + +# ── variant counter ────────────────────────────────────────────────────────── + +def test_two_tier_variant_counter(db): + _seed(db) + insert_candles(db, make_candles(EXCHANGE, "ETH/USDT", TF, N_CANDLES, seed=8, start_price=200.0)) + bot = _new_bot(db) + bot, s1 = _run(db, bot) + assert (s1["variants"], s1["variants_on_slice"]) == (1, 1) + bot = _set(db, bot, backtest_from=s1["data_from"], backtest_to=s1["data_to"]) + + # Same slice, unchanged data, same config → no new variant + bot, s2 = _run(db, bot) + assert (s2["variants"], s2["variants_on_slice"]) == (1, 1) + + # Same slice, tweaked strategy → variant #2 on this slice, 2 in total + bot = _set(db, bot, max_positions=2) + bot, s3 = _run(db, bot) + assert (s3["variants"], s3["variants_on_slice"]) == (2, 2) + + # Other pair (new slice) → slice counter restarts, total keeps counting + bot = _set(db, bot, symbols=["ETH/USDT"]) + bot, s4 = _run(db, bot) + assert s4["slice_key"] != s3["slice_key"] + assert (s4["variants"], s4["variants_on_slice"]) == (3, 1) + + # Layout / routing / pin changes are not strategy variants + bot = _set(db, bot, ui_layout={"x": 1}, live_allocation_pct=50) + bot, s5 = _run(db, bot) + assert (s5["variants"], s5["variants_on_slice"]) == (3, 1) + + +def test_cache_wipe_resets_both_counters(db): + _seed(db) + bot = _new_bot(db) + bot, s1 = _run(db, bot) + bot = _set(db, bot, max_positions=2) + bot, s2 = _run(db, bot) + assert s2["variants"] == 2 + db.execute(text("DELETE FROM bot_config_runs WHERE bot_name = :bn"), {"bn": BOT}) + db.commit() + bot, s3 = _run(db, bot) + assert (s3["variants"], s3["variants_on_slice"]) == (1, 1)