diff --git a/cli/autotrader_bridge.py b/cli/autotrader_bridge.py new file mode 100644 index 0000000..fd457cf --- /dev/null +++ b/cli/autotrader_bridge.py @@ -0,0 +1,154 @@ +"""Read-only bridge to an autoresearch strategy project. + +Python port of ACC's ``server/src/autotrader-bridge.ts``. Parses results.tsv, +reads strategy.py, queries git log, and checks data readiness — without ever +mutating the project. Used by ``hl autoresearch status``. +""" +from __future__ import annotations + +import subprocess +from dataclasses import dataclass, field +from pathlib import Path +from typing import List, Optional + + +@dataclass +class AutoresearchResult: + commit: str + score: float + sharpe: float + max_dd: float + status: str # "kept" | "discarded" | "baseline" + description: str + + +@dataclass +class GitEntry: + hash: str + message: str + date: str + + +@dataclass +class LabStatus: + name: str + path: str + branch: str + data_ready: bool + best_score: Optional[float] + best_commit: str + total_experiments: int + results: List[AutoresearchResult] = field(default_factory=list) + strategy_preview: str = "" + git_log: List[GitEntry] = field(default_factory=list) + + +def _autotrader_cache_dir() -> Path: + return Path.home() / ".cache" / "autotrader" / "data" + + +def get_results(project_dir: Path) -> List[AutoresearchResult]: + """Parse results.tsv (commit, score, sharpe, max_dd, status, description).""" + tsv = project_dir / "results.tsv" + if not tsv.exists(): + return [] + out: List[AutoresearchResult] = [] + lines = tsv.read_text().strip().splitlines() + for line in lines[1:]: # skip header + if not line.strip(): + continue + parts = line.split("\t") + + def _f(idx: int) -> float: + try: + return float(parts[idx]) + except (IndexError, ValueError): + return 0.0 + + out.append( + AutoresearchResult( + commit=parts[0] if len(parts) > 0 else "", + score=_f(1), + sharpe=_f(2), + max_dd=_f(3), + status=parts[4] if len(parts) > 4 else "discarded", + description=parts[5] if len(parts) > 5 else "", + ) + ) + return out + + +def get_strategy_preview(project_dir: Path, limit: int = 2000) -> str: + p = project_dir / "strategy.py" + if not p.exists(): + return "" + try: + return p.read_text()[:limit] + except OSError: + return "" + + +def get_branch(project_dir: Path) -> str: + try: + return subprocess.run( + ["git", "branch", "--show-current"], + cwd=project_dir, text=True, capture_output=True, timeout=3, + ).stdout.strip() or "unknown" + except (subprocess.SubprocessError, OSError): + return "unknown" + + +def get_git_log(project_dir: Path, count: int = 10) -> List[GitEntry]: + try: + out = subprocess.run( + ["git", "log", "--oneline", "--format=%h\t%s\t%ci", f"-{count}"], + cwd=project_dir, text=True, capture_output=True, timeout=5, + ).stdout.strip() + except (subprocess.SubprocessError, OSError): + return [] + if not out: + return [] + entries: List[GitEntry] = [] + for line in out.splitlines(): + parts = line.split("\t") + entries.append(GitEntry( + hash=parts[0] if len(parts) > 0 else "", + message=parts[1] if len(parts) > 1 else "", + date=parts[2] if len(parts) > 2 else "", + )) + return entries + + +def check_data_ready(project_dir: Path) -> bool: + """Crypto majors are the minimum bar for a runnable backtest.""" + cache = _autotrader_cache_dir() + return all( + (cache / f"{sym}_1h.parquet").exists() for sym in ("BTC", "ETH", "SOL") + ) + + +def best_result(results: List[AutoresearchResult]) -> Optional[AutoresearchResult]: + best: Optional[AutoresearchResult] = None + for r in results: + if best is None or r.score > best.score: + best = r + return best + + +def get_status(project_dir: Path) -> LabStatus: + """Full read-only status for an autoresearch project directory.""" + project_dir = Path(project_dir).expanduser().resolve() + results = get_results(project_dir) + best = best_result(results) + return LabStatus( + name=project_dir.name, + path=str(project_dir), + branch=get_branch(project_dir), + data_ready=check_data_ready(project_dir), + best_score=best.score if best else None, + best_commit=best.commit if best else "", + total_experiments=len(results), + results=results, + strategy_preview=get_strategy_preview(project_dir), + git_log=get_git_log(project_dir), + ) diff --git a/cli/commands/autoresearch.py b/cli/commands/autoresearch.py new file mode 100644 index 0000000..04e7baa --- /dev/null +++ b/cli/commands/autoresearch.py @@ -0,0 +1,512 @@ +"""hl autoresearch — autonomous strategy research loop. + +Karpathy-style propose → commit → eval → keep/discard loop over a strategy +project (see `hl strategy new`). Two proposers ship: + + --agent demo parametric: perturbs module-level numeric constants in + strategy.py by a random factor in [0.8, 1.25]. + --agent llm Anthropic Claude rewrites strategy.py end-to-end, one parameter + per experiment. Resolves ANTHROPIC_API_KEY from + env → ~/.nunchi/anthropic_api_key → interactive prompt. + +Commands: + hl autoresearch run [--iterations N] run the loop + hl autoresearch results show last N rows of results.tsv + hl autoresearch status best score / commit / #experiments + hl autoresearch tail [--follow] tail the run's JSONL event log + +Ported from nunchi-cli (which ported it from house/skills/autoresearch). +""" +from __future__ import annotations + +import json +import os +import random +import re +import shutil +import subprocess +import sys +import time +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional + +import typer + +autoresearch_app = typer.Typer( + name="autoresearch", + help="Autonomous strategy research loop (Karpathy autoresearch pattern).", + no_args_is_help=True, + add_completion=False, +) + + +def _strategies_root() -> Path: + override = os.environ.get("NUNCHI_STRATEGIES_DIR") + if override: + return Path(override).expanduser().resolve() + return Path.home() / ".nunchi" / "strategies" + + +def runs_root() -> Path: + return Path.home() / ".nunchi" / "autoresearch_runs" + + +def _python_runner(strategy_dir: Path) -> list[str]: + if shutil.which("uv") and (strategy_dir / "pyproject.toml").exists(): + return ["uv", "run", "python"] + return [sys.executable] + + +def _git(cmd: list[str], cwd: Path) -> subprocess.CompletedProcess: + return subprocess.run(["git"] + cmd, cwd=cwd, check=True, text=True, capture_output=True) + + +def _run_backtest(strategy_dir: Path, timeout: int = 240) -> dict: + """Run backtest.py in the strategy dir; parse `key: value` lines for metrics.""" + log_path = strategy_dir / "run.log" + runner = _python_runner(strategy_dir) + try: + proc = subprocess.run( + runner + ["backtest.py"], + cwd=strategy_dir, timeout=timeout, text=True, capture_output=True, + ) + log_path.write_text(proc.stdout + "\n--- stderr ---\n" + proc.stderr) + except subprocess.TimeoutExpired: + log_path.write_text("TIMEOUT") + return {"score": None, "sharpe": None, "max_dd": None, "returncode": -1, "error": "timeout"} + + metrics: dict = {"score": None, "sharpe": None, "max_dd": None, "returncode": proc.returncode} + for line in proc.stdout.splitlines(): + for key in ("score", "sharpe", "max_drawdown_pct"): + m = re.match(rf"^{key}:\s+(-?[\d.]+)", line.strip()) + if m: + target = "max_dd" if key == "max_drawdown_pct" else key + try: + metrics[target] = float(m.group(1)) + except ValueError: + pass + if proc.returncode != 0: + metrics["error"] = proc.stderr.splitlines()[-1] if proc.stderr else "non-zero exit" + return metrics + + +# Match module-level constants like LOOKBACK = 24 or STOP_LOSS_PCT = 0.03 +_PARAM_PATTERN = re.compile(r"^(?P[A-Z_][A-Z0-9_]*)\s*=\s*(?P-?\d+(?:\.\d+)?)\s*$", re.M) + +# Default model for --agent llm. Override with ANTHROPIC_MODEL or --model. +_DEFAULT_LLM_MODEL = "claude-sonnet-4-6" + +_ANTHROPIC_KEYSTORE = Path.home() / ".nunchi" / "anthropic_api_key" + + +def _resolve_anthropic_key(*, interactive: bool = True) -> str: + """Resolve the Anthropic API key. + + Order: env ANTHROPIC_API_KEY → ~/.nunchi/anthropic_api_key → interactive prompt. + """ + key = os.environ.get("ANTHROPIC_API_KEY", "").strip() + if key: + return key + if _ANTHROPIC_KEYSTORE.exists(): + try: + stored = _ANTHROPIC_KEYSTORE.read_text().strip() + except OSError: + stored = "" + if stored: + os.environ["ANTHROPIC_API_KEY"] = stored + return stored + if not interactive or not sys.stdin.isatty(): + typer.echo( + "error: --agent llm needs an Anthropic API key.\n" + " Set ANTHROPIC_API_KEY in env, write it to " + f"{_ANTHROPIC_KEYSTORE}, or run interactively to be prompted.", + err=True, + ) + raise typer.Exit(code=2) + typer.echo("--agent llm needs an Anthropic API key.") + typer.echo(" Create one at: https://console.anthropic.com/settings/keys") + entered = typer.prompt("ANTHROPIC_API_KEY", hide_input=True).strip() + if not entered: + typer.echo("error: empty API key", err=True) + raise typer.Exit(code=2) + if not entered.startswith("sk-ant-"): + typer.echo( + "warning: key doesn't start with 'sk-ant-' — Anthropic keys usually do.", + err=True, + ) + if typer.confirm(f"Save to {_ANTHROPIC_KEYSTORE} for future runs?", default=True): + _ANTHROPIC_KEYSTORE.parent.mkdir(parents=True, exist_ok=True) + _ANTHROPIC_KEYSTORE.write_text(entered) + try: + _ANTHROPIC_KEYSTORE.chmod(0o600) + except OSError: + pass + typer.echo(f"✓ saved to {_ANTHROPIC_KEYSTORE} (0600)") + os.environ["ANTHROPIC_API_KEY"] = entered + return entered + + +def _llm_propose( + *, + strategy_py: Path, + results_tsv: Path, + program_md: Optional[Path], + iteration: int, + model: str, + api_key: str, +) -> tuple[str, str]: + """LLM-driven proposal — Claude rewrites strategy.py with a single change.""" + try: + import anthropic + except ImportError as e: + raise RuntimeError( + "--agent llm requires the anthropic SDK. Install:\n" + " pip install 'anthropic>=0.40.0'" + ) from e + + strategy_text = strategy_py.read_text() + history = "" + if results_tsv.exists(): + lines = results_tsv.read_text().splitlines() + if len(lines) > 1: + history = "\n".join(lines[:1] + lines[-20:]) + program_blob = "" + if program_md and program_md.exists(): + program_blob = program_md.read_text() + + system = ( + "You are an autonomous AI researcher running the autoresearch loop " + "(Karpathy-style propose → eval → keep/discard) on a Hyperliquid " + "trading strategy. Read program.md, the current strategy.py, and the " + "results.tsv history. Propose EXACTLY ONE single-parameter change — " + "either tweak ONE module-level constant or rewrite ONE small block of " + "logic. NEVER change multiple things at once (attribution + search-" + "space discipline). Prefer simplicity. If results.tsv shows ≥5 " + "consecutive non-improvements, switch to a contrarian / regime-shift " + "/ radical proposal (Council Mode). Output the COMPLETE replacement " + "strategy.py inside ... and a one-line human " + "description inside .... Do not include any other prose." + ) + user = ( + f"# program.md\n{program_blob or '(not present — infer from strategy.py)'}\n\n" + f"# current strategy.py\n```python\n{strategy_text}\n```\n\n" + f"# results.tsv (header + last 20 rows)\n{history or '(empty — this is the first proposal)'}\n\n" + f"Iteration #{iteration}. Propose ONE change.\n" + "Return ONLY the two tagged blocks:\n" + "\n\n\n" + "one-line summary of the single change" + ) + + client = anthropic.Anthropic(api_key=api_key) + msg = client.messages.create( + model=model, + max_tokens=8192, + system=system, + messages=[{"role": "user", "content": user}], + ) + body = "".join(getattr(b, "text", "") for b in msg.content) + sm = re.search(r"\s*(.*?)\s*", body, re.DOTALL) + if not sm: + raise RuntimeError( + f"llm response missing ... block. First 500 chars:\n{body[:500]}" + ) + new_text = sm.group(1) + # Strip optional ```python fences if the model wrapped the code + new_text = re.sub(r"^```(?:python)?\s*\n", "", new_text) + new_text = re.sub(r"\n```\s*$", "", new_text) + if not new_text.strip(): + raise RuntimeError("llm returned an empty block") + dm = re.search(r"\s*(.*?)\s*", body, re.DOTALL) + desc = (dm.group(1).strip() if dm else "llm proposal")[:200] + return desc, new_text + + +def _demo_propose(strategy_py: Path, rng: random.Random) -> tuple[str, str]: + """Pick a numeric module-level constant and perturb it by a factor in [0.8, 1.25].""" + text = strategy_py.read_text() + matches = list(_PARAM_PATTERN.finditer(text)) + if not matches: + return ("no-op: no module-level numeric params found", text) + pick = rng.choice(matches) + name = pick.group("name") + raw = pick.group("val") + val = float(raw) + factor = rng.uniform(0.8, 1.25) + new = val * factor + if val >= 0 and new < 0: + new = abs(new) + if "." not in raw: + new_str = str(max(1, int(round(new)))) + else: + new_str = f"{new:.6f}".rstrip("0").rstrip(".") + if new_str == "": + new_str = "0.0" + new_text = text[: pick.start()] + f"{name} = {new_str}" + text[pick.end():] + desc = f"perturb {name}: {raw} → {new_str} (x{factor:.3f})" + return desc, new_text + + +@autoresearch_app.command("run") +def autoresearch_run( + name: str = typer.Argument(..., help="Strategy name (create with `hl strategy new`)"), + iterations: int = typer.Option(5, "--iterations", "-n"), + seed: Optional[int] = typer.Option(None, "--seed"), + agent: str = typer.Option("demo", "--agent", help="demo | llm — demo perturbs numeric constants; llm uses Anthropic Claude"), + model: Optional[str] = typer.Option(None, "--model", help=f"Override LLM model when --agent llm (default {_DEFAULT_LLM_MODEL}, or $ANTHROPIC_MODEL)"), + tag: Optional[str] = typer.Option(None, "--tag", help="Branch tag; default today's date"), + run_id: Optional[str] = typer.Option(None, "--run-id", help="Override auto-generated run id"), + json_output: bool = typer.Option(False, "--json"), +): + """Run the autoresearch loop. Prints a JSONL event stream + writes results.tsv.""" + strategy_dir = _strategies_root() / name + if not strategy_dir.exists(): + typer.echo(f"error: strategy {name!r} not found at {strategy_dir}", err=True) + typer.echo(f"hint: hl strategy new {name}", err=True) + raise typer.Exit(code=2) + + if agent not in ("demo", "llm"): + typer.echo(f"error: --agent={agent!r} not supported. Choose 'demo' or 'llm'.", err=True) + raise typer.Exit(code=2) + + strategy_py = strategy_dir / "strategy.py" + results_tsv = strategy_dir / "results.tsv" + program_md = strategy_dir / "program.md" + if not strategy_py.exists(): + typer.echo(f"error: {strategy_py} missing", err=True) + raise typer.Exit(code=2) + if not results_tsv.exists(): + results_tsv.write_text("commit\tscore\tsharpe\tmax_dd\tstatus\tdescription\n") + + api_key: Optional[str] = None + llm_model = model or os.environ.get("ANTHROPIC_MODEL", "").strip() or _DEFAULT_LLM_MODEL + if agent == "llm": + api_key = _resolve_anthropic_key() + try: + import anthropic # noqa: F401 (early check before the baseline spends time) + except ImportError: + typer.echo( + "error: --agent llm requires the anthropic SDK. Install with:\n" + " pip install 'anthropic>=0.40.0'", + err=True, + ) + raise typer.Exit(code=2) + + run_id = run_id or uuid.uuid4().hex[:12] + run_tag = tag or datetime.now(timezone.utc).strftime("%Y%m%d") + runs_root().mkdir(parents=True, exist_ok=True) + run_log = runs_root() / f"{run_id}.jsonl" + + def emit(event: str, **payload): + rec = {"ts": time.time(), "event": event, **payload} + with run_log.open("a") as f: + f.write(json.dumps(rec) + "\n") + if not json_output: + visible = {k: v for k, v in payload.items() if k not in ("log",)} + typer.echo(f"[{event}] " + json.dumps(visible)) + + emit( + "run_start", + run_id=run_id, + strategy=name, + iterations=iterations, + agent=agent, + tag=run_tag, + model=llm_model if agent == "llm" else None, + ) + + in_git = (strategy_dir / ".git").exists() + branch = f"autoresearch/{run_tag}-{run_id}" + if in_git: + try: + _git(["checkout", "-b", branch], cwd=strategy_dir) + except subprocess.CalledProcessError as e: + emit("git_branch_failed", error=(e.stderr or str(e))[:200]) + in_git = False + + emit("baseline_start") + baseline = _run_backtest(strategy_dir) + emit("baseline_done", **baseline) + head = "BASELINE" + if in_git: + try: + head = _git(["rev-parse", "--short", "HEAD"], cwd=strategy_dir).stdout.strip() + except subprocess.CalledProcessError: + pass + with results_tsv.open("a") as f: + f.write( + f"{head}\t{baseline.get('score') if baseline.get('score') is not None else '?'}" + f"\t{baseline.get('sharpe') if baseline.get('sharpe') is not None else '?'}" + f"\t{baseline.get('max_dd') if baseline.get('max_dd') is not None else '?'}" + f"\tbaseline\tinitial scaffold\n" + ) + best_score: float = baseline.get("score") if baseline.get("score") is not None else float("-inf") + + rng = random.Random(seed) + + for i in range(1, iterations + 1): + try: + if agent == "demo": + desc, new_text = _demo_propose(strategy_py, rng) + else: # agent == "llm" + assert api_key is not None + desc, new_text = _llm_propose( + strategy_py=strategy_py, + results_tsv=results_tsv, + program_md=program_md, + iteration=i, + model=llm_model, + api_key=api_key, + ) + except Exception as e: + emit("proposal_failed", iteration=i, error=str(e)[:500]) + continue + emit("proposed", iteration=i, description=desc) + strategy_py.write_text(new_text) + + commit = "DRY" + if in_git: + try: + _git(["add", "strategy.py"], cwd=strategy_dir) + _git(["commit", "-m", f"autoresearch {i}: {desc}"], cwd=strategy_dir) + commit = _git(["rev-parse", "--short", "HEAD"], cwd=strategy_dir).stdout.strip() + except subprocess.CalledProcessError as e: + emit("commit_failed", iteration=i, error=(e.stderr or str(e))[:200]) + + metrics = _run_backtest(strategy_dir) + emit("eval_result", iteration=i, **metrics) + + score = metrics.get("score") + keep = score is not None and score > best_score + status = "kept" if keep else "discarded" + if keep: + best_score = score + emit("kept", iteration=i, score=score, commit=commit) + else: + emit("discarded", iteration=i, score=score, commit=commit) + if in_git and commit != "DRY": + try: + _git(["reset", "--hard", "HEAD~1"], cwd=strategy_dir) + except subprocess.CalledProcessError as e: + emit("revert_failed", iteration=i, error=(e.stderr or str(e))[:200]) + + with results_tsv.open("a") as f: + f.write( + f"{commit}\t{score if score is not None else '?'}" + f"\t{metrics.get('sharpe') if metrics.get('sharpe') is not None else '?'}" + f"\t{metrics.get('max_dd') if metrics.get('max_dd') is not None else '?'}" + f"\t{status}\t{desc}\n" + ) + + emit("done", run_id=run_id, best_score=best_score if best_score != float("-inf") else None) + + summary = { + "run_id": run_id, + "strategy": name, + "iterations": iterations, + "best_score": best_score if best_score != float("-inf") else None, + "results_tsv": str(results_tsv), + "log": str(run_log), + "branch": branch if in_git else None, + } + if json_output: + typer.echo(json.dumps(summary, indent=2)) + else: + typer.echo(f"\n✓ autoresearch done — run_id={run_id} best_score={summary['best_score']}") + typer.echo(f" results: {results_tsv}") + typer.echo(f" log: {run_log}") + + +@autoresearch_app.command("results") +def autoresearch_results( + name: str = typer.Argument(...), + limit: int = typer.Option(20, "--limit", "-n"), + json_output: bool = typer.Option(False, "--json"), +): + """Show the last N results from results.tsv.""" + path = _strategies_root() / name / "results.tsv" + if not path.exists(): + typer.echo(f"error: {path} not found", err=True) + raise typer.Exit(code=2) + lines = path.read_text().splitlines() + if not lines: + typer.echo("(empty)") + return + header = lines[0].split("\t") + rows = [dict(zip(header, line.split("\t"))) for line in lines[1:][-limit:]] + if json_output: + typer.echo(json.dumps(rows, indent=2)) + return + typer.echo(f"{'Commit':<12} {'Score':<10} {'Sharpe':<10} {'MaxDD':<10} {'Status':<10} Description") + typer.echo(f"{'-'*12} {'-'*10} {'-'*10} {'-'*10} {'-'*10} {'-'*40}") + for r in rows: + typer.echo( + f"{r.get('commit','')[:12]:<12} {r.get('score',''):<10} {r.get('sharpe',''):<10}" + f" {r.get('max_dd',''):<10} {r.get('status',''):<10} {r.get('description','')}" + ) + + +@autoresearch_app.command("status") +def autoresearch_status( + name: str = typer.Argument(...), + json_output: bool = typer.Option(False, "--json"), +): + """Show best score / commit / experiment count for a strategy project. + + Read-only summary via the autotrader bridge (Python port of ACC's + autotrader-bridge.ts results.tsv parser). + """ + project_root = str(Path(__file__).resolve().parent.parent.parent) + if project_root not in sys.path: + sys.path.insert(0, project_root) + from cli.autotrader_bridge import get_status + + strategy_dir = _strategies_root() / name + if not strategy_dir.exists(): + typer.echo(f"error: strategy {name!r} not found at {strategy_dir}", err=True) + raise typer.Exit(code=2) + + st = get_status(strategy_dir) + if json_output: + typer.echo(json.dumps({ + "name": st.name, + "path": st.path, + "branch": st.branch, + "data_ready": st.data_ready, + "best_score": st.best_score, + "best_commit": st.best_commit, + "total_experiments": st.total_experiments, + }, indent=2)) + return + + typer.echo(f"\033[1m{st.name}\033[0m ({st.path})") + typer.echo(f" branch: {st.branch}") + typer.echo(f" data ready: {st.data_ready} (BTC/ETH/SOL parquet present)") + typer.echo(f" experiments: {st.total_experiments}") + typer.echo(f" best score: {st.best_score if st.best_score is not None else '—'}") + typer.echo(f" best commit: {st.best_commit or '—'}") + if st.git_log: + typer.echo(" recent commits:") + for e in st.git_log[:5]: + typer.echo(f" {e.hash:<10} {e.message}") + + +@autoresearch_app.command("tail") +def autoresearch_tail( + run_id: str = typer.Argument(..., help="Run ID returned by `hl autoresearch run`"), + follow: bool = typer.Option(False, "--follow", "-f"), +): + """Tail a run's JSONL event log.""" + path = runs_root() / f"{run_id}.jsonl" + if not path.exists(): + typer.echo(f"error: {path} not found", err=True) + raise typer.Exit(code=2) + if not follow: + typer.echo(path.read_text()) + return + proc = subprocess.Popen(["tail", "-n", "+1", "-f", str(path)]) + try: + proc.wait() + except KeyboardInterrupt: + proc.terminate() diff --git a/cli/commands/strategy.py b/cli/commands/strategy.py new file mode 100644 index 0000000..a2b706a --- /dev/null +++ b/cli/commands/strategy.py @@ -0,0 +1,426 @@ +"""hl strategy — strategy scaffold, lifecycle, and live load. + +Each strategy is a project directory holding the autotrader scaffold (program.md ++ strategy.py + backtest.py + prepare.py + benchmarks/ + results.tsv). The +autoresearch loop (`hl autoresearch run`) mutates strategy.py and logs to +results.tsv. Once a strategy is found, `hl strategy load` wraps the SAME hourly +`on_bar` strategy in the AutoBarStrategy tick adapter and runs it through the +live TradingEngine. + +The scaffold was ported from Nunchi-trade/house (apps/autotrader) via nunchi-cli. + +Commands: + hl strategy new scaffold a new strategy project from the template + hl strategy list list local strategies + registered runtime strategies + hl strategy show show metadata + latest results + hl strategy path print the strategy directory path + hl strategy prepare download/prepare backtest data (runs prepare.py) + hl strategy load run the strategy live (on_bar -> on_tick bridge) +""" +from __future__ import annotations + +import json +import logging +import os +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Optional + +import typer + +strategy_app = typer.Typer( + name="strategy", + help="Strategy scaffold, lifecycle, and live load (autotrader template). See `hl strategies` for the runtime registry.", + no_args_is_help=True, + add_completion=False, +) + + +def strategies_root() -> Path: + """Default root for local strategy projects. Override with NUNCHI_STRATEGIES_DIR.""" + override = os.environ.get("NUNCHI_STRATEGIES_DIR") + if override: + return Path(override).expanduser().resolve() + return Path.home() / ".nunchi" / "strategies" + + +def template_root() -> Path: + """Built-in autotrader scaffold template.""" + return Path(__file__).resolve().parent.parent.parent / "spawn" / "templates" / "strategy" + + +def _python_runner(strategy_dir: Path) -> list[str]: + """Prefer `uv run python` if uv is installed and the strategy has a + pyproject.toml; else the current interpreter.""" + if shutil.which("uv") and (strategy_dir / "pyproject.toml").exists(): + return ["uv", "run", "python"] + return [sys.executable] + + +@strategy_app.command("new") +def strategy_new( + name: str = typer.Argument(..., help="Strategy name (also used as dir + branch tag)"), + dir: Optional[Path] = typer.Option(None, "--dir", "-d", help="Parent directory override (default ~/.nunchi/strategies/)"), + no_git: bool = typer.Option(False, "--no-git", help="Skip git init"), + json_output: bool = typer.Option(False, "--json"), +): + """Scaffold a new strategy project from the autotrader template.""" + parent = (dir or strategies_root()).expanduser().resolve() + target = parent / name + + if target.exists(): + typer.echo(f"error: {target} already exists", err=True) + raise typer.Exit(code=2) + + template = template_root() + if not template.exists(): + typer.echo(f"error: template not found at {template}", err=True) + raise typer.Exit(code=2) + + parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(template, target, ignore=shutil.ignore_patterns("__pycache__")) + + # .gitignore experiment artifacts so `git reset --hard` (discard path) won't + # roll them back. + (target / ".gitignore").write_text( + "# autoresearch experiment artifacts — untracked so discard-resets don't clobber them\n" + "results.tsv\n" + "run.log\n" + "council_log.md\n" + "uv.lock\n" + "__pycache__/\n" + ".venv/\n" + ".pytest_cache/\n" + ) + + # Seed results.tsv header (autoresearch loop appends rows here). + (target / "results.tsv").write_text("commit\tscore\tsharpe\tmax_dd\tstatus\tdescription\n") + + git_initialized = False + if not no_git: + try: + subprocess.run(["git", "init", "-q"], cwd=target, check=True) + subprocess.run(["git", "add", "-A"], cwd=target, check=True) + subprocess.run( + ["git", "commit", "-q", "-m", f"scaffold strategy {name} from autotrader template"], + cwd=target, check=True, + ) + git_initialized = True + except (subprocess.CalledProcessError, FileNotFoundError) as e: + typer.echo(f"warn: git init failed ({e}); continuing without git", err=True) + + payload = { + "name": name, + "path": str(target), + "files": sorted(p.name for p in target.iterdir() if not p.name.startswith(".")), + "git": git_initialized, + "template_source": "Nunchi-trade/house:apps/autotrader (via nunchi-cli)", + } + if json_output: + typer.echo(json.dumps(payload, indent=2)) + else: + typer.echo(f"✓ scaffolded {name} at {target}") + typer.echo(f" git: {'initialized' if git_initialized else 'skipped'}") + typer.echo(f" next: hl strategy prepare {name} (download data)") + typer.echo(f" hl autoresearch run {name} (start the loop)") + typer.echo(f" hl strategy load {name} --mock (dry-run the bridge)") + + +@strategy_app.command("list") +def strategy_list( + json_output: bool = typer.Option(False, "--json"), +): + """List local strategy projects + runtime-registered strategies.""" + root = strategies_root() + local: list[dict] = [] + if root.exists(): + for entry in sorted(root.iterdir()): + if not entry.is_dir(): + continue + results_tsv = entry / "results.tsv" + latest_score: Optional[float] = None + iterations = 0 + if results_tsv.exists(): + lines = results_tsv.read_text().splitlines()[1:] # skip header + iterations = len(lines) + if lines: + parts = lines[-1].split("\t") + if len(parts) >= 2: + try: + latest_score = float(parts[1]) + except ValueError: + pass + local.append({ + "name": entry.name, + "path": str(entry), + "iterations": iterations, + "latest_score": latest_score, + }) + + registered: list[str] = [] + try: + project_root = str(Path(__file__).resolve().parent.parent.parent) + if project_root not in sys.path: + sys.path.insert(0, project_root) + from cli.strategy_registry import STRATEGY_REGISTRY + registered = sorted(STRATEGY_REGISTRY.keys()) + except Exception as e: + if not json_output: + typer.echo(f"warn: could not load runtime registry: {e}", err=True) + + if json_output: + typer.echo(json.dumps({"local": local, "registered": registered}, indent=2)) + return + + if local: + typer.echo(f"\033[1mLocal strategy projects ({root}):\033[0m") + typer.echo(f"{'Name':<24} {'Iterations':<12} {'Latest Score':<14} Path") + typer.echo(f"{'-'*24} {'-'*12} {'-'*14} {'-'*40}") + for s in local: + score = f"{s['latest_score']:.4f}" if s["latest_score"] is not None else "—" + typer.echo(f"\033[36m{s['name']:<24}\033[0m {s['iterations']:<12} {score:<14} {s['path']}") + else: + typer.echo("(no local strategies — run `hl strategy new ` to scaffold one)") + + typer.echo("") + typer.echo(f"\033[1mRuntime registry ({len(registered)} strategies — `hl strategies` for full table):\033[0m") + for name in registered: + typer.echo(f" • {name}") + + +@strategy_app.command("show") +def strategy_show( + name: str = typer.Argument(...), + json_output: bool = typer.Option(False, "--json"), +): + """Show strategy metadata + last 5 results.""" + path = strategies_root() / name + if not path.exists(): + typer.echo(f"error: {path} does not exist", err=True) + raise typer.Exit(code=2) + + program = (path / "program.md").read_text() if (path / "program.md").exists() else "(no program.md)" + strategy_lines = (path / "strategy.py").read_text().count("\n") if (path / "strategy.py").exists() else 0 + results_tsv = path / "results.tsv" + iterations = 0 + rows: list[dict] = [] + if results_tsv.exists(): + lines = results_tsv.read_text().splitlines() + if lines: + header = lines[0].split("\t") + iterations = len(lines) - 1 + for line in lines[1:][-5:]: + parts = line.split("\t") + rows.append(dict(zip(header, parts))) + + payload = { + "name": name, + "path": str(path), + "strategy_py_lines": strategy_lines, + "iterations": iterations, + "last_5": rows, + } + if json_output: + typer.echo(json.dumps(payload, indent=2)) + return + + typer.echo(f"\033[1m{name}\033[0m ({path})") + typer.echo(f" strategy.py: {strategy_lines} lines") + typer.echo(f" iterations: {iterations}") + if rows: + typer.echo(" last 5 results:") + for r in rows: + typer.echo( + f" {r.get('commit', '')[:12]:<12} score={r.get('score', '?'):<10}" + f" status={r.get('status', '?'):<10} {r.get('description', '')}" + ) + typer.echo("\n--- program.md (first 500 chars) ---") + typer.echo(program[:500]) + + +@strategy_app.command("path") +def strategy_path(name: str = typer.Argument(...)): + """Print the strategy directory path. Useful for `cd $(hl strategy path foo)`.""" + typer.echo(str(strategies_root() / name)) + + +@strategy_app.command("prepare") +def strategy_prepare( + name: str = typer.Argument(...), + symbols: Optional[str] = typer.Option(None, "--symbols", help="Comma-separated symbol list (e.g. BTC,BTCSWP,GOLD)"), +): + """Download/prepare backtest data (delegates to prepare.py inside the strategy dir).""" + strategy_dir = strategies_root() / name + if not strategy_dir.exists(): + typer.echo(f"error: {strategy_dir} not found", err=True) + raise typer.Exit(code=2) + runner = _python_runner(strategy_dir) + cmd = runner + ["prepare.py"] + if symbols: + # prepare.py takes space-separated --symbols; split the comma list. + cmd += ["--symbols"] + [s.strip() for s in symbols.split(",") if s.strip()] + proc = subprocess.run(cmd, cwd=strategy_dir) + raise typer.Exit(code=proc.returncode) + + +def _best_score(strategy_dir: Path) -> Optional[float]: + """Read the best (max) score from results.tsv — the drift baseline.""" + results_tsv = strategy_dir / "results.tsv" + if not results_tsv.exists(): + return None + best: Optional[float] = None + for line in results_tsv.read_text().splitlines()[1:]: + parts = line.split("\t") + if len(parts) >= 2: + try: + s = float(parts[1]) + except ValueError: + continue + if best is None or s > best: + best = s + return best + + +@strategy_app.command("load") +def strategy_load( + name: str = typer.Argument(..., help="Strategy project name (created via `hl strategy new`)"), + instrument: str = typer.Option( + "ETH-PERP", "--instrument", "-i", + help="Instrument to trade (ETH-PERP, BTC-PERP, BTCSWP-USDYP, xyz commodity instruments)", + ), + tick_interval: float = typer.Option(10.0, "--tick", "-t", help="Seconds between ticks"), + ticks_per_hour: int = typer.Option( + 360, "--ticks-per-hour", + help="Ticks folded into one on_bar boundary (360 = 1h @ 10s). Lower for fast tests.", + ), + mainnet: bool = typer.Option(False, "--mainnet", help="Use mainnet (default: testnet)"), + dry_run: bool = typer.Option(False, "--dry-run", help="Run the bridge but place no real orders"), + mock: bool = typer.Option(False, "--mock", help="Use mock market data (no HL connection)"), + max_ticks: int = typer.Option(0, "--max-ticks", help="Stop after N ticks (0 = forever)"), + resume: bool = typer.Option(True, "--resume/--fresh", help="Resume from saved state or start fresh"), + data_dir: Optional[str] = typer.Option(None, "--data-dir", help="State + trade log dir (default data/cli/)"), + strategy_path: Optional[Path] = typer.Option( + None, "--strategy-path", + help="Override path to strategy.py (default ~/.nunchi/strategies//strategy.py)", + ), +): + """Run an autoresearch strategy LIVE via the on_bar -> on_tick bridge. + + Wraps the project's hourly `on_bar` strategy in AutoBarStrategy (rolling + tick->bar buffer, signed-USD target -> order translation, drift detection, + reduce_only/safe_mode guardrails) and runs it through the standard + TradingEngine — the same loop `hl run` uses. + """ + project_root = str(Path(__file__).resolve().parent.parent.parent) + if project_root not in sys.path: + sys.path.insert(0, project_root) + + from cli.config import TradingConfig + from cli.strategy_registry import resolve_instrument + from sdk.strategy_sdk.autobar_adapter import AutoBarStrategy + + strat_dir = strategies_root() / name + strat_py = strategy_path.expanduser().resolve() if strategy_path else (strat_dir / "strategy.py") + if not strat_py.exists(): + typer.echo(f"error: strategy.py not found at {strat_py}", err=True) + typer.echo(f"hint: hl strategy new {name}", err=True) + raise typer.Exit(code=2) + + cfg = TradingConfig() + cfg.strategy = name + cfg.instrument = resolve_instrument(instrument) + cfg.tick_interval = tick_interval + cfg.mainnet = mainnet + cfg.dry_run = dry_run + cfg.max_ticks = max_ticks + cfg.data_dir = data_dir or f"data/cli/{name}" + + logging.basicConfig( + level=getattr(logging, cfg.log_level.upper(), logging.INFO), + format="%(asctime)s %(name)-14s %(levelname)-5s %(message)s", + datefmt="%H:%M:%S", + ) + + # ── Network guard: prevent wrong-chain accidents (mirrors run.py) ── + if cfg.mainnet: + if os.environ.get("HL_TESTNET", "true").lower() == "true": + typer.echo( + "FATAL: --mainnet set but HL_TESTNET=true in environment. Refusing to start.", + err=True, + ) + raise typer.Exit(code=1) + else: + if os.environ.get("HL_TESTNET", "true").lower() == "false": + typer.echo( + "FATAL: testnet mode but HL_TESTNET=false in environment. Pass --mainnet or fix env.", + err=True, + ) + raise typer.Exit(code=1) + + # Drift baseline: best kept backtest score from results.tsv. + backtest_score = _best_score(strat_dir) + + def _retrain_hook(reason: str, stats: dict) -> None: + # Retrain-trigger hook: the runner just logs + drops a marker file other + # tooling (or an autoresearch supervisor) can poll. Kept real but simple. + logging.getLogger("autobar").warning( + "RETRAIN TRIGGER [%s]: %s | %s", name, reason, json.dumps(stats), + ) + try: + marker = strat_dir / "retrain.flag" + marker.write_text(json.dumps({"reason": reason, **stats}, indent=2)) + typer.echo(f" ↳ wrote retrain marker: {marker}") + except OSError: + pass + + strategy_instance = AutoBarStrategy( + strategy_id=name, + strategy_path=str(strat_py), + ticks_per_hour=ticks_per_hour, + backtest_score=backtest_score, + retrain_callback=_retrain_hook, + ) + + # Build HL adapter (mirrors run.py). + if mock or dry_run: + from cli.hl_adapter import DirectMockProxy + hl = DirectMockProxy() + typer.echo(f"Mode: {'DRY RUN' if dry_run else 'MOCK'}") + else: + from cli.hl_adapter import DirectHLProxy + from parent.hl_proxy import HLProxy + private_key = cfg.get_private_key() + raw_hl = HLProxy(private_key=private_key, testnet=not cfg.mainnet) + hl = DirectHLProxy(raw_hl) + typer.echo(f"Mode: LIVE ({'mainnet' if cfg.mainnet else 'testnet'})") + + builder_cfg = cfg.get_builder_config() + builder_info = builder_cfg.to_builder_info() + + typer.echo(f"Strategy: {name} (on_bar -> on_tick bridge)") + typer.echo(f" source: {strat_py}") + typer.echo(f"Instrument: {cfg.instrument}") + typer.echo(f"Tick: {cfg.tick_interval}s bar boundary every {ticks_per_hour} ticks") + typer.echo( + f"Drift base: backtest score = " + f"{backtest_score if backtest_score is not None else 'n/a (no results.tsv)'}" + ) + if cfg.max_ticks > 0: + typer.echo(f"Max ticks: {cfg.max_ticks}") + typer.echo("") + + from cli.engine import TradingEngine + + engine = TradingEngine( + hl=hl, + strategy=strategy_instance, + instrument=cfg.instrument, + tick_interval=cfg.tick_interval, + dry_run=cfg.dry_run, + data_dir=cfg.data_dir, + risk_limits=cfg.to_risk_limits(), + builder=builder_info, + ) + engine.run(max_ticks=cfg.max_ticks, resume=resume) diff --git a/cli/main.py b/cli/main.py index 6253f07..9783c23 100644 --- a/cli/main.py +++ b/cli/main.py @@ -35,6 +35,8 @@ from cli.commands.skills import skills_app from cli.commands.journal import journal_app from cli.commands.keys import keys_app +from cli.commands.strategy import strategy_app +from cli.commands.autoresearch import autoresearch_app app.command("run", help="Start autonomous trading with a strategy")(run_cmd) app.command("status", help="Show positions, PnL, and risk state")(status_cmd) @@ -53,6 +55,8 @@ app.add_typer(skills_app, name="skills", help="Skill discovery and registry") app.add_typer(journal_app, name="journal", help="Trade journal — structured position records with reasoning") app.add_typer(keys_app, name="keys", help="Unified key management across backends") +app.add_typer(strategy_app, name="strategy", help="Strategy scaffold, lifecycle, and live load (autoresearch -> live bridge)") +app.add_typer(autoresearch_app, name="autoresearch", help="Autonomous strategy research loop (propose -> eval -> keep/discard)") def main(): diff --git a/sdk/strategy_sdk/autobar_adapter.py b/sdk/strategy_sdk/autobar_adapter.py new file mode 100644 index 0000000..2c2d70e --- /dev/null +++ b/sdk/strategy_sdk/autobar_adapter.py @@ -0,0 +1,506 @@ +"""AutoBarStrategy — bridge an autoresearch (hourly `on_bar`) strategy into the +tick-level `on_tick` engine. + +An autoresearch project (see `hl strategy new` / `hl autoresearch run`) exposes a +``strategy.py`` with:: + + def on_bar(bar_data: dict[str, BarData], portfolio: PortfolioState) -> list[Signal] + +where each ``Signal.target_position`` is a *signed USD notional* target for a +symbol (``+`` long, ``-`` short), and ``BarData`` carries a rolling history +DataFrame. That contract is hourly; agent-cli's TradingEngine is tick-level +(``BaseStrategy.on_tick`` every ``tick_interval`` seconds). + +This adapter is the faithful Python port of ACC's +``server/src/strategy-deployer.ts`` translation layer: + + * Maintain a rolling deque of ~360 ten-second ticks (= 1 hour) per instrument. + * Detect an hourly bar boundary (wall-clock hour change, with a tick-count + fallback so it also fires under mock/replay where the clock barely moves). + * On a boundary, fold the ticks into one OHLCV close, append to the hourly + history, build ``BarData`` + ``PortfolioState`` and call the external + ``on_bar`` exactly once. Hold the resulting target between bars (no + re-evaluation mid-bar). + * Translate each ``Signal.target_position`` (signed USD) into a delta vs the + engine's live ``context.position_notional`` and emit an IOC + ``StrategyDecision`` for the residual. + +It deliberately does NOT modify the external strategy — it only adapts the call +shape — so the SAME ``strategy.py`` the autoresearch loop backtested runs live. +""" +from __future__ import annotations + +import importlib.util +import logging +import math +import sys +from collections import deque +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional + +from common.models import ( + MarketSnapshot, + StrategyDecision, + instrument_to_asset, +) +from sdk.strategy_sdk.base import BaseStrategy, StrategyContext + +log = logging.getLogger("autobar") + +# 10s ticks * 360 = 1 hour. Mirrors strategy-deployer.ts (deque maxlen 360). +TICKS_PER_HOUR = 360 +# Hourly history depth handed to on_bar via BarData.history. The autotrader +# scaffold uses LOOKBACK_BARS=500; match it so live history shape == backtest. +DEFAULT_HISTORY_BARS = 500 +MS_PER_HOUR = 3_600_000 + + +def _load_external_strategy(strategy_py: Path): + """Import an external strategy.py and return an instantiated `Strategy()`. + + The file lives outside the package tree (``~/.nunchi/strategies//``), + so we load it by path. Its directory is prepended to ``sys.path`` first so + its own ``from prepare import Signal, BarData, PortfolioState`` resolves + against the sibling ``prepare.py`` in the scaffold. + """ + strategy_py = strategy_py.expanduser().resolve() + if not strategy_py.exists(): + raise FileNotFoundError(f"strategy file not found: {strategy_py}") + + strat_dir = str(strategy_py.parent) + if strat_dir not in sys.path: + sys.path.insert(0, strat_dir) + + # Unique module name so two strategies don't collide in sys.modules. + mod_name = f"_autobar_ext_{strategy_py.parent.name}_{strategy_py.stem}" + spec = importlib.util.spec_from_file_location(mod_name, strategy_py) + if spec is None or spec.loader is None: + raise ImportError(f"cannot build import spec for {strategy_py}") + module = importlib.util.module_from_spec(spec) + sys.modules[mod_name] = module + spec.loader.exec_module(module) + + if not hasattr(module, "Strategy"): + raise AttributeError( + f"{strategy_py} does not define a `Strategy` class " + "(autoresearch scaffold contract)" + ) + return module + + +class _TickBar: + """Accumulates 10s ticks into a single OHLCV hourly bar.""" + + __slots__ = ("open", "high", "low", "close", "volume", "funding_rate", "n") + + def __init__(self, price: float, funding_rate: float = 0.0) -> None: + self.open = price + self.high = price + self.low = price + self.close = price + self.volume = 0.0 + self.funding_rate = funding_rate + self.n = 1 + + def update(self, price: float, funding_rate: float, volume_delta: float) -> None: + self.high = max(self.high, price) + self.low = min(self.low, price) + self.close = price + self.funding_rate = funding_rate + self.volume += max(0.0, volume_delta) + self.n += 1 + + +class AutoBarStrategy(BaseStrategy): + """Adapts an external hourly ``on_bar`` strategy to ``on_tick``. + + Args: + strategy_id: engine strategy id (also the project name). + strategy_path: path to the external ``strategy.py`` exposing ``Strategy`` + with ``on_bar(bar_data, portfolio) -> list[Signal]``. Defaults to + ``~/.nunchi/strategies//strategy.py``. + ticks_per_hour: ticks that fold into one bar AND the tick-count fallback + boundary. Lower it for fast tests/replay (e.g. 10). + history_bars: hourly bars kept in ``BarData.history``. + initial_capital: equity floor used to seed ``PortfolioState`` before the + engine reports real account value. + backtest_score: the strategy's kept backtest score, used as the drift + baseline (read from results.tsv by the runner). ``None`` disables + score-vs-live drift comparison. + drift_window: number of completed bars of live returns to keep for the + rolling live-Sharpe drift estimate. + retrain_callback: optional ``callable(reason: str, stats: dict)`` invoked + once when drift crosses the threshold (retrain-trigger hook). + """ + + def __init__( + self, + strategy_id: str = "autobar", + strategy_path: Optional[str] = None, + ticks_per_hour: int = TICKS_PER_HOUR, + history_bars: int = DEFAULT_HISTORY_BARS, + initial_capital: float = 100_000.0, + backtest_score: Optional[float] = None, + drift_window: int = 168, + retrain_callback: Optional[Callable[[str, Dict[str, Any]], None]] = None, + **_ignored: Any, + ) -> None: + super().__init__(strategy_id=strategy_id) + + if strategy_path is None: + strategy_path = str( + Path.home() / ".nunchi" / "strategies" / strategy_id / "strategy.py" + ) + self.strategy_path = Path(strategy_path) + + module = _load_external_strategy(self.strategy_path) + self._ext_module = module + self._ext = module.Strategy() + # The scaffold's dataclasses live in the sibling prepare.py; pull them in + # by way of the strategy module's own imports so we build the exact types + # on_bar expects. + self._Signal = getattr(module, "Signal", None) or self._import_from_prepare("Signal") + self._BarData = getattr(module, "BarData", None) or self._import_from_prepare("BarData") + self._PortfolioState = ( + getattr(module, "PortfolioState", None) + or self._import_from_prepare("PortfolioState") + ) + + self.ticks_per_hour = max(1, int(ticks_per_hour)) + self.history_bars = max(1, int(history_bars)) + self.initial_capital = float(initial_capital) + + # Rolling tick buffer + hourly history, per instrument (multi-symbol safe). + self._cur_bar: Dict[str, _TickBar] = {} + self._ticks_in_bar: Dict[str, int] = {} + self._hourly: Dict[str, deque] = {} + self._last_hour_idx: Dict[str, int] = {} + # Last computed target USD notional per symbol — held between bars. + self._target_usd: Dict[str, float] = {} + # avg entry price per symbol, for PortfolioState.entry_prices. + self._entry_px: Dict[str, float] = {} + + # ── Guardrails ── + self.backtest_score = backtest_score + self.drift_window = max(8, int(drift_window)) + self._live_returns: deque = deque(maxlen=self.drift_window) + self._last_bar_equity: Optional[float] = None + self.retrain_callback = retrain_callback + self._retrain_fired = False + self.drift_state: Dict[str, Any] = { + "live_sharpe": None, + "live_return_pct": None, + "backtest_score": backtest_score, + "diverged": False, + "bars_observed": 0, + } + + # ------------------------------------------------------------------ helpers + def _import_from_prepare(self, name: str): + prep = self.strategy_path.parent / "prepare.py" + if not prep.exists(): + raise ImportError( + f"{name} not found in strategy module and no prepare.py at {prep}" + ) + spec = importlib.util.spec_from_file_location( + f"_autobar_prep_{self.strategy_path.parent.name}", prep + ) + if spec is None or spec.loader is None: + raise ImportError(f"cannot import prepare.py at {prep}") + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + obj = getattr(mod, name, None) + if obj is None: + raise ImportError(f"{name} not defined in {prep}") + return obj + + def _hour_index(self, snapshot: MarketSnapshot) -> int: + """Wall-clock hour bucket. 0 if no timestamp (replay/mock fallback).""" + if snapshot.timestamp_ms > 0: + return snapshot.timestamp_ms // MS_PER_HOUR + return 0 + + def _history_df(self, instrument: str): + """Build the BarData.history DataFrame from completed hourly bars. + + Includes an ``isfr_rate`` column (0.0 live) so scaffolds that read + ``history["isfr_rate"]`` don't KeyError — matching prepare.py's schema. + """ + import pandas as pd + + bars = list(self._hourly.get(instrument, ())) + if not bars: + return pd.DataFrame( + columns=[ + "timestamp", "open", "high", "low", + "close", "volume", "funding_rate", "isfr_rate", + ] + ) + return pd.DataFrame(bars) + + # -------------------------------------------------------------------- bridge + def on_tick( + self, + snapshot: MarketSnapshot, + context: Optional[StrategyContext] = None, + ) -> List[StrategyDecision]: + if snapshot.mid_price <= 0: + return [] + + instrument = snapshot.instrument + symbol = instrument_to_asset(instrument) + + # 1) Fold this tick into the in-progress bar. + cur = self._cur_bar.get(instrument) + if cur is None: + self._cur_bar[instrument] = _TickBar(snapshot.mid_price, snapshot.funding_rate) + self._ticks_in_bar[instrument] = 1 + else: + cur.update(snapshot.mid_price, snapshot.funding_rate, snapshot.volume_24h * 0.0) + self._ticks_in_bar[instrument] = self._ticks_in_bar.get(instrument, 0) + 1 + + # 2) Decide if this tick closes a bar. Two triggers (either fires): + # a) wall-clock hour rolled over (real-time deployment), or + # b) we've accumulated `ticks_per_hour` ticks (replay/mock/backfill). + hour_idx = self._hour_index(snapshot) + prev_hour = self._last_hour_idx.get(instrument) + wallclock_boundary = ( + prev_hour is not None and hour_idx > prev_hour and snapshot.timestamp_ms > 0 + ) + self._last_hour_idx[instrument] = hour_idx + count_boundary = self._ticks_in_bar.get(instrument, 0) >= self.ticks_per_hour + + if not (wallclock_boundary or count_boundary): + # Mid-bar: hold the last target. Re-issue the residual only if the + # engine reports we've drifted off target (e.g. partial fill). + return self._residual_decisions(snapshot, context) + + # 3) Bar boundary — finalize the bar and run on_bar exactly once. + closed = self._cur_bar.pop(instrument) + self._ticks_in_bar[instrument] = 0 + if closed is None: + return [] + + bar_row = { + "timestamp": (hour_idx * MS_PER_HOUR) if snapshot.timestamp_ms > 0 + else snapshot.timestamp_ms, + "open": closed.open, + "high": closed.high, + "low": closed.low, + "close": closed.close, + "volume": closed.volume, + "funding_rate": closed.funding_rate, + "isfr_rate": 0.0, + } + hq = self._hourly.setdefault(instrument, deque(maxlen=self.history_bars)) + hq.append(bar_row) + + # Start the next in-progress bar seeded with the current tick. + self._cur_bar[instrument] = _TickBar(snapshot.mid_price, snapshot.funding_rate) + self._ticks_in_bar[instrument] = 1 + + decisions = self._run_on_bar(symbol, instrument, snapshot, context) + + # 4) Guardrail: update drift estimate on each completed bar. + self._update_drift(context) + + return decisions + + def _run_on_bar( + self, + symbol: str, + instrument: str, + snapshot: MarketSnapshot, + context: Optional[StrategyContext], + ) -> List[StrategyDecision]: + """Build BarData+PortfolioState, call external on_bar, translate signals.""" + history_df = self._history_df(instrument) + + last = history_df.iloc[-1] + bar = self._BarData( + symbol=symbol, + timestamp=int(last["timestamp"]), + open=float(last["open"]), + high=float(last["high"]), + low=float(last["low"]), + close=float(last["close"]), + volume=float(last["volume"]), + funding_rate=float(last["funding_rate"]), + history=history_df, + ) + + # PortfolioState from live engine context. positions are signed USD + # notional keyed by bare symbol — matching the backtest engine. + pos_notional = float(context.position_notional) if context else self._target_usd.get(symbol, 0.0) + equity = self.initial_capital + if context is not None: + # account value isn't on context; approximate equity with capital + + # realized + unrealized so position-sizing-by-equity strategies work. + equity = self.initial_capital + float(context.realized_pnl) + float(context.unrealized_pnl) + positions = {symbol: pos_notional} if abs(pos_notional) > 0.0 else {} + entry_prices = dict(self._entry_px) + if abs(pos_notional) > 0.0 and symbol not in entry_prices: + entry_prices[symbol] = snapshot.mid_price + + portfolio = self._PortfolioState( + cash=max(0.0, equity - sum(abs(v) for v in positions.values())), + positions=positions, + entry_prices=entry_prices, + equity=equity, + timestamp=bar.timestamp, + ) + + try: + signals = self._ext.on_bar({symbol: bar}, portfolio) or [] + except Exception as e: # never let a strategy bug kill the tick loop + log.error("on_bar raised for %s: %s", self.strategy_id, e, exc_info=True) + return [] + + decisions: List[StrategyDecision] = [] + for sig in signals: + sig_symbol = getattr(sig, "symbol", symbol) + if sig_symbol != symbol: + # Single-instrument engine run; ignore cross-symbol signals here. + # (Multi-symbol live trading runs one engine per instrument.) + log.debug("ignoring signal for %s (engine bound to %s)", sig_symbol, symbol) + continue + target_usd = float(getattr(sig, "target_position", 0.0)) + self._target_usd[symbol] = target_usd + if target_usd == 0.0: + self._entry_px.pop(symbol, None) + else: + self._entry_px[symbol] = snapshot.mid_price + decisions.extend(self._target_to_decision(target_usd, snapshot, context)) + + return decisions + + def _target_to_decision( + self, + target_usd: float, + snapshot: MarketSnapshot, + context: Optional[StrategyContext], + ) -> List[StrategyDecision]: + """Translate a signed-USD target into an IOC order for the residual. + + delta_usd = target_usd - current_position_notional + size_base = |delta_usd| / mid_price + side = buy if delta_usd > 0 else sell + + Mirrors strategy-deployer.ts: USD target -> delta vs current -> order. + """ + current_usd = float(context.position_notional) if context else 0.0 + + # Guardrail: reduce_only / safe_mode — never *increase* exposure. + if context is not None and (context.reduce_only or context.safe_mode): + if abs(target_usd) >= abs(current_usd): + if context.safe_mode: + log.warning("safe_mode: suppressing new/added risk (target=%.2f cur=%.2f)", + target_usd, current_usd) + return [] + # Allow a reducing order toward the smaller target only. + + delta_usd = target_usd - current_usd + if abs(delta_usd) < 1.0: # < $1 change — skip (matches backtest engine) + return [] + + mid = snapshot.mid_price + if mid <= 0: + return [] + size_base = abs(delta_usd) / mid + if size_base <= 0: + return [] + + side = "buy" if delta_usd > 0 else "sell" + return [ + StrategyDecision( + action="place_order", + instrument=snapshot.instrument, + side=side, + size=size_base, + limit_price=mid, + order_type="Ioc", + meta={ + "source": "autobar", + "target_usd": target_usd, + "delta_usd": delta_usd, + }, + ) + ] + + def _residual_decisions( + self, + snapshot: MarketSnapshot, + context: Optional[StrategyContext], + ) -> List[StrategyDecision]: + """Between bars: re-assert the held target if live position drifted. + + on_bar is NOT re-run; we only top up / trim toward the last target if a + partial fill (or external move) left a residual > $1. This keeps the held + target without re-evaluating signals mid-bar. + """ + symbol = instrument_to_asset(snapshot.instrument) + if symbol not in self._target_usd: + return [] + return self._target_to_decision(self._target_usd[symbol], snapshot, context) + + # --------------------------------------------------------------- guardrails + def _update_drift(self, context: Optional[StrategyContext]) -> None: + """Track live realized return / Sharpe per bar and compare to the + strategy's backtest score. Warn (and fire the retrain hook once) on + divergence. + + Divergence rule (intentionally simple): once we have a full window of + live bars, flag if the annualized live Sharpe is negative while the kept + backtest score was positive, OR if live Sharpe falls below half the + backtest score. Both are coarse but actionable "the live regime no + longer matches what we optimized" signals. + """ + if context is None: + return + equity = self.initial_capital + float(context.realized_pnl) + float(context.unrealized_pnl) + if self._last_bar_equity is not None and self._last_bar_equity > 0: + self._live_returns.append((equity - self._last_bar_equity) / self._last_bar_equity) + self._last_bar_equity = equity + self.drift_state["bars_observed"] += 1 + + n = len(self._live_returns) + if n < self.drift_window: + return + + import numpy as np + + r = np.array(self._live_returns, dtype=float) + live_ret_pct = float((np.prod(1.0 + r) - 1.0) * 100.0) + std = r.std() + live_sharpe = float((r.mean() / std) * math.sqrt(8760)) if std > 0 else 0.0 + self.drift_state["live_sharpe"] = round(live_sharpe, 4) + self.drift_state["live_return_pct"] = round(live_ret_pct, 4) + + if self.backtest_score is None: + return + + diverged = False + reason = "" + if self.backtest_score > 0 and live_sharpe < 0: + diverged = True + reason = ( + f"live Sharpe {live_sharpe:.2f} negative vs positive backtest " + f"score {self.backtest_score:.2f}" + ) + elif live_sharpe < 0.5 * self.backtest_score: + diverged = True + reason = ( + f"live Sharpe {live_sharpe:.2f} < 50% of backtest score " + f"{self.backtest_score:.2f}" + ) + + self.drift_state["diverged"] = diverged + if diverged: + log.warning("DRIFT [%s]: %s (live_ret=%.2f%% over %d bars)", + self.strategy_id, reason, live_ret_pct, n) + if self.retrain_callback is not None and not self._retrain_fired: + self._retrain_fired = True + try: + self.retrain_callback(reason, dict(self.drift_state)) + except Exception as e: + log.error("retrain_callback raised: %s", e) diff --git a/spawn/__init__.py b/spawn/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spawn/templates/strategy/backtest.py b/spawn/templates/strategy/backtest.py new file mode 100644 index 0000000..1e61cfb --- /dev/null +++ b/spawn/templates/strategy/backtest.py @@ -0,0 +1,45 @@ +""" +Run backtest. Usage: uv run backtest.py +Imports strategy from strategy.py, runs on validation data, prints metrics. +This file is fixed — do not modify. +""" + +import time +import signal as sig + +from prepare import load_data, run_backtest, compute_score, TIME_BUDGET + +# Timeout guard +def timeout_handler(signum, frame): + print("TIMEOUT: backtest exceeded time budget") + exit(1) + +sig.signal(sig.SIGALRM, timeout_handler) +sig.alarm(TIME_BUDGET + 30) # 30s grace for startup + +t_start = time.time() + +from strategy import Strategy + +strategy = Strategy() +data = load_data("val") + +print(f"Loaded {sum(len(df) for df in data.values())} bars across {len(data)} symbols") +print(f"Symbols: {list(data.keys())}") + +result = run_backtest(strategy, data) +score = compute_score(result) + +t_end = time.time() + +print("---") +print(f"score: {score:.6f}") +print(f"sharpe: {result.sharpe:.6f}") +print(f"total_return_pct: {result.total_return_pct:.6f}") +print(f"max_drawdown_pct: {result.max_drawdown_pct:.6f}") +print(f"num_trades: {result.num_trades}") +print(f"win_rate_pct: {result.win_rate_pct:.6f}") +print(f"profit_factor: {result.profit_factor:.6f}") +print(f"annual_turnover: {result.annual_turnover:.2f}") +print(f"backtest_seconds: {result.backtest_seconds:.1f}") +print(f"total_seconds: {t_end - t_start:.1f}") diff --git a/spawn/templates/strategy/benchmarks/avellaneda_mm.py b/spawn/templates/strategy/benchmarks/avellaneda_mm.py new file mode 100644 index 0000000..6e016b1 --- /dev/null +++ b/spawn/templates/strategy/benchmarks/avellaneda_mm.py @@ -0,0 +1,87 @@ +"""Avellaneda-Stoikov inventory-aware market maker — ported from agent-cli.""" +import math +import numpy as np +from prepare import Signal, PortfolioState, BarData + +GAMMA = 0.1 +K = 1.5 +POSITION_SIZE_PCT = 0.08 +MAX_INVENTORY_PCT = 0.25 +MIN_SPREAD_BPS = 20.0 # wider for hourly bars (not tick-level) +MAX_SPREAD_BPS = 500.0 +VOL_WINDOW = 30 +ACTIVE_SYMBOLS = ["BTC", "ETH", "SOL"] + +class Strategy: + def __init__(self): + self.entry_prices = {} + + def _compute_vol(self, closes): + if len(closes) < 3: + return 0.001 + log_rets = np.diff(np.log(closes[-VOL_WINDOW:])) + return max(np.std(log_rets), 1e-6) + + def on_bar(self, bar_data: dict, portfolio: PortfolioState) -> list: + signals = [] + equity = portfolio.equity if portfolio.equity > 0 else portfolio.cash + + for symbol in ACTIVE_SYMBOLS: + if symbol not in bar_data: + continue + bd = bar_data[symbol] + if len(bd.history) < VOL_WINDOW: + continue + + closes = bd.history["close"].values + mid = bd.close + sigma = self._compute_vol(closes) * mid + + current_pos = portfolio.positions.get(symbol, 0.0) + max_inv = equity * MAX_INVENTORY_PCT + q = current_pos / max_inv if max_inv > 0 else 0.0 + + # Reservation price: skew away from inventory + T = 1.0 + r_price = mid - q * GAMMA * sigma**2 * T + + # Optimal spread + spread = GAMMA * sigma**2 * T + if GAMMA > 0: + spread += (2.0 / GAMMA) * math.log(1.0 + GAMMA / K) + + half_spread = max(mid * MIN_SPREAD_BPS / 10000, min(spread / 2, mid * MAX_SPREAD_BPS / 10000)) + + bid_price = r_price - half_spread + ask_price = r_price + half_spread + + # Size scaled by inventory utilization + utilization = abs(current_pos) / max_inv if max_inv > 0 else 0 + size = equity * POSITION_SIZE_PCT * max(0.1, 1.0 - utilization) + + # Use reservation price vs mid as directional signal + target = current_pos + price_diff_pct = (r_price - mid) / mid + if price_diff_pct > 0.001: # reservation above mid → buy + target = size + elif price_diff_pct < -0.001: # reservation below mid → sell + target = -size + elif abs(price_diff_pct) < 0.0003 and current_pos != 0: + target = 0.0 + + # Stop loss + if current_pos != 0 and symbol in self.entry_prices: + entry = self.entry_prices[symbol] + pnl = (mid - entry) / entry + if current_pos < 0: pnl = -pnl + if pnl < -0.03: + target = 0.0 + + if abs(target - current_pos) > 1.0: + signals.append(Signal(symbol=symbol, target_position=target)) + if target != 0 and current_pos == 0: + self.entry_prices[symbol] = mid + elif target == 0: + self.entry_prices.pop(symbol, None) + + return signals diff --git a/spawn/templates/strategy/benchmarks/funding_arb.py b/spawn/templates/strategy/benchmarks/funding_arb.py new file mode 100644 index 0000000..f6072ab --- /dev/null +++ b/spawn/templates/strategy/benchmarks/funding_arb.py @@ -0,0 +1,65 @@ +"""Funding rate carry strategy — ported from agent-cli.""" +import numpy as np +from prepare import Signal, PortfolioState, BarData + +ACTIVE_SYMBOLS = ["BTC", "ETH", "SOL"] +POSITION_SIZE_PCT = 0.10 +FUNDING_ENTRY_THRESHOLD = 0.00005 # lower threshold for hourly data (funding is small per-hour) +FUNDING_EXIT_THRESHOLD = 0.00001 # exit when funding normalizes +LOOKBACK = 24 # hours to average funding +STOP_LOSS_PCT = 0.03 +MAX_EXPOSURE_PCT = 0.30 + +class Strategy: + def __init__(self): + self.entry_prices = {} + + def on_bar(self, bar_data: dict, portfolio: PortfolioState) -> list: + signals = [] + equity = portfolio.equity if portfolio.equity > 0 else portfolio.cash + total_exposure = sum(abs(v) for v in portfolio.positions.values()) + + for symbol in ACTIVE_SYMBOLS: + if symbol not in bar_data: + continue + bd = bar_data[symbol] + if len(bd.history) < LOOKBACK: + continue + + funding_rates = bd.history["funding_rate"].values[-LOOKBACK:] + avg_funding = np.mean(funding_rates) + current_pos = portfolio.positions.get(symbol, 0.0) + mid = bd.close + + size = equity * POSITION_SIZE_PCT + remaining_capacity = equity * MAX_EXPOSURE_PCT - total_exposure + abs(current_pos) + size = min(size, max(0, remaining_capacity)) + + target = current_pos + + # Carry trade: short when funding high (shorts get paid), + # long when funding negative (longs get paid) + if avg_funding > FUNDING_ENTRY_THRESHOLD: + target = -size + elif avg_funding < -FUNDING_ENTRY_THRESHOLD: + target = size + elif abs(avg_funding) < FUNDING_EXIT_THRESHOLD and current_pos != 0: + target = 0.0 + + # Stop loss + if current_pos != 0 and symbol in self.entry_prices: + entry = self.entry_prices[symbol] + pnl_pct = (mid - entry) / entry + if current_pos < 0: + pnl_pct = -pnl_pct + if pnl_pct < -STOP_LOSS_PCT: + target = 0.0 + + if abs(target - current_pos) > 1.0: + signals.append(Signal(symbol=symbol, target_position=target)) + if target != 0 and current_pos == 0: + self.entry_prices[symbol] = mid + elif target == 0: + self.entry_prices.pop(symbol, None) + + return signals diff --git a/spawn/templates/strategy/benchmarks/isfr_funding_divergence.py b/spawn/templates/strategy/benchmarks/isfr_funding_divergence.py new file mode 100644 index 0000000..7029d2d --- /dev/null +++ b/spawn/templates/strategy/benchmarks/isfr_funding_divergence.py @@ -0,0 +1,51 @@ +"""ISFR + funding divergence strategy. + +Uses ISFR as a macro stress feature and Hyperliquid funding as a local perp +microstructure feature. The intent is to give AutoResearch a concrete template +for searching ISFR-conditioned strategies without making ISFR a tradeable asset. +""" + +import numpy as np +from prepare import Signal, PortfolioState, BarData + +ACTIVE_SYMBOLS = ["BTC", "ETH", "SOL"] +LOOKBACK = 72 +POSITION_SIZE_PCT = 0.08 +ISFR_Z_ENTRY = 1.0 +FUNDING_Z_ENTRY = 0.8 +MAX_EXPOSURE_PCT = 0.30 + + +class Strategy: + def on_bar(self, bar_data: dict, portfolio: PortfolioState) -> list: + signals = [] + equity = portfolio.equity if portfolio.equity > 0 else portfolio.cash + total_exposure = sum(abs(v) for v in portfolio.positions.values()) + + for symbol in ACTIVE_SYMBOLS: + if symbol not in bar_data: + continue + bd: BarData = bar_data[symbol] + if len(bd.history) < LOOKBACK or "isfr_rate" not in bd.history: + continue + + funding = bd.history["funding_rate"].values[-LOOKBACK:] + isfr = bd.history["isfr_rate"].values[-LOOKBACK:] + if np.std(funding) == 0 or np.std(isfr) == 0: + continue + + funding_z = (funding[-1] - np.mean(funding)) / np.std(funding) + isfr_z = (isfr[-1] - np.mean(isfr)) / np.std(isfr) + current_pos = portfolio.positions.get(symbol, 0.0) + target_notional = current_pos + + if total_exposure < equity * MAX_EXPOSURE_PCT: + if isfr_z < -ISFR_Z_ENTRY and funding_z < FUNDING_Z_ENTRY: + target_notional = equity * POSITION_SIZE_PCT + elif isfr_z > ISFR_Z_ENTRY or funding_z > FUNDING_Z_ENTRY: + target_notional = -equity * POSITION_SIZE_PCT + + if abs(target_notional - current_pos) > 1.0: + signals.append(Signal(symbol=symbol, target_position=target_notional)) + + return signals diff --git a/spawn/templates/strategy/benchmarks/mean_reversion.py b/spawn/templates/strategy/benchmarks/mean_reversion.py new file mode 100644 index 0000000..638fe51 --- /dev/null +++ b/spawn/templates/strategy/benchmarks/mean_reversion.py @@ -0,0 +1,65 @@ +"""Mean reversion z-score strategy — ported from agent-cli.""" +import numpy as np +from prepare import Signal, PortfolioState, BarData + +ACTIVE_SYMBOLS = ["BTC", "ETH", "SOL"] +WINDOW = 24 +ENTRY_ZSCORE = 2.0 +EXIT_ZSCORE = 0.5 +POSITION_SIZE_PCT = 0.10 +STOP_LOSS_PCT = 0.04 + +class Strategy: + def __init__(self): + self.entry_prices = {} + + def on_bar(self, bar_data: dict, portfolio: PortfolioState) -> list: + signals = [] + equity = portfolio.equity if portfolio.equity > 0 else portfolio.cash + + for symbol in ACTIVE_SYMBOLS: + if symbol not in bar_data: + continue + bd = bar_data[symbol] + if len(bd.history) < WINDOW: + continue + + closes = bd.history["close"].values[-WINDOW:] + sma = np.mean(closes) + std = np.std(closes) + mid = bd.close + current_pos = portfolio.positions.get(symbol, 0.0) + + if std <= 0: + continue + + zscore = (mid - sma) / std + size = equity * POSITION_SIZE_PCT + target = current_pos + + # Enter on extreme z-scores + if zscore > ENTRY_ZSCORE: + target = -size # overbought → short + elif zscore < -ENTRY_ZSCORE: + target = size # oversold → long + # Exit when z-score normalizes + elif abs(zscore) < EXIT_ZSCORE and current_pos != 0: + target = 0.0 + + # Stop loss + if current_pos != 0 and symbol in self.entry_prices: + entry = self.entry_prices[symbol] + pnl_pct = (mid - entry) / entry + if current_pos < 0: + pnl_pct = -pnl_pct + if pnl_pct < -STOP_LOSS_PCT: + target = 0.0 + + if abs(target - current_pos) > 1.0: + signals.append(Signal(symbol=symbol, target_position=target)) + if target != 0 and current_pos == 0: + self.entry_prices[symbol] = mid + elif target == 0: + self.entry_prices.pop(symbol, None) + + return signals diff --git a/spawn/templates/strategy/benchmarks/momentum_breakout.py b/spawn/templates/strategy/benchmarks/momentum_breakout.py new file mode 100644 index 0000000..7bf86b1 --- /dev/null +++ b/spawn/templates/strategy/benchmarks/momentum_breakout.py @@ -0,0 +1,86 @@ +"""Momentum breakout with volume confirmation — ported from agent-cli.""" +import numpy as np +from prepare import Signal, PortfolioState, BarData + +ACTIVE_SYMBOLS = ["BTC", "ETH", "SOL"] +LOOKBACK = 48 +BREAKOUT_THRESHOLD = 0.008 # 0.8% for hourly +VOLUME_SURGE_MULT = 1.0 # no volume filter (volume data may be spotty) +TRAILING_STOP_BPS = 200 # 2% trailing stop +POSITION_SIZE_PCT = 0.10 +MAX_HOLD_BARS = 72 + +class Strategy: + def __init__(self): + self.entry_prices = {} + self.peak_prices = {} + self.bars_held = {} + + def on_bar(self, bar_data: dict, portfolio: PortfolioState) -> list: + signals = [] + equity = portfolio.equity if portfolio.equity > 0 else portfolio.cash + + for symbol in ACTIVE_SYMBOLS: + if symbol not in bar_data: + continue + bd = bar_data[symbol] + if len(bd.history) < LOOKBACK: + continue + + highs = bd.history["high"].values[-LOOKBACK:] + lows = bd.history["low"].values[-LOOKBACK:] + volumes = bd.history["volume"].values[-LOOKBACK:] + + period_high = np.max(highs) + period_low = np.min(lows) + avg_vol = np.mean(volumes) if len(volumes) > 0 else 1 + mid = bd.close + current_pos = portfolio.positions.get(symbol, 0.0) + size = equity * POSITION_SIZE_PCT + target = current_pos + + vol_surge = bd.volume > avg_vol * VOLUME_SURGE_MULT if avg_vol > 0 else False + + # Breakout entry + if current_pos == 0: + up_break = (mid - period_high) / period_high if period_high > 0 else 0 + dn_break = (period_low - mid) / period_low if period_low > 0 else 0 + + if up_break > BREAKOUT_THRESHOLD and vol_surge: + target = size + elif dn_break > BREAKOUT_THRESHOLD and vol_surge: + target = -size + else: + # Track holding time + self.bars_held[symbol] = self.bars_held.get(symbol, 0) + 1 + + # Trailing stop + if symbol not in self.peak_prices: + self.peak_prices[symbol] = mid + if current_pos > 0: + self.peak_prices[symbol] = max(self.peak_prices[symbol], mid) + stop = self.peak_prices[symbol] * (1 - TRAILING_STOP_BPS / 10000) + if mid < stop: + target = 0.0 + else: + self.peak_prices[symbol] = min(self.peak_prices[symbol], mid) + stop = self.peak_prices[symbol] * (1 + TRAILING_STOP_BPS / 10000) + if mid > stop: + target = 0.0 + + # Max hold time + if self.bars_held.get(symbol, 0) > MAX_HOLD_BARS: + target = 0.0 + + if abs(target - current_pos) > 1.0: + signals.append(Signal(symbol=symbol, target_position=target)) + if target != 0 and current_pos == 0: + self.entry_prices[symbol] = mid + self.peak_prices[symbol] = mid + self.bars_held[symbol] = 0 + elif target == 0: + self.entry_prices.pop(symbol, None) + self.peak_prices.pop(symbol, None) + self.bars_held.pop(symbol, None) + + return signals diff --git a/spawn/templates/strategy/benchmarks/regime_mm.py b/spawn/templates/strategy/benchmarks/regime_mm.py new file mode 100644 index 0000000..edf54d8 --- /dev/null +++ b/spawn/templates/strategy/benchmarks/regime_mm.py @@ -0,0 +1,99 @@ +"""Volatility-regime adaptive strategy — ported from agent-cli.""" +import math +import numpy as np +from prepare import Signal, PortfolioState, BarData + +VOL_WINDOW = 48 +ACTIVE_SYMBOLS = ["BTC", "ETH", "SOL"] + +# Regime params: (vol_threshold, spread_bps, size_mult, stop_mult) +REGIMES = [ + (0.30, 10, 1.5, 0.02), # I_low: tight spread, big size, tight stop + (0.60, 25, 1.0, 0.03), # II_normal + (1.00, 50, 0.5, 0.05), # III_high: wide spread, small size + (float("inf"), 100, 0.2, 0.08), # IV_extreme: survival +] +HYSTERESIS = 3 +BASE_SIZE_PCT = 0.08 + +class Strategy: + def __init__(self): + self.entry_prices = {} + self.regime_idx = {s: 1 for s in ACTIVE_SYMBOLS} + self.down_count = {s: 0 for s in ACTIVE_SYMBOLS} + self.down_candidate = {s: -1 for s in ACTIVE_SYMBOLS} + + def _classify(self, symbol, ann_vol): + target = 0 + for i, (thresh, _, _, _) in enumerate(REGIMES): + if ann_vol < thresh: + target = i + break + + curr = self.regime_idx[symbol] + if target > curr: + self.regime_idx[symbol] = target + self.down_count[symbol] = 0 + elif target < curr: + if target == self.down_candidate[symbol]: + self.down_count[symbol] += 1 + else: + self.down_candidate[symbol] = target + self.down_count[symbol] = 1 + if self.down_count[symbol] >= HYSTERESIS: + self.regime_idx[symbol] = target + self.down_count[symbol] = 0 + else: + self.down_count[symbol] = 0 + + return REGIMES[self.regime_idx[symbol]] + + def on_bar(self, bar_data: dict, portfolio: PortfolioState) -> list: + signals = [] + equity = portfolio.equity if portfolio.equity > 0 else portfolio.cash + + for symbol in ACTIVE_SYMBOLS: + if symbol not in bar_data: + continue + bd = bar_data[symbol] + if len(bd.history) < VOL_WINDOW: + continue + + closes = bd.history["close"].values + log_rets = np.diff(np.log(closes[-VOL_WINDOW:])) + ann_vol = np.std(log_rets) * math.sqrt(8760) if len(log_rets) > 1 else 0.5 + + _, spread_bps, size_mult, stop_mult = self._classify(symbol, ann_vol) + mid = bd.close + current_pos = portfolio.positions.get(symbol, 0.0) + + half_spread = mid * spread_bps / 10000 + size = equity * BASE_SIZE_PCT * size_mult + + # Simple momentum signal within regime-adaptive framework + sma_fast = np.mean(closes[-12:]) + sma_slow = np.mean(closes[-48:]) if len(closes) >= 48 else np.mean(closes) + + target = current_pos + if sma_fast > sma_slow * (1 + spread_bps / 20000): + target = size + elif sma_fast < sma_slow * (1 - spread_bps / 20000): + target = -size + + # Stop loss based on regime + if current_pos != 0 and symbol in self.entry_prices: + entry = self.entry_prices[symbol] + pnl_pct = (mid - entry) / entry + if current_pos < 0: + pnl_pct = -pnl_pct + if pnl_pct < -stop_mult: + target = 0.0 + + if abs(target - current_pos) > 1.0: + signals.append(Signal(symbol=symbol, target_position=target)) + if target != 0 and current_pos == 0: + self.entry_prices[symbol] = mid + elif target == 0: + self.entry_prices.pop(symbol, None) + + return signals diff --git a/spawn/templates/strategy/prepare.py b/spawn/templates/strategy/prepare.py new file mode 100644 index 0000000..5b0c8c2 --- /dev/null +++ b/spawn/templates/strategy/prepare.py @@ -0,0 +1,812 @@ +""" +Autotrader backtesting engine. Fixed evaluation harness — DO NOT MODIFY. +Downloads Hyperliquid historical data, runs backtests, computes scores. + +Usage: + python prepare.py # download data + python prepare.py --symbols BTC # download specific symbols +""" + +import os +import sys +import time +import math +import signal +import argparse +from dataclasses import dataclass, field + +import numpy as np +import pandas as pd +import requests +import pyarrow.parquet as pq + +# --------------------------------------------------------------------------- +# Constants (fixed, do not modify) +# --------------------------------------------------------------------------- + +TIME_BUDGET = 120 # backtest time budget in seconds (2 minutes) +INITIAL_CAPITAL = 100_000.0 # $100K starting capital +MAKER_FEE = 0.0002 # 2 bps +TAKER_FEE = 0.0005 # 5 bps +SLIPPAGE_BPS = 1.0 # 1 bps simulated slippage +MAX_LEVERAGE = 20 # max leverage allowed +LOOKBACK_BARS = 500 # history buffer provided to strategy +BAR_INTERVAL = "1h" + +# Crypto majors (CryptoCompare histohour, no geo-restrictions). +CRYPTO_SYMBOLS = ["BTC", "ETH", "SOL"] + +# HIP-3 instruments. Data comes from the Hyperliquid info API (candleSnapshot + +# fundingHistory) using the dex-prefixed coin names, NOT CryptoCompare. +# +# Coin names + the dex serving them were discovered live on 2026-06-04 via: +# curl ... -d '{"type":"perpDexs"}' # list dexs +# curl ... -d '{"type":"meta","dex":""}' # list coins +# curl ... -d '{"type":"candleSnapshot","req":{...}}' # confirm OHLCV +# +# yex (Nunchi yield-perp dex, TESTNET): yex:VXX, yex:US3M, yex:BTCSWP +# xyz (tradfi/commodity dex, MAINNET) : GOLD, SILVER, COPPER, PLATINUM, +# PALLADIUM, NATGAS, BRENTOIL, CL (WTI crude) ← all confirmed returning +# hourly candles. (WHEAT/CORN/URANIUM/ALUMINIUM are listed but returned 0 +# candles on 2026-06-04, so they are NOT enabled by default — add them to +# SYMBOL_SOURCES once they carry history.) +# +# Each entry: bare symbol -> {"coin": , "network": "mainnet"|"testnet"}. +SYMBOL_SOURCES: dict = { + "BTCSWP": {"coin": "yex:BTCSWP", "network": "testnet"}, + "GOLD": {"coin": "xyz:GOLD", "network": "mainnet"}, + "SILVER": {"coin": "xyz:SILVER", "network": "mainnet"}, + "COPPER": {"coin": "xyz:COPPER", "network": "mainnet"}, + "PLATINUM": {"coin": "xyz:PLATINUM", "network": "mainnet"}, + "PALLADIUM": {"coin": "xyz:PALLADIUM", "network": "mainnet"}, + "NATGAS": {"coin": "xyz:NATGAS", "network": "mainnet"}, + "BRENTOIL": {"coin": "xyz:BRENTOIL", "network": "mainnet"}, + "CL": {"coin": "xyz:CL", "network": "mainnet"}, # WTI crude oil +} + +HIP3_SYMBOLS = list(SYMBOL_SOURCES.keys()) + +# Full symbol universe. Crypto first (CryptoCompare), then HIP-3 (HL info API). +SYMBOLS = CRYPTO_SYMBOLS + HIP3_SYMBOLS + +# Date splits (UTC timestamps). Overridable via env for instruments whose +# listing history is short (commodities + BTCSWP only carry data from late +# 2025 / early 2026 — confirmed 2026-06-04), so a recent-only window can be +# backtested without editing this file: +# AUTOTRADER_TRAIN_START / _TRAIN_END / _VAL_START / _VAL_END / +# _TEST_START / _TEST_END (YYYY-MM-DD) +TRAIN_START = os.environ.get("AUTOTRADER_TRAIN_START", "2023-06-01") +TRAIN_END = os.environ.get("AUTOTRADER_TRAIN_END", "2024-06-30") +VAL_START = os.environ.get("AUTOTRADER_VAL_START", "2024-07-01") +VAL_END = os.environ.get("AUTOTRADER_VAL_END", "2025-03-31") +TEST_START = os.environ.get("AUTOTRADER_TEST_START", "2025-04-01") +TEST_END = os.environ.get("AUTOTRADER_TEST_END", "2025-12-31") + +HOURS_PER_YEAR = 8760 + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +CACHE_DIR = os.path.join(os.path.expanduser("~"), ".cache", "autotrader") +DATA_DIR = os.path.join(CACHE_DIR, "data") +ISFR_HISTORY_PATH_ENV = "NUNCHI_ISFR_HISTORY_PATH" +ISFR_CACHE_FILE = os.path.join(DATA_DIR, "ISFR_1h.parquet") + +# --------------------------------------------------------------------------- +# Data types +# --------------------------------------------------------------------------- + +@dataclass +class BarData: + symbol: str + timestamp: int + open: float + high: float + low: float + close: float + volume: float + funding_rate: float + history: pd.DataFrame # last LOOKBACK_BARS bars + isfr_rate: float = 0.0 + +@dataclass +class Signal: + symbol: str + target_position: float # target USD notional (signed: +long, -short) + order_type: str = "market" + +@dataclass +class PortfolioState: + cash: float + positions: dict # symbol -> signed USD notional + entry_prices: dict # symbol -> avg entry price + equity: float = 0.0 + timestamp: int = 0 + +@dataclass +class BacktestResult: + sharpe: float = 0.0 + total_return_pct: float = 0.0 + max_drawdown_pct: float = 0.0 + num_trades: int = 0 + win_rate_pct: float = 0.0 + profit_factor: float = 0.0 + annual_turnover: float = 0.0 + backtest_seconds: float = 0.0 + equity_curve: list = field(default_factory=list) + trade_log: list = field(default_factory=list) + +# --------------------------------------------------------------------------- +# Data download +# --------------------------------------------------------------------------- + +HL_INFO_URL = "https://api.hyperliquid.xyz/info" +HL_INFO_URL_TESTNET = "https://api.hyperliquid-testnet.xyz/info" +CRYPTOCOMPARE_URL = "https://min-api.cryptocompare.com/data/v2/histohour" + + +def _hl_base_url(network: str = "mainnet") -> str: + """Return the HL info endpoint for a network. yex (Nunchi) is testnet-only; + xyz commodities are mainnet.""" + return HL_INFO_URL_TESTNET if network == "testnet" else HL_INFO_URL + +def _download_cryptocompare_candles(symbol: str, start_ms: int, end_ms: int) -> pd.DataFrame: + """Download hourly OHLCV from CryptoCompare (no geo-restrictions).""" + all_rows = [] + # CryptoCompare uses 'toTs' (end timestamp in seconds) and returns up to 2000 bars + current_end = end_ms // 1000 + start_s = start_ms // 1000 + + while current_end > start_s: + params = { + "fsym": symbol, + "tsym": "USD", + "limit": 2000, + "toTs": current_end, + } + resp = requests.get(CRYPTOCOMPARE_URL, params=params, timeout=30) + resp.raise_for_status() + data = resp.json() + bars = data.get("Data", {}).get("Data", []) + if not bars: + break + for bar in bars: + ts_s = bar["time"] + if ts_s < start_s: + continue + all_rows.append({ + "timestamp": ts_s * 1000, + "open": float(bar["open"]), + "high": float(bar["high"]), + "low": float(bar["low"]), + "close": float(bar["close"]), + "volume": float(bar.get("volumefrom", 0)), + }) + # Move window back + earliest = bars[0]["time"] + if earliest >= current_end: + break + current_end = earliest - 1 + time.sleep(0.3) + + if not all_rows: + return pd.DataFrame() + df = pd.DataFrame(all_rows).sort_values("timestamp").drop_duplicates("timestamp").reset_index(drop=True) + return df + + +def _download_hl_funding(symbol: str, start_ms: int, end_ms: int, + base_url: str = HL_INFO_URL) -> pd.DataFrame: + """Download funding rate history from Hyperliquid. + + `symbol` is the HL coin name (e.g. "BTC" or a dex-prefixed "yex:BTCSWP" / + "xyz:GOLD"). `base_url` selects mainnet vs testnet. + """ + all_rows = [] + current = start_ms + while current < end_ms: + body = { + "type": "fundingHistory", + "coin": symbol, + "startTime": current, + "endTime": min(current + 30 * 24 * 3600 * 1000, end_ms), + } + try: + resp = requests.post(base_url, json=body, timeout=30) + resp.raise_for_status() + data = resp.json() + if not data: + break + for row in data: + all_rows.append({ + "timestamp": int(row["time"]), + "funding_rate": float(row["fundingRate"]), + }) + current = int(data[-1]["time"]) + 1 + except Exception: + break + time.sleep(0.2) + + if not all_rows: + return pd.DataFrame(columns=["timestamp", "funding_rate"]) + return pd.DataFrame(all_rows) + + +def _normalize_isfr_frame(df: pd.DataFrame) -> pd.DataFrame: + """Normalize user-provided ISFR history into timestamp-ms + decimal rate.""" + if df.empty: + return pd.DataFrame(columns=["timestamp", "isfr_rate"]) + + timestamp_col = "timestamp" if "timestamp" in df.columns else "time" if "time" in df.columns else "date" if "date" in df.columns else None + value_col = None + for candidate in ("isfr_rate", "isfr", "isfr_score", "composite_bps", "rate_bps"): + if candidate in df.columns: + value_col = candidate + break + if timestamp_col is None or value_col is None: + return pd.DataFrame(columns=["timestamp", "isfr_rate"]) + + out = df[[timestamp_col, value_col]].copy() + out.columns = ["timestamp", "isfr_rate"] + if not np.issubdtype(out["timestamp"].dtype, np.number): + out["timestamp"] = pd.to_datetime(out["timestamp"], utc=True, errors="coerce").astype("int64") // 1_000_000 + out["timestamp"] = pd.to_numeric(out["timestamp"], errors="coerce") + out["isfr_rate"] = pd.to_numeric(out["isfr_rate"], errors="coerce") + out = out.dropna().sort_values("timestamp").drop_duplicates("timestamp") + if out.empty: + return pd.DataFrame(columns=["timestamp", "isfr_rate"]) + + # Accept seconds, milliseconds, bps, percent-like scores (1.37), or decimals. + if out["timestamp"].max() < 10_000_000_000: + out["timestamp"] = out["timestamp"] * 1000 + median_abs = out["isfr_rate"].abs().median() + if median_abs > 10: + out["isfr_rate"] = out["isfr_rate"] / 10_000.0 + elif median_abs > 0.5: + out["isfr_rate"] = out["isfr_rate"] / 100.0 + return out[["timestamp", "isfr_rate"]].reset_index(drop=True) + + +def _load_isfr_history_from_path(path: str) -> pd.DataFrame: + if not path: + return pd.DataFrame(columns=["timestamp", "isfr_rate"]) + expanded = os.path.expanduser(path) + if not os.path.exists(expanded): + return pd.DataFrame(columns=["timestamp", "isfr_rate"]) + try: + if expanded.endswith(".parquet"): + raw = pd.read_parquet(expanded) + else: + raw = pd.read_csv(expanded) + except Exception: + return pd.DataFrame(columns=["timestamp", "isfr_rate"]) + return _normalize_isfr_frame(raw) + + +def _synthetic_isfr_proxy(start_ms: int, end_ms: int) -> pd.DataFrame: + """Deterministic ISFR proxy for local research when canonical history is absent.""" + timestamps = np.arange(start_ms, end_ms + 1, 3600 * 1000, dtype=np.int64) + if len(timestamps) == 0: + return pd.DataFrame(columns=["timestamp", "isfr_rate"]) + hours = (timestamps - timestamps[0]) / (3600 * 1000) + base = 0.0137 + cycle = 0.0014 * np.sin(hours / (24 * 9)) + stress = 0.0008 * np.sin(hours / (24 * 31) + 0.7) + pulse = 0.0005 * np.maximum(0, np.sin(hours / (24 * 5) - 1.2)) + return pd.DataFrame({"timestamp": timestamps, "isfr_rate": base + cycle + stress + pulse}) + + +def _load_or_build_isfr_series(start_ms: int, end_ms: int) -> pd.DataFrame: + path = os.environ.get(ISFR_HISTORY_PATH_ENV, "").strip() + source = _load_isfr_history_from_path(path) + if source.empty and os.path.exists(ISFR_CACHE_FILE): + try: + source = pd.read_parquet(ISFR_CACHE_FILE) + except Exception: + source = pd.DataFrame(columns=["timestamp", "isfr_rate"]) + if source.empty: + source = _synthetic_isfr_proxy(start_ms, end_ms) + source = _normalize_isfr_frame(source) + if not source.empty: + os.makedirs(DATA_DIR, exist_ok=True) + source.to_parquet(ISFR_CACHE_FILE, index=False) + return source + + +def _merge_isfr(df: pd.DataFrame, isfr: pd.DataFrame) -> pd.DataFrame: + if df.empty: + return df + if isfr.empty: + df["isfr_rate"] = 0.0 + return df + left = df.drop(columns=["isfr_rate"], errors="ignore").drop_duplicates(subset=["timestamp"]).sort_values("timestamp") + right = isfr.drop_duplicates(subset=["timestamp"]).sort_values("timestamp") + merged = pd.merge_asof(left, right, on="timestamp", direction="backward") + merged["isfr_rate"] = merged["isfr_rate"].bfill().fillna(0.0) + return merged + + +def _download_hl_candles(symbol: str, interval: str, start_ms: int, end_ms: int, + base_url: str = HL_INFO_URL) -> pd.DataFrame: + """Download OHLCV candles from Hyperliquid. + + `symbol` is the HL coin name (e.g. "BTC" or a dex-prefixed "yex:BTCSWP" / + "xyz:GOLD"). `base_url` selects mainnet vs testnet. + """ + all_rows = [] + current = start_ms + chunk_ms = 30 * 24 * 3600 * 1000 # 30 days + while current < end_ms: + body = { + "type": "candleSnapshot", + "req": { + "coin": symbol, + "interval": interval, + "startTime": current, + "endTime": min(current + chunk_ms, end_ms), + } + } + try: + resp = requests.post(base_url, json=body, timeout=30) + resp.raise_for_status() + data = resp.json() + if not data: + current += chunk_ms + continue + for row in data: + all_rows.append({ + "timestamp": int(row["t"]), + "open": float(row["o"]), + "high": float(row["h"]), + "low": float(row["l"]), + "close": float(row["c"]), + "volume": float(row["v"]), + }) + current = int(data[-1]["t"]) + 3600 * 1000 + except Exception: + current += chunk_ms + time.sleep(0.2) + return pd.DataFrame(all_rows) + + +def _download_hip3_symbol(symbol: str, start_ms: int, end_ms: int) -> pd.DataFrame: + """Download OHLCV + funding for a HIP-3 instrument (BTCSWP / commodities). + + Uses the HL info API with the real dex-prefixed coin name and the right + network (yex=testnet, xyz=mainnet) per SYMBOL_SOURCES. Returns OHLCV merged + with funding (funding columns may be all-zero for HIP-3 dexs that don't + publish funding history yet — that's fine, the engine treats 0 funding as + no carry). + """ + src = SYMBOL_SOURCES[symbol] + coin = src["coin"] + base_url = _hl_base_url(src.get("network", "mainnet")) + print(f" {symbol}: downloading HIP-3 candles ({coin} @ {src.get('network','mainnet')})...") + df = _download_hl_candles(coin, "1h", start_ms, end_ms, base_url=base_url) + if df.empty: + return df + df = df.drop_duplicates(subset=["timestamp"]).sort_values("timestamp").reset_index(drop=True) + + funding = _download_hl_funding(coin, start_ms, end_ms, base_url=base_url) + if not funding.empty: + funding = funding.drop_duplicates(subset=["timestamp"]).sort_values("timestamp") + df = pd.merge_asof(df, funding, on="timestamp", direction="backward") + if "funding_rate" not in df.columns: + df["funding_rate"] = 0.0 + df["funding_rate"] = df["funding_rate"].fillna(0.0) + return df + + +def download_data(symbols=None): + """Download historical OHLCV + funding data for all symbols.""" + os.makedirs(DATA_DIR, exist_ok=True) + if symbols is None: + symbols = SYMBOLS + + start_ms = int(pd.Timestamp(TRAIN_START, tz="UTC").timestamp() * 1000) + end_ms = int(pd.Timestamp(TEST_END, tz="UTC").timestamp() * 1000) + isfr = _load_or_build_isfr_series(start_ms, end_ms) + print(f" ISFR: loaded {len(isfr)} hourly points") + + for symbol in symbols: + filepath = os.path.join(DATA_DIR, f"{symbol}_1h.parquet") + if os.path.exists(filepath): + existing = pd.read_parquet(filepath) + if "isfr_rate" not in existing.columns: + existing = _merge_isfr(existing, isfr) + existing.to_parquet(filepath, index=False) + print(f" {symbol}: added ISFR feature to {len(existing)} cached bars") + else: + print(f" {symbol}: already have {len(existing)} bars") + continue + + if symbol in SYMBOL_SOURCES: + # HIP-3 instrument (BTCSWP / commodities) — HL info API, not CC. + df = _download_hip3_symbol(symbol, start_ms, end_ms) + if df.empty: + print(f" {symbol}: NO DATA AVAILABLE, skipping") + continue + else: + print(f" {symbol}: downloading candles from CryptoCompare...") + + # Use CryptoCompare for reliable historical OHLCV (no geo-restrictions) + df = _download_cryptocompare_candles(symbol, start_ms, end_ms) + if len(df) < 100: + print(f" {symbol}: CryptoCompare insufficient ({len(df)} bars), trying HL...") + df = _download_hl_candles(symbol, "1h", start_ms, end_ms) + + if df.empty: + print(f" {symbol}: NO DATA AVAILABLE, skipping") + continue + + # Download funding rates + print(f" {symbol}: downloading funding rates...") + funding = _download_hl_funding(symbol, start_ms, end_ms) + + # Merge + df = df.drop_duplicates(subset=["timestamp"]).sort_values("timestamp").reset_index(drop=True) + if not funding.empty: + funding = funding.drop_duplicates(subset=["timestamp"]).sort_values("timestamp") + # Merge nearest — funding is every 8h, candles every 1h + df = pd.merge_asof(df, funding, on="timestamp", direction="backward") + if "funding_rate" not in df.columns: + df["funding_rate"] = 0.0 + df["funding_rate"] = df["funding_rate"].fillna(0.0) + + df = _merge_isfr(df, isfr) + + df.to_parquet(filepath, index=False) + print(f" {symbol}: saved {len(df)} bars to {filepath}") + + +def load_data(split: str = "val") -> dict: + """Load OHLCV+funding data for the given split. Returns {symbol: DataFrame}.""" + splits = { + "train": (TRAIN_START, TRAIN_END), + "val": (VAL_START, VAL_END), + "test": (TEST_START, TEST_END), + } + assert split in splits, f"split must be one of {list(splits.keys())}" + start_str, end_str = splits[split] + start_ms = int(pd.Timestamp(start_str, tz="UTC").timestamp() * 1000) + end_ms = int(pd.Timestamp(end_str, tz="UTC").timestamp() * 1000) + + result = {} + for symbol in SYMBOLS: + filepath = os.path.join(DATA_DIR, f"{symbol}_1h.parquet") + if not os.path.exists(filepath): + continue + df = pd.read_parquet(filepath) + if "isfr_rate" not in df.columns: + df["isfr_rate"] = 0.0 + mask = (df["timestamp"] >= start_ms) & (df["timestamp"] < end_ms) + split_df = df[mask].reset_index(drop=True) + if len(split_df) > 0: + result[symbol] = split_df + return result + +# --------------------------------------------------------------------------- +# Backtesting engine (DO NOT CHANGE) +# --------------------------------------------------------------------------- + +def run_backtest(strategy, data: dict) -> BacktestResult: + """ + Run strategy over data. Returns BacktestResult with full metrics. + Enforces TIME_BUDGET. + """ + t_start = time.time() + + # Build unified timeline + all_timestamps = set() + for symbol, df in data.items(): + all_timestamps.update(df["timestamp"].tolist()) + timestamps = sorted(all_timestamps) + + if not timestamps: + return BacktestResult() + + # Index data by (symbol, timestamp) for fast lookup + indexed = {} + for symbol, df in data.items(): + indexed[symbol] = df.set_index("timestamp") + + # Portfolio state + portfolio = PortfolioState( + cash=INITIAL_CAPITAL, + positions={}, + entry_prices={}, + equity=INITIAL_CAPITAL, + timestamp=0, + ) + + equity_curve = [INITIAL_CAPITAL] + hourly_returns = [] + trade_log = [] + total_volume = 0.0 + prev_equity = INITIAL_CAPITAL + + # History buffers + history_buffers = {symbol: [] for symbol in data} + + for ts in timestamps: + elapsed = time.time() - t_start + if elapsed > TIME_BUDGET: + break + + portfolio.timestamp = ts + + # Build bar data + bar_data = {} + for symbol in data: + if symbol not in indexed or ts not in indexed[symbol].index: + continue + row = indexed[symbol].loc[ts] + if isinstance(row, pd.DataFrame): + row = row.iloc[0] + + # Update history buffer + bar_dict = { + "timestamp": ts, + "open": row["open"], + "high": row["high"], + "low": row["low"], + "close": row["close"], + "volume": row["volume"], + "funding_rate": row.get("funding_rate", 0.0), + "isfr_rate": row.get("isfr_rate", 0.0), + } + history_buffers[symbol].append(bar_dict) + if len(history_buffers[symbol]) > LOOKBACK_BARS: + history_buffers[symbol] = history_buffers[symbol][-LOOKBACK_BARS:] + + hist_df = pd.DataFrame(history_buffers[symbol]) + + bar_data[symbol] = BarData( + symbol=symbol, + timestamp=ts, + open=row["open"], + high=row["high"], + low=row["low"], + close=row["close"], + volume=row["volume"], + funding_rate=row.get("funding_rate", 0.0), + history=hist_df, + isfr_rate=row.get("isfr_rate", 0.0), + ) + + if not bar_data: + continue + + # Update portfolio equity (mark-to-market) + unrealized_pnl = 0.0 + for sym, pos_notional in portfolio.positions.items(): + if sym in bar_data: + current_price = bar_data[sym].close + entry_price = portfolio.entry_prices.get(sym, current_price) + if entry_price > 0: + price_change = (current_price - entry_price) / entry_price + unrealized_pnl += pos_notional * price_change + + portfolio.equity = portfolio.cash + sum(abs(v) for v in portfolio.positions.values()) + unrealized_pnl + + # Apply funding rates (on open positions) + for sym, pos_notional in list(portfolio.positions.items()): + if sym in bar_data: + fr = bar_data[sym].funding_rate + # Funding: longs pay when positive, shorts receive + # Applied every 8h, but we have hourly bars so scale by 1/8 + funding_payment = pos_notional * fr / 8.0 + portfolio.cash -= funding_payment + + # Get signals from strategy + try: + signals = strategy.on_bar(bar_data, portfolio) + except Exception: + signals = [] + + # Execute signals + for sig in (signals or []): + if sig.symbol not in bar_data: + continue + + current_price = bar_data[sig.symbol].close + current_pos = portfolio.positions.get(sig.symbol, 0.0) + delta = sig.target_position - current_pos + + if abs(delta) < 1.0: # < $1 change, skip + continue + + # Check leverage constraint + new_positions = dict(portfolio.positions) + new_positions[sig.symbol] = sig.target_position + total_exposure = sum(abs(v) for v in new_positions.values()) + if total_exposure > portfolio.equity * MAX_LEVERAGE: + continue + + # Apply slippage and fees + slippage = current_price * SLIPPAGE_BPS / 10000 + fee_rate = TAKER_FEE + if delta > 0: # buying + exec_price = current_price + slippage + else: # selling + exec_price = current_price - slippage + + fee = abs(delta) * fee_rate + portfolio.cash -= fee + total_volume += abs(delta) + + # Update position + if sig.target_position == 0: + # Closing position — realize PnL + if sig.symbol in portfolio.entry_prices: + entry = portfolio.entry_prices[sig.symbol] + if entry > 0: + pnl = current_pos * (exec_price - entry) / entry + portfolio.cash += abs(current_pos) + pnl + del portfolio.entry_prices[sig.symbol] + if sig.symbol in portfolio.positions: + del portfolio.positions[sig.symbol] + trade_log.append(("close", sig.symbol, delta, exec_price, pnl if 'pnl' in dir() else 0)) + else: + if current_pos == 0: + # Opening new position + portfolio.cash -= abs(sig.target_position) + portfolio.positions[sig.symbol] = sig.target_position + portfolio.entry_prices[sig.symbol] = exec_price + trade_log.append(("open", sig.symbol, delta, exec_price, 0)) + else: + # Modifying position + old_notional = abs(current_pos) + old_entry = portfolio.entry_prices.get(sig.symbol, exec_price) + # Realize PnL on reduced portion + if abs(sig.target_position) < abs(current_pos): + reduced = abs(current_pos) - abs(sig.target_position) + if old_entry > 0: + pnl = (current_pos / abs(current_pos)) * reduced * (exec_price - old_entry) / old_entry + else: + pnl = 0 + portfolio.cash += reduced + pnl + elif abs(sig.target_position) > abs(current_pos): + added = abs(sig.target_position) - abs(current_pos) + portfolio.cash -= added + # Weighted average entry + if old_notional + added > 0: + new_entry = (old_entry * old_notional + exec_price * added) / (old_notional + added) + portfolio.entry_prices[sig.symbol] = new_entry + portfolio.positions[sig.symbol] = sig.target_position + trade_log.append(("modify", sig.symbol, delta, exec_price, 0)) + + # Recalculate equity after trades + unrealized_pnl = 0.0 + for sym, pos_notional in portfolio.positions.items(): + if sym in bar_data: + current_price = bar_data[sym].close + entry_price = portfolio.entry_prices.get(sym, current_price) + if entry_price > 0: + price_change = (current_price - entry_price) / entry_price + unrealized_pnl += pos_notional * price_change + + current_equity = portfolio.cash + sum(abs(v) for v in portfolio.positions.values()) + unrealized_pnl + equity_curve.append(current_equity) + + # Hourly return + if prev_equity > 0: + hourly_returns.append((current_equity - prev_equity) / prev_equity) + prev_equity = current_equity + + # Liquidation check + if current_equity < INITIAL_CAPITAL * 0.01: + break + + t_end = time.time() + + # Compute metrics + returns = np.array(hourly_returns) if hourly_returns else np.array([0.0]) + eq = np.array(equity_curve) + + # Sharpe ratio (annualized from hourly) + if returns.std() > 0: + sharpe = (returns.mean() / returns.std()) * np.sqrt(HOURS_PER_YEAR) + else: + sharpe = 0.0 + + # Total return + final_equity = eq[-1] if len(eq) > 0 else INITIAL_CAPITAL + total_return_pct = (final_equity - INITIAL_CAPITAL) / INITIAL_CAPITAL * 100 + + # Max drawdown + peak = np.maximum.accumulate(eq) + drawdown = (peak - eq) / np.where(peak > 0, peak, 1) + max_drawdown_pct = drawdown.max() * 100 + + # Win rate and profit factor + trade_pnls = [t[4] for t in trade_log if t[0] == "close"] + num_trades = len(trade_log) + if trade_pnls: + wins = [p for p in trade_pnls if p > 0] + losses = [p for p in trade_pnls if p < 0] + win_rate_pct = len(wins) / len(trade_pnls) * 100 if trade_pnls else 0 + gross_profit = sum(wins) if wins else 0 + gross_loss = abs(sum(losses)) if losses else 1e-10 + profit_factor = gross_profit / gross_loss + else: + win_rate_pct = 0.0 + profit_factor = 0.0 + + # Annual turnover + data_hours = len(timestamps) + if data_hours > 0: + annual_turnover = total_volume * (HOURS_PER_YEAR / data_hours) + else: + annual_turnover = 0.0 + + return BacktestResult( + sharpe=sharpe, + total_return_pct=total_return_pct, + max_drawdown_pct=max_drawdown_pct, + num_trades=num_trades, + win_rate_pct=win_rate_pct, + profit_factor=profit_factor, + annual_turnover=annual_turnover, + backtest_seconds=t_end - t_start, + equity_curve=equity_curve, + trade_log=trade_log, + ) + +# --------------------------------------------------------------------------- +# Evaluation metric (DO NOT CHANGE — this is the fixed metric) +# --------------------------------------------------------------------------- + +def compute_score(result: BacktestResult) -> float: + """ + Composite risk-adjusted score (HIGHER is better). + + score = sharpe * sqrt(trade_count_factor) - drawdown_penalty - turnover_penalty + + Hard cutoffs for degenerate strategies. + + NOTE (all-symbol support): the penalties are instrument-AGNOSTIC — they act on + the portfolio-level equity curve (Sharpe, drawdown, turnover) regardless of + which symbols traded, and the √8760 annualization holds because every symbol + (crypto + HIP-3 commodities + BTCSWP) is sampled on the SAME 1h bar grid. So + no per-instrument re-tuning was needed when BTCSWP/commodities were added. + """ + # Hard cutoffs + if result.num_trades < 10: + return -999.0 + if result.max_drawdown_pct > 50.0: + return -999.0 + final_equity = result.equity_curve[-1] if result.equity_curve else INITIAL_CAPITAL + if final_equity < INITIAL_CAPITAL * 0.5: + return -999.0 + + # Trade count factor: full credit at 50+ trades + trade_count_factor = min(result.num_trades / 50.0, 1.0) + + # Drawdown penalty: no penalty below 15%, then 5x per additional percent + drawdown_penalty = max(0, result.max_drawdown_pct - 15.0) * 0.05 + + # Turnover penalty: penalize excessive churning (>500x annual) + turnover_ratio = result.annual_turnover / INITIAL_CAPITAL if INITIAL_CAPITAL > 0 else 0 + turnover_penalty = max(0, turnover_ratio - 500) * 0.001 + + score = result.sharpe * math.sqrt(trade_count_factor) - drawdown_penalty - turnover_penalty + return score + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Prepare data for autotrader") + parser.add_argument("--symbols", nargs="+", default=None, help="Symbols to download (default: all)") + args = parser.parse_args() + + print(f"Cache directory: {CACHE_DIR}") + print() + + print("Downloading data...") + download_data(args.symbols) + print() + print("Done! Ready to backtest.") diff --git a/spawn/templates/strategy/program.md b/spawn/templates/strategy/program.md new file mode 100644 index 0000000..faa23e9 --- /dev/null +++ b/spawn/templates/strategy/program.md @@ -0,0 +1,137 @@ +# autotrader + +Autonomous trading strategy research on Hyperliquid perpetual futures. + +## Context + +This project adapts Karpathy's autoresearch pattern for trading strategy discovery. +The owner (Nunchi) has existing production strategies that were designed for **tick-level market making** (20-second intervals). Those strategies underperform when ported to **hourly directional trading** on this backtest harness. + +Your job: **discover novel hourly-timeframe strategies** that outperform both the simple baseline AND the existing production strategies. + +## Current Leaderboard (your target to beat) + +``` +RANK STRATEGY SCORE SHARPE RETURN MAX_DD TRADES +1. simple_momentum 2.724 2.724 +42.6% 7.6% 9081 ← BASELINE TO BEAT +2. funding_arb -0.191 -0.191 -1.3% 9.4% 1403 +3. regime_mm -0.322 -0.322 -3.1% 11.2% 12854 +4. mean_reversion -3.964 -3.380 -26.2% 26.7% 3185 +5. avellaneda_mm -999 (no trades — MM strategy doesn't port to hourly) +6. momentum_breakout -999 (no trades — breakout too tight for hourly) +``` + +The baseline momentum strategy scores 2.724. **Your goal is to beat 2.724.** + +## Existing Strategy Concepts (from production codebase) + +These concepts are proven in live trading at tick-level. Adapt them for hourly: + +1. **Avellaneda-Stoikov**: reservation_price = mid - q * gamma * sigma^2 * T. Skew quotes based on inventory. The key insight: use inventory-awareness to size positions. +2. **Vol regime classification**: Bin vol into 4 regimes (low/normal/high/extreme) with hysteresis. Adjust size and stops per regime. Immediate upshift, delayed downshift. +3. **Funding rate carry**: Short when funding high (shorts get paid), long when negative. The carry component is real P&L on Hyperliquid perps. +4. **Cross-venue funding arb**: When HL funding diverges from median, bias quotes. Asymmetric sizing: favor the side collecting premium. +5. **ISFR as stress/fair-rate input**: Use `isfr_rate` as a macro DeFi stress feature. Explore ISFR drops as risk-on entries, ISFR spikes as risk-off exits/shorts, and ISFR + funding divergence as a fair-funding signal. +6. **Momentum breakout**: Enter on price breaking N-period range with volume confirmation. Trailing stops. +7. **Risk multipliers**: Vol bin → size multiplier. Drawdown bin → spread/stop multiplier. Green/yellow/orange/red zones. + +## Setup + +To set up a new experiment, work with the user to: + +1. **Agree on a run tag**: propose a tag based on today's date (e.g. `mar10`). The branch `autotrader/` must not already exist. +2. **Create the branch**: `git checkout -b autotrader/` from current master. +3. **Read the in-scope files**: `prepare.py`, `strategy.py`, `backtest.py`, this file. +4. **Verify data exists**: `ls ~/.cache/autotrader/data/` +5. **Initialize results.tsv**: `echo -e "commit\tscore\tsharpe\tmax_dd\tstatus\tdescription" > results.tsv` +6. **Confirm and go**. + +## Experimentation + +Each experiment runs a backtest on historical Hyperliquid perp data (BTC, ETH, SOL, hourly bars, Jul 2024 - Mar 2025). Launch: `uv run backtest.py`. + +**What you CAN do:** +- Modify `strategy.py` — this is the only file you edit. Everything is fair game. + +**What you CANNOT do:** +- Modify `prepare.py`, `backtest.py`, or anything in `benchmarks/`. +- Install new packages. Only numpy, pandas, scipy, and standard library. +- Look at test set data. + +**The goal: get the highest `score`.** Higher is better. Baseline is 2.724. + +## Output format + +``` +grep "^score:" run.log +``` + +## Results TSV + +``` +commit score sharpe max_dd status description +``` + +## The experiment loop + +LOOP FOREVER: + +1. Look at git state +2. Modify `strategy.py` with an experimental idea +3. git commit +4. `uv run backtest.py > run.log 2>&1` +5. `grep "^score:\|^sharpe:\|^max_drawdown_pct:" run.log` +6. If empty → crashed. `tail -n 50 run.log`, fix or skip. +7. Record in results.tsv (untracked) +8. If score IMPROVED (higher than best so far): keep +9. If score equal or worse: `git reset --hard HEAD~1` + +## Strategy Research Directions + +Start with these high-probability ideas: + +### Tier 1 — Most Likely to Improve Score +- **Add SOL with lower weight** — diversification should help Sharpe +- **Vol-regime adaptive sizing** — reduce positions in high vol, increase in low vol (proven concept from production) +- **Multi-timeframe momentum** — require 12h, 24h, 48h agreement before entry +- **ATR-based trailing stops** — current fixed stops are suboptimal +- **Funding carry overlay** — add carry component on top of momentum +- **ISFR-conditioned entries** — long BTC/ETH/SOL when ISFR drops below its rolling mean while price momentum confirms + +### Tier 2 — Worth Exploring +- **EMA crossover instead of raw momentum** — smoother signals, fewer whipsaws +- **Cross-asset lead-lag** — BTC momentum predicts ETH/SOL 1-6h later +- **Dynamic threshold** — adjust momentum entry threshold by recent vol +- **Inverse vol position sizing** — proven in production risk framework +- **Ensemble voting** — combine 3+ signals, only enter when majority agree +- **ISFR + funding divergence** — treat ISFR as fair funding and fade perps when venue funding trades rich to the index + +### Tier 3 — Radical / Novel +- **Pure mean reversion on funding rate** — trade the mean reversion of funding itself +- **Correlation regime switching** — different strategies for high/low BTC-ETH correlation +- **Pairs trading** — long ETH/short BTC (or vice versa) on relative value +- **Time-of-day patterns** — are there hourly seasonality patterns? +- **Volatility breakout** — enter when realized vol breaks above/below its own SMA +- **Machine learning lite** — rolling linear regression of features → direction + +## Data Available + +- BTC, ETH, SOL hourly OHLCV + funding rates +- ISFR hourly feature as `isfr_rate` on each `BarData` and in `bar_data[symbol].history` +- Val period: 2024-07-01 to 2025-03-31 +- History buffer: last 500 bars via `bar_data[symbol].history` DataFrame +- Columns: timestamp, open, high, low, close, volume, funding_rate, isfr_rate + +## Scoring Formula (from prepare.py) + +``` +score = sharpe * sqrt(trade_count_factor) - drawdown_penalty - turnover_penalty +trade_count_factor = min(num_trades / 50, 1.0) +drawdown_penalty = max(0, max_drawdown_pct - 15) * 0.05 +turnover_penalty = max(0, annual_turnover/capital - 500) * 0.001 +Hard cutoffs: <10 trades → -999, >50% drawdown → -999, lost >50% → -999 +``` + +## NEVER STOP + +Once the experiment loop has begun, do NOT pause to ask the human if you should continue. You are autonomous. If you run out of ideas, think harder. The loop runs until interrupted. diff --git a/spawn/templates/strategy/pyproject.toml b/spawn/templates/strategy/pyproject.toml new file mode 100644 index 0000000..94d01a6 --- /dev/null +++ b/spawn/templates/strategy/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "autotrader" +version = "0.1.0" +description = "Autonomous trading strategy research (autoresearch for Hyperliquid)" +requires-python = ">=3.10" +dependencies = [ + "numpy>=1.26.0", + "pandas>=2.1.0", + "scipy>=1.11.0", + "requests>=2.31.0", + "pyarrow>=14.0.0", +] diff --git a/spawn/templates/strategy/run_benchmarks.py b/spawn/templates/strategy/run_benchmarks.py new file mode 100644 index 0000000..3066eaf --- /dev/null +++ b/spawn/templates/strategy/run_benchmarks.py @@ -0,0 +1,44 @@ +"""Run all benchmark strategies and print leaderboard.""" +import sys +import importlib +import time +from prepare import load_data, run_backtest, compute_score + +BENCHMARKS = [ + "benchmarks.avellaneda_mm", + "benchmarks.regime_mm", + "benchmarks.funding_arb", + "benchmarks.isfr_funding_divergence", + "benchmarks.mean_reversion", + "benchmarks.momentum_breakout", +] + +data = load_data("val") +print(f"Loaded {sum(len(df) for df in data.values())} bars across {len(data)} symbols\n") + +results = [] +for name in BENCHMARKS: + short = name.split(".")[-1] + try: + mod = importlib.import_module(name) + strategy = mod.Strategy() + t0 = time.time() + result = run_backtest(strategy, data) + score = compute_score(result) + dt = time.time() - t0 + results.append((short, score, result.sharpe, result.total_return_pct, + result.max_drawdown_pct, result.num_trades, result.win_rate_pct, dt)) + print(f" {short:25s} score={score:8.4f} sharpe={result.sharpe:6.3f} " + f"ret={result.total_return_pct:7.2f}% dd={result.max_drawdown_pct:5.2f}% " + f"trades={result.num_trades:5d} wr={result.win_rate_pct:5.1f}% ({dt:.1f}s)") + except Exception as e: + print(f" {short:25s} CRASHED: {e}") + results.append((short, -999, 0, 0, 0, 0, 0, 0)) + +print("\n" + "=" * 80) +print("LEADERBOARD (sorted by score, higher is better)") +print("=" * 80) +results.sort(key=lambda x: x[1], reverse=True) +for i, (name, score, sharpe, ret, dd, trades, wr, dt) in enumerate(results, 1): + print(f" {i}. {name:25s} score={score:8.4f} sharpe={sharpe:6.3f} " + f"ret={ret:7.2f}% dd={dd:5.2f}% trades={trades:5d}") diff --git a/spawn/templates/strategy/strategy.py b/spawn/templates/strategy/strategy.py new file mode 100644 index 0000000..ffecc9c --- /dev/null +++ b/spawn/templates/strategy/strategy.py @@ -0,0 +1,77 @@ +""" +Autotrader strategy file. This is the ONLY file the agent modifies. + +Start simple. Beat the existing Nunchi production strategies. +The agent should discover novel strategies, not just tune parameters. + +Usage: imported by backtest.py — do not run directly. +""" + +import numpy as np +import pandas as pd +from prepare import Signal, PortfolioState, BarData + +# --------------------------------------------------------------------------- +# Parameters +# --------------------------------------------------------------------------- + +LOOKBACK = 24 +POSITION_SIZE_PCT = 0.10 +STOP_LOSS_PCT = 0.03 +TAKE_PROFIT_PCT = 0.06 +MOMENTUM_THRESHOLD = 0.02 +ISFR_DROP_THRESHOLD = 0.0005 +ISFR_SPIKE_THRESHOLD = 0.0005 +ACTIVE_SYMBOLS = ["BTC", "ETH", "SOL"] + +# --------------------------------------------------------------------------- +# Strategy +# --------------------------------------------------------------------------- + +class Strategy: + def __init__(self): + self.entry_prices = {} + + def on_bar(self, bar_data: dict, portfolio: PortfolioState) -> list: + signals = [] + equity = portfolio.equity if portfolio.equity > 0 else portfolio.cash + + for symbol in ACTIVE_SYMBOLS: + if symbol not in bar_data: + continue + bd = bar_data[symbol] + if len(bd.history) < LOOKBACK: + continue + + closes = bd.history["close"].values[-LOOKBACK:] + isfr_series = bd.history.get("isfr_rate", pd.Series([0.0])).values[-LOOKBACK:] + returns = (closes[-1] - closes[0]) / closes[0] + isfr_delta = 0.0 + if len(isfr_series) >= LOOKBACK: + isfr_delta = float(isfr_series[-1] - np.mean(isfr_series)) + + current_pos = portfolio.positions.get(symbol, 0.0) + target_notional = current_pos + + if returns > MOMENTUM_THRESHOLD or isfr_delta < -ISFR_DROP_THRESHOLD: + target_notional = equity * POSITION_SIZE_PCT + elif returns < -MOMENTUM_THRESHOLD or isfr_delta > ISFR_SPIKE_THRESHOLD: + target_notional = -equity * POSITION_SIZE_PCT + + if current_pos != 0 and symbol in self.entry_prices: + entry = self.entry_prices[symbol] + if entry > 0: + pnl_pct = (bd.close - entry) / entry + if current_pos < 0: + pnl_pct = -pnl_pct + if pnl_pct < -STOP_LOSS_PCT or pnl_pct > TAKE_PROFIT_PCT: + target_notional = 0.0 + + if abs(target_notional - current_pos) > 1.0: + signals.append(Signal(symbol=symbol, target_position=target_notional)) + if target_notional != 0 and current_pos == 0: + self.entry_prices[symbol] = bd.close + elif target_notional == 0: + self.entry_prices.pop(symbol, None) + + return signals