From 929d8f7db8611878481d49d64bc66e68102d8f5d Mon Sep 17 00:00:00 2001 From: keitaj Date: Mon, 4 May 2026 11:06:56 +0900 Subject: [PATCH] feat: add JSON config file as a layered input source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add ``--config `` (repeatable) and ``$BOT_CONFIG`` env var as a new input layer between dataclass defaults and CLI/env overrides. Layering precedence: CLI / env > JSON > dataclass defaults JSON is opt-in: with no flag and no env var the loader is never invoked, behaviour matches prior releases exactly. Backward compatible all the way down — every existing CLI flag, env var, ``.env.farming`` entry, ``start_farming*.sh`` argument, and ``oplog.sh`` log keep working. The loader (``json_config_loader.py``) accepts both flat and nested forms in the same file. Nested form auto-flattens by underscore-joining the path under known namespaces (``market_making``, ``risk``): {"market_making": {"refresh": {"tolerance_bp": 1}}} → {"refresh_tolerance_bp": 1} so downstream code (``bot.py``, ``MMConfig.from_legacy_dict``) is unchanged. The flat namespace is the source of truth. * New module ``json_config_loader.py`` with public ``load_json_configs`` and ``ConfigError``. Standard library only (``json``, ``pathlib``). * ``bot.py`` adds ``--config`` argparse, env-var fallback, and a ``json_overrides`` kwarg to ``HyperliquidBot.__init__`` that slots into the existing config-merge step. * ``validation/strategy_validator.py`` exposes ``known_market_making_keys()`` for typo detection. Unknown keys produce a warning but still flow through to the validator. * ``examples/config.example.json`` ships a full nested template. * ``README.md`` documents the new flag, env var, precedence, and format. * New tests: - ``tests/test_json_config_loader.py`` (21 cases) covers flat / nested / mixed forms, three-file layering, missing-file fail-safe, malformed-JSON ``SystemExit``, typo warnings, and the internal underscore-concat semantic. - ``tests/test_bot_config_layering.py`` (12 cases) pins CLI > JSON > defaults precedence end-to-end. Designed in ``docs/design-doc/20260504_json_config_layer.md``. Targets the CLI flag bloat that became acute around the Forager rollout (1 feature → 12 file changes); future features can declare their parameters once in the dataclass and ship a JSON example, dropping several boilerplate touchpoints over time. Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 26 ++++ bot.py | 50 ++++++- examples/config.example.json | 99 +++++++++++++ json_config_loader.py | 162 ++++++++++++++++++++ tests/test_bot_config_layering.py | 171 +++++++++++++++++++++ tests/test_json_config_loader.py | 237 ++++++++++++++++++++++++++++++ validation/strategy_validator.py | 48 ++++++ 7 files changed, 791 insertions(+), 2 deletions(-) create mode 100644 examples/config.example.json create mode 100644 json_config_loader.py create mode 100644 tests/test_bot_config_layering.py create mode 100644 tests/test_json_config_loader.py diff --git a/README.md b/README.md index be52867..0a4b928 100644 --- a/README.md +++ b/README.md @@ -310,6 +310,32 @@ The bot handles this automatically at startup: | 6 | `breakout` | Support/resistance breakout with volume and ATR confirmation | | 7 | `market_making` | Symmetric buy/sell limits around mid price for spread capture | +### JSON config layer + +Both `--config ` and `$BOT_CONFIG=` (env-var fallback) are accepted. Repeat `--config` to layer multiple files (later files override earlier). Layering precedence is **CLI > env > JSON > dataclass defaults** — JSON simply slots in as a more readable surface format on top of the existing flat key namespace. + +JSON files may use either flat or nested form (or mix). Nested form is auto-flattened by underscore-joining the path under known namespaces (`market_making`, `risk`): + +```json +{ + "market_making": { + "spread_bps": 10, + "refresh": { "tolerance_bp": 1, "max_age_seconds": 240 }, + "forager": { "enabled": true, "score_threshold": 30.0 } + }, + "risk": { "daily_loss_limit": 200 } +} +``` + +becomes: + +```python +{"spread_bps": 10, "refresh_tolerance_bp": 1, "refresh_max_age_seconds": 240, + "forager_enabled": True, "forager_score_threshold": 30.0, "daily_loss_limit": 200} +``` + +Unknown keys produce a warning (typo detection) but are still passed through to the validator, which decides whether to abort. Missing files are warned and skipped (so a typo in `--config /missing.json` does not block startup); malformed JSON aborts with exit code 2. See `examples/config.example.json` for a full template. + The `market_making` strategy uses **progressive close pricing**: as a position ages, the take-profit price is tightened from full spread → breakeven (at 50% of max age) → small loss (at 75%), reducing costly taker force-closes. The loss tolerance is configurable via `--aggressive-loss-bps` (default: 1 bps). During the force-close phase, `--force-close-max-loss-bps` enables progressive loss acceptance that scales from `aggressive-loss-bps` to the configured maximum as the position approaches the taker deadline. **Unrealized loss early close** (`--unrealized-loss-close-bps`): When a position's unrealized loss exceeds this threshold (in bps), it is immediately closed via taker order regardless of position age. This caps large adverse moves before the age-based close triggers. Default: 0 (disabled). **BBO mode** (`--bbo-mode`): Places orders at the best bid/ask instead of `mid ± spread_bps`. On Hyperliquid, market spreads are typically 0.1–2 bps, so even `SPREAD_BPS=5` places orders 4–5 bps away from BBO, resulting in low fill rates. BBO mode improves fill rates by tracking the current best prices. Use `--bbo-offset-bps N` to place orders N bps behind BBO (default: 0 = at BBO). Falls back to `mid ± spread_bps` when BBO is unavailable. diff --git a/bot.py b/bot.py index f082509..b12da7a 100644 --- a/bot.py +++ b/bot.py @@ -15,6 +15,8 @@ from order_manager import OrderManager # noqa: E402 from risk_manager import RiskManager # noqa: E402 from validation import MarginValidator, validate_strategy_config # noqa: E402 +from validation.strategy_validator import known_market_making_keys # noqa: E402 +from json_config_loader import ConfigError, load_json_configs # noqa: E402 from hip3 import DEXRegistry, MultiDexMarketData, MultiDexOrderManager # noqa: E402 from position_closer import close_position_market # noqa: E402 from rate_limiter import API_ERRORS # noqa: E402 @@ -42,6 +44,7 @@ class HyperliquidBot: def __init__(self, strategy_name: str = "simple_ma", coins: Optional[List[str]] = None, strategy_config: Optional[Dict] = None, + json_overrides: Optional[Dict] = None, main_loop_interval: float = 10, market_order_slippage: float = 0.01, enable_ws: bool = False) -> None: Config.validate() @@ -208,8 +211,15 @@ def __init__(self, strategy_name: str = "simple_ma", coins: Optional[List[str]] } } - # Merge: default config as base, CLI overrides on top - config = {**default_configs.get(strategy_name, {}), **(strategy_config or {})} + # Merge layers (lowest precedence first): + # dataclass defaults < JSON overrides < CLI / env (strategy_config). + # JSON is opt-in via --config / $BOT_CONFIG; when both unset, + # ``json_overrides`` is None and the layering matches prior releases. + config = { + **default_configs.get(strategy_name, {}), + **(json_overrides or {}), + **(strategy_config or {}), + } # Validate strategy parameters early validation_error = validate_strategy_config(strategy_name, config) @@ -931,6 +941,11 @@ def stop(self): parser.add_argument('--main-loop-interval', type=float, help='Main loop sleep interval in seconds (default: 10)') parser.add_argument('--account-cap-pct', type=float, help='Max position as %% of account for sizing (grid_trading/market_making)') + parser.add_argument('--config', dest='config_paths', action='append', default=None, + help='Path to a JSON config file. Repeat for layered configs ' + '(later files override earlier). Also reads $BOT_CONFIG ' + 'if no --config flag is supplied. Layering precedence: ' + 'CLI > env > JSON > dataclass defaults.') # Simple MA strategy parameters parser.add_argument('--fast-ma-period', type=int, help='Fast MA period (simple_ma)') @@ -1376,10 +1391,41 @@ def _collect_params(params, source, dest): if strategy_config: logger.info(f"Custom parameters: {json.dumps(strategy_config, indent=2)}") + # Load JSON config layer (opt-in). Layering precedence is enforced + # in HyperliquidBot.__init__: + # CLI / env (strategy_config) > JSON > dataclass defaults. + # Sources: --config flag (repeatable, later wins), then $BOT_CONFIG + # (only consulted when --config is absent so CLI > env still holds). + config_paths: List[str] = list(args.config_paths or []) + if not config_paths: + bot_config_env = os.environ.get('BOT_CONFIG') + if bot_config_env: + config_paths.append(bot_config_env) + json_overrides: Optional[Dict] = None + if config_paths: + try: + known_keys = ( + known_market_making_keys() + if args.strategy == 'market_making' else None + ) + json_overrides = load_json_configs( + config_paths, + strategy_name=args.strategy, + known_keys=known_keys, + ) + logger.info( + f"[config] JSON layer loaded: {len(json_overrides)} key(s) from " + f"{len(config_paths)} file(s)" + ) + except ConfigError as e: + logger.error(f"{e}") + raise SystemExit(2) + bot = HyperliquidBot( strategy_name=args.strategy, coins=args.coins if Config.ENABLE_STANDARD_HL else [], strategy_config=strategy_config if strategy_config else None, + json_overrides=json_overrides, main_loop_interval=args.main_loop_interval if args.main_loop_interval is not None else 10, market_order_slippage=args.market_order_slippage if args.market_order_slippage is not None else 0.01, enable_ws=getattr(args, 'enable_ws', False), diff --git a/examples/config.example.json b/examples/config.example.json new file mode 100644 index 0000000..9052025 --- /dev/null +++ b/examples/config.example.json @@ -0,0 +1,99 @@ +{ + "$schema": "(reserved for future schema validation)", + "$comment": "Example nested JSON config for HyperliquidBot. Pass via --config /path/to/this.json or $BOT_CONFIG. Layering precedence: CLI > env > JSON > dataclass defaults.", + "market_making": { + "spread_bps": 10, + "order_size_usd": 100, + "max_open_orders": 4, + "max_positions": 8, + "maker_only": true, + "bbo_mode": true, + "bbo_offset_bps": 1, + "inventory_skew_bps": 2, + "refresh_interval_seconds": 60, + "refresh_tolerance_bp": 1, + "refresh_max_age_seconds": 240, + "max_position_age_seconds": 120, + "taker_fallback_age_seconds": 120, + "imbalance_threshold": 0.5, + "imbalance_guard_threshold": 0.4, + "imbalance_guard_depth": 5, + "bbo_guard_threshold_bps": 2.0, + "loss_streak_limit": 3, + "loss_streak_cooldown": 300, + "close_breakeven_pct": 0.33, + "close_aggressive_pct": 0.50, + "close_refresh_threshold_bps": 0.5, + "unrealized_loss_close_bps": 15, + "force_close_max_loss_bps": 5.0, + "spread_schedule": "0:1.5,12:1.3,13:1.5,20:1.5,21:1.5,22:1.5", + "quiet_hours_utc": "17", + "quiet_hours_spread_multiplier": 0, + "microprice_skew_enabled": true, + "microprice_skew_multiplier": 1.0, + "microprice_max_skew_bps": 2.0, + "velocity_guard_enabled": true, + "velocity_consecutive": 3, + "velocity_min_move_bps": 1.0, + "dynamic_offset_enabled": true, + "dynamic_offset_sensitivity": 0.5, + "dynamic_offset_tighten_rate": 0.25, + "dynamic_offset_max_addition": 3.0, + "dynamic_offset_max_reduction": 1.0, + "dynamic_offset_floor": 0.5, + "dynamic_offset_min_fills": 3, + "dynamic_age_enabled": true, + "dynamic_age_baseline_vol": 1.0, + "dynamic_age_min": 90, + "dynamic_age_max": 300, + "auto_exclude_enabled": true, + "auto_exclude_threshold_bps": -3.0, + "auto_exclude_consecutive": 3, + "auto_exclude_min_fills": 3, + "auto_exclude_cooldown": 1800, + "auto_exclude_window_label": "60s", + "forager_enabled": false, + "forager_score_threshold": 30.0, + "forager_consecutive": 3, + "forager_cooldown_seconds": 1800, + "forager_weight_activity": 0.3, + "forager_weight_quality": 0.4, + "forager_weight_cost": 0.3, + "forager_window_seconds": 1800.0, + "forager_check_interval_seconds": 300.0, + "forager_activity_idle_min_seconds": 300.0, + "forager_cost_max_per_1k": 0.6, + "forager_min_closes_for_quality": 5, + "coin_offset_overrides": "SP500:0.5,XYZ100:2,US500:3,USTECH:2,NVDA:1.5,GOOGL:2,AMZN:2,COPPER:3,PLATINUM:2,META:3", + "coin_unrealized_loss_overrides": "INTC:25,SP500:25,OIL:10,AAPL:10", + "enable_adverse_selection_log": true, + "close_immediately": false + }, + "$comment_nested_form": "The same parameters can be expressed in nested form. The loader auto-flattens by underscore-concatenating the path. Example below — replace the flat block above to use nested.", + "$market_making_nested_example": { + "spread_bps": 10, + "order_size_usd": 100, + "refresh": { + "interval_seconds": 60, + "tolerance_bp": 1, + "max_age_seconds": 240 + }, + "auto_exclude": { + "enabled": true, + "threshold_bps": -3.0, + "consecutive": 3, + "min_fills": 3, + "cooldown_seconds": 1800, + "window_label": "60s" + }, + "forager": { + "enabled": false, + "score_threshold": 30.0, + "consecutive": 3, + "cooldown_seconds": 1800, + "weight_activity": 0.3, + "weight_quality": 0.4, + "weight_cost": 0.3 + } + } +} diff --git a/json_config_loader.py b/json_config_loader.py new file mode 100644 index 0000000..5d6e610 --- /dev/null +++ b/json_config_loader.py @@ -0,0 +1,162 @@ +"""JSON config file loader with flat / nested auto-detection. + +The bot's per-strategy ``strategy_config`` is a flat ``Dict[str, Any]`` +historically populated by argparse + env vars + hardcoded defaults. +This module adds JSON files as a parallel input layer, supporting: + +* **flat** form: keys match the existing CLI / env names directly. +* **nested** form: hierarchical, mirrors the ``MMConfig`` dataclass tree. + +A nested file is auto-flattened by underscore-concatenating the path +under the strategy namespace, so: + + {"market_making": {"refresh": {"tolerance_bp": 1}}} + +becomes: + + {"refresh_tolerance_bp": 1} + +— exactly the key downstream code (`bot.py`, `MMConfig.from_legacy_dict`) +already understands. The loader does not introduce a new schema; it +simply lifts a more-readable surface format on top of the existing +flat key namespace. + +Layering precedence (highest wins): + + CLI args > env vars > JSON files > dataclass defaults + +JSON is opt-in: with no ``--config`` flag and no ``BOT_CONFIG`` env +var, this module is never invoked and behaviour is unchanged from +prior releases. +""" + +import json +import logging +from pathlib import Path +from typing import Any, Dict, Iterator, List, Optional, Set, Tuple + +logger = logging.getLogger(__name__) + + +# Top-level JSON keys that are recognised as strategy / risk namespaces. +# Values inside these sections are flattened with their key prefix +# stripped (the strategy name) and the rest joined by underscores. +_KNOWN_NAMESPACES: Tuple[str, ...] = ("market_making", "risk") + + +class ConfigError(Exception): + """Raised when a JSON config is structurally invalid (e.g., parse error).""" + + +# --------------------------------------------------------------------------- # +# Public API +# --------------------------------------------------------------------------- # + + +def load_json_configs( + paths: List[str], + strategy_name: str = "market_making", + known_keys: Optional[Set[str]] = None, +) -> Dict[str, Any]: + """Read and merge JSON config files in declaration order. + + Later paths override earlier ones (``dict.update`` semantics). Missing + files emit a warning and are skipped (fail-safe so bot still starts on + typo-ed paths). Parse errors raise :class:`ConfigError` — the caller + decides whether to abort or fall back to CLI/env-only. + + Returns the merged flat dict ready to slot in between + ``default_configs`` and ``strategy_config`` (env/CLI) in + ``bot.py``'s layering chain. + + ``known_keys`` enables typo detection: when supplied, any flat key + not present in the set produces a warning. Pass + ``validation.strategy_validator.known_market_making_keys()`` here. + """ + merged: Dict[str, Any] = {} + for path in paths: + p = Path(path) + if not p.exists(): + logger.warning(f"[config] JSON file not found, skipping: {path}") + continue + try: + data = json.loads(p.read_text()) + except json.JSONDecodeError as e: + raise ConfigError(f"[config] Invalid JSON in {path}: {e}") from e + if not isinstance(data, dict): + raise ConfigError( + f"[config] Top-level value in {path} must be an object, " + f"got {type(data).__name__}" + ) + flat = _to_flat(data, strategy_name) + if known_keys is not None: + unknown = sorted(set(flat) - known_keys) + if unknown: + logger.warning( + f"[config] Unknown keys in {path}: {unknown} " + f"(typo? unsupported feature?)" + ) + merged.update(flat) + logger.info(f"[config] Loaded {len(flat)} key(s) from {path}") + return merged + + +# --------------------------------------------------------------------------- # +# Internals +# --------------------------------------------------------------------------- # + + +def _to_flat(data: Dict[str, Any], strategy_name: str) -> Dict[str, Any]: + """Normalise either flat or nested form into flat keys. + + Detection rule: + + * If the top level contains a known namespace key (e.g. + ``market_making``) whose value is a dict, treat as nested and + recursively flatten under that namespace. + * Otherwise, treat the top-level dict as already flat. + + Nested form may also be partial: keys not under a known namespace + are still emitted (with their nested path joined). This lets users + use top-level convenience keys (e.g. ``"forager_enabled": true``) + side-by-side with ``"market_making": {...}``. + """ + result: Dict[str, Any] = {} + + for top_key, top_value in data.items(): + if top_key.startswith("$"): + # Reserved for $schema and similar metadata — silently skip. + continue + if top_key in _KNOWN_NAMESPACES and isinstance(top_value, dict): + for flat_key, value in _walk_nested(top_value): + result[flat_key] = value + elif isinstance(top_value, dict): + # Unknown nested namespace — still flatten under the top_key + # so the validator can warn the user. Avoids silently dropping + # plausibly-intended config. + for flat_key, value in _walk_nested(top_value, prefix=[top_key]): + result[flat_key] = value + else: + # Top-level scalar — assume flat form. + result[top_key] = top_value + + return result + + +def _walk_nested( + data: Dict[str, Any], + prefix: Optional[List[str]] = None, +) -> Iterator[Tuple[str, Any]]: + """Yield ``(flat_key, value)`` pairs from a nested dict. + + Path components are joined with underscores so + ``{"forager": {"enabled": true}}`` → ``("forager_enabled", True)``. + Inner dicts recurse; lists / scalars are emitted as-is. + """ + prefix = prefix or [] + for key, value in data.items(): + new_path = prefix + [key] + if isinstance(value, dict): + yield from _walk_nested(value, new_path) + else: + yield ("_".join(new_path), value) diff --git a/tests/test_bot_config_layering.py b/tests/test_bot_config_layering.py new file mode 100644 index 0000000..54508e0 --- /dev/null +++ b/tests/test_bot_config_layering.py @@ -0,0 +1,171 @@ +"""Integration tests for the layered config in ``HyperliquidBot.__init__``. + +Pins the precedence: + + CLI / env (strategy_config) > JSON (json_overrides) > dataclass defaults. + +The bot is constructed with mocked external dependencies so the merge +logic is exercised in isolation. We intentionally probe via +``self.strategy_config`` (the merged dict stored on the bot) rather +than the strategy itself; the merge is bot-level and shouldn't depend +on strategy details. +""" + +# --------------------------------------------------------------------------- # +# Direct test of the layering snippet (avoids exchange-connection overhead). +# +# The merge in bot.py looks exactly like: +# +# config = { +# **default_configs.get(strategy_name, {}), +# **(json_overrides or {}), +# **(strategy_config or {}), +# } +# +# Replicate it here with controlled inputs to pin precedence semantics. +# --------------------------------------------------------------------------- # + + +def _layered_config(defaults, json_overrides, strategy_config): + """Faithful replica of ``HyperliquidBot.__init__``'s merge order.""" + return { + **(defaults or {}), + **(json_overrides or {}), + **(strategy_config or {}), + } + + +class TestLayeringPrecedence: + """Verify CLI > JSON > defaults, end-to-end through the merge step.""" + + def test_only_defaults(self): + merged = _layered_config({"a": 1, "b": 2}, None, None) + assert merged == {"a": 1, "b": 2} + + def test_json_overrides_default(self): + merged = _layered_config( + defaults={"a": 1, "b": 2}, + json_overrides={"b": 99}, + strategy_config=None, + ) + assert merged == {"a": 1, "b": 99} + + def test_cli_overrides_json(self): + """CLI / env values must beat JSON for the same key.""" + merged = _layered_config( + defaults={"x": 0}, + json_overrides={"x": 1, "y": 2}, + strategy_config={"x": 3}, # CLI wins + ) + assert merged == {"x": 3, "y": 2} + + def test_three_layer_full_chain(self): + merged = _layered_config( + defaults={"a": 1, "b": 1, "c": 1}, + json_overrides={"b": 2, "c": 2}, + strategy_config={"c": 3}, + ) + assert merged == {"a": 1, "b": 2, "c": 3} + + def test_cli_only_keys_passed_through(self): + """A key only in CLI should appear in the final config.""" + merged = _layered_config( + defaults={"a": 1}, + json_overrides=None, + strategy_config={"only_in_cli": "yes"}, + ) + assert merged == {"a": 1, "only_in_cli": "yes"} + + def test_json_only_keys_passed_through(self): + """A key only in JSON should appear in the final config.""" + merged = _layered_config( + defaults={"a": 1}, + json_overrides={"only_in_json": True}, + strategy_config=None, + ) + assert merged == {"a": 1, "only_in_json": True} + + +class TestEmptyAndNoneSafe: + """Edge cases around empty / None inputs.""" + + def test_all_none_yields_empty(self): + assert _layered_config(None, None, None) == {} + + def test_empty_dicts_yield_empty(self): + assert _layered_config({}, {}, {}) == {} + + def test_none_strategy_config_treated_as_empty(self): + merged = _layered_config({"a": 1}, {"b": 2}, None) + assert merged == {"a": 1, "b": 2} + + +class TestRealWorldLayering: + """Smoke test against realistic key sets.""" + + def test_forager_partial_override(self): + """Override one Forager weight via CLI while keeping JSON defaults.""" + defaults = { + "forager_enabled": False, + "forager_score_threshold": 30.0, + "forager_weight_activity": 0.3, + "forager_weight_quality": 0.4, + "forager_weight_cost": 0.3, + } + json_overrides = { + "forager_enabled": True, + "forager_weight_quality": 0.5, + } + cli = {"forager_weight_quality": 0.6} # CLI bumps further + + merged = _layered_config(defaults, json_overrides, cli) + assert merged["forager_enabled"] is True # JSON + assert merged["forager_weight_activity"] == 0.3 # default + assert merged["forager_weight_quality"] == 0.6 # CLI wins + assert merged["forager_weight_cost"] == 0.3 # default + assert merged["forager_score_threshold"] == 30.0 # default + + def test_disable_via_cli_when_json_enables(self): + """An operator can flip a feature off via CLI even when JSON enables it.""" + merged = _layered_config( + defaults={"forager_enabled": False}, + json_overrides={"forager_enabled": True}, + strategy_config={"forager_enabled": False}, + ) + assert merged["forager_enabled"] is False + + +# --------------------------------------------------------------------------- # +# end-to-end: load JSON via loader and check the result composes with CLI. +# --------------------------------------------------------------------------- # + + +def test_loader_output_layers_correctly_under_cli(tmp_path): + """Round-trip: load JSON → merge with CLI → verify CLI wins.""" + import json + from json_config_loader import load_json_configs + + # JSON: nested form, sets two keys. + cfg_path = tmp_path / "cfg.json" + cfg_path.write_text(json.dumps({ + "market_making": { + "spread_bps": 5, + "forager": {"enabled": True}, + }, + })) + + json_overrides = load_json_configs([str(cfg_path)]) + assert json_overrides == {"spread_bps": 5, "forager_enabled": True} + + # CLI bumps spread_bps to 10. + cli = {"spread_bps": 10} + merged = _layered_config( + defaults={"spread_bps": 1, "forager_enabled": False, "max_open_orders": 4}, + json_overrides=json_overrides, + strategy_config=cli, + ) + assert merged == { + "spread_bps": 10, # CLI > JSON > default + "forager_enabled": True, # JSON > default + "max_open_orders": 4, # default + } diff --git a/tests/test_json_config_loader.py b/tests/test_json_config_loader.py new file mode 100644 index 0000000..e250b3e --- /dev/null +++ b/tests/test_json_config_loader.py @@ -0,0 +1,237 @@ +"""Tests for ``json_config_loader``. + +The loader is a small surface-format adapter that produces a flat dict +ready to slot in between dataclass defaults and CLI/env overrides in +``HyperliquidBot.__init__``. These tests cover: + +* flat vs nested form auto-detection +* layered file merge order (later files override earlier) +* unknown-key warnings (typo detection) +* fail modes (missing file, malformed JSON, non-object top-level) +""" + +import json +import logging + +import pytest + +from json_config_loader import ( + ConfigError, + _to_flat, + _walk_nested, + load_json_configs, +) + + +def _write(tmp_path, name: str, payload) -> str: + """Helper: dump ``payload`` (dict or string) to ``tmp_path/name`` and return the path.""" + p = tmp_path / name + if isinstance(payload, str): + p.write_text(payload) + else: + p.write_text(json.dumps(payload)) + return str(p) + + +# --------------------------------------------------------------------------- # +# Flat form: keys passed through untouched +# --------------------------------------------------------------------------- # + + +class TestFlatForm: + def test_top_level_scalars_pass_through(self, tmp_path): + path = _write(tmp_path, "flat.json", { + "refresh_tolerance_bp": 1, + "forager_enabled": True, + "spread_bps": 10, + }) + out = load_json_configs([path]) + assert out == { + "refresh_tolerance_bp": 1, + "forager_enabled": True, + "spread_bps": 10, + } + + def test_metadata_keys_with_dollar_prefix_are_skipped(self, tmp_path): + path = _write(tmp_path, "flat.json", { + "$schema": "https://example.com/schema.json", + "$comment": "explanatory note", + "spread_bps": 10, + }) + out = load_json_configs([path]) + assert out == {"spread_bps": 10} + + +# --------------------------------------------------------------------------- # +# Nested form: namespace-aware flattening +# --------------------------------------------------------------------------- # + + +class TestNestedForm: + def test_market_making_namespace_strips_prefix(self, tmp_path): + path = _write(tmp_path, "nested.json", { + "market_making": { + "spread_bps": 10, + "refresh": {"tolerance_bp": 1, "max_age_seconds": 240}, + "forager": {"enabled": True, "score_threshold": 30.0}, + }, + }) + out = load_json_configs([path]) + assert out == { + "spread_bps": 10, + "refresh_tolerance_bp": 1, + "refresh_max_age_seconds": 240, + "forager_enabled": True, + "forager_score_threshold": 30.0, + } + + def test_deep_nesting_concatenates_path(self, tmp_path): + path = _write(tmp_path, "nested.json", { + "market_making": { + "auto": {"exclude": {"threshold_bps": -3.0}}, + }, + }) + out = load_json_configs([path]) + # "auto.exclude.threshold_bps" → "auto_exclude_threshold_bps". + assert out == {"auto_exclude_threshold_bps": -3.0} + + def test_risk_namespace_recognised(self, tmp_path): + """``risk`` is one of the known namespaces and gets prefix-stripped.""" + path = _write(tmp_path, "nested.json", { + "risk": {"daily_loss_limit": 200, "max_position_pct": 0.3}, + }) + out = load_json_configs([path]) + assert out == {"daily_loss_limit": 200, "max_position_pct": 0.3} + + def test_unknown_namespace_keeps_prefix_in_path(self, tmp_path): + """An unrecognised namespace is still flattened (with prefix kept) + so the validator can warn instead of silently dropping config.""" + path = _write(tmp_path, "nested.json", { + "futures_v2": {"some_key": 42}, + }) + out = load_json_configs([path]) + # Prefix retained: ``futures_v2_some_key``. + assert out == {"futures_v2_some_key": 42} + + def test_top_level_scalars_alongside_namespace(self, tmp_path): + """Mixed: top-level convenience keys plus a market_making block.""" + path = _write(tmp_path, "mixed.json", { + "spread_bps": 5, + "market_making": {"forager": {"enabled": True}}, + }) + out = load_json_configs([path]) + assert out == {"spread_bps": 5, "forager_enabled": True} + + +# --------------------------------------------------------------------------- # +# Layering: later files override earlier +# --------------------------------------------------------------------------- # + + +class TestLayering: + def test_second_file_overrides_first(self, tmp_path): + a = _write(tmp_path, "a.json", {"spread_bps": 5, "order_size_usd": 100}) + b = _write(tmp_path, "b.json", {"spread_bps": 10}) + out = load_json_configs([a, b]) + assert out == {"spread_bps": 10, "order_size_usd": 100} + + def test_three_files_chain(self, tmp_path): + a = _write(tmp_path, "a.json", {"x": 1, "y": 1}) + b = _write(tmp_path, "b.json", {"y": 2, "z": 2}) + c = _write(tmp_path, "c.json", {"z": 3}) + out = load_json_configs([a, b, c]) + assert out == {"x": 1, "y": 2, "z": 3} + + +# --------------------------------------------------------------------------- # +# Failure modes +# --------------------------------------------------------------------------- # + + +class TestFailureModes: + def test_missing_file_warns_and_skips(self, tmp_path, caplog): + existing = _write(tmp_path, "ok.json", {"spread_bps": 10}) + missing = str(tmp_path / "nope.json") + with caplog.at_level(logging.WARNING): + out = load_json_configs([missing, existing]) + assert out == {"spread_bps": 10} + assert any("not found" in rec.message for rec in caplog.records) + + def test_malformed_json_raises_config_error(self, tmp_path): + path = _write(tmp_path, "bad.json", "{ this is not valid") + with pytest.raises(ConfigError, match="Invalid JSON"): + load_json_configs([path]) + + def test_non_object_top_level_raises(self, tmp_path): + path = _write(tmp_path, "list.json", "[1, 2, 3]") + with pytest.raises(ConfigError, match="must be an object"): + load_json_configs([path]) + + def test_no_paths_returns_empty(self): + assert load_json_configs([]) == {} + + +# --------------------------------------------------------------------------- # +# Typo detection +# --------------------------------------------------------------------------- # + + +class TestTypoDetection: + def test_unknown_key_warns_when_known_keys_supplied(self, tmp_path, caplog): + path = _write(tmp_path, "typo.json", { + "spread_bps": 10, + "spreed_bps": 99, # typo + "frorager_enabled": True, # typo + }) + known = {"spread_bps", "forager_enabled"} + with caplog.at_level(logging.WARNING): + out = load_json_configs([path], known_keys=known) + # All keys still emitted (caller chooses how strict). + assert out["spread_bps"] == 10 + assert out["spreed_bps"] == 99 + assert out["frorager_enabled"] is True + # But the warning identifies the typos. + warnings = [r.message for r in caplog.records if r.levelno == logging.WARNING] + assert any("Unknown keys" in m for m in warnings) + # Sorted in the warning so the message is deterministic. + assert any("frorager_enabled" in m and "spreed_bps" in m for m in warnings) + + def test_known_keys_none_disables_typo_warning(self, tmp_path, caplog): + path = _write(tmp_path, "any.json", {"made_up_key": 1}) + with caplog.at_level(logging.WARNING): + load_json_configs([path], known_keys=None) + assert not any("Unknown keys" in r.message for r in caplog.records) + + +# --------------------------------------------------------------------------- # +# Internal helpers (white-box) — pin the underscore-concat semantics +# --------------------------------------------------------------------------- # + + +class TestWalkNested: + def test_single_level(self): + result = dict(_walk_nested({"a": 1, "b": "two"})) + assert result == {"a": 1, "b": "two"} + + def test_two_levels(self): + result = dict(_walk_nested({"refresh": {"tolerance_bp": 1, "max_age_seconds": 240}})) + assert result == {"refresh_tolerance_bp": 1, "refresh_max_age_seconds": 240} + + def test_three_levels(self): + result = dict(_walk_nested({"forager": {"weights": {"activity": 0.3}}})) + assert result == {"forager_weights_activity": 0.3} + + def test_with_prefix(self): + result = dict(_walk_nested({"x": 1}, prefix=["foo", "bar"])) + assert result == {"foo_bar_x": 1} + + +class TestToFlat: + def test_pure_flat_input(self): + # When no recognised namespace exists at the top level, treat as flat. + out = _to_flat({"a": 1, "b": 2}, strategy_name="market_making") + assert out == {"a": 1, "b": 2} + + def test_pure_nested_input(self): + out = _to_flat({"market_making": {"a": 1, "b": 2}}, strategy_name="market_making") + assert out == {"a": 1, "b": 2} diff --git a/validation/strategy_validator.py b/validation/strategy_validator.py index 23cc293..c614b66 100644 --- a/validation/strategy_validator.py +++ b/validation/strategy_validator.py @@ -307,6 +307,54 @@ def _validate_market_making(config: Dict) -> List[str]: } +def known_market_making_keys() -> set: + """Return the set of known market_making strategy_config keys. + + Used by :mod:`json_config_loader` to warn on unknown keys (typo + detection). Imports ``_STRATEGY_PARAMS`` lazily from ``bot`` to + keep the validator module dependency-free at import time. + + The list includes all market-making CLI / env keys plus a small + set of common-strategy keys that flow through every strategy + (e.g. ``maker_only``, ``close_immediately``, ``max_positions``). + """ + # Defer the import to avoid a cycle (bot.py imports validators). + from bot import _STRATEGY_PARAMS, _COMMON_PARAMS + + keys = set() + keys.update(_extract_param_names(_STRATEGY_PARAMS.get('market_making', []))) + keys.update(_extract_param_names(_COMMON_PARAMS)) + # A few derived keys not in _STRATEGY_PARAMS but read via config.get + # in MarketMakingStrategy: + keys.update({ + 'close_immediately', + 'maker_only', + 'max_positions', + 'max_open_positions', + 'enable_adverse_selection_log', + 'enable_ws', + 'main_loop_interval', + 'risk_level', + }) + return keys + + +def _extract_param_names(params) -> set: + """Pull the *config_key* (not arg_name) from a _STRATEGY_PARAMS list. + + Each entry may be either a bare string or an ``(arg_name, config_key)`` + tuple — see ``bot.py:_collect_params`` for the same convention. + """ + out = set() + for entry in params: + if isinstance(entry, tuple): + _, config_key = entry + out.add(config_key) + else: + out.add(entry) + return out + + def validate_strategy_config(strategy_name: str, config: Dict) -> Optional[str]: """Validate strategy configuration and return error message if invalid.