diff --git a/README.md b/README.md index 3772664..9c6e42f 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,23 @@ tape. Normal `.backtest(...)` remains defensive and backward-compatible. Cython/C++ remains deferred because the larger benchmark still points to facade/report overhead rather than pure Numba kernels. +Latest Phase 31 intrabar execution benchmark: + +| Route | Workload | Runtime | Throughput | Ratio | Parity | +|---|---:|---:|---:|---:|---| +| `close_target_v2_pure_kernel` | 25,000 bars | 0.0115s | 2,171,235 bars/s | baseline | baseline | +| `intrabar_bracket_v1_minimal` | 25,000 bars, 2,000 fills | 0.0118s | 2,113,511 bars/s | 1.03x close-target | oracle-checked | +| `intrabar_bracket_v1_audit` | 25,000 bars, fill ledger | 0.0527s | 474,245 bars/s | 4.46x minimal | pass | +| `intrabar_reference_python` | 25,000 bars | 0.2759s | 90,626 bars/s | 23.32x slower than minimal | truth model | +| `fill_replay_v1_kernel` | 25,000 bars, 2,000 fills | 0.0111s | 2,259,396 bars/s | 0.94x minimal | accounting | +| `native_event_explicit_orders_facade` | 25,000 bars, 2,000 market orders | 0.0761s | 328,311 bars/s | 6.44x minimal | speed reference | + +Phase 31 adds execution-contract certification for close-target, fast intrabar +SL/TP/trailing, and explicit fill replay paths. The fast intrabar kernel is +about 23.3x faster than the readable Python oracle on the committed benchmark +while preserving the oracle semantics through targeted parity tests and audit +second-pass checks. + Ecosystem positioning: | Tool | Core strength | Runtime model | QuantBT role beside it | diff --git a/__init__.py b/__init__.py index d0bfe73..caa4935 100644 --- a/__init__.py +++ b/__init__.py @@ -47,7 +47,7 @@ from .backtester import BacktestEngine from .portfolio import MultiSymbolPortfolio -from .endpoint import EndpointConfig, QuantBTEndpoint, QuantBTPreparedContext, format_metrics_report +from .endpoint import EndpointConfig, PreparedIntrabarRunner, QuantBTEndpoint, QuantBTPreparedContext, format_metrics_report from .walkforward import ( DuplicatePruner, EarlyStoppingCallback, @@ -90,6 +90,48 @@ from .adapters.nautilus import NautilusBacktestEngine from .core.types import BacktestResult from .core.results import BacktestResultV2, OptionBacktestResult +from .core.execution_contract import ( + EXECUTION_CONTRACT_REGISTRY, + AmbiguityPolicy, + ExecutionContract, + FillPhase, + FundingPhase, + IntrabarSameBarPolicy, + LiquidationPriority, + MarketFillPolicy, + SignalPhase, + StopGapPolicy, + TakeProfitGapPolicy, + TrailingUpdatePhase, + get_execution_contract, +) +from .core.market_tape import MarketValidationCertificate, PreparedMarketTape, prepare_market_tape +from .core.intrabar_reference import ( + IntrabarEventFlag, + IntrabarFill, + IntrabarFillReason, + IntrabarIntentTape, + IntrabarLevelMode, + IntrabarReferenceResult, + IntrabarSizingMode, + run_intrabar_reference, +) +from .core.intrabar_kernel import ( + FillReplayTape, + NativeFillReplayResult, + NativeIntrabarKernelResult, + run_fill_replay_kernel, + run_intrabar_kernel, +) +from .core.certification import ( + AlphaExecutionClassification, + CertificationLevel, + alpha_report_markdown, + build_alpha_certification_report, + certify_result_metadata, + classify_alpha_source, + scan_alpha_directory, +) from .core.orders import ( BasketIntent, Fill, @@ -515,6 +557,8 @@ "BacktestResultV2", "BracketOrderSpec", "AccountConfig", + "AlphaExecutionClassification", + "AmbiguityPolicy", "ArbExecutionPolicy", "ArbitrageLeg", "ArbitragePlan", @@ -527,6 +571,7 @@ "BasketLegSpec", "BasketSpec", "CalendarSpreadSpec", + "CertificationLevel", "CarryModel", "CarryModelKind", "ContractType", @@ -534,22 +579,40 @@ "CostModelKind", "CrossExchangeArbSpec", "DcaGridSpec", + "EXECUTION_CONTRACT_REGISTRY", "ExecutionConfig", + "ExecutionContract", "FeeModel", "Fill", + "FillReplayTape", "FillPricePolicy", + "FillPhase", + "FundingPhase", "FundingArbitrageSpec", "FrozenBasketPlan", "HedgePolicy", "HedgePolicyKind", "IndexBasketArbSpec", "InstrumentSpec", + "IntrabarEventFlag", + "IntrabarFill", + "IntrabarFillReason", + "IntrabarIntentTape", + "IntrabarLevelMode", + "IntrabarReferenceResult", + "IntrabarSizingMode", + "IntrabarSameBarPolicy", "LifecycleModel", "LifecycleModelKind", "LiquiditySide", + "LiquidationPriority", "MarginMode", "MarginModel", "MarginModelKind", + "MarketFillPolicy", + "MarketValidationCertificate", + "NativeFillReplayResult", + "NativeIntrabarKernelResult", "OmsMode", "OrderAction", "OrderActivationPolicy", @@ -561,32 +624,48 @@ "OptionsVolArbSpec", "PackageExecutionKind", "PackageRejection", + "PreparedMarketTape", + "PreparedIntrabarRunner", "SameBarPolicy", "SignalModel", "SignalModelKind", "SignalSpec", + "SignalPhase", "SizingPolicy", "SizingPolicyKind", "SpotPerpCashCarrySpec", "SpreadFormula", "SpreadFormulaKind", "StatArbPairSpec", + "StopGapPolicy", "StructuredOrderPlan", + "TakeProfitGapPolicy", "TimeInForce", "Trade", + "TrailingUpdatePhase", "TriangularArbSpec", + "alpha_report_markdown", "build_arbitrage_order_plan", + "build_alpha_certification_report", "build_bracket_order_plan", "build_quantity_constraints", "build_dca_grid_order_plan", "build_frozen_basket_orders", + "certify_result_metadata", + "classify_alpha_source", + "get_execution_contract", "order_intents_to_lifecycle_commands", + "prepare_market_tape", "normalize_portfolio_mode", "normalize_portfolio_sizing_mode", "normalize_rebalance_policy", "portfolio_capability_matrix", "quantize_signed_quantity", "round_down_to_step", + "run_fill_replay_kernel", + "run_intrabar_kernel", + "run_intrabar_reference", + "scan_alpha_directory", "SUPPORTED_DEPTH_MODELS", "l2_replay_available", "simulate_nautilus_order_package_depth", diff --git a/backends/native_vectorized.py b/backends/native_vectorized.py index b2968b7..e4399c7 100644 --- a/backends/native_vectorized.py +++ b/backends/native_vectorized.py @@ -8,6 +8,7 @@ from dataclasses import dataclass, field from typing import Dict, List, Optional, Union +import warnings import numpy as np import pandas as pd @@ -24,7 +25,15 @@ ) from ..core.constraints import build_quantity_constraints, quantize_target_units_matrix from ..core.results import BacktestResultV2 -from ..core.schema import AccountConfig, BasketLegSpec, BasketSpec, ExecutionConfig, InstrumentSpec +from ..core.schema import ( + AccountConfig, + BasketLegSpec, + BasketSpec, + ExecutionConfig, + FillPricePolicy, + InstrumentSpec, + SameBarPolicy, +) from ..core.vectorized import _engine_units_v2 from ..core.arbitrage import ( ArbitrageSpec, @@ -64,6 +73,22 @@ def __post_init__(self) -> None: raise ValueError("fee_rate must be >= 0") elif float(self.fee_rate) < 0.0: raise ValueError("fee_rate must be >= 0") + unsupported = [] + if self.execution.fill_price_policy is not FillPricePolicy.CLOSE: + unsupported.append(f"fill_price_policy={self.execution.fill_price_policy.value!r}") + if self.execution.same_bar_policy is not SameBarPolicy.CONSERVATIVE: + unsupported.append(f"same_bar_policy={self.execution.same_bar_policy.value!r}") + if self.execution.allow_partial_fill: + unsupported.append("allow_partial_fill=True") + if self.execution.min_order_notional > 0.0: + unsupported.append("min_order_notional") + if not self.execution.reject_on_insufficient_margin: + unsupported.append("reject_on_insufficient_margin=False") + if unsupported: + raise NotImplementedError( + "native_vectorized is the close_target_v2 contract and does not support " + + ", ".join(unsupported) + ) class NativeVectorizedBackend: @@ -78,6 +103,59 @@ class NativeVectorizedBackend: def __init__(self, config: NativeVectorizedConfig): self.config = config + @staticmethod + def _close_target_metadata( + *, + symbol_list: List[str], + idx: pd.DatetimeIndex, + high_low_source: str, + first_bar_policy: str, + ) -> Dict: + signature = market_data_signature(idx, symbol_list) + return { + "backend": "native_vectorized", + "backend_alias": "native_vectorized", + "engine": "close_target_v2", + "engine_id": "close_target_v2", + "kernel_version": "units_v2", + "execution_contract": { + "engine_id": "close_target_v2", + "signal_phase": "bar_close", + "fill_phase": "same_close", + "intrabar_exit_model": "none", + "market_fill_policy": "close", + "timeline": "mark close[t-1]->close[t], rebalance target at close[t]", + "accounting_certified": True, + "execution_generated_by_engine": True, + }, + "signal_phase": "bar_close", + "fill_phase": "same_close", + "intrabar_exit_model": "none", + "first_bar_target_policy": first_bar_policy, + "high_low_source": high_low_source, + "data_signature": signature, + } + + @staticmethod + def _high_low_source(highs, lows) -> str: + if highs is None and lows is None: + return "close_fallback_uncertified_intrabar_risk" + if highs is None: + return "high_close_fallback_uncertified_intrabar_risk" + if lows is None: + return "low_close_fallback_uncertified_intrabar_risk" + return "provided" + + @staticmethod + def _warn_high_low_fallback(high_low_source: str) -> None: + if high_low_source != "provided": + warnings.warn( + "native_vectorized close_target_v2 received missing high/low data and will use close fallback; " + "intrabar liquidation/risk is uncertified for this run. Pass explicit highs/lows for certified risk.", + RuntimeWarning, + stacklevel=3, + ) + def prepare_market_arrays( self, datetime_index: Union[pd.DatetimeIndex, pd.Series], @@ -98,10 +176,12 @@ def prepare_market_arrays( idx = validate_datetime(datetime_index) symbol_list = symbols or list(closes.keys()) close_dict = align_series(closes, symbol_list, idx) + high_low_source = self._high_low_source(highs, lows) + self._warn_high_low_fallback(high_low_source) high_dict = align_series(highs, symbol_list, idx, fallback=close_dict) low_dict = align_series(lows, symbol_list, idx, fallback=close_dict) funding_dict = prepare_funding(funding_rate if self.config.use_funding else 0.0, symbol_list, idx) - return build_market_arrays( + market = build_market_arrays( symbols=symbol_list, idx=idx, closes_dict=close_dict, @@ -109,6 +189,7 @@ def prepare_market_arrays( lows_dict=low_dict, funding_dict=funding_dict, ) + return market def run_target_units( self, @@ -135,6 +216,8 @@ def run_target_units( raise ValueError("symbols, target_units, and closes must contain the same keys") close_dict = align_series(closes, symbol_list, idx) + high_low_source = self._high_low_source(highs, lows) + self._warn_high_low_fallback(high_low_source) high_dict = align_series(highs, symbol_list, idx, fallback=close_dict) low_dict = align_series(lows, symbol_list, idx, fallback=close_dict) target_dict = align_series(target_units, symbol_list, idx, fill_val=0.0) @@ -168,6 +251,7 @@ def run_target_units( slot_size=slot_size, min_qty=min_qty, min_notional=min_notional, + high_low_source=high_low_source, ) def _run_target_arrays( @@ -191,6 +275,7 @@ def _run_target_arrays( min_notional: Optional[Union[float, Dict[str, float]]] = None, market_arrays: Optional[PreparedMarketArrays] = None, raw_signal_matrix: Optional[np.ndarray] = None, + high_low_source: str = "provided", ) -> BacktestResultV2: contract_sizes = self._per_symbol_array(contract_size, symbol_list, default=1.0) constraints = build_quantity_constraints( @@ -273,6 +358,22 @@ def _run_target_arrays( index=idx, ) + metadata = self._close_target_metadata( + symbol_list=symbol_list, + idx=idx, + high_low_source=high_low_source, + first_bar_policy="target_units[0]_not_executed; first executable rebalance occurs at bar index 1", + ) + metadata.update( + { + "fee_rate_oneway": self._fee_rate_metadata(fee_rates, symbol_list), + "slippage_bps": self.config.execution.slippage_bps, + "initial_buying_power": self.config.account.initial_capital * float(np.mean(leverages)), + "liquidation_reason": int(liq_reason), + "quantity_constraints": constraints.as_dict(), + } + ) + return BacktestResultV2( equity=equity, returns=returns, @@ -287,15 +388,7 @@ def _run_target_arrays( funding=funding, margin=margin, diagnostics=diagnostics, - metadata={ - "backend": "native_vectorized", - "engine": "units_v2", - "fee_rate_oneway": self._fee_rate_metadata(fee_rates, symbol_list), - "slippage_bps": self.config.execution.slippage_bps, - "initial_buying_power": self.config.account.initial_capital * float(np.mean(leverages)), - "liquidation_reason": int(liq_reason), - "quantity_constraints": constraints.as_dict(), - }, + metadata=metadata, ) def run_signals( @@ -336,6 +429,9 @@ def run_signals( symbol_list = symbols or list(positions.keys()) pos_dict = None if raw_signal_matrix is not None else align_series(positions, symbol_list, idx, fill_val=0.0) close_dict = None if market_arrays is not None else align_series(closes, symbol_list, idx) + high_low_source = "prepared_market_arrays" if market_arrays is not None else self._high_low_source(highs, lows) + if market_arrays is None: + self._warn_high_low_fallback(high_low_source) alloc = self._per_symbol_mapping(alloc_per_trade, symbol_list, default=100_000.0) if ht in ("signal_notional", "signal"): @@ -390,6 +486,7 @@ def run_signals( slot_size=slot_size, min_qty=min_qty, min_notional=min_notional, + high_low_source=high_low_source, ) if close_dict is None: diff --git a/benchmarks/README.md b/benchmarks/README.md index 371111b..dfea896 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -88,3 +88,17 @@ python3 benchmarks/gamma_scalping_backtestsample.py \ first-class delta-hedged combined-equity accounting. - Cython/C++ should only be considered after a larger profile shows pure kernels, not pandas/tape/report facade work, dominating runtime. + +Phase 31 intrabar execution: + +```bash +python3 benchmarks/run_phase31_intrabar.py --rows 25000 --repeats 3 +python3 benchmarks/run_phase31_intrabar.py --rows 512 --repeats 1 +``` + +- `phase31_intrabar_benchmark.*` compares the new fast + `intrabar_bracket_v1` kernel against the close-target pure kernel, the Python + intrabar oracle, fill replay, and the native-event explicit-order facade. +- Use the fast intrabar route for single-symbol next-open SL/TP/trailing + research. Use `report_level="audit"` for fill-ledger certification and + `report_level="minimal"` for WFO/optimizer loops. diff --git a/benchmarks/phase31_intrabar_benchmark.json b/benchmarks/phase31_intrabar_benchmark.json new file mode 100644 index 0000000..f39742f --- /dev/null +++ b/benchmarks/phase31_intrabar_benchmark.json @@ -0,0 +1,108 @@ +{ + "rows": 25000, + "repeats": 3, + "seed": 31, + "records": [ + { + "route": "close_target_v2_pure_kernel", + "rows": 25000, + "symbols": 1, + "fills_or_orders": 0, + "warmup_seconds": 0.21664978098124266, + "runtime_seconds": 0.011514185927808285, + "runtime_min_seconds": 0.011514185927808285, + "runtime_max_seconds": 0.03456737520173192, + "bars_per_second": 2171234.6975066373, + "ratio_vs_close_target": 1.0, + "ratio_vs_intrabar_minimal": null, + "speedup_vs_reference": null, + "parity": "baseline", + "notes": "" + }, + { + "route": "intrabar_bracket_v1_minimal", + "rows": 25000, + "symbols": 1, + "fills_or_orders": 2000, + "warmup_seconds": 0.043024857994169, + "runtime_seconds": 0.011828660033643246, + "runtime_min_seconds": 0.011828660033643246, + "runtime_max_seconds": 0.012280617840588093, + "bars_per_second": 2113510.7382319416, + "ratio_vs_close_target": 1.0273118836022497, + "ratio_vs_intrabar_minimal": null, + "speedup_vs_reference": 23.321355362486532, + "parity": "oracle_checked_in_tests", + "notes": "" + }, + { + "route": "intrabar_bracket_v1_audit", + "rows": 25000, + "symbols": 1, + "fills_or_orders": 2000, + "warmup_seconds": 0.05681353295221925, + "runtime_seconds": 0.05271531501784921, + "runtime_min_seconds": 0.05271531501784921, + "runtime_max_seconds": 0.056146749295294285, + "bars_per_second": 474245.48239036597, + "ratio_vs_close_target": 4.578292842269876, + "ratio_vs_intrabar_minimal": 4.4565753743801535, + "speedup_vs_reference": 5.23302163732173, + "parity": "pass", + "notes": "two_pass_sparse_fills" + }, + { + "route": "intrabar_reference_python", + "rows": 25000, + "symbols": 1, + "fills_or_orders": 2000, + "warmup_seconds": 0.25595735386013985, + "runtime_seconds": 0.27586038410663605, + "runtime_min_seconds": 0.27586038410663605, + "runtime_max_seconds": 0.3258694182150066, + "bars_per_second": 90625.55350584899, + "ratio_vs_close_target": 23.958305505593465, + "ratio_vs_intrabar_minimal": 23.321355362486532, + "speedup_vs_reference": null, + "parity": "truth_model", + "notes": "" + }, + { + "route": "fill_replay_v1_kernel", + "rows": 25000, + "symbols": 1, + "fills_or_orders": 2000, + "warmup_seconds": 0.018238954711705446, + "runtime_seconds": 0.011064901947975159, + "runtime_min_seconds": 0.011064901947975159, + "runtime_max_seconds": 0.01129651814699173, + "bars_per_second": 2259396.4336552406, + "ratio_vs_close_target": 0.9609799613580978, + "ratio_vs_intrabar_minimal": 0.9354315633811611, + "speedup_vs_reference": null, + "parity": "accounting_only", + "notes": "" + }, + { + "route": "native_event_explicit_orders_facade", + "rows": 25000, + "symbols": 1, + "fills_or_orders": 2000, + "warmup_seconds": 0.09161354415118694, + "runtime_seconds": 0.07614740869030356, + "runtime_min_seconds": 0.07614740869030356, + "runtime_max_seconds": 0.08270613476634026, + "bars_per_second": 328310.58114763454, + "ratio_vs_close_target": 6.613355834944222, + "ratio_vs_intrabar_minimal": 6.437534638219714, + "speedup_vs_reference": null, + "parity": "speed_reference_not_semantic_claim", + "notes": "full_facade_order_replay" + } + ], + "summary": { + "intrabar_minimal_speedup_vs_reference": 23.321355362486532, + "intrabar_audit_ratio_vs_minimal": 4.4565753743801535, + "intrabar_minimal_ratio_vs_close_target": 1.0273118836022497 + } +} \ No newline at end of file diff --git a/benchmarks/phase31_intrabar_benchmark.md b/benchmarks/phase31_intrabar_benchmark.md new file mode 100644 index 0000000..bc1d125 --- /dev/null +++ b/benchmarks/phase31_intrabar_benchmark.md @@ -0,0 +1,22 @@ +# Phase 31 Intrabar Benchmark + +- Rows: `25000` +- Repeats: `3` +- Seed: `31` + +| Route | Runtime | Bars/s | Ratio vs close-target | Ratio vs intrabar minimal | Speedup vs Python oracle | Fills/orders | Parity | Notes | +|---|---:|---:|---:|---:|---:|---:|---|---| +| `close_target_v2_pure_kernel` | 0.011514s | 2,171,235 | 1.00x | - | - | 0 | baseline | | +| `intrabar_bracket_v1_minimal` | 0.011829s | 2,113,511 | 1.03x | - | 23.32x | 2000 | oracle_checked_in_tests | | +| `intrabar_bracket_v1_audit` | 0.052715s | 474,245 | 4.58x | 4.46x | 5.23x | 2000 | pass | two_pass_sparse_fills | +| `intrabar_reference_python` | 0.275860s | 90,626 | 23.96x | 23.32x | - | 2000 | truth_model | | +| `fill_replay_v1_kernel` | 0.011065s | 2,259,396 | 0.96x | 0.94x | - | 2000 | accounting_only | | +| `native_event_explicit_orders_facade` | 0.076147s | 328,311 | 6.61x | 6.44x | - | 2000 | speed_reference_not_semantic_claim | full_facade_order_replay | + +## Summary + +- Fast intrabar minimal vs Python oracle: `23.32x` faster. +- Fast intrabar audit vs minimal: `4.46x` runtime ratio. +- Fast intrabar minimal vs close-target pure kernel: `1.03x` runtime ratio. + +Interpretation: close-target remains the fastest narrow contract. The new intrabar kernel is the fast path for alpha logic that needs next-open entry, intrabar SL/TP/trailing, and audit fills without falling back to Python event loops. diff --git a/benchmarks/run_phase31_intrabar.py b/benchmarks/run_phase31_intrabar.py new file mode 100644 index 0000000..b998ab5 --- /dev/null +++ b/benchmarks/run_phase31_intrabar.py @@ -0,0 +1,362 @@ +#!/usr/bin/env python3 +""" +Phase 31D intrabar execution benchmark and certification summary. +""" + +from __future__ import annotations + +import argparse +import gc +import json +import statistics +import sys +import time +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Dict, List + +import numpy as np +import pandas as pd + + +PACKAGE_DIR = Path(__file__).resolve().parents[1] +PROJECT_DIR = PACKAGE_DIR.parent +if str(PROJECT_DIR) not in sys.path: + sys.path.insert(0, str(PROJECT_DIR)) + +from quantbt import ( # noqa: E402 + AccountConfig, + BacktestEngineV2, + ExecutionContract, + FillReplayTape, + IntrabarIntentTape, + OrderIntent, + OrderSide, + OrderType, + prepare_market_tape, + run_fill_replay_kernel, + run_intrabar_kernel, + run_intrabar_reference, +) +from quantbt.core.vectorized import _engine_units_v2 # noqa: E402 + + +@dataclass +class Phase31BenchmarkRecord: + route: str + rows: int + symbols: int + fills_or_orders: int + warmup_seconds: float + runtime_seconds: float + runtime_min_seconds: float + runtime_max_seconds: float + bars_per_second: float + ratio_vs_close_target: float | None = None + ratio_vs_intrabar_minimal: float | None = None + speedup_vs_reference: float | None = None + parity: str = "n/a" + notes: str = "" + + +def run_benchmark(*, rows: int = 25_000, repeats: int = 3, seed: int = 31) -> Dict: + df, intent = _make_intrabar_fixture(rows=rows, seed=seed) + tape = prepare_market_tape(data=df, symbols=["BTC"], use_funding=False) + account = AccountConfig(initial_capital=100_000.0, leverage=10.0) + contract = ExecutionContract.intrabar_bracket(close_on_last_bar=True) + + records: list[Phase31BenchmarkRecord] = [] + close_stats = _measure(lambda: _run_close_target_kernel(tape, intent, account), repeats=repeats) + records.append(_record("close_target_v2_pure_kernel", rows, 1, 0, close_stats, baseline=close_stats["best"], parity="baseline")) + + minimal_stats = _measure( + lambda: run_intrabar_kernel(tape=tape, intent=intent, account=account, contract=contract, report_level="minimal"), + repeats=repeats, + ) + minimal_result = run_intrabar_kernel(tape=tape, intent=intent, account=account, contract=contract, report_level="minimal") + records.append( + _record( + "intrabar_bracket_v1_minimal", + rows, + 1, + minimal_result.fill_count, + minimal_stats, + baseline=close_stats["best"], + parity="oracle_checked_in_tests", + ) + ) + + audit_stats = _measure( + lambda: run_intrabar_kernel(tape=tape, intent=intent, account=account, contract=contract, report_level="audit"), + repeats=repeats, + ) + audit_result = run_intrabar_kernel(tape=tape, intent=intent, account=account, contract=contract, report_level="audit") + records.append( + _record( + "intrabar_bracket_v1_audit", + rows, + 1, + audit_result.fill_count, + audit_stats, + baseline=close_stats["best"], + intrabar_minimal=minimal_stats["best"], + parity="pass" if np.allclose(audit_result.equity, minimal_result.equity, atol=1e-9, rtol=0.0) else "fail", + notes="two_pass_sparse_fills", + ) + ) + + reference_stats = _measure( + lambda: run_intrabar_reference(tape=tape, intent=intent, account=account, contract=contract), + repeats=max(1, min(2, repeats)), + ) + records.append( + _record( + "intrabar_reference_python", + rows, + 1, + audit_result.fill_count, + reference_stats, + baseline=close_stats["best"], + intrabar_minimal=minimal_stats["best"], + parity="truth_model", + ) + ) + + fill_tape = FillReplayTape.from_frame(audit_result.fills_report) + fill_replay_stats = _measure( + lambda: run_fill_replay_kernel(tape=tape, fill_tape=fill_tape, account=account), + repeats=repeats, + ) + records.append( + _record( + "fill_replay_v1_kernel", + rows, + 1, + len(fill_tape.bar_index), + fill_replay_stats, + baseline=close_stats["best"], + intrabar_minimal=minimal_stats["best"], + parity="accounting_only", + ) + ) + + native_event_stats = _measure(lambda: _run_native_event_orders(df, audit_result.fills_report), repeats=max(1, min(2, repeats))) + records.append( + _record( + "native_event_explicit_orders_facade", + rows, + 1, + int(len(audit_result.fills_report)), + native_event_stats, + baseline=close_stats["best"], + intrabar_minimal=minimal_stats["best"], + parity="speed_reference_not_semantic_claim", + notes="full_facade_order_replay", + ) + ) + + reference = next(r for r in records if r.route == "intrabar_reference_python") + for record in records: + if record.route.startswith("intrabar_bracket_v1"): + record.speedup_vs_reference = reference.runtime_seconds / record.runtime_seconds + + return { + "rows": rows, + "repeats": repeats, + "seed": seed, + "records": [asdict(record) for record in records], + "summary": _summary(records), + } + + +def make_markdown(report: Dict) -> str: + lines = [ + "# Phase 31 Intrabar Benchmark", + "", + f"- Rows: `{report['rows']}`", + f"- Repeats: `{report['repeats']}`", + f"- Seed: `{report['seed']}`", + "", + "| Route | Runtime | Bars/s | Ratio vs close-target | Ratio vs intrabar minimal | Speedup vs Python oracle | Fills/orders | Parity | Notes |", + "|---|---:|---:|---:|---:|---:|---:|---|---|", + ] + for record in report["records"]: + lines.append( + "| `{route}` | {runtime:.6f}s | {bps:,.0f} | {rclose} | {rmin} | {speedup} | {fills} | {parity} | {notes} |".format( + route=record["route"], + runtime=record["runtime_seconds"], + bps=record["bars_per_second"], + rclose=_fmt_ratio(record["ratio_vs_close_target"]), + rmin=_fmt_ratio(record["ratio_vs_intrabar_minimal"]), + speedup=_fmt_ratio(record["speedup_vs_reference"]), + fills=record["fills_or_orders"], + parity=record["parity"], + notes=record["notes"] or "", + ) + ) + lines.extend( + [ + "", + "## Summary", + "", + f"- Fast intrabar minimal vs Python oracle: `{_fmt_ratio(report['summary']['intrabar_minimal_speedup_vs_reference'])}` faster.", + f"- Fast intrabar audit vs minimal: `{_fmt_ratio(report['summary']['intrabar_audit_ratio_vs_minimal'])}` runtime ratio.", + f"- Fast intrabar minimal vs close-target pure kernel: `{_fmt_ratio(report['summary']['intrabar_minimal_ratio_vs_close_target'])}` runtime ratio.", + "", + "Interpretation: close-target remains the fastest narrow contract. The new intrabar kernel is the fast path for alpha logic that needs next-open entry, intrabar SL/TP/trailing, and audit fills without falling back to Python event loops.", + ] + ) + return "\n".join(lines) + "\n" + + +def _make_intrabar_fixture(*, rows: int, seed: int): + rng = np.random.default_rng(seed) + idx = pd.date_range("2020-01-01", periods=rows, freq="1h", tz="UTC") + ret = rng.normal(0.0, 0.0015, size=rows) + close = 100.0 * np.exp(np.cumsum(ret)) + open_ = np.r_[close[0], close[:-1] * (1.0 + rng.normal(0.0, 0.0002, size=rows - 1))] + high = np.maximum(open_, close) * (1.0 + rng.uniform(0.0005, 0.006, size=rows)) + low = np.minimum(open_, close) * (1.0 - rng.uniform(0.0005, 0.006, size=rows)) + df = pd.DataFrame({"open": open_, "high": high, "low": low, "close": close, "volume": 100.0}, index=idx) + entry_side = np.zeros(rows, dtype=np.int8) + entry_size = np.zeros(rows, dtype=np.float64) + entry_side[5::50] = 1 + entry_size[5::50] = 1.0 + entry_side[30::50] = -1 + entry_size[30::50] = 1.0 + stop = np.full(rows, 0.012, dtype=np.float64) + tp = np.full(rows, 0.018, dtype=np.float64) + trailing = np.full(rows, 0.010, dtype=np.float64) + technical_exit = np.zeros(rows, dtype=np.bool_) + technical_exit[45::50] = True + intent = IntrabarIntentTape.from_arrays( + entry_side=entry_side, + entry_size=entry_size, + stop_value=stop, + take_profit_value=tp, + trailing_value=trailing, + technical_exit=technical_exit, + ) + return df, intent + + +def _run_close_target_kernel(tape, intent, account): + target = np.zeros((tape.n_bars, 1), dtype=np.float64) + current = 0.0 + for i in range(tape.n_bars): + if intent.entry_side[i] != 0 and intent.entry_size[i] > 0.0: + current = float(intent.entry_side[i]) * float(intent.entry_size[i]) + target[i, 0] = current + return _engine_units_v2( + tape.n_bars, + 1, + tape.highs, + tape.lows, + tape.closes, + target, + tape.funding_rates, + tape.funding_event_mask, + account.initial_capital, + np.array([account.leverage], dtype=np.float64), + account.maintenance_ratio, + np.array([0.0], dtype=np.float64), + np.array([1.0], dtype=np.float64), + 0.0, + False, + )[0][-1] + + +def _run_native_event_orders(df: pd.DataFrame, fills: pd.DataFrame): + orders = [] + idx = df.index + for row in fills.itertuples(index=False): + bar = int(row.bar_index) + side = OrderSide.BUY if int(row.side) > 0 else OrderSide.SELL + orders.append(OrderIntent(idx[bar], "BTC", side, OrderType.MARKET, qty=float(row.qty))) + engine = BacktestEngineV2( + data=df, + symbols=["BTC"], + backend="native_event", + orders=orders, + account=AccountConfig(initial_capital=100_000.0, leverage=10.0), + use_funding=False, + fee_rate=0.0, + ) + return engine.result.equity.iloc[-1] + + +def _measure(workload, *, repeats: int) -> Dict[str, float]: + gc.collect() + start = time.perf_counter() + workload() + warmup = time.perf_counter() - start + runtimes = [] + for _ in range(max(1, repeats)): + gc.collect() + start = time.perf_counter() + workload() + runtimes.append(time.perf_counter() - start) + return { + "best": float(min(runtimes)), + "worst": float(max(runtimes)), + "median": float(statistics.median(runtimes)), + "warmup": float(warmup), + } + + +def _record(route, rows, symbols, fills, stats, *, baseline, intrabar_minimal=None, parity="n/a", notes=""): + runtime = stats["best"] + return Phase31BenchmarkRecord( + route=route, + rows=rows, + symbols=symbols, + fills_or_orders=int(fills), + warmup_seconds=float(stats["warmup"]), + runtime_seconds=float(runtime), + runtime_min_seconds=float(stats["best"]), + runtime_max_seconds=float(stats["worst"]), + bars_per_second=float(rows / runtime) if runtime > 0 else float("inf"), + ratio_vs_close_target=float(runtime / baseline) if baseline and runtime else None, + ratio_vs_intrabar_minimal=float(runtime / intrabar_minimal) if intrabar_minimal and runtime else None, + parity=parity, + notes=notes, + ) + + +def _summary(records: List[Phase31BenchmarkRecord]) -> Dict: + lookup = {record.route: record for record in records} + minimal = lookup["intrabar_bracket_v1_minimal"] + audit = lookup["intrabar_bracket_v1_audit"] + reference = lookup["intrabar_reference_python"] + close_target = lookup["close_target_v2_pure_kernel"] + return { + "intrabar_minimal_speedup_vs_reference": reference.runtime_seconds / minimal.runtime_seconds, + "intrabar_audit_ratio_vs_minimal": audit.runtime_seconds / minimal.runtime_seconds, + "intrabar_minimal_ratio_vs_close_target": minimal.runtime_seconds / close_target.runtime_seconds, + } + + +def _fmt_ratio(value) -> str: + if value is None: + return "-" + return f"{float(value):.2f}x" + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description="Run Phase 31 intrabar benchmark.") + parser.add_argument("--rows", type=int, default=25_000) + parser.add_argument("--repeats", type=int, default=3) + parser.add_argument("--seed", type=int, default=31) + parser.add_argument("--json-out", type=Path, default=PACKAGE_DIR / "benchmarks" / "phase31_intrabar_benchmark.json") + parser.add_argument("--md-out", type=Path, default=PACKAGE_DIR / "benchmarks" / "phase31_intrabar_benchmark.md") + args = parser.parse_args(argv) + + report = run_benchmark(rows=args.rows, repeats=args.repeats, seed=args.seed) + args.json_out.write_text(json.dumps(report, indent=2), encoding="utf-8") + args.md_out.write_text(make_markdown(report), encoding="utf-8") + print(make_markdown(report)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/core/__init__.py b/core/__init__.py index da78751..98496cf 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -3,6 +3,48 @@ from .vectorized import _engine_units_v2 from .types import BacktestResult from .results import BacktestResultV2 +from .execution_contract import ( + EXECUTION_CONTRACT_REGISTRY, + AmbiguityPolicy, + ExecutionContract, + FillPhase, + FundingPhase, + IntrabarSameBarPolicy, + LiquidationPriority, + MarketFillPolicy, + SignalPhase, + StopGapPolicy, + TakeProfitGapPolicy, + TrailingUpdatePhase, + get_execution_contract, +) +from .market_tape import MarketValidationCertificate, PreparedMarketTape, prepare_market_tape +from .intrabar_reference import ( + IntrabarEventFlag, + IntrabarFill, + IntrabarFillReason, + IntrabarIntentTape, + IntrabarLevelMode, + IntrabarReferenceResult, + IntrabarSizingMode, + run_intrabar_reference, +) +from .intrabar_kernel import ( + FillReplayTape, + NativeFillReplayResult, + NativeIntrabarKernelResult, + run_fill_replay_kernel, + run_intrabar_kernel, +) +from .certification import ( + AlphaExecutionClassification, + CertificationLevel, + alpha_report_markdown, + build_alpha_certification_report, + certify_result_metadata, + classify_alpha_source, + scan_alpha_directory, +) from .orders import ( BasketIntent, Fill, @@ -111,6 +153,8 @@ "BacktestResultV2", "BracketOrderSpec", "AccountConfig", + "AlphaExecutionClassification", + "AmbiguityPolicy", "ArbExecutionPolicy", "ArbitrageLeg", "ArbitragePlan", @@ -123,6 +167,7 @@ "BasketLegSpec", "BasketSpec", "CalendarSpreadSpec", + "CertificationLevel", "CarryModel", "CarryModelKind", "ContractType", @@ -130,23 +175,41 @@ "CostModelKind", "CrossExchangeArbSpec", "DcaGridSpec", + "EXECUTION_CONTRACT_REGISTRY", "ExecutionConfig", + "ExecutionContract", "FeeModel", "Fill", "FillPricePolicy", + "FillPhase", + "FundingPhase", "FundingArbitrageSpec", "FrozenBasketPlan", "HedgePolicy", "HedgePolicyKind", "IndexBasketArbSpec", "InstrumentSpec", + "IntrabarEventFlag", + "IntrabarFill", + "IntrabarFillReason", + "IntrabarIntentTape", + "IntrabarLevelMode", + "IntrabarReferenceResult", + "IntrabarSizingMode", + "IntrabarSameBarPolicy", + "FillReplayTape", "LifecycleModel", "LifecycleModelKind", "LiquiditySide", + "LiquidationPriority", "MarginMode", "MarginModel", "MarginModelKind", + "MarketFillPolicy", + "MarketValidationCertificate", "NautilusExecutionDepthConfig", + "NativeFillReplayResult", + "NativeIntrabarKernelResult", "NativeActiveOrderSnapshot", "NativeEventStrategyError", "NativeEventStrategyProtocol", @@ -164,26 +227,41 @@ "PackageExecutionKind", "PackageDepthPreflightResult", "PackageRejection", + "PreparedMarketTape", "SameBarPolicy", "SignalModel", "SignalModelKind", "SignalSpec", + "SignalPhase", "SizingPolicy", "SizingPolicyKind", "SpotPerpCashCarrySpec", "SpreadFormula", "SpreadFormulaKind", "StatArbPairSpec", + "StopGapPolicy", "StructuredOrderPlan", + "TakeProfitGapPolicy", "TimeInForce", "Trade", + "TrailingUpdatePhase", "TriangularArbSpec", + "alpha_report_markdown", "build_arbitrage_order_plan", + "build_alpha_certification_report", "build_bracket_order_plan", "build_dca_grid_order_plan", "build_frozen_basket_orders", + "certify_result_metadata", + "classify_alpha_source", + "get_execution_contract", "order_intents_to_lifecycle_commands", + "prepare_market_tape", "round_down_to_step", + "run_intrabar_reference", + "run_intrabar_kernel", + "run_fill_replay_kernel", + "scan_alpha_directory", "SUPPORTED_DEPTH_MODELS", "l2_replay_available", "simulate_nautilus_order_package_depth", diff --git a/core/certification.py b/core/certification.py new file mode 100644 index 0000000..1453f21 --- /dev/null +++ b/core/certification.py @@ -0,0 +1,274 @@ +""" +Alpha execution-contract certification helpers. + +These helpers are intentionally lightweight and conservative. They do not try +to prove a strategy has no look-ahead bias from source text alone; they identify +which execution contract a file appears to require and what certification level +an already-run result can claim from its metadata. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from enum import IntEnum +from pathlib import Path +import re +from typing import Dict, Iterable, List, Optional, Sequence + + +class CertificationLevel(IntEnum): + LEGACY = 0 + ACCOUNTING_REPLAY = 1 + ENGINE_CAUSAL = 2 + CROSS_BACKEND = 3 + EXTERNAL_VALIDATION = 4 + + +LEVEL_DESCRIPTIONS = { + CertificationLevel.LEGACY: "legacy_or_unspecified_execution_contract", + CertificationLevel.ACCOUNTING_REPLAY: "explicit_fills_accounted_but_fill_generation_not_certified", + CertificationLevel.ENGINE_CAUSAL: "engine_owned_causal_execution_with_oracle_or_kernel_parity", + CertificationLevel.CROSS_BACKEND: "native_engine_matches_native_event_on_known_scenarios", + CertificationLevel.EXTERNAL_VALIDATION: "external_or_lower_timeframe_validation_available", +} + + +INTRABAR_MARKERS = ( + "exit_price", + "exit_type", + "stop_loss", + "stoploss", + "take_profit", + "takeprofit", + "trailing", + "trailing_stop", + "use_sl", + "use_tp", + "slpercent", + "tppercent", + "high[", + "low[", +) +FILL_REPLAY_MARKERS = ("fill_replay", "fills_df", "compact_fill", "bar_index", "sequence") +GRID_MARKERS = ("dca_ladder", "grid", "safety_order", "take_profit_price", "stop_loss_price") +NEXT_OPEN_MARKERS = ("next_open", "open[t+1]", "shift(1)", "open.shift") +CLOSE_TARGET_MARKERS = ("native_vectorized", "signal_notional", "pos_weight", "target_weight") + + +@dataclass(frozen=True) +class AlphaExecutionClassification: + alpha_id: str + path: str + required_engine: str + current_backend: str + certification_status: str + certification_level: int + markers: tuple[str, ...] = () + notes: tuple[str, ...] = () + uses_intrabar_high_low: bool = False + uses_stop: bool = False + uses_take_profit: bool = False + uses_trailing: bool = False + uses_custom_exit_price: bool = False + uses_explicit_fills: bool = False + uses_grid_or_dca: bool = False + metadata: Dict = field(default_factory=dict) + + def to_dict(self) -> Dict: + return asdict(self) + + +def classify_alpha_source(source: str, *, alpha_id: str = "unknown", path: str = "") -> AlphaExecutionClassification: + text = source.lower() + markers = _matched_markers(text) + current_backend = _detect_current_backend(text) + uses_explicit_fills = any(marker in text for marker in FILL_REPLAY_MARKERS) + uses_grid_or_dca = any(marker in text for marker in GRID_MARKERS) + uses_stop = any(marker in text for marker in ("stop_loss", "stoploss", "slpercent", "use_sl")) + uses_take_profit = any(marker in text for marker in ("take_profit", "takeprofit", "tppercent", "use_tp")) + uses_trailing = "trailing" in text + uses_custom_exit_price = "exit_price" in text or "exit_type" in text + uses_intrabar_high_low = bool(re.search(r"\bhigh\s*\[|\blow\s*\[|df\s*\[\s*['\"]high|df\s*\[\s*['\"]low", text)) + + if uses_grid_or_dca: + required_engine = "event_lifecycle_v2" + level = CertificationLevel.LEGACY + status = "needs_specialized_event_or_nautilus_certification" + elif uses_explicit_fills and not (uses_stop or uses_take_profit or uses_trailing): + required_engine = "fill_replay_v1" + level = CertificationLevel.ACCOUNTING_REPLAY + status = "can_start_with_accounting_replay" + elif uses_stop or uses_take_profit or uses_trailing or uses_custom_exit_price or uses_intrabar_high_low: + required_engine = "intrabar_bracket_v1" + level = CertificationLevel.LEGACY + status = "requires_intrabar_migration" + elif any(marker in text for marker in NEXT_OPEN_MARKERS): + required_engine = "next_open_v1" + level = CertificationLevel.LEGACY + status = "requires_next_open_contract" + elif any(marker in text for marker in CLOSE_TARGET_MARKERS): + required_engine = "close_target_v2" + level = CertificationLevel.ENGINE_CAUSAL if current_backend in {"native_vectorized", "close_target_v2"} else CertificationLevel.LEGACY + status = "close_target_candidate" + else: + required_engine = "unknown" + level = CertificationLevel.LEGACY + status = "manual_review_required" + + notes = _notes_for_classification(required_engine, current_backend, markers) + return AlphaExecutionClassification( + alpha_id=alpha_id, + path=path, + required_engine=required_engine, + current_backend=current_backend, + certification_status=status, + certification_level=int(level), + markers=tuple(markers), + notes=tuple(notes), + uses_intrabar_high_low=uses_intrabar_high_low, + uses_stop=uses_stop, + uses_take_profit=uses_take_profit, + uses_trailing=uses_trailing, + uses_custom_exit_price=uses_custom_exit_price, + uses_explicit_fills=uses_explicit_fills, + uses_grid_or_dca=uses_grid_or_dca, + ) + + +def scan_alpha_directory(root: str | Path, *, suffixes: Sequence[str] = (".py", ".ipynb", ".md"), max_bytes: int = 2_000_000) -> List[AlphaExecutionClassification]: + base = Path(root) + if not base.exists(): + raise FileNotFoundError(str(base)) + out: list[AlphaExecutionClassification] = [] + for path in sorted(p for p in base.rglob("*") if p.is_file() and p.suffix.lower() in suffixes): + if any(part.startswith(".") for part in path.relative_to(base).parts): + continue + if path.stat().st_size > max_bytes: + out.append( + AlphaExecutionClassification( + alpha_id=path.stem, + path=str(path), + required_engine="unknown", + current_backend="unknown", + certification_status="skipped_large_file", + certification_level=int(CertificationLevel.LEGACY), + notes=("file exceeds scanner max_bytes",), + ) + ) + continue + text = path.read_text(encoding="utf-8", errors="ignore") + out.append(classify_alpha_source(text, alpha_id=path.stem, path=str(path))) + return out + + +def certify_result_metadata(metadata: Dict) -> Dict: + engine = str(metadata.get("engine_id") or metadata.get("engine") or "").lower() + backend = str(metadata.get("backend") or metadata.get("backend_alias") or "").lower() + if engine == "fill_replay_v1": + level = CertificationLevel.ACCOUNTING_REPLAY + status = "accounting_certified" + elif engine == "intrabar_bracket_v1": + level = CertificationLevel.ENGINE_CAUSAL + status = "engine_causal_certified" + if metadata.get("cross_backend_parity_passed"): + level = CertificationLevel.CROSS_BACKEND + status = "cross_backend_certified" + elif backend == "nautilus" or "nautilus" in engine: + level = CertificationLevel.EXTERNAL_VALIDATION + status = "external_validation_route" + elif engine == "close_target_v2": + level = CertificationLevel.ENGINE_CAUSAL + status = "close_target_certified" + if str(metadata.get("certification_status", "")).startswith("uncertified"): + level = CertificationLevel.LEGACY + status = str(metadata.get("certification_status")) + else: + level = CertificationLevel.LEGACY + status = "uncertified_or_unknown" + return { + "engine_id": engine or "unknown", + "backend": backend or "unknown", + "certification_level": int(level), + "certification_label": f"LEVEL {int(level)}", + "certification_status": status, + "description": LEVEL_DESCRIPTIONS[level], + } + + +def build_alpha_certification_report(items: Iterable[AlphaExecutionClassification]) -> Dict: + rows = [item.to_dict() for item in items] + by_engine: Dict[str, int] = {} + by_status: Dict[str, int] = {} + for row in rows: + by_engine[row["required_engine"]] = by_engine.get(row["required_engine"], 0) + 1 + by_status[row["certification_status"]] = by_status.get(row["certification_status"], 0) + 1 + return { + "total": len(rows), + "by_required_engine": by_engine, + "by_status": by_status, + "items": rows, + } + + +def alpha_report_markdown(report: Dict) -> str: + lines = [ + "# Alpha Execution Certification Report", + "", + f"- Total files scanned: `{report['total']}`", + "", + "## By Required Engine", + "", + "| Engine | Count |", + "|---|---:|", + ] + for engine, count in sorted(report["by_required_engine"].items()): + lines.append(f"| `{engine}` | {count} |") + lines.extend(["", "## By Status", "", "| Status | Count |", "|---|---:|"]) + for status, count in sorted(report["by_status"].items()): + lines.append(f"| `{status}` | {count} |") + lines.extend(["", "## Files", "", "| Alpha | Required engine | Current backend | Status | Markers |", "|---|---|---|---|---|"]) + for item in report["items"]: + markers = ", ".join(item["markers"][:8]) + if len(item["markers"]) > 8: + markers += ", ..." + lines.append( + f"| `{item['alpha_id']}` | `{item['required_engine']}` | `{item['current_backend']}` | " + f"`{item['certification_status']}` | {markers or '-'} |" + ) + return "\n".join(lines) + "\n" + + +def _matched_markers(text: str) -> list[str]: + all_markers = sorted(set(INTRABAR_MARKERS + FILL_REPLAY_MARKERS + GRID_MARKERS + NEXT_OPEN_MARKERS + CLOSE_TARGET_MARKERS)) + return [marker for marker in all_markers if marker in text] + + +def _detect_current_backend(text: str) -> str: + if "nautilus_validation" in text or "backend=\"nautilus\"" in text or "backend='nautilus'" in text: + return "nautilus" + if "intrabar_bracket" in text: + return "native_intrabar" + if "fill_replay" in text: + return "fill_replay_v1" + if "native_event" in text: + return "native_event" + if "native_vectorized" in text: + return "native_vectorized" + if "%_equity" in text or "pct_equity" in text: + return "legacy_pct_equity" + if "backtestengine(" in text: + return "legacy" + return "unknown" + + +def _notes_for_classification(required_engine: str, current_backend: str, markers: Sequence[str]) -> list[str]: + notes: list[str] = [] + if required_engine == "intrabar_bracket_v1" and current_backend in {"native_vectorized", "legacy", "legacy_pct_equity"}: + notes.append("intrabar markers found on a close-target/legacy route; migrate to intrabar intent or fill replay") + if required_engine == "fill_replay_v1": + notes.append("accounting can be validated from explicit fills, but fill generation remains alpha-owned") + if required_engine == "event_lifecycle_v2": + notes.append("multi-order/grid/DCA behavior should stay on event lifecycle or Nautilus validation") + if not markers: + notes.append("no execution-sensitive markers detected; manual review still required before production certification") + return notes diff --git a/core/execution_contract.py b/core/execution_contract.py new file mode 100644 index 0000000..386cb45 --- /dev/null +++ b/core/execution_contract.py @@ -0,0 +1,197 @@ +""" +Execution contract taxonomy for QuantBT backtest engines. + +The contract object is deliberately small and serializable. It describes what a +backend promises to simulate; hot kernels receive integer codes compiled from +these records in later phases. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +from enum import Enum +from typing import Dict, Mapping + + +class SignalPhase(str, Enum): + BAR_OPEN = "bar_open" + BAR_CLOSE = "bar_close" + + +class FillPhase(str, Enum): + SAME_OPEN = "same_open" + SAME_CLOSE = "same_close" + NEXT_OPEN = "next_open" + NEXT_CLOSE = "next_close" + + +class MarketFillPolicy(str, Enum): + CLOSE = "close" + OPEN = "open" + NEXT_OPEN = "next_open" + + +class StopGapPolicy(str, Enum): + OPEN_WORSE_THAN_TRIGGER = "open_worse_than_trigger" + + +class TakeProfitGapPolicy(str, Enum): + LIMIT_PRICE_CONSERVATIVE = "limit_price_conservative" + OPEN_PRICE_IMPROVEMENT = "open_price_improvement" + + +class IntrabarSameBarPolicy(str, Enum): + CONSERVATIVE = "conservative" + STOP_FIRST = "stop_first" + TP_FIRST = "tp_first" + OHLC_PATH = "ohlc_path" + OLHC_PATH = "olhc_path" + REJECT_AMBIGUOUS = "reject_ambiguous" + LOWER_TIMEFRAME_REQUIRED = "lower_timeframe_required" + + +class TrailingUpdatePhase(str, Enum): + NONE = "none" + NEXT_BAR = "next_bar" + + +class FundingPhase(str, Enum): + POSITION_AT_EVENT = "position_at_event" + POSITION_AT_CLOSE = "position_at_close" + + +class LiquidationPriority(str, Enum): + LIQUIDATION_FIRST_AT_GAP = "liquidation_first_at_gap" + USER_STOP_FIRST = "user_stop_first" + + +class AmbiguityPolicy(str, Enum): + FLAG_AND_CONSERVATIVE = "flag_and_conservative" + REJECT = "reject" + LOWER_TIMEFRAME_REQUIRED = "lower_timeframe_required" + + +@dataclass(frozen=True) +class ExecutionContract: + engine_id: str + signal_phase: SignalPhase + entry_fill_phase: FillPhase + market_fill_policy: MarketFillPolicy + stop_gap_policy: StopGapPolicy = StopGapPolicy.OPEN_WORSE_THAN_TRIGGER + take_profit_gap_policy: TakeProfitGapPolicy = TakeProfitGapPolicy.LIMIT_PRICE_CONSERVATIVE + same_bar_policy: IntrabarSameBarPolicy = IntrabarSameBarPolicy.CONSERVATIVE + trailing_update_phase: TrailingUpdatePhase = TrailingUpdatePhase.NONE + funding_phase: FundingPhase = FundingPhase.POSITION_AT_EVENT + liquidation_priority: LiquidationPriority = LiquidationPriority.LIQUIDATION_FIRST_AT_GAP + close_on_last_bar: bool = True + ambiguity_policy: AmbiguityPolicy = AmbiguityPolicy.FLAG_AND_CONSERVATIVE + strict_data: bool = True + + def __post_init__(self) -> None: + if not self.engine_id: + raise ValueError("engine_id is required") + + @classmethod + def close_target(cls) -> "ExecutionContract": + return cls( + engine_id="close_target_v2", + signal_phase=SignalPhase.BAR_CLOSE, + entry_fill_phase=FillPhase.SAME_CLOSE, + market_fill_policy=MarketFillPolicy.CLOSE, + trailing_update_phase=TrailingUpdatePhase.NONE, + close_on_last_bar=False, + ) + + @classmethod + def next_open(cls) -> "ExecutionContract": + return cls( + engine_id="next_open_v1", + signal_phase=SignalPhase.BAR_CLOSE, + entry_fill_phase=FillPhase.NEXT_OPEN, + market_fill_policy=MarketFillPolicy.NEXT_OPEN, + trailing_update_phase=TrailingUpdatePhase.NONE, + ) + + @classmethod + def intrabar_bracket( + cls, + *, + same_bar_policy: IntrabarSameBarPolicy = IntrabarSameBarPolicy.CONSERVATIVE, + trailing_update_phase: TrailingUpdatePhase = TrailingUpdatePhase.NEXT_BAR, + take_profit_gap_policy: TakeProfitGapPolicy = TakeProfitGapPolicy.LIMIT_PRICE_CONSERVATIVE, + close_on_last_bar: bool = True, + ) -> "ExecutionContract": + return cls( + engine_id="intrabar_bracket_v1", + signal_phase=SignalPhase.BAR_CLOSE, + entry_fill_phase=FillPhase.NEXT_OPEN, + market_fill_policy=MarketFillPolicy.NEXT_OPEN, + same_bar_policy=same_bar_policy, + trailing_update_phase=trailing_update_phase, + take_profit_gap_policy=take_profit_gap_policy, + close_on_last_bar=close_on_last_bar, + ) + + @classmethod + def fill_replay(cls) -> "ExecutionContract": + return cls( + engine_id="fill_replay_v1", + signal_phase=SignalPhase.BAR_CLOSE, + entry_fill_phase=FillPhase.NEXT_OPEN, + market_fill_policy=MarketFillPolicy.NEXT_OPEN, + ) + + @classmethod + def event_lifecycle(cls) -> "ExecutionContract": + return cls( + engine_id="event_lifecycle_v2", + signal_phase=SignalPhase.BAR_CLOSE, + entry_fill_phase=FillPhase.NEXT_OPEN, + market_fill_policy=MarketFillPolicy.NEXT_OPEN, + ) + + def to_metadata(self) -> Dict: + payload = asdict(self) + for key, value in list(payload.items()): + if isinstance(value, Enum): + payload[key] = value.value + return payload + + @classmethod + def from_metadata(cls, metadata: Mapping | "ExecutionContract") -> "ExecutionContract": + if isinstance(metadata, ExecutionContract): + return metadata + payload = dict(metadata or {}) + if not payload: + raise ValueError("execution contract metadata is empty") + return cls( + engine_id=str(payload["engine_id"]), + signal_phase=SignalPhase(payload["signal_phase"]), + entry_fill_phase=FillPhase(payload["entry_fill_phase"]), + market_fill_policy=MarketFillPolicy(payload["market_fill_policy"]), + stop_gap_policy=StopGapPolicy(payload.get("stop_gap_policy", StopGapPolicy.OPEN_WORSE_THAN_TRIGGER.value)), + take_profit_gap_policy=TakeProfitGapPolicy(payload.get("take_profit_gap_policy", TakeProfitGapPolicy.LIMIT_PRICE_CONSERVATIVE.value)), + same_bar_policy=IntrabarSameBarPolicy(payload.get("same_bar_policy", IntrabarSameBarPolicy.CONSERVATIVE.value)), + trailing_update_phase=TrailingUpdatePhase(payload.get("trailing_update_phase", TrailingUpdatePhase.NONE.value)), + funding_phase=FundingPhase(payload.get("funding_phase", FundingPhase.POSITION_AT_EVENT.value)), + liquidation_priority=LiquidationPriority(payload.get("liquidation_priority", LiquidationPriority.LIQUIDATION_FIRST_AT_GAP.value)), + close_on_last_bar=bool(payload.get("close_on_last_bar", True)), + ambiguity_policy=AmbiguityPolicy(payload.get("ambiguity_policy", AmbiguityPolicy.FLAG_AND_CONSERVATIVE.value)), + strict_data=bool(payload.get("strict_data", True)), + ) + + +EXECUTION_CONTRACT_REGISTRY: Dict[str, ExecutionContract] = { + "close_target_v2": ExecutionContract.close_target(), + "next_open_v1": ExecutionContract.next_open(), + "intrabar_bracket_v1": ExecutionContract.intrabar_bracket(), + "fill_replay_v1": ExecutionContract.fill_replay(), + "event_lifecycle_v2": ExecutionContract.event_lifecycle(), +} + + +def get_execution_contract(engine_id: str) -> ExecutionContract: + key = str(engine_id).lower().strip() + if key not in EXECUTION_CONTRACT_REGISTRY: + raise KeyError(f"unknown execution contract {engine_id!r}") + return EXECUTION_CONTRACT_REGISTRY[key] diff --git a/core/intrabar_kernel.py b/core/intrabar_kernel.py new file mode 100644 index 0000000..d6a1273 --- /dev/null +++ b/core/intrabar_kernel.py @@ -0,0 +1,1143 @@ +""" +Fast Numba kernels for Phase 31 intrabar execution contracts. + +The public Python reference oracle remains the readability source of truth. +This module mirrors that state machine with primitive arrays only: no Python +objects are created inside hot loops, and sparse fills are generated only by an +optional deterministic second pass. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Dict, Optional, Sequence + +import numpy as np +import pandas as pd +from numba import njit + +from .execution_contract import ExecutionContract, IntrabarSameBarPolicy, TakeProfitGapPolicy +from .intrabar_reference import IntrabarFill, IntrabarFillReason, IntrabarIntentTape, IntrabarLevelMode, IntrabarSizingMode, _validate_intrabar_contract_supported +from .market_tape import PreparedMarketTape +from .schema import AccountConfig + + +LEVEL_ABSOLUTE_PRICE = 1 +LEVEL_PRICE_DISTANCE = 2 +LEVEL_PERCENT_DISTANCE = 3 + +SAME_BAR_CONSERVATIVE = 1 +SAME_BAR_STOP_FIRST = 2 +SAME_BAR_TP_FIRST = 3 +SAME_BAR_OHLC_PATH = 4 +SAME_BAR_OLHC_PATH = 5 +SAME_BAR_REJECT_AMBIGUOUS = 6 + +TP_LIMIT_CONSERVATIVE = 1 +TP_OPEN_PRICE_IMPROVEMENT = 2 + +FILL_ENTRY = 1 +FILL_TECHNICAL_EXIT = 2 +FILL_REVERSAL_EXIT = 3 +FILL_REVERSAL_ENTRY = 4 +FILL_STOP_LOSS = 5 +FILL_TAKE_PROFIT = 6 +FILL_LIQUIDATION = 7 +FILL_FINAL_CLOSE = 8 + +FLAG_ENTRY_FILLED = 1 << 0 +FLAG_EXIT_FILLED = 1 << 1 +FLAG_STOP_FILLED = 1 << 2 +FLAG_TP_FILLED = 1 << 3 +FLAG_TECH_EXIT = 1 << 4 +FLAG_REVERSAL = 1 << 5 +FLAG_AMBIGUOUS = 1 << 6 +FLAG_FUNDING = 1 << 7 +FLAG_LIQUIDATION = 1 << 8 +FLAG_REJECTED = 1 << 9 +FLAG_ENTRY_SUPPRESSED = 1 << 10 + +SIZING_UNITS = 1 +SIZING_FIXED_NOTIONAL = 2 +SIZING_PCT_EQUITY = 3 +SIZING_RISK_PER_TRADE = 4 + +BAR_TS_CLOSE = 1 +BAR_TS_OPEN = 2 + + +@dataclass(frozen=True) +class NativeIntrabarKernelResult: + equity: pd.Series + position: pd.Series + average_entry: pd.Series + active_stop: pd.Series + active_take_profit: pd.Series + fees: pd.Series + funding: pd.Series + event_flags: pd.Series + initial_margin: pd.Series + maintenance_margin: pd.Series + fills: tuple[IntrabarFill, ...] = () + fills_report: pd.DataFrame = field(default_factory=pd.DataFrame) + ambiguity_count: int = 0 + rejected_count: int = 0 + fill_count: int = 0 + liquidated: bool = False + liquidation_bar: int = -1 + report_level: str = "standard" + metadata: Dict = field(default_factory=dict) + + +@dataclass(frozen=True) +class FillReplayTape: + bar_index: np.ndarray + sequence: np.ndarray + side: np.ndarray + qty: np.ndarray + price: np.ndarray + fee: np.ndarray + reason: np.ndarray + + @classmethod + def from_frame(cls, frame: pd.DataFrame, *, fee_rate: float = 0.0, contract_size: float = 1.0) -> "FillReplayTape": + required = {"bar_index", "side", "qty", "price"} + missing = sorted(required - set(frame.columns)) + if missing: + raise ValueError(f"fill replay frame is missing columns {missing}") + sequence = frame["sequence"] if "sequence" in frame else pd.Series(np.arange(len(frame)), index=frame.index) + price = pd.to_numeric(frame["price"], errors="raise").to_numpy(dtype=np.float64) + qty = pd.to_numeric(frame["qty"], errors="raise").to_numpy(dtype=np.float64) + if "fee" in frame: + fee = pd.to_numeric(frame["fee"], errors="raise").to_numpy(dtype=np.float64) + else: + fee = np.abs(qty) * price * float(contract_size) * float(fee_rate) + reason = _reason_series_to_codes(frame["reason"]) if "reason" in frame else np.zeros(len(frame), dtype=np.int16) + return cls( + bar_index=np.ascontiguousarray(pd.to_numeric(frame["bar_index"], errors="raise").to_numpy(dtype=np.int64)), + sequence=np.ascontiguousarray(pd.to_numeric(sequence, errors="raise").to_numpy(dtype=np.int64)), + side=np.ascontiguousarray(np.sign(pd.to_numeric(frame["side"], errors="raise").to_numpy(dtype=np.float64)).astype(np.int8)), + qty=np.ascontiguousarray(qty, dtype=np.float64), + price=np.ascontiguousarray(price, dtype=np.float64), + fee=np.ascontiguousarray(fee, dtype=np.float64), + reason=np.ascontiguousarray(reason, dtype=np.int16), + ) + + +@dataclass(frozen=True) +class NativeFillReplayResult: + equity: pd.Series + position: pd.Series + fees: pd.Series + event_flags: pd.Series + fill_count: int + metadata: Dict = field(default_factory=dict) + + +def run_intrabar_kernel( + *, + tape: PreparedMarketTape, + intent: IntrabarIntentTape, + account: AccountConfig, + contract: Optional[ExecutionContract] = None, + fee_rate: float = 0.0, + slippage_rate: float = 0.0, + contract_size: float = 1.0, + sizing_mode: IntrabarSizingMode | str = IntrabarSizingMode.UNITS, + fixed_notional: float = 0.0, + equity_fraction: float = 0.0, + risk_fraction: float = 0.0, + qty_step: float = 0.0, + min_qty: float = 0.0, + min_notional: float = 0.0, + tick_size: float = 0.0, + report_level: str = "standard", +) -> NativeIntrabarKernelResult: + """ + Run the fast single-symbol `intrabar_bracket_v1` Numba kernel. + + `report_level="audit"` triggers the second pass and materializes sparse + fills. `minimal` and `standard` keep fill accounting as counters/flags only. + """ + if tape.n_symbols != 1: + raise NotImplementedError("intrabar fast kernel v1 supports exactly one symbol") + if len(intent.entry_side) != tape.n_bars: + raise ValueError("intent length must match market tape length") + if fee_rate < 0.0 or slippage_rate < 0.0: + raise ValueError("fee_rate and slippage_rate must be >= 0") + level = _normalize_report_level(report_level) + contract = contract or ExecutionContract.intrabar_bracket() + if contract.engine_id != "intrabar_bracket_v1": + raise ValueError("run_intrabar_kernel requires intrabar_bracket_v1 contract") + _validate_intrabar_contract_supported(contract) + if contract.same_bar_policy is IntrabarSameBarPolicy.REJECT_AMBIGUOUS: + raise NotImplementedError("fast intrabar kernel v1 does not support REJECT_AMBIGUOUS; use the reference oracle for debug rejection") + sizing_mode_value = IntrabarSizingMode(sizing_mode) + + arrays = _run_intrabar_pass( + record_fills=False, + fill_capacity=1, + tape=tape, + intent=intent, + account=account, + contract=contract, + fee_rate=fee_rate, + slippage_rate=slippage_rate, + contract_size=contract_size, + sizing_mode=sizing_mode_value, + fixed_notional=fixed_notional, + equity_fraction=equity_fraction, + risk_fraction=risk_fraction, + qty_step=qty_step, + min_qty=min_qty, + min_notional=min_notional, + tick_size=tick_size, + ) + ( + equity, + position, + avg_entry, + active_stop, + active_tp, + fees, + funding, + flags, + initial_margin, + maintenance_margin, + fill_count, + ambiguity_count, + rejected_count, + liquidated, + liquidation_bar, + _fill_bar, + _fill_seq, + _fill_side, + _fill_qty, + _fill_price, + _fill_fee, + _fill_reason, + ) = arrays + + fills: tuple[IntrabarFill, ...] = () + fills_report = pd.DataFrame() + if level == "audit": + audit = _run_intrabar_pass( + record_fills=True, + fill_capacity=int(fill_count), + tape=tape, + intent=intent, + account=account, + contract=contract, + fee_rate=fee_rate, + slippage_rate=slippage_rate, + contract_size=contract_size, + sizing_mode=sizing_mode_value, + fixed_notional=fixed_notional, + equity_fraction=equity_fraction, + risk_fraction=risk_fraction, + qty_step=qty_step, + min_qty=min_qty, + min_notional=min_notional, + tick_size=tick_size, + ) + _assert_intrabar_audit_parity(arrays, audit) + fills = _materialize_intrabar_fills( + timestamps_ns=tape.timestamps_ns, + fill_bar=audit[15], + fill_seq=audit[16], + fill_side=audit[17], + fill_qty=audit[18], + fill_price=audit[19], + fill_fee=audit[20], + fill_reason=audit[21], + fill_count=int(fill_count), + ) + fills_report = _fills_to_report(fills) + + idx = pd.DatetimeIndex(pd.to_datetime(tape.timestamps_ns, utc=True)) + symbol = tape.symbols[0] + metadata = { + "engine": "intrabar_bracket_v1", + "engine_id": "intrabar_bracket_v1", + "backend": "native_intrabar", + "backend_alias": "native_intrabar", + "kernel_version": "intrabar_numba_v1", + "execution_contract": contract.to_metadata(), + "data_signature": tape.signature, + "validation_certificate": tape.validation_certificate.__dict__.copy(), + "report_level": level, + "two_pass_audit": level == "audit", + "fill_count": int(fill_count), + "ambiguity_count": int(ambiguity_count), + "rejected_count": int(rejected_count), + "liquidated": bool(liquidated), + "liquidation_bar": int(liquidation_bar), + "funding_timing_certified": True, + "funding_event_alignment": "exact_bar_timestamp", + "bar_timestamp_semantics": tape.bar_timestamp_semantics, + "funding_event_price_reference": "open" if tape.bar_timestamp_semantics == "open" else "close", + "sizing_mode": sizing_mode_value.value, + "sizing": { + "fixed_notional": float(fixed_notional), + "equity_fraction": float(equity_fraction), + "risk_fraction": float(risk_fraction), + }, + "quantity_constraints": { + "qty_step": float(qty_step), + "min_qty": float(min_qty), + "min_notional": float(min_notional), + "tick_size": float(tick_size), + }, + } + return NativeIntrabarKernelResult( + equity=pd.Series(equity, index=idx, name="equity"), + position=pd.Series(position, index=idx, name=f"Position_{symbol}"), + average_entry=pd.Series(avg_entry, index=idx, name="average_entry"), + active_stop=pd.Series(active_stop, index=idx, name="active_stop"), + active_take_profit=pd.Series(active_tp, index=idx, name="active_take_profit"), + fees=pd.Series(fees, index=idx, name="fees"), + funding=pd.Series(funding, index=idx, name="funding"), + event_flags=pd.Series(flags, index=idx, name="event_flags"), + initial_margin=pd.Series(initial_margin, index=idx, name="initial_margin"), + maintenance_margin=pd.Series(maintenance_margin, index=idx, name="maintenance_margin"), + fills=fills, + fills_report=fills_report, + ambiguity_count=int(ambiguity_count), + rejected_count=int(rejected_count), + fill_count=int(fill_count), + liquidated=bool(liquidated), + liquidation_bar=int(liquidation_bar), + report_level=level, + metadata=metadata, + ) + + +def run_fill_replay_kernel( + *, + tape: PreparedMarketTape, + fill_tape: FillReplayTape, + account: AccountConfig, + contract_size: float = 1.0, +) -> NativeFillReplayResult: + """Replay explicit fills through fast accounting without certifying signal generation.""" + if tape.n_symbols != 1: + raise NotImplementedError("fill replay v1 supports exactly one symbol") + _validate_fill_replay_tape(fill_tape, tape.n_bars) + equity, position, fees, flags = _engine_fill_replay_v1( + tape.opens[:, 0], + tape.closes[:, 0], + fill_tape.bar_index, + fill_tape.sequence, + fill_tape.side, + fill_tape.qty, + fill_tape.price, + fill_tape.fee, + account.initial_capital, + float(contract_size), + ) + idx = pd.DatetimeIndex(pd.to_datetime(tape.timestamps_ns, utc=True)) + metadata = { + "engine": "fill_replay_v1", + "engine_id": "fill_replay_v1", + "backend": "native_intrabar", + "accounting_certified": True, + "price_accounting_certified": True, + "fee_accounting_certified": True, + "funding_certified": False, + "margin_certified": False, + "liquidation_certified": False, + "execution_generation_certified": False, + "causality_certified": False, + "data_signature": tape.signature, + "fill_count": int(len(fill_tape.bar_index)), + } + return NativeFillReplayResult( + equity=pd.Series(equity, index=idx, name="equity"), + position=pd.Series(position, index=idx, name=f"Position_{tape.symbols[0]}"), + fees=pd.Series(fees, index=idx, name="fees"), + event_flags=pd.Series(flags, index=idx, name="event_flags"), + fill_count=int(len(fill_tape.bar_index)), + metadata=metadata, + ) + + +def _run_intrabar_pass( + *, + record_fills: bool, + fill_capacity: int, + tape, + intent, + account, + contract, + fee_rate, + slippage_rate, + contract_size, + sizing_mode, + fixed_notional, + equity_fraction, + risk_fraction, + qty_step, + min_qty, + min_notional, + tick_size, +): + stop_value = _optional_float_array(intent.stop_value, tape.n_bars) + tp_value = _optional_float_array(intent.take_profit_value, tape.n_bars) + trailing_value = _optional_float_array(intent.trailing_value, tape.n_bars) + exit_long = _optional_bool_array(intent.exit_long if intent.exit_long is not None else intent.technical_exit, tape.n_bars) + exit_short = _optional_bool_array(intent.exit_short if intent.exit_short is not None else intent.technical_exit, tape.n_bars) + fill_bar = np.zeros(max(1, int(fill_capacity)), dtype=np.int64) + fill_seq = np.zeros(max(1, int(fill_capacity)), dtype=np.int16) + fill_side = np.zeros(max(1, int(fill_capacity)), dtype=np.int8) + fill_qty = np.zeros(max(1, int(fill_capacity)), dtype=np.float64) + fill_price = np.zeros(max(1, int(fill_capacity)), dtype=np.float64) + fill_fee = np.zeros(max(1, int(fill_capacity)), dtype=np.float64) + fill_reason = np.zeros(max(1, int(fill_capacity)), dtype=np.int16) + return _engine_intrabar_bracket_v1( + tape.opens[:, 0], + tape.highs[:, 0], + tape.lows[:, 0], + tape.closes[:, 0], + np.ascontiguousarray(intent.entry_side, dtype=np.int8), + np.ascontiguousarray(intent.entry_size, dtype=np.float64), + stop_value, + tp_value, + trailing_value, + exit_long, + exit_short, + tape.funding_rates[:, 0], + tape.funding_event_mask, + _bar_timestamp_semantics_code(tape.bar_timestamp_semantics), + float(account.initial_capital), + float(account.leverage), + float(account.maintenance_ratio), + float(account.margin_buffer), + float(contract_size), + float(fee_rate), + float(slippage_rate), + _sizing_mode_code(sizing_mode), + float(fixed_notional), + float(equity_fraction), + float(risk_fraction), + float(qty_step), + float(min_qty), + float(min_notional), + float(tick_size), + _level_mode_code(intent.level_mode), + _same_bar_policy_code(contract.same_bar_policy), + _tp_policy_code(contract.take_profit_gap_policy), + bool(contract.close_on_last_bar), + bool(record_fills), + fill_bar, + fill_seq, + fill_side, + fill_qty, + fill_price, + fill_fee, + fill_reason, + ) + + +@njit(cache=True, nogil=True) +def _engine_intrabar_bracket_v1( + opens, + highs, + lows, + closes, + entry_side, + entry_size, + stop_value, + tp_value, + trailing_value, + exit_long, + exit_short, + funding_rates, + funding_mask, + bar_timestamp_semantics, + initial_capital, + leverage, + maintenance_ratio, + margin_buffer, + contract_size, + fee_rate, + slippage_rate, + sizing_mode, + fixed_notional, + equity_fraction, + risk_fraction, + qty_step, + min_qty, + min_notional, + tick_size, + level_mode, + same_bar_policy, + tp_gap_policy, + close_on_last_bar, + record_fills, + fill_bar, + fill_seq, + fill_side, + fill_qty, + fill_price, + fill_fee, + fill_reason, +): + n = closes.shape[0] + equity_arr = np.zeros(n, dtype=np.float64) + pos_arr = np.zeros(n, dtype=np.float64) + avg_arr = np.zeros(n, dtype=np.float64) + stop_arr = np.zeros(n, dtype=np.float64) + tp_arr = np.zeros(n, dtype=np.float64) + fee_arr = np.zeros(n, dtype=np.float64) + funding_arr = np.zeros(n, dtype=np.float64) + flags_arr = np.zeros(n, dtype=np.uint16) + init_margin = np.zeros(n, dtype=np.float64) + maint_margin = np.zeros(n, dtype=np.float64) + + equity = initial_capital + position = 0.0 + avg_entry = 0.0 + active_stop = np.nan + active_tp = np.nan + fill_count = 0 + ambiguity_count = 0 + rejected_count = 0 + liquidated = False + liquidation_bar = -1 + + equity_arr[0] = equity + for t in range(1, n): + if liquidated: + equity_arr[t] = 0.0 + continue + + seq = 0 + open_ref = opens[t] + close_ref = closes[t] + last_ref = open_ref + + if position != 0.0: + equity += position * (open_ref - closes[t - 1]) * contract_size + + if position != 0.0 and _maintenance_breached_numba(equity, position, open_ref, contract_size, maintenance_ratio): + side = -1 if position > 0.0 else 1 + price = _market_price_numba(open_ref, side, slippage_rate, tick_size) + qty = abs(position) + fee = qty * price * contract_size * fee_rate + equity += position * (price - open_ref) * contract_size - fee + fee_arr[t] += fee + fill_count = _record_fill_numba(record_fills, fill_count, t, seq, side, qty, price, fee, FILL_LIQUIDATION, fill_bar, fill_seq, fill_side, fill_qty, fill_price, fill_fee, fill_reason) + flags_arr[t] |= FLAG_EXIT_FILLED | FLAG_LIQUIDATION + liquidated = True + liquidation_bar = t + equity = 0.0 + equity_arr[t] = 0.0 + continue + + if bar_timestamp_semantics == BAR_TS_OPEN and position != 0.0 and funding_mask[t]: + funding_cost = position * open_ref * contract_size * funding_rates[t] + equity -= funding_cost + funding_arr[t] = funding_cost + flags_arr[t] |= FLAG_FUNDING + + pending_side = entry_side[t - 1] + pending_size = entry_size[t - 1] + pending_exit = (position > 0.0 and exit_long[t - 1]) or (position < 0.0 and exit_short[t - 1]) + exit_same_side_conflict = pending_exit and pending_side != 0 and position != 0.0 and _sign_numba(position) == pending_side + + if position != 0.0 and (pending_exit or (pending_side != 0 and _sign_numba(position) != pending_side)): + reason = FILL_REVERSAL_EXIT if pending_side != 0 and _sign_numba(position) != pending_side else FILL_TECHNICAL_EXIT + side = -1 if position > 0.0 else 1 + price = _market_price_numba(open_ref, side, slippage_rate, tick_size) + qty = abs(position) + fee = qty * price * contract_size * fee_rate + equity += position * (price - open_ref) * contract_size - fee + fee_arr[t] += fee + fill_count = _record_fill_numba(record_fills, fill_count, t, seq, side, qty, price, fee, reason, fill_bar, fill_seq, fill_side, fill_qty, fill_price, fill_fee, fill_reason) + seq += 1 + flags_arr[t] |= FLAG_EXIT_FILLED + if reason == FILL_TECHNICAL_EXIT: + flags_arr[t] |= FLAG_TECH_EXIT + else: + flags_arr[t] |= FLAG_REVERSAL + position = 0.0 + avg_entry = 0.0 + active_stop = np.nan + active_tp = np.nan + + if pending_side != 0 and pending_size > 0.0 and position == 0.0: + side = 1 if pending_side > 0 else -1 + price = _market_price_numba(open_ref, side, slippage_rate, tick_size) + if exit_same_side_conflict: + qty = 0.0 + else: + qty = _compile_entry_quantity_numba( + pending_size, + price, + equity, + contract_size, + sizing_mode, + fixed_notional, + equity_fraction, + risk_fraction, + stop_value[t - 1], + level_mode, + side, + tick_size, + ) + qty = abs(_quantize_signed_quantity_numba(qty, price, contract_size, qty_step, min_qty, min_notional)) + if exit_same_side_conflict: + flags_arr[t] |= FLAG_ENTRY_SUPPRESSED + equity_arr[t] = equity + pos_arr[t] = position + avg_arr[t] = avg_entry + stop_arr[t] = 0.0 if not np.isfinite(active_stop) else active_stop + tp_arr[t] = 0.0 if not np.isfinite(active_tp) else active_tp + continue + if qty <= 0.0: + flags_arr[t] |= FLAG_REJECTED + rejected_count += 1 + equity_arr[t] = equity + pos_arr[t] = position + avg_arr[t] = avg_entry + stop_arr[t] = 0.0 if not np.isfinite(active_stop) else active_stop + tp_arr[t] = 0.0 if not np.isfinite(active_tp) else active_tp + continue + if not _has_initial_margin_numba(equity, qty, price, contract_size, leverage, margin_buffer): + flags_arr[t] |= FLAG_REJECTED + rejected_count += 1 + equity_arr[t] = equity + pos_arr[t] = position + avg_arr[t] = avg_entry + stop_arr[t] = 0.0 if not np.isfinite(active_stop) else active_stop + tp_arr[t] = 0.0 if not np.isfinite(active_tp) else active_tp + continue + fee = qty * price * contract_size * fee_rate + equity -= fee + fee_arr[t] += fee + position = qty * side + avg_entry = price + last_ref = price + active_stop, active_tp = _initial_bracket_numba(stop_value[t - 1], tp_value[t - 1], trailing_value[t - 1], side, price, level_mode, tick_size) + reason = FILL_REVERSAL_ENTRY if (flags_arr[t] & FLAG_REVERSAL) != 0 else FILL_ENTRY + fill_count = _record_fill_numba(record_fills, fill_count, t, seq, side, qty, price, fee, reason, fill_bar, fill_seq, fill_side, fill_qty, fill_price, fill_fee, fill_reason) + seq += 1 + flags_arr[t] |= FLAG_ENTRY_FILLED + + if position != 0.0: + exit_side, exit_price, exit_reason, ambiguous = _resolve_intrabar_exit_numba( + 1 if position > 0.0 else -1, + open_ref, + highs[t], + lows[t], + active_stop, + active_tp, + same_bar_policy, + tp_gap_policy, + slippage_rate, + tick_size, + ) + if exit_reason != 0: + if ambiguous: + flags_arr[t] |= FLAG_AMBIGUOUS + ambiguity_count += 1 + qty = abs(position) + fee = qty * exit_price * contract_size * fee_rate + equity += position * (exit_price - last_ref) * contract_size - fee + fee_arr[t] += fee + fill_count = _record_fill_numba(record_fills, fill_count, t, seq, exit_side, qty, exit_price, fee, exit_reason, fill_bar, fill_seq, fill_side, fill_qty, fill_price, fill_fee, fill_reason) + seq += 1 + flags_arr[t] |= FLAG_EXIT_FILLED + if exit_reason == FILL_STOP_LOSS: + flags_arr[t] |= FLAG_STOP_FILLED + else: + flags_arr[t] |= FLAG_TP_FILLED + position = 0.0 + avg_entry = 0.0 + active_stop = np.nan + active_tp = np.nan + + if position != 0.0: + if _maintenance_breached_worst_numba(equity, position, last_ref, highs[t], lows[t], contract_size, maintenance_ratio): + side = -1 if position > 0.0 else 1 + worst = lows[t] if position > 0.0 else highs[t] + price = _market_price_numba(worst, side, slippage_rate, tick_size) + qty = abs(position) + fee = qty * price * contract_size * fee_rate + equity += position * (price - last_ref) * contract_size - fee + fee_arr[t] += fee + fill_count = _record_fill_numba(record_fills, fill_count, t, seq, side, qty, price, fee, FILL_LIQUIDATION, fill_bar, fill_seq, fill_side, fill_qty, fill_price, fill_fee, fill_reason) + flags_arr[t] |= FLAG_EXIT_FILLED | FLAG_LIQUIDATION + liquidated = True + liquidation_bar = t + equity = 0.0 + position = 0.0 + avg_entry = 0.0 + active_stop = np.nan + active_tp = np.nan + else: + equity += position * (close_ref - last_ref) * contract_size + active_stop = _update_trailing_numba(trailing_value[t], position, close_ref, active_stop, level_mode, tick_size) + + if liquidated: + equity_arr[t] = 0.0 + pos_arr[t] = 0.0 + avg_arr[t] = 0.0 + stop_arr[t] = 0.0 + tp_arr[t] = 0.0 + continue + + if bar_timestamp_semantics == BAR_TS_CLOSE and position != 0.0 and funding_mask[t]: + funding_cost = position * close_ref * contract_size * funding_rates[t] + equity -= funding_cost + funding_arr[t] = funding_cost + flags_arr[t] |= FLAG_FUNDING + + equity_arr[t] = equity + pos_arr[t] = position + avg_arr[t] = avg_entry + stop_arr[t] = 0.0 if not np.isfinite(active_stop) else active_stop + tp_arr[t] = 0.0 if not np.isfinite(active_tp) else active_tp + init_margin[t] = abs(position) * close_ref * contract_size / leverage + maint_margin[t] = abs(position) * close_ref * contract_size * maintenance_ratio + + if close_on_last_bar and position != 0.0 and not liquidated: + t = n - 1 + side = -1 if position > 0.0 else 1 + price = _market_price_numba(closes[t], side, slippage_rate, tick_size) + qty = abs(position) + fee = qty * price * contract_size * fee_rate + equity += position * (price - closes[t]) * contract_size - fee + fee_arr[t] += fee + fill_count = _record_fill_numba(record_fills, fill_count, t, 99, side, qty, price, fee, FILL_FINAL_CLOSE, fill_bar, fill_seq, fill_side, fill_qty, fill_price, fill_fee, fill_reason) + position = 0.0 + equity_arr[t] = equity + pos_arr[t] = 0.0 + avg_arr[t] = 0.0 + stop_arr[t] = 0.0 + tp_arr[t] = 0.0 + init_margin[t] = 0.0 + maint_margin[t] = 0.0 + + return ( + equity_arr, + pos_arr, + avg_arr, + stop_arr, + tp_arr, + fee_arr, + funding_arr, + flags_arr, + init_margin, + maint_margin, + fill_count, + ambiguity_count, + rejected_count, + liquidated, + liquidation_bar, + fill_bar, + fill_seq, + fill_side, + fill_qty, + fill_price, + fill_fee, + fill_reason, + ) + + +@njit(cache=True, nogil=True) +def _engine_fill_replay_v1(opens, closes, fill_bar, fill_seq, fill_side, fill_qty, fill_price, fill_fee, initial_capital, contract_size): + n = closes.shape[0] + equity_arr = np.zeros(n, dtype=np.float64) + pos_arr = np.zeros(n, dtype=np.float64) + fee_arr = np.zeros(n, dtype=np.float64) + flags_arr = np.zeros(n, dtype=np.uint16) + equity = initial_capital + position = 0.0 + ptr = 0 + n_fills = fill_bar.shape[0] + prev_close = opens[0] + for t in range(n): + current_ref = opens[t] + if t > 0 and position != 0.0: + equity += position * (opens[t] - prev_close) * contract_size + while ptr < n_fills and fill_bar[ptr] == t: + price = fill_price[ptr] + side = fill_side[ptr] + qty = fill_qty[ptr] + fee = fill_fee[ptr] + if position != 0.0: + equity += position * (price - current_ref) * contract_size + equity -= fee + fee_arr[t] += fee + position += side * qty + current_ref = price + flags_arr[t] |= FLAG_ENTRY_FILLED if side > 0 else FLAG_EXIT_FILLED + ptr += 1 + if position != 0.0: + equity += position * (closes[t] - current_ref) * contract_size + equity_arr[t] = equity + pos_arr[t] = position + prev_close = closes[t] + return equity_arr, pos_arr, fee_arr, flags_arr + + +@njit(cache=True, nogil=True) +def _market_price_numba(price, side, slippage_rate, tick_size): + raw = price * (1.0 + slippage_rate if side > 0 else 1.0 - slippage_rate) + return _quantize_price_numba(raw, side, tick_size) + + +@njit(cache=True, nogil=True) +def _sign_numba(value): + if value > 0.0: + return 1 + if value < 0.0: + return -1 + return 0 + + +@njit(cache=True, nogil=True) +def _has_initial_margin_numba(equity, qty, price, contract_size, leverage, margin_buffer): + required = abs(qty) * price * contract_size / leverage + return equity >= required * (1.0 + margin_buffer) + + +@njit(cache=True, nogil=True) +def _maintenance_breached_numba(equity, position, price, contract_size, maintenance_ratio): + maintenance = abs(position) * price * contract_size * maintenance_ratio + return maintenance > 0.0 and equity <= maintenance + + +@njit(cache=True, nogil=True) +def _maintenance_breached_worst_numba(equity, position, reference_price, high, low, contract_size, maintenance_ratio): + worst = low if position > 0.0 else high + worst_equity = equity + position * (worst - reference_price) * contract_size + maintenance = abs(position) * worst * contract_size * maintenance_ratio + return maintenance > 0.0 and worst_equity <= maintenance + + +@njit(cache=True, nogil=True) +def _initial_bracket_numba(stop_value, tp_value, trailing_value, side, fill_price, level_mode, tick_size): + stop = np.nan + tp = np.nan + if np.isfinite(stop_value) and stop_value > 0.0: + stop = _level_price_numba(fill_price, side, stop_value, level_mode, True, tick_size) + if np.isfinite(tp_value) and tp_value > 0.0: + tp = _level_price_numba(fill_price, side, tp_value, level_mode, False, tick_size) + if np.isfinite(trailing_value) and trailing_value > 0.0: + trailing_stop = _level_price_numba(fill_price, side, trailing_value, level_mode, True, tick_size) + if not np.isfinite(stop): + stop = trailing_stop + elif side > 0: + stop = max(stop, trailing_stop) + else: + stop = min(stop, trailing_stop) + return stop, tp + + +@njit(cache=True, nogil=True) +def _level_price_numba(price, side, value, level_mode, is_stop, tick_size): + direction = -1.0 if (side > 0 and is_stop) or (side < 0 and not is_stop) else 1.0 + if level_mode == LEVEL_ABSOLUTE_PRICE: + return _quantize_price_numba(value, -side, tick_size) + if level_mode == LEVEL_PRICE_DISTANCE: + return _quantize_price_numba(price + direction * value, -side, tick_size) + return _quantize_price_numba(price * (1.0 + direction * value), -side, tick_size) + + +@njit(cache=True, nogil=True) +def _resolve_intrabar_exit_numba(side, open_price, high, low, stop_price, tp_price, same_bar_policy, tp_gap_policy, slippage_rate, tick_size): + has_stop = np.isfinite(stop_price) and stop_price > 0.0 + has_tp = np.isfinite(tp_price) and tp_price > 0.0 + if side > 0: + stop_hit = has_stop and low <= stop_price + tp_hit = has_tp and high >= tp_price + stop_gap = has_stop and open_price <= stop_price + tp_gap = has_tp and open_price >= tp_price + exit_side = -1 + else: + stop_hit = has_stop and high >= stop_price + tp_hit = has_tp and low <= tp_price + stop_gap = has_stop and open_price >= stop_price + tp_gap = has_tp and open_price <= tp_price + exit_side = 1 + if not stop_hit and not tp_hit: + return 0, 0.0, 0, False + ambiguous = stop_hit and tp_hit + if ambiguous and same_bar_policy == SAME_BAR_REJECT_AMBIGUOUS: + return 0, 0.0, -1, True + stop_first = ( + same_bar_policy == SAME_BAR_CONSERVATIVE + or same_bar_policy == SAME_BAR_STOP_FIRST + or (side > 0 and same_bar_policy == SAME_BAR_OLHC_PATH) + or (side < 0 and same_bar_policy == SAME_BAR_OHLC_PATH) + ) + if stop_hit and ((not tp_hit) or stop_first): + price = open_price if stop_gap else stop_price + return exit_side, _market_price_numba(price, exit_side, slippage_rate, tick_size), FILL_STOP_LOSS, ambiguous + if tp_hit: + price = open_price if tp_gap and tp_gap_policy == TP_OPEN_PRICE_IMPROVEMENT else tp_price + return exit_side, _quantize_price_numba(price, exit_side, tick_size), FILL_TAKE_PROFIT, ambiguous + return 0, 0.0, 0, False + + +@njit(cache=True, nogil=True) +def _update_trailing_numba(trailing_value, position, close_price, current_stop, level_mode, tick_size): + if not np.isfinite(trailing_value) or trailing_value <= 0.0: + return current_stop + side = 1 if position > 0.0 else -1 + candidate = _level_price_numba(close_price, side, trailing_value, level_mode, True, tick_size) + if not np.isfinite(current_stop): + return candidate + return max(current_stop, candidate) if side > 0 else min(current_stop, candidate) + + +@njit(cache=True, nogil=True) +def _quantize_price_numba(price, side, tick_size): + if tick_size <= 0.0 or not np.isfinite(price): + return price + if side > 0: + return np.ceil((price / tick_size) - 1e-12) * tick_size + return np.floor((price / tick_size) + 1e-12) * tick_size + + +@njit(cache=True, nogil=True) +def _compile_entry_quantity_numba(size_weight, fill_price, equity, contract_size, sizing_mode, fixed_notional, equity_fraction, risk_fraction, stop_value, level_mode, side, tick_size): + weight = abs(size_weight) + if sizing_mode == SIZING_UNITS: + return weight + if fill_price <= 0.0 or contract_size <= 0.0: + return 0.0 + if sizing_mode == SIZING_FIXED_NOTIONAL: + return fixed_notional * weight / (fill_price * contract_size) + if sizing_mode == SIZING_PCT_EQUITY: + return equity * equity_fraction * weight / (fill_price * contract_size) + if sizing_mode == SIZING_RISK_PER_TRADE: + if not np.isfinite(stop_value) or stop_value <= 0.0: + return 0.0 + stop_price = _level_price_numba(fill_price, side, stop_value, level_mode, True, tick_size) + stop_distance = abs(fill_price - stop_price) + if stop_distance <= 0.0: + return 0.0 + return equity * risk_fraction * weight / (stop_distance * contract_size) + return 0.0 + + +@njit(cache=True, nogil=True) +def _quantize_signed_quantity_numba(qty, price, contract_size, qty_step, min_qty, min_notional): + if qty == 0.0: + return 0.0 + sign = 1.0 if qty > 0.0 else -1.0 + abs_q = abs(qty) + if qty_step > 0.0: + abs_q = np.floor((abs_q / qty_step) + 1e-12) * qty_step + if abs_q <= 0.0: + return 0.0 + if min_qty > 0.0 and abs_q + 1e-12 < min_qty: + return 0.0 + if min_notional > 0.0 and abs_q * price * contract_size + 1e-12 < min_notional: + return 0.0 + return sign * abs_q + + +@njit(cache=True, nogil=True) +def _record_fill_numba(record, count, bar, seq, side, qty, price, fee, reason, fill_bar, fill_seq, fill_side, fill_qty, fill_price, fill_fee, fill_reason): + if record and count < fill_bar.shape[0]: + fill_bar[count] = bar + fill_seq[count] = seq + fill_side[count] = side + fill_qty[count] = qty + fill_price[count] = price + fill_fee[count] = fee + fill_reason[count] = reason + return count + 1 + + +def _optional_float_array(value, n: int) -> np.ndarray: + if value is None: + return np.full(n, np.nan, dtype=np.float64) + return np.ascontiguousarray(value, dtype=np.float64) + + +def _optional_bool_array(value, n: int) -> np.ndarray: + if value is None: + return np.zeros(n, dtype=np.bool_) + return np.ascontiguousarray(value, dtype=np.bool_) + + +def _level_mode_code(mode) -> int: + value = mode.value if hasattr(mode, "value") else str(mode) + if value == IntrabarLevelMode.ABSOLUTE_PRICE.value: + return LEVEL_ABSOLUTE_PRICE + if value == IntrabarLevelMode.PRICE_DISTANCE.value: + return LEVEL_PRICE_DISTANCE + if value == IntrabarLevelMode.PERCENT_DISTANCE.value: + return LEVEL_PERCENT_DISTANCE + raise NotImplementedError(f"unsupported intrabar level mode={mode!r}") + + +def _sizing_mode_code(mode) -> int: + value = mode.value if hasattr(mode, "value") else str(mode) + mapping = { + IntrabarSizingMode.UNITS.value: SIZING_UNITS, + IntrabarSizingMode.FIXED_NOTIONAL.value: SIZING_FIXED_NOTIONAL, + IntrabarSizingMode.PCT_EQUITY.value: SIZING_PCT_EQUITY, + IntrabarSizingMode.RISK_PER_TRADE.value: SIZING_RISK_PER_TRADE, + } + if value not in mapping: + raise NotImplementedError(f"unsupported intrabar sizing_mode={mode!r}") + return mapping[value] + + +def _bar_timestamp_semantics_code(value: str) -> int: + semantics = str(value or "close").lower().strip() + if semantics == "close": + return BAR_TS_CLOSE + if semantics == "open": + return BAR_TS_OPEN + raise ValueError("bar_timestamp_semantics must be 'open' or 'close'") + + +def _same_bar_policy_code(policy) -> int: + value = policy.value if hasattr(policy, "value") else str(policy) + mapping = { + IntrabarSameBarPolicy.CONSERVATIVE.value: SAME_BAR_CONSERVATIVE, + IntrabarSameBarPolicy.STOP_FIRST.value: SAME_BAR_STOP_FIRST, + IntrabarSameBarPolicy.TP_FIRST.value: SAME_BAR_TP_FIRST, + IntrabarSameBarPolicy.OHLC_PATH.value: SAME_BAR_OHLC_PATH, + IntrabarSameBarPolicy.OLHC_PATH.value: SAME_BAR_OLHC_PATH, + IntrabarSameBarPolicy.REJECT_AMBIGUOUS.value: SAME_BAR_REJECT_AMBIGUOUS, + } + if value not in mapping: + raise NotImplementedError(f"unsupported same-bar policy={policy!r}") + return mapping[value] + + +def _tp_policy_code(policy) -> int: + value = policy.value if hasattr(policy, "value") else str(policy) + if value == TakeProfitGapPolicy.LIMIT_PRICE_CONSERVATIVE.value: + return TP_LIMIT_CONSERVATIVE + if value == TakeProfitGapPolicy.OPEN_PRICE_IMPROVEMENT.value: + return TP_OPEN_PRICE_IMPROVEMENT + raise NotImplementedError(f"unsupported take-profit gap policy={policy!r}") + + +def _normalize_report_level(report_level: str) -> str: + level = str(report_level or "standard").lower().strip() + aliases = {"full": "audit", "debug": "audit", "optimizer": "minimal", "scoring": "minimal"} + level = aliases.get(level, level) + if level not in {"minimal", "standard", "audit"}: + raise ValueError("report_level must be minimal, standard, or audit") + return level + + +def _assert_intrabar_audit_parity(first, second, atol: float = 1e-9) -> None: + for i, name in enumerate(("equity", "position", "average_entry", "active_stop", "active_take_profit", "fees", "funding", "flags")): + if not np.allclose(first[i], second[i], atol=atol, rtol=0.0): + raise AssertionError(f"intrabar audit replay drifted from pass 1 for {name}") + for i, name in ((10, "fill_count"), (11, "ambiguity_count"), (12, "rejected_count"), (13, "liquidated"), (14, "liquidation_bar")): + if first[i] != second[i]: + raise AssertionError(f"intrabar audit replay drifted from pass 1 for {name}") + + +def _materialize_intrabar_fills( + *, + timestamps_ns: np.ndarray, + fill_bar: np.ndarray, + fill_seq: np.ndarray, + fill_side: np.ndarray, + fill_qty: np.ndarray, + fill_price: np.ndarray, + fill_fee: np.ndarray, + fill_reason: np.ndarray, + fill_count: int, +) -> tuple[IntrabarFill, ...]: + idx = pd.DatetimeIndex(pd.to_datetime(timestamps_ns, utc=True)) + out = [] + for i in range(fill_count): + bar = int(fill_bar[i]) + out.append( + IntrabarFill( + bar_index=bar, + sequence=int(fill_seq[i]), + timestamp=pd.Timestamp(idx[bar]), + side=int(fill_side[i]), + qty=float(fill_qty[i]), + price=float(fill_price[i]), + fee=float(fill_fee[i]), + reason=_reason_code_to_enum(int(fill_reason[i])), + ) + ) + return tuple(out) + + +def _fills_to_report(fills: Sequence[IntrabarFill]) -> pd.DataFrame: + return pd.DataFrame( + [ + { + "bar_index": fill.bar_index, + "sequence": fill.sequence, + "timestamp": fill.timestamp, + "side": fill.side, + "qty": fill.qty, + "price": fill.price, + "fee": fill.fee, + "reason": fill.reason.value, + } + for fill in fills + ] + ) + + +def _reason_code_to_enum(code: int) -> IntrabarFillReason: + mapping = { + FILL_ENTRY: IntrabarFillReason.ENTRY, + FILL_TECHNICAL_EXIT: IntrabarFillReason.TECHNICAL_EXIT, + FILL_REVERSAL_EXIT: IntrabarFillReason.REVERSAL_EXIT, + FILL_REVERSAL_ENTRY: IntrabarFillReason.REVERSAL_ENTRY, + FILL_STOP_LOSS: IntrabarFillReason.STOP_LOSS, + FILL_TAKE_PROFIT: IntrabarFillReason.TAKE_PROFIT, + FILL_LIQUIDATION: IntrabarFillReason.LIQUIDATION, + FILL_FINAL_CLOSE: IntrabarFillReason.FINAL_CLOSE, + } + return mapping.get(code, IntrabarFillReason.ENTRY) + + +def _reason_series_to_codes(series: pd.Series) -> np.ndarray: + out = np.zeros(len(series), dtype=np.int16) + mapping = {reason.value: code for code, reason in ( + (FILL_ENTRY, IntrabarFillReason.ENTRY), + (FILL_TECHNICAL_EXIT, IntrabarFillReason.TECHNICAL_EXIT), + (FILL_REVERSAL_EXIT, IntrabarFillReason.REVERSAL_EXIT), + (FILL_REVERSAL_ENTRY, IntrabarFillReason.REVERSAL_ENTRY), + (FILL_STOP_LOSS, IntrabarFillReason.STOP_LOSS), + (FILL_TAKE_PROFIT, IntrabarFillReason.TAKE_PROFIT), + (FILL_LIQUIDATION, IntrabarFillReason.LIQUIDATION), + (FILL_FINAL_CLOSE, IntrabarFillReason.FINAL_CLOSE), + )} + for i, value in enumerate(series.astype(str)): + out[i] = mapping.get(value, 0) + return out + + +def _validate_fill_replay_tape(fill_tape: FillReplayTape, n_bars: int) -> None: + if not (len(fill_tape.bar_index) == len(fill_tape.sequence) == len(fill_tape.side) == len(fill_tape.qty) == len(fill_tape.price) == len(fill_tape.fee)): + raise ValueError("fill replay arrays must have matching lengths") + if len(fill_tape.bar_index) == 0: + return + if np.any(fill_tape.bar_index < 0) or np.any(fill_tape.bar_index >= n_bars): + raise ValueError("fill replay bar_index is out of market tape range") + if not np.isfinite(fill_tape.qty).all() or not np.isfinite(fill_tape.price).all() or not np.isfinite(fill_tape.fee).all(): + raise ValueError("fill replay qty/price/fee must be finite") + if np.any(fill_tape.qty <= 0.0) or np.any(fill_tape.price <= 0.0) or np.any(fill_tape.fee < 0.0): + raise ValueError("fill replay qty/price must be positive and fee non-negative") + prev_bar = int(fill_tape.bar_index[0]) + prev_seq = int(fill_tape.sequence[0]) + for bar, seq in zip(fill_tape.bar_index[1:], fill_tape.sequence[1:]): + bar_i = int(bar) + seq_i = int(seq) + if bar_i < prev_bar or (bar_i == prev_bar and seq_i < prev_seq): + raise ValueError("fill replay tape must be sorted by bar_index then sequence") + prev_bar = bar_i + prev_seq = seq_i diff --git a/core/intrabar_reference.py b/core/intrabar_reference.py new file mode 100644 index 0000000..8679b4e --- /dev/null +++ b/core/intrabar_reference.py @@ -0,0 +1,717 @@ +""" +Readable Python oracle for the Phase 31 intrabar execution contract. + +This is not a performance engine. It is the reference state machine used to +prove the later Numba kernel. Strategy output is intentionally compact: +entry side/size plus optional stop, take-profit, trailing distance, and +technical-exit arrays. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum, IntFlag +from typing import Dict, Optional, Sequence + +import numpy as np +import pandas as pd + +from .execution_contract import ExecutionContract, IntrabarSameBarPolicy, TakeProfitGapPolicy +from .constraints import quantize_signed_quantity +from .market_tape import PreparedMarketTape +from .schema import AccountConfig + + +class IntrabarLevelMode(str, Enum): + ABSOLUTE_PRICE = "absolute_price" + PRICE_DISTANCE = "price_distance" + PERCENT_DISTANCE = "percent_distance" + + +class IntrabarSizingMode(str, Enum): + UNITS = "units" + FIXED_NOTIONAL = "fixed_notional" + PCT_EQUITY = "pct_equity" + RISK_PER_TRADE = "risk_per_trade" + + +class IntrabarFillReason(str, Enum): + ENTRY = "entry" + TECHNICAL_EXIT = "technical_exit" + REVERSAL_EXIT = "reversal_exit" + REVERSAL_ENTRY = "reversal_entry" + STOP_LOSS = "stop_loss" + TAKE_PROFIT = "take_profit" + LIQUIDATION = "liquidation" + FINAL_CLOSE = "final_close" + + +class IntrabarEventFlag(IntFlag): + NONE = 0 + ENTRY_FILLED = 1 << 0 + EXIT_FILLED = 1 << 1 + STOP_FILLED = 1 << 2 + TP_FILLED = 1 << 3 + TECH_EXIT = 1 << 4 + REVERSAL = 1 << 5 + AMBIGUOUS = 1 << 6 + FUNDING = 1 << 7 + LIQUIDATION = 1 << 8 + REJECTED = 1 << 9 + ENTRY_SUPPRESSED = 1 << 10 + + +@dataclass(frozen=True) +class IntrabarIntentTape: + entry_side: np.ndarray + entry_size: np.ndarray + stop_value: Optional[np.ndarray] = None + take_profit_value: Optional[np.ndarray] = None + trailing_value: Optional[np.ndarray] = None + technical_exit: Optional[np.ndarray] = None + exit_long: Optional[np.ndarray] = None + exit_short: Optional[np.ndarray] = None + level_mode: IntrabarLevelMode = IntrabarLevelMode.PERCENT_DISTANCE + + def __post_init__(self) -> None: + n = len(self.entry_side) + if len(self.entry_size) != n: + raise ValueError("entry_size must have the same length as entry_side") + for name in ("stop_value", "take_profit_value", "trailing_value", "technical_exit", "exit_long", "exit_short"): + value = getattr(self, name) + if value is not None and len(value) != n: + raise ValueError(f"{name} must have the same length as entry_side") + + @classmethod + def from_arrays( + cls, + *, + entry_side: Sequence, + entry_size: Sequence, + stop_value: Optional[Sequence] = None, + take_profit_value: Optional[Sequence] = None, + trailing_value: Optional[Sequence] = None, + technical_exit: Optional[Sequence] = None, + exit_long: Optional[Sequence] = None, + exit_short: Optional[Sequence] = None, + level_mode: IntrabarLevelMode = IntrabarLevelMode.PERCENT_DISTANCE, + ) -> "IntrabarIntentTape": + legacy_exit = None if technical_exit is None else np.ascontiguousarray(technical_exit, dtype=np.bool_) + return cls( + entry_side=np.ascontiguousarray(entry_side, dtype=np.int8), + entry_size=np.ascontiguousarray(entry_size, dtype=np.float64), + stop_value=_optional_float_array(stop_value), + take_profit_value=_optional_float_array(take_profit_value), + trailing_value=_optional_float_array(trailing_value), + technical_exit=legacy_exit, + exit_long=legacy_exit if exit_long is None and legacy_exit is not None else _optional_bool_array(exit_long), + exit_short=legacy_exit if exit_short is None and legacy_exit is not None else _optional_bool_array(exit_short), + level_mode=level_mode, + ) + + +@dataclass(frozen=True) +class IntrabarFill: + bar_index: int + sequence: int + timestamp: pd.Timestamp + side: int + qty: float + price: float + fee: float + reason: IntrabarFillReason + + +@dataclass(frozen=True) +class IntrabarReferenceResult: + equity: pd.Series + position: pd.Series + average_entry: pd.Series + active_stop: pd.Series + active_take_profit: pd.Series + fees: pd.Series + funding: pd.Series + event_flags: pd.Series + fills: tuple[IntrabarFill, ...] + ambiguity_count: int + rejected_count: int = 0 + liquidated: bool = False + liquidation_bar: int = -1 + metadata: Dict = field(default_factory=dict) + + +def run_intrabar_reference( + *, + tape: PreparedMarketTape, + intent: IntrabarIntentTape, + account: AccountConfig, + contract: Optional[ExecutionContract] = None, + fee_rate: float = 0.0, + slippage_rate: float = 0.0, + contract_size: float = 1.0, + sizing_mode: IntrabarSizingMode | str = IntrabarSizingMode.UNITS, + fixed_notional: float = 0.0, + equity_fraction: float = 0.0, + risk_fraction: float = 0.0, + qty_step: float = 0.0, + min_qty: float = 0.0, + min_notional: float = 0.0, + tick_size: float = 0.0, +) -> IntrabarReferenceResult: + """ + Execute a single-symbol intrabar bracket tape with causal next-open timing. + + Decision arrays at index `t-1` become executable at `open[t]`. + """ + if tape.n_symbols != 1: + raise NotImplementedError("Phase 31B intrabar oracle certifies single-symbol tapes only") + if len(intent.entry_side) != tape.n_bars: + raise ValueError("intent length must match market tape length") + if account.initial_capital <= 0.0: + raise ValueError("initial_capital must be > 0") + if fee_rate < 0.0 or slippage_rate < 0.0: + raise ValueError("fee_rate and slippage_rate must be >= 0") + contract = contract or ExecutionContract.intrabar_bracket() + if contract.engine_id != "intrabar_bracket_v1": + raise ValueError("run_intrabar_reference requires intrabar_bracket_v1 contract") + _validate_intrabar_contract_supported(contract) + sizing_code = IntrabarSizingMode(sizing_mode) + + idx = pd.DatetimeIndex(pd.to_datetime(tape.timestamps_ns, utc=True)) + opens = tape.opens[:, 0] + highs = tape.highs[:, 0] + lows = tape.lows[:, 0] + closes = tape.closes[:, 0] + funding_rates = tape.funding_rates[:, 0] + funding_mask = tape.funding_event_mask + timestamp_semantics = str(getattr(tape, "bar_timestamp_semantics", "close")).lower().strip() + if timestamp_semantics not in {"open", "close"}: + raise ValueError("bar_timestamp_semantics must be 'open' or 'close'") + funding_at_open = timestamp_semantics == "open" + + n = tape.n_bars + equity_arr = np.zeros(n, dtype=np.float64) + pos_arr = np.zeros(n, dtype=np.float64) + avg_arr = np.zeros(n, dtype=np.float64) + stop_arr = np.zeros(n, dtype=np.float64) + tp_arr = np.zeros(n, dtype=np.float64) + fee_arr = np.zeros(n, dtype=np.float64) + funding_arr = np.zeros(n, dtype=np.float64) + flags_arr = np.zeros(n, dtype=np.uint16) + + equity = float(account.initial_capital) + position = 0.0 + avg_entry = 0.0 + active_stop = np.nan + active_tp = np.nan + fills: list[IntrabarFill] = [] + ambiguity_count = 0 + rejected_count = 0 + liquidated = False + liquidation_bar = -1 + + equity_arr[0] = equity + for t in range(1, n): + if liquidated: + equity_arr[t] = 0.0 + pos_arr[t] = 0.0 + avg_arr[t] = 0.0 + stop_arr[t] = 0.0 + tp_arr[t] = 0.0 + continue + + seq = 0 + open_ref = float(opens[t]) + close_ref = float(closes[t]) + last_ref = open_ref + if position != 0.0: + equity += position * (open_ref - float(closes[t - 1])) * contract_size + + if position != 0.0 and _maintenance_breached(equity, position, open_ref, contract_size, account.maintenance_ratio): + side = -1 if position > 0.0 else 1 + price = _market_price(open_ref, side, slippage_rate, tick_size=tick_size) + fee = abs(position) * price * contract_size * fee_rate + equity += position * (price - open_ref) * contract_size - fee + fee_arr[t] += fee + fills.append(_fill(t, seq, idx[t], side, abs(position), price, fee, IntrabarFillReason.LIQUIDATION)) + flags_arr[t] |= int(IntrabarEventFlag.EXIT_FILLED | IntrabarEventFlag.LIQUIDATION) + liquidated = True + liquidation_bar = t + equity = 0.0 + position = 0.0 + avg_entry = 0.0 + active_stop = np.nan + active_tp = np.nan + equity_arr[t] = 0.0 + pos_arr[t] = 0.0 + avg_arr[t] = 0.0 + stop_arr[t] = 0.0 + tp_arr[t] = 0.0 + continue + + if funding_at_open and position != 0.0 and funding_mask[t]: + funding_cost = position * open_ref * contract_size * funding_rates[t] + equity -= funding_cost + funding_arr[t] = funding_cost + flags_arr[t] |= int(IntrabarEventFlag.FUNDING) + + pending_side = int(intent.entry_side[t - 1]) + pending_size = float(intent.entry_size[t - 1]) + pending_exit = _pending_exit(intent, t - 1, position) + exit_same_side_conflict = bool( + pending_exit and pending_side != 0 and position != 0.0 and np.sign(position) == pending_side + ) + + if position != 0.0 and (pending_exit or (pending_side != 0 and np.sign(position) != pending_side)): + reason = IntrabarFillReason.REVERSAL_EXIT if pending_side != 0 and np.sign(position) != pending_side else IntrabarFillReason.TECHNICAL_EXIT + side = -1 if position > 0.0 else 1 + price = _market_price(open_ref, side, slippage_rate, tick_size=tick_size) + fee = abs(position) * price * contract_size * fee_rate + equity += position * (price - open_ref) * contract_size - fee + fee_arr[t] += fee + fills.append(_fill(t, seq, idx[t], side, abs(position), price, fee, reason)) + seq += 1 + flags_arr[t] |= int(IntrabarEventFlag.EXIT_FILLED) + if reason is IntrabarFillReason.TECHNICAL_EXIT: + flags_arr[t] |= int(IntrabarEventFlag.TECH_EXIT) + else: + flags_arr[t] |= int(IntrabarEventFlag.REVERSAL) + position = 0.0 + avg_entry = 0.0 + active_stop = np.nan + active_tp = np.nan + + if pending_side != 0 and pending_size > 0.0 and position == 0.0: + side = 1 if pending_side > 0 else -1 + price = _market_price(open_ref, side, slippage_rate, tick_size=tick_size) + if exit_same_side_conflict: + qty = 0.0 + else: + qty = _compile_entry_quantity( + size_weight=float(pending_size), + fill_price=price, + equity=equity, + contract_size=contract_size, + sizing_mode=sizing_code, + fixed_notional=fixed_notional, + equity_fraction=equity_fraction, + risk_fraction=risk_fraction, + stop_value=None if intent.stop_value is None else float(intent.stop_value[t - 1]), + level_mode=intent.level_mode, + side=side, + tick_size=tick_size, + ) + qty = abs( + quantize_signed_quantity( + qty, + price, + contract_size=contract_size, + qty_step=qty_step, + min_qty=min_qty, + min_notional=min_notional, + ) + ) + if exit_same_side_conflict: + flags_arr[t] |= int(IntrabarEventFlag.ENTRY_SUPPRESSED) + equity_arr[t] = equity + pos_arr[t] = position + avg_arr[t] = avg_entry + stop_arr[t] = 0.0 if not np.isfinite(active_stop) else active_stop + tp_arr[t] = 0.0 if not np.isfinite(active_tp) else active_tp + continue + if qty <= 0.0: + flags_arr[t] |= int(IntrabarEventFlag.REJECTED) + rejected_count += 1 + equity_arr[t] = equity + pos_arr[t] = position + avg_arr[t] = avg_entry + stop_arr[t] = 0.0 if not np.isfinite(active_stop) else active_stop + tp_arr[t] = 0.0 if not np.isfinite(active_tp) else active_tp + continue + if not _has_initial_margin(equity, qty, price, contract_size, account.leverage, account.margin_buffer): + flags_arr[t] |= int(IntrabarEventFlag.REJECTED) + rejected_count += 1 + equity_arr[t] = equity + pos_arr[t] = position + avg_arr[t] = avg_entry + stop_arr[t] = 0.0 if not np.isfinite(active_stop) else active_stop + tp_arr[t] = 0.0 if not np.isfinite(active_tp) else active_tp + continue + fee = qty * price * contract_size * fee_rate + equity -= fee + fee_arr[t] += fee + position = qty * side + avg_entry = price + last_ref = price + active_stop, active_tp = _initial_bracket(intent, t - 1, side, price, tick_size=tick_size) + reason = IntrabarFillReason.REVERSAL_ENTRY if flags_arr[t] & int(IntrabarEventFlag.REVERSAL) else IntrabarFillReason.ENTRY + fills.append(_fill(t, seq, idx[t], side, qty, price, fee, reason)) + seq += 1 + flags_arr[t] |= int(IntrabarEventFlag.ENTRY_FILLED) + + if position != 0.0: + exit_info = _resolve_intrabar_exit( + side=1 if position > 0.0 else -1, + open_price=open_ref, + high=float(highs[t]), + low=float(lows[t]), + stop_price=active_stop, + tp_price=active_tp, + same_bar_policy=contract.same_bar_policy, + take_profit_gap_policy=contract.take_profit_gap_policy, + slippage_rate=slippage_rate, + tick_size=tick_size, + ) + if exit_info is not None: + exit_side, exit_price, reason, ambiguous = exit_info + if ambiguous: + flags_arr[t] |= int(IntrabarEventFlag.AMBIGUOUS) + ambiguity_count += 1 + qty = abs(position) + fee = qty * exit_price * contract_size * fee_rate + equity += position * (exit_price - last_ref) * contract_size - fee + fee_arr[t] += fee + fills.append(_fill(t, seq, idx[t], exit_side, qty, exit_price, fee, reason)) + seq += 1 + flags_arr[t] |= int(IntrabarEventFlag.EXIT_FILLED) + if reason is IntrabarFillReason.STOP_LOSS: + flags_arr[t] |= int(IntrabarEventFlag.STOP_FILLED) + else: + flags_arr[t] |= int(IntrabarEventFlag.TP_FILLED) + position = 0.0 + avg_entry = 0.0 + active_stop = np.nan + active_tp = np.nan + + if position != 0.0: + if _maintenance_breached_at_worst( + equity, + position, + last_ref, + high=float(highs[t]), + low=float(lows[t]), + contract_size=contract_size, + maintenance_ratio=account.maintenance_ratio, + ): + side = -1 if position > 0.0 else 1 + worst = float(lows[t]) if position > 0.0 else float(highs[t]) + price = _market_price(worst, side, slippage_rate, tick_size=tick_size) + fee = abs(position) * price * contract_size * fee_rate + equity += position * (price - last_ref) * contract_size - fee + fee_arr[t] += fee + fills.append(_fill(t, seq, idx[t], side, abs(position), price, fee, IntrabarFillReason.LIQUIDATION)) + flags_arr[t] |= int(IntrabarEventFlag.EXIT_FILLED | IntrabarEventFlag.LIQUIDATION) + liquidated = True + liquidation_bar = t + equity = 0.0 + position = 0.0 + avg_entry = 0.0 + active_stop = np.nan + active_tp = np.nan + else: + equity += position * (close_ref - last_ref) * contract_size + active_stop = _update_trailing(intent, t, position, close_ref, active_stop, tick_size=tick_size) + + if liquidated: + equity_arr[t] = 0.0 + pos_arr[t] = 0.0 + avg_arr[t] = 0.0 + stop_arr[t] = 0.0 + tp_arr[t] = 0.0 + continue + + if not funding_at_open and position != 0.0 and funding_mask[t]: + funding_cost = position * close_ref * contract_size * funding_rates[t] + equity -= funding_cost + funding_arr[t] = funding_cost + flags_arr[t] |= int(IntrabarEventFlag.FUNDING) + + equity_arr[t] = equity + pos_arr[t] = position + avg_arr[t] = avg_entry + stop_arr[t] = 0.0 if not np.isfinite(active_stop) else active_stop + tp_arr[t] = 0.0 if not np.isfinite(active_tp) else active_tp + + if contract.close_on_last_bar and position != 0.0: + t = n - 1 + side = -1 if position > 0.0 else 1 + price = _market_price(float(closes[t]), side, slippage_rate, tick_size=tick_size) + fee = abs(position) * price * contract_size * fee_rate + equity += position * (price - float(closes[t])) * contract_size - fee + fee_arr[t] += fee + fills.append(_fill(t, 99, idx[t], side, abs(position), price, fee, IntrabarFillReason.FINAL_CLOSE)) + position = 0.0 + equity_arr[t] = equity + pos_arr[t] = 0.0 + avg_arr[t] = 0.0 + stop_arr[t] = 0.0 + tp_arr[t] = 0.0 + + return IntrabarReferenceResult( + equity=pd.Series(equity_arr, index=idx, name="equity"), + position=pd.Series(pos_arr, index=idx, name=f"Position_{tape.symbols[0]}"), + average_entry=pd.Series(avg_arr, index=idx, name="average_entry"), + active_stop=pd.Series(stop_arr, index=idx, name="active_stop"), + active_take_profit=pd.Series(tp_arr, index=idx, name="active_take_profit"), + fees=pd.Series(fee_arr, index=idx, name="fees"), + funding=pd.Series(funding_arr, index=idx, name="funding"), + event_flags=pd.Series(flags_arr, index=idx, name="event_flags"), + fills=tuple(fills), + ambiguity_count=int(ambiguity_count), + rejected_count=int(rejected_count), + liquidated=bool(liquidated), + liquidation_bar=int(liquidation_bar), + metadata={ + "engine": "intrabar_reference_v1", + "engine_id": "intrabar_reference_v1", + "execution_contract": contract.to_metadata(), + "data_signature": tape.signature, + "fill_count": len(fills), + "ambiguity_count": int(ambiguity_count), + "rejected_count": int(rejected_count), + "liquidated": bool(liquidated), + "liquidation_bar": int(liquidation_bar), + "oracle": True, + "funding_timing_certified": True, + "funding_event_alignment": "exact_bar_timestamp", + "bar_timestamp_semantics": timestamp_semantics, + "funding_event_price_reference": "open" if funding_at_open else "close", + "sizing_mode": sizing_code.value, + "quantity_constraints": { + "qty_step": float(qty_step), + "min_qty": float(min_qty), + "min_notional": float(min_notional), + "tick_size": float(tick_size), + }, + }, + ) + + +def _optional_float_array(value) -> Optional[np.ndarray]: + if value is None: + return None + return np.ascontiguousarray(value, dtype=np.float64) + + +def _optional_bool_array(value) -> Optional[np.ndarray]: + if value is None: + return None + return np.ascontiguousarray(value, dtype=np.bool_) + + +def _fill(bar, seq, ts, side, qty, price, fee, reason) -> IntrabarFill: + return IntrabarFill( + bar_index=int(bar), + sequence=int(seq), + timestamp=pd.Timestamp(ts), + side=int(side), + qty=float(qty), + price=float(price), + fee=float(fee), + reason=reason, + ) + + +def _market_price(open_price: float, side: int, slippage_rate: float, *, tick_size: float = 0.0) -> float: + raw = float(open_price * (1.0 + slippage_rate if side > 0 else 1.0 - slippage_rate)) + return _quantize_price(raw, side, tick_size) + + +def _has_initial_margin(equity: float, qty: float, price: float, contract_size: float, leverage: float, margin_buffer: float) -> bool: + required = abs(qty) * price * contract_size / leverage + return bool(equity >= required * (1.0 + margin_buffer)) + + +def _maintenance_breached(equity: float, position: float, price: float, contract_size: float, maintenance_ratio: float) -> bool: + maintenance = abs(position) * price * contract_size * maintenance_ratio + return bool(maintenance > 0.0 and equity <= maintenance) + + +def _maintenance_breached_at_worst( + equity: float, + position: float, + reference_price: float, + *, + high: float, + low: float, + contract_size: float, + maintenance_ratio: float, +) -> bool: + worst = low if position > 0.0 else high + worst_equity = equity + position * (worst - reference_price) * contract_size + maintenance = abs(position) * worst * contract_size * maintenance_ratio + return bool(maintenance > 0.0 and worst_equity <= maintenance) + + +def _initial_bracket(intent: IntrabarIntentTape, signal_bar: int, side: int, fill_price: float, *, tick_size: float = 0.0) -> tuple[float, float]: + stop = np.nan + tp = np.nan + if intent.stop_value is not None and np.isfinite(intent.stop_value[signal_bar]) and intent.stop_value[signal_bar] > 0.0: + stop = _level_price(fill_price, side, float(intent.stop_value[signal_bar]), intent.level_mode, is_stop=True, tick_size=tick_size) + if ( + intent.take_profit_value is not None + and np.isfinite(intent.take_profit_value[signal_bar]) + and intent.take_profit_value[signal_bar] > 0.0 + ): + tp = _level_price(fill_price, side, float(intent.take_profit_value[signal_bar]), intent.level_mode, is_stop=False, tick_size=tick_size) + if intent.trailing_value is not None and np.isfinite(intent.trailing_value[signal_bar]) and intent.trailing_value[signal_bar] > 0.0: + trailing_stop = _level_price(fill_price, side, float(intent.trailing_value[signal_bar]), intent.level_mode, is_stop=True, tick_size=tick_size) + stop = trailing_stop if not np.isfinite(stop) else (max(stop, trailing_stop) if side > 0 else min(stop, trailing_stop)) + return stop, tp + + +def _level_price(price: float, side: int, value: float, mode: IntrabarLevelMode, *, is_stop: bool, tick_size: float = 0.0) -> float: + direction = -1.0 if (side > 0 and is_stop) or (side < 0 and not is_stop) else 1.0 + if mode is IntrabarLevelMode.ABSOLUTE_PRICE: + return _quantize_price(float(value), -side, tick_size) + if mode is IntrabarLevelMode.PRICE_DISTANCE: + return _quantize_price(float(price + direction * value), -side, tick_size) + if mode is IntrabarLevelMode.PERCENT_DISTANCE: + return _quantize_price(float(price * (1.0 + direction * value)), -side, tick_size) + raise NotImplementedError(f"unsupported level mode={mode!r}") + + +def _resolve_intrabar_exit( + *, + side: int, + open_price: float, + high: float, + low: float, + stop_price: float, + tp_price: float, + same_bar_policy: IntrabarSameBarPolicy, + take_profit_gap_policy: TakeProfitGapPolicy, + slippage_rate: float, + tick_size: float = 0.0, +): + has_stop = np.isfinite(stop_price) and stop_price > 0.0 + has_tp = np.isfinite(tp_price) and tp_price > 0.0 + if side > 0: + stop_hit = has_stop and low <= stop_price + tp_hit = has_tp and high >= tp_price + stop_gap = has_stop and open_price <= stop_price + tp_gap = has_tp and open_price >= tp_price + exit_side = -1 + else: + stop_hit = has_stop and high >= stop_price + tp_hit = has_tp and low <= tp_price + stop_gap = has_stop and open_price >= stop_price + tp_gap = has_tp and open_price <= tp_price + exit_side = 1 + if not stop_hit and not tp_hit: + return None + ambiguous = bool(stop_hit and tp_hit) + if ambiguous and same_bar_policy is IntrabarSameBarPolicy.REJECT_AMBIGUOUS: + raise ValueError("same bar stop/take-profit ambiguity requires lower timeframe or explicit policy") + stop_first = same_bar_policy in { + IntrabarSameBarPolicy.CONSERVATIVE, + IntrabarSameBarPolicy.STOP_FIRST, + IntrabarSameBarPolicy.OLHC_PATH if side > 0 else IntrabarSameBarPolicy.OHLC_PATH, + } + if stop_hit and (not tp_hit or stop_first): + price = open_price if stop_gap else stop_price + price = _market_price(float(price), exit_side, slippage_rate, tick_size=tick_size) + return exit_side, price, IntrabarFillReason.STOP_LOSS, ambiguous + if tp_hit: + if tp_gap and take_profit_gap_policy is TakeProfitGapPolicy.OPEN_PRICE_IMPROVEMENT: + price = open_price + else: + price = tp_price + return exit_side, _quantize_price(float(price), exit_side, tick_size), IntrabarFillReason.TAKE_PROFIT, ambiguous + return None + + +def _update_trailing(intent: IntrabarIntentTape, signal_bar: int, position: float, close_price: float, current_stop: float, *, tick_size: float = 0.0) -> float: + if intent.trailing_value is None: + return current_stop + value = float(intent.trailing_value[signal_bar]) + if not np.isfinite(value) or value <= 0.0: + return current_stop + side = 1 if position > 0.0 else -1 + candidate = _level_price(close_price, side, value, intent.level_mode, is_stop=True, tick_size=tick_size) + if not np.isfinite(current_stop): + return candidate + return max(current_stop, candidate) if side > 0 else min(current_stop, candidate) + + +def _pending_exit(intent: IntrabarIntentTape, signal_bar: int, position: float) -> bool: + if position > 0.0 and intent.exit_long is not None: + return bool(intent.exit_long[signal_bar]) + if position < 0.0 and intent.exit_short is not None: + return bool(intent.exit_short[signal_bar]) + if intent.technical_exit is not None: + return bool(intent.technical_exit[signal_bar]) + return False + + +def _compile_entry_quantity( + *, + size_weight: float, + fill_price: float, + equity: float, + contract_size: float, + sizing_mode: IntrabarSizingMode, + fixed_notional: float, + equity_fraction: float, + risk_fraction: float, + stop_value: Optional[float], + level_mode: IntrabarLevelMode, + side: int, + tick_size: float = 0.0, +) -> float: + weight = abs(float(size_weight)) + if sizing_mode is IntrabarSizingMode.UNITS: + return weight + if sizing_mode is IntrabarSizingMode.FIXED_NOTIONAL: + notional = float(fixed_notional) * weight + return notional / (fill_price * contract_size) if fill_price > 0.0 and contract_size > 0.0 else 0.0 + if sizing_mode is IntrabarSizingMode.PCT_EQUITY: + notional = float(equity) * float(equity_fraction) * weight + return notional / (fill_price * contract_size) if fill_price > 0.0 and contract_size > 0.0 else 0.0 + if sizing_mode is IntrabarSizingMode.RISK_PER_TRADE: + if stop_value is None or not np.isfinite(stop_value) or stop_value <= 0.0: + return 0.0 + stop_price = _level_price(fill_price, side, float(stop_value), level_mode, is_stop=True, tick_size=tick_size) + stop_distance = abs(fill_price - stop_price) + risk_budget = float(equity) * float(risk_fraction) * weight + return risk_budget / (stop_distance * contract_size) if stop_distance > 0.0 and contract_size > 0.0 else 0.0 + raise NotImplementedError(f"unsupported intrabar sizing_mode={sizing_mode!r}") + + +def _quantize_price(price: float, side: int, tick_size: float) -> float: + tick = float(tick_size) + if tick <= 0.0 or not np.isfinite(price): + return float(price) + if side > 0: + return float(np.ceil((float(price) / tick) - 1e-12) * tick) + return float(np.floor((float(price) / tick) + 1e-12) * tick) + + +def _validate_intrabar_contract_supported(contract: ExecutionContract) -> None: + from .execution_contract import ( + AmbiguityPolicy, + FillPhase, + FundingPhase, + LiquidationPriority, + MarketFillPolicy, + SignalPhase, + StopGapPolicy, + TrailingUpdatePhase, + ) + + if contract.signal_phase is not SignalPhase.BAR_CLOSE: + raise NotImplementedError("intrabar_bracket_v1 supports signal_phase=bar_close only") + if contract.entry_fill_phase is not FillPhase.NEXT_OPEN: + raise NotImplementedError("intrabar_bracket_v1 supports entry_fill_phase=next_open only") + if contract.market_fill_policy is not MarketFillPolicy.NEXT_OPEN: + raise NotImplementedError("intrabar_bracket_v1 supports market_fill_policy=next_open only") + if contract.stop_gap_policy is not StopGapPolicy.OPEN_WORSE_THAN_TRIGGER: + raise NotImplementedError("intrabar_bracket_v1 supports stop_gap_policy=open_worse_than_trigger only") + if contract.trailing_update_phase is not TrailingUpdatePhase.NEXT_BAR: + raise NotImplementedError("intrabar_bracket_v1 supports trailing_update_phase=next_bar only") + if contract.funding_phase is not FundingPhase.POSITION_AT_EVENT: + raise NotImplementedError("intrabar_bracket_v1 supports funding_phase=position_at_event only") + if contract.liquidation_priority is not LiquidationPriority.LIQUIDATION_FIRST_AT_GAP: + raise NotImplementedError("intrabar_bracket_v1 supports liquidation_priority=liquidation_first_at_gap only") + if contract.ambiguity_policy not in {AmbiguityPolicy.FLAG_AND_CONSERVATIVE, AmbiguityPolicy.REJECT}: + raise NotImplementedError("intrabar_bracket_v1 supports ambiguity_policy flag_and_conservative or reject only") diff --git a/core/market_tape.py b/core/market_tape.py new file mode 100644 index 0000000..a01a2d8 --- /dev/null +++ b/core/market_tape.py @@ -0,0 +1,480 @@ +""" +Strict market tape preparation for execution-certified engines. + +This module intentionally does not reuse the compatibility preprocessor. The +existing preprocessor is permissive for legacy notebooks; Phase 31 engines need +explicit validation and a certificate before kernels run. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import hashlib +from typing import Dict, Optional, Sequence, Union + +import numpy as np +import pandas as pd + + +SeriesMap = Dict[str, pd.Series] +FrameMap = Dict[str, pd.DataFrame] + + +@dataclass(frozen=True) +class MarketValidationCertificate: + signature: str + row_count: int + symbol_count: int + timezone: str + first_timestamp_ns: int + last_timestamp_ns: int + finite_ok: bool + ohlc_ok: bool + monotonic_ok: bool + unique_ok: bool + alignment_ok: bool + bar_timestamp_semantics: str = "close" + validator_version: str = "market_tape_v1" + + +@dataclass(frozen=True) +class PreparedMarketTape: + timestamps_ns: np.ndarray + symbols: tuple[str, ...] + opens: np.ndarray + highs: np.ndarray + lows: np.ndarray + closes: np.ndarray + volumes: np.ndarray + funding_rates: np.ndarray + funding_event_mask: np.ndarray + bar_timestamp_semantics: str + signature: str + validation_certificate: MarketValidationCertificate + + @property + def n_bars(self) -> int: + return int(self.opens.shape[0]) + + @property + def n_symbols(self) -> int: + return int(self.opens.shape[1]) + + +def prepare_market_tape( + *, + data: Optional[Union[pd.DataFrame, FrameMap]] = None, + opens: Optional[Union[pd.Series, SeriesMap]] = None, + highs: Optional[Union[pd.Series, SeriesMap]] = None, + lows: Optional[Union[pd.Series, SeriesMap]] = None, + closes: Optional[Union[pd.Series, SeriesMap]] = None, + volumes: Optional[Union[pd.Series, SeriesMap]] = None, + datetime_index: Optional[pd.DatetimeIndex] = None, + symbols: Optional[Sequence[str]] = None, + funding_rate: Union[float, pd.Series, Dict[str, Union[float, pd.Series]]] = 0.0, + funding_event_timestamps: Optional[Union[pd.DatetimeIndex, Sequence]] = None, + funding_event_rates: Optional[Union[Sequence, pd.Series, Dict[str, Union[Sequence, pd.Series]]]] = None, + use_funding: bool = True, + validation_mode: str = "strict", + missing_funding_policy: str = "raise", + source_timezone: Optional[str] = None, + bar_timestamp_semantics: str = "close", +) -> PreparedMarketTape: + """ + Build a strict, immutable OHLCV/funding tape. + + `validation_mode="strict"` rejects unsorted, duplicate, missing, NaN, and + invalid OHLC data. It does not forward-fill or fallback high/low to close. + """ + mode = str(validation_mode).lower().strip() + if mode not in {"strict", "trusted_prepared", "debug"}: + raise ValueError("validation_mode must be strict, trusted_prepared, or debug") + timestamp_semantics = _normalize_bar_timestamp_semantics(bar_timestamp_semantics) + if isinstance(data, PreparedMarketTape): + if data.bar_timestamp_semantics != timestamp_semantics: + raise ValueError( + "prepared market tape bar_timestamp_semantics does not match requested semantics" + ) + return data + + frames, symbol_list = _frames_from_inputs( + data=data, + opens=opens, + highs=highs, + lows=lows, + closes=closes, + volumes=volumes, + datetime_index=datetime_index, + symbols=symbols, + source_timezone=source_timezone, + ) + if not symbol_list: + raise ValueError("at least one symbol is required") + idx = frames[symbol_list[0]].index + _validate_index(idx, name=symbol_list[0]) + for symbol in symbol_list[1:]: + if not frames[symbol].index.equals(idx): + raise ValueError(f"symbol {symbol!r} index is not aligned to {symbol_list[0]!r}") + + n = len(idx) + m = len(symbol_list) + opens_m = np.empty((n, m), dtype=np.float64) + highs_m = np.empty((n, m), dtype=np.float64) + lows_m = np.empty((n, m), dtype=np.float64) + closes_m = np.empty((n, m), dtype=np.float64) + volumes_m = np.empty((n, m), dtype=np.float64) + for j, symbol in enumerate(symbol_list): + frame = frames[symbol] + _validate_ohlcv_frame(frame, symbol) + opens_m[:, j] = frame["open"].to_numpy(dtype=np.float64) + highs_m[:, j] = frame["high"].to_numpy(dtype=np.float64) + lows_m[:, j] = frame["low"].to_numpy(dtype=np.float64) + closes_m[:, j] = frame["close"].to_numpy(dtype=np.float64) + volumes_m[:, j] = frame["volume"].to_numpy(dtype=np.float64) + + ohlcv = np.stack((opens_m, highs_m, lows_m, closes_m, volumes_m), axis=2) + finite_ok = bool(np.isfinite(ohlcv).all()) + if not finite_ok: + raise ValueError("OHLCV contains NaN or infinite values") + ohlc_ok = bool( + ( + (lows_m <= opens_m) + & (lows_m <= closes_m) + & (highs_m >= opens_m) + & (highs_m >= closes_m) + & (highs_m >= lows_m) + & (opens_m > 0.0) + & (highs_m > 0.0) + & (lows_m > 0.0) + & (closes_m > 0.0) + & (volumes_m >= 0.0) + ).all() + ) + if not ohlc_ok: + raise ValueError("invalid OHLCV invariant") + + timestamps_ns = idx.view("int64").astype(np.int64, copy=True) + funding_m, funding_mask = _prepare_funding_matrix( + funding_rate=funding_rate, + funding_event_timestamps=funding_event_timestamps, + funding_event_rates=funding_event_rates, + use_funding=use_funding, + symbols=symbol_list, + idx=idx, + missing_funding_policy=missing_funding_policy, + source_timezone=source_timezone, + ) + signature = _signature( + timestamps_ns, + symbol_list, + opens_m, + highs_m, + lows_m, + closes_m, + volumes_m, + funding_m, + funding_mask.astype(np.float64), + metadata=f"bar_timestamp_semantics={timestamp_semantics}", + ) + cert = MarketValidationCertificate( + signature=signature, + row_count=int(n), + symbol_count=int(m), + timezone=str(idx.tz), + first_timestamp_ns=int(timestamps_ns[0]), + last_timestamp_ns=int(timestamps_ns[-1]), + finite_ok=finite_ok, + ohlc_ok=ohlc_ok, + monotonic_ok=True, + unique_ok=True, + alignment_ok=True, + bar_timestamp_semantics=timestamp_semantics, + ) + arrays = (timestamps_ns, opens_m, highs_m, lows_m, closes_m, volumes_m, funding_m, funding_mask) + for arr in arrays: + arr.setflags(write=False) + return PreparedMarketTape( + timestamps_ns=np.ascontiguousarray(timestamps_ns), + symbols=tuple(symbol_list), + opens=np.ascontiguousarray(opens_m), + highs=np.ascontiguousarray(highs_m), + lows=np.ascontiguousarray(lows_m), + closes=np.ascontiguousarray(closes_m), + volumes=np.ascontiguousarray(volumes_m), + funding_rates=np.ascontiguousarray(funding_m), + funding_event_mask=np.ascontiguousarray(funding_mask), + bar_timestamp_semantics=timestamp_semantics, + signature=signature, + validation_certificate=cert, + ) + + +def _frames_from_inputs( + *, + data, + opens, + highs, + lows, + closes, + volumes, + datetime_index, + symbols, + source_timezone, +) -> tuple[FrameMap, list[str]]: + if data is not None: + if isinstance(data, pd.DataFrame): + symbol_list = list(symbols or ["DEFAULT"]) + if len(symbol_list) != 1: + raise ValueError("single DataFrame market tape requires one symbol") + return {symbol_list[0]: _standard_frame(data, datetime_index, source_timezone=source_timezone)}, symbol_list + symbol_list = list(symbols or data.keys()) + return {symbol: _standard_frame(data[symbol], datetime_index=None, source_timezone=source_timezone) for symbol in symbol_list}, symbol_list + + if closes is None: + raise ValueError("closes or data is required") + if isinstance(closes, pd.Series): + symbol_list = list(symbols or ["DEFAULT"]) + if len(symbol_list) != 1: + raise ValueError("single Series market tape requires one symbol") + symbol = symbol_list[0] + idx = _strict_index(datetime_index if datetime_index is not None else closes.index, name=symbol, source_timezone=source_timezone) + frame = pd.DataFrame( + { + "open": _series_for_symbol(opens, symbol, idx, required=True), + "high": _series_for_symbol(highs, symbol, idx, required=True), + "low": _series_for_symbol(lows, symbol, idx, required=True), + "close": _align_exact(closes, idx, "close"), + "volume": _series_for_symbol(volumes, symbol, idx, required=False), + }, + index=idx, + ) + return {symbol: frame}, symbol_list + + symbol_list = list(symbols or closes.keys()) + idx = _strict_index(datetime_index if datetime_index is not None else closes[symbol_list[0]].index, name=symbol_list[0], source_timezone=source_timezone) + frames = {} + for symbol in symbol_list: + frames[symbol] = pd.DataFrame( + { + "open": _series_for_symbol(opens, symbol, idx, required=True), + "high": _series_for_symbol(highs, symbol, idx, required=True), + "low": _series_for_symbol(lows, symbol, idx, required=True), + "close": _align_exact(closes[symbol], idx, "close"), + "volume": _series_for_symbol(volumes, symbol, idx, required=False), + }, + index=idx, + ) + return frames, symbol_list + + +def _standard_frame(data: pd.DataFrame, datetime_index=None, *, source_timezone: Optional[str] = None) -> pd.DataFrame: + frame = data.copy().rename( + columns={ + "Datetime": "timestamp", + "Date": "timestamp", + "Timestamp": "timestamp", + "Open": "open", + "High": "high", + "Low": "low", + "Close": "close", + "Volume": "volume", + } + ) + if datetime_index is not None: + frame.index = _strict_index(datetime_index, name="datetime_index", source_timezone=source_timezone) + elif "timestamp" in frame.columns: + frame = frame.set_index(_strict_index(frame["timestamp"], name="timestamp", source_timezone=source_timezone)) + else: + frame.index = _strict_index(frame.index, name="data", source_timezone=source_timezone) + required = {"open", "high", "low", "close"} + missing = sorted(required - set(frame.columns)) + if missing: + raise ValueError(f"market data is missing required columns {missing}") + if "volume" not in frame.columns: + frame["volume"] = 0.0 + frame = frame[["open", "high", "low", "close", "volume"]].copy() + frame.index = _strict_index(frame.index, name="data", source_timezone=source_timezone) + return frame + + +def _strict_index(value, *, name: str, source_timezone: Optional[str] = None) -> pd.DatetimeIndex: + raw = pd.DatetimeIndex(pd.to_datetime(value, errors="raise")) + if raw.tz is None: + if source_timezone is None: + raise ValueError(f"{name} index is timezone-naive; pass source_timezone for strict market tape") + raw = raw.tz_localize(source_timezone) + idx = raw.tz_convert("UTC") + _validate_index(idx, name=name) + return idx + + +def _validate_index(idx: pd.DatetimeIndex, *, name: str) -> None: + if len(idx) == 0: + raise ValueError(f"{name} index is empty") + values = idx.view("int64") + if not bool(np.all(values[1:] > values[:-1])): + if bool(pd.Index(values).duplicated().any()): + raise ValueError(f"{name} index contains duplicate timestamps") + raise ValueError(f"{name} index must be strictly increasing") + if idx.tz is None: + raise ValueError(f"{name} index must be timezone-aware") + + +def _validate_ohlcv_frame(frame: pd.DataFrame, symbol: str) -> None: + if len(frame) == 0: + raise ValueError(f"{symbol} market data is empty") + missing = [col for col in ("open", "high", "low", "close", "volume") if col not in frame] + if missing: + raise ValueError(f"{symbol} market data is missing columns {missing}") + + +def _series_for_symbol(data, symbol: str, idx: pd.DatetimeIndex, *, required: bool) -> pd.Series: + if data is None: + if required: + raise ValueError(f"{symbol} requires explicit open/high/low/close for strict market tape") + return pd.Series(0.0, index=idx, name="volume") + if isinstance(data, pd.Series): + series = data + else: + if symbol not in data: + if required: + raise KeyError(f"{symbol!r} missing from strict market tape input") + return pd.Series(0.0, index=idx, name="volume") + series = data[symbol] + return _align_exact(series, idx, symbol) + + +def _align_exact(series: pd.Series, idx: pd.DatetimeIndex, name: str) -> pd.Series: + s = series.copy() + s.index = _strict_index(s.index, name=name) + if not s.index.equals(idx): + raise ValueError(f"{name} series index is not exactly aligned") + return pd.to_numeric(s, errors="raise").astype(float) + + +def _prepare_funding_matrix( + *, + funding_rate, + funding_event_timestamps, + funding_event_rates, + use_funding: bool, + symbols: list[str], + idx: pd.DatetimeIndex, + missing_funding_policy: str, + source_timezone: Optional[str], +) -> tuple[np.ndarray, np.ndarray]: + n = len(idx) + m = len(symbols) + funding = np.zeros((n, m), dtype=np.float64) + mask = np.zeros(n, dtype=np.bool_) + if not use_funding: + return funding, mask + policy = str(missing_funding_policy or "raise").lower().strip() + if policy not in {"raise", "zero"}: + raise ValueError("missing_funding_policy must be raise or zero") + if funding_event_timestamps is not None or funding_event_rates is not None: + if funding_event_timestamps is None or funding_event_rates is None: + raise ValueError("funding_event_timestamps and funding_event_rates must be provided together") + return _funding_from_events( + event_timestamps=funding_event_timestamps, + event_rates=funding_event_rates, + symbols=symbols, + idx=idx, + source_timezone=source_timezone, + ) + if isinstance(funding_rate, dict): + for j, symbol in enumerate(symbols): + if symbol not in funding_rate: + if policy == "zero": + continue + raise KeyError(f"funding_rate dict is missing symbol {symbol!r}") + value = funding_rate[symbol] + if isinstance(value, pd.Series): + funding[:, j] = _align_exact(value, idx, f"funding:{symbol}").to_numpy(dtype=np.float64) + else: + scalar = float(value) + if policy != "zero": + raise ValueError("strict funding requires event timestamps/rates or an aligned Series; scalar funding is not event-causal") + funding[:, j] = scalar + elif isinstance(funding_rate, pd.Series): + series = _align_exact(funding_rate, idx, "funding") + funding[:, :] = series.to_numpy(dtype=np.float64)[:, None] + else: + scalar = float(funding_rate) + if scalar != 0.0 or policy != "zero": + raise ValueError("strict funding requires funding events or an aligned Series; use_funding=False or missing_funding_policy='zero' for no funding") + funding[:, :] = 0.0 + mask[1:] = funding[1:].any(axis=1) + return funding, mask + + +def _funding_from_events( + *, + event_timestamps, + event_rates, + symbols: list[str], + idx: pd.DatetimeIndex, + source_timezone: Optional[str], +) -> tuple[np.ndarray, np.ndarray]: + event_idx = _strict_index(event_timestamps, name="funding_events", source_timezone=source_timezone) + if len(event_idx) == 0: + return np.zeros((len(idx), len(symbols)), dtype=np.float64), np.zeros(len(idx), dtype=np.bool_) + event_ns = event_idx.view("int64") + if isinstance(event_rates, dict): + rates_by_symbol = {} + for symbol in symbols: + if symbol not in event_rates: + raise KeyError(f"funding_event_rates dict is missing symbol {symbol!r}") + rates_by_symbol[symbol] = _event_rate_values(event_rates[symbol], event_idx, symbol) + else: + values = _event_rate_values(event_rates, event_idx, "funding_events") + rates_by_symbol = {symbol: values for symbol in symbols} + + funding = np.zeros((len(idx), len(symbols)), dtype=np.float64) + mask = np.zeros(len(idx), dtype=np.bool_) + idx_ns = idx.view("int64") + for k, ts_ns in enumerate(event_ns): + bar = int(np.searchsorted(idx_ns, ts_ns, side="left")) + if bar >= len(idx_ns) or idx_ns[bar] != ts_ns: + raise ValueError("funding events must align exactly to a market bar timestamp for POSITION_AT_EVENT certification") + if bar == 0: + raise ValueError("funding event at the first bar cannot be certified because no prior position interval exists") + for j, symbol in enumerate(symbols): + rate = float(rates_by_symbol[symbol][k]) + if rate != 0.0: + funding[bar, j] += rate + mask[bar] = True + return funding, mask + + +def _event_rate_values(value, event_idx: pd.DatetimeIndex, name: str) -> np.ndarray: + if isinstance(value, pd.Series): + series = value.copy() + series.index = _strict_index(series.index, name=f"funding_event_rates:{name}") + if not series.index.equals(event_idx): + raise ValueError(f"funding event rates for {name} must align exactly to funding_event_timestamps") + return pd.to_numeric(series, errors="raise").to_numpy(dtype=np.float64) + arr = np.asarray(value, dtype=np.float64) + if arr.ndim == 0: + raise ValueError("funding_event_rates scalar is not valid; pass one rate per funding event") + if len(arr) != len(event_idx): + raise ValueError("funding_event_rates length must match funding_event_timestamps") + return np.ascontiguousarray(arr, dtype=np.float64) + + +def _normalize_bar_timestamp_semantics(value: str) -> str: + semantics = str(value or "close").lower().strip() + if semantics not in {"open", "close"}: + raise ValueError("bar_timestamp_semantics must be 'open' or 'close'") + return semantics + + +def _signature(timestamps_ns: np.ndarray, symbols: list[str], *arrays: np.ndarray, metadata: str = "") -> str: + h = hashlib.sha256() + h.update(np.ascontiguousarray(timestamps_ns).view(np.uint8)) + h.update("|".join(symbols).encode("utf-8")) + if metadata: + h.update(str(metadata).encode("utf-8")) + for arr in arrays: + h.update(np.ascontiguousarray(arr).view(np.uint8)) + return h.hexdigest() diff --git a/core/preprocessor.py b/core/preprocessor.py index 1dd6a46..924de04 100644 --- a/core/preprocessor.py +++ b/core/preprocessor.py @@ -104,7 +104,12 @@ def prepare_funding( out: Dict[str, pd.Series] = {} for sym in symbols: if isinstance(fr_input, dict): - val = fr_input.get(sym, 0.0001) + if sym not in fr_input: + raise KeyError( + f"funding_rate dict is missing symbol {sym!r}; pass 0.0 explicitly " + "or set use_funding=False to avoid synthetic funding defaults" + ) + val = fr_input[sym] elif isinstance(fr_input, pd.Series): val = fr_input else: diff --git a/docs/README.md b/docs/README.md index fb06efb..dff1653 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,6 +8,9 @@ Use this page as the first stop when deciding which QuantBT document to read. |---|---| | Choose the right backend | [Backend selection](backend_selection.md) | | Call QuantBT from notebooks/services | [Endpoint contract](endpoint.md) | +| Choose the correct execution timing contract | [Execution contracts](execution_contracts.md) | +| Migrate SL/TP/trailing alphas to the fast intrabar route | [Fast intrabar](fast_intrabar.md) | +| Audit alpha source files before certification | [Alpha certification](alpha_certification.md) | | Understand vectorized vs event-driven tradeoffs | [Vectorized vs event-driven](vectorized_vs_event_driven.md) | | Validate leverage, buying power, liquidation, funding | [Margin and leverage](margin_leverage.md) | | Understand market/limit/stop fill behavior | [Order fill policies](order_fill_policies.md) | @@ -21,6 +24,8 @@ Use this page as the first stop when deciding which QuantBT document to read. | Strategy type | Preferred route | Why | |---|---|---| | Single-symbol signal research | `QuantBTEndpoint.signal_notional(...)` or `.pct_equity(...)` | Fast scalar signal backtests with stable notebook API | +| Single-symbol SL/TP/trailing | `QuantBTEndpoint.intrabar_bracket(...)` | Strict next-open entry with high/low intrabar exit semantics | +| Existing explicit fill tape | `QuantBTEndpoint.fill_replay(...)` | Accounting replay for old alphas before causal migration | | Explicit orders | `QuantBTEndpoint.orders(...)` | Market/limit/stop order lifecycle and fill reports | | DCA/grid | `QuantBTEndpoint.dca_ladder(...)` | Structural levels, high/low touch detection, trigger-price fills | | Portfolio matrix | `QuantBTEndpoint.portfolio(...)` | Multi-symbol positions with portfolio-level accounting | @@ -47,3 +52,5 @@ For production-like research: is needed. 4. Save `result.metadata`, order/fill reports, config, and benchmark artifacts with the strategy output. +5. For execution-sensitive alphas, run the alpha certification scanner and do + not claim production certification below Level 2. diff --git a/docs/alpha_certification.md b/docs/alpha_certification.md new file mode 100644 index 0000000..1a59451 --- /dev/null +++ b/docs/alpha_certification.md @@ -0,0 +1,113 @@ +# Alpha Execution Certification + +Phase 31D adds lightweight tooling to classify alpha source files by the +execution contract they appear to require. + +The scanner is intentionally conservative. It does not prove a strategy is free +of look-ahead bias. It finds execution-sensitive markers and tells the reviewer +which QuantBT route should be used before making production-style claims. + +## CLI + +```bash +PYTHONPATH=/root/bobby/pool_alpha \ +python3 quantbt/tools/audit_alpha_execution_contracts.py \ + /root/bobby/pool_alpha/alphas_storage/TA \ + --json-out /tmp/alpha_contracts.json \ + --md-out /tmp/alpha_contracts.md +``` + +Default outputs are written under `benchmarks/out/`, which is ignored by git for +local scans. + +## Python API + +```python +from quantbt import ( + scan_alpha_directory, + build_alpha_certification_report, + alpha_report_markdown, +) + +items = scan_alpha_directory("/root/bobby/pool_alpha/alphas_storage/TA") +report = build_alpha_certification_report(items) +markdown = alpha_report_markdown(report) +``` + +To classify a string directly: + +```python +from quantbt import classify_alpha_source + +item = classify_alpha_source(source_text, alpha_id="my_alpha") +print(item.required_engine) +``` + +## Classification Output + +Each row contains: + +```text +alpha_id +path +required_engine +current_backend +certification_status +certification_level +markers +notes +uses_stop / uses_take_profit / uses_trailing / uses_explicit_fills / uses_grid_or_dca +``` + +Typical required engines: + +| Required engine | Meaning | +|---|---| +| `close_target_v2` | plain target signal route | +| `next_open_v1` | next-open timing required | +| `intrabar_bracket_v1` | SL/TP/trailing/high-low exit semantics required | +| `fill_replay_v1` | alpha emits fills; replay accounting first | +| `event_lifecycle_v2` | grid/DCA/package order lifecycle required | +| `unknown` | manual review required | + +## Certification Metadata + +Result metadata can be summarized with: + +```python +from quantbt import certify_result_metadata + +cert = certify_result_metadata(result.metadata) +``` + +Levels: + +```text +0 legacy_or_unspecified_execution_contract +1 explicit_fills_accounted_but_fill_generation_not_certified +2 engine_owned_causal_execution_with_oracle_or_kernel_parity +3 native_engine_matches_native_event_on_known_scenarios +4 external_or_lower_timeframe_validation_available +``` + +## Migration Workflow + +1. Scan the alpha source directory. +2. Treat `requires_intrabar_migration` as a hard warning, not a cosmetic note. +3. For old alphas with explicit `exit_price`, first replay existing fills with + `fill_replay` to lock accounting. +4. Convert the alpha output into compact `entry_signal`, `stop_value`, + `take_profit_value`, `trailing_value`, and `technical_exit` intent columns. +5. Compare `intrabar_bracket_reference` against `intrabar_bracket` on a small + sample. +6. Use `report_level="audit"` for stakeholder/debug runs and + `report_level="minimal"` for optimizer loops. +7. Add Nautilus/lower-timeframe validation only when the strategy needs Level 4 + evidence. + +## Production Claim Rule + +Do not call an execution-sensitive alpha production-certified just because it +runs through a vectorized endpoint. A stop-loss/take-profit/trailing strategy +should reach Level 2 at minimum. For investor or stakeholder reports, Level 3 +or Level 4 evidence is preferred. diff --git a/docs/endpoint.md b/docs/endpoint.md index dbf39a2..e631a80 100644 --- a/docs/endpoint.md +++ b/docs/endpoint.md @@ -54,6 +54,9 @@ bt.metrics # alias for bt.full_report() |---|---|---|---| | `QuantBTEndpoint.pct_equity()` | `pct_equity` | `legacy` | legacy `%_equity` signal where notional is recomputed from live equity | | `QuantBTEndpoint.signal_notional()` | `signal_notional` | `native_vectorized` | fast single-symbol signal research with fixed units between signal changes | +| `QuantBTEndpoint.intrabar_bracket()` | `intrabar_bracket` | `native_intrabar` | fast Phase 31C Numba kernel for next-open SL/TP/trailing/reversal semantics | +| `QuantBTEndpoint.intrabar_bracket_reference()` | `intrabar_bracket_reference` | `intrabar_reference` | readable Phase 31B oracle for next-open SL/TP/trailing/reversal semantics | +| `QuantBTEndpoint.fill_replay()` | `fill_replay` | `native_intrabar` | fast accounting replay from explicit fills | | `QuantBTEndpoint.dca_ladder()` | `dca_ladder` | `legacy` | structural DCA/grid levels with high/low limit-touch simulation | | `QuantBTEndpoint.orders()` | `orders` | `native_event` | explicit `OrderIntent` market/limit/stop simulation | | `QuantBTEndpoint.basket()` | `basket` | `native_event` | pair/basket entry with frozen hedge-ratio units | @@ -82,6 +85,29 @@ Use `backend="auto"` when service code wants QuantBT to choose the safest route: - `nautilus_validation` routes to Nautilus; - other signal modes route to native vectorized. +`native_vectorized` is explicitly the `close_target_v2` execution contract: +signals are interpreted as target exposure at the same bar close, with no +engine-owned intrabar SL/TP/trailing path. Results include contract metadata +such as `engine_id`, `signal_phase`, `fill_phase`, `intrabar_exit_model`, +`kernel_version`, and `data_signature`. If a close-target run receives columns +that look like intrabar execution artifacts (`exit_price`, `stop_loss`, +`take_profit`, `trailing`, etc.), QuantBT marks the run as uncertified for those +intrabar semantics instead of silently implying correctness. + +`intrabar_bracket` is the Phase 31C fast Numba implementation of +`intrabar_bracket_v1`; `intrabar_bracket_reference` is the readable Python +oracle for the same semantics. Both use strict market tape validation. They +model: signal at bar close, entry at next bar open, gap-aware stop-loss, +limit-style take-profit, same-bar SL/TP ambiguity, trailing-stop updates that +only become effective on the next bar, technical exits, reversals as two +fee/slippage legs, initial-margin rejection, simple single-symbol liquidation, +and optional final close. + +For the full contract taxonomy and certification workflow, read +[`execution_contracts.md`](execution_contracts.md), +[`fast_intrabar.md`](fast_intrabar.md), and +[`alpha_certification.md`](alpha_certification.md). + ## Nautilus Support Matrix Services can inspect current Nautilus adapter coverage before constructing a @@ -378,6 +404,238 @@ Routing: For plain market rebalance signals, native vectorized and native event should match equity closely. Use event mode when fill-level diagnostics matter. +## Intrabar Bracket, Fast And Reference + +Use this for execution-certified alpha logic that depends on SL/TP/trailing +behavior inside the bar. `intrabar_bracket(...)` is the Phase 31C fast Numba +kernel. `intrabar_bracket_reference(...)` keeps the readable Phase 31B Python +oracle for debugging and parity checks. + +```python +bt = QuantBTEndpoint.intrabar_bracket( + initial_capital=20_000, + leverage=5, + fee_rate=0.0002, # one-way fee + slippage_bps=1.0, # source of truth for intrabar slippage + tick_size=0.01, # optional conservative price quantization + use_funding=False, + close_on_last_bar=True, + report_level="standard", +) + +result = bt.backtest( + data=df, + signal_col="entry_signal", # signed qty: +1.0 long, -1.0 short, 0 no new entry + symbols=["ETHUSDT"], + intent_cols={ + "stop_value": "sl_pct", + "take_profit_value": "tp_pct", + "trailing_value": "trail_pct", + "exit_long": "exit_long", + "exit_short": "exit_short", + }, +) + +bt.show_metrics() +fills = bt.fills_report +``` + +Input contract: + +- `data`: strict single-symbol OHLCV DataFrame with timezone-aware + `DatetimeIndex` or timestamp column; +- required market columns: `open`, `high`, `low`, `close`; +- no sorting, deduplication, forward-fill, or high/low fallback is performed; +- `signal` or `signal_col`: compact signed entry size, where the sign is side + and absolute value is quantity; +- optional `intent_cols`: map strategy column names into `stop_value`, + `take_profit_value`, `trailing_value`, `exit_long`, and `exit_short`; +- legacy `technical_exit` is still accepted and maps to both long/short exits, + but new alphas should use side-specific exits; +- intrabar slippage uses `slippage_bps`; legacy `slippage` is converted with a + deprecation warning, and passing both raises; +- `tick_size` is optional and quantizes entry, stop, take-profit, and trailing + prices conservatively; +- default `level_mode="percent_distance"` interprets `0.05` as 5 percent from + fill price. Use `level_mode="price_distance"` or `"absolute_price"` when + supplying distance/level values in price units. + +Execution contract: + +- signal at close of bar `t`; +- entry/technical exit/reversal at open of bar `t + 1`; +- stop gaps fill at the open when the open is worse than trigger; +- take-profit is limit-conservative by default; +- same-bar SL/TP conflict is flagged and resolved conservatively; +- trailing stop is updated after the bar close and only applies from the next + bar; +- reversal pays two legs: close old position and open new position; +- result metadata contains `validation_certificate`, `data_signature`, + `execution_contract`, `fills_report`, `kernel_version`, and report-level + details. + +Report levels: + +- `minimal`: optimizer/WFO path. Keeps equity, position, fees/funding, counters, + and event flags; no fill ledger materialization. +- `standard`: default notebook/service path. Adds diagnostics such as active + stop/TP and margin series; still no fill DataFrame. +- `audit`: runs a deterministic second pass, allocates sparse fill arrays sized + exactly to real `fill_count`, materializes `result.fills` and + `bt.fills_report`, and asserts parity against pass 1. + +Use the reference endpoint for differential debugging: + +```python +ref = QuantBTEndpoint.intrabar_bracket_reference( + initial_capital=20_000, + leverage=5, + fee_rate=0.0002, + slippage_bps=1.0, + use_funding=False, +) + +ref_result = ref.backtest( + data=df, + signal_col="entry_signal", + symbols=["ETHUSDT"], + intent_cols={"stop_value": "sl_pct", "take_profit_value": "tp_pct"}, +) +``` + +Prepared runner for optimizers: + +```python +bt = QuantBTEndpoint.intrabar_bracket( + initial_capital=20_000, + leverage=5, + fee_rate=0.0002, + slippage_bps=1.0, + use_funding=False, + report_level="minimal", +) + +runner = bt.prepare_intrabar(data=df, symbols=["ETHUSDT"]) +intent = alpha.generate(runner.market, params) +result = runner.run(intent, report_level="minimal") +audit = runner.run(intent, report_level="audit") +``` + +Funding for intrabar routes is event-causal only when the funding timestamp +matches an exact market bar timestamp. Mid-bar funding events are rejected and +require a smaller timeframe. Use `use_funding=False` when no funding is part of +the test, pass an aligned funding Series with non-zero values only on funding +bars, or pass exact-boundary `funding_event_timestamps` plus +`funding_event_rates` to `backtest(...)` / `prepare_intrabar(...)`. + +Also declare the bar timestamp convention when funding is enabled: +`bar_timestamp_semantics="close"` is the default and applies funding after the +bar's intrabar path on the remaining close position. Use +`bar_timestamp_semantics="open"` for bar-open timestamped crypto feeds; funding +then applies before pending exit/entry orders at `open[t]`. + +Custom execution contracts can be passed directly and are preserved in +metadata: + +```python +from quantbt import ExecutionContract, IntrabarSameBarPolicy + +contract = ExecutionContract.intrabar_bracket( + same_bar_policy=IntrabarSameBarPolicy.TP_FIRST, + close_on_last_bar=False, +) + +bt = QuantBTEndpoint.intrabar_bracket(execution_contract=contract) +``` + +Unsupported contract fields raise `NotImplementedError`; they are not silently +reset to defaults. + +## Fill Replay + +Use this when an old alpha already emitted explicit fills and QuantBT should +only validate/account them. This route certifies accounting, not fill +generation. + +```python +bt = QuantBTEndpoint.fill_replay( + initial_capital=20_000, + leverage=5, + contract_size=1.0, +) + +result = bt.backtest( + data=df, + symbols=["ETHUSDT"], + fill_replay=fills_df, +) +``` + +`fills_df` must be sorted by `bar_index`, then `sequence`, and contain: + +```text +bar_index +side # +1 buy, -1 sell +qty +price +``` + +This route is Level 1 certification by design: QuantBT certifies accounting from +the supplied fills, while the alpha or external system remains responsible for +causal fill generation. + +## Alpha Execution Audit + +Use the scanner before migrating old alpha directories: + +```bash +PYTHONPATH=/root/bobby/pool_alpha \ +python3 quantbt/tools/audit_alpha_execution_contracts.py \ + /root/bobby/pool_alpha/alphas_storage/TA \ + --json-out /tmp/alpha_contracts.json \ + --md-out /tmp/alpha_contracts.md +``` + +Or from Python: + +```python +from quantbt import ( + scan_alpha_directory, + build_alpha_certification_report, + alpha_report_markdown, +) + +items = scan_alpha_directory("/root/bobby/pool_alpha/alphas_storage/TA") +report = build_alpha_certification_report(items) +print(alpha_report_markdown(report)) +``` + +Certification levels: + +| Level | Meaning | +|---:|---| +| 0 | legacy or unspecified execution contract | +| 1 | explicit-fill accounting replay | +| 2 | engine-causal QuantBT execution | +| 3 | native cross-backend parity | +| 4 | external validation, usually Nautilus/lower-timeframe route | + +Optional columns: + +```text +sequence +fee +reason +``` + +If `fee` is omitted, `fee_rate` is used to compute one-way fees from notional. +Result metadata declares: + +```text +accounting_certified = true +execution_generation_certified = false +``` + ## DCA / Grid Ladder Use this when `signal` is a structural ladder level, not a dynamic position diff --git a/docs/execution_contracts.md b/docs/execution_contracts.md new file mode 100644 index 0000000..20754c5 --- /dev/null +++ b/docs/execution_contracts.md @@ -0,0 +1,126 @@ +# QuantBT Execution Contracts + +Execution contract is the promise between a strategy signal and the backtest +engine. It says when the signal is known, when an order may be submitted, which +price can fill, and which accounting rules are certified. + +This matters because a single `pos_weight` column can mean very different +things: + +- target exposure at the current close; +- signal observed at close and filled next open; +- entry plus stop-loss/take-profit/trailing behavior inside the next bar; +- explicit fills already generated by an external alpha; +- multi-order package lifecycle such as DCA/grid/basket/arbitrage. + +QuantBT now makes those contracts explicit. Old endpoints remain compatible, +but execution-sensitive alphas should use the narrowest contract that matches +their domain. + +## Contract Map + +| Contract | Endpoint | Signal phase | Fill phase | Intrabar semantics | Certification target | +|---|---|---|---|---|---| +| `close_target_v2` | `signal_notional`, `pct_equity`, native vectorized | bar close | bar close target accounting | none | Level 2 for close-target signals | +| `next_open_v1` | reserved / event routes | bar close | next bar open | entry/exit only | future specialized route | +| `intrabar_bracket_v1` | `intrabar_bracket`, `intrabar_bracket_reference` | bar close | next bar open, then high/low path | SL/TP/trailing/reversal | Level 2 now, Level 3+ with parity bundle | +| `fill_replay_v1` | `fill_replay` | explicit fills supplied | explicit fill tape | alpha-owned | Level 1 accounting replay | +| `event_lifecycle_v2` | `orders`, `basket`, `arbitrage`, DCA/grid routes | command/event driven | market/limit/stop lifecycle | order-owned | Level 2-4 depending parity | +| `nautilus` | `nautilus_validation`, order/package validation | event driven | Nautilus simulation | external engine | Level 4 route when configured | + +## Close Target + +Use close-target routes when the strategy already produced a target exposure +series and does not claim stop-loss, take-profit, trailing-stop, or custom +intrabar exit prices. + +The model is intentionally simple: + +```text +signal[t] -> target exposure at close[t] +PnL[t+1] -> position[t] * (close[t+1] - close[t]) +``` + +This is correct for research signals that do not depend on the inside of the +bar. It is not a certified route for alphas that generate `exit_price`, +`exit_type`, `slpercent`, `tppercent`, or high/low-based exits. If such columns +are present on a close-target run, QuantBT records an uncertified status in +metadata instead of silently claiming intrabar correctness. + +## Intrabar Bracket + +Use `intrabar_bracket_v1` when a strategy emits compact entry intent and the +engine must own: + +- next-open entry; +- gap-aware stop-loss; +- limit-style take-profit; +- same-bar SL/TP conflict policy; +- trailing-stop update timing; +- technical exits; +- reversals as two fee-paying legs; +- optional final close; +- simple single-symbol liquidation checks. + +The canonical timing is: + +```text +strategy observes bar t close +entry/reversal/technical exit may fill at open[t + 1] +stop/take-profit may fill within bar t + 1 using high/low +trailing stop updates after bar t + 1 close +``` + +The Python reference oracle is readable and should be used when debugging a new +alpha migration. The Numba kernel is the production research path once parity is +verified. + +## Fill Replay + +Use `fill_replay_v1` when an alpha already emitted exact fills. QuantBT then +replays the fill tape for accounting only: + +```text +bar_index, sequence, side, qty, price +``` + +This certifies PnL/fee/funding/margin accounting from the supplied fills. It +does not certify how those fills were generated. The right label is Level 1 +unless paired with a separate causal fill-generation proof. + +## Event Lifecycle + +Use native event or Nautilus routes when the strategy has explicit order +lifecycle requirements: + +- market/limit/stop orders; +- order activation/cancel/replace; +- bracket/OCO packages; +- basket all-or-none packages; +- arbitrage multi-leg execution; +- DCA/grid state machines. + +These routes are broader but slower than the narrow intrabar kernel. They are +the right place for order/package semantics that cannot be reduced to a compact +single-symbol bracket tape. + +## Certification Levels + +| Level | Label | Meaning | +|---:|---|---| +| 0 | Legacy / unspecified | Backtest ran, but execution-sensitive claims are not certified | +| 1 | Accounting replay | Explicit fills were replayed correctly; fill generation remains alpha-owned | +| 2 | Engine causal | QuantBT owns causal execution semantics with oracle/kernel parity | +| 3 | Cross-backend | Native route matches another QuantBT route on known scenarios | +| 4 | External validation | Nautilus/lower-timeframe/external bundle is available | + +Production-style claims should be at least Level 2. Execution-sensitive +stakeholder reports should prefer Level 3 or Level 4 when feasible. + +## Rule Of Thumb + +- Pure signal target: use close target. +- Entry at next bar and SL/TP/trailing inside the bar: use intrabar bracket. +- Old alpha emits fills: use fill replay first, then migrate to causal engine. +- Multi-leg package or grid state: use event lifecycle or Nautilus validation. +- If unsure, scan the alpha and treat the result as a migration hint, not proof. diff --git a/docs/fast_intrabar.md b/docs/fast_intrabar.md new file mode 100644 index 0000000..c34e29c --- /dev/null +++ b/docs/fast_intrabar.md @@ -0,0 +1,202 @@ +# Fast Intrabar Bracket And Fill Replay + +Phase 31 adds a strict single-symbol intrabar route for alphas that previously +mixed signal generation and exit-price simulation in notebook code. + +The goal is not to make OHLC bars pretend to be tick data. The goal is to make +the exact assumptions explicit, deterministic, fast, and auditable. + +## Fast Kernel + +```python +from quantbt import QuantBTEndpoint + +bt = QuantBTEndpoint.intrabar_bracket( + initial_capital=20_000, + leverage=5, + fee_rate=0.0002, # one-way + slippage_bps=1.0, + tick_size=0.01, + use_funding=False, + close_on_last_bar=True, + report_level="audit", +) + +result = bt.backtest( + data=df, + signal_col="entry_signal", + symbols=["ETHUSDT"], + intent_cols={ + "stop_value": "sl_pct", + "take_profit_value": "tp_pct", + "trailing_value": "trail_pct", + "exit_long": "exit_long", + "exit_short": "exit_short", + }, +) + +fills = bt.fills_report +result.show_metrics() +``` + +## Required Data + +`data` must be a strict single-symbol OHLCV frame: + +```text +DatetimeIndex or timestamp column +open +high +low +close +volume optional +``` + +Strict means no silent sorting, deduplication, or high/low fallback. The engine +should not “repair” an execution-sensitive tape because that can hide data +leakage and timestamp mistakes. + +## Intent Columns + +The minimal signal is a signed entry size: + +```text ++1.0 -> open long one unit on next bar open +-1.0 -> open short one unit on next bar open + 0.0 -> no new entry +``` + +Optional intent columns: + +```text +stop_value +take_profit_value +trailing_value +exit_long +exit_short +``` + +`level_mode="percent_distance"` is the default. Under this mode `0.02` means a +2 percent distance from entry price. `price_distance` and `absolute_price` are +available when the strategy emits price distances or absolute levels. + +Legacy `technical_exit` is still accepted for compatibility and maps to both +long and short exits. New alphas should emit `exit_long` and `exit_short`, so an +exit signal cannot accidentally close the wrong side. + +Sizing options: + +```text +units +fixed_notional +pct_equity +risk_per_trade +``` + +After sizing, the same shared quantity constraints used by the event and +portfolio backends are applied: `qty_step`/`lot_size`/`slot_size`, `min_qty`, +and `min_notional`. If `tick_size` is provided, entry, stop, take-profit, and +trailing prices are quantized conservatively to the exchange tick. + +## Execution Semantics + +The engine uses this timing: + +```text +entry signal known at close[t] +entry fills at open[t + 1] +stop and TP evaluated inside bar t + 1 using high/low +technical exit fills at open[t + 1] +trailing stop updates after the bar close +reversal = close old position + open new position +``` + +Same-bar stop/TP ambiguity is resolved conservatively by default. For a long, +if both stop and take-profit are touched in the same OHLC bar, the stop wins. +For a short, the same conservative loss-first rule applies. + +Gap stops fill at the open when the open is worse than the trigger. Take-profit +uses limit-style behavior by default. + +## Report Levels + +| Level | Best use | Fill ledger | +|---|---|---| +| `minimal` | optimizer and WFO loops | no | +| `standard` | notebooks and normal services | no | +| `audit` | migration/certification/debugging | yes | + +`audit` runs a deterministic second pass. The first pass computes accounting; +the second pass materializes sparse fill arrays sized exactly to the observed +fill count. The audit path asserts accounting parity with the first pass. + +## Python Oracle + +```python +ref = QuantBTEndpoint.intrabar_bracket_reference( + initial_capital=20_000, + leverage=5, + fee_rate=0.0002, + slippage_bps=1.0, +) +ref_result = ref.backtest(data=df, signal_col="entry_signal") +``` + +Use the reference route to inspect behavior when migrating an alpha. Use the +Numba route for real sweeps after parity tests pass. + +## Prepared Runner + +```python +runner = bt.prepare_intrabar(data=df, symbols=["ETHUSDT"]) +result = runner.run(intent, report_level="minimal") +audit = runner.run(intent, report_level="audit") +``` + +The prepared runner caches strict OHLCV arrays, timestamps, funding arrays, +quantity constraints, validation certificate, data signature, and frozen profile +metadata. Use this in WFO/Optuna loops where market tape is fixed and only the +intent changes. + +Funding events must align exactly to a market bar timestamp for +`POSITION_AT_EVENT` certification. Mid-bar funding events are rejected rather +than approximated from the end-of-bar position. + +Timestamp semantics must also be explicit. The default +`bar_timestamp_semantics="close"` means each OHLC timestamp labels the bar +close, so funding is applied after intrabar execution on the remaining close +position. For common crypto feeds whose OHLC timestamp labels the bar open, +pass `bar_timestamp_semantics="open"`; funding is then applied after open-gap +marking and before pending orders at `open[t]`. + +## Fill Replay + +```python +bt = QuantBTEndpoint.fill_replay(initial_capital=20_000, leverage=5) +result = bt.backtest(data=df, fill_replay=fills_df, symbols=["ETHUSDT"]) +``` + +`fills_df` must contain: + +```text +bar_index +side +qty +price +sequence optional +fee optional +``` + +Fill replay is Level 1 certification: accounting is tested, but fill generation +is still owned by the alpha or external system that produced the tape. + +## What This Does Not Claim + +- It is not tick or L2 order-book simulation. +- It is not a multi-symbol cross-margin intrabar engine. +- It is not the DCA/grid state machine. +- It does not claim portfolio, arbitrage, grid, DCA, options, or shared + cross-margin intrabar correctness. +- It does not make a look-ahead alpha valid. + +For those cases, use native event or Nautilus package validation. diff --git a/endpoint.py b/endpoint.py index 2b8a72c..c30aab0 100644 --- a/endpoint.py +++ b/endpoint.py @@ -10,8 +10,11 @@ from __future__ import annotations from dataclasses import asdict, dataclass, field, is_dataclass, replace +import hashlib +import json from pathlib import Path from typing import Dict, Optional, Sequence, Union +import warnings import numpy as np import pandas as pd @@ -43,6 +46,16 @@ NautilusExecutionDepthConfig, simulate_nautilus_order_package_depth, ) +from .core.execution_contract import ExecutionContract +from .core.constraints import build_quantity_constraints +from .core.intrabar_reference import ( + IntrabarIntentTape, + IntrabarLevelMode, + IntrabarSizingMode, + run_intrabar_reference, +) +from .core.intrabar_kernel import FillReplayTape, run_fill_replay_kernel, run_intrabar_kernel +from .core.market_tape import PreparedMarketTape, prepare_market_tape from .core.orders import OrderCommand, OrderIntent, order_intents_to_lifecycle_commands from .core.results import BacktestResultV2, OptionBacktestResult from .core.schema import AccountConfig, BasketLegSpec, BasketSpec, ExecutionConfig, InstrumentSpec, OrderSide, OrderType, TimeInForce @@ -195,6 +208,81 @@ def v2_fee_rate(self) -> float: return self.fee / 2.0 if self.fee_rate is None else float(self.fee_rate) +@dataclass(frozen=True) +class PreparedIntrabarRunner: + """Prepared single-symbol intrabar runner for repeated WFO/Optuna runs.""" + + endpoint: "QuantBTEndpoint" + tape: PreparedMarketTape + symbol: str + contract: ExecutionContract + profile_metadata: Dict + + @property + def market(self) -> PreparedMarketTape: + return self.tape + + def run(self, intent: IntrabarIntentTape, *, report_level: Optional[str] = None) -> BacktestResultV2: + config = self.endpoint.config + level = report_level or config.report_level + kernel = run_intrabar_kernel( + tape=self.tape, + intent=intent, + account=config.account, + contract=self.contract, + fee_rate=config.v2_fee_rate, + slippage_rate=float(config.execution.slippage_rate), + contract_size=_scalar_for_symbol(config.contract_size, self.symbol), + **self.endpoint._intrabar_execution_kwargs(self.symbol), + report_level=level, + ) + idx = kernel.equity.index + returns = kernel.equity.pct_change().replace([np.inf, -np.inf], np.nan).fillna(0.0) + diagnostics = pd.DataFrame( + { + "average_entry": kernel.average_entry, + "active_stop": kernel.active_stop, + "active_take_profit": kernel.active_take_profit, + "event_flags": kernel.event_flags, + "initial_margin": kernel.initial_margin, + "maintenance_margin": kernel.maintenance_margin, + "fees": kernel.fees, + "funding": kernel.funding, + }, + index=idx, + ) + metadata = { + **kernel.metadata, + "input_mode": "intrabar_intent", + "symbol": self.symbol, + "phase": "31F_prepared_intrabar_runner", + "prepared_runner": True, + "profile_metadata": dict(self.profile_metadata), + "fills_report": kernel.fills_report, + "positions_report": pd.DataFrame({f"Position_{self.symbol}": kernel.position}, index=idx), + } + result = BacktestResultV2( + equity=kernel.equity, + returns=returns, + positions=pd.DataFrame({f"Position_{self.symbol}": kernel.position.to_numpy(dtype=float)}, index=idx), + closes=pd.DataFrame({f"Close_{self.symbol}": self.tape.closes[:, 0]}, index=idx), + symbols=[self.symbol], + initial_capital=float(config.account.initial_capital), + leverage=float(config.account.leverage), + liquidated=bool(kernel.liquidated), + liquidation_bar=int(kernel.liquidation_bar), + fills=kernel.fills, + fees=kernel.fees, + funding=kernel.funding, + margin=diagnostics[["initial_margin", "maintenance_margin"]], + diagnostics=diagnostics, + metadata=metadata, + ) + self.endpoint.engine = kernel + self.endpoint._store_result(result) + return self.endpoint.result + + class QuantBTEndpoint: """ Stable notebook/service facade for all QuantBT backtest modes. @@ -246,6 +334,54 @@ def prepare_service_context( symbols=symbols, ) + def prepare_intrabar( + self, + *, + data, + datetime_index=None, + symbols: Optional[Sequence[str]] = None, + funding_event_timestamps=None, + funding_event_rates=None, + ) -> PreparedIntrabarRunner: + """ + Prepare strict intrabar market tape once and reuse it for many intents. + + This is an opt-in service/WFO helper. Normal `.backtest(...)` remains + backward-compatible, while optimizer loops can avoid rebuilding OHLCV, + funding, validation certificate, data signature, and quantity profiles + on every trial. + """ + symbol_list = list(symbols or self.config.symbols or ["DEFAULT"]) + if len(symbol_list) != 1: + raise ValueError("prepare_intrabar currently supports exactly one symbol") + tape = prepare_market_tape( + data=data, + datetime_index=datetime_index, + symbols=symbol_list, + funding_rate=self.config.funding_rate, + funding_event_timestamps=funding_event_timestamps, + funding_event_rates=funding_event_rates, + use_funding=self.config.use_funding, + validation_mode="strict", + missing_funding_policy=str(self.config.metadata.get("missing_funding_policy", "raise")), + source_timezone=self.config.metadata.get("source_timezone"), + bar_timestamp_semantics=str(self.config.metadata.get("bar_timestamp_semantics", "close")), + ) + contract = _execution_contract_from_config(self.config) + symbol = symbol_list[0] + profile = { + "mode": self.config.mode, + "backend": self.config.backend, + "account": asdict(self.config.account), + "execution": asdict(self.config.execution), + "fee_rate": self.config.v2_fee_rate, + "contract_size": _scalar_for_symbol(self.config.contract_size, symbol), + "intrabar": self._intrabar_execution_kwargs(symbol), + "data_signature": tape.signature, + } + profile["prepared_signature"] = _prepared_profile_signature(tape.signature, profile) + return PreparedIntrabarRunner(endpoint=self, tape=tape, symbol=symbol, contract=contract, profile_metadata=profile) + @classmethod def pct_equity(cls, **kwargs) -> "QuantBTEndpoint": """ @@ -276,6 +412,106 @@ def signal_notional(cls, backend: str = "native_vectorized", **kwargs) -> "Quant """ return cls(_config_from_kwargs(mode="signal_notional", sizing="signal_notional", backend=backend, **kwargs)) + @classmethod + def intrabar_bracket_reference( + cls, + *, + level_mode: Union[str, IntrabarLevelMode] = IntrabarLevelMode.PERCENT_DISTANCE, + intrabar_sizing_mode: Union[str, IntrabarSizingMode] = IntrabarSizingMode.UNITS, + close_on_last_bar: bool = True, + execution_contract: Optional[ExecutionContract] = None, + **kwargs, + ) -> "QuantBTEndpoint": + """ + Create the Phase 31B readable intrabar reference endpoint. + + This endpoint is the causal Python oracle for `intrabar_bracket_v1`. + Strategy output can stay compact: pass a signed `signal`/`signal_col` + where positive means long entry size, negative means short entry size, + and zero means no new entry. Optional stop, take-profit, trailing, and + technical-exit arrays are supplied through `intent_cols` at run time. + + It is intentionally not the future Numba production kernel. Use it to + verify SL/TP/trailing/reversal semantics and audit fill timing before + promoting an alpha to the fast intrabar backend. + """ + metadata = dict(kwargs.pop("metadata", {})) + mode_value = level_mode.value if hasattr(level_mode, "value") else str(level_mode) + metadata.setdefault("intrabar_level_mode", mode_value) + metadata.setdefault("intrabar_sizing_mode", IntrabarSizingMode(intrabar_sizing_mode).value) + metadata.setdefault("execution_contract_id", "intrabar_bracket_v1") + contract = execution_contract or ExecutionContract.intrabar_bracket(close_on_last_bar=close_on_last_bar) + metadata.setdefault("execution_contract", contract.to_metadata()) + return cls( + _config_from_kwargs( + mode="intrabar_bracket_reference", + backend="intrabar_reference", + sizing="intrabar_intent", + metadata=metadata, + **kwargs, + ) + ) + + @classmethod + def intrabar_bracket( + cls, + *, + level_mode: Union[str, IntrabarLevelMode] = IntrabarLevelMode.PERCENT_DISTANCE, + intrabar_sizing_mode: Union[str, IntrabarSizingMode] = IntrabarSizingMode.UNITS, + close_on_last_bar: bool = True, + execution_contract: Optional[ExecutionContract] = None, + report_level: str = "standard", + **kwargs, + ) -> "QuantBTEndpoint": + """ + Create the Phase 31C fast Numba intrabar bracket endpoint. + + Use the same compact input contract as + `intrabar_bracket_reference(...)`. `report_level="minimal"` is meant + for optimizers, `standard` returns diagnostics, and `audit` runs a + deterministic second pass to materialize exact sparse fills. + """ + metadata = dict(kwargs.pop("metadata", {})) + mode_value = level_mode.value if hasattr(level_mode, "value") else str(level_mode) + metadata.setdefault("intrabar_level_mode", mode_value) + metadata.setdefault("intrabar_sizing_mode", IntrabarSizingMode(intrabar_sizing_mode).value) + metadata.setdefault("execution_contract_id", "intrabar_bracket_v1") + contract = execution_contract or ExecutionContract.intrabar_bracket(close_on_last_bar=close_on_last_bar) + metadata.setdefault("execution_contract", contract.to_metadata()) + return cls( + _config_from_kwargs( + mode="intrabar_bracket", + backend="native_intrabar", + sizing="intrabar_intent", + report_level=report_level, + metadata=metadata, + **kwargs, + ) + ) + + @classmethod + def fill_replay(cls, *, report_level: str = "audit", **kwargs) -> "QuantBTEndpoint": + """ + Create a fast accounting replay endpoint for explicit fills. + + Use `backtest(data=df, fill_replay=FillReplayTape_or_DataFrame)`. This + certifies accounting from supplied fills but does not certify how those + fills were generated. + """ + metadata = dict(kwargs.pop("metadata", {})) + metadata.setdefault("execution_contract_id", "fill_replay_v1") + metadata.setdefault("execution_contract", ExecutionContract.fill_replay().to_metadata()) + return cls( + _config_from_kwargs( + mode="fill_replay", + backend="native_intrabar", + sizing="explicit_fills", + report_level=report_level, + metadata=metadata, + **kwargs, + ) + ) + @classmethod def dca_ladder(cls, **kwargs) -> "QuantBTEndpoint": """ @@ -931,6 +1167,11 @@ def backtest( instruments: Optional[Union[OptionInstrumentRegistry, Sequence[OptionInstrumentSpec], Dict[str, OptionInstrumentSpec]]] = None, packages: Optional[Sequence[OptionPackageIntent]] = None, strategy_run: Optional[OptionStrategyRun] = None, + intent: Optional[IntrabarIntentTape] = None, + intent_cols: Optional[Dict[str, str]] = None, + funding_event_timestamps=None, + funding_event_rates=None, + fill_replay: Optional[Union[FillReplayTape, pd.DataFrame]] = None, underlying: Optional[Union[pd.DataFrame, pd.Series]] = None, hedge_policy: Optional[OptionHedgeConfig] = None, net_option_delta: Optional[pd.Series] = None, @@ -1005,6 +1246,37 @@ def backtest( datetime_index=datetime_index, symbols=symbols, ) + if mode == "intrabar_bracket_reference": + return self._run_intrabar_bracket_reference( + data=data, + signal=signal, + signal_col=signal_col, + datetime_index=datetime_index, + symbols=symbols, + intent=intent, + intent_cols=intent_cols, + funding_event_timestamps=funding_event_timestamps, + funding_event_rates=funding_event_rates, + ) + if mode == "intrabar_bracket": + return self._run_intrabar_bracket_fast( + data=data, + signal=signal, + signal_col=signal_col, + datetime_index=datetime_index, + symbols=symbols, + intent=intent, + intent_cols=intent_cols, + funding_event_timestamps=funding_event_timestamps, + funding_event_rates=funding_event_rates, + ) + if mode == "fill_replay": + return self._run_fill_replay( + data=data, + datetime_index=datetime_index, + symbols=symbols, + fill_replay=fill_replay, + ) if mode in ("single_signal", "pct_equity", "signal_notional", "dca_ladder", "nautilus_validation"): return self._run_single(data=data, signal=signal, signal_col=signal_col, datetime_index=datetime_index, symbols=symbols) if mode == "orders": @@ -1249,6 +1521,234 @@ def _run_options( self._store_result(self.engine.result) return self.result + def _run_intrabar_bracket_reference(self, data, signal, signal_col, datetime_index, symbols, intent, intent_cols, funding_event_timestamps=None, funding_event_rates=None): + tape, intent, symbol = self._prepare_intrabar_run(data, signal, signal_col, datetime_index, symbols, intent, intent_cols, funding_event_timestamps, funding_event_rates) + contract = _execution_contract_from_config(self.config) + oracle = run_intrabar_reference( + tape=tape, + intent=intent, + account=self.config.account, + contract=contract, + fee_rate=self.config.v2_fee_rate, + slippage_rate=float(self.config.execution.slippage_rate), + contract_size=_scalar_for_symbol(self.config.contract_size, symbol), + **self._intrabar_execution_kwargs(symbol), + ) + idx = oracle.equity.index + returns = oracle.equity.pct_change().replace([np.inf, -np.inf], np.nan).fillna(0.0) + diagnostics = pd.DataFrame( + { + "average_entry": oracle.average_entry, + "active_stop": oracle.active_stop, + "active_take_profit": oracle.active_take_profit, + "event_flags": oracle.event_flags, + "fees": oracle.fees, + "funding": oracle.funding, + }, + index=idx, + ) + metadata = { + **oracle.metadata, + "backend": "intrabar_reference", + "backend_alias": "intrabar_bracket_reference", + "engine_id": "intrabar_reference_v1", + "input_mode": "intrabar_intent", + "symbol": symbol, + "validation_certificate": asdict(tape.validation_certificate), + "strict_market_tape": True, + "phase": "31B_python_reference_oracle", + "fills_report": _intrabar_fills_to_frame(oracle.fills), + "positions_report": pd.DataFrame({f"Position_{symbol}": oracle.position}, index=idx), + } + result = BacktestResultV2( + equity=oracle.equity, + returns=returns, + positions=pd.DataFrame({f"Position_{symbol}": oracle.position.to_numpy(dtype=float)}, index=idx), + closes=pd.DataFrame({f"Close_{symbol}": tape.closes[:, 0]}, index=idx), + symbols=[symbol], + initial_capital=float(self.config.account.initial_capital), + leverage=float(self.config.account.leverage), + fills=oracle.fills, + fees=oracle.fees, + funding=oracle.funding, + diagnostics=diagnostics, + metadata=metadata, + ) + self.engine = oracle + self._store_result(result) + return self.result + + def _run_intrabar_bracket_fast(self, data, signal, signal_col, datetime_index, symbols, intent, intent_cols, funding_event_timestamps=None, funding_event_rates=None): + tape, intent, symbol = self._prepare_intrabar_run(data, signal, signal_col, datetime_index, symbols, intent, intent_cols, funding_event_timestamps, funding_event_rates) + contract = _execution_contract_from_config(self.config) + kernel = run_intrabar_kernel( + tape=tape, + intent=intent, + account=self.config.account, + contract=contract, + fee_rate=self.config.v2_fee_rate, + slippage_rate=float(self.config.execution.slippage_rate), + contract_size=_scalar_for_symbol(self.config.contract_size, symbol), + **self._intrabar_execution_kwargs(symbol), + report_level=self.config.report_level, + ) + idx = kernel.equity.index + returns = kernel.equity.pct_change().replace([np.inf, -np.inf], np.nan).fillna(0.0) + diagnostics = pd.DataFrame( + { + "average_entry": kernel.average_entry, + "active_stop": kernel.active_stop, + "active_take_profit": kernel.active_take_profit, + "event_flags": kernel.event_flags, + "initial_margin": kernel.initial_margin, + "maintenance_margin": kernel.maintenance_margin, + "fees": kernel.fees, + "funding": kernel.funding, + }, + index=idx, + ) + metadata = { + **kernel.metadata, + "input_mode": "intrabar_intent", + "symbol": symbol, + "phase": "31C_numba_intrabar_kernel", + "fills_report": kernel.fills_report, + "positions_report": pd.DataFrame({f"Position_{symbol}": kernel.position}, index=idx), + } + result = BacktestResultV2( + equity=kernel.equity, + returns=returns, + positions=pd.DataFrame({f"Position_{symbol}": kernel.position.to_numpy(dtype=float)}, index=idx), + closes=pd.DataFrame({f"Close_{symbol}": tape.closes[:, 0]}, index=idx), + symbols=[symbol], + initial_capital=float(self.config.account.initial_capital), + leverage=float(self.config.account.leverage), + liquidated=bool(kernel.liquidated), + liquidation_bar=int(kernel.liquidation_bar), + fills=kernel.fills, + fees=kernel.fees, + funding=kernel.funding, + margin=diagnostics[["initial_margin", "maintenance_margin"]], + diagnostics=diagnostics, + metadata=metadata, + ) + self.engine = kernel + self._store_result(result) + return self.result + + def _run_fill_replay(self, data, datetime_index, symbols, fill_replay): + if fill_replay is None: + raise ValueError("fill_replay endpoint requires fill_replay=FillReplayTape or DataFrame") + symbol_list = list(symbols or self.config.symbols or ["DEFAULT"]) + if len(symbol_list) != 1: + raise ValueError("fill_replay currently supports exactly one symbol") + symbol = symbol_list[0] + tape = prepare_market_tape( + data=data, + datetime_index=datetime_index, + symbols=symbol_list, + funding_rate=self.config.funding_rate, + use_funding=False, + validation_mode="strict", + source_timezone=self.config.metadata.get("source_timezone"), + bar_timestamp_semantics=str(self.config.metadata.get("bar_timestamp_semantics", "close")), + ) + if isinstance(fill_replay, FillReplayTape): + fill_tape = fill_replay + elif isinstance(fill_replay, pd.DataFrame): + fill_tape = FillReplayTape.from_frame( + fill_replay, + fee_rate=self.config.v2_fee_rate, + contract_size=_scalar_for_symbol(self.config.contract_size, symbol), + ) + else: + raise TypeError("fill_replay must be a FillReplayTape or pandas DataFrame") + replay = run_fill_replay_kernel( + tape=tape, + fill_tape=fill_tape, + account=self.config.account, + contract_size=_scalar_for_symbol(self.config.contract_size, symbol), + ) + idx = replay.equity.index + returns = replay.equity.pct_change().replace([np.inf, -np.inf], np.nan).fillna(0.0) + metadata = { + **replay.metadata, + "symbol": symbol, + "phase": "31C_fill_replay_kernel", + "fills_report": fill_replay.copy() if isinstance(fill_replay, pd.DataFrame) else pd.DataFrame(), + } + result = BacktestResultV2( + equity=replay.equity, + returns=returns, + positions=pd.DataFrame({f"Position_{symbol}": replay.position.to_numpy(dtype=float)}, index=idx), + closes=pd.DataFrame({f"Close_{symbol}": tape.closes[:, 0]}, index=idx), + symbols=[symbol], + initial_capital=float(self.config.account.initial_capital), + leverage=float(self.config.account.leverage), + fees=replay.fees, + diagnostics=pd.DataFrame({"event_flags": replay.event_flags, "fees": replay.fees}, index=idx), + metadata=metadata, + ) + self.engine = replay + self._store_result(result) + return self.result + + def _prepare_intrabar_run(self, data, signal, signal_col, datetime_index, symbols, intent, intent_cols, funding_event_timestamps=None, funding_event_rates=None): + symbol_list = list(symbols or self.config.symbols or ["DEFAULT"]) + if len(symbol_list) != 1: + raise ValueError(f"{self.config.mode} currently supports exactly one symbol") + symbol = symbol_list[0] + tape = prepare_market_tape( + data=data, + datetime_index=datetime_index, + symbols=symbol_list, + funding_rate=self.config.funding_rate, + funding_event_timestamps=funding_event_timestamps, + funding_event_rates=funding_event_rates, + use_funding=self.config.use_funding, + validation_mode="strict", + missing_funding_policy=str(self.config.metadata.get("missing_funding_policy", "raise")), + source_timezone=self.config.metadata.get("source_timezone"), + bar_timestamp_semantics=str(self.config.metadata.get("bar_timestamp_semantics", "close")), + ) + lookup_frame = None if isinstance(data, PreparedMarketTape) else _strict_lookup_frame(data, datetime_index, source_timezone=self.config.metadata.get("source_timezone")) + if intent is None: + level_mode = IntrabarLevelMode(str(self.config.metadata.get("intrabar_level_mode", IntrabarLevelMode.PERCENT_DISTANCE.value))) + intent = _intrabar_intent_from_endpoint_input( + frame=lookup_frame, + index=pd.DatetimeIndex(pd.to_datetime(tape.timestamps_ns, utc=True)), + signal=signal, + signal_col=signal_col, + intent_cols=intent_cols or {}, + level_mode=level_mode, + ) + return tape, intent, symbol + + def _intrabar_execution_kwargs(self, symbol: str) -> Dict: + constraints = build_quantity_constraints( + [symbol], + instruments=self.config.instruments, + qty_step=self.config.qty_step, + lot_size=self.config.lot_size, + slot_size=self.config.slot_size, + min_qty=self.config.min_qty, + min_notional=self.config.min_notional, + ) + sizing_mode = IntrabarSizingMode(str(self.config.metadata.get("intrabar_sizing_mode", IntrabarSizingMode.UNITS.value))) + fixed_notional = float(self.config.metadata.get("fixed_notional", self.config.alloc_per_trade if not isinstance(self.config.alloc_per_trade, dict) else self.config.alloc_per_trade.get(symbol, 0.0))) + equity_fraction = float(self.config.metadata.get("equity_fraction", self.config.alloc_per_trade if not isinstance(self.config.alloc_per_trade, dict) else self.config.alloc_per_trade.get(symbol, 0.0))) + risk_fraction = float(self.config.metadata.get("risk_fraction", 0.0)) + return { + "sizing_mode": sizing_mode, + "fixed_notional": fixed_notional, + "equity_fraction": equity_fraction, + "risk_fraction": risk_fraction, + "qty_step": float(constraints.qty_step[0]), + "min_qty": float(constraints.min_qty[0]), + "min_notional": float(constraints.min_notional[0]), + "tick_size": _tick_size_for_symbol(self.config.instruments, symbol, self.config.metadata.get("tick_size", 0.0)), + } + def _run_single(self, data, signal, signal_col, datetime_index, symbols): frame, idx, sig = _normalize_single_data(data=data, signal=signal, signal_col=signal_col, datetime_index=datetime_index) backend = _resolve_backend(self.config) @@ -1305,6 +1805,16 @@ def _run_single(self, data, signal, signal_col, datetime_index, symbols): min_qty=self.config.min_qty, min_notional=self.config.min_notional, ) + markers = _intrabar_marker_columns(frame) + if backend == "native_vectorized" and markers: + warnings.warn( + "native_vectorized is close_target_v2 and does not certify intrabar SL/TP/trailing columns " + f"{markers}; use a future intrabar/fill-replay/event backend for those semantics.", + RuntimeWarning, + stacklevel=2, + ) + self.engine.result.metadata["intrabar_misuse_markers"] = markers + self.engine.result.metadata["certification_status"] = "uncertified_intrabar_columns_on_close_target" self._store_result(self.engine.result) return self.result @@ -2068,6 +2578,20 @@ def _normalize_result_contract(result) -> None: metadata["orders_count"] = len(getattr(result, "orders", ())) if "fills_count" not in metadata: metadata["fills_count"] = len(getattr(result, "fills", ())) + engine = str(metadata.get("engine", "unknown")) + backend = str(metadata.get("backend", metadata.get("backend_alias", "unknown"))) + metadata.setdefault("backend_alias", backend) + metadata.setdefault("engine_id", engine) + metadata.setdefault("kernel_version", engine) + metadata.setdefault( + "execution_contract", + { + "engine_id": metadata["engine_id"], + "signal_phase": metadata.get("signal_phase", "unspecified"), + "fill_phase": metadata.get("fill_phase", "unspecified"), + "intrabar_exit_model": metadata.get("intrabar_exit_model", "unspecified"), + }, + ) def _attach_endpoint_run_config(result, config: EndpointConfig) -> None: @@ -2113,6 +2637,7 @@ def _sync_applied_nautilus_config(payload: Dict, metadata: Dict) -> None: def _endpoint_run_config_payload(config: EndpointConfig) -> Dict: + intrabar_mode = str(config.mode).lower().strip() in {"intrabar_bracket", "intrabar_bracket_reference", "fill_replay"} payload = { "mode": config.mode, "backend": config.backend, @@ -2121,7 +2646,7 @@ def _endpoint_run_config_payload(config: EndpointConfig) -> Dict: "account": _jsonable(asdict(config.account)), "execution": { **_jsonable(asdict(config.execution)), - "legacy_slippage_rate": float(config.slippage), + "legacy_slippage_rate": None if intrabar_mode else float(config.slippage), "slippage_bps": float(config.execution.slippage_bps), }, "fees": { @@ -2253,6 +2778,16 @@ def _fmt_int(value) -> str: def _config_from_kwargs(**kwargs) -> EndpointConfig: + mode_name = str(kwargs.get("mode", "")).lower().strip() + metadata = dict(kwargs.pop("metadata", {}) or {}) + if "tick_size" in kwargs: + metadata.setdefault("tick_size", kwargs.pop("tick_size")) + if "source_timezone" in kwargs: + metadata.setdefault("source_timezone", kwargs.pop("source_timezone")) + if "missing_funding_policy" in kwargs: + metadata.setdefault("missing_funding_policy", kwargs.pop("missing_funding_policy")) + if "bar_timestamp_semantics" in kwargs: + metadata.setdefault("bar_timestamp_semantics", kwargs.pop("bar_timestamp_semantics")) hedge_type_alias = kwargs.pop("hedge_type", None) if hedge_type_alias is not None and "sizing" not in kwargs: kwargs["sizing"] = hedge_type_alias @@ -2268,10 +2803,27 @@ def _config_from_kwargs(**kwargs) -> EndpointConfig: maintenance_ratio=0.005 if maintenance_ratio is None else float(maintenance_ratio), ) + legacy_slippage_supplied = "slippage" in kwargs + legacy_slippage_value = kwargs.get("slippage") slippage_bps = kwargs.pop("slippage_bps", None) execution = kwargs.pop("execution", None) + if slippage_bps is not None and execution is not None: + raise ValueError("pass either execution=ExecutionConfig(...) or slippage_bps=..., not both") + if slippage_bps is not None and legacy_slippage_supplied: + raise ValueError("pass either slippage_bps or legacy slippage, not both") if execution is None: - execution = ExecutionConfig(slippage_bps=0.0 if slippage_bps is None else float(slippage_bps)) + if slippage_bps is not None: + execution = ExecutionConfig(slippage_bps=float(slippage_bps)) + elif mode_name in {"intrabar_bracket", "intrabar_bracket_reference"} and legacy_slippage_supplied: + warnings.warn( + "QuantBT intrabar endpoints use slippage_bps as the source of truth; " + "legacy slippage was converted to slippage_bps for compatibility.", + DeprecationWarning, + stacklevel=3, + ) + execution = ExecutionConfig(slippage_bps=float(legacy_slippage_value) * 10_000.0) + else: + execution = ExecutionConfig(slippage_bps=0.0) dca_kwargs = kwargs.pop("dca_kwargs", {}) for key in ( @@ -2287,7 +2839,7 @@ def _config_from_kwargs(**kwargs) -> EndpointConfig: if key in kwargs: dca_kwargs[key] = kwargs.pop(key) - return EndpointConfig(account=account, execution=execution, dca_kwargs=dca_kwargs, **kwargs) + return EndpointConfig(account=account, execution=execution, dca_kwargs=dca_kwargs, metadata=metadata, **kwargs) def _pop_dataclass_kwargs(kwargs: Dict, dataclass_type) -> Dict: @@ -2876,6 +3428,171 @@ def _walkforward_scoring_config(config: EndpointConfig, target_mode: str) -> End raise NotImplementedError(f"endpoint scoring is not implemented for walk-forward target_mode={target_mode!r}") +def _strict_lookup_frame(data, datetime_index=None, *, source_timezone: Optional[str] = None) -> pd.DataFrame: + if not isinstance(data, pd.DataFrame): + raise ValueError("intrabar endpoint requires a DataFrame when intent is not supplied explicitly") + frame = data.copy().rename( + columns={ + "Datetime": "timestamp", + "Date": "timestamp", + "Timestamp": "timestamp", + "Open": "open", + "High": "high", + "Low": "low", + "Close": "close", + "Volume": "volume", + } + ) + if datetime_index is not None: + frame.index = _endpoint_strict_index(datetime_index, source_timezone=source_timezone) + elif "timestamp" in frame.columns: + frame = frame.set_index(_endpoint_strict_index(frame["timestamp"], source_timezone=source_timezone)) + else: + frame.index = _endpoint_strict_index(frame.index, source_timezone=source_timezone) + return frame + + +def _endpoint_strict_index(value, *, source_timezone: Optional[str] = None) -> pd.DatetimeIndex: + raw = pd.DatetimeIndex(pd.to_datetime(value, errors="raise")) + if raw.tz is None: + if source_timezone is None: + raise ValueError("intrabar endpoint received timezone-naive data; pass metadata={'source_timezone': ...}") + raw = raw.tz_localize(source_timezone) + return raw.tz_convert("UTC") + + +def _execution_contract_from_config(config: EndpointConfig) -> ExecutionContract: + meta = config.metadata.get("execution_contract") + if meta: + return ExecutionContract.from_metadata(meta) + contract_id = str(config.metadata.get("execution_contract_id", "intrabar_bracket_v1")) + if contract_id == "intrabar_bracket_v1": + return ExecutionContract.intrabar_bracket() + return ExecutionContract.from_metadata({"engine_id": contract_id}) + + +def _tick_size_for_symbol(instruments, symbol: str, default: float = 0.0) -> float: + if isinstance(default, dict): + fallback = float(default.get(symbol, 0.0)) + else: + fallback = float(default or 0.0) + if instruments is None: + return fallback + if isinstance(instruments, dict): + inst = instruments.get(symbol) + return fallback if inst is None else float(getattr(inst, "tick_size", fallback)) + for inst in instruments: + if getattr(inst, "symbol", None) == symbol: + return float(getattr(inst, "tick_size", fallback)) + return fallback + + +def _prepared_profile_signature(data_signature: str, profile: Dict) -> str: + payload = dict(profile) + payload["data_signature"] = data_signature + raw = json.dumps(_jsonable(payload), sort_keys=True, default=str).encode("utf-8") + return hashlib.sha256(raw).hexdigest() + + +def _intrabar_intent_from_endpoint_input( + *, + frame: Optional[pd.DataFrame], + index: pd.DatetimeIndex, + signal, + signal_col: Optional[str], + intent_cols: Dict[str, str], + level_mode: IntrabarLevelMode, +) -> IntrabarIntentTape: + signed_signal = None + if signal is not None: + signed_signal = _strict_series_values(signal, index, name="signal") + elif signal_col is not None: + signed_signal = _strict_frame_col_values(frame, index, signal_col, dtype=float) + elif "entry_signal" in intent_cols: + signed_signal = _strict_frame_col_values(frame, index, intent_cols["entry_signal"], dtype=float) + elif "signal" in intent_cols: + signed_signal = _strict_frame_col_values(frame, index, intent_cols["signal"], dtype=float) + + if "entry_side" in intent_cols: + entry_side = np.sign(_strict_frame_col_values(frame, index, intent_cols["entry_side"], dtype=float)).astype(np.int8) + elif signed_signal is not None: + entry_side = np.sign(signed_signal).astype(np.int8) + else: + raise ValueError("intrabar endpoint requires signal/signal_col or intent_cols['entry_side']") + + if "entry_size" in intent_cols: + entry_size = np.abs(_strict_frame_col_values(frame, index, intent_cols["entry_size"], dtype=float)) + elif signed_signal is not None: + entry_size = np.abs(signed_signal) + else: + raise ValueError("intrabar endpoint requires intent_cols['entry_size'] when no signed signal is supplied") + + return IntrabarIntentTape.from_arrays( + entry_side=entry_side, + entry_size=entry_size, + stop_value=_optional_intent_col(frame, index, intent_cols, "stop_value"), + take_profit_value=_optional_intent_col(frame, index, intent_cols, "take_profit_value"), + trailing_value=_optional_intent_col(frame, index, intent_cols, "trailing_value"), + technical_exit=_optional_intent_col(frame, index, intent_cols, "technical_exit", dtype=bool), + exit_long=_optional_intent_col(frame, index, intent_cols, "exit_long", dtype=bool), + exit_short=_optional_intent_col(frame, index, intent_cols, "exit_short", dtype=bool), + level_mode=level_mode, + ) + + +def _strict_series_values(series, index: pd.DatetimeIndex, *, name: str) -> np.ndarray: + if not isinstance(series, pd.Series): + series = pd.Series(series, index=index) + s = series.copy() + s.index = pd.DatetimeIndex(pd.to_datetime(s.index, errors="raise", utc=True)) + if not s.index.equals(index): + raise ValueError(f"{name} index must exactly match the strict market tape index") + return pd.to_numeric(s, errors="raise").to_numpy(dtype=np.float64) + + +def _strict_frame_col_values(frame: Optional[pd.DataFrame], index: pd.DatetimeIndex, col: str, *, dtype=float) -> np.ndarray: + if frame is None: + raise ValueError(f"intent column {col!r} requires DataFrame data") + if col not in frame.columns: + raise ValueError(f"intent column {col!r} not found in data") + if not pd.DatetimeIndex(frame.index).equals(index): + raise ValueError(f"intent column {col!r} index must exactly match the strict market tape index") + if dtype is bool: + return frame[col].fillna(False).astype(bool).to_numpy(dtype=np.bool_) + return pd.to_numeric(frame[col], errors="raise").to_numpy(dtype=np.float64) + + +def _optional_intent_col(frame: Optional[pd.DataFrame], index: pd.DatetimeIndex, cols: Dict[str, str], key: str, *, dtype=float): + col = cols.get(key) + if col is None: + return None + return _strict_frame_col_values(frame, index, col, dtype=dtype) + + +def _scalar_for_symbol(value, symbol: str, default: float = 1.0) -> float: + if isinstance(value, dict): + return float(value.get(symbol, default)) + return float(default if value is None else value) + + +def _intrabar_fills_to_frame(fills) -> pd.DataFrame: + rows = [] + for fill in fills: + rows.append( + { + "bar_index": int(fill.bar_index), + "sequence": int(fill.sequence), + "timestamp": pd.Timestamp(fill.timestamp), + "side": int(fill.side), + "qty": float(fill.qty), + "price": float(fill.price), + "fee": float(fill.fee), + "reason": fill.reason.value if hasattr(fill.reason, "value") else str(fill.reason), + } + ) + return pd.DataFrame(rows) + + def _slice_wf_data_to_index(data, index: pd.DatetimeIndex): if isinstance(data, pd.DataFrame): return data.reindex(index).copy() @@ -2906,6 +3623,31 @@ def _normalize_single_data(data, signal, signal_col, datetime_index): return frame, frame.index, sig +def _intrabar_marker_columns(frame: pd.DataFrame) -> list[str]: + markers = { + "exit_price", + "exit_type", + "stop_loss", + "stoploss", + "sl", + "take_profit", + "takeprofit", + "tp", + "trailing", + "trailing_stop", + "use_sl", + "use_tp", + "slpercent", + "tppercent", + } + found = [] + for col in frame.columns: + key = str(col).lower() + if key in markers or "trailing" in key or "stop_loss" in key or "take_profit" in key: + found.append(str(col)) + return found + + def _normalize_symbol_data(data, closes, highs, lows, datetime_index, symbols): if closes is not None: symbol_list = list(symbols or closes.keys()) diff --git a/engines.py b/engines.py index 9c08136..6603039 100644 --- a/engines.py +++ b/engines.py @@ -209,12 +209,20 @@ def _run_native_event(self) -> BacktestResultV2: ) if self.strategy is not None: + opens, volumes = _market_open_volume( + data=self.data, + datetime_index=idx, + closes=closes, + symbols=symbols, + ) return backend.run_strategy( datetime_index=idx, strategy=self.strategy, closes=closes, highs=highs, lows=lows, + opens=opens, + volumes=volumes, funding_rate=self.funding_rate, contract_size=self.contract_size, leverage=self.leverage, @@ -729,6 +737,40 @@ def _market_data( return idx, close_map, high_map, low_map, symbol_list +def _market_open_volume( + data: Optional[Union[pd.DataFrame, Dict[str, Union[pd.DataFrame, pd.Series]]]], + datetime_index: pd.DatetimeIndex, + closes: SeriesMap, + symbols: List[str], +) -> Tuple[SeriesMap, SeriesMap]: + opens: SeriesMap = {} + volumes: SeriesMap = {} + if isinstance(data, pd.DataFrame): + if len(symbols) != 1: + raise ValueError("single DataFrame reactive run requires one symbol") + frame = _extract_frame_ohlcv(data, datetime_index) + opens[symbols[0]] = frame["open"] + volumes[symbols[0]] = frame["volume"] + return opens, volumes + if isinstance(data, dict): + for symbol in symbols: + value = data[symbol] + if isinstance(value, pd.DataFrame): + frame = _extract_frame_ohlcv(value, datetime_index) + opens[symbol] = frame["open"] + volumes[symbol] = frame["volume"] + else: + close = closes[symbol] + opens[symbol] = close + volumes[symbol] = pd.Series(0.0, index=close.index, name="volume") + return opens, volumes + for symbol in symbols: + close = closes[symbol] + opens[symbol] = close + volumes[symbol] = pd.Series(0.0, index=close.index, name="volume") + return opens, volumes + + def _extract_frame_ohlc( data: pd.DataFrame, datetime_index: Optional[Union[pd.DatetimeIndex, pd.Series]], diff --git a/tests/test_phase31_merge_blockers.py b/tests/test_phase31_merge_blockers.py new file mode 100644 index 0000000..5f10abb --- /dev/null +++ b/tests/test_phase31_merge_blockers.py @@ -0,0 +1,436 @@ +from dataclasses import replace + +import numpy as np +import pandas as pd +import pytest + +from quantbt import ( + AccountConfig, + ExecutionContract, + ExecutionConfig, + FillPhase, + FillReplayTape, + IntrabarEventFlag, + IntrabarFillReason, + IntrabarIntentTape, + IntrabarSizingMode, + QuantBTEndpoint, + IntrabarSameBarPolicy, + TakeProfitGapPolicy, + prepare_market_tape, + run_fill_replay_kernel, + run_intrabar_kernel, + run_intrabar_reference, +) + + +def _frame(rows, *, tz="UTC"): + idx = pd.date_range("2024-01-01", periods=len(rows), freq="1h", tz=tz) + return pd.DataFrame(rows, index=idx) + + +def test_phase31e_intrabar_uses_slippage_bps_as_source_of_truth(): + df = _frame( + [ + {"open": 100.0, "high": 100.0, "low": 100.0, "close": 100.0, "entry": 1.0}, + {"open": 100.0, "high": 101.0, "low": 99.0, "close": 100.0, "entry": 0.0}, + ] + ) + bt = QuantBTEndpoint.intrabar_bracket( + initial_capital=10_000.0, + execution=ExecutionConfig(slippage_bps=10.0), + fee_rate=0.0, + use_funding=False, + close_on_last_bar=False, + report_level="audit", + ) + + bt.backtest(data=df, signal_col="entry", symbols=["BTC"]) + + assert bt.fills_report.iloc[0]["price"] == pytest.approx(100.1) + assert bt.result.metadata["run_config"]["execution"]["slippage_bps"] == 10.0 + + +def test_phase31e_legacy_slippage_conflict_raises(): + with pytest.raises(ValueError, match="either slippage_bps or legacy slippage"): + QuantBTEndpoint.intrabar_bracket(slippage=0.0001, slippage_bps=1.0) + + +def test_phase31e_scalar_funding_rejected_unless_zero_policy_or_disabled(): + df = _frame( + [ + {"open": 100.0, "high": 100.0, "low": 100.0, "close": 100.0}, + {"open": 100.0, "high": 100.0, "low": 100.0, "close": 100.0}, + ] + ) + with pytest.raises(ValueError, match="strict funding"): + prepare_market_tape(data=df, symbols=["BTC"], funding_rate=0.0001, use_funding=True) + + tape = prepare_market_tape(data=df, symbols=["BTC"], funding_rate=0.0, use_funding=True, missing_funding_policy="zero") + assert not tape.funding_event_mask.any() + + +def test_phase31g_funding_event_requires_exact_bar_boundary_and_applies_there(): + df = _frame( + [ + {"open": 100.0, "high": 100.0, "low": 100.0, "close": 100.0}, + {"open": 100.0, "high": 100.0, "low": 100.0, "close": 100.0}, + {"open": 100.0, "high": 100.0, "low": 100.0, "close": 100.0}, + ] + ) + with pytest.raises(ValueError, match="align exactly"): + prepare_market_tape( + data=df, + symbols=["BTC"], + use_funding=True, + funding_event_timestamps=[pd.Timestamp("2024-01-01 00:30", tz="UTC")], + funding_event_rates=[0.001], + ) + + tape = prepare_market_tape( + data=df, + symbols=["BTC"], + use_funding=True, + funding_event_timestamps=[pd.Timestamp("2024-01-01 01:00", tz="UTC")], + funding_event_rates=[0.001], + ) + + assert tape.funding_event_mask.tolist() == [False, True, False] + assert tape.funding_rates[1, 0] == pytest.approx(0.001) + + +def test_phase31h_funding_timestamp_semantics_open_vs_close_and_kernel_parity(): + df = _frame( + [ + {"open": 100.0, "high": 100.0, "low": 100.0, "close": 100.0}, + {"open": 100.0, "high": 120.0, "low": 100.0, "close": 120.0}, + {"open": 200.0, "high": 200.0, "low": 200.0, "close": 200.0}, + ] + ) + funding_ts = [df.index[2]] + funding_rate = [0.01] + intent = IntrabarIntentTape.from_arrays( + entry_side=[1, 0, 0], + entry_size=[1.0, 0.0, 0.0], + exit_long=[False, True, False], + ) + account = AccountConfig(initial_capital=10_000.0) + contract = ExecutionContract.intrabar_bracket(close_on_last_bar=False) + + tape_open = prepare_market_tape( + data=df, + symbols=["BTC"], + use_funding=True, + funding_event_timestamps=funding_ts, + funding_event_rates=funding_rate, + bar_timestamp_semantics="open", + ) + tape_close = prepare_market_tape( + data=df, + symbols=["BTC"], + use_funding=True, + funding_event_timestamps=funding_ts, + funding_event_rates=funding_rate, + bar_timestamp_semantics="close", + ) + + open_oracle = run_intrabar_reference(tape=tape_open, intent=intent, account=account, contract=contract) + close_oracle = run_intrabar_reference(tape=tape_close, intent=intent, account=account, contract=contract) + open_kernel = run_intrabar_kernel(tape=tape_open, intent=intent, account=account, contract=contract, report_level="audit") + close_kernel = run_intrabar_kernel(tape=tape_close, intent=intent, account=account, contract=contract, report_level="audit") + + assert open_oracle.funding.iloc[2] == pytest.approx(2.0) + assert close_oracle.funding.iloc[2] == pytest.approx(0.0) + assert open_oracle.equity.iloc[-1] == pytest.approx(10_098.0) + assert close_oracle.equity.iloc[-1] == pytest.approx(10_100.0) + np.testing.assert_allclose(open_kernel.equity.to_numpy(), open_oracle.equity.to_numpy(), atol=1e-9, rtol=0.0) + np.testing.assert_allclose(close_kernel.equity.to_numpy(), close_oracle.equity.to_numpy(), atol=1e-9, rtol=0.0) + assert open_kernel.metadata["bar_timestamp_semantics"] == "open" + assert close_kernel.metadata["funding_event_price_reference"] == "close" + + +def test_phase31h_endpoint_propagates_bar_timestamp_semantics_to_intrabar_tape(): + df = _frame( + [ + {"open": 100.0, "high": 100.0, "low": 100.0, "close": 100.0, "entry": 1.0, "exit": False}, + {"open": 100.0, "high": 120.0, "low": 100.0, "close": 120.0, "entry": 0.0, "exit": True}, + {"open": 200.0, "high": 200.0, "low": 200.0, "close": 200.0, "entry": 0.0, "exit": True}, + ] + ) + bt = QuantBTEndpoint.intrabar_bracket( + initial_capital=10_000.0, + fee_rate=0.0, + use_funding=True, + bar_timestamp_semantics="open", + close_on_last_bar=False, + report_level="audit", + ) + + result = bt.backtest( + data=df, + signal_col="entry", + symbols=["BTC"], + intent_cols={"exit_long": "exit"}, + funding_event_timestamps=[df.index[2]], + funding_event_rates=[0.01], + ) + + assert result.metadata["validation_certificate"]["bar_timestamp_semantics"] == "open" + assert result.metadata["bar_timestamp_semantics"] == "open" + assert result.funding.iloc[2] == pytest.approx(2.0) + + +def test_phase31e_dynamic_trailing_uses_value_at_t_not_t_minus_1(): + df = _frame( + [ + {"open": 100.0, "high": 100.0, "low": 100.0, "close": 100.0}, + {"open": 100.0, "high": 111.0, "low": 99.0, "close": 110.0}, + {"open": 110.0, "high": 111.0, "low": 100.0, "close": 108.0}, + ] + ) + tape = prepare_market_tape(data=df, symbols=["BTC"], use_funding=False) + intent = IntrabarIntentTape.from_arrays( + entry_side=[1, 0, 0], + entry_size=[1.0, 0.0, 0.0], + trailing_value=[0.20, 0.05, 0.05], + ) + + result = run_intrabar_reference(tape=tape, intent=intent, account=AccountConfig(initial_capital=10_000.0)) + kernel = run_intrabar_kernel(tape=tape, intent=intent, account=AccountConfig(initial_capital=10_000.0), report_level="audit") + + assert result.fills[1].reason is IntrabarFillReason.STOP_LOSS + assert result.fills[1].price == pytest.approx(104.5) + np.testing.assert_allclose(kernel.equity.to_numpy(), result.equity.to_numpy(), atol=1e-9, rtol=0.0) + + +def test_phase31e_fixed_notional_pct_equity_risk_sizing_and_qty_filters(): + df = _frame( + [ + {"open": 100.0, "high": 100.0, "low": 100.0, "close": 100.0}, + {"open": 100.0, "high": 102.0, "low": 98.0, "close": 101.0}, + ] + ) + tape = prepare_market_tape(data=df, symbols=["BTC"], use_funding=False) + intent = IntrabarIntentTape.from_arrays(entry_side=[1, 0], entry_size=[1.0, 0.0], stop_value=[0.05, np.nan]) + account = AccountConfig(initial_capital=10_000.0, leverage=10.0) + + fixed = run_intrabar_kernel( + tape=tape, + intent=intent, + account=account, + contract=ExecutionContract.intrabar_bracket(close_on_last_bar=False), + sizing_mode=IntrabarSizingMode.FIXED_NOTIONAL, + fixed_notional=1_000.0, + qty_step=0.25, + report_level="audit", + ) + assert fixed.fills[0].qty == pytest.approx(10.0) + + pct = run_intrabar_kernel( + tape=tape, + intent=intent, + account=account, + contract=ExecutionContract.intrabar_bracket(close_on_last_bar=False), + sizing_mode=IntrabarSizingMode.PCT_EQUITY, + equity_fraction=0.10, + qty_step=0.25, + report_level="audit", + ) + assert pct.fills[0].qty == pytest.approx(10.0) + + risk = run_intrabar_kernel( + tape=tape, + intent=intent, + account=account, + contract=ExecutionContract.intrabar_bracket(close_on_last_bar=False), + sizing_mode=IntrabarSizingMode.RISK_PER_TRADE, + risk_fraction=0.01, + qty_step=0.5, + report_level="audit", + ) + assert risk.fills[0].qty == pytest.approx(20.0) + + rejected = run_intrabar_kernel( + tape=tape, + intent=intent, + account=account, + contract=ExecutionContract.intrabar_bracket(close_on_last_bar=False), + sizing_mode=IntrabarSizingMode.FIXED_NOTIONAL, + fixed_notional=1.0, + min_notional=5.0, + report_level="audit", + ) + assert rejected.rejected_count == 1 + assert rejected.fill_count == 0 + + +def test_phase31e_exit_long_short_are_side_specific_and_same_side_entry_is_exit_only(): + df = _frame( + [ + {"open": 100.0, "high": 100.0, "low": 100.0, "close": 100.0}, + {"open": 100.0, "high": 102.0, "low": 99.0, "close": 101.0}, + {"open": 101.0, "high": 102.0, "low": 100.0, "close": 101.0}, + ] + ) + tape = prepare_market_tape(data=df, symbols=["BTC"], use_funding=False) + intent = IntrabarIntentTape.from_arrays( + entry_side=[1, 1, 0], + entry_size=[1.0, 1.0, 0.0], + exit_long=[False, True, False], + exit_short=[True, False, False], + ) + + result = run_intrabar_kernel( + tape=tape, + intent=intent, + account=AccountConfig(initial_capital=10_000.0), + contract=ExecutionContract.intrabar_bracket(close_on_last_bar=False), + report_level="audit", + ) + + assert [fill.reason for fill in result.fills] == [IntrabarFillReason.ENTRY, IntrabarFillReason.TECHNICAL_EXIT] + assert result.position.iloc[-1] == 0.0 + assert result.rejected_count == 0 + assert int(result.event_flags.iloc[2]) & int(IntrabarEventFlag.ENTRY_SUPPRESSED) + + +def test_phase31e_strict_timezone_rejects_naive_and_localizes_source_timezone(): + naive = pd.DataFrame( + [{"open": 1.0, "high": 1.0, "low": 1.0, "close": 1.0}], + index=pd.date_range("2024-01-01", periods=1, freq="1h"), + ) + with pytest.raises(ValueError, match="timezone-naive"): + prepare_market_tape(data=naive, symbols=["BTC"], use_funding=False) + + tape = prepare_market_tape(data=naive, symbols=["BTC"], use_funding=False, source_timezone="Asia/Ho_Chi_Minh") + ts = pd.Timestamp(tape.timestamps_ns[0], tz="UTC") + assert ts == pd.Timestamp("2023-12-31 17:00", tz="UTC") + + +def test_phase31e_unsupported_contract_field_raises(): + df = _frame( + [ + {"open": 100.0, "high": 100.0, "low": 100.0, "close": 100.0}, + {"open": 100.0, "high": 101.0, "low": 99.0, "close": 100.0}, + ] + ) + tape = prepare_market_tape(data=df, symbols=["BTC"], use_funding=False) + intent = IntrabarIntentTape.from_arrays(entry_side=[1, 0], entry_size=[1.0, 0.0]) + bad = replace(ExecutionContract.intrabar_bracket(), entry_fill_phase=FillPhase.SAME_CLOSE) + + with pytest.raises(NotImplementedError, match="entry_fill_phase"): + run_intrabar_kernel(tape=tape, intent=intent, account=AccountConfig(initial_capital=10_000.0), contract=bad) + + +def test_phase31g_endpoint_preserves_full_execution_contract_policies(): + df = _frame( + [ + {"open": 100.0, "high": 100.0, "low": 100.0, "close": 100.0, "entry": 1.0, "tp": 0.05}, + {"open": 100.0, "high": 106.0, "low": 99.0, "close": 105.0, "entry": 0.0, "tp": np.nan}, + ] + ) + contract = ExecutionContract.intrabar_bracket( + same_bar_policy=IntrabarSameBarPolicy.TP_FIRST, + take_profit_gap_policy=TakeProfitGapPolicy.OPEN_PRICE_IMPROVEMENT, + close_on_last_bar=False, + ) + bt = QuantBTEndpoint.intrabar_bracket( + execution_contract=contract, + initial_capital=10_000.0, + fee_rate=0.0, + slippage_bps=0.0, + use_funding=False, + report_level="audit", + ) + + result = bt.backtest(data=df, signal_col="entry", symbols=["BTC"], intent_cols={"take_profit_value": "tp"}) + restored = ExecutionContract.from_metadata(result.metadata["execution_contract"]) + + assert restored.same_bar_policy is IntrabarSameBarPolicy.TP_FIRST + assert restored.take_profit_gap_policy is TakeProfitGapPolicy.OPEN_PRICE_IMPROVEMENT + assert restored.close_on_last_bar is False + + +def test_phase31e_fill_replay_certification_is_granular(): + df = _frame( + [ + {"open": 100.0, "high": 101.0, "low": 99.0, "close": 100.0}, + {"open": 100.0, "high": 103.0, "low": 99.0, "close": 102.0}, + ] + ) + tape = prepare_market_tape(data=df, symbols=["BTC"], use_funding=False) + fills = FillReplayTape.from_frame(pd.DataFrame([{"bar_index": 1, "side": 1, "qty": 1.0, "price": 100.0, "fee": 0.0}])) + + result = run_fill_replay_kernel(tape=tape, fill_tape=fills, account=AccountConfig(initial_capital=10_000.0)) + + assert result.metadata["price_accounting_certified"] is True + assert result.metadata["fee_accounting_certified"] is True + assert result.metadata["funding_certified"] is False + assert result.metadata["margin_certified"] is False + assert result.metadata["execution_generation_certified"] is False + + +def test_phase31f_prepared_intrabar_runner_matches_normal_endpoint_and_freezes_profile(): + df = _frame( + [ + {"open": 100.0, "high": 100.0, "low": 100.0, "close": 100.0, "entry": 1.0, "sl": 0.05}, + {"open": 100.0, "high": 101.0, "low": 94.0, "close": 98.0, "entry": 0.0, "sl": np.nan}, + {"open": 98.0, "high": 99.0, "low": 97.0, "close": 98.0, "entry": 0.0, "sl": np.nan}, + ] + ) + bt = QuantBTEndpoint.intrabar_bracket(initial_capital=10_000.0, fee_rate=0.0, use_funding=False, report_level="audit") + normal = bt.backtest(data=df, signal_col="entry", symbols=["BTC"], intent_cols={"stop_value": "sl"}) + runner = bt.prepare_intrabar(data=df, symbols=["BTC"]) + intent = IntrabarIntentTape.from_arrays(entry_side=df["entry"].to_numpy(), entry_size=np.abs(df["entry"].to_numpy()), stop_value=df["sl"].to_numpy()) + prepared = runner.run(intent, report_level="audit") + + np.testing.assert_allclose(prepared.equity.to_numpy(), normal.equity.to_numpy(), atol=1e-9, rtol=0.0) + assert prepared.metadata["prepared_runner"] is True + assert prepared.metadata["profile_metadata"]["data_signature"] == normal.metadata["data_signature"] + assert "prepared_signature" in prepared.metadata["profile_metadata"] + + +def test_phase31g_data_signature_changes_with_volume_and_funding(): + base = _frame( + [ + {"open": 100.0, "high": 101.0, "low": 99.0, "close": 100.0, "volume": 1.0}, + {"open": 100.0, "high": 101.0, "low": 99.0, "close": 100.0, "volume": 1.0}, + ] + ) + changed_volume = base.copy() + changed_volume.iloc[1, changed_volume.columns.get_loc("volume")] = 2.0 + funding = pd.Series([0.0, 0.001], index=base.index) + no_funding = pd.Series([0.0, 0.0], index=base.index) + + sig_base = prepare_market_tape(data=base, symbols=["BTC"], use_funding=False).signature + sig_volume = prepare_market_tape(data=changed_volume, symbols=["BTC"], use_funding=False).signature + sig_funding = prepare_market_tape(data=base, symbols=["BTC"], use_funding=True, funding_rate=funding).signature + sig_no_funding = prepare_market_tape(data=base, symbols=["BTC"], use_funding=True, funding_rate=no_funding).signature + sig_open_semantics = prepare_market_tape(data=base, symbols=["BTC"], use_funding=False, bar_timestamp_semantics="open").signature + + assert sig_base != sig_volume + assert sig_funding != sig_no_funding + assert sig_base != sig_open_semantics + + +def test_phase31g_tick_size_quantizes_entry_stop_tp_and_trailing(): + df = _frame( + [ + {"open": 100.0, "high": 100.0, "low": 100.0, "close": 100.0, "entry": 1.0, "sl": 0.051, "trail": 0.033}, + {"open": 100.03, "high": 102.0, "low": 96.0, "close": 101.07, "entry": 0.0, "sl": np.nan, "trail": 0.033}, + {"open": 101.0, "high": 102.0, "low": 97.0, "close": 100.0, "entry": 0.0, "sl": np.nan, "trail": 0.033}, + ] + ) + bt = QuantBTEndpoint.intrabar_bracket( + initial_capital=10_000.0, + fee_rate=0.0, + slippage_bps=0.0, + use_funding=False, + tick_size=0.05, + report_level="audit", + ) + bt.backtest(data=df, signal_col="entry", symbols=["BTC"], intent_cols={"stop_value": "sl", "trailing_value": "trail"}) + + assert bt.fills_report.iloc[0]["price"] == pytest.approx(100.05) + ticks = bt.fills_report["price"].to_numpy(dtype=float) / 0.05 + np.testing.assert_allclose(ticks, np.round(ticks), atol=1e-9, rtol=0.0) diff --git a/tests/test_phase31a_execution_correctness_contract.py b/tests/test_phase31a_execution_correctness_contract.py new file mode 100644 index 0000000..4bc604d --- /dev/null +++ b/tests/test_phase31a_execution_correctness_contract.py @@ -0,0 +1,154 @@ +from __future__ import annotations + +import pandas as pd +import pytest + +from quantbt import ( + BacktestEngineV2, + ExecutionConfig, + FillPricePolicy, + NativeVectorizedBackend, + NativeVectorizedConfig, + OrderCommand, + OrderSide, + OrderType, + QuantBTEndpoint, + TimeInForce, +) +from quantbt.core.preprocessor import prepare_funding +from quantbt.core.schema import AccountConfig + + +def _ohlcv() -> pd.DataFrame: + idx = pd.date_range("2024-01-01", periods=4, freq="1h", tz="UTC") + return pd.DataFrame( + { + "open": [100.0, 101.0, 102.0, 103.0], + "high": [101.0, 102.0, 103.0, 104.0], + "low": [99.0, 100.0, 101.0, 102.0], + "close": [100.0, 101.0, 102.0, 103.0], + "volume": [10.0, 11.0, 12.0, 13.0], + }, + index=idx, + ) + + +def test_phase31a_native_vectorized_declares_close_target_contract_metadata(): + idx = pd.date_range("2024-01-01", periods=3, freq="1h", tz="UTC") + close = pd.Series([100.0, 101.0, 102.0], index=idx) + target = pd.Series([0.0, 1.0, 1.0], index=idx) + backend = NativeVectorizedBackend( + NativeVectorizedConfig( + account=AccountConfig(initial_capital=10_000.0, leverage=5.0), + use_funding=False, + ) + ) + + result = backend.run_target_units( + datetime_index=idx, + target_units={"BTC": target}, + closes={"BTC": close}, + highs={"BTC": close}, + lows={"BTC": close}, + ) + + assert result.metadata["engine"] == "close_target_v2" + assert result.metadata["engine_id"] == "close_target_v2" + assert result.metadata["kernel_version"] == "units_v2" + assert result.metadata["execution_contract"]["signal_phase"] == "bar_close" + assert result.metadata["execution_contract"]["fill_phase"] == "same_close" + assert result.metadata["intrabar_exit_model"] == "none" + assert result.metadata["first_bar_target_policy"].startswith("target_units[0]_not_executed") + assert result.metadata["data_signature"].length == 3 + + +def test_phase31a_funding_dict_missing_symbol_is_explicit_error(): + idx = pd.date_range("2024-01-01", periods=3, freq="1h", tz="UTC") + with pytest.raises(KeyError, match="missing symbol 'ETH'"): + prepare_funding({"BTC": 0.0}, ["BTC", "ETH"], idx) + + +def test_phase31a_native_vectorized_rejects_unsupported_execution_config(): + with pytest.raises(NotImplementedError, match="close_target_v2"): + NativeVectorizedConfig( + account=AccountConfig(initial_capital=10_000.0), + execution=ExecutionConfig(fill_price_policy=FillPricePolicy.NEXT_OPEN), + ) + + +def test_phase31a_missing_high_low_is_marked_uncertified_instead_of_silent(): + idx = pd.date_range("2024-01-01", periods=3, freq="1h", tz="UTC") + close = pd.Series([100.0, 101.0, 102.0], index=idx) + target = pd.Series([0.0, 1.0, 1.0], index=idx) + backend = NativeVectorizedBackend( + NativeVectorizedConfig( + account=AccountConfig(initial_capital=10_000.0, leverage=5.0), + use_funding=False, + ) + ) + + with pytest.warns(RuntimeWarning, match="missing high/low"): + result = backend.run_target_units( + datetime_index=idx, + target_units={"BTC": target}, + closes={"BTC": close}, + ) + + assert result.metadata["high_low_source"] == "close_fallback_uncertified_intrabar_risk" + + +def test_phase31a_endpoint_warns_when_close_target_receives_intrabar_columns(): + df = _ohlcv() + df["pos_weight"] = [0.0, 1.0, 1.0, 0.0] + df["exit_price"] = [0.0, 0.0, 99.0, 0.0] + endpoint = QuantBTEndpoint.signal_notional( + backend="native_vectorized", + initial_capital=10_000.0, + leverage=5.0, + use_funding=False, + alloc_per_trade=1_000.0, + ) + + with pytest.warns(RuntimeWarning, match="close_target_v2"): + result = endpoint.backtest(data=df, signal_col="pos_weight", symbols=["BTC"]) + + assert result.metadata["intrabar_misuse_markers"] == ["exit_price"] + assert result.metadata["certification_status"] == "uncertified_intrabar_columns_on_close_target" + + +def test_phase31a_reactive_event_context_receives_open_and_volume_from_facade(): + df = _ohlcv() + + class Strategy: + def __init__(self): + self.seen = [] + + def on_bar_close(self, context): + self.seen.append((context.bar_index, float(context.open[0]), float(context.volume[0]))) + if context.bar_index == 0: + return [ + OrderCommand( + timestamp=context.timestamp, + symbol="BTC", + side=OrderSide.BUY, + order_type=OrderType.MARKET, + qty=1.0, + tif=TimeInForce.IOC, + order_id="entry", + ) + ] + return [] + + strategy = Strategy() + BacktestEngineV2( + data=df, + signals=pd.Series(0.0, index=df.index), + backend="native_event", + account=AccountConfig(initial_capital=10_000.0, leverage=5.0), + strategy=strategy, + symbols=["BTC"], + use_funding=False, + ) + + assert strategy.seen[0] == (0, 100.0, 10.0) + assert strategy.seen[1] == (1, 101.0, 11.0) diff --git a/tests/test_phase31b_market_tape_intrabar_oracle.py b/tests/test_phase31b_market_tape_intrabar_oracle.py new file mode 100644 index 0000000..9b7313b --- /dev/null +++ b/tests/test_phase31b_market_tape_intrabar_oracle.py @@ -0,0 +1,194 @@ +from __future__ import annotations + +import numpy as np +import pandas as pd +import pytest + +from quantbt import ( + AccountConfig, + ExecutionContract, + IntrabarFillReason, + IntrabarIntentTape, + IntrabarLevelMode, + get_execution_contract, + prepare_market_tape, + run_intrabar_reference, + QuantBTEndpoint, +) + + +def _frame(rows) -> pd.DataFrame: + idx = pd.date_range("2024-01-01", periods=len(rows), freq="1h", tz="UTC") + return pd.DataFrame(rows, index=idx) + + +def test_phase31b_execution_contract_registry_exposes_core_contracts(): + contract = get_execution_contract("intrabar_bracket_v1") + + assert contract.engine_id == "intrabar_bracket_v1" + assert contract.signal_phase.value == "bar_close" + assert contract.entry_fill_phase.value == "next_open" + assert ExecutionContract.close_target().to_metadata()["engine_id"] == "close_target_v2" + + +def test_phase31b_prepare_market_tape_strict_certificate_and_immutable_arrays(): + df = _frame( + [ + {"open": 100.0, "high": 101.0, "low": 99.0, "close": 100.0, "volume": 10.0}, + {"open": 101.0, "high": 102.0, "low": 100.0, "close": 101.0, "volume": 11.0}, + ] + ) + + tape = prepare_market_tape(data=df, symbols=["BTC"], funding_rate=0.0, use_funding=False) + + assert tape.symbols == ("BTC",) + assert tape.validation_certificate.row_count == 2 + assert tape.validation_certificate.ohlc_ok is True + assert tape.opens.flags.writeable is False + with pytest.raises(ValueError): + tape.opens[0, 0] = 1.0 + + +def test_phase31b_prepare_market_tape_rejects_duplicate_unsorted_missing_and_invalid_ohlc(): + duplicate_idx = pd.DatetimeIndex( + [ + pd.Timestamp("2024-01-01 00:00", tz="UTC"), + pd.Timestamp("2024-01-01 00:00", tz="UTC"), + ] + ) + duplicate = pd.DataFrame( + {"open": [1.0, 1.0], "high": [1.0, 1.0], "low": [1.0, 1.0], "close": [1.0, 1.0]}, + index=duplicate_idx, + ) + with pytest.raises(ValueError, match="duplicate"): + prepare_market_tape(data=duplicate) + + missing = _frame([{"open": 1.0, "high": 1.0, "close": 1.0}]) + with pytest.raises(ValueError, match="missing"): + prepare_market_tape(data=missing) + + invalid = _frame([{"open": 100.0, "high": 99.0, "low": 98.0, "close": 100.0}]) + with pytest.raises(ValueError, match="invalid OHLCV"): + prepare_market_tape(data=invalid) + + +def test_phase31b_prepare_market_tape_funding_dict_requires_symbols(): + df = _frame( + [ + {"open": 1.0, "high": 1.0, "low": 1.0, "close": 1.0}, + {"open": 1.0, "high": 1.0, "low": 1.0, "close": 1.0}, + ] + ) + funding = pd.Series(0.0, index=df.index) + with pytest.raises(KeyError, match="ETH"): + prepare_market_tape(data={"BTC": df, "ETH": df}, funding_rate={"BTC": funding}) + + +def test_phase31b_intrabar_oracle_conservative_same_bar_stop_tp_conflict(): + df = _frame( + [ + {"open": 100.0, "high": 100.0, "low": 100.0, "close": 100.0}, + {"open": 100.0, "high": 110.0, "low": 94.0, "close": 100.0}, + {"open": 100.0, "high": 101.0, "low": 99.0, "close": 100.0}, + ] + ) + tape = prepare_market_tape(data=df, symbols=["BTC"], use_funding=False) + intent = IntrabarIntentTape.from_arrays( + entry_side=[1, 0, 0], + entry_size=[1.0, 0.0, 0.0], + stop_value=[0.05, np.nan, np.nan], + take_profit_value=[0.08, np.nan, np.nan], + level_mode=IntrabarLevelMode.PERCENT_DISTANCE, + ) + + result = run_intrabar_reference(tape=tape, intent=intent, account=AccountConfig(initial_capital=10_000.0)) + + assert [fill.reason for fill in result.fills] == [IntrabarFillReason.ENTRY, IntrabarFillReason.STOP_LOSS] + assert result.fills[1].price == 95.0 + assert result.ambiguity_count == 1 + assert result.position.iloc[-1] == 0.0 + assert result.equity.iloc[-1] == 9_995.0 + + +def test_phase31b_intrabar_oracle_trailing_update_is_next_bar_not_same_bar(): + df = _frame( + [ + {"open": 100.0, "high": 100.0, "low": 100.0, "close": 100.0}, + {"open": 100.0, "high": 111.0, "low": 99.0, "close": 110.0}, + {"open": 110.0, "high": 111.0, "low": 104.0, "close": 108.0}, + ] + ) + tape = prepare_market_tape(data=df, symbols=["BTC"], use_funding=False) + intent = IntrabarIntentTape.from_arrays( + entry_side=[1, 0, 0], + entry_size=[1.0, 0.0, 0.0], + stop_value=[0.10, np.nan, np.nan], + trailing_value=[0.05, 0.05, 0.05], + ) + + result = run_intrabar_reference(tape=tape, intent=intent, account=AccountConfig(initial_capital=10_000.0)) + + assert len(result.fills) == 2 + assert result.fills[1].bar_index == 2 + assert result.fills[1].price == 104.5 + assert result.equity.iloc[-1] == 10_004.5 + + +def test_phase31b_intrabar_oracle_reversal_is_two_legs_with_two_fees(): + df = _frame( + [ + {"open": 100.0, "high": 100.0, "low": 100.0, "close": 100.0}, + {"open": 100.0, "high": 101.0, "low": 99.0, "close": 100.0}, + {"open": 102.0, "high": 103.0, "low": 101.0, "close": 102.0}, + ] + ) + tape = prepare_market_tape(data=df, symbols=["BTC"], use_funding=False) + intent = IntrabarIntentTape.from_arrays( + entry_side=[1, -1, 0], + entry_size=[1.0, 2.0, 0.0], + ) + + result = run_intrabar_reference( + tape=tape, + intent=intent, + account=AccountConfig(initial_capital=10_000.0), + contract=ExecutionContract.intrabar_bracket(close_on_last_bar=False), + fee_rate=0.001, + ) + + assert [fill.reason for fill in result.fills[:3]] == [ + IntrabarFillReason.ENTRY, + IntrabarFillReason.REVERSAL_EXIT, + IntrabarFillReason.REVERSAL_ENTRY, + ] + assert result.fees.iloc[1] == pytest.approx(0.1) + assert result.fees.iloc[2] == pytest.approx(0.102 + 0.204) + assert result.position.iloc[-1] == -2.0 + + +def test_phase31b_endpoint_runs_intrabar_reference_with_compact_intent_cols(): + df = _frame( + [ + {"open": 100.0, "high": 100.0, "low": 100.0, "close": 100.0, "entry": 1.0, "sl_pct": 0.05, "tp_pct": 0.08}, + {"open": 100.0, "high": 110.0, "low": 94.0, "close": 100.0, "entry": 0.0, "sl_pct": np.nan, "tp_pct": np.nan}, + {"open": 100.0, "high": 101.0, "low": 99.0, "close": 100.0, "entry": 0.0, "sl_pct": np.nan, "tp_pct": np.nan}, + ] + ) + bt = QuantBTEndpoint.intrabar_bracket_reference( + initial_capital=10_000.0, + fee_rate=0.0, + slippage_bps=0.0, + use_funding=False, + ) + + result = bt.backtest( + data=df, + signal_col="entry", + symbols=["BTC"], + intent_cols={"stop_value": "sl_pct", "take_profit_value": "tp_pct"}, + ) + + assert result.metadata["engine_id"] == "intrabar_reference_v1" + assert result.metadata["validation_certificate"]["ohlc_ok"] is True + assert bt.fills_report["reason"].tolist() == ["entry", "stop_loss"] + assert bt.show_metrics()["final_equity"] == pytest.approx(9_995.0) diff --git a/tests/test_phase31c_intrabar_kernel.py b/tests/test_phase31c_intrabar_kernel.py new file mode 100644 index 0000000..fdf9629 --- /dev/null +++ b/tests/test_phase31c_intrabar_kernel.py @@ -0,0 +1,271 @@ +from __future__ import annotations + +import time + +import numpy as np +import pandas as pd +import pytest + +from quantbt import ( + AccountConfig, + ExecutionContract, + FillReplayTape, + IntrabarFillReason, + IntrabarIntentTape, + QuantBTEndpoint, + prepare_market_tape, + run_fill_replay_kernel, + run_intrabar_kernel, + run_intrabar_reference, +) + + +def _frame(rows) -> pd.DataFrame: + idx = pd.date_range("2024-01-01", periods=len(rows), freq="1h", tz="UTC") + return pd.DataFrame(rows, index=idx) + + +def _assert_kernel_matches_reference(df, intent, *, account=None, fee_rate=0.0, slippage_rate=0.0, close_on_last_bar=True): + tape = prepare_market_tape(data=df, symbols=["BTC"], use_funding=False) + account = account or AccountConfig(initial_capital=10_000.0, leverage=10.0) + contract = ExecutionContract.intrabar_bracket(close_on_last_bar=close_on_last_bar) + reference = run_intrabar_reference( + tape=tape, + intent=intent, + account=account, + contract=contract, + fee_rate=fee_rate, + slippage_rate=slippage_rate, + ) + kernel = run_intrabar_kernel( + tape=tape, + intent=intent, + account=account, + contract=contract, + fee_rate=fee_rate, + slippage_rate=slippage_rate, + report_level="audit", + ) + np.testing.assert_allclose(kernel.equity.to_numpy(), reference.equity.to_numpy(), atol=1e-9, rtol=0.0) + np.testing.assert_allclose(kernel.position.to_numpy(), reference.position.to_numpy(), atol=1e-9, rtol=0.0) + np.testing.assert_allclose(kernel.fees.to_numpy(), reference.fees.to_numpy(), atol=1e-9, rtol=0.0) + np.testing.assert_array_equal(kernel.event_flags.to_numpy(), reference.event_flags.to_numpy()) + assert [fill.reason for fill in kernel.fills] == [fill.reason for fill in reference.fills] + assert kernel.fill_count == len(reference.fills) + return reference, kernel + + +def test_phase31c_kernel_matches_oracle_same_bar_ambiguity_and_audit_fills(): + df = _frame( + [ + {"open": 100.0, "high": 100.0, "low": 100.0, "close": 100.0}, + {"open": 100.0, "high": 110.0, "low": 94.0, "close": 100.0}, + {"open": 100.0, "high": 101.0, "low": 99.0, "close": 100.0}, + ] + ) + intent = IntrabarIntentTape.from_arrays( + entry_side=[1, 0, 0], + entry_size=[1.0, 0.0, 0.0], + stop_value=[0.05, np.nan, np.nan], + take_profit_value=[0.08, np.nan, np.nan], + ) + + reference, kernel = _assert_kernel_matches_reference(df, intent) + + assert kernel.ambiguity_count == reference.ambiguity_count == 1 + assert kernel.fills_report["reason"].tolist() == ["entry", "stop_loss"] + assert kernel.metadata["two_pass_audit"] is True + + +def test_phase31c_kernel_slippage_is_marked_from_actual_fill_price(): + df = _frame( + [ + {"open": 100.0, "high": 100.0, "low": 100.0, "close": 100.0}, + {"open": 100.0, "high": 103.0, "low": 99.0, "close": 102.0}, + ] + ) + intent = IntrabarIntentTape.from_arrays(entry_side=[1, 0], entry_size=[1.0, 0.0]) + + reference, kernel = _assert_kernel_matches_reference( + df, + intent, + fee_rate=0.0, + slippage_rate=0.001, + close_on_last_bar=False, + ) + + assert reference.fills[0].price == pytest.approx(100.1) + assert kernel.equity.iloc[-1] == pytest.approx(10_001.9) + + +def test_phase31c_kernel_trailing_and_reversal_match_oracle(): + df = _frame( + [ + {"open": 100.0, "high": 100.0, "low": 100.0, "close": 100.0}, + {"open": 100.0, "high": 111.0, "low": 99.0, "close": 110.0}, + {"open": 110.0, "high": 112.0, "low": 104.0, "close": 108.0}, + {"open": 107.0, "high": 108.0, "low": 101.0, "close": 103.0}, + ] + ) + intent = IntrabarIntentTape.from_arrays( + entry_side=[1, -1, 0, 0], + entry_size=[1.0, 2.0, 0.0, 0.0], + trailing_value=[0.05, 0.05, 0.05, 0.05], + ) + + _reference, kernel = _assert_kernel_matches_reference(df, intent, fee_rate=0.001) + + reasons = [fill.reason for fill in kernel.fills] + assert IntrabarFillReason.REVERSAL_EXIT in reasons + assert IntrabarFillReason.REVERSAL_ENTRY in reasons + + +def test_phase31c_kernel_rejects_insufficient_initial_margin(): + df = _frame( + [ + {"open": 100.0, "high": 100.0, "low": 100.0, "close": 100.0}, + {"open": 100.0, "high": 101.0, "low": 99.0, "close": 100.0}, + ] + ) + tape = prepare_market_tape(data=df, symbols=["BTC"], use_funding=False) + intent = IntrabarIntentTape.from_arrays(entry_side=[1, 0], entry_size=[200.0, 0.0]) + + result = run_intrabar_kernel( + tape=tape, + intent=intent, + account=AccountConfig(initial_capital=1_000.0, leverage=1.0), + contract=ExecutionContract.intrabar_bracket(close_on_last_bar=False), + report_level="audit", + ) + + assert result.rejected_count == 1 + assert result.fill_count == 0 + assert result.position.iloc[-1] == 0.0 + + +def test_phase31c_kernel_liquidates_unprotected_intrabar_breach(): + df = _frame( + [ + {"open": 100.0, "high": 100.0, "low": 100.0, "close": 100.0}, + {"open": 100.0, "high": 101.0, "low": 40.0, "close": 80.0}, + {"open": 80.0, "high": 81.0, "low": 79.0, "close": 80.0}, + ] + ) + tape = prepare_market_tape(data=df, symbols=["BTC"], use_funding=False) + intent = IntrabarIntentTape.from_arrays(entry_side=[1, 0, 0], entry_size=[10.0, 0.0, 0.0]) + + result = run_intrabar_kernel( + tape=tape, + intent=intent, + account=AccountConfig(initial_capital=1_000.0, leverage=2.0, maintenance_ratio=1.5), + contract=ExecutionContract.intrabar_bracket(close_on_last_bar=False), + report_level="audit", + ) + + assert result.liquidated is True + assert result.liquidation_bar == 1 + assert result.equity.iloc[-1] == 0.0 + assert result.fills[-1].reason == IntrabarFillReason.LIQUIDATION + + +def test_phase31c_fill_replay_reconstructs_accounting_from_explicit_fills(): + df = _frame( + [ + {"open": 100.0, "high": 101.0, "low": 99.0, "close": 100.0}, + {"open": 100.0, "high": 103.0, "low": 99.0, "close": 102.0}, + {"open": 102.0, "high": 104.0, "low": 101.0, "close": 103.0}, + ] + ) + tape = prepare_market_tape(data=df, symbols=["BTC"], use_funding=False) + fills = FillReplayTape.from_frame( + pd.DataFrame( + [ + {"bar_index": 1, "sequence": 0, "side": 1, "qty": 1.0, "price": 100.0, "fee": 0.1, "reason": "entry"}, + {"bar_index": 2, "sequence": 0, "side": -1, "qty": 1.0, "price": 103.0, "fee": 0.103, "reason": "take_profit"}, + ] + ) + ) + + result = run_fill_replay_kernel( + tape=tape, + fill_tape=fills, + account=AccountConfig(initial_capital=10_000.0), + ) + + assert result.equity.iloc[-1] == pytest.approx(10_002.797) + assert result.position.iloc[-1] == 0.0 + assert result.metadata["accounting_certified"] is True + assert result.metadata["execution_generation_certified"] is False + + +def test_phase31c_fast_endpoint_supports_standard_and_audit_report_levels(): + df = _frame( + [ + {"open": 100.0, "high": 100.0, "low": 100.0, "close": 100.0, "entry": 1.0, "sl": 0.05}, + {"open": 100.0, "high": 101.0, "low": 94.0, "close": 98.0, "entry": 0.0, "sl": np.nan}, + {"open": 98.0, "high": 99.0, "low": 97.0, "close": 98.0, "entry": 0.0, "sl": np.nan}, + ] + ) + + standard = QuantBTEndpoint.intrabar_bracket(initial_capital=10_000.0, fee_rate=0.0, slippage_bps=0.0, use_funding=False, report_level="standard") + standard_result = standard.backtest(data=df, signal_col="entry", symbols=["BTC"], intent_cols={"stop_value": "sl"}) + assert standard_result.metadata["engine_id"] == "intrabar_bracket_v1" + assert standard_result.metadata["report_level"] == "standard" + assert standard.fills_report.empty + + audit = QuantBTEndpoint.intrabar_bracket(initial_capital=10_000.0, fee_rate=0.0, slippage_bps=0.0, use_funding=False, report_level="audit") + audit_result = audit.backtest(data=df, signal_col="entry", symbols=["BTC"], intent_cols={"stop_value": "sl"}) + assert audit_result.metadata["report_level"] == "audit" + assert audit.fills_report["reason"].tolist() == ["entry", "stop_loss"] + assert audit.show_metrics()["final_equity"] == pytest.approx(9_995.0) + + +def test_phase31c_fill_replay_endpoint_accepts_single_dataframe_argument(): + df = _frame( + [ + {"open": 100.0, "high": 101.0, "low": 99.0, "close": 100.0}, + {"open": 100.0, "high": 103.0, "low": 99.0, "close": 102.0}, + ] + ) + fills = pd.DataFrame([{"bar_index": 1, "sequence": 0, "side": 1, "qty": 1.0, "price": 100.0, "fee": 0.0}]) + + bt = QuantBTEndpoint.fill_replay(initial_capital=10_000.0) + result = bt.backtest(data=df, symbols=["BTC"], fill_replay=fills) + + assert result.metadata["engine_id"] == "fill_replay_v1" + assert result.equity.iloc[-1] == pytest.approx(10_002.0) + + +def test_phase31c_warm_kernel_is_materially_faster_than_python_oracle(): + n = 2_000 + idx = pd.date_range("2024-01-01", periods=n, freq="1h", tz="UTC") + base = 100.0 + np.sin(np.arange(n) / 20.0) + df = pd.DataFrame( + { + "open": base, + "high": base + 1.0, + "low": base - 1.0, + "close": base + 0.2, + }, + index=idx, + ) + entry_side = np.zeros(n, dtype=np.int8) + entry_size = np.zeros(n, dtype=np.float64) + entry_side[::40] = 1 + entry_size[::40] = 1.0 + intent = IntrabarIntentTape.from_arrays(entry_side=entry_side, entry_size=entry_size, stop_value=np.full(n, 0.03)) + tape = prepare_market_tape(data=df, symbols=["BTC"], use_funding=False) + account = AccountConfig(initial_capital=10_000.0, leverage=10.0) + contract = ExecutionContract.intrabar_bracket(close_on_last_bar=True) + + run_intrabar_kernel(tape=tape, intent=intent, account=account, contract=contract, report_level="minimal") + start = time.perf_counter() + fast = run_intrabar_kernel(tape=tape, intent=intent, account=account, contract=contract, report_level="minimal") + fast_seconds = time.perf_counter() - start + + start = time.perf_counter() + slow = run_intrabar_reference(tape=tape, intent=intent, account=account, contract=contract) + slow_seconds = time.perf_counter() - start + + np.testing.assert_allclose(fast.equity.to_numpy(), slow.equity.to_numpy(), atol=1e-9, rtol=0.0) + assert fast_seconds < slow_seconds diff --git a/tests/test_phase31d_certification.py b/tests/test_phase31d_certification.py new file mode 100644 index 0000000..0d9a736 --- /dev/null +++ b/tests/test_phase31d_certification.py @@ -0,0 +1,84 @@ +import json +from pathlib import Path + +from quantbt import ( + CertificationLevel, + alpha_report_markdown, + build_alpha_certification_report, + certify_result_metadata, + classify_alpha_source, + scan_alpha_directory, +) +from quantbt.benchmarks.run_phase31_intrabar import make_markdown, run_benchmark +from quantbt.tools.audit_alpha_execution_contracts import main as audit_main + + +def test_phase31d_classifies_execution_sensitive_sources(): + close = classify_alpha_source("native = QuantBTEndpoint.pct_equity(); signal = df['pos_weight']") + assert close.required_engine == "close_target_v2" + assert close.current_backend == "legacy_pct_equity" + assert close.certification_level == int(CertificationLevel.LEGACY) + + intrabar = classify_alpha_source("df['exit_price'] = df['low']; slpercent = 2.0; tppercent = 3.0") + assert intrabar.required_engine == "intrabar_bracket_v1" + assert intrabar.uses_stop + assert intrabar.uses_take_profit + assert intrabar.uses_custom_exit_price + + replay = classify_alpha_source("fills_df = compact_fill[['bar_index', 'sequence', 'qty', 'price']]") + assert replay.required_engine == "fill_replay_v1" + assert replay.certification_level == int(CertificationLevel.ACCOUNTING_REPLAY) + + grid = classify_alpha_source("hedge_type='dca_ladder'; safety_order = 1; grid = True") + assert grid.required_engine == "event_lifecycle_v2" + assert grid.uses_grid_or_dca + + +def test_phase31d_certifies_result_metadata_levels(): + assert certify_result_metadata({"engine_id": "fill_replay_v1"})["certification_level"] == 1 + assert certify_result_metadata({"engine_id": "intrabar_bracket_v1"})["certification_level"] == 2 + assert certify_result_metadata({"engine_id": "intrabar_bracket_v1", "cross_backend_parity_passed": True})["certification_level"] == 3 + assert certify_result_metadata({"backend": "nautilus"})["certification_level"] == 4 + assert certify_result_metadata({"engine_id": "close_target_v2", "certification_status": "uncertified_intrabar_columns_on_close_target"})["certification_level"] == 0 + + +def test_phase31d_scanner_and_report_markdown(tmp_path: Path): + (tmp_path / "alpha_close.py").write_text("signal_notional = True\npos_weight = 1", encoding="utf-8") + (tmp_path / "alpha_intrabar.py").write_text("exit_price = df['low']\ntrailing_stop = 0.01", encoding="utf-8") + (tmp_path / "skip.bin").write_bytes(b"ignored") + + items = scan_alpha_directory(tmp_path) + report = build_alpha_certification_report(items) + markdown = alpha_report_markdown(report) + + assert report["total"] == 2 + assert report["by_required_engine"]["close_target_v2"] == 1 + assert report["by_required_engine"]["intrabar_bracket_v1"] == 1 + assert "alpha_intrabar" in markdown + + +def test_phase31d_audit_cli_writes_artifacts(tmp_path: Path): + source_dir = tmp_path / "alphas" + source_dir.mkdir() + (source_dir / "alpha.py").write_text("fill_replay(fills_df)", encoding="utf-8") + json_out = tmp_path / "report.json" + md_out = tmp_path / "report.md" + + rc = audit_main([str(source_dir), "--json-out", str(json_out), "--md-out", str(md_out)]) + + assert rc == 0 + payload = json.loads(json_out.read_text(encoding="utf-8")) + assert payload["total"] == 1 + assert payload["items"][0]["required_engine"] == "fill_replay_v1" + assert "Alpha Execution Certification Report" in md_out.read_text(encoding="utf-8") + + +def test_phase31d_benchmark_smoke_is_parity_safe(): + report = run_benchmark(rows=512, repeats=1, seed=31) + routes = {row["route"]: row for row in report["records"]} + + assert routes["intrabar_bracket_v1_audit"]["parity"] == "pass" + assert routes["intrabar_bracket_v1_minimal"]["runtime_seconds"] > 0.0 + assert routes["fill_replay_v1_kernel"]["fills_or_orders"] == routes["intrabar_bracket_v1_audit"]["fills_or_orders"] + assert report["summary"]["intrabar_minimal_speedup_vs_reference"] > 0.0 + assert "Phase 31 Intrabar Benchmark" in make_markdown(report) diff --git a/tools/audit_alpha_execution_contracts.py b/tools/audit_alpha_execution_contracts.py new file mode 100644 index 0000000..1ce44e5 --- /dev/null +++ b/tools/audit_alpha_execution_contracts.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +""" +Scan alpha source files and classify required QuantBT execution contracts. + +Example: + PYTHONPATH=/root/bobby/pool_alpha python3 quantbt/tools/audit_alpha_execution_contracts.py \ + /root/bobby/pool_alpha/alphas_storage/TA \ + --json-out /tmp/alpha_contracts.json \ + --md-out /tmp/alpha_contracts.md +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import sys + + +PACKAGE_DIR = Path(__file__).resolve().parents[1] +PROJECT_DIR = PACKAGE_DIR.parent +if str(PROJECT_DIR) not in sys.path: + sys.path.insert(0, str(PROJECT_DIR)) + +from quantbt import alpha_report_markdown, build_alpha_certification_report, scan_alpha_directory # noqa: E402 + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description="Audit alpha files for QuantBT execution-contract requirements.") + parser.add_argument("root", type=Path, help="Alpha source directory to scan") + parser.add_argument("--json-out", type=Path, default=PACKAGE_DIR / "benchmarks" / "out" / "alpha_execution_contracts.json") + parser.add_argument("--md-out", type=Path, default=PACKAGE_DIR / "benchmarks" / "out" / "alpha_execution_contracts.md") + parser.add_argument("--max-bytes", type=int, default=2_000_000) + args = parser.parse_args(argv) + + items = scan_alpha_directory(args.root, max_bytes=args.max_bytes) + report = build_alpha_certification_report(items) + + args.json_out.parent.mkdir(parents=True, exist_ok=True) + args.md_out.parent.mkdir(parents=True, exist_ok=True) + args.json_out.write_text(json.dumps(report, indent=2, default=str), encoding="utf-8") + args.md_out.write_text(alpha_report_markdown(report), encoding="utf-8") + print(f"scanned={report['total']} json={args.json_out} markdown={args.md_out}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/upgrade/implement.md b/upgrade/implement.md index 58c0010..d9d16d4 100644 --- a/upgrade/implement.md +++ b/upgrade/implement.md @@ -4421,7 +4421,9 @@ Sau khi runner này hoàn thành, có thể viết lại `grid_long_only` và `g ## Phase 31 - Execution Correctness And Fast Intrabar Upgrade -Status: planning, awaiting approval. +Status: complete for the current single-symbol OHLC intrabar scope. Phase 31A, +Phase 31B, Phase 31C, and Phase 31D implemented on +`feat/31-execution-correctness-intrabar`. Source design document: @@ -4497,6 +4499,50 @@ Acceptance: - Silent execution ambiguity becomes explicit metadata, warning, or error. - No intrabar engine implementation yet. +Implementation notes after Phase 31A: + +- `native_vectorized` now declares the close-target execution contract via + metadata: + - `engine="close_target_v2"`; + - `engine_id="close_target_v2"`; + - `backend_alias="native_vectorized"`; + - `kernel_version="units_v2"`; + - `signal_phase="bar_close"`; + - `fill_phase="same_close"`; + - `intrabar_exit_model="none"`; + - `first_bar_target_policy`; + - `data_signature`. +- `NativeVectorizedConfig` fails fast on unsupported execution config for the + close-target contract: + - non-close fill price policy; + - non-conservative same-bar policy; + - partial fills; + - min order notional; + - disabling insufficient-margin rejection. +- Funding dictionaries no longer synthesize `0.0001` for missing symbols; the + caller must pass the symbol explicitly, pass scalar funding, or disable + funding. +- Missing high/low on native-vectorized close-target runs is now marked with + `high_low_source="close_fallback_uncertified_intrabar_risk"` and emits a + bounded warning. Phase 31B will replace this compatibility fallback with + strict prepared-tape certification. +- Reactive native-event facade now passes `open` and `volume` from the input + frame into strategy context. +- Close-target endpoint warns and marks runs as + `uncertified_intrabar_columns_on_close_target` if the input dataframe + contains likely intrabar artifacts such as `exit_price`, `stop_loss`, + `take_profit`, or `trailing`. + +Validation after Phase 31A: + +- `tests/test_phase31a_execution_correctness_contract.py`: `6 passed`. +- Targeted regression: + `tests/test_phase2_native_vectorized.py`, + `tests/test_endpoint.py`, + `tests/test_phase30d_native_event_reactive_runner.py`, + `tests/test_phase30e_native_event_incremental_runner.py`, + `tests/test_phase9_performance_parity.py`: `42 passed`. + ### Phase 31B - Strict Prepared Market Tape And Python Intrabar Oracle Scope: @@ -4540,6 +4586,51 @@ Acceptance: - Prepared and non-prepared market inputs produce identical canonical arrays. - No Numba intrabar kernel is promoted until oracle tests are stable. +Implementation notes after Phase 31B: + +- Added `core/execution_contract.py` with the execution contract registry: + `close_target_v2`, `next_open_v1`, `intrabar_bracket_v1`, + `fill_replay_v1`, and `event_lifecycle_v2`. +- Added `core/market_tape.py` with strict immutable `PreparedMarketTape` and + `MarketValidationCertificate`. + - It rejects unsorted or duplicate timestamps. + - It rejects missing OHLC, NaN/inf, invalid OHLC invariants, non-positive + prices, and negative volume. + - It does not sort, deduplicate, forward-fill, back-fill, or synthesize + high/low from close. + - Funding dicts must explicitly cover every symbol. +- Added `core/intrabar_reference.py` as the readable single-symbol truth model + for `intrabar_bracket_v1`. + - Signal at close becomes executable at next open. + - Stops are gap-aware. + - Take-profit is limit-conservative by default. + - Same-bar SL/TP ambiguity is flagged and conservatively resolved. + - Trailing updates are effective from the next bar, not the same bar. + - Reversal pays two explicit legs: exit old position and enter new position. + - Optional final close is controlled by the execution contract. +- Added public exports from `quantbt` and `quantbt.core`. +- Added `QuantBTEndpoint.intrabar_bracket_reference(...)`. + - The endpoint accepts a compact signed `signal` / `signal_col` where sign is + side and absolute value is entry quantity. + - Optional SL/TP/trailing/technical-exit columns are mapped through + `intent_cols` instead of requiring a wide fixed strategy schema. + - The endpoint returns `BacktestResultV2`, normalized `fills_report`, + diagnostics, validation certificate, and normal `show_metrics()` / + `full_report()` compatibility. + +Validation after Phase 31B: + +- `tests/test_phase31b_market_tape_intrabar_oracle.py`: strict tape, + execution-contract registry, funding dict strictness, same-bar ambiguity, + next-bar trailing, reversal double-fee accounting, and public endpoint smoke. + +Remaining for Phase 31C: + +- Promote the oracle semantics into a Numba fast intrabar kernel. +- Add sparse audit ledger / second-pass fill replay. +- Add parity tests between oracle, native event, and the new Numba kernel. +- Add benchmark gates for `minimal`, `standard`, and `audit` report levels. + ### Phase 31C - Numba Fast Intrabar Kernel, Audit Ledger, And Fill Replay Scope: @@ -4583,6 +4674,40 @@ Acceptance: - No Python objects are created in hot loops. - Audit mode is deterministic and preserves exact fill sequence. +Implementation notes after Phase 31C: + +- Added `core/intrabar_kernel.py`: + - `_engine_intrabar_bracket_v1` is a Numba single-symbol linear intrabar + kernel for `intrabar_bracket_v1`; + - `run_intrabar_kernel(...)` wraps the kernel and returns + `NativeIntrabarKernelResult`; + - `report_level="minimal"` and `standard` avoid sparse fill materialization; + - `report_level="audit"` runs deterministic pass 2, allocates exact-size + sparse fill arrays, materializes fills/report, and asserts pass-1 parity; + - `FillReplayTape` and `run_fill_replay_kernel(...)` provide + `fill_replay_v1` accounting migration. +- Corrected the Python oracle accounting for entry slippage: + - PnL after entry now marks from actual fill price, not raw bar open; + - this makes fee/slippage legs explicit and gives the Numba kernel a correct + parity target. +- Added simple single-symbol margin/risk semantics to oracle and kernel: + - initial margin rejection with account leverage and margin buffer; + - conservative intrabar maintenance breach liquidation for unprotected paths; + - full venue mark-price liquidation remains future certification work. +- Added public endpoints: + - `QuantBTEndpoint.intrabar_bracket(...)` for the fast Numba route; + - `QuantBTEndpoint.fill_replay(...)` for explicit-fill accounting replay. +- Added public exports from `quantbt` and `quantbt.core`: + `run_intrabar_kernel`, `NativeIntrabarKernelResult`, `FillReplayTape`, + `run_fill_replay_kernel`, and `NativeFillReplayResult`. + +Validation after Phase 31C: + +- `tests/test_phase31c_intrabar_kernel.py` covers oracle parity, audit second + pass, slippage accounting, trailing/reversal behavior, insufficient-margin + rejection, single-symbol liquidation, fill replay accounting, endpoint + standard/audit routes, fill replay endpoint, and warm-kernel speed smoke. + ### Phase 31D - Certification, Alpha Audit Tooling, And Docs Scope: @@ -4641,3 +4766,1026 @@ Explicit non-goals for Phase 31: - No generic multi-order grid engine inside the intrabar kernel. - No options Greeks/portfolio option execution in this kernel. - No Cython/C++ until prepared tape, lazy result, and Numba kernels are profiled. + +Implementation notes after Phase 31D: + +- Added `core/certification.py`: + - `CertificationLevel` with Level 0-4 labels; + - `classify_alpha_source(...)` for conservative source classification; + - `scan_alpha_directory(...)` for `.py`, `.ipynb`, and `.md` alpha inventory; + - `build_alpha_certification_report(...)` and `alpha_report_markdown(...)`; + - `certify_result_metadata(...)` to summarize result metadata into a + stakeholder-readable certification label. +- Added `tools/audit_alpha_execution_contracts.py`: + - writes JSON and Markdown alpha execution-contract audit reports; + - intentionally treats source scanning as a migration hint, not proof of + causality or absence of look-ahead bias. +- Added Phase 31 benchmark harness: + - `benchmarks/run_phase31_intrabar.py`; + - committed `phase31_intrabar_benchmark.json` and + `phase31_intrabar_benchmark.md` after running the standard 25k-bar profile. +- Added docs: + - `docs/execution_contracts.md`; + - `docs/fast_intrabar.md`; + - `docs/alpha_certification.md`; + - endpoint and benchmark documentation links. +- Public exports now include certification helpers from `quantbt` and + `quantbt.core`. + +Validation after Phase 31D: + +- `tests/test_phase31d_certification.py` covers source classification, metadata + certification levels, directory scan/report generation, CLI artifact writes, + and benchmark smoke parity. +- Phase 31A/B/C/D targeted tests pass together with endpoint smoke tests. +- Full unit regression excluding real-data tests passes. +- Benchmark standard profile compares: + - close-target pure kernel; + - fast intrabar minimal; + - fast intrabar audit; + - Python intrabar oracle; + - fill replay kernel; + - native-event explicit-order facade. + +Phase 31 certification conclusion: + +- Completed: + - semantic freeze and contract manifest; + - strict market tape validation; + - close-target misuse metadata; + - Python intrabar oracle; + - Numba fast intrabar bracket kernel; + - audit ledger second pass; + - fill replay accounting kernel; + - public endpoint routes; + - source scanner and certification docs; + - reproducible benchmark report. +- Production readiness: + - practical Level 2 for single-symbol linear next-open SL/TP/trailing + intrabar research when the alpha emits compact intent columns and benchmark + parity passes; + - Level 1 for old explicit-fill alphas using `fill_replay`; + - Level 3/4 still requires native-event/Nautilus/lower-timeframe parity + artifacts for each concrete strategy and venue. +- Coverage of + `quantbt_phase17_execution_correctness_fast_intrabar_upgrade.md` is roughly + 80 percent of the requested institutional methodology: the core semantic, + oracle, kernel, audit, docs, and scanner work is done. Deferred pieces are + intentionally outside the current single-symbol OHLC bracket kernel: + standalone `next_open_v1` facade, broad native-event/Nautilus parity bundles, + lower-timeframe/tick validation, shared cross-margin intrabar semantics, + venue-specific liquidation/funding event ordering, and generic DCA/grid + state machines. + + +# Upgrade after Phase 31; +## QuantBT Execution Correctness — Các sửa đổi bắt buộc trước khi merge + +Status: implemented as two compact follow-up phases on +`feat/31-execution-correctness-intrabar`. + +### Phase 31G - Final Merge Blockers From Sol Review + +Status: implemented. + +Final blockers reviewed: + +1. Funding timing semantics. + - Decision: keep `FundingPhase.POSITION_AT_EVENT` only for funding events + whose timestamp matches an exact market bar timestamp. + - Mid-bar funding events now raise and require a smaller timeframe. + - Added explicit `bar_timestamp_semantics`: + - `close` default: OHLC timestamp is the bar close, funding applies after + intrabar execution on the remaining close position; + - `open`: OHLC timestamp is the bar open, funding applies after open-gap + marking and before pending exit/entry orders at `open[t]`. + - `bar_timestamp_semantics` is part of the strict market tape signature, so + prepared caches cannot be reused across open/close timestamp contracts. + - Kernel/reference metadata records + `funding_timing_certified=true` and + `funding_event_alignment="exact_bar_timestamp"`. +2. Execution contract propagation. + - Added `ExecutionContract.from_metadata(...)`. + - `QuantBTEndpoint.intrabar_bracket(...)` and + `.intrabar_bracket_reference(...)` accept `execution_contract=contract`. + - Endpoint and `PreparedIntrabarRunner` restore the full contract from + metadata rather than reconstructing only `close_on_last_bar`. + - Unsupported fields still raise `NotImplementedError`. +3. Data signature completeness. + - Strict market tape signature now includes: + - timestamps; + - symbols; + - open/high/low/close; + - volume; + - funding rates; + - funding event mask; + - bar timestamp semantics. + - Prepared intrabar runner also freezes a `prepared_signature` containing + market signature plus account/execution/sizing/constraint profile metadata. + +Technical debt handled in the same pass: + +- `exit + same-side entry` now emits `ENTRY_SUPPRESSED` instead of counting as + a rejected order. +- Dynamic trailing has explicit Python-oracle vs Numba parity coverage. +- Added optional `tick_size` conservative price quantization for entry, SL, TP, + and trailing levels. +- Docs now state the current certified scope as fast, deterministic, audited + **single-symbol intrabar** execution only. +- Added tests for: + - funding position phase; + - open-vs-close bar timestamp funding semantics; + - execution-contract propagation; + - signature changes from volume/funding; + - signature changes from bar timestamp semantics; + - prepared runner vs normal endpoint parity; + - minimal/audit parity through the existing audit tests; + - tick-size price quantization. + +Merge gate after Phase 31G: + +```bash +pytest -q tests/test_phase31*.py +pytest -q +python3 benchmarks/run_phase31_intrabar.py --rows 25000 --repeats 3 +``` + +Validation after Phase 31G: + +- `tests/test_phase31*.py`: 42 passed. +- Full `pytest -q`: 470 passed, 1 skipped. +- Phase31 benchmark: fast intrabar minimal 25k bars in 0.0118s, about 2.11M + bars/s and 23.32x faster than the Python oracle. + +Merge certification scope: + +> Fast, deterministic, and audited single-symbol intrabar execution kernel. + +### Phase 31E - Merge Blocker Execution Correctness + +Implemented: + +- `slippage_bps` is the source of truth for intrabar endpoints: + - fast/reference intrabar routes now pass `config.execution.slippage_rate`; + - legacy `slippage` on intrabar factories is converted once with a + deprecation warning; + - passing both `slippage` and `slippage_bps` raises; + - intrabar run config records `legacy_slippage_rate=None`. +- Funding is event-causal in strict market tape: + - scalar funding is rejected in strict mode; + - zero funding requires `use_funding=False` or + `missing_funding_policy="zero"`; + - `funding_event_timestamps` and `funding_event_rates` are supported; + - events must match an exact market bar timestamp; + - `bar_timestamp_semantics="close"` applies funding after intrabar execution + on the close position; + - `bar_timestamp_semantics="open"` applies funding before pending open + orders at `open[t]`. +- Strict timezone: + - naive market data is rejected unless `source_timezone` is provided; + - source timezone is localized first, then converted to UTC; + - data signatures are created after UTC normalization. +- Dynamic trailing now uses `trailing_value[t]` at close `t`; the new trailing + level only affects later bars. +- Intrabar intent supports side-specific `exit_long` and `exit_short`; + legacy `technical_exit` remains a compatibility alias for both sides. +- Same-side `exit + entry` conflict is `exit only`; opposite entry remains a + two-leg reversal. +- Intrabar sizing compiler added: + - `units`; + - `fixed_notional`; + - `pct_equity`; + - `risk_per_trade`. +- Shared quantity constraints are applied to intrabar entry quantity: + - `qty_step` / `lot_size` / `slot_size`; + - `min_qty`; + - `min_notional`. +- Unsupported `ExecutionContract` fields are rejected with + `NotImplementedError` instead of being silently ignored. +- Fill replay certification metadata is granular: + - price and fee accounting are certified; + - funding, margin, liquidation, generation, and causality are explicitly not + certified by the current fill replay implementation. + +### Phase 31F - Prepared Intrabar Runner, Docs, And Regression Lock + +Implemented: + +- Added `PreparedIntrabarRunner`: + - `runner = bt.prepare_intrabar(data=df, symbols=[...])`; + - `runner.run(intent, report_level="minimal")`; + - `runner.run(intent, report_level="audit")`; + - caches strict OHLCV arrays, funding arrays, validation certificate, data + signature, quantity constraints, and frozen profile metadata. +- Added endpoint support for event funding inputs: + - `funding_event_timestamps`; + - `funding_event_rates`. +- Added docs for: + - `slippage_bps`; + - funding events; + - side-specific exits; + - intrabar sizing; + - prepared runner usage. +- Added `tests/test_phase31_merge_blockers.py` covering the Sol blocker list + implemented in this compact follow-up. + +Validation: + +- `tests/test_phase31_merge_blockers.py`: 11 passed. +- Phase31 A/B/C/D/E/F targeted suite: 39 passed. +- Endpoint/native smoke suite: 41 passed. + +Remaining outside this compact follow-up: + +- A full `QuantBTProfile` / profile registry façade for every strategy family. +- Output contract classes for portfolio, grid, DCA, arbitrage, and options. +- Broad automatic output routing across all execution families. +- L2/tick/lower-timeframe validation and Nautilus Level 4 bundles for each + concrete alpha. + +## 1. Các lỗi phải sửa trước + +### 1.1 `slippage_bps` là nguồn cấu hình duy nhất + +Intrabar kernel phải lấy slippage từ: + +```python +slippage_rate = execution.slippage_bps / 10_000.0 +``` + +Không được đồng thời duy trì hai nguồn: + +```python +slippage +slippage_bps +``` + +Nếu legacy API truyền `slippage`, chỉ convert tại compatibility adapter và phát cảnh báo deprecation. Nếu cả hai cùng được truyền, phải raise error. + +--- + +### 1.2 Funding phải dựa trên event thực tế + +Không broadcast một scalar funding rate lên mọi bar. + +Intrabar backend chỉ nhận: + +```python +funding_event_timestamps +funding_event_rates +``` + +hoặc một series đã có: + +```text +rate != 0 chỉ tại funding event +``` + +Funding được áp khi event timestamp khớp chính xác một market bar timestamp: + +```text +funding_event_timestamp == market_bar_timestamp +``` + +Nếu OHLC timestamp là bar close, dùng `bar_timestamp_semantics="close"` để +funding áp sau intrabar path trên position còn lại tại close. Nếu OHLC timestamp +là bar open, dùng `bar_timestamp_semantics="open"` để funding áp trước pending +orders tại `open[t]`. + +Thiếu funding của symbol phải raise trong strict mode. Chỉ dùng zero khi: + +```python +use_funding=False +``` + +hoặc người dùng khai báo rõ: + +```python +missing_funding_policy="zero" +``` + +--- + +### 1.3 Dynamic trailing phải dùng giá trị tại `t` + +Tại `close[t]`, trailing mới phải được tính từ: + +```python +trailing_value[t] +``` + +không phải: + +```python +trailing_value[t - 1] +``` + +Trailing stop vừa cập nhật chỉ có hiệu lực từ bar `t+1`. Không được dùng stop mới để kiểm tra lại `high[t]` hoặc `low[t]`. + +--- + +### 1.4 Thêm sizing compiler và quantity constraints + +Alpha không nên trả direct quantity trừ khi khai báo rõ: + +```python +sizing_mode="units" +``` + +Phải hỗ trợ tối thiểu: + +```text +UNITS +FIXED_NOTIONAL +PCT_EQUITY +RISK_PER_TRADE +``` + +Ví dụ fixed notional: + +$$ +q = +\frac{ +Notional \times SizeWeight +}{ +FillPrice \times ContractSize +} +$$ + +Ví dụ risk per trade: + +$$ +q = +\frac{ +Equity \times RiskFraction \times SizeWeight +}{ +StopDistance \times ContractSize +} +$$ + +Sau khi tính raw quantity, engine phải áp: + +```text +qty_step +min_qty +min_notional +max_qty +available_margin +``` + +Việc quantize quantity phải dùng cùng một hàm cho reference oracle, Numba kernel và event backend. + +--- + +### 1.5 Tách `exit_long` và `exit_short` + +Không dùng một boolean chung: + +```python +technical_exit +``` + +Thay bằng: + +```python +exit_long +exit_short +``` + +Quy tắc: + +```text +exit_long chỉ tác động khi đang long +exit_short chỉ tác động khi đang short +``` + +Phải định nghĩa conflict policy khi cùng bar có exit và entry: + +```text +EXIT_ONLY +EXIT_THEN_REENTER +REVERSAL +REJECT_CONFLICT +``` + +Default khuyến nghị: + +```text +opposite entry -> reversal +same-side entry -> bỏ qua +exit + same-side entry -> exit only +``` + +Mọi reversal phải được account thành hai fill legs riêng biệt. + +--- + +### 1.6 Strict timezone + +Không được tự hiểu naive datetime là UTC. + +Strict mode: + +```python +if index.tz is None and source_timezone is None: + raise MarketDataError(...) +``` + +Nếu có: + +```python +source_timezone="Asia/Ho_Chi_Minh" +``` + +thì localize trước, sau đó mới convert UTC. + +Data signature phải được tạo sau khi timezone đã chuẩn hóa. + +--- + +### 1.7 Enforce hoặc reject mọi execution contract field + +Mọi field public trong `ExecutionContract` phải thuộc một trong hai trạng thái: + +```text +được backend thực thi đầy đủ +hoặc bị reject bằng NotImplementedError +``` + +Không được âm thầm bỏ qua các field như: + +```text +stop_gap_policy +take_profit_gap_policy +same_bar_policy +trailing_update_phase +funding_phase +liquidation_priority +ambiguity_policy +fill_price_policy +``` + +Mỗi backend nên khai báo capability: + +```python +BackendCapabilities( + supported_fill_phases=..., + supports_intrabar_stop=True, + supports_trailing=True, + supports_partial_fill=False, + supports_cross_margin=False, +) +``` + +Endpoint validate contract trước khi chạy kernel. + +--- + +### 1.8 Sửa certification của fill replay + +`fill_replay` chỉ được chứng nhận cho những domain mà implementation thực sự xử lý. + +Metadata nên tách riêng: + +```json +{ + "price_accounting_certified": true, + "fee_accounting_certified": true, + "funding_certified": false, + "margin_certified": false, + "liquidation_certified": false, + "execution_generation_certified": false, + "causality_certified": false +} +``` + +Chỉ nâng certification sau khi bổ sung implementation và parity tests tương ứng. + +--- + +### 1.9 Thêm `PreparedIntrabarRunner` + +Data preparation, validation và profile compilation chỉ chạy một lần: + +```python +runner = QuantBT.intrabar( + profile=profile, +).prepare( + data=df, + symbol="ETHUSDT", + funding=funding_events, +) +``` + +Mỗi trial chỉ cần: + +```python +intent = alpha.generate(runner.market, params) + +result = runner.run( + intent, + report_level="minimal", +) +``` + +Best candidate mới chạy: + +```python +audit = runner.run( + intent, + report_level="audit", +) +``` + +Prepared runner phải cache: + +```text +OHLCV contiguous arrays +timestamps +funding event arrays +instrument constraints +compiled execution codes +validation certificate +data signature +reusable buffers +``` + +Không được build lại DataFrame, funding mask hoặc instrument arrays trong mỗi Optuna trial. + +--- + +## 2. Kiến trúc dùng chung cho mọi alpha + +Không nên biến `IntrabarAlphaOutput` thành output duy nhất cho mọi chiến lược. + +Intrabar, target-position, grid, DCA, arbitrage và portfolio có execution semantics khác nhau. Ép tất cả vào một schema sẽ lặp lại lỗi thiết kế cũ của `pos_weight`. + +Nên dùng một façade chung nhưng nhiều output contract chuyên biệt. + +```text +QuantBT + ├── SharedProfile + ├── PreparedRunner + ├── AlphaOutput protocol + └── Backend/kernel registry +``` + +### 2.1 Profile dùng chung dạng composition + +```python +@dataclass(frozen=True) +class QuantBTProfile: + market: MarketProfile + account: AccountProfile + execution: ExecutionProfile + sizing: SizingProfile + portfolio: PortfolioProfile | None = None + reporting: ReportingProfile = ReportingProfile() +``` + +Profile được khai báo một lần cho từng môi trường/thị trường: + +```python +VN30F_PROFILE +BINANCE_PERP_PROFILE +VN_STOCK_PROFILE +DERIBIT_OPTION_PROFILE +``` + +Mọi alpha dùng cùng thị trường chỉ tham chiếu profile đó, không khai báo lại fee, leverage, slippage, contract size hoặc quantity constraints. + +### 2.2 Output contract theo họ chiến lược + +```python +class AlphaOutput(Protocol): + execution_family: str +``` + +Các output cụ thể: + +```text +TargetPositionOutput +NextOpenSignalOutput +IntrabarAlphaOutput +PortfolioTargetOutput +OrderIntentOutput +GridPlanOutput +DCAPlanOutput +ArbitrageOutput +OptionStrategyOutput +``` + +#### `IntrabarAlphaOutput` + +Dùng cho single-position hoặc simple multi-symbol SL/TP/trailing: + +```python +@dataclass(frozen=True) +class IntrabarAlphaOutput: + entry_side: np.ndarray + size_weight: np.ndarray + + stop_value: np.ndarray | None + take_profit_value: np.ndarray | None + trailing_value: np.ndarray | None + + exit_long: np.ndarray | None + exit_short: np.ndarray | None + + level_mode: LevelMode + signal_mode: SignalMode = SignalMode.PULSE +``` + +#### `PortfolioTargetOutput` + +Dùng cho cross-sectional allocation: + +```python +@dataclass(frozen=True) +class PortfolioTargetOutput: + target_weights: np.ndarray + rebalance_mask: np.ndarray +``` + +#### `OrderIntentOutput` + +Dùng cho generic order lifecycle: + +```python +@dataclass(frozen=True) +class OrderIntentOutput: + commands: CompactOrderCommandTape +``` + +#### Grid và DCA + +Không nên ép grid/DCA thành một `entry_side`. + +Chúng cần output riêng: + +```python +@dataclass(frozen=True) +class GridPlanOutput: + level_prices: np.ndarray + level_sizes: np.ndarray + side: np.ndarray + cancel_replace_mask: np.ndarray +``` + +```python +@dataclass(frozen=True) +class DCAPlanOutput: + trigger_prices: np.ndarray + order_sizes: np.ndarray + take_profit_rules: np.ndarray + stop_rules: np.ndarray +``` + +#### Arbitrage + +Arbitrage phải biểu diễn một basket atomic hoặc coordinated legs: + +```python +@dataclass(frozen=True) +class ArbitrageOutput: + basket_entry: np.ndarray + basket_exit: np.ndarray + leg_weights: np.ndarray + hedge_ratios: np.ndarray + execution_policy: BasketExecutionPolicy +``` + +Không được chạy từng leg độc lập rồi gọi đó là arbitrage backtest chuẩn. + +--- + +## 3. Không nên tạo endpoint ngầm bằng global state khi import + +Không nên làm: + +```python +import quantbt +``` + +rồi package âm thầm giữ một global profile hoặc global endpoint. + +Global mutable state sẽ gây vấn đề: + +```text +khó tái lập kết quả +không thread-safe +khó chạy nhiều thị trường trong một process +Optuna trials có thể dùng nhầm profile +tests ảnh hưởng lẫn nhau +khó biết result dùng config nào +``` + +Nên dùng explicit façade nhưng khai báo rất ngắn: + +```python +qbt = QuantBT(profile=BINANCE_PERP_PROFILE) +runner = qbt.prepare(data=df, symbol="ETHUSDT") +``` + +Sau đó dùng lại `runner` cho mọi alpha: + +```python +result_a = runner.run(alpha_a.generate(runner.market, params_a)) +result_b = runner.run(alpha_b.generate(runner.market, params_b)) +result_c = runner.run(alpha_c.generate(runner.market, params_c)) +``` + +Có thể thêm profile registry: + +```python +qbt = QuantBT.from_profile("binance_perp_default") +``` + +hoặc YAML: + +```yaml +profile: binance_perp_default +``` + +Nhưng profile cuối cùng phải được đóng băng vào result metadata để bảo đảm reproducibility. + +--- + +## 4. Routing tự động nhưng không được mơ hồ + +Runner có thể tự route theo kiểu output: + +```python +result = runner.run(alpha_output) +``` + +Ví dụ: + +```text +IntrabarAlphaOutput -> intrabar_bracket_v1 +TargetPositionOutput -> close_target_v2 +PortfolioTargetOutput -> native_portfolio_v3 +GridPlanOutput -> grid kernel/event backend +DCAPlanOutput -> DCA kernel +ArbitrageOutput -> basket/arbitrage backend +OrderIntentOutput -> event_lifecycle_v2 +``` + +Nếu profile và output không tương thích: + +```python +raise ExecutionContractError(...) +``` + +Không được fallback âm thầm sang backend khác. + +--- + +## 5. API sử dụng cuối cùng + +Khai báo profile một lần: + +```python +profile = QuantBTProfile( + market=BinancePerpetualMarketProfile( + symbol="ETHUSDT", + contract_size=1.0, + qty_step=0.001, + min_qty=0.001, + min_notional=5.0, + ), + account=AccountProfile( + initial_capital=100_000.0, + leverage=3.0, + maintenance_ratio=0.005, + ), + execution=IntrabarExecutionProfile( + signal_phase="close", + fill_phase="next_open", + fee_rate=0.0004, + slippage_bps=2.0, + same_bar_policy="conservative", + close_on_last_bar=True, + ), + sizing=FixedNotionalSizing( + notional_per_trade=10_000.0, + ), +) +``` + +Prepare một lần: + +```python +runner = QuantBT(profile).prepare( + data=df, + funding=funding_events, +) +``` + +Mỗi alpha chỉ còn: + +```python +intent = alpha.generate( + market=runner.market, + params=params, +) + +result = runner.run( + intent, + report_level="minimal", +) +``` + +Audit: + +```python +audit = runner.run( + intent, + report_level="audit", +) +``` + +Đây nên là API chính. Các low-level endpoint vẫn được giữ cho advanced use cases và backward compatibility. + +--- + +## 6. Regression tests phải bổ sung + +```text +test_intrabar_uses_slippage_bps_as_source_of_truth +test_legacy_slippage_conflict_raises +test_scalar_funding_rejected +test_funding_applied_only_at_event +test_funding_crosses_missing_exact_hour +test_dynamic_trailing_uses_value_at_t +test_new_trailing_not_applied_to_same_bar +test_fixed_notional_sizing +test_pct_equity_sizing +test_risk_per_trade_sizing +test_qty_step_rounding +test_min_qty_rejection +test_min_notional_rejection +test_exit_long_only_affects_long +test_exit_short_only_affects_short +test_exit_entry_conflict_policy +test_reversal_has_two_fill_legs +test_naive_timezone_rejected +test_source_timezone_localized_then_converted +test_unsupported_contract_field_raises +test_fill_replay_certification_is_granular +test_prepared_intrabar_matches_normal_endpoint +test_minimal_and_audit_equity_parity +test_profile_metadata_is_frozen_in_result +test_output_type_routes_to_expected_backend +test_incompatible_profile_output_raises +``` + +--- + +## 7. Cách chạy test + +### Chạy toàn bộ test suite + +```bash +pytest -q +``` + +### Dừng ngay tại lỗi đầu tiên + +```bash +pytest -q -x +``` + +### Chạy các test Phase 31 hiện tại + +```bash +pytest -q tests/test_phase31*.py +``` + +### Chạy riêng intrabar kernel và oracle + +```bash +pytest -q \ + tests/test_phase31_intrabar_reference.py \ + tests/test_phase31c_intrabar_kernel.py +``` + +Nếu tên file thực tế khác, kiểm tra bằng: + +```bash +find tests -maxdepth 1 -type f | sort | grep -E "phase31|intrabar|fill_replay" +``` + +### Chạy các regression tests mới + +Khuyến nghị đặt trong: + +```text +tests/test_phase31_merge_blockers.py +tests/test_phase31_profiles_and_runner.py +``` + +Sau đó chạy: + +```bash +pytest -q \ + tests/test_phase31_merge_blockers.py \ + tests/test_phase31_profiles_and_runner.py +``` + +### Chạy test với output đầy đủ + +```bash +pytest -vv -s tests/test_phase31_merge_blockers.py +``` + +### Chạy một test cụ thể + +```bash +pytest -q \ + tests/test_phase31_merge_blockers.py::test_dynamic_trailing_uses_value_at_t +``` + +### Chạy tests liên quan funding + +```bash +pytest -q -k "funding" +``` + +### Chạy tests liên quan sizing + +```bash +pytest -q -k "sizing or quantity or min_notional or qty_step" +``` + +### Chạy tests liên quan intrabar + +```bash +pytest -q -k "intrabar or trailing or same_bar or reversal" +``` + +### Chạy coverage + +```bash +pytest \ + --cov=quantbt \ + --cov-report=term-missing \ + --cov-report=html +``` + +Nếu package import trực tiếp từ repository root: + +```bash +PYTHONPATH=. pytest -q +``` + +### Chạy benchmark sau khi tests pass + +```bash +python benchmarks/run_phase17_intrabar.py +``` + +Hoặc benchmark hiện có trên nhánh: + +```bash +find benchmarks -maxdepth 1 -type f | sort | grep -E "phase17|phase31|intrabar" +``` + +Rồi chạy file tìm được: + +```bash +python benchmarks/.py +``` + +Benchmark phải chạy hai lần: + +```text +cold JIT compile +warm execution +``` + +Chỉ dùng warm execution cho performance gate. + +--- + +## 8. Merge gate + +Chỉ merge vào `dev` khi: + +* [ ] Tất cả lỗi P0 phía trên đã sửa. +* [ ] Mọi execution contract field được enforce hoặc reject. +* [ ] Python oracle và Numba kernel parity. +* [ ] Minimal và audit mode cho cùng equity/accounting. +* [ ] Prepared và non-prepared endpoint parity. +* [ ] Sizing và quantity constraints có regression tests. +* [ ] Funding chỉ áp tại event. +* [ ] Dynamic trailing dùng `t`. +* [ ] Strict timezone hoạt động. +* [ ] Fill replay certification không overclaim. +* [ ] Full test suite pass. +* [ ] Benchmark không vượt performance threshold đã đặt. +* [ ] Result metadata lưu profile, execution contract, kernel version và data signature. + +Kiến trúc nên chốt theo nguyên tắc: + +> **Một façade và một profile dùng lại cho nhiều alpha, nhưng mỗi họ chiến lược phải có output contract và backend phù hợp riêng. Không dùng một schema duy nhất để ép target-position, intrabar, portfolio, grid, DCA và arbitrage vào cùng semantics.**