Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions __init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@
GridDCATrialOutput,
JsonlOptimizationLogger,
MissingOptimizationMetricError,
MultiSeedOptimization,
ObjectiveResult,
OptionPackageGenericEvaluator,
OptionTrialOutput,
Expand All @@ -96,6 +97,7 @@
PreparedPortfolioEvaluator,
PreparedSignalEvaluator,
ReportMetricObjective,
RobustSelectionConfig,
SamplerConfig,
SearchSpaceInfo,
SelectedCandidate,
Expand Down Expand Up @@ -584,11 +586,13 @@
"DuplicatePruner",
"CONSTRAINTS_USER_ATTR",
"JsonlOptimizationLogger",
"MultiSeedOptimization",
"ObjectiveResult",
"OptimizationConfig",
"OptimizationResult",
"OptimizationTrialRecord",
"OptunaOptimizer",
"RobustSelectionConfig",
"SamplerConfig",
"SearchSpaceInfo",
"SingleObjectiveEarlyStopping",
Expand Down
173 changes: 173 additions & 0 deletions docs/optimization.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ from quantbt import (
ReportMetricObjective,
SharpeObjective,
CandidateSelector,
RobustSelectionConfig,
MultiSeedOptimization,
)
```

Expand Down Expand Up @@ -195,6 +197,72 @@ unseeded sampler behavior, matching `optuna.samplers.TPESampler()` defaults.
This is useful for exploratory legacy-style searches. Keep an explicit integer
seed for reproducible studies and stakeholder audit runs.

## Search Assurance

Use `initial_trials` to force historical champions or hand-picked baselines to
be evaluated by the current evaluator before sampled trials:

```python
result = optimizer.optimize(
param_ranges=param_ranges,
fixed_params={"issl": True},
initial_trials=[
hyhy,
hrhr,
hyhy_migrate_quantbt,
],
candidate_selector=CandidateSelector(mode="feasible_best"),
)
```

Warm-start trials are tagged as `quantbt_source="warm_start"` and are exposed
through:

```python
result.baseline_trials
result.search_diagnostics["baseline_rank"]
result.search_regression
```

For single-objective studies, QuantBT applies a baseline floor: if the selected
candidate is worse than the best feasible warm-start on the primary objective,
`selected_params` is reset to the warm-start baseline and
`search_regression=True`.

When strategy params contain inactive/noisy branches, pass an
`effective_params_builder` so duplicate detection and diagnostics can use the
semantic parameter set:

```python
def effective(params):
out = dict(params)
if not out["istp"]:
out.pop("tppercent", None)
if not out["usevol"]:
out.pop("rvol", None)
out.pop("len_vol", None)
return out

result = optimizer.optimize(
param_ranges=param_ranges,
fixed_params=fixed,
initial_trials=[hyhy],
effective_params_builder=effective,
)
```

Use `early_stopping_min_trials` when early stopping is enabled so the study
cannot stop before a minimum exploration floor:

```python
OptimizationConfig(
study_name="delta_rsi",
n_trials=1_500,
early_stopping_rounds=300,
early_stopping_min_trials=800,
)
```

This fallback is intentionally used for early arbitrage, grid/DCA, and option
package workflows until a specialized prepared evaluator is worth adding.

Expand Down Expand Up @@ -335,6 +403,111 @@ selector when production params are required.
`CandidateSelector(mode="pareto_first")` filters infeasible Pareto trials before
selection.

### Robust Plateau Selection

For practical alpha research, the highest Optuna trial can be an isolated
sample. QuantBT therefore exposes a post-search plateau selector:

```python
selector = CandidateSelector(
mode="robust_plateau",
config=RobustSelectionConfig(
top_quantile=0.10,
min_trades=100,
max_drawdown_pct=25.0,
neighborhood_radius=0.10,
min_neighbor_count=8,
seed_consensus=3,
instability_penalty=0.25,
worst_weight=0.25,
drawdown_penalty=0.0,
),
)

result = optimizer.optimize(
param_ranges=param_ranges,
candidate_selector=selector,
)
```

The sampler still optimizes the raw objective. The selector runs only after the
study is complete:

```text
completed trials
-> formal feasibility filter
-> optional metric filters such as min trades / max drawdown
-> top objective quantile
-> parameter-neighborhood scoring
-> medoid candidate from the best plateau
```

The robust score is selection-only:

```text
score =
median(objective in neighborhood)
+ worst_weight * worst(objective in neighborhood)
- instability_penalty * std(objective in neighborhood)
- drawdown_penalty * median(max_drawdown_pct)
+ size_bonus * log(1 + neighbor_count)
```

This is designed to avoid selecting a single lucky spike that does not survive
nearby parameter perturbation. It does not change the objective surface seen by
Optuna.

`result.robust_candidates` records the ranked plateau candidates, including
neighbor count, seed consensus count, objective dispersion, and selected medoid
trial number.

### Multi-Seed Search

One random TPE trajectory is not enough evidence that a parameter region is
stable. `MultiSeedOptimization` reruns the same evaluator under several sampler
seeds, aggregates the completed trial records, then applies the same selector:

```python
multi = MultiSeedOptimization(
evaluator=evaluator,
config=OptimizationConfig(
study_name="delta_rsi_intrabar_multiseed",
n_trials=600,
seed=None,
show_progress_bar=False,
early_stopping_rounds=None,
),
sampler_config=SamplerConfig(
name="tpe",
kwargs={
"n_startup_trials": 120,
"multivariate": False,
"group": False,
},
),
seeds=(None, 41, 42, 43, 44),
)

result = multi.optimize(
param_ranges=param_ranges,
fixed_params={"issl": True},
initial_trials=[known_good_params],
candidate_selector=selector,
)
```

`result.seed_results` stores best/selected params per seed. Trial metadata also
contains `quantbt_seed`, so robust plateau selection can require a region to be
seen across several seeds via `seed_consensus`.

For a normal single-study `OptunaOptimizer` result without seed metadata, the
selector treats the study as one consensus group. Set `seed_consensus > 1` only
when using aggregated multi-seed trial records.

Warm-start baseline floor still applies: if the robust candidate is worse than
the best feasible historical baseline on the primary objective, QuantBT returns
the baseline and sets `search_regression=True`.

## Reproducibility Safety

Phase 32 final merge rules are conservative:
Expand Down
5 changes: 4 additions & 1 deletion optimization/__init__.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""Domain-agnostic optimization API for QuantBT."""

from .callbacks import JsonlOptimizationLogger, SingleObjectiveEarlyStopping
from .candidate_selection import CandidateSelector, SelectedCandidate, constraints_feasible
from .candidate_selection import CandidateSelector, RobustSelectionConfig, SelectedCandidate, constraints_feasible
from .config import OptimizationConfig, SamplerConfig
from .constraints import CONSTRAINTS_USER_ATTR, constraints_from_trial, set_trial_constraints
from .evaluator import TrialEvaluator
Expand Down Expand Up @@ -31,6 +31,7 @@
result_full_report,
)
from .optimizer import OptunaOptimizer
from .multiseed import MultiSeedOptimization
from .result import ObjectiveResult, OptimizationResult, OptimizationTrialRecord
from .samplers import build_sampler
from .space import (
Expand All @@ -52,6 +53,7 @@
"GridDCATrialOutput",
"JsonlOptimizationLogger",
"MissingOptimizationMetricError",
"MultiSeedOptimization",
"ObjectiveResult",
"OptionPackageGenericEvaluator",
"OptionTrialOutput",
Expand All @@ -63,6 +65,7 @@
"PreparedPortfolioEvaluator",
"PreparedSignalEvaluator",
"ReportMetricObjective",
"RobustSelectionConfig",
"SamplerConfig",
"SearchSpaceInfo",
"SelectedCandidate",
Expand Down
9 changes: 7 additions & 2 deletions optimization/callbacks.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,19 +11,23 @@
class SingleObjectiveEarlyStopping:
"""Stop a single-objective Optuna study after best-value stagnation."""

def __init__(self, patience: int, direction: str, min_delta: float = 1e-4):
def __init__(self, patience: int, direction: str, min_delta: float = 1e-4, min_trials: int = 0):
if patience <= 0:
raise ValueError("patience must be positive")
direction = str(direction).lower().strip()
if direction not in {"maximize", "minimize"}:
raise ValueError("direction must be maximize or minimize")
if min_delta < 0.0:
raise ValueError("min_delta must be >= 0")
if min_trials < 0:
raise ValueError("min_trials must be >= 0")
self.patience = int(patience)
self.direction = direction
self.min_delta = float(min_delta)
self.min_trials = int(min_trials)
self._best: Optional[float] = None
self._stale = 0
self._completed = 0

def __call__(self, study, trial) -> None:
try:
Expand All @@ -36,12 +40,13 @@ def __call__(self, study, trial) -> None:
current = float(study.best_value)
except Exception:
return
self._completed += 1
if self._is_improved(current):
self._best = current
self._stale = 0
else:
self._stale += 1
if self._stale >= self.patience:
if self._completed >= self.min_trials and self._stale >= self.patience:
study.stop()

def _is_improved(self, current: float) -> bool:
Expand Down
Loading
Loading