From 19814cb16b74a24c3061e3eb109dc0a2e557a1f6 Mon Sep 17 00:00:00 2001 From: Matt Spinola Date: Sun, 2 Aug 2026 21:53:02 -0400 Subject: [PATCH] feat(validation): deflated_expectancy, the number the monitor should anchor to MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docs/edge_monitor.md carried this at the top of its open list since the monitor shipped: EdgeBaseline took a deflated_expectancy float and nothing in the package produced one. deflated_sharpe corrects a Sharpe and returns a PROBABILITY, which is the right output for a gate and useless to a monitor. A monitor needs a number in R, and without one the only anchor available was the sample mean, which is the number the parameters were optimized on. The conversion rides on the bar deflated_sharpe already uses. SR0 is the expected maximum per-trade Sharpe of N noise trials; carry it back into R by the winner's own sigma and subtract: deflated = mu - sigma * SR0 Both functions now call one _expected_max_sharpe, so two corrections for one search cannot disagree about how big the search was. Three choices worth recording: * It takes trial LOGS, not trial Sharpes, unlike deflated_sharpe. The Sharpes are computed inside so their clock cannot be got wrong: multiplying a per-month Sharpe by a per-trade sigma gives a haircut in no units at all, silently, which is the v0.4.0 units bug in a new costume. * It is a bias correction, NOT a significance test, and the docstring, the __str__ and the property name all say so. It removes the selection bias a search of this size is EXPECTED to produce, so a pure-noise winner still clears zero roughly half the time: measured 56% / 47% / 44% for N = 5 / 20 / 100, against deflated_sharpe correctly calling 0% of the same draws significant. I named the property `survives` first, which reads as a verdict it does not deliver; it is `is_positive`, and the measurement that caught this is pinned as its reproducer. * EdgeBaseline.from_log accepts the result OBJECT as well as a float, because handing over the wrong field of a result you already computed anchors the monitor to the pre-correction number while reporting deflated=True. Two findings from wiring the example to the real function rather than a stand-in: * The worked example faked this step with `raw_mean * 0.8` under a comment starting "Pretend". With a real 64-config search the haircut is 26% of the raw edge rather than 20%, and the naive baseline flatters by 36% rather than 25%. Every figure in tutorial §14 moved and is re-pinned. * tests/test_edge_monitor_example.py REBUILT the baseline it exists to pin instead of importing it, so all five assertions passed unchanged while the example they guard printed entirely different numbers. It now imports promoted_book(). A guard that reconstructs what it guards is not a guard. The payoff is now a test rather than a claim: run one untouched, fully healthy live book against both baselines and the naive one returns DEGRADED where the deflated one returns HOLDING. An inflated reference does not merely mis-scale the ratios, it manufactures alarms on books that never decayed. 373 tests pass, ruff clean, mkdocs --strict clean. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 58 +++++-- docs/architecture.md | 3 +- docs/edge_monitor.md | 98 +++++++++--- docs/tutorial.md | 99 +++++++----- examples/edge_monitor.py | 51 +++++-- src/crucible/validation/__init__.py | 8 +- src/crucible/validation/monitor.py | 25 +++- src/crucible/validation/pbo.py | 184 ++++++++++++++++++++++- tests/test_deflated_expectancy.py | 224 ++++++++++++++++++++++++++++ tests/test_edge_monitor_example.py | 101 ++++++++----- 10 files changed, 721 insertions(+), 130 deletions(-) create mode 100644 tests/test_deflated_expectancy.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0303a54..6f725c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,17 +6,43 @@ adhere to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] -### Fixed -- **The opportunity-set channel switched itself off for any log without `entry_date`.** - `_trades_per_year` read `entry_date` only, so a log built from a return column and an - exit date, which is an ordinary shape, produced `trades_per_year=None` on the baseline - and `frequency_ratio=None` in every verdict. It said so in `reasons` and nothing was - wrong-but-silent, yet the effect was that the one channel covering a signal that stops - firing was dead by default for a whole class of books. It now falls back to `exit_date` - (a rate is `n / span`, and either column dates the same trades closely enough), with - `entry_date` still preferred when both are present. +### Added +- **`validation.deflated_expectancy`**, closing the gap that stood at the top of + `docs/edge_monitor.md`'s open list since the monitor shipped. `EdgeBaseline` took a + `deflated_expectancy` float and nothing in the package produced one: `deflated_sharpe` + corrects a Sharpe and returns a **probability**, which is the right output for a gate + and useless to a monitor. A monitor needs a number in R to anchor to, and without one + the only available anchor was the sample mean, which is the number the parameters were + optimized on. + + The conversion rides on the bar `deflated_sharpe` already uses: `SR0`, the expected + maximum per-trade Sharpe of N noise trials, carried back into R by the winner's own + sigma and subtracted, `deflated = mu - sigma * SR0`. Both now call one + `_expected_max_sharpe`, so two corrections for one search cannot disagree about how big + the search was. `EdgeBaseline.from_log` accepts the `DeflatedExpectancy` object as well + as a float, because handing over the wrong field of a result you already computed + anchors the monitor to the pre-correction number while reporting `deflated=True`. + + **It takes trial LOGS, not trial Sharpes**, unlike `deflated_sharpe`. The Sharpes are + computed inside so their clock cannot be got wrong: multiplying a per-month Sharpe by a + per-trade sigma yields a haircut in no units at all, silently, which is the v0.4.0 units + bug in a new costume. + + **It is a bias correction, not a significance test**, and the docstring, the `__str__` + and the property name all say so. It removes the selection bias a search of this size is + *expected* to produce, so a pure-noise winner still clears zero roughly half the time + (measured at 56% / 47% / 44% for N = 5 / 20 / 100, against `deflated_sharpe` correctly + calling 0% of the same draws significant). The result's property is `is_positive` rather + than `survives` for exactly that reason. A correction that leaves nothing raises when it + reaches `EdgeBaseline`, whose existing refusal now names deflation as a cause. ### Changed +- **`examples/edge_monitor.py` now runs a real 64-config search** and deflates the winner, + where it previously applied a hardcoded `raw_mean * 0.8` under a comment beginning + "Pretend". Every figure in tutorial §14 moved as a result (the haircut is 26% of the raw + edge, not 20%, and the naive baseline flatters by 36%, not 25%). The example now exposes + `promoted_book()` so the tutorial's numbers have one source. + - **The CUSUM's false-alarm budget is now stated in CALENDAR TIME.** New `Thresholds.monitor_arl0_years` (default 25), converted using the baseline's own firing rate; `monitor_arl0_trades` stays as the fallback when the rate is unknown. @@ -46,6 +72,20 @@ adhere to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). docstring and [docs/edge_monitor.md](docs/edge_monitor.md) now carry the graded table, and two tests pin it so the claim cannot quietly drift back. +### Fixed +- **`tests/test_edge_monitor_example.py` rebuilt the baseline it was supposed to pin**, + instead of importing it. So the "reproducibility guard" for tutorial §14 passed + unchanged while the example it guards printed entirely different numbers. It now imports + `promoted_book()`. A guard that reconstructs what it guards is not a guard. +- **The opportunity-set channel switched itself off for any log without `entry_date`.** + `_trades_per_year` read `entry_date` only, so a log built from a return column and an + exit date, which is an ordinary shape, produced `trades_per_year=None` on the baseline + and `frequency_ratio=None` in every verdict. It said so in `reasons` and nothing was + wrong-but-silent, yet the effect was that the one channel covering a signal that stops + firing was dead by default for a whole class of books. It now falls back to `exit_date` + (a rate is `n / span`, and either column dates the same trades closely enough), with + `entry_date` still preferred when both are present. + ## [0.5.0] - 2026-08-02 The edge-monitor release. The gauntlet asks "is this edge real?" once, over a fixed log; diff --git a/docs/architecture.md b/docs/architecture.md index 04a79b9..4eb6dd9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -70,7 +70,7 @@ flowchart TD | Module | Package? | Purpose | Public entry points | |---|---|---|---| | **`edge`** | `edge/` (trade_log, simulator, metrics, stats) | Produce and describe the `TradeLog`; the honesty layer (CI + p-value). | `TradeLog`, `barrier_trades`, `edge_report`, `reality_check`, `bootstrap_ci`, `block_bootstrap_pvalue`, `random_entry_null` | -| **`validation`** | `validation/` (holdout, walk_forward, permutation, pbo, search_space, gate, gauntlet, thresholds, diagnostics, monitor) | Out-of-sample survival, data-mining corrections, the audited gauntlet, and the post-promotion decay monitor. | `holdout`, `walk_forward`, `sign_permutation_pvalue`, `sidak_correction`, `spa_test`, `pbo_cscv`, `deflated_sharpe`, `SearchSpaceLog`, `run_gauntlet`, `Thresholds`, `edge_monitor`, `EdgeBaseline` | +| **`validation`** | `validation/` (holdout, walk_forward, permutation, pbo, search_space, gate, gauntlet, thresholds, diagnostics, monitor) | Out-of-sample survival, data-mining corrections, the audited gauntlet, and the post-promotion decay monitor. | `holdout`, `walk_forward`, `sign_permutation_pvalue`, `sidak_correction`, `spa_test`, `pbo_cscv`, `deflated_sharpe`, `deflated_expectancy`, `SearchSpaceLog`, `run_gauntlet`, `Thresholds`, `edge_monitor`, `EdgeBaseline` | | **`breadth`** | `breadth.py` (single file) | How many *independent* bets a correlated set of return streams holds. | `effective_n`, `participation_ratio`, `Breadth` | | **`ml`** | `ml/` (ic, decay, redundancy, pit) | The same honesty aimed at a model's scores — a predictions frame, not a `TradeLog`. | `information_coefficient`, `alpha_gate`, `quantile_decay`, `fold_ic`, `redundancy_droplist`, `asof_window` | | **`report`** | `report/` (tearsheet, scorecards) | Self-contained HTML tearsheets. Plotly, behind the `[report]` extra; **not** re-exported at top level. | `tearsheet`, `gauntlet_report`, `fullrange_scorecard`, `monitor_panel` | @@ -144,6 +144,7 @@ simulator), this is the output you must return. | `EdgeReport` | `edge/metrics.py:130` | dataclass | `n, win_rate, expectancy, profit_factor, payoff_ratio, sqn` (+ excursion optionals) | | `PBOResult` | `validation/pbo.py:68` | dataclass | `pbo, logits, oos_below_zero, degradation_slope/_r2, n_configs, n_splits, n_blocks`; `.label` ROBUST/GUARDED/OVERFIT | | `DeflatedSharpe` | `validation/pbo.py:194` | dataclass | `observed_sharpe, deflated_sharpe, sr0_threshold, n_trials, n_obs, skew, kurtosis`; `.label` SIGNIFICANT/MARGINAL/NOT SIGNIFICANT | +| `DeflatedExpectancy` | `validation/pbo.py:324` | dataclass | `observed_expectancy, deflated_expectancy, haircut, sigma, sr0_threshold, n_trials, n_trades, n_scored`; `.is_positive`, `.retained`. A bias correction in R, not a test | | `Breadth` | `breadth.py:22` | frozen | `n_eff, n_assets, eigenvalues, loadings, corr`; `.redundancy` = n_assets/n_eff | | `Fold` / `WalkForwardResult` | `validation/walk_forward.py:62,75` | dataclass | fold detail; `folds, stitched, param_grid` | | `HoldoutResult` | `validation/holdout.py:51` | dataclass | early/late `Verdict`s | diff --git a/docs/edge_monitor.md b/docs/edge_monitor.md index a8c0f26..554eac4 100644 --- a/docs/edge_monitor.md +++ b/docs/edge_monitor.md @@ -95,6 +95,9 @@ already produces the corrected version of exactly that figure: `deflated_sharpe` is a **deflated** expectancy is measuring decay from a number that was defensible in the first place. +`validation.deflated_expectancy` now does that conversion; see +[Deflating the baseline](#deflating-the-baseline) below. + Worth flagging in the reference numbers: OOS expectancy (`+0.158%`) is *above* IS (`+0.134%`). That is backwards from the usual optimization bias. Either the search was narrow (small honest N), the in-sample window was hostile, or the out-of-sample period @@ -160,10 +163,13 @@ crucible emits HOLDING / SLIPPING / DEGRADED. It does not emit "cut to half size ### What shipped ```python -from crucible.validation import EdgeBaseline, cusum_design, edge_monitor, empirical_arl +from crucible.validation import ( + EdgeBaseline, cusum_design, deflated_expectancy, edge_monitor, empirical_arl, +) # ONCE, at promotion. Freeze the result. -base = EdgeBaseline.from_log(validated_log, deflated_expectancy=0.08, n_variants=64) +corrected = deflated_expectancy(validated_log.r, [t.r for t in trials], n_trials=64) +base = EdgeBaseline.from_log(validated_log, deflated_expectancy=corrected, n_variants=64) design = cusum_design(base) # k and h derived from Thresholds, not typed in verdict = edge_monitor(live_log, base) @@ -246,12 +252,71 @@ so is not. The rolling ratio and the firing-rate ratio cap out at `SLIPPING`. This is the part worth keeping if nothing else here survives review, and [`examples/edge_monitor.py`](https://github.com/mspinola/crucible/blob/main/examples/edge_monitor.py) -shows why. On a book whose true edge is fully intact and a quarter **above** baseline, -the 200-trade trailing read ranges from -31% to 240% of baseline on noise alone and dips -under the 50% line in 9% of windows, while the CUSUM peaks at 53% of its threshold and +shows why. On a book whose true edge is fully intact and **above** baseline, +the 200-trade trailing read ranges from -25% to 197% of baseline on noise alone and dips +under the 50% line in 11% of windows, while the CUSUM peaks at 82% of its threshold and never fires. A "cut at 50% of baseline" rule would have cut a healthy book on whichever window you happened to read. +## Deflating the baseline + +Defect 1 was the largest gap on this page for as long as it stood: the argument for +building the monitor here rather than copying the reference implementation rested on +anchoring to a search-corrected number, and nothing in the package produced one. +`deflated_sharpe` corrects a Sharpe and returns a **probability**, which is the right +output for a gate and useless to a monitor. A monitor needs a number in R. + +`validation.deflated_expectancy` writes the conversion, on the bar `deflated_sharpe` +already uses: + +``` +SR0 = expected MAXIMUM per-trade Sharpe of N noise trials + (Bailey/López de Prado, scaled by the spread of the trial Sharpes) +deflated = mu - sigma * SR0 +``` + +The bar lives in Sharpe units, so it is carried back into R by the winner's own sigma +before being subtracted. Both functions now call one `_expected_max_sharpe`, so the two +corrections for one search cannot disagree about how big the search was. + +```python +from crucible.validation import deflated_expectancy, EdgeBaseline + +d = deflated_expectancy(winner.r, [t.r for t in every_variant_tried], n_trials=log) +base = EdgeBaseline.from_log(winner, deflated_expectancy=d, n_variants=log.n_variants) +``` + +Three decisions worth recording. + +**It takes trial LOGS, not trial Sharpes**, unlike `deflated_sharpe`. The Sharpes are +computed inside, so their clock cannot be got wrong. A per-month Sharpe and a per-trade +Sharpe are different numbers on different scales, and multiplying the wrong one by a +per-trade sigma produces a haircut in no units at all, silently. That is the v0.4.0 units +bug in a new costume, and the fix is to not accept the ambiguous input. + +**It is a bias correction, not a significance test, and the docstring says so in those +words.** It removes the selection bias a search of this size is *expected* to produce. +The realized maximum sits above its own mean about half the time, so a pure-noise winner +still clears zero here roughly as often as not: measured at 56% / 47% / 44% for N = 5 / +20 / 100, while `deflated_sharpe` correctly calls 0% of the same draws significant +(reproducer: +`tests/test_deflated_expectancy.py::test_the_haircut_is_a_bias_correction_not_a_test`). +The result object's property is therefore named `is_positive` rather than `survives`, +because the first draft called it `survives` and that reads as a verdict it does not +deliver. Establish the edge is real with the gauntlet; use this to decide what to anchor +to afterwards. + +**It over-corrects a genuine edge, deliberately.** A winner chosen partly for real signal +carries less selection bias than the pure-luck maximum being subtracted, so the deflated +number sits below the truth. For a monitor baseline that is the safer direction: too low +a bar makes the monitor slow to call decay, too high a bar makes it cry wolf, and a +spurious alarm forces a re-optimization that taxes the honest N of the next verdict. + +A correction that leaves nothing raises rather than returning a smaller baseline. +`EdgeBaseline` already refused a non-positive expectancy; the message now names deflation +as a cause, because a book whose edge does not survive its own search is not a monitoring +problem. + ## Settled These were the open questions this page carried before #109. Merging answered them, so @@ -268,25 +333,14 @@ they are recorded here with what decided them rather than left looking live. Ordered by how much each one undercuts the argument for the module. -1. **Nothing deflates an expectancy.** `deflated_expectancy` is still a number the - caller supplies. `deflated_sharpe` corrects a Sharpe ratio, not a per-trade mean, so - the conversion is unwritten. This is the largest gap: anchoring to a search-corrected - baseline was the whole reason to build this here rather than copy the reference - implementation, and until it exists that advantage is a docstring rather than a - feature. -2. **Nothing calls it.** The monitor needs a caller to freeze an `EdgeBaseline` at the - moment of promotion and hold it. In this stack that is `crucible_stack.orchestrate` - (which owns the `DeploymentLedger` and the promotion event) or `livebook`. Neither - knows the module exists, so today it runs only in its own tests. Sibling-repo work, - not crucible's. -3. **The firing-rate channel is uncalibrated.** It compares a ratio to a threshold, so +1. **The firing-rate channel is uncalibrated.** It compares a ratio to a threshold, so it can only ever say `SLIPPING`. Trade arrivals are approximately Poisson, so an arrival-process test would give it a stated false-alarm rate and let it stand beside the CUSUM. -4. **It has never met real decay.** Only synthetic decay, generated to test it. The ARL +2. **It has never met real decay.** Only synthetic decay, generated to test it. The ARL figures are design targets, not field results. The first honest test is the first promoted book that genuinely degrades. -5. **The README still does not mention the module,** though it enumerates every other +3. **The README still does not mention the module,** though it enumerates every other subpackage's API with a worked block. The worked example, tutorial §14 and the `report.monitor_panel` block all exist now; the README is the last documentation gap. @@ -301,11 +355,11 @@ amount of code. The improvement over the reference version is not the detector. It is what the detector is anchored to: a search-corrected expectancy instead of the optimized in-sample one, denominated in R instead of percent of account, watched alongside the opportunity set -rather than in isolation. Two of those three are built. The first is not, which is why -it sits at the top of [Still open](#still-open). +rather than in isolation. All three are now built, the first of them last and only after +this page had carried it as the top open item for several revisions. The 2020 dip in the reference output is the uncalibrated alarm firing while the calibrated one stayed silent. That is not evidence the monitor works, and the same pattern reproduces here: in §14 of the tutorial, a book whose edge never decayed shows a -trailing read swinging between -31% and 240% of baseline. Hence the rule that only a +trailing read swinging between -25% and 197% of baseline. Hence the rule that only a detector with a stated false-alarm rate may escalate to `DEGRADED`. diff --git a/docs/tutorial.md b/docs/tutorial.md index 38dd1fa..3d80417 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -584,15 +584,27 @@ the complementary question: given that you searched N configs and kept the best- series' own **skew and kurtosis** (fat left tails widen the error bar). Read `≥ 95%` like a passed significance test. -Where the permutation test corrects the *p-value* for the search, these correct the *IS ranking* -and the *Sharpe* for it, the same multiple-testing disease, caught two more ways. Capital-free -(stdlib `NormalDist`, no scipy). - -Unlike the trade-log tests, these two aren't drawn on the report or wired into the gauntlet. +- **Deflated expectancy** (`deflated_expectancy`). The same bar, `SR0`, subtracted rather than + tested against: `deflated = mu − sigma × SR0`, a per-trade edge in **R** with the search's + expected luck removed. It exists because a probability cannot anchor a monitor (§14) and the + raw sample mean is the number the parameters were optimized on. Pass the trial **logs**, not + trial Sharpes, so the per-trade clock can't be got wrong. + + Read it as a **bias correction, not a test**. It strips the selection bias a search of this + size is *expected* to produce, so a pure-noise winner still clears zero about half the time, + where `deflated_sharpe` correctly calls none of them significant. Establish the edge is real + with the gauntlet, then use this to decide what to anchor to. + +Where the permutation test corrects the *p-value* for the search, these correct the *IS ranking*, +the *Sharpe*, and the *expectancy* for it, the same multiple-testing disease caught three more +ways. Capital-free (stdlib `NormalDist`, no scipy). + +Unlike the trade-log tests, these aren't drawn on the report or wired into the gauntlet. They need the **whole search** as input (`pbo_cscv` a `T×N` periods×configs matrix. -`deflated_sharpe` the winner's Sharpe plus the trial count), which a single `TradeLog` doesn't -carry. Call them yourself with that matrix, same story as the block bootstrap (§3): a standalone -check that runs on a different object than the one the report shows. +`deflated_sharpe` the winner's Sharpe plus the trial count, `deflated_expectancy` one R series +per config), which a single `TradeLog` doesn't carry. Call them yourself with that matrix, same +story as the block bootstrap (§3): a standalone check that runs on a different object than the +one the report shows. > **Sources.** **PBO / CSCV**: Bailey, Borwein, López de Prado & Zhu (2017), "The Probability of > Backtest Overfitting," *Journal of Computational Finance*. **AFML Ch. 11–12**. **Deflated / @@ -1229,32 +1241,45 @@ The runnable version is seeded and synthetic, so these numbers reproduce exactly. ```python -from crucible.validation import EdgeBaseline, cusum_design, edge_monitor, empirical_arl +from crucible.validation import ( + EdgeBaseline, cusum_design, deflated_expectancy, edge_monitor, empirical_arl, +) -validated = synthetic_book(2000, seed=1) # the book that passed -base = EdgeBaseline.from_log(validated, deflated_expectancy=..., n_variants=64) +validated = max(trials, key=sharpe) # the config the search kept +corrected = deflated_expectancy(validated.r, [t.r for t in trials], n_trials=64) +base = EdgeBaseline.from_log(validated, deflated_expectancy=corrected, n_variants=64) verdict = edge_monitor(live_book, base) # HOLDING | SLIPPING | DEGRADED ``` ### Step 1: freeze the baseline, and deflate it (§5, §11) +The example runs a real 64-config search and keeps the best, which is the config whose *luck* +ran highest as well as whose edge did. That is the whole problem in one line. + ``` -in-sample mean +0.2088R over 2000 trades -deflated baseline +0.1671R deflated=True -naive baseline +0.2088R deflated=False +in-sample mean +0.2751R over 2000 trades +search haircut -0.0723R (SR0 0.0503 x sigma 1.4386, N=64) +deflated baseline +0.2027R deflated=True +naive baseline +0.2751R deflated=False firing rate 150 trades/yr ``` The in-sample mean is the number the parameters were **optimized on**, so it is biased high in exactly the way §5 describes. Anchoring the monitor to it makes "half the baseline" mean half of -something inflated. Here a 20% search correction moves the reference from `+0.2088R` to -`+0.1671R`, so every ratio measured against the naive figure is flattered by **25%**. +something inflated. `deflated_expectancy` subtracts the expected best-of-64 under the null, +`mu − sigma × SR0`, moving the reference from `+0.2751R` to `+0.2027R`: the naive figure flatters +every ratio measured against it by **36%**, and **74%** of the raw edge survives. + +That correction is not cosmetic. Run the same untouched, fully healthy live book against both +baselines and the naive one returns **DEGRADED** while the deflated one returns **HOLDING** +(`test_the_deflation_is_what_keeps_the_healthy_book_holding`). An inflated reference does not +merely mis-scale the ratios, it manufactures alarms on books that never decayed. -Two properties keep this honest. `EdgeBaseline` is **frozen at promotion**, and `edge_monitor` -has *no parameter from which one could be rebuilt*, because a baseline recomputed from current -data re-fits onto the drifted reality and can never fire. And an undeflated baseline is allowed -but never silent: `deflated=False` rides in the verdict rather than being validated away, the -same discipline `variant_count()` applies to a typed-in N (§5b). +Two further properties keep this honest. `EdgeBaseline` is **frozen at promotion**, and +`edge_monitor` has *no parameter from which one could be rebuilt*, because a baseline recomputed +from current data re-fits onto the drifted reality and can never fire. And an undeflated baseline +is allowed but never silent: `deflated=False` rides in the verdict rather than being validated +away, the same discipline `variant_count()` applies to a typed-in N (§5b). ### Step 2: design the detector, then check its claims @@ -1262,14 +1287,14 @@ You state the shift worth catching and the false-alarm budget; `k` and `h` follo ``` detects a shift to 50% of baseline -k = +0.1253R h = 37.71 sigma +k = +0.1520R h = 35.09 sigma nominal ARL0 = 3,752 trades = 25.0 years (mean, between false alarms) at 150 trades/yr -nominal ARL1 = 806 trades = 5.4 years (mean, to detect the design shift) +nominal ARL1 = 658 trades = 4.4 years (mean, to detect the design shift) ``` `k` is the textbook midpoint `(mu_0 + mu_1)/2`. **The budget is stated in calendar time, and the cost comes back in the same unit:** you are buying one false alarm per 25 years and paying -about 5.4 years to notice a halved edge. Both are numbers to argue with *before* deployment. +about 4.4 years to notice a halved edge. Both are numbers to argue with *before* deployment. That the budget is in years rather than trades matters more than it looks. `monitor_arl0_years` is converted using the baseline's own firing rate, so one default means the same thing to every @@ -1281,15 +1306,15 @@ Those ARLs come from a Gaussian approximation, and trade R is emphatically not G check rather than assume, by resampling the book's own returns: ``` -in control mean 3,973 / median 2,951 trades vs nominal mean 3,752 (1.06x) -edge halved mean 811 / median 686 trades vs nominal mean 806 (1.01x) +in control mean 4,087 / median 3,244 trades vs nominal mean 3,752 (1.09x) +edge halved mean 632 / median 500 trades vs nominal mean 658 (0.96x) ``` Close here, because this synthetic book is only mildly skewed. **Do not read that as general.** On a real pooled trend book (skew about +5, single trades near +39R) the in-control figure runs about **2.0x** nominal, and the error grows with skew and with the size of the budget. -Note which line stays accurate: the **shifted** one, at 1.01x. Skew inflates the in-control run +Note which line stays accurate: the **shifted** one, at 0.96x. Skew inflates the in-control run length, because in control the statistic hovers near zero and only alarms via rare large excursions, which is exactly where a fat tail bites. Under a real shift it reaches the boundary by drift, where tail shape barely matters. So a skewed book gets *fewer* false alarms than @@ -1302,28 +1327,28 @@ are strongly right-skewed. Quoting one against someone else's other misstates la | Live book | Verdict | Trailing 200-trade read | Firing rate | CUSUM | |---|---|---|---|---| -| edge intact | **HOLDING** | 110% of baseline | 100% | peak 67% of threshold | -| edge halved | **DEGRADED** | -16% | 100% | alarm at live trade 1,138 | -| signal drying up | **SLIPPING** | 88% (intact) | **33%** | peak 53%, silent | +| edge intact | **HOLDING** | 91% of baseline | 100% | peak 82% of threshold | +| edge halved | **DEGRADED** | -13% | 100% | alarm at live trade 276 | +| signal drying up | **SLIPPING** | 72% (intact) | **33%** | peak 61%, silent | The third row is the failure mode an expectancy-only monitor cannot see. That book's per-trade -edge is **fine** (88% of baseline, well inside noise); the signal simply stopped firing, 50 +edge is **fine** (72% of baseline, well inside noise); the signal simply stopped firing, 50 trades a year against a baseline of 150. Annual R falls by two thirds with per-trade expectancy untouched. Opportunity-set decay is a distinct failure from edge decay and needs its own channel. ### Step 4: why the soft channel cannot say DEGRADED -The "edge intact" book was generated with the edge **fully intact**, a quarter *above* the -deflated baseline. It never decayed. Yet: +The "edge intact" book was generated with the edge **fully intact**, 6% *above* the deflated +baseline. It never decayed. Yet: ``` -trailing 200-trade read ranges -31% to 240% of baseline, on noise alone -it dips below the 50% line in 9% of windows -the CUSUM peaks at 53% of threshold and never fires +trailing 200-trade read ranges -25% to 197% of baseline, on noise alone +it dips below the 50% line in 11% of windows +the CUSUM peaks at 82% of threshold and never fires ``` A "cut size at 50% of baseline" rule would have cut this healthy book on whichever of those -windows you happened to read. Today's read is 110%, which looks reassuring, and that is the same +windows you happened to read. Today's read is 91%, which looks reassuring, and that is the same coin landing the other way up. Both are noise. That is why the split is structural rather than advisory: **only a detector with a stated diff --git a/examples/edge_monitor.py b/examples/edge_monitor.py index b905ee9..0d621ef 100644 --- a/examples/edge_monitor.py +++ b/examples/edge_monitor.py @@ -14,6 +14,7 @@ EdgeBaseline, Thresholds, cusum_design, + deflated_expectancy, edge_monitor, empirical_arl, rolling_expectancy, @@ -42,25 +43,49 @@ def thin(trades: TradeLog, keep_every: int) -> TradeLog: return TradeLog(trades.frame.iloc[::keep_every].reset_index(drop=True)) -def main(): - # ── 1. the validated book, and the baseline frozen from it ────────────────── - validated = synthetic_book(2000, seed=1, start="2015-01-01") - raw_mean = float(validated.r.mean()) - - # The number the parameters were optimized on is biased high. Pretend the - # search correction knocked 20% off it; anchoring to the corrected figure is +def promoted_book(): + """The search, the config it kept, and the baseline frozen from it. + + A function rather than inline setup so the tutorial's figures have exactly ONE + source: `tests/test_edge_monitor_example.py` imports this instead of rebuilding + it. A test that rebuilds the setup it is meant to pin passes happily while the + example it guards prints something else, which is what happened here. + """ + # A real search: 64 configs scored, the best kept. The others are neighbouring + # variants of the same book, which is what a parameter sweep actually looks + # like, and their SPREAD is what sets how high a Sharpe luck alone could reach. + rng = np.random.default_rng(99) + trials = [synthetic_book(2000, edge=e, seed=1 + i, start="2015-01-01") + for i, e in enumerate(rng.uniform(0.85, 1.0, 64))] + # The config the search KEEPS is the best in sample, which is the one whose luck + # ran highest as well as whose edge ran highest. That is the whole problem. + validated = max(trials, key=lambda t: t.r.mean() / t.r.std(ddof=1)) + + # The number the parameters were optimized on is biased high. Subtract what a + # 64-wide search could have found by luck; anchoring to the corrected figure is # the whole reason to run this in crucible rather than by hand. - honest = EdgeBaseline.from_log(validated, deflated_expectancy=raw_mean * 0.8, + corrected = deflated_expectancy(validated.r, [t.r for t in trials], n_trials=64) + honest = EdgeBaseline.from_log(validated, deflated_expectancy=corrected, n_variants=64) + return validated, corrected, honest + + +def main(): + # ── 1. the search, the winner, and the baseline frozen from it ────────────── + validated, corrected, honest = promoted_book() + raw_mean = float(validated.r.mean()) naive = EdgeBaseline.from_log(validated) # deflated=False, and it says so print("1) THE FROZEN BASELINE") print(f" in-sample mean {raw_mean:+.4f}R over {validated.n} trades") + print(f" search haircut {-corrected.haircut:+.4f}R " + f"(SR0 {corrected.sr0_threshold:.4f} x sigma {corrected.sigma:.4f}, N=64)") print(f" deflated baseline {honest.expectancy:+.4f}R deflated={honest.deflated}") print(f" naive baseline {naive.expectancy:+.4f}R deflated={naive.deflated}") print(f" firing rate {honest.trades_per_year:.0f} trades/yr") - print(" The naive baseline is 25% higher, so every ratio measured against it") - print(" is flattered by the same amount. Build this ONCE, at promotion.") + print(f" The naive baseline is {naive.expectancy / honest.expectancy - 1:.0%} higher, " + "so every ratio measured") + print(" against it is flattered by the same amount. Build this ONCE, at promotion.") # ── 2. the detector, designed rather than typed in ────────────────────────── design = cusum_design(honest) @@ -93,10 +118,12 @@ def main(): roll = rolling_expectancy(healthy, v.window).dropna() below = float((roll < honest.expectancy * Thresholds().monitor_slip_ratio).mean()) + healthy_mean = float(healthy.r.mean()) print("\n5) THE REASON THE SOFT CHANNEL CANNOT SAY 'DEGRADED'") print(" The 'edge intact' book was generated with edge=1.0, so its TRUE per-trade") - print(f" edge is {raw_mean:+.4f}R, a quarter ABOVE the {honest.expectancy:+.4f}R " - "baseline. It never decayed.") + print(f" edge is {healthy_mean:+.4f}R, " + f"{healthy_mean / honest.expectancy - 1:.0%} ABOVE the " + f"{honest.expectancy:+.4f}R baseline. It never decayed.") print(f" Even so, its trailing {v.window}-trade read dips below the 50% line in " f"{below:.0%} of windows,") print(f" ranging {roll.min() / honest.expectancy:.0%} to " diff --git a/src/crucible/validation/__init__.py b/src/crucible/validation/__init__.py index f0b96a7..9738961 100644 --- a/src/crucible/validation/__init__.py +++ b/src/crucible/validation/__init__.py @@ -12,8 +12,9 @@ permutation sign-permutation p-value, data-mining correction, White's Reality Check + Hansen's SPA (its more powerful successor) across every variant you searched - pbo probability of backtest overfitting (CSCV) + deflated Sharpe — - how much the ACT OF SELECTING the best config overfit + pbo probability of backtest overfitting (CSCV), deflated Sharpe, and + deflated expectancy: how much the ACT OF SELECTING the best config + overfit, as a probability and as an edge in R search_space the search ledger — an honest N for the data-mining correction, counting every variant tried (not just the winner you kept) gate an audited, un-overridable pass/fail gate (and a Gauntlet of them) @@ -52,8 +53,10 @@ rolling_expectancy, ) from crucible.validation.pbo import ( + DeflatedExpectancy, DeflatedSharpe, PBOResult, + deflated_expectancy, deflated_sharpe, pbo_cscv, ) @@ -85,6 +88,7 @@ "sign_permutation_pvalue", "sidak_correction", "variant_count", "whites_reality_check", "spa_test", "pbo_cscv", "PBOResult", "deflated_sharpe", "DeflatedSharpe", + "deflated_expectancy", "DeflatedExpectancy", "SearchSpaceLog", "Gate", "GateCheck", "Gauntlet", "fold_dispersion", "walk_forward_efficiency", diff --git a/src/crucible/validation/monitor.py b/src/crucible/validation/monitor.py index 4e8b687..ff13800 100644 --- a/src/crucible/validation/monitor.py +++ b/src/crucible/validation/monitor.py @@ -109,7 +109,10 @@ def __post_init__(self) -> None: raise ValueError( f"baseline expectancy must be positive and finite, got {self.expectancy}. " "Monitoring the decay of an edge that was never positive is meaningless; " - "the thing to run on a non-positive log is the gauntlet, not the monitor." + "the thing to run on a non-positive log is the gauntlet, not the monitor. " + "If this came from `deflated_expectancy`, the search's noise ceiling " + "exceeded the edge: the answer is not to monitor a smaller number, it is " + "that the book did not survive its own search." ) if not math.isfinite(self.sigma) or self.sigma <= 0: raise ValueError(f"baseline sigma must be positive and finite, got {self.sigma}") @@ -117,7 +120,7 @@ def __post_init__(self) -> None: raise ValueError(f"baseline needs at least 2 trades, got {self.n_trades}") @classmethod - def from_log(cls, trades: TradeLog, *, deflated_expectancy: Optional[float] = None, + def from_log(cls, trades: TradeLog, *, deflated_expectancy: Optional[object] = None, n_variants: Optional[int] = None, trades_per_year: Optional[float] = None) -> "EdgeBaseline": """Measure a baseline from the log the edge was validated on. Call this ONCE, at @@ -127,18 +130,30 @@ def from_log(cls, trades: TradeLog, *, deflated_expectancy: Optional[float] = No Supplying it sets `deflated=True`. Omitting it falls back to the log's sample mean, which is the number the parameters were optimized on and is therefore biased high; that is recorded as `deflated=False`, not silently accepted. + + Accepts either a float or the `DeflatedExpectancy` that + `crucible.validation.deflated_expectancy` returns, read by attribute. Prefer + passing the object: it carries an `observed_expectancy` alongside the deflated + one, and handing over the wrong field of a result you already computed anchors + the monitor to the pre-correction number while reporting `deflated=True`, which + is worse than not deflating at all. + + A correction that leaves nothing (`deflated_expectancy <= 0`) raises here, via + the same check that refuses a negative raw edge. That is not a monitor to build + with a smaller number; it is a book whose edge did not survive its own search, + and the honest response is to not deploy it. """ r = trades.r if trades_per_year is None: trades_per_year = _trades_per_year(trades) + value = getattr(deflated_expectancy, "deflated_expectancy", deflated_expectancy) return cls( - expectancy=float(deflated_expectancy if deflated_expectancy is not None - else r.mean()), + expectancy=float(value if value is not None else r.mean()), sigma=float(r.std(ddof=1)), n_trades=int(r.size), trades_per_year=trades_per_year, n_variants=n_variants, - deflated=deflated_expectancy is not None, + deflated=value is not None, ) diff --git a/src/crucible/validation/pbo.py b/src/crucible/validation/pbo.py index 700b01e..4399bfe 100644 --- a/src/crucible/validation/pbo.py +++ b/src/crucible/validation/pbo.py @@ -225,6 +225,24 @@ def __str__(self) -> str: ]) +def _expected_max_sharpe(sd_trials: float, n_trials: int) -> float: + """The multiple-testing bar: expected MAXIMUM Sharpe of `n_trials` noise trials. + + Bailey/López de Prado's two-point blend of the (1 - 1/N) and (1 - 1/(N e)) standard + normal quantiles, scaled by the dispersion of the trial Sharpes. The dispersion is + what makes this a property of *this* search rather than of N alone: a sweep over + near-identical variants has little spread to exploit and earns a small bar, while a + sweep whose configs scatter widely could have hit a high Sharpe by luck. + + Shared by `deflated_sharpe` (which asks whether the winner CLEARS this bar) and + `deflated_expectancy` (which subtracts it). + """ + return float(sd_trials * ( + (1.0 - _EULER_GAMMA) * _NORM.inv_cdf(1.0 - 1.0 / n_trials) + + _EULER_GAMMA * _NORM.inv_cdf(1.0 - 1.0 / (n_trials * np.e)) + )) + + def _psr(sr: float, sr_star: float, n: int, skew: float, kurt: float) -> float: """Probabilistic Sharpe Ratio: P(true SR > sr_star) given `n` observations and the return distribution's skew/kurtosis (Bailey & López de Prado 2012). The @@ -285,13 +303,7 @@ def deflated_sharpe(trial_sharpes: Returns, *, returns: Returns, skew = float((z ** 3).mean()) kurt = float((z ** 4).mean()) # non-excess: 3.0 under normality - var_sr = float(np.var(sr_trials, ddof=1)) - # Expected max of N standard-normal draws (Bailey/LdP): a two-point blend of the - # (1 - 1/N) and (1 - 1/(N e)) quantiles, scaled by the trial-Sharpe dispersion. - sr0 = np.sqrt(var_sr) * ( - (1.0 - _EULER_GAMMA) * _NORM.inv_cdf(1.0 - 1.0 / N) - + _EULER_GAMMA * _NORM.inv_cdf(1.0 - 1.0 / (N * np.e)) - ) + sr0 = _expected_max_sharpe(float(np.std(sr_trials, ddof=1)), N) dsr = _psr(sr_obs, float(sr0), n, skew, kurt) return DeflatedSharpe( @@ -303,3 +315,161 @@ def deflated_sharpe(trial_sharpes: Returns, *, returns: Returns, skew=skew, kurtosis=kurt, ) + + +# --------------------------------------------------------------------------- # +# Deflated expectancy +# --------------------------------------------------------------------------- # + +@dataclass +class DeflatedExpectancy: + """A per-trade edge with the search's expected luck subtracted, in R. + + The companion to `DeflatedSharpe`, and deliberately a different KIND of answer. + `deflated_sharpe` returns a probability ("does this clear the bar?"), which is the + right output for a gate. A monitor cannot anchor to a probability: it needs a + *number* to compare live trades against, and anchoring to the raw sample mean anchors + to the number the search optimized. That gap is what this closes. + """ + observed_expectancy: float # raw per-trade mean R, the number the search maximized + deflated_expectancy: float # what survives the multiple-testing bar. May be <= 0. + haircut: float # observed - deflated, in R + sigma: float # per-trade dispersion in R + sr0_threshold: float # the bar, in per-trade Sharpe units + n_trials: int # honest N the correction was priced against + n_trades: int + n_scored: int # trial logs actually supplied + + @property + def is_positive(self) -> bool: + """Whether any edge is left after the haircut. Named for exactly what it checks. + + **This is not a significance test and must not be read as one.** Subtracting the + EXPECTED maximum removes the average selection bias, and the realized maximum + lands above its own mean about half the time, so a pure-noise winner clears zero + here roughly as often as not (measured: 56% / 47% / 44% for N = 5 / 20 / 100, + against `deflated_sharpe` correctly calling 0% of them significant. Reproducer: + `tests/test_deflated_expectancy.py::test_the_haircut_is_a_bias_correction_not_a_test`). + + Ask `deflated_sharpe` whether the edge is real. Ask this how much of it to + believe once you already know that it is. + """ + return self.deflated_expectancy > 0 + + @property + def retained(self) -> float: + """Fraction of the raw edge left after the haircut. Negative when it exceeded it.""" + return (self.deflated_expectancy / self.observed_expectancy + if self.observed_expectancy != 0 else float("nan")) + + def __str__(self) -> str: + return "\n".join([ + "=" * 60, + " DEFLATED EXPECTANCY (per trade, in R)", + "=" * 60, + f"trials searched : {self.n_trials} scored: {self.n_scored}", + f"trades / sigma : {self.n_trades} / {self.sigma:.3f} R", + "-" * 60, + f"observed expectancy : {self.observed_expectancy:+.4f} R (optimized on)", + f"search haircut : {-self.haircut:+.4f} R " + f"(SR0 {self.sr0_threshold:+.3f} x sigma)", + f"deflated expectancy : {self.deflated_expectancy:+.4f} R " + f"[{'EDGE REMAINS' if self.is_positive else 'NOTHING LEFT'}]", + f" retained : {self.retained:5.1%} of the raw edge", + "-" * 60, + " a bias correction, NOT a significance test:", + " ask deflated_sharpe whether the edge is real.", + "=" * 60, + ]) + + +def deflated_expectancy(returns: Returns, trial_returns: Sequence[Returns], *, + n_trials: Optional[object] = None) -> DeflatedExpectancy: + """Subtract what the search could have found by luck from a per-trade expectancy. + + `returns` is the winning config's own per-TRADE R series; `trial_returns` is one + per-trade R series per config you SCORED, the winner included. Both are per-trade, + and that is load-bearing. + + The bar is `deflated_sharpe`'s SR0: the expected maximum per-trade Sharpe of N noise + trials, scaled by the spread of the trial Sharpes. Converting it back to R and + subtracting gives the edge that survives:: + + deflated = mu - sigma * SR0 + + **This takes trial LOGS, not trial Sharpes, unlike `deflated_sharpe`.** The Sharpes + are computed here so their clock cannot be got wrong. A per-MONTH Sharpe and a + per-TRADE Sharpe are different numbers on different scales, and multiplying the wrong + one by a per-trade sigma yields a haircut in no units at all, silently. That is the + v0.4.0 units bug in a new costume, and the way to not have it is to not accept the + ambiguous input. + + `n_trials` separates configs TRIED from configs SCORED exactly as in + `deflated_sharpe`: pass an int or a `SearchSpaceLog`, and the bar rises to match. + Omitting it prices the correction at the number of logs supplied, which is right only + when every variant produced a scoreable result. + + **The result can be non-positive, and is returned rather than raised.** A haircut + exceeding the raw edge means the search did not beat its own noise ceiling. That is + a finding about the search, so the caller gets to see it; what refuses is + `EdgeBaseline`, which will not build a monitor around a non-positive edge. + + **This is a bias correction, not a significance test.** It removes the selection bias + a search of this size is EXPECTED to produce, which leaves a positive number for a + pure-noise winner about half the time (see `DeflatedExpectancy.is_positive`). Run it + on a book you have already established is real, to decide what to anchor to. Run + `deflated_sharpe` to establish that in the first place. Using this as a screen would + pass roughly half of pure noise. + + It errs toward over-correcting a genuine edge, because a winner chosen partly for + real signal carries less selection bias than the pure-luck maximum being subtracted. + For a monitor baseline that is the safer direction: too low a bar makes the monitor + slow to call decay, too high a bar makes it cry wolf, and a spurious alarm here + forces a re-optimization that taxes the honest N of the next verdict. + """ + r = np.asarray(returns, dtype=float) + r = r[np.isfinite(r)] + n = len(r) + if n < 2: + raise ValueError(f"need >= 2 trades to measure an expectancy, got {n}") + + sr_trials = [] + for t in trial_returns: + a = np.asarray(t, dtype=float) + a = a[np.isfinite(a)] + if a.size < 2: + continue # too thin to score, same posture as a NaN Sharpe + sd_t = a.std(ddof=1) + sr_trials.append(float(a.mean() / sd_t) if sd_t > 0 else 0.0) + scored = len(sr_trials) + if scored < 2: + raise ValueError( + f"need >= 2 scoreable trial logs to estimate the search's variance, got {scored}. " + "With one trial there is no search to correct for, and the expected maximum " + "of a single draw is not defined by this approximation." + ) + + ledger_n = getattr(n_trials, "session_n_variants", None) + N = scored if n_trials is None else int(ledger_n if ledger_n is not None else n_trials) + if N < scored: + raise ValueError( + f"n_trials={N} is fewer than the {scored} trial logs supplied; a search " + "cannot have tried fewer configs than it scored") + + sigma = float(r.std(ddof=1)) + if sigma <= 0: + raise ValueError("winning log has zero dispersion; there is no Sharpe to deflate") + + sr0 = _expected_max_sharpe(float(np.std(np.asarray(sr_trials), ddof=1)), N) + haircut = sigma * sr0 + mu = float(r.mean()) + return DeflatedExpectancy( + observed_expectancy=mu, + deflated_expectancy=mu - haircut, + haircut=float(haircut), + sigma=sigma, + sr0_threshold=float(sr0), + n_trials=N, + n_trades=n, + n_scored=scored, + ) diff --git a/tests/test_deflated_expectancy.py b/tests/test_deflated_expectancy.py new file mode 100644 index 0000000..34d8109 --- /dev/null +++ b/tests/test_deflated_expectancy.py @@ -0,0 +1,224 @@ +"""Deflating a per-trade expectancy for the size of the search that found it. + +`deflated_sharpe` answers "is it real?" with a probability. A monitor cannot anchor to a +probability, so this answers "how much of it should I believe?" with a number in R. The +two share one bar (`_expected_max_sharpe`) and must not drift apart. +""" +import numpy as np +import pytest + +from crucible.edge import TradeLog +from crucible.validation import ( + DeflatedExpectancy, + EdgeBaseline, + deflated_expectancy, + deflated_sharpe, +) +from crucible.validation.search_space import SearchSpaceLog + + +def _trials(mu, n_trials, n_trades, seed=0, sigma=1.0): + rng = np.random.default_rng(seed) + return [rng.normal(mu, sigma, n_trades) for _ in range(n_trials)] + + +def _winner(trials): + sr = [t.mean() / t.std(ddof=1) for t in trials] + return trials[int(np.argmax(sr))] + + +# ── the arithmetic ────────────────────────────────────────────────────────────────── + +def test_the_haircut_is_sigma_times_the_bar(): + """deflated = mu - sigma * SR0. The bar is in Sharpe units, so it has to be carried + back into R before it can be subtracted from an expectancy.""" + t = _trials(0.2, 8, 2000, seed=1) + d = deflated_expectancy(t[0], t) + assert d.haircut == pytest.approx(d.sigma * d.sr0_threshold) + assert d.deflated_expectancy == pytest.approx(d.observed_expectancy - d.haircut) + assert d.observed_expectancy == pytest.approx(float(np.mean(t[0]))) + assert d.sigma == pytest.approx(float(np.std(t[0], ddof=1))) + + +def test_it_shares_one_bar_with_the_deflated_sharpe(): + """Same SR0, reached from both directions. Two corrections for one search that + disagreed about how big the search was would be worse than either alone.""" + t = _trials(0.2, 10, 1500, seed=2) + w = _winner(t) + de = deflated_expectancy(w, t) + ds = deflated_sharpe([x.mean() / x.std(ddof=1) for x in t], returns=w) + assert de.sr0_threshold == pytest.approx(ds.sr0_threshold) + + +def test_a_bigger_search_takes_a_bigger_haircut(): + t = _trials(0.2, 6, 3000, seed=3) + bars = [deflated_expectancy(t[0], t, n_trials=N).haircut + for N in (6, 50, 500, 5000)] + assert bars == sorted(bars) + assert bars[0] < bars[-1] + + +def test_retained_is_the_surviving_fraction(): + t = _trials(0.3, 5, 4000, seed=4) + d = deflated_expectancy(t[0], t) + assert d.retained == pytest.approx(d.deflated_expectancy / d.observed_expectancy) + assert 0 < d.retained < 1 + + +# ── what it is, and what it is not ────────────────────────────────────────────────── + +def test_the_haircut_is_a_bias_correction_not_a_test(): + """The reproducer for the figures quoted in `DeflatedExpectancy.is_positive`. + + Under pure noise the raw mean of the winner is positive essentially always (it was + selected for being the maximum). Subtracting the EXPECTED maximum removes the average + selection bias, and the realized maximum sits above its own mean about half the time, + so the deflated number clears zero roughly as often as not. `deflated_sharpe`, which + is an actual test, calls none of them significant. + + This is why `is_positive` must not be read as a pass. + """ + rng = np.random.default_rng(7) + reps = 400 + out = {} + for n_trials in (5, 20, 100): + pos_raw = pos_def = sig = 0 + for _ in range(reps): + r = rng.normal(0.0, 1.0, (n_trials, 400)) + sr = r.mean(1) / r.std(1, ddof=1) + win = r[int(np.argmax(sr))] + d = deflated_expectancy(win, list(r)) + pos_raw += d.observed_expectancy > 0 + pos_def += d.is_positive + sig += deflated_sharpe(list(sr), returns=win).deflated_sharpe >= 0.95 + out[n_trials] = (pos_raw / reps, pos_def / reps, sig / reps) + + for n_trials, (raw, deflated, significant) in out.items(): + assert raw >= 0.95, f"N={n_trials}: the winner's raw mean should be positive" + assert 0.35 <= deflated <= 0.65, f"N={n_trials}: expected ~half, got {deflated:.0%}" + assert significant <= 0.02, f"N={n_trials}: DSR should reject noise, got {significant:.0%}" + + # and the correction still does its job: it strips essentially all of the fake edge + assert out[100][1] < out[100][0] + + +def test_it_over_corrects_a_real_edge_which_is_the_safe_direction_for_a_monitor(): + """A winner chosen partly for real signal carries less selection bias than the + pure-luck maximum being subtracted, so the deflated number sits BELOW the truth.""" + true_mu = 0.20 + t = _trials(true_mu, 40, 2000, seed=11) + d = deflated_expectancy(_winner(t), t) + assert d.deflated_expectancy < d.observed_expectancy + assert d.deflated_expectancy < true_mu * 1.02 + + +# ── the honest N ──────────────────────────────────────────────────────────────────── + +def test_a_search_space_log_is_accepted_directly(): + log = SearchSpaceLog(scope="trend:arm_x_regime") + for i in range(60): + log.record({"variant": i}) + t = _trials(0.2, 5, 2000, seed=5) + assert (deflated_expectancy(t[0], t, n_trials=log).n_trials + == deflated_expectancy(t[0], t, n_trials=60).n_trials == 60) + + +def test_claiming_fewer_trials_than_were_scored_is_refused(): + t = _trials(0.2, 9, 1000, seed=6) + with pytest.raises(ValueError, match="cannot have tried fewer configs than it scored"): + deflated_expectancy(t[0], t, n_trials=3) + + +def test_omitting_the_honest_n_prices_it_at_what_was_supplied(): + t = _trials(0.2, 7, 1000, seed=8) + assert deflated_expectancy(t[0], t).n_trials == 7 + + +# ── refusals ──────────────────────────────────────────────────────────────────────── + +def test_one_trial_is_not_a_search(): + """The expected maximum of a single draw is not defined by this approximation, and + there is no multiple testing to correct for anyway.""" + t = _trials(0.2, 1, 500, seed=9) + with pytest.raises(ValueError, match=">= 2 scoreable trial logs"): + deflated_expectancy(t[0], t) + + +def test_trials_too_thin_to_score_are_dropped_not_counted(): + t = _trials(0.2, 4, 800, seed=10) + d = deflated_expectancy(t[0], t + [np.array([0.5]), np.array([])]) + assert d.n_scored == 4 + + +def test_a_flat_winner_has_no_sharpe_to_deflate(): + with pytest.raises(ValueError, match="zero dispersion"): + deflated_expectancy(np.ones(100), _trials(0.2, 4, 500, seed=12)) + + +def test_too_few_trades_to_measure_an_expectancy(): + with pytest.raises(ValueError, match=">= 2 trades"): + deflated_expectancy([0.5], _trials(0.2, 4, 500, seed=13)) + + +# ── the seam into the monitor ─────────────────────────────────────────────────────── + +def test_the_baseline_takes_the_result_object_and_marks_itself_deflated(): + t = _trials(0.25, 8, 3000, seed=14) + w = _winner(t) + d = deflated_expectancy(w, t) + b = EdgeBaseline.from_log(TradeLog.from_arrays(w), deflated_expectancy=d) + assert b.deflated is True + assert b.expectancy == pytest.approx(d.deflated_expectancy) + assert b.expectancy < float(np.mean(w)) # strictly below the optimized number + + +def test_a_float_still_works_and_agrees_with_the_object(): + t = _trials(0.25, 8, 3000, seed=14) + w = _winner(t) + d = deflated_expectancy(w, t) + log = TradeLog.from_arrays(w) + assert (EdgeBaseline.from_log(log, deflated_expectancy=d.deflated_expectancy) + == EdgeBaseline.from_log(log, deflated_expectancy=d)) + + +def test_omitting_it_still_falls_back_to_the_optimized_mean(): + t = _trials(0.25, 8, 3000, seed=14) + b = EdgeBaseline.from_log(TradeLog.from_arrays(t[0])) + assert b.deflated is False + assert b.expectancy == pytest.approx(float(np.mean(t[0]))) + + +def test_a_correction_that_leaves_nothing_refuses_to_become_a_baseline(): + """Not a monitor to build with a smaller number: a book that did not survive its + own search. The refusal names the deflation as a possible cause.""" + rng = np.random.default_rng(15) + trials = [rng.normal(0.0, 1.0, 300) for _ in range(80)] + w = _winner(trials) + d = deflated_expectancy(w, trials, n_trials=100_000) + assert not d.is_positive + with pytest.raises(ValueError, match="did not survive its own search"): + EdgeBaseline.from_log(TradeLog.from_arrays(w), deflated_expectancy=d) + + +def test_the_deflated_baseline_lowers_the_monitors_bar_rather_than_raising_it(): + """The point of the whole exercise. Anchoring to the optimized mean tells the monitor + to expect more than the book can deliver, and it alarms on a book performing exactly + as it truly should.""" + from crucible.validation import cusum_design + + t = _trials(0.25, 30, 3000, seed=16) + w = _winner(t) + log = TradeLog.from_arrays(w) + raw = EdgeBaseline.from_log(log) + corrected = EdgeBaseline.from_log(log, deflated_expectancy=deflated_expectancy(w, t)) + assert corrected.expectancy < raw.expectancy + assert cusum_design(corrected).k_r < cusum_design(raw).k_r + + +def test_the_result_is_a_frozen_readable_record(): + t = _trials(0.2, 6, 1200, seed=17) + d = deflated_expectancy(t[0], t) + assert isinstance(d, DeflatedExpectancy) + text = str(d) + assert "DEFLATED EXPECTANCY" in text + assert "NOT a significance test" in text diff --git a/tests/test_edge_monitor_example.py b/tests/test_edge_monitor_example.py index d8e1e11..e5aeca1 100644 --- a/tests/test_edge_monitor_example.py +++ b/tests/test_edge_monitor_example.py @@ -3,6 +3,12 @@ The tutorial quotes these exact numbers, so they must not drift silently. A dependency bump that moves an ARL or flips a verdict should fail here, not surface as a wrong number in the published page. + +The baseline comes from `examples.edge_monitor.promoted_book`, NOT from a copy of +its setup. An earlier version of this file rebuilt the baseline itself, so when the +example switched from a hardcoded 20% haircut to a real `deflated_expectancy` every +assertion here still passed while every figure in the example changed. A guard that +reconstructs what it guards is not a guard. """ import sys from pathlib import Path @@ -19,96 +25,121 @@ empirical_arl, rolling_expectancy, ) -from examples.edge_monitor import main, synthetic_book, thin # noqa: E402 +from examples.edge_monitor import ( # noqa: E402 + main, + promoted_book, + synthetic_book, + thin, +) @pytest.fixture(scope="module") -def baseline(): - validated = synthetic_book(2000, seed=1, start="2015-01-01") - return validated, EdgeBaseline.from_log( - validated, deflated_expectancy=float(validated.r.mean()) * 0.8, n_variants=64) +def promoted(): + return promoted_book() + +def test_step1_the_search_and_the_deflated_baseline(promoted): + validated, corrected, base = promoted + assert validated.r.mean() == pytest.approx(0.2751, abs=1e-4) -def test_step1_frozen_baseline(baseline): - validated, base = baseline - assert validated.r.mean() == pytest.approx(0.2088, abs=1e-4) - assert base.expectancy == pytest.approx(0.1671, abs=1e-4) + # the haircut is sigma x SR0, priced against the honest N of 64 + assert corrected.n_trials == 64 + assert corrected.sr0_threshold == pytest.approx(0.0503, abs=1e-4) + assert corrected.sigma == pytest.approx(1.4386, abs=1e-4) + assert corrected.haircut == pytest.approx(0.0723, abs=1e-4) + assert corrected.retained == pytest.approx(0.737, abs=1e-3) + assert corrected.is_positive + + assert base.expectancy == pytest.approx(0.2027, abs=1e-4) assert base.trades_per_year == pytest.approx(150, abs=1) assert base.deflated is True - # the naive alternative, and the 25% flattery the tutorial quotes + + # the naive alternative, and the flattery the tutorial quotes naive = EdgeBaseline.from_log(validated) assert naive.deflated is False - assert naive.expectancy / base.expectancy == pytest.approx(1.25, abs=1e-3) + assert naive.expectancy / base.expectancy == pytest.approx(1.357, abs=1e-3) -def test_step2_design_and_its_measured_arls(baseline): - validated, base = baseline +def test_step2_design_and_its_measured_arls(promoted): + validated, _, base = promoted d = cusum_design(base) - assert d.k_r == pytest.approx(0.1253, abs=1e-4) - assert d.h_std == pytest.approx(37.71, abs=0.05) + assert d.k_r == pytest.approx(0.1520, abs=1e-4) + assert d.h_std == pytest.approx(35.09, abs=0.05) # the budget is calendar time: 25 years at this book's 150 trades/yr assert d.arl0_basis == "years" assert d.arl0 == pytest.approx(3_752, abs=5) assert d.arl0 / base.trades_per_year == pytest.approx(25.0, abs=0.1) - assert d.arl1 == pytest.approx(806, abs=5) - assert d.arl1 / base.trades_per_year == pytest.approx(5.4, abs=0.1) + assert d.arl1 == pytest.approx(658, abs=5) + assert d.arl1 / base.trades_per_year == pytest.approx(4.4, abs=0.1) # k is the textbook midpoint (mu_0 + mu_1) / 2 assert d.k_r == pytest.approx(0.75 * base.expectancy, rel=1e-9) in_control = empirical_arl(d, validated.r, baseline_expectancy=base.expectancy, n_sims=300, seed=7) - assert in_control.mean_run == pytest.approx(3_973, abs=60) - assert in_control.median_run == pytest.approx(2_951, abs=60) - assert in_control.inflation == pytest.approx(1.06, abs=0.02) + assert in_control.mean_run == pytest.approx(4_087, abs=60) + assert in_control.median_run == pytest.approx(3_244, abs=60) + assert in_control.inflation == pytest.approx(1.09, abs=0.02) halved = empirical_arl(d, validated.r, baseline_expectancy=base.expectancy, shift=0.5, n_sims=300, seed=7) - assert halved.mean_run == pytest.approx(811, abs=20) - assert halved.median_run == pytest.approx(686, abs=20) + assert halved.mean_run == pytest.approx(632, abs=20) + assert halved.median_run == pytest.approx(500, abs=20) # the right skew the tutorial calls out: median well under mean assert halved.median_run < halved.mean_run * 0.9 -def test_step3_three_live_books(baseline): - _, base = baseline +def test_step3_three_live_books(promoted): + _, _, base = promoted healthy = synthetic_book(1300, edge=1.0, seed=2, start="2022-01-01") halved = synthetic_book(1300, edge=0.5, seed=3, start="2022-01-01") drying = thin(healthy, 3) intact = edge_monitor(healthy, base) assert intact.label == "HOLDING" - assert intact.edge_ratio == pytest.approx(1.10, abs=0.01) - assert intact.cusum_peak / intact.cusum_h == pytest.approx(0.67, abs=0.01) + assert intact.edge_ratio == pytest.approx(0.91, abs=0.01) + assert intact.cusum_peak / intact.cusum_h == pytest.approx(0.82, abs=0.01) decayed = edge_monitor(halved, base) assert decayed.label == "DEGRADED" - assert decayed.alarm_index == 1138 - assert decayed.edge_ratio == pytest.approx(-0.16, abs=0.01) + assert decayed.alarm_index == 276 + assert decayed.edge_ratio == pytest.approx(-0.13, abs=0.01) thinned = edge_monitor(drying, base) assert thinned.label == "SLIPPING" assert thinned.n_live == 434 assert thinned.frequency_ratio == pytest.approx(0.33, abs=0.01) # the point of the row: per-trade edge is INTACT, only the rate collapsed - assert thinned.edge_ratio == pytest.approx(0.88, abs=0.01) + assert thinned.edge_ratio == pytest.approx(0.72, abs=0.01) assert not thinned.cusum_alarm -def test_step4_the_soft_channel_is_noisy_on_a_healthy_book(baseline): - _, base = baseline +def test_step4_the_soft_channel_is_noisy_on_a_healthy_book(promoted): + _, _, base = promoted healthy = synthetic_book(1300, edge=1.0, seed=2, start="2022-01-01") roll = rolling_expectancy(healthy, Thresholds().monitor_window).dropna() / base.expectancy - assert roll.min() == pytest.approx(-0.31, abs=0.01) - assert roll.max() == pytest.approx(2.40, abs=0.01) + assert roll.min() == pytest.approx(-0.25, abs=0.01) + assert roll.max() == pytest.approx(1.97, abs=0.01) below = float((roll < Thresholds().monitor_slip_ratio).mean()) - assert below == pytest.approx(0.09, abs=0.01) + assert below == pytest.approx(0.11, abs=0.01) # ...while the calibrated detector stays silent on the same book assert not edge_monitor(healthy, base).cusum_alarm +def test_the_deflation_is_what_keeps_the_healthy_book_holding(promoted): + """The payoff, stated as a test. Anchored to the optimized mean, a book whose true + edge never moved reads as decayed; anchored to the corrected one, it reads HOLDING.""" + validated, _, base = promoted + healthy = synthetic_book(1300, edge=1.0, seed=2, start="2022-01-01") + naive = EdgeBaseline.from_log(validated) + + assert edge_monitor(healthy, base).label == "HOLDING" + assert edge_monitor(healthy, naive).label == "DEGRADED" + + def test_the_example_runs(capsys): main() out = capsys.readouterr().out - for expected in ("HOLDING", "DEGRADED", "SLIPPING", "THE FROZEN BASELINE"): + for expected in ("HOLDING", "DEGRADED", "SLIPPING", "THE FROZEN BASELINE", + "search haircut"): assert expected in out