From 942f367f2f6aff269804d2600ba34814c661bf61 Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Sat, 10 Oct 2026 05:36:00 -0400 Subject: [PATCH] docs: verify legacy migration paths with shared inputs --- docs/user-guide/empyrical-migration.md | 98 +++++++++++ docs/user-guide/migration.md | 67 ++++++- examples/legacy_migration_comparison.py | 225 ++++++++++++++++++++++++ mkdocs.yml | 1 + 4 files changed, 390 insertions(+), 1 deletion(-) create mode 100644 docs/user-guide/empyrical-migration.md create mode 100644 examples/legacy_migration_comparison.py diff --git a/docs/user-guide/empyrical-migration.md b/docs/user-guide/empyrical-migration.md new file mode 100644 index 00000000..aa99e0c6 --- /dev/null +++ b/docs/user-guide/empyrical-migration.md @@ -0,0 +1,98 @@ +# Migrating Selected Empyrical Metrics + +Diagnostic covers the portfolio metrics listed below, but it does not replace +every Empyrical workflow. Move one calculation at a time, retain its input +cadence and missing-value policy, and compare the result on your own returns. +For Pyfolio returns analysis or Alphalens factor analysis, use the +[separate migration guide](migration.md). + +If portfolio metrics are one part of a larger research workflow, Diagnostic +also documents [purged cross-validation and multiple-trial Sharpe +assessment](cross-validation.md). That capability is a reason to evaluate it, +not evidence that every Empyrical function has an equivalent. + +## Run the comparison + +From a checkout of this repository, run the published packages on one +deterministic returns series with Python 3.12: + +```bash +uv run --no-project --python 3.12 \ + --with 'ml4t-diagnostic==0.1.8' \ + --with 'empyrical-reloaded==0.5.12' \ + --with 'pytz==2026.5' \ + python examples/legacy_migration_comparison.py --task empyrical +``` + +The script at `examples/legacy_migration_comparison.py` calls real Empyrical +functions and Diagnostic `PortfolioAnalysis` on the same 252 daily decimal +returns. It checks complete-data agreement, then replaces one return with +NaN. These are observed results for versions 0.5.12 and 0.1.8, not a general +equivalence guarantee. +The separate `pytz` pin supplies a runtime import used by Empyrical 0.5.12 +but absent from that release's dependency metadata. + +
+ +| Empyrical call | Diagnostic result from `PortfolioAnalysis(...).compute_summary_stats()` | Complete-input result in both | +|---|---|---:| +| `annual_return(..., annualization=252)` | `.annual_return` | 0.079730 | +| `annual_volatility(..., annualization=252)` | `.annual_volatility` | 0.150155 | +| `sharpe_ratio(..., annualization=252)` | `.sharpe_ratio` | 0.585568 | +| `sortino_ratio(..., annualization=252)` | `.sortino_ratio` | 0.890397 | +| `max_drawdown(...)` | `.max_drawdown` | -0.110216 | +| `calmar_ratio(..., annualization=252)` | `.calmar_ratio` | 0.723394 | +| `omega_ratio(...)` | `.omega_ratio` | 1.098490 | +| `tail_ratio(...)` | `.tail_ratio` | 1.218861 | +| `value_at_risk(..., cutoff=0.05)` | `.var_95` | -0.013656 | +| `conditional_value_at_risk(..., cutoff=0.05)` | `.cvar_95` | -0.017801 | + +
+ +Both drawdown and the observed 5% VaR/CVaR are **negative return values**, not +positive loss magnitudes. Empyrical uses a left-tail `cutoff=0.05` while +Diagnostic labels the same selection as 95% confidence. Check the sign before +feeding either result into a risk limit. + +## Preserve cadence, risk-free units, and missing-value behavior + +Empyrical defaults to `period="daily"` (252 periods per year). Diagnostic +defaults to `periods_per_year=252`. For monthly returns, pass +`period="monthly"` to Empyrical and `periods_per_year=12` to Diagnostic. +The script's separate twelve-month input produced annual return `0.037051` +in both. Neither API can infer the intended annualization from an unlabelled +NumPy return array. + +Both Sharpe defaults use zero risk-free rate. For nonzero rates, Empyrical's +`risk_free` argument is a **per-period return**; Diagnostic's `risk_free` +argument is an **annual rate**. The script converted a 2% annual rate to +`(1 + 0.02) ** (1 / 252) - 1` before calling Empyrical and passed `0.02` +to Diagnostic. Both produced Sharpe `0.453682`. Passing `0.02` directly to +both functions would not compare the same assumption. + +With one NaN at return index 10, nine of the ten listed metric pairs still +matched on this input. Empyrical `value_at_risk` returned `nan` because it +uses a percentile that includes NaN; Diagnostic `.var_95` returned +`-0.013664` using a NaN-aware percentile. Both annual-return values were +`0.071988`; the input still had 252 periods and 251 valid returns. Choose a +missing-value policy before migration rather than relying on this one example. + +## Metrics that need separate review + +Benchmark-relative `alpha` and `beta` are available from Diagnostic when you +pass aligned benchmark returns. On the [Pyfolio comparison](migration.md#pyfolio-portfolio-and-performance-analysis), +beta matched at `0.795601`, while annualized alpha was `0.100369` in Pyfolio's +Empyrical calculation and `0.095663` in Diagnostic. Empyrical compounds the +mean periodic alpha; Diagnostic multiplies the fitted daily intercept by 252. +Choose the annualization definition before replacing alpha in a report. + +The similarly named stability statistics are not equivalent. On the complete +input, Empyrical `stability_of_timeseries` returned `0.252783` and Diagnostic +`.stability` returned `0.247302`. Empyrical fits cumulative log returns; +Diagnostic fits cumulative simple wealth. Do not relabel one as the other. + +This assessment does not cover Empyrical's `gpd_risk_estimates`, +`perf_attrib`, date aggregation, or the `roll_*` function family as drop-in +replacements. Diagnostic has other portfolio and factor analyses, but their +inputs and result definitions must be checked separately. Keep Empyrical for +those tasks until a specific migration is verified. diff --git a/docs/user-guide/migration.md b/docs/user-guide/migration.md index e501adc9..1d61db56 100644 --- a/docs/user-guide/migration.md +++ b/docs/user-guide/migration.md @@ -3,6 +3,36 @@ Diagnostic is not a drop-in replacement for either package. Choose a task below, convert its input shape, and compare result definitions before comparing numbers. The examples use synthetic data and run with the base `ml4t-diagnostic` package. +For selected performance metrics, see the separate +[Empyrical migration assessment](empyrical-migration.md). + +One reason to evaluate Diagnostic beyond these matching values is its +[purged cross-validation workflow](cross-validation.md), which combines +chronological folds with multiple-trial Sharpe assessment. For portfolio +reporting, its [backtest bridge](backtest-tearsheets.md) also accepts a +completed `ml4t-backtest` result. These are separate capabilities, not +claims that the legacy tear sheets or factor panels are reproduced. + +## Run the shared-input comparison + +The package-only examples below show each Diagnostic input shape. To compare the +published legacy and Diagnostic APIs on the same deterministic inputs, run this +script from a checkout of this repository on Python 3.12: + +```bash +uv run --no-project --python 3.12 \ + --with 'ml4t-diagnostic==0.1.8' \ + --with 'pyfolio-reloaded==0.9.9' \ + --with 'alphalens-reloaded==0.4.6' \ + --with 'empyrical-reloaded==0.5.12' \ + python examples/legacy_migration_comparison.py --task all +``` + +The comparison script at `examples/legacy_migration_comparison.py` uses the +published packages, not the source checkout. It asserts the shared +Sharpe, drawdown, and IC results and prints other measured values. The figures +below are from that command with the stated versions; they are not performance +benchmarks or a promise that every legacy output matches. ## Alphalens factor analysis @@ -67,9 +97,19 @@ print(f"One-day IC: {signal.ic['1D']:.3f}") The assertion establishes the expected direction on these synthetic data; it is not a significance test for a real factor. Use the [signal quickstart](../getting-started/quickstart.md) to interpret IC and spread. +In the shared-input comparison, 600 factor rows across 50 business dates and +12 assets use one-day forward returns from the same wide price table. Both calls +use five quantiles, no outlier filter, and ungrouped Spearman IC. Alphalens +kept all 600 rows; its mean IC and Diagnostic's `SignalResult.ic["1D"]` were +both `0.318881`. Diagnostic also returned a top-minus-bottom quantile spread +of `0.004636`. The agreement is for this ungrouped input and definition. + Alphalens also supports grouped and group-adjusted IC, configurable zero-aware bucketing, and its own complete factor tear sheet. Those options -are not implied by a Diagnostic signal result. +are not implied by a Diagnostic signal result. In particular, do not interpret +Diagnostic's default global quantiles as Alphalens `group_adjust=True` or +`binning_by_group=True`. Keep the legacy path when those panels or conventions +are required. Factor *construction* is a separate task. If needed, [ML4T Engineer's feature guide](https://github.com/ml4t/engineer/blob/145ccafd2ca36dc61f976cdbeaaf147af21e7c2e/docs/user-guide/features.md) @@ -122,6 +162,31 @@ print(f"Portfolio Sharpe: {summary.sharpe_ratio:.2f}") Use the [portfolio workflow](workflows.md) and [backtest report guide](backtest-tearsheets.md) for the supported output checks. +The shared-input comparison passes 252 aligned business-day returns and benchmark +returns to Pyfolio `perf_stats()` and Diagnostic `PortfolioAnalysis`, with zero +risk-free rate and 252 periods per year. Return, volatility, drawdown, and +alpha values use decimal-return units; Sharpe ratio and beta are dimensionless: + +
+ +| Metric | Pyfolio 0.9.9 | Diagnostic 0.1.8 | +|---|---:|---:| +| Annual return | 0.079730 | 0.079730 | +| Annual volatility | 0.150155 | 0.150155 | +| Sharpe ratio | 0.585568 | 0.585568 | +| Maximum drawdown | -0.110216 | -0.110216 | +| Alpha | 0.100369 | 0.095663 | +| Beta | 0.795601 | 0.795601 | + +
+ +The alpha difference is expected under the two implementations: Pyfolio's +Empyrical calculation compounds mean periodic alpha, while Diagnostic +multiplies its fitted daily intercept by 252. Do not compare annualized alpha +without choosing a convention. The example has no positions or transactions; +Pyfolio's optional Zipline position multipliers were unavailable and did not +affect these returns-based values. + Pyfolio's full sheet can additionally include return cones, significant events, round trips reconstructed from fills, capacity analysis using market data, and optional factor-attribution panels. Diagnostic has related diff --git a/examples/legacy_migration_comparison.py b/examples/legacy_migration_comparison.py new file mode 100644 index 00000000..2e7afb2d --- /dev/null +++ b/examples/legacy_migration_comparison.py @@ -0,0 +1,225 @@ +"""Compare published legacy and Diagnostic APIs on shared synthetic inputs. + +Run from the repository root with the pinned-package command in the migration guide. +""" + +from __future__ import annotations + +import argparse +from importlib.metadata import version + +import numpy as np +import pandas as pd + +from ml4t.diagnostic import analyze_signal +from ml4t.diagnostic.evaluation import PortfolioAnalysis + + +def returns_input() -> tuple[pd.Series, pd.Series]: + dates = pd.bdate_range("2025-01-02", periods=252) + rng = np.random.default_rng(42) + benchmark = pd.Series(rng.normal(0.0003, 0.007, len(dates)), index=dates) + strategy = pd.Series( + 0.0002 + 0.8 * benchmark.to_numpy() + rng.normal(0, 0.008, len(dates)), + index=dates, + ) + return strategy, benchmark + + +def compare_portfolios(strategy: pd.Series, benchmark: pd.Series) -> None: + import pyfolio.timeseries as pyfolio_timeseries + + legacy = pyfolio_timeseries.perf_stats(strategy, factor_returns=benchmark) + current = PortfolioAnalysis( + returns=strategy.to_numpy(), + benchmark=benchmark.to_numpy(), + dates=strategy.index.to_numpy(), + risk_free=0.0, + periods_per_year=252, + ).compute_summary_stats() + print("Pyfolio / Diagnostic daily portfolio summary") + for label, attribute in ( + ("Annual return", "annual_return"), + ("Annual volatility", "annual_volatility"), + ("Sharpe ratio", "sharpe_ratio"), + ("Max drawdown", "max_drawdown"), + ("Alpha", "alpha"), + ("Beta", "beta"), + ): + print(f"{label}: {legacy[label]:.6f} / {getattr(current, attribute):.6f}") + np.testing.assert_allclose(legacy["Sharpe ratio"], current.sharpe_ratio) + np.testing.assert_allclose(legacy["Max drawdown"], current.max_drawdown) + + +def signal_input() -> tuple[pd.Series, pd.DataFrame]: + dates = pd.bdate_range("2025-01-02", periods=60) + assets = [f"asset_{index:02d}" for index in range(12)] + rng = np.random.default_rng(7) + scores = rng.normal(size=(50, len(assets))) + factor_index = pd.MultiIndex.from_product([dates[:50], assets], names=["date", "asset"]) + factor = pd.Series(scores.ravel(), index=factor_index, name="factor") + prices = np.full((len(dates), len(assets)), 100.0) + for day in range(1, len(dates)): + lagged_scores = scores[day - 1] if day - 1 < len(scores) else 0.0 + prices[day] = prices[day - 1] * ( + 1 + 0.002 * lagged_scores + rng.normal(scale=0.005, size=len(assets)) + ) + return factor, pd.DataFrame(prices, index=dates, columns=assets) + + +def current_signal(factor: pd.Series, prices: pd.DataFrame): + factor_long = factor.reset_index() + price_long = ( + prices.rename_axis("date") + .stack() + .rename("price") + .rename_axis(index=["date", "asset"]) + .reset_index() + ) + return analyze_signal( + factor_long, + price_long, + periods=(1,), + quantiles=5, + filter_zscore=None, + min_assets=10, + ) + + +def compare_signals(factor: pd.Series, prices: pd.DataFrame) -> None: + import alphalens.performance as alphalens_performance + import alphalens.utils as alphalens_utils + + cleaned = alphalens_utils.get_clean_factor_and_forward_returns( + factor, + prices, + quantiles=5, + periods=(1,), + filter_zscore=None, + max_loss=0.35, + ) + legacy_ic = alphalens_performance.factor_information_coefficient(cleaned)["1D"].mean() + current = current_signal(factor, prices) + print("Alphalens / Diagnostic one-day signal analysis") + print(f"Input factor rows: {len(factor)}") + print(f"Cleaned Alphalens rows: {len(cleaned)}") + print(f"Mean Spearman IC: {legacy_ic:.6f} / {current.ic['1D']:.6f}") + print(f"Diagnostic spread: {current.spread['1D']:.6f}") + np.testing.assert_allclose(legacy_ic, current.ic["1D"]) + + +def show_diagnostic_only() -> None: + strategy, _ = returns_input() + summary = PortfolioAnalysis( + returns=strategy.to_numpy(), + dates=strategy.index.to_numpy(), + periods_per_year=252, + ).compute_summary_stats() + signal = current_signal(*signal_input()) + assert np.isfinite(summary.sharpe_ratio) + assert np.isfinite(signal.ic["1D"]) + print(f"Diagnostic Sharpe ratio: {summary.sharpe_ratio:.6f}") + print(f"Diagnostic one-day signal IC: {signal.ic['1D']:.6f}") + + +def compare_metrics(strategy: pd.Series) -> None: + import empyrical + + metrics = ( + ("annual_return", "annual_return", lambda v: empyrical.annual_return(v, annualization=252)), + ( + "annual_volatility", + "annual_volatility", + lambda v: empyrical.annual_volatility(v, annualization=252), + ), + ("sharpe_ratio", "sharpe_ratio", lambda v: empyrical.sharpe_ratio(v, annualization=252)), + ("sortino_ratio", "sortino_ratio", lambda v: empyrical.sortino_ratio(v, annualization=252)), + ("max_drawdown", "max_drawdown", empyrical.max_drawdown), + ("calmar_ratio", "calmar_ratio", lambda v: empyrical.calmar_ratio(v, annualization=252)), + ("omega_ratio", "omega_ratio", empyrical.omega_ratio), + ("tail_ratio", "tail_ratio", empyrical.tail_ratio), + ("value_at_risk", "var_95", empyrical.value_at_risk), + ("conditional_value_at_risk", "cvar_95", empyrical.conditional_value_at_risk), + ) + print("Empyrical / Diagnostic daily metrics") + for label, values in (("complete", strategy), ("one NaN", strategy.copy())): + if label == "one NaN": + values.iloc[10] = np.nan + current = PortfolioAnalysis( + returns=values.to_numpy(), + dates=values.index.to_numpy(), + periods_per_year=252, + ).compute_summary_stats() + print(f"Input: {label}, rows={len(values)}, valid={values.notna().sum()}") + for name, current_name, legacy_function in metrics: + old = float(legacy_function(values)) + new = float(getattr(current, current_name)) + print(f"{name}: {old:.6f} / {new:.6f}") + if label == "complete": + np.testing.assert_allclose(old, new) + old_stability = empyrical.stability_of_timeseries(values) + print(f"stability_of_timeseries: {old_stability:.6f} / {current.stability:.6f}") + + monthly = pd.Series( + [0.01, -0.02, 0.015, 0.004, -0.006, 0.012, 0.003, -0.005, 0.011, 0.008, -0.004, 0.009], + index=pd.date_range("2025-01-01", periods=12, freq="MS"), + ) + legacy_monthly = empyrical.annual_return(monthly, period="monthly") + current_monthly = ( + PortfolioAnalysis( + returns=monthly.to_numpy(), dates=monthly.index.to_numpy(), periods_per_year=12 + ) + .compute_summary_stats() + .annual_return + ) + print(f"Monthly annual return: {legacy_monthly:.6f} / {current_monthly:.6f}") + np.testing.assert_allclose(legacy_monthly, current_monthly) + + annual_risk_free = 0.02 + periodic_risk_free = (1 + annual_risk_free) ** (1 / 252) - 1 + legacy_sharpe = empyrical.sharpe_ratio( + strategy, risk_free=periodic_risk_free, annualization=252 + ) + current_sharpe = ( + PortfolioAnalysis( + returns=strategy.to_numpy(), risk_free=annual_risk_free, periods_per_year=252 + ) + .compute_summary_stats() + .sharpe_ratio + ) + print(f"Sharpe with 2% annual risk-free: {legacy_sharpe:.6f} / {current_sharpe:.6f}") + np.testing.assert_allclose(legacy_sharpe, current_sharpe) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--task", + choices=("diagnostic", "all", "pyfolio", "alphalens", "empyrical"), + default="diagnostic", + ) + task = parser.parse_args().task + packages = ["ml4t-diagnostic", "numpy", "pandas", "scipy", "polars"] + if task in ("all", "empyrical"): + packages.append("empyrical-reloaded") + if task in ("all", "pyfolio"): + packages.append("pyfolio-reloaded") + if task in ("all", "alphalens"): + packages.append("alphalens-reloaded") + for package in packages: + print(f"{package}=={version(package)}") + if task == "diagnostic": + show_diagnostic_only() + return + if task in ("all", "pyfolio", "empyrical"): + strategy, benchmark = returns_input() + if task in ("all", "pyfolio"): + compare_portfolios(strategy, benchmark) + if task in ("all", "empyrical"): + compare_metrics(strategy) + if task in ("all", "alphalens"): + compare_signals(*signal_input()) + + +if __name__ == "__main__": + main() diff --git a/mkdocs.yml b/mkdocs.yml index f072e52e..879646aa 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -113,6 +113,7 @@ nav: - Feature Selection: user-guide/feature-selection.md - Workflows: user-guide/workflows.md - From Alphalens and Pyfolio: user-guide/migration.md + - From Empyrical Metrics: user-guide/empyrical-migration.md - HTML Backtest Reports: user-guide/backtest-tearsheets.md - Trade Diagnostics: user-guide/trade-analysis.md - Trade-SHAP Dashboard: DASHBOARD.md