From 0b35b42d4f2cae585d558fee991cbbd25446bd95 Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Tue, 20 Oct 2026 16:41:23 -0400 Subject: [PATCH 01/19] fix(features)!: preserve exact integer price returns Select native arithmetic at schema planning time, subtract widened integer ticks before floating division, and retain float dtype/formula and logarithmic domains. Require Polars 2.0 for the native schema dispatch API. Add eager/lazy/grouped regression cases and an executable migration recipe. Refs #107 --- docs/user-guide/features.md | 38 +++++ pyproject.toml | 2 +- .../engineer/features/utils/arithmetic.py | 41 ++++- tests/test_returns_precision.py | 143 ++++++++++++++++++ uv.lock | 2 +- 5 files changed, 218 insertions(+), 8 deletions(-) create mode 100644 tests/test_returns_precision.py diff --git a/docs/user-guide/features.md b/docs/user-guide/features.md index a4f0271b..e1c03cfc 100644 --- a/docs/user-guide/features.md +++ b/docs/user-guide/features.md @@ -85,6 +85,44 @@ result = compute_features(df, [ Unknown parameters, repeated output names, and output names that replace input columns raise `ValueError` before feature execution. +### Derive precise price returns + +The next minor version requires Polars 2.0 or newer. Return expressions select their +native arithmetic from the column dtype while planning a query, without collecting +lazy data. Keep integer ticks as integers until after the expression evaluates; +casting large prices to floating point first can erase a one-tick change. + + +```python +import polars as pl +from ml4t.engineer.features.utils.arithmetic import returns, log_returns + +prices = pl.DataFrame({ + "asset": ["A", "A", "A"], + "close": pl.Series([2**53, 2**53 + 1, 2**53 + 2], dtype=pl.Int64), +}) +result = prices.lazy().with_columns( + returns("close").over("asset").alias("return"), + log_returns("close").over("asset").alias("log_return"), +).collect() +assert result["return"][0] is None +assert result["return"][1] == 1 / 2**53 +assert result["log_return"][1] > 0 +``` + +Both expressions retain the input row count and leave the first `periods` rows +undefined in each asset. Integer differences are computed before floating division; +floating input keeps its existing ratio formula and dtype. Log returns retain tiny +changes without turning a near-total loss into an artificial negative infinity. +`percentage_change` uses the same simple-return calculation multiplied by 100. + +When replacing an Empyrical price-return call, note that its output omits the first +undefined observation. Trim that documented leading row only if the consuming +workflow needs that shape; do not drop every null or compress gaps. These are price +expressions, not portfolio statistics. Use explicit grouping for multiple assets and +validate the declared bar grid before deriving returns as described below. Row-only +data must already have the intended observation order. + ### Calendar and bar grid The calendar-aware default targets the next minor version (0.2), separate from the 0.1.7 patch. diff --git a/pyproject.toml b/pyproject.toml index 4240cb8f..b593d2e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,7 +76,7 @@ dependencies = [ "ml4t-specs>=0.2.0,<0.3.0", # Data processing (Polars-first) - "polars>=0.20.0", + "polars>=2.0.0", "pandas>=2.0.0", "numpy>=1.24.0,<2.6", diff --git a/src/ml4t/engineer/features/utils/arithmetic.py b/src/ml4t/engineer/features/utils/arithmetic.py index 15100ac5..f0503084 100644 --- a/src/ml4t/engineer/features/utils/arithmetic.py +++ b/src/ml4t/engineer/features/utils/arithmetic.py @@ -9,6 +9,32 @@ from ml4t.engineer.core.exceptions import InvalidParameterError +def _price_return(column: str, periods: int, *, logarithmic: bool) -> pl.Expr: + def build(price: pl.Expr, dtype: pl.DataType) -> pl.Expr: + previous = price.shift(periods) + ratio = price / previous + if not dtype.is_integer(): + return ratio.log() if logarithmic else ratio - 1 + + wide = price.cast(pl.Int128) + lagged = wide.shift(periods) + if dtype == pl.Int128: + # Opposite-sign Int128 differences can exceed even the widest native integer. + same_sign = ((wide >= 0) & (lagged >= 0)) | ((wide <= 0) & (lagged <= 0)) + safe_price = pl.when(same_sign).then(wide).otherwise(0) + safe_previous = pl.when(same_sign).then(lagged).otherwise(0) + difference = (safe_price - safe_previous).cast(pl.Float64) + gain = pl.when(same_sign).then(difference / previous).otherwise(ratio - 1) + else: + gain = (wide - lagged).cast(pl.Float64) / previous + if not logarithmic: + return gain + # log1p retains tiny ticks, while the ratio retains near-total losses and domains. + return pl.when(gain.abs() < 0.5).then(gain.log1p()).otherwise(ratio.log()) + + return pl.col(column).pipe_with_dtype(build) + + def returns(column: str, periods: int = 1) -> pl.Expr: """Price returns over N periods. @@ -16,7 +42,10 @@ def returns(column: str, periods: int = 1) -> pl.Expr: This is one of the most fundamental calculations in quantitative finance. Formula: - Return = (Price[t] / Price[t-periods]) - 1 + Return = (Price[t] - Price[t-periods]) / Price[t-periods] + + Integer differences are retained before floating conversion. Floating inputs + retain their existing ratio calculation and output dtype. Args: column: Name of the column containing price data @@ -30,7 +59,7 @@ def returns(column: str, periods: int = 1) -> pl.Expr: Example: >>> import polars as pl - >>> import ml4t.engineer.features.ta as qta + >>> from ml4t.engineer.features.utils import arithmetic as qta >>> >>> df = pl.DataFrame({ ... "close": [100, 105, 103, 107, 110] @@ -44,7 +73,7 @@ def returns(column: str, periods: int = 1) -> pl.Expr: if periods < 1: raise InvalidParameterError(f"periods must be >= 1, got {periods}") - return (pl.col(column) / pl.col(column).shift(periods)) - 1 + return _price_return(column, periods, logarithmic=False) def log_returns(column: str, periods: int = 1) -> pl.Expr: @@ -69,7 +98,7 @@ def log_returns(column: str, periods: int = 1) -> pl.Expr: Example: >>> import polars as pl - >>> import ml4t.engineer.features.ta as qta + >>> from ml4t.engineer.features.utils import arithmetic as qta >>> >>> df = pl.DataFrame({ ... "close": [100, 105, 103, 107, 110] @@ -82,7 +111,7 @@ def log_returns(column: str, periods: int = 1) -> pl.Expr: if periods < 1: raise InvalidParameterError(f"periods must be >= 1, got {periods}") - return (pl.col(column) / pl.col(column).shift(periods)).log() + return _price_return(column, periods, logarithmic=True) def percentage_change(column: str, periods: int = 1) -> pl.Expr: @@ -106,7 +135,7 @@ def percentage_change(column: str, periods: int = 1) -> pl.Expr: Example: >>> import polars as pl - >>> import ml4t.engineer.features.ta as qta + >>> from ml4t.engineer.features.utils import arithmetic as qta >>> >>> df = pl.DataFrame({ ... "close": [100, 105, 103, 107, 110] diff --git a/tests/test_returns_precision.py b/tests/test_returns_precision.py new file mode 100644 index 00000000..6502a97d --- /dev/null +++ b/tests/test_returns_precision.py @@ -0,0 +1,143 @@ +"""Public price expressions must not discard exact integer ticks before division.""" + +import math + +import numpy as np +import polars as pl +import pytest +from polars.testing import assert_frame_equal + +from ml4t.engineer import compute_features +from ml4t.engineer.core.exceptions import InvalidParameterError +from ml4t.engineer.features.utils.arithmetic import ( + log_returns, + percentage_change, + returns, +) + +INTEGER_CASES = [ + (pl.Int64, [2**53, 2**53 + 1, 2**53 + 2]), + (pl.Int64, [2**63 - 1, 2**63 - 2, 2**63 - 3]), + (pl.UInt64, [2**63, 2**63 + 1, 2**63 + 2]), + (pl.UInt64, [2**64 - 1, 2**64 - 2, 2**64 - 3]), + (pl.Int64, [-(2**63), 2**63 - 1, -(2**63)]), + (pl.UInt64, [2**64 - 1, 0, 1, 2**64 - 1]), + (pl.UInt64, [2**64 - 1, 1, 2]), + (pl.Int128, [2**127 - 1, 2**127 - 2, 2**127 - 3]), + (pl.Int128, [-(2**127), 2**127 - 1, -(2**127)]), + (pl.Int8, [-128, 127, -128]), + (pl.UInt8, [250, 5, 10]), + (pl.Int64, [100, 101, None, 99, 103]), + (pl.Int64, [0, 0, 10, 0, -10]), + (pl.Int64, [-100, -101, -99, -102]), + (pl.Int64, []), + (pl.Int64, [2**53 + 1]), +] +METHODS = [returns, log_returns, percentage_change] + + +def _expected(prices, periods, method): + result = [] + for index, current in enumerate(prices): + previous = prices[index - periods] if index >= periods else None + if current is None or previous is None: + result.append(None) + continue + with np.errstate(divide="ignore", invalid="ignore"): + change = (current - previous) / previous if previous else np.divide(current, previous) + if method is log_returns: + value = ( + np.log1p(change) if abs(change) < 0.5 else np.log(np.divide(current, previous)) + ) + else: + value = change * (100 if method is percentage_change else 1) + result.append(float(value)) + return result + + +def _assert_values(actual, expected): + assert len(actual) == len(expected) + for observed, wanted in zip(actual, expected, strict=True): + if wanted is None: + assert observed is None + elif math.isnan(wanted): + assert math.isnan(observed) + else: + assert observed == pytest.approx(wanted, rel=2e-15, abs=0) + + +@pytest.mark.parametrize("dtype,prices", INTEGER_CASES) +@pytest.mark.parametrize("method", METHODS) +@pytest.mark.parametrize("periods", [1, 2]) +@pytest.mark.parametrize("lazy", [False, True]) +def test_exact_integer_prices_preserve_ticks_limits_and_missingness( + dtype, prices, method, periods, lazy +): + frame = pl.DataFrame({"close": pl.Series(prices, dtype=dtype)}) + before = frame.clone() + data = frame.lazy() if lazy else frame + produced = data.select(method("close", periods).alias("return")) + if lazy: + assert isinstance(produced, pl.LazyFrame) + assert produced.collect_schema()["return"] == pl.Float64 + produced = produced.collect() + _assert_values(produced["return"].to_list(), _expected(prices, periods, method)) + assert_frame_equal(frame, before) + + +@pytest.mark.parametrize("method", METHODS) +@pytest.mark.parametrize("dtype", [pl.Float32, pl.Float64]) +@pytest.mark.parametrize("lazy", [False, True]) +def test_float_prices_keep_native_dtype_domain_and_missing_conventions(method, dtype, lazy): + frame = pl.DataFrame( + {"close": pl.Series([100.5, 101.25, None, float("nan"), 0, -1], dtype=dtype)} + ) + source = pl.col("close") + ratio = source / source.shift(1) + expected = ratio.log() if method is log_returns else (ratio - 1) + if method is percentage_change: + expected = expected * 100 + data = frame.lazy() if lazy else frame + produced = data.select(method("close").alias("return")) + control = data.select(expected.alias("return")) + if lazy: + produced, control = produced.collect(), control.collect() + assert_frame_equal(produced, control) + + +@pytest.mark.parametrize("method", METHODS) +@pytest.mark.parametrize("lazy", [False, True]) +def test_public_price_validation_then_grouped_returns_does_not_mix_assets(method, lazy): + frame = pl.DataFrame( + { + "asset": ["A", "B", "A", "B", "A", "B"], + "close": [2**53, 100, 2**53 + 1, 101, 2**53 + 2, 99], + } + ).with_columns( + pl.col("close").alias("open"), + pl.col("close").alias("high"), + pl.col("close").alias("low"), + pl.lit(100).alias("volume"), + ) + before = frame.clone() + source = frame.lazy() if lazy else frame + validated = compute_features(source, [], group_col="asset", calendar=None) + produced = validated.with_columns(method("close").over("asset").alias("return")) + if lazy: + produced = produced.collect(engine="streaming") + for asset, prices in (("A", [2**53, 2**53 + 1, 2**53 + 2]), ("B", [100, 101, 99])): + _assert_values( + produced.filter(pl.col("asset") == asset)["return"].to_list(), + _expected(prices, 1, method), + ) + assert_frame_equal(frame, before) + + +@pytest.mark.parametrize("method", METHODS) +@pytest.mark.parametrize("periods", [0, -1]) +def test_invalid_lag_fails_before_input_or_lazy_plan_is_changed(method, periods): + frame = pl.DataFrame({"close": [100, 101]}) + before = frame.clone() + with pytest.raises(InvalidParameterError, match="periods must be >= 1"): + frame.lazy().select(method("close", periods)) + assert_frame_equal(frame, before) diff --git a/uv.lock b/uv.lock index 4448424a..5214931d 100644 --- a/uv.lock +++ b/uv.lock @@ -1734,7 +1734,7 @@ requires-dist = [ { name = "pandas-market-calendars", marker = "extra == 'dev'", specifier = ">=5.5.0" }, { name = "plotly", marker = "extra == 'all'", specifier = ">=5.15.0" }, { name = "plotly", marker = "extra == 'viz'", specifier = ">=5.15.0" }, - { name = "polars", specifier = ">=0.20.0" }, + { name = "polars", specifier = ">=2.0.0" }, { name = "pre-commit", marker = "extra == 'all'", specifier = ">=3.3.0" }, { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=3.3.0" }, { name = "pyarrow", marker = "extra == 'all'", specifier = ">=14.0.0" }, From a96d32f334c3862aaf46dcbc737b3bab2222e7f7 Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Tue, 20 Oct 2026 16:53:26 -0400 Subject: [PATCH 02/19] build: replace vulnerable certifi in contributor lock The required dependency audit on PR113 reproduced GHSA-6q48-6c7w-9gw2 in certifi 2026.7.22. Upgrade only certifi to patched 2026.10.19; both runtime and complete contributor lock exports now pass pip-audit. Existing dependency PR106 retains the affected version and does not supply this repair. --- uv.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 5214931d..1f175b32 100644 --- a/uv.lock +++ b/uv.lock @@ -128,11 +128,11 @@ filecache = [ [[package]] name = "certifi" -version = "2026.7.22" +version = "2026.10.19" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/24167ea9858356b47a87a50d39908bfdb72ceeefe0041586e704e5376b3a/certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55", size = 138112, upload-time = "2026-07-22T03:35:12.644Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl", hash = "sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775", size = 136983, upload-time = "2026-07-22T03:35:11.276Z" }, + { url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" }, ] [[package]] From e17cfec2e3425d0aeeafce89a16d15a1f88be1ee Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Tue, 20 Oct 2026 17:19:11 -0400 Subject: [PATCH 03/19] build: resolve newly published Pillow security advisories The fresh contributor audit reproduces eight Pillow12.3 advisories published after the earlier clean local audit. Lock the patched13.0 release without ignoring vulnerabilities or changing runtime requirements. Core requirements are unchanged; focused numerical and executable documentation checks pass. --- uv.lock | 139 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 72 insertions(+), 67 deletions(-) diff --git a/uv.lock b/uv.lock index 1f175b32..6525bd3a 100644 --- a/uv.lock +++ b/uv.lock @@ -2198,73 +2198,78 @@ wheels = [ [[package]] name = "pillow" -version = "12.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1c/3d/bb7fca845737cf9d7dbde16ed1843984665ff2e0a518f5db43e77ec540b9/pillow-12.3.0.tar.gz", hash = "sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce", size = 47025035, upload-time = "2026-07-01T11:56:38.965Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/37/bf/fb3ebff8ddcb76aac5a01389251bbbb9519922a9b520d8247c1ca864a25d/pillow-12.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965", size = 5345969, upload-time = "2026-07-01T11:54:06.397Z" }, - { url = "https://files.pythonhosted.org/packages/d8/66/9a386a92561f402389a4fc70c18838bf6d35eb5eb5c6850b4b2dc64f5048/pillow-12.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7", size = 4780323, upload-time = "2026-07-01T11:54:09.351Z" }, - { url = "https://files.pythonhosted.org/packages/25/27/ac8f99618ffd3dde21db0f4d4b1d2ab00c0880595bfd17df103f7f39fd0c/pillow-12.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9", size = 6266838, upload-time = "2026-07-01T11:54:11.71Z" }, - { url = "https://files.pythonhosted.org/packages/84/21/a35af28dcc61f37ed850a2d64c65c701321dfbf25085e469d5559360cbbf/pillow-12.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91", size = 6940830, upload-time = "2026-07-01T11:54:13.732Z" }, - { url = "https://files.pythonhosted.org/packages/eb/51/8b08617af3ad95e33ce6d7dd2c99ed6c8298f7fb131636303956be022e25/pillow-12.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c", size = 6344383, upload-time = "2026-07-01T11:54:15.756Z" }, - { url = "https://files.pythonhosted.org/packages/1d/72/cf78ac9780bb93c28328f408973845a309d4d145041665f734572ced1b52/pillow-12.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df", size = 7052934, upload-time = "2026-07-01T11:54:17.721Z" }, - { url = "https://files.pythonhosted.org/packages/20/20/25e0f4dc178a6bc0696793720055519a0de89e7661dae886992decbd2f81/pillow-12.3.0-cp312-cp312-win32.whl", hash = "sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f", size = 6472684, upload-time = "2026-07-01T11:54:19.839Z" }, - { url = "https://files.pythonhosted.org/packages/45/89/da2f7971a317f83d807fdd4065c0af40208e59e692cc43d315a71a0e96d1/pillow-12.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09", size = 7227137, upload-time = "2026-07-01T11:54:22.025Z" }, - { url = "https://files.pythonhosted.org/packages/de/47/4845a0a6c0dbf1db8456bd9fc791f13c5ced7ced20606d08a0aacfd25b49/pillow-12.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510", size = 2568267, upload-time = "2026-07-01T11:54:24.051Z" }, - { url = "https://files.pythonhosted.org/packages/9d/ac/31fb64e1e7efb5a4b50cd3d92049ba89ac6e4d8d3bb6a74e15048ca3353e/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89", size = 4161684, upload-time = "2026-07-01T11:54:25.934Z" }, - { url = "https://files.pythonhosted.org/packages/87/b4/9805e23d2b4d77842b468513841fda254ee42f0289d25088340e4ff46e2d/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace", size = 4255487, upload-time = "2026-07-01T11:54:27.935Z" }, - { url = "https://files.pythonhosted.org/packages/df/39/ecf519435a200c693fe053a6ee4d835b41cf963a4dfc2551c4e637cb2a71/pillow-12.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec", size = 3696433, upload-time = "2026-07-01T11:54:29.813Z" }, - { url = "https://files.pythonhosted.org/packages/42/92/2fc3ffad878ae8dd5469ec1bc8eb83b71f48e13efdf68f02709003982a32/pillow-12.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66", size = 5345889, upload-time = "2026-07-01T11:54:31.97Z" }, - { url = "https://files.pythonhosted.org/packages/10/76/8803c13605b763d33d156c4678fc77f8443389c0c51c8aef707bb02015f4/pillow-12.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35", size = 4780109, upload-time = "2026-07-01T11:54:34.026Z" }, - { url = "https://files.pythonhosted.org/packages/1f/01/e18aff37cb0b4aac47ac90f016d347a49aca667ef97f190b06ac2aabc928/pillow-12.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65", size = 6263736, upload-time = "2026-07-01T11:54:36.131Z" }, - { url = "https://files.pythonhosted.org/packages/f7/62/de5bdd77d935331f4f802edc11e4d82950f642caad6cb2f949837b8560e2/pillow-12.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3", size = 6937129, upload-time = "2026-07-01T11:54:38.216Z" }, - { url = "https://files.pythonhosted.org/packages/70/4d/105627a13300c5e0df1d174230b32fd1273062c96f7745fd552b945d1e1d/pillow-12.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a", size = 6339562, upload-time = "2026-07-01T11:54:40.354Z" }, - { url = "https://files.pythonhosted.org/packages/6b/1d/f13de01a553988ab895ba1c722e06cf3144d4f57656fd5b81b6d881f1179/pillow-12.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e", size = 7049439, upload-time = "2026-07-01T11:54:42.489Z" }, - { url = "https://files.pythonhosted.org/packages/c9/f9/066794cca041b969964f779ee5fa66a9498bbf34248ac39c5d7954e4198f/pillow-12.3.0-cp313-cp313-win32.whl", hash = "sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f", size = 6473287, upload-time = "2026-07-01T11:54:44.9Z" }, - { url = "https://files.pythonhosted.org/packages/a6/9b/7a58e61d62be561da3a356fe2384d4059a6345fc130e23ef1c36a5b81d24/pillow-12.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8", size = 7239691, upload-time = "2026-07-01T11:54:47.141Z" }, - { url = "https://files.pythonhosted.org/packages/aa/b0/c4ed4f0ef8f8fa5ee8351537db6650bb8189f7e118842978dd6589065692/pillow-12.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b", size = 2568185, upload-time = "2026-07-01T11:54:49.137Z" }, - { url = "https://files.pythonhosted.org/packages/dc/01/001f65b68192f0228cc1dbbc8d2530ab5d58b61037ba0587f946fea607cd/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330", size = 4161736, upload-time = "2026-07-01T11:54:51.156Z" }, - { url = "https://files.pythonhosted.org/packages/1a/d2/0219746d0fd16fc8a84498e79452375be3797d3ce4044596ce565164b84f/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217", size = 4255435, upload-time = "2026-07-01T11:54:53.414Z" }, - { url = "https://files.pythonhosted.org/packages/c8/02/8d0bc62ef0302318c46ff2a512822d2610e81c7aa46c9b3abe6cbaca5ad0/pillow-12.3.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930", size = 3696262, upload-time = "2026-07-01T11:54:55.739Z" }, - { url = "https://files.pythonhosted.org/packages/85/e2/73c77d218410b14f5f2d565e8a998d5317b7b9c75368d29985139f7a46f0/pillow-12.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8", size = 5350344, upload-time = "2026-07-01T11:54:57.657Z" }, - { url = "https://files.pythonhosted.org/packages/c7/da/32c752228ae345f489e3a42499d817b6c3996da7e8a3bc7a04fc806b243b/pillow-12.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0", size = 4780131, upload-time = "2026-07-01T11:54:59.713Z" }, - { url = "https://files.pythonhosted.org/packages/b1/9d/8b2c807dbef61a5197c047afe99823787eb66f63daf9fb2432f91d6f0462/pillow-12.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321", size = 6263757, upload-time = "2026-07-01T11:55:01.778Z" }, - { url = "https://files.pythonhosted.org/packages/5c/44/c85361f65dbe00eea8576ee467c768d25129989efb76e94f205e9ca9bb46/pillow-12.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b", size = 6936962, upload-time = "2026-07-01T11:55:03.93Z" }, - { url = "https://files.pythonhosted.org/packages/18/7e/e483414b35800b86b6f08dbbc7803fb5cd52c4d6f897f47d53ea2c7e6f65/pillow-12.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198", size = 6339171, upload-time = "2026-07-01T11:55:05.989Z" }, - { url = "https://files.pythonhosted.org/packages/f0/f4/68c491844841ede6bed70189546b3ee9731cf9f2cbad396faff5e1ccba45/pillow-12.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130", size = 7048116, upload-time = "2026-07-01T11:55:08.131Z" }, - { url = "https://files.pythonhosted.org/packages/a3/34/77f3f793fed8efc7d243f21b33c5a3f0d1c97ee70346d3db855587e155ff/pillow-12.3.0-cp314-cp314-win32.whl", hash = "sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a", size = 6467209, upload-time = "2026-07-01T11:55:10.408Z" }, - { url = "https://files.pythonhosted.org/packages/f1/e0/492879f69d94f91f60fc8cd05ba03650e9520afebb2fb7aa12777d7c7f38/pillow-12.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d", size = 7237707, upload-time = "2026-07-01T11:55:12.745Z" }, - { url = "https://files.pythonhosted.org/packages/c9/ac/6b11f2875f1c2ac040d84e1bbf9cf22a88038f901ca1037898b280b38365/pillow-12.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838", size = 2565995, upload-time = "2026-07-01T11:55:14.736Z" }, - { url = "https://files.pythonhosted.org/packages/52/69/c2208e56af9bfc1913afb24020297a691eb1d4ef688474c8a04913f65e04/pillow-12.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e", size = 5352503, upload-time = "2026-07-01T11:55:17.076Z" }, - { url = "https://files.pythonhosted.org/packages/07/70/e5686d753e898a45d778ff1718dba8516ead6ab6b95d85fc8c4b70650cf2/pillow-12.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17", size = 4782956, upload-time = "2026-07-01T11:55:19.448Z" }, - { url = "https://files.pythonhosted.org/packages/d5/37/25c6692f06927ee973ff18c8d9ee98ad0b4d84ee67a09610c2dd1447958e/pillow-12.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385", size = 6322855, upload-time = "2026-07-01T11:55:21.613Z" }, - { url = "https://files.pythonhosted.org/packages/cc/91/420637fcb8f1bc11029e403b4538e6694744428d8246118e45719f944556/pillow-12.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c", size = 6989642, upload-time = "2026-07-01T11:55:24.006Z" }, - { url = "https://files.pythonhosted.org/packages/10/08/b94d7811281ccf0d143a1cf768d1c49e1e54af63e7b708ab2ee3eb87face/pillow-12.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d", size = 6391281, upload-time = "2026-07-01T11:55:26.252Z" }, - { url = "https://files.pythonhosted.org/packages/d2/87/24233f785f55474dc02ce3e739c5528a77e3a862e9333d1dd7a25cc31f70/pillow-12.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931", size = 7096716, upload-time = "2026-07-01T11:55:28.318Z" }, - { url = "https://files.pythonhosted.org/packages/23/26/fcb2f6e37175b04f53570b59937867e2b80ee1685e744023153028fc14f9/pillow-12.3.0-cp314-cp314t-win32.whl", hash = "sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7", size = 6474125, upload-time = "2026-07-01T11:55:30.956Z" }, - { url = "https://files.pythonhosted.org/packages/90/de/3634abee5f1c9e13c56787b7d5517b0ba8d6de51700b95578cf338349c9f/pillow-12.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c", size = 7242939, upload-time = "2026-07-01T11:55:34.044Z" }, - { url = "https://files.pythonhosted.org/packages/ce/2a/fd13f8eb24de5714a6eb444a3d67e2842c6c576e159a43793adf23051351/pillow-12.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45", size = 2567506, upload-time = "2026-07-01T11:55:35.988Z" }, - { url = "https://files.pythonhosted.org/packages/5d/dc/8fdce34ec725a33c81c6ba122b904d6b9024e50ea9ac7bede62fab54506c/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139", size = 4162063, upload-time = "2026-07-01T11:55:37.941Z" }, - { url = "https://files.pythonhosted.org/packages/76/66/2044b9a63d3b84ff048228dfcb7cd9bf0df983e8470971bf7d4c57b693de/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402", size = 4255549, upload-time = "2026-07-01T11:55:40.022Z" }, - { url = "https://files.pythonhosted.org/packages/52/7e/1f67e6f4ece6b582ee4b539decbcc9f848dc245a93ed8cd7338bafef72f1/pillow-12.3.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c", size = 3696331, upload-time = "2026-07-01T11:55:41.98Z" }, - { url = "https://files.pythonhosted.org/packages/12/40/d306fc2c8e4d45d7f175c77edca7063be7b86fe7fe6e68f4353bf71d808c/pillow-12.3.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f", size = 5350370, upload-time = "2026-07-01T11:55:44.028Z" }, - { url = "https://files.pythonhosted.org/packages/dd/44/668fb1437e8ce420f62d6106eb66e44a5971602a4d794615bdf79315d82d/pillow-12.3.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701", size = 4780147, upload-time = "2026-07-01T11:55:46.073Z" }, - { url = "https://files.pythonhosted.org/packages/0c/08/93fa2e70e30a2d81547e481b6ee2bb9522117221fb1e0ce4b5df70967677/pillow-12.3.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace", size = 6273659, upload-time = "2026-07-01T11:55:48.264Z" }, - { url = "https://files.pythonhosted.org/packages/f8/6d/043e96ff814fc31a33077e4cba86082167db520c93632afdf2042febbb0c/pillow-12.3.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4", size = 6947439, upload-time = "2026-07-01T11:55:50.503Z" }, - { url = "https://files.pythonhosted.org/packages/af/92/ba71d2ee2ac0edf3fa33bd9d5ee9ee080da70b1766f3ca3934f9938ddac9/pillow-12.3.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39", size = 6353577, upload-time = "2026-07-01T11:55:52.697Z" }, - { url = "https://files.pythonhosted.org/packages/0f/ce/e63064e2122923ff687c8ad792d0d736a7b3920a56a46982e81a7fdd25d6/pillow-12.3.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71", size = 7060394, upload-time = "2026-07-01T11:55:55.149Z" }, - { url = "https://files.pythonhosted.org/packages/54/76/a09cc3ccc8d773a7283d34c38bec1708f9e3cc932093cbc4c5e71ac4060b/pillow-12.3.0-cp315-cp315-win32.whl", hash = "sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827", size = 6467375, upload-time = "2026-07-01T11:55:57.769Z" }, - { url = "https://files.pythonhosted.org/packages/3e/03/1846c49ba3b1d5550392a4bbd06d6fb4578e1cd91a803198b5c90f5f7d53/pillow-12.3.0-cp315-cp315-win_amd64.whl", hash = "sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5", size = 7237048, upload-time = "2026-07-01T11:55:59.975Z" }, - { url = "https://files.pythonhosted.org/packages/fb/bb/89f35dcc79610423f9f195504d7def7f0d1416a711541b42867e25fe3412/pillow-12.3.0-cp315-cp315-win_arm64.whl", hash = "sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658", size = 2566006, upload-time = "2026-07-01T11:56:02.143Z" }, - { url = "https://files.pythonhosted.org/packages/30/88/707027ba09942dfa2c28759b5c222d769290a41c6d20ea60ec250801941f/pillow-12.3.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf", size = 5352509, upload-time = "2026-07-01T11:56:04.2Z" }, - { url = "https://files.pythonhosted.org/packages/b0/6d/00352fa25332c2569cd387851f568cc5a4b75a9adbfb37ac4fbce4c02eec/pillow-12.3.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64", size = 4783167, upload-time = "2026-07-01T11:56:06.631Z" }, - { url = "https://files.pythonhosted.org/packages/13/4f/9e049dfa21af7c22427275720e2490267ba8138120add5c4c574deb69782/pillow-12.3.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e", size = 6329237, upload-time = "2026-07-01T11:56:08.868Z" }, - { url = "https://files.pythonhosted.org/packages/36/16/cf6eeaae8d0fce8dd390a33437cf68c5d5bd73834a2bc6e2f14efda0ab45/pillow-12.3.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777", size = 6997047, upload-time = "2026-07-01T11:56:11.379Z" }, - { url = "https://files.pythonhosted.org/packages/1e/69/dbf769bdd55f48bf5733cac28edc6364ffaa072ec9ba336266e4fe66be55/pillow-12.3.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1", size = 6400440, upload-time = "2026-07-01T11:56:13.908Z" }, - { url = "https://files.pythonhosted.org/packages/a0/e1/ffc9cfc2eea0d178da8018e18e959301ad9d6bc9f3edb7181e748a474b97/pillow-12.3.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9", size = 7105895, upload-time = "2026-07-01T11:56:16.575Z" }, - { url = "https://files.pythonhosted.org/packages/18/f0/a5595c1e8c3ae44b9828cb2f0fa8155e5095ef04d6327b8f61cf44a3df85/pillow-12.3.0-cp315-cp315t-win32.whl", hash = "sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8", size = 6474384, upload-time = "2026-07-01T11:56:18.855Z" }, - { url = "https://files.pythonhosted.org/packages/e4/04/62bcd9f844984c5938d3b05264a61d797a29d3e0812341a8204af70bbdee/pillow-12.3.0-cp315-cp315t-win_amd64.whl", hash = "sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418", size = 7243537, upload-time = "2026-07-01T11:56:21.214Z" }, - { url = "https://files.pythonhosted.org/packages/3d/68/1f3066acedf37673694a7141381d8f811ae97f30d34413d236abe7d489f1/pillow-12.3.0-cp315-cp315t-win_arm64.whl", hash = "sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59", size = 2567491, upload-time = "2026-07-01T11:56:23.506Z" }, +version = "13.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/a4/9cc30690a394a8305e8be4c9036aa16d0dfddadd09079860b582f562496d/pillow-13.0.0.tar.gz", hash = "sha256:cdcd1ba258178f7356d74170c9a81ba4eab6a5d55748bc402a5a2a6fe6be3510", size = 47058631, upload-time = "2026-10-16T22:18:09.718Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/b8/58d79332229482eb0d27c82973850808c19d37e5bdc14443d3417bd373f5/pillow-13.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:67c8d44ad89fa7eb88de5d6ef6199b1e03dd24974f85a67639fc8badb801fe79", size = 5568643, upload-time = "2026-10-16T22:15:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/39/4b/f2ed53d83aa44ac19599ac0a54897f867fafb95f406980d1e24083f8cbe3/pillow-13.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc481f742ccda20e9ed760b8f978dff50f2a4e9bc253ac167fb0a05b0783ef52", size = 4852329, upload-time = "2026-10-16T22:15:46.284Z" }, + { url = "https://files.pythonhosted.org/packages/3b/bb/9e30e8ed18c20800ee327a12107d5249ad8dc444867b3adfcc21654e3c71/pillow-13.0.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8bf295fe459343bc6093c8e5e0df60bfa189e0fcb775a22e2c2488fdeccc6c0d", size = 6519309, upload-time = "2026-10-16T22:15:48.14Z" }, + { url = "https://files.pythonhosted.org/packages/e6/99/fdf65cc539e266ab93546c261bdf98411fe165c6e4e13f9dbf597097192b/pillow-13.0.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:42d63be3af4f9b951b199b0a9d23365d37ab14687e6d77b74940432c8515c4ca", size = 7144542, upload-time = "2026-10-16T22:15:50.093Z" }, + { url = "https://files.pythonhosted.org/packages/26/9b/f38b7c4d8f599537e2d1a6d546cec9beef7536ff8f5c49a3f9c78179d8ac/pillow-13.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4bd024d567eca931b668978d672dba914d4b8e2628e2c3073a2ad178de8a2620", size = 6618042, upload-time = "2026-10-16T22:15:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a8/7c/45ee28531f89ee4582967d8a17f7a26d71ca64cd902b93c269590cd60a00/pillow-13.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:23edf4ec4b6f4f240e8df6e4985e9ee2ced86474632433e55fb1e2548744bf40", size = 7228137, upload-time = "2026-10-16T22:15:54.009Z" }, + { url = "https://files.pythonhosted.org/packages/01/bc/9c2a969cb1b70201d04fa17fd5aedf02a732870b8da86bceaaa30b0ca385/pillow-13.0.0-cp312-cp312-win32.whl", hash = "sha256:02464da86f561f410e13b061ec37c3068a9cb557a095494157742d07b3c24a27", size = 6515423, upload-time = "2026-10-16T22:15:56.25Z" }, + { url = "https://files.pythonhosted.org/packages/ff/cd/73a79ecdc822076002eb24b5e1e4a530b0bb5310fcb6d239abbec877110b/pillow-13.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:ff3fe9eb621fbb9db4c4be33ab946f6bac5f98ab50ca8d3260503653656c961b", size = 7272166, upload-time = "2026-10-16T22:15:58.597Z" }, + { url = "https://files.pythonhosted.org/packages/4a/a3/be5a1236efd085b3984e4305da8f179ae9dab80c991d2ad78f2bd5c4c843/pillow-13.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:c3bc8ee0271387e0b1e3ad0ff893645990a656e79bf9c42f4f734a3642b11d51", size = 2698943, upload-time = "2026-10-16T22:16:00.637Z" }, + { url = "https://files.pythonhosted.org/packages/82/e2/9132d38df320e8099e2642d9fa538f18a7dfebf68a875a31725dc2a115aa/pillow-13.0.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:e98b4f91f99b24b1da72e10621ba181f7281922384a7403094debb4199de6c00", size = 4190620, upload-time = "2026-10-16T22:16:02.527Z" }, + { url = "https://files.pythonhosted.org/packages/cb/be/436dac0996876f65ef1916ffb711c5ef4af2ebc57414b6f16642e07f6b47/pillow-13.0.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:34fb3db1149c879343861831728c243c432d4a5afc8f1878ca7612ef8de2a033", size = 4290286, upload-time = "2026-10-16T22:16:04.861Z" }, + { url = "https://files.pythonhosted.org/packages/60/16/7c1007162e4169ea0d5bec47e2e0b2174f56c50cf5b8c02a1dc23e41037e/pillow-13.0.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:2d50500a4a3bc6469adcdb319cf05d688b08ca3bf0f22a05339f3b25b433a599", size = 3723169, upload-time = "2026-10-16T22:16:06.956Z" }, + { url = "https://files.pythonhosted.org/packages/08/bc/578ab2eaa04705a8978bd9be0fe15c3d43a8fc95097c4f7e39540e79e4fa/pillow-13.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ef5fe6b7db3f501c56000ba8fe7c7e1a35ca1d45824fbeb95a67daa85cff4dbe", size = 5568469, upload-time = "2026-10-16T22:16:08.664Z" }, + { url = "https://files.pythonhosted.org/packages/98/c8/ccede3dcc4f274b80fb49199eed1ef34ba05566c8b71976b9d1f111da2e3/pillow-13.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f11a3c1358bb3c68012a7b1f41b223c3db9f7bfde90b481df028cfe1c3363313", size = 4852187, upload-time = "2026-10-16T22:16:10.693Z" }, + { url = "https://files.pythonhosted.org/packages/fb/3c/cd0770e96d20bfd990db024bcbd43897ff3e88da1f4782cf90ea231b1d4f/pillow-13.0.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19b66ca0a4d25cc0200b381ce824c7ea090db61427315e73cfb05dbf1b35c369", size = 6515908, upload-time = "2026-10-16T22:16:12.896Z" }, + { url = "https://files.pythonhosted.org/packages/ef/fe/f5cdaa2d1c4086e757430f368c271a58ee29cfe7378e7ae7b9d964015c25/pillow-13.0.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5f56499c1a74d2b862beb0c33b8016f99d095858455df9d21583bbbc1c15b582", size = 7142544, upload-time = "2026-10-16T22:16:14.844Z" }, + { url = "https://files.pythonhosted.org/packages/fb/b5/a54f6d44adbcd23737c8c12953f42be107e4cca64e247c0ca3a5b563061c/pillow-13.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a2a14cd73d5e6c927f644b0110ee81f59c4ef506259ec9183904747f2caacc81", size = 6613097, upload-time = "2026-10-16T22:16:16.893Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b4/9e6c1207c21b2e91db0d15c16d30f3701ce5add1bd4da0e787761e66cb6e/pillow-13.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:08b826a9bf1265c76de04a42061d61ea6b109418b7826f28ff1ecdfaa26c5502", size = 7224846, upload-time = "2026-10-16T22:16:19.195Z" }, + { url = "https://files.pythonhosted.org/packages/78/9f/d3c3cb0518f3dd1eee0ef501535f53d15597a53c5c4118d178086143397a/pillow-13.0.0-cp313-cp313-win32.whl", hash = "sha256:12b0fa15879fe5a9d9eebe66e4e2d74128a773b1a4448010c9c714f45e04b9d8", size = 6516013, upload-time = "2026-10-16T22:16:21.389Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ed/1b16fb66a37e1b9e41370b93b2c614c7da761a5b2a41dc333f7af4639962/pillow-13.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:9dda0788a9932776a75d8bcc020076edc308ce2508b51f6739a45e3158b3fc42", size = 7284777, upload-time = "2026-10-16T22:16:23.393Z" }, + { url = "https://files.pythonhosted.org/packages/87/7d/8d02eba9f7ecc87f8e90f731e032cc4773802f2a843763afd0c66a535ff4/pillow-13.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:40a266371ece581e5de13b6635ef78d7487824af80561b6ca27694a88c19bc0f", size = 2698826, upload-time = "2026-10-16T22:16:25.382Z" }, + { url = "https://files.pythonhosted.org/packages/8e/45/fbad9c7f5e2f1745c373a434d46072bc84eddfefaa28b852425ac957fda6/pillow-13.0.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:44629d4498c9cb6b2051806155f22141df0d8a929c56134b166655232dfb2f8c", size = 4190721, upload-time = "2026-10-16T22:16:27.524Z" }, + { url = "https://files.pythonhosted.org/packages/a2/bf/2074902a259a90692c5dbf132885d21671644cbbd3f690047516243a55db/pillow-13.0.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:bf97ba1b5542050e9b0c74e632323b6d8136cc99d64e193abebee22fe6013c76", size = 4290236, upload-time = "2026-10-16T22:16:29.562Z" }, + { url = "https://files.pythonhosted.org/packages/ff/a5/3d1bb93b8ea38d15d21356c1b8e9f50971ac592aaa248f7fb71739094c0e/pillow-13.0.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:aa694c49787040943570d61c9b8fc01271bd2a2cfdd55f0d896f8aecdce1a981", size = 3723093, upload-time = "2026-10-16T22:16:31.249Z" }, + { url = "https://files.pythonhosted.org/packages/84/a1/be060135d85187b25fa8b2573786c0e45c62ae27527fff5257675eb97f4a/pillow-13.0.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:49c12af515073e9bf511e4ef736743bf70d531a643668722279314b937742ba4", size = 5572253, upload-time = "2026-10-16T22:16:33.103Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f4/0bf6f1ac9ba7ee2ffe9914aeb410a1035a0c1a3c42f122388d7fc950bc4b/pillow-13.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d9b0a2bee3f3e03e9f29057aedb807dbf1e1d96ee1b9d0f422d9db6c8583271d", size = 4852269, upload-time = "2026-10-16T22:16:35.102Z" }, + { url = "https://files.pythonhosted.org/packages/83/16/b603274617b3e6728da3a0133d4fba1ea8d7fbe0c4ea5fef9cbffa398b8b/pillow-13.0.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5933fcea5a5ba64c686f55c886a115cbfd92f2900460784a1e79909fdbafd7b6", size = 6516058, upload-time = "2026-10-16T22:16:37.228Z" }, + { url = "https://files.pythonhosted.org/packages/1d/f2/1371707ce438ba9a43f9d24089828494ec6f9396e28cd7903aa704dd6306/pillow-13.0.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e91c2b1cbeae768c98752e67e717e283d68dae45717b41f2d91eaee648e37b7d", size = 7142574, upload-time = "2026-10-16T22:16:39.172Z" }, + { url = "https://files.pythonhosted.org/packages/b9/a9/62cf68587f910bc87c5951bcb6e9430c7643a65076adf5f18b4bf350e2bd/pillow-13.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f6299e494e46253d40c34653cac369412f44ca53b5f85d848d3f95b1ed0546c", size = 6614295, upload-time = "2026-10-16T22:16:41.117Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c9/9e9b48178649730299be0bafbf5f810a5b6977538f5a92d47e4ba5f5d608/pillow-13.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f9bd7d260d986f3553518c5d723544346fa48031f8aef3265d786aeb59845bc0", size = 7224841, upload-time = "2026-10-16T22:16:43.118Z" }, + { url = "https://files.pythonhosted.org/packages/8f/48/c44e91384b1b6c072a714a63d00608acc16a6a968735e478e4be4384e5ac/pillow-13.0.0-cp314-cp314-win32.whl", hash = "sha256:2e31c4e68cffd7b1f116127626813b82fcba19edf6b940a7b0c266d6484d0055", size = 6508694, upload-time = "2026-10-16T22:16:45.883Z" }, + { url = "https://files.pythonhosted.org/packages/50/e2/7ad17747517e4c813dca78257d6bb593466f8231baf1384bc98d92745edb/pillow-13.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:f7cbb7a64e7ec664775195e4f81923bc8a93d9451b963f844e59118f63147a7a", size = 7281830, upload-time = "2026-10-16T22:16:47.725Z" }, + { url = "https://files.pythonhosted.org/packages/41/9f/e5b51f2ce8ef4563215334035403ce61a244f93a2e65c24a940131618f87/pillow-13.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:a6aaa7871aebca081eb5c45f83e845bbd9712ae99fa9286dbfdcee835e63f4b4", size = 2695468, upload-time = "2026-10-16T22:16:49.515Z" }, + { url = "https://files.pythonhosted.org/packages/97/73/a53502f731f6c84805a4ce87a8881e86d07a65aee64728b900d112dc703e/pillow-13.0.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7b999816b5850a35ada08443e6e03945f1f6488144a83afe2d37b240a552aafa", size = 5578196, upload-time = "2026-10-16T22:16:51.291Z" }, + { url = "https://files.pythonhosted.org/packages/21/50/7e949959a56b5559c78ef172b70fc6025706acb289f3f648a15329e4fa6f/pillow-13.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:67871a13cf47275201553b536c025da777f997fa562a19eb798af4ad4b97c18f", size = 4857276, upload-time = "2026-10-16T22:16:53.399Z" }, + { url = "https://files.pythonhosted.org/packages/c9/52/43ded0ac08e77e39d17bd6d894162572201b5e8fdc4a022b14c55a1960a4/pillow-13.0.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8ca8f69b7c77fa96dad99f8638c1fc4f1c0bed4acaa1baf7a3c31bdb80a0ab1", size = 6607008, upload-time = "2026-10-16T22:16:55.525Z" }, + { url = "https://files.pythonhosted.org/packages/0d/2e/4a57ed216d5bf877387cba4d44c02aa87abe51ad2e06404aa6b597c4da39/pillow-13.0.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6bce6a29c3898191f4c9998e4d4e8f69790caeabde46e54da4bc235ce5bde683", size = 7224281, upload-time = "2026-10-16T22:16:57.738Z" }, + { url = "https://files.pythonhosted.org/packages/5c/3d/2929b248e135752d63aaeabf53f6de797e905b7b6263f9c1e2adade20aba/pillow-13.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:816752a5ef76a80484c8c441c85593b7ea9304f6231fc5776ad1fbff7e773058", size = 6695600, upload-time = "2026-10-16T22:16:59.842Z" }, + { url = "https://files.pythonhosted.org/packages/bc/33/fb0da992cf084ed1ee6ee05e96b6191e35c351eb707ca2df874feb24ce47/pillow-13.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5bdb38ba220b61b4ec31dd1046dea6b016d3c6850a23f43ed8b58332b2fc668e", size = 7300589, upload-time = "2026-10-16T22:17:01.686Z" }, + { url = "https://files.pythonhosted.org/packages/bb/af/c30dd9ef6a3dce84ecb1b2393cac629047a32e66279056c6cb46c3ffd65c/pillow-13.0.0-cp314-cp314t-win32.whl", hash = "sha256:06903aac074d00c988e77a1fc435286dc498506d4ca8d6f69df9babddf4ecc15", size = 6517204, upload-time = "2026-10-16T22:17:03.722Z" }, + { url = "https://files.pythonhosted.org/packages/85/d5/ef3a8ae7609c8ab26570abf18194882534b480d245a9148b895a2262c004/pillow-13.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:9a635068d9793c7f0d929f39f18b58f1bd86c236e8e7b64167b86af9232ebca0", size = 7286695, upload-time = "2026-10-16T22:17:05.543Z" }, + { url = "https://files.pythonhosted.org/packages/84/3c/0861db0c69a03689ebe3e7599d35668a414deb5aaafd8255215e4e62e3d3/pillow-13.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0d1a9cf7bc6917538b20f8d06b9ab2147e542f68849e96fc58eb66bdf6acc692", size = 2698098, upload-time = "2026-10-16T22:17:07.674Z" }, + { url = "https://files.pythonhosted.org/packages/6e/eb/e86a41962a3a31ad495a3d6a912d9002b01bba4a6fa71c28703abdca7f69/pillow-13.0.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7d5e097eabd5d4915c27ac9f6e9325bb1e01a7c94d7348b5c4451d3fc30e8521", size = 4191213, upload-time = "2026-10-16T22:17:09.449Z" }, + { url = "https://files.pythonhosted.org/packages/dd/04/7e7445e706afec84ef06dc25b2c77f9cecff242c23730c62928ad1089d4a/pillow-13.0.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:51ebb529fc5e8abdd55302f6c3f8943037348ec2384329a3e55677f69ae2fdfc", size = 4290650, upload-time = "2026-10-16T22:17:11.196Z" }, + { url = "https://files.pythonhosted.org/packages/46/85/d534089068ac40c0f61f650389a296c812eab745a8a733f81feec39ae955/pillow-13.0.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:c87a04e7f152ee4db5dbe15f0bfa3fe628a1aeedbe66b81c43b87f52c813f9ac", size = 3723078, upload-time = "2026-10-16T22:17:12.772Z" }, + { url = "https://files.pythonhosted.org/packages/ce/2b/00b5c3b401df538b5dbe311d3899730fae5ba1e21cb8b80a128887ac6747/pillow-13.0.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:20722cd5375533ad754053363a80b8fb30c79d37930a6f7228030e237ce3bd5d", size = 5572292, upload-time = "2026-10-16T22:17:14.539Z" }, + { url = "https://files.pythonhosted.org/packages/72/f5/a6b0610f946e9b54ae888ba7ad29d6e2f8d9b11f69e212d70416b7186c09/pillow-13.0.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:04a63720f79e22fd54d897c233a68314c50ca0603360f54ce2bd5db8f0f43782", size = 4852222, upload-time = "2026-10-16T22:17:16.521Z" }, + { url = "https://files.pythonhosted.org/packages/1e/d0/c0ff25bb0427ec70f32aac995453321fc3c4494a46c7292ff02d9d260b04/pillow-13.0.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4f8db54177035a2b5f717f94475b74907e12a96dc74f6d487f3966b0a4694071", size = 6527349, upload-time = "2026-10-16T22:17:18.534Z" }, + { url = "https://files.pythonhosted.org/packages/85/11/e46fa0bcc0fc117f1eacc206b3d51f04d583fab328b4b740672d0438d83f/pillow-13.0.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f36aa77f74b14a2103622b196402c71330e4e7cf4122ee45c854e9a01e0eecf", size = 7152873, upload-time = "2026-10-16T22:17:20.415Z" }, + { url = "https://files.pythonhosted.org/packages/7c/53/2bdeca705f3a58679fa9201233193c86a30238340ca29b3a152c3fcc719b/pillow-13.0.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:149a1c323ba366f90b48e6f1a6db6582eb50512df374588de56fc909abc0f0f3", size = 6626871, upload-time = "2026-10-16T22:17:22.361Z" }, + { url = "https://files.pythonhosted.org/packages/14/de/3829030f209722ade3753bb456b805a0e7f885cf885b483407365c82e9ca/pillow-13.0.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e892354a424b1e82a78ba16785822bdd40de06f19446a3d4a28047df1a4f8932", size = 7236439, upload-time = "2026-10-16T22:17:24.352Z" }, + { url = "https://files.pythonhosted.org/packages/44/b1/67fc37388382bc036a74f04da5bb6fcbf90ec3ac1f45893e06176c373fd3/pillow-13.0.0-cp315-cp315-win32.whl", hash = "sha256:8e459b613cf31f8980f584387b4fe47bf08e076e288d61aa8eba1f7fd4ddbb81", size = 6508688, upload-time = "2026-10-16T22:17:26.314Z" }, + { url = "https://files.pythonhosted.org/packages/44/05/fdb6a79aee10861165526b41b7b440e36ee1ff58613b2fbefe3acbf06d4f/pillow-13.0.0-cp315-cp315-win_amd64.whl", hash = "sha256:d4737f50d23ddd11d6685faacbb9252887dde0bc7997127757247983d74399d6", size = 7281107, upload-time = "2026-10-16T22:17:28.198Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9c/6c6f7dc2a566467baf121394e83de25ac287cffe34cc030726beb4be0224/pillow-13.0.0-cp315-cp315-win_arm64.whl", hash = "sha256:2310e2adfc13c6d457d7d0f2ffdd5c56a74a317650cf765887df8d515a647ac0", size = 2695513, upload-time = "2026-10-16T22:17:30.013Z" }, + { url = "https://files.pythonhosted.org/packages/23/16/e9120defdb8bff693b582d035312acc6f530cdc2bc5238a87f410efccd82/pillow-13.0.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:21c32061a5512dc13f87cc7303e120e4d77c2f70df89a35af0e3ce246e5ac57f", size = 5578129, upload-time = "2026-10-16T22:17:31.642Z" }, + { url = "https://files.pythonhosted.org/packages/47/22/76faf2bb2448f48a84cf894c1e2bb1090d7d2b1aa521ece131cb2f05d631/pillow-13.0.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:1c774b9c0ba0d0b356f71530677b39028878a2cae5a93e0a13779cfd17120f88", size = 4857428, upload-time = "2026-10-16T22:17:33.43Z" }, + { url = "https://files.pythonhosted.org/packages/b3/64/0861ea4e03c50b14af9ed9dd8b9f388ab42f445c623a9fcbab19aef7aa55/pillow-13.0.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20822140c9a2ba9c5e42df342b0a892ab2e102fc0e3db967a133bb9ce3f898bb", size = 6612582, upload-time = "2026-10-16T22:17:35.415Z" }, + { url = "https://files.pythonhosted.org/packages/5c/20/cd1e9b3f60a9f3d5ebe46da36955c26004092a501f7f2c42f9fcc26611bc/pillow-13.0.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ffea6b2986a797c84b6d1d27d4f43e0b83c73fc16281a930891cd237cf0d08fe", size = 7230176, upload-time = "2026-10-16T22:17:37.362Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ef/ffb0a2fc85690714fb7ac662956949645d57f15bf4426f520da361e7b6e3/pillow-13.0.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:05cd2cd39055d0b9650060e4513ef4a9a0aea870083950bd046a0d6490f93a22", size = 6703640, upload-time = "2026-10-16T22:17:39.313Z" }, + { url = "https://files.pythonhosted.org/packages/e0/c2/6402ee3ccb3363aeed8fdb3f9fe56703ac5377a539f0b81b53999d7e121f/pillow-13.0.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:c6fd031333dedcad7ce3e03d244093f43504f209e86925ebdb2b868fa4037630", size = 7309231, upload-time = "2026-10-16T22:17:41.021Z" }, + { url = "https://files.pythonhosted.org/packages/fc/96/d6b82e8b8c06ad143b66b12e0b31051e4bf489b4fabca1f390a14fe1ebde/pillow-13.0.0-cp315-cp315t-win32.whl", hash = "sha256:c50c80e3591e5cf9a8dfa6007d4f10732f02e32263850999286e56ccd650cfb2", size = 6517200, upload-time = "2026-10-16T22:17:42.886Z" }, + { url = "https://files.pythonhosted.org/packages/60/d4/0062e6005e9b270b165015aafd343364ac378ec5a685431ea42baad33357/pillow-13.0.0-cp315-cp315t-win_amd64.whl", hash = "sha256:adb3c2124c68e480f4632d06cb7caa3b48a51678475f483b4edb1c6dee29ea43", size = 7287488, upload-time = "2026-10-16T22:17:45.56Z" }, + { url = "https://files.pythonhosted.org/packages/e4/81/b8eb349746be7bce47dd95337c6634e84910af91b59aaa9159c034c614fa/pillow-13.0.0-cp315-cp315t-win_arm64.whl", hash = "sha256:4a25756244bdcce3b50580baadf320fa48297f0eeb8e114ba64139a40d4d0fc8", size = 2698073, upload-time = "2026-10-16T22:17:47.556Z" }, + { url = "https://files.pythonhosted.org/packages/60/da/f6d644e631c5b370c5d0b899f3ccadd2780aab070b29a110527cf4e1fd2c/pillow-13.0.0-pp312-pypy312_pp80-macosx_10_15_x86_64.whl", hash = "sha256:8619685cad8efc4c38c66d3b36aad854523297c43d748b3b6f0cc03c90ba3463", size = 5523188, upload-time = "2026-10-16T22:17:59.334Z" }, + { url = "https://files.pythonhosted.org/packages/e4/b0/2594f202d743a35d4fe80682bfd96f277e5ca7aecb27cf64ef5bfc1d000b/pillow-13.0.0-pp312-pypy312_pp80-macosx_11_0_arm64.whl", hash = "sha256:7e91e97a929232cdb828889ee6ad0e48d7f40f4ec38818f4e4a8968b2b1af22e", size = 4806808, upload-time = "2026-10-16T22:18:01.188Z" }, + { url = "https://files.pythonhosted.org/packages/8e/ed/d504165ef79af6fafe25c76726cf30f544cc236002f7ed054f1d51adc4ff/pillow-13.0.0-pp312-pypy312_pp80-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93ff9c9c3663ce23aaa5d0b6b6abac55359e2a0c06c7e30f3771829af816be63", size = 5464358, upload-time = "2026-10-16T22:18:02.779Z" }, + { url = "https://files.pythonhosted.org/packages/7b/fe/a10aba5503ca13e9e009b8bd0a1a4ce5584bdce246245fd6f3a83701369c/pillow-13.0.0-pp312-pypy312_pp80-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e65dd6e75de67b2362a4fa3e16aa627595f65054dc4ec73d64680c84d592fa61", size = 6043514, upload-time = "2026-10-16T22:18:04.538Z" }, + { url = "https://files.pythonhosted.org/packages/c0/37/66b12dc476041fe523bca29c9d58fbf1ea92a4eaf73acfb9e993948f1191/pillow-13.0.0-pp312-pypy312_pp80-win_amd64.whl", hash = "sha256:567f20f1c75897d7663830da6ea8af719e03d712b1d9ee7f813319f9ece42e5a", size = 7366030, upload-time = "2026-10-16T22:18:06.638Z" }, ] [[package]] From 890b1eeec1523553e42c152a72ffba98a2188c57 Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Tue, 20 Oct 2026 18:35:25 -0400 Subject: [PATCH 04/19] feat(features): add explicit finite-window Pipeline factor methods Preserve native TA-Lib defaults while providing separately registered finite weighting, Cutler RSI, population volatility and technical field definitions. Keep missing-value denominators, warmup and grouped lazy execution explicit. Refs #108. --- docs/user-guide/features.md | 61 ++++ src/ml4t/engineer/core/lookbacks.py | 16 + src/ml4t/engineer/features/AGENTS.md | 14 +- src/ml4t/engineer/features/__init__.py | 2 + src/ml4t/engineer/features/rolling.py | 437 +++++++++++++++++++++++++ tests/compat/test_zipline_factors.py | 308 +++++++++++++++++ tests/core/test_registry.py | 4 +- tests/test_all_features_integration.py | 2 +- 8 files changed, 834 insertions(+), 10 deletions(-) create mode 100644 src/ml4t/engineer/features/rolling.py create mode 100644 tests/compat/test_zipline_factors.py diff --git a/docs/user-guide/features.md b/docs/user-guide/features.md index e1c03cfc..886a0364 100644 --- a/docs/user-guide/features.md +++ b/docs/user-guide/features.md @@ -123,6 +123,67 @@ expressions, not portfolio statistics. Use explicit grouping for multiple assets validate the declared bar grid before deriving returns as described below. Row-only data must already have the intended observation order. +### Explicit finite-window methods + +The next minor version adds the following registered methods for workflows that need +finite observation windows rather than the native recursive or TA-Lib definitions. +Existing `rsi`, `ema`, `macd`, `aroon`, `bollinger_bands` and risk features keep their +defaults. These additions do not provide a Pipeline graph interpreter. + +| Feature | Definition and missingness | +|---|---| +| `rsi_cutler` | Simple gains/losses over `period` changes (`period + 1` prices). Omit changes adjacent to missing quotes; a flat window is undefined. | +| `finite_mean` | Require `period` rows, then divide by the count of nonmissing prices. | +| `finite_weighted_mean` | Sum valid price-times-volume products, divided by the independently summed volume. Missing prices do not remove denominator weights. | +| `average_dollar_volume` | Sum valid price-times-volume products, divided by the fixed `period` row count. Input price, share and currency basis must already agree. | +| `linear_weighted_mean` | Weights `1..period`, oldest to newest. Omit missing products but retain every denominator weight. | +| `finite_ewma` | Normalize `decay_rate**age` over exactly `period` prices. Missing prices propagate; this is not recursive EMA. | +| `finite_ewm_std` | The same finite weights, with weighted bias correction. Insufficient effective degrees of freedom yield NaN. | +| `annualized_volatility` | Population standard deviation (`ddof=0`) of supplied `returns`, omitting missing values after the full window. Multiply by `sqrt(annualization_factor)`, default 252. Calendar choice does not set this scale. | +| `finite_max_drawdown` | Select the first largest absolute price drop, then divide by its trough price. This explicit legacy definition is not a peak-denominator portfolio drawdown. | +| `finite_bollinger_bands` | NaN-omitting mean and population standard deviation; fields `lower`, `middle`, `upper`, with multiplier `k`. | +| `finite_aroon` | First extreme wins ties; fields `down`, `up`. Scale the position in `period` prices by `100 / (period - 1)`. | +| `fast_stochastic` | Unsmoothed K from the latest close and NaN-omitting high/low extrema. Zero range remains undefined. | +| `finite_ichimoku` | Unshifted `tenkan_sen`, `kijun_sen`, `senkou_span_a`, `senkou_span_b`. `chikou_position` indexes from the oldest price in the window, unlike a chart displacement. | +| `finite_macd_signal` | Finite normalized fast/slow exponential means, followed by a finite signal mean. Length-one means use their sole observation. | +| `finite_true_range` | Two-price true range with NaN-omitting high/low/previous-close components. There is no implicit larger-window reduction. | + +Except for Cutler RSI and the composed MACD signal, `period` counts input rows and the +first structurally usable row is `period - 1`. Missing observations still occupy their +original row positions. All methods return Float64 expressions and retain every input +row. Multi-output features use the usual `_` column names. Ichimoku +propagates missing high/low values within each field's subwindow; all fields require +the declared full window. Its zero tenkan/kijun lengths select that full window, +matching the legacy `[-0:]` slice, while `chikou_position` must be in `[0, period)`. + + +```python +import math +import polars as pl +from ml4t.engineer import compute_features + +prices = pl.DataFrame({"close": [10.0, 13.0, 12.0, 14.0, 13.0]}).with_columns( + pl.col("close").alias("open"), + pl.col("close").alias("high"), + pl.col("close").alias("low"), + pl.lit(100.0).alias("volume"), +) +result = compute_features(prices.lazy(), [ + {"name": "rsi_cutler", "params": {"period": 3}, "output": "cutler"}, + {"name": "rsi", "params": {"period": 3}, "output": "wilder"}, +], calendar=None).collect() +assert math.isclose(result["cutler"][4], 50.0, rel_tol=1e-14, abs_tol=0.0) +assert result["wilder"][4] != result["cutler"][4] +assert result.height == prices.height +``` + +Use explicit grouping or `compute_features` for multiple assets. A `calendar=None` +call counts the supplied observation rows without checking schedules. With a named +calendar, declare the complete grid before computing any feature. The finite methods +do not infer frequency, select source revisions, fill gaps or make future information +eligible. They currently support observation/expected-bar windows; do not reinterpret +their kernels as session or elapsed windows. + ### Calendar and bar grid The calendar-aware default targets the next minor version (0.2), separate from the 0.1.7 patch. diff --git a/src/ml4t/engineer/core/lookbacks.py b/src/ml4t/engineer/core/lookbacks.py index 4818e414..543d18dd 100644 --- a/src/ml4t/engineer/core/lookbacks.py +++ b/src/ml4t/engineer/core/lookbacks.py @@ -70,6 +70,22 @@ def _ulcer_index(params: Mapping[str, Any]) -> int: FEATURE_LOOKBACKS: dict[str, LookbackCalculator] = { + # Explicit finite-window methods, independent of the native TA-Lib defaults. + "rsi_cutler": _parameter("period"), + "finite_mean": _parameter("period", -1), + "finite_weighted_mean": _parameter("period", -1), + "average_dollar_volume": _parameter("period", -1), + "linear_weighted_mean": _parameter("period", -1), + "finite_ewma": _parameter("period", -1), + "finite_ewm_std": _parameter("period", -1), + "annualized_volatility": _parameter("period", -1), + "finite_max_drawdown": _parameter("period", -1), + "finite_bollinger_bands": _parameter("period", -1), + "finite_aroon": _parameter("period", -1), + "fast_stochastic": _parameter("period", -1), + "finite_ichimoku": _parameter("period", -1), + "finite_macd_signal": _sum("slow_period", "signal_period", adjustment=-2), + "finite_true_range": _constant(1), # Volume "ad": _constant(0), "adosc": _parameter("slowperiod", -1), diff --git a/src/ml4t/engineer/features/AGENTS.md b/src/ml4t/engineer/features/AGENTS.md index 6435dea9..c2df0f09 100644 --- a/src/ml4t/engineer/features/AGENTS.md +++ b/src/ml4t/engineer/features/AGENTS.md @@ -7,17 +7,17 @@ This directory contains the registry-backed indicator implementations used by | Category | Count | Location | Examples | |----------|-------|----------|----------| -| momentum | 31 | `momentum/` | `rsi`, `macd`, `stoch`, `adx`, `mfi` | -| trend | 10 | `trend/` | `sma`, `ema`, `wma`, `kama`, `donchian_channels` | -| volatility | 15 | `volatility/` | `atr`, `bollinger_bands`, `yang_zhang_volatility`, `garch_forecast` | +| momentum | 35 | `momentum/`, `rolling.py` | `rsi`, `rsi_cutler`, `macd`, `finite_macd_signal` | +| trend | 15 | `trend/`, `rolling.py` | `sma`, `ema`, `finite_mean`, `finite_ewma` | +| volatility | 18 | `volatility/`, `rolling.py` | `atr`, `bollinger_bands`, `annualized_volatility` | | microstructure | 15 | `microstructure/` | `kyle_lambda`, `amihud_illiquidity`, `realized_spread` | | ml | 14 | `ml/` plus `fdiff.py` | lag features, entropy, Fourier, percentile-rank features | -| statistics | 14 | `statistics/` | `stddev`, `var`, `linearreg`, `structural_break` | -| volume | 3 | `volume/` | `obv`, `ad`, `adosc` | +| statistics | 15 | `statistics/`, `rolling.py` | `stddev`, `var`, `finite_ewm_std`, `structural_break` | +| volume | 4 | `volume/`, `rolling.py` | `obv`, `ad`, `adosc`, `average_dollar_volume` | | price_transform | 5 | `price_transform/` | `avgprice`, `typprice`, `medprice`, `wclprice`, `midprice` | | math | 3 | `math/` | rolling `max_`, `min_`, `sum_` | | regime | 4 | `regime.py` | `hurst_exponent`, `choppiness_index`, `fractal_efficiency` | -| risk | 6 | `risk.py` | drawdown, downside deviation, tail ratio | +| risk | 7 | `risk.py`, `rolling.py` | drawdown, downside deviation, tail ratio | ## Standalone Helpers @@ -70,6 +70,6 @@ durations. Do not resample, infer missing sessions or reinterpret recursive indi ## Trust Signals -- 120 total features across 11 categories +- 135 total features across 11 categories - 60 TA-Lib validated indicators at `1e-6` tolerance - registry metadata supports discovery by category, normalization, and TA-Lib parity diff --git a/src/ml4t/engineer/features/__init__.py b/src/ml4t/engineer/features/__init__.py index 2f77ed86..9506a7fc 100644 --- a/src/ml4t/engineer/features/__init__.py +++ b/src/ml4t/engineer/features/__init__.py @@ -37,6 +37,7 @@ price_transform, # noqa: F401 regime, risk, + rolling, statistics, # noqa: F401 trend, # noqa: F401 volatility, # noqa: F401 @@ -50,4 +51,5 @@ "fdiff", "regime", "risk", + "rolling", ] diff --git a/src/ml4t/engineer/features/rolling.py b/src/ml4t/engineer/features/rolling.py new file mode 100644 index 00000000..5b588819 --- /dev/null +++ b/src/ml4t/engineer/features/rolling.py @@ -0,0 +1,437 @@ +"""Finite observation-window methods with explicit missing-value definitions. + +These methods are independent of the recursively seeded TA-Lib indicators. +Window sizes count input observations, including missing observations. Expressions +must be applied within the caller's asset groups, as compute_features does. +""" + +import math + +import numpy as np +import numpy.typing as npt +import polars as pl + +from ml4t.engineer._numba import jit +from ml4t.engineer.core.decorators import feature + + +def _period(period: int, *, minimum: int = 1) -> int: + if isinstance(period, bool) or not isinstance(period, int) or period < minimum: + raise ValueError(f"period must be an integer >= {minimum}, got {period!r}") + return period + + +def _ready(expression: pl.Expr, period: int) -> pl.Expr: + return pl.when(pl.int_range(pl.len()) >= period - 1).then(expression) + + +def _numeric(column: str) -> pl.Expr: + return pl.col(column).cast(pl.Float64).fill_nan(None) + + +@feature( + name="rsi_cutler", + category="momentum", + description="Finite simple-average RSI over period changes, omitting missing changes", + normalized=True, + value_range=(0.0, 100.0), + formula="100 * sum(positive changes) / sum(absolute changes)", +) +def rsi_cutler(close: str, period: int = 14) -> pl.Expr: + """Cutler RSI over period changes (period+1 prices); flat windows yield NaN. + + Missing quotes invalidate their adjacent changes. The remaining changes in + the full observation window are reduced without filling the missing prices. + Integer changes are subtracted in Int128 before floating conversion. + """ + _period(period) + + def change(price: pl.Expr, dtype: pl.DataType) -> pl.Expr: + if dtype.is_integer() and dtype != pl.Int128: + return price.cast(pl.Int128).diff().cast(pl.Float64) + return price.diff().cast(pl.Float64) + + delta = pl.col(close).pipe_with_dtype(change).fill_nan(None) + gain = delta.clip(lower_bound=0).rolling_mean(period, min_samples=1) + loss = (-delta.clip(upper_bound=0)).rolling_mean(period, min_samples=1) + return _ready(100 * gain / (gain + loss), period + 1) + + +@feature( + name="finite_mean", + category="trend", + description="Full-window simple mean with NaN/null omission and valid-count denominator", + formula="sum(valid prices) / count(valid prices)", +) +def finite_mean(close: str, period: int = 20) -> pl.Expr: + """Require period rows, then average the nonmissing values in that window.""" + _period(period) + return _ready(_numeric(close).rolling_mean(period, min_samples=1), period) + + +@feature( + name="finite_weighted_mean", + category="trend", + description="Finite weighted mean with independently reduced weight denominator", + input_type="OHLCV", + formula="nansum(close * volume) / nansum(volume)", +) +def finite_weighted_mean(close: str, volume: str, period: int = 20) -> pl.Expr: + """Weights paired with missing prices remain in the independent denominator. + + This explicit definition reproduces WeightedAverageValue/VWAP in Pipeline. + Callers requiring a paired-valid denominator must mask weights themselves. + """ + _period(period) + numerator = (_numeric(close) * _numeric(volume)).rolling_sum(period, min_samples=0) + denominator = _numeric(volume).rolling_sum(period, min_samples=0) + return _ready(numerator / denominator, period) + + +@feature( + name="average_dollar_volume", + category="volume", + description="Finite price-times-volume sum divided by the full observation count", + input_type="OHLCV", + formula="nansum(close * volume) / period", +) +def average_dollar_volume(close: str, volume: str, period: int = 20) -> pl.Expr: + """Mean dollar volume with fixed-count denominator; no currency/basis conversion.""" + _period(period) + total = (_numeric(close) * _numeric(volume)).rolling_sum(period, min_samples=0) + return _ready(total / period, period) + + +@jit(nopython=True, cache=True) # type: ignore[misc] +def _weighted_windows( + values: npt.NDArray[np.float64], + weights: npt.NDArray[np.float64], + variance: bool, + omit: bool, +) -> npt.NDArray[np.float64]: + width = len(weights) + result = np.full(len(values), np.nan) + weight_sum = weights.sum() + correction_denominator = weight_sum * weight_sum - (weights * weights).sum() + for end in range(width - 1, len(values)): + total = 0.0 + missing = False + for j in range(width): + value = values[end - width + 1 + j] + if np.isnan(value): + missing = True + else: + total += weights[j] * value + if missing and not omit: + continue + mean = total / weight_sum + if not variance: + result[end] = mean + elif correction_denominator > 0: + centered = 0.0 + for j in range(width): + delta = values[end - width + 1 + j] - mean + centered += weights[j] * delta * delta + result[end] = np.sqrt(centered * weight_sum / correction_denominator) + return result + + +def _weighted_expression( + close: str | pl.Expr, + weights: npt.NDArray[np.float64], + *, + variance: bool = False, + omit: bool = False, +) -> pl.Expr: + expression = pl.col(close) if isinstance(close, str) else close + return expression.cast(pl.Float64).map_batches( + lambda values: pl.Series(_weighted_windows(values.to_numpy(), weights, variance, omit)), + return_dtype=pl.Float64, + ) + + +@feature( + name="linear_weighted_mean", + category="trend", + description="Finite linear weights 1..period with fixed denominator and NaN omission", + formula="nansum(price * [1..period]) / (period * (period+1) / 2)", +) +def linear_weighted_mean(close: str, period: int = 20) -> pl.Expr: + """Linear weighted mean; a missing price does not remove its denominator weight.""" + _period(period) + return _weighted_expression(close, np.arange(1, period + 1, dtype=np.float64), omit=True) + + +def _exponential_weights(period: int, decay_rate: float) -> npt.NDArray[np.float64]: + _period(period) + if ( + isinstance(decay_rate, bool) + or not isinstance(decay_rate, int | float) + or not math.isfinite(decay_rate) + or not 0 < decay_rate <= 1 + ): + raise ValueError("decay_rate must be finite and in (0, 1]") + # The common power cancels in both statistics. Newest weight 1 avoids + # needless underflow from the reference's decay**(period+1)..decay**2. + return np.power(float(decay_rate), np.arange(period - 1, -1, -1, dtype=np.float64)) + + +@feature( + name="finite_ewma", + category="trend", + description="Finite normalized exponential weighted mean; missing values propagate", + formula="sum(price * decay_rate**age) / sum(decay_rate**age)", +) +def finite_ewma(close: str, period: int = 20, decay_rate: float = 0.9) -> pl.Expr: + """Finite-window EWMA, distinct from recursively seeded ema.""" + return _weighted_expression(close, _exponential_weights(period, decay_rate)) + + +@feature( + name="finite_ewm_std", + category="statistics", + description="Finite exponential weighted standard deviation with weighted bias correction", + formula="sqrt(weighted_variance * sum(w)**2 / (sum(w)**2-sum(w**2)))", +) +def finite_ewm_std(close: str, period: int = 20, decay_rate: float = 0.9) -> pl.Expr: + """Bias-corrected finite weighted std; undefined effective degrees yield NaN.""" + return _weighted_expression(close, _exponential_weights(period, decay_rate), variance=True) + + +@feature( + name="annualized_volatility", + category="volatility", + description="Population standard deviation of supplied returns with explicit annualization", + input_type="returns", + formula="nanstd(returns, ddof=0) * sqrt(annualization_factor)", +) +def annualized_volatility( + returns: str, period: int = 252, annualization_factor: float = 252.0 +) -> pl.Expr: + """Full-window population volatility; period counts supplied return observations.""" + _period(period) + if ( + isinstance(annualization_factor, bool) + or not isinstance(annualization_factor, int | float) + or not math.isfinite(annualization_factor) + or annualization_factor <= 0 + ): + raise ValueError("annualization_factor must be positive and finite") + volatility = _numeric(returns).rolling_std(period, min_samples=1, ddof=0) + return _ready(volatility * math.sqrt(annualization_factor), period) + + +@jit(nopython=True, cache=True) # type: ignore[misc] +def _drawdown_windows(values: npt.NDArray[np.float64], period: int) -> npt.NDArray[np.float64]: + result = np.full(len(values), np.nan) + for end in range(period - 1, len(values)): + peak = np.nan + largest = -np.inf + selected_peak = np.nan + selected_end = np.nan + for j in range(end - period + 1, end + 1): + value = values[j] + if np.isnan(value): + continue + peak = np.fmax(peak, value) + difference = peak - value + if difference > largest: + largest = difference + selected_peak, selected_end = peak, value + if selected_end != 0: + result[end] = (selected_peak - selected_end) / selected_end + elif selected_peak > 0: + result[end] = np.inf + return result + + +@feature( + name="finite_max_drawdown", + category="risk", + description="Pipeline absolute-drop selection with trough-price denominator", + formula="(peak - trough) / trough at first largest absolute price drop", +) +def finite_max_drawdown(close: str, period: int = 20) -> pl.Expr: + """Legacy finite drawdown definition, not portfolio/peak-denominator drawdown.""" + _period(period) + return ( + pl.col(close) + .cast(pl.Float64) + .map_batches( + lambda values: pl.Series(_drawdown_windows(values.to_numpy(), period)), + return_dtype=pl.Float64, + ) + ) + + +@feature( + name="finite_bollinger_bands", + category="volatility", + description="Finite NaN-omitting mean and population-standard-deviation bands", + formula="middle=nanmean(close); lower/upper=middle +/- k*nanstd(close,ddof=0)", +) +def finite_bollinger_bands(close: str, period: int = 20, k: float = 2.0) -> dict[str, pl.Expr]: + """Population bands requiring period rows; no smoothing or sample correction.""" + _period(period) + if isinstance(k, bool) or not isinstance(k, int | float) or not math.isfinite(k) or k < 0: + raise ValueError("k must be finite and nonnegative") + middle = finite_mean(close, period) + distance = _ready(_numeric(close).rolling_std(period, min_samples=1, ddof=0), period) * k + return {"lower": middle - distance, "middle": middle, "upper": middle + distance} + + +@jit(nopython=True, cache=True) # type: ignore[misc] +def _extreme_position( + values: npt.NDArray[np.float64], period: int, maximum: bool +) -> npt.NDArray[np.float64]: + result = np.full(len(values), np.nan) + for end in range(period - 1, len(values)): + chosen = np.nan + index = -1 + for j in range(period): + value = values[end - period + 1 + j] + if not np.isnan(value) and ( + index == -1 or (value > chosen if maximum else value < chosen) + ): + chosen, index = value, j + if index >= 0: + result[end] = 100.0 * index / (period - 1) + return result + + +@feature( + name="finite_aroon", + category="momentum", + description="Finite Aroon with first extreme ties and period-1 denominator", + input_type="HL", + normalized=True, + value_range=(0.0, 100.0), + formula="100 * first extreme position / (period-1)", +) +def finite_aroon(high: str, low: str, period: int = 25) -> dict[str, pl.Expr]: + """First-tie Aroon fields; period counts prices, not changes.""" + _period(period, minimum=2) + + def position(column: str, maximum: bool) -> pl.Expr: + return ( + pl.col(column) + .cast(pl.Float64) + .map_batches( + lambda values: pl.Series(_extreme_position(values.to_numpy(), period, maximum)), + return_dtype=pl.Float64, + ) + ) + + return {"down": position(low, False), "up": position(high, True)} + + +@feature( + name="fast_stochastic", + category="momentum", + description="Unsmoothed finite stochastic K with NaN-omitting extrema", + input_type="HLC", + normalized=True, + value_range=(0.0, 100.0), + formula="100 * (close - nanmin(low)) / (nanmax(high) - nanmin(low))", +) +def fast_stochastic(high: str, low: str, close: str, period: int = 14) -> pl.Expr: + """Unsmoothed K; zero ranges stay undefined rather than being replaced by zero.""" + _period(period) + upper = _numeric(high).rolling_max(period, min_samples=1) + lower = _numeric(low).rolling_min(period, min_samples=1) + return _ready(100 * (_numeric(close) - lower) / (upper - lower), period) + + +@feature( + name="finite_ichimoku", + category="trend", + description="Finite unshifted Ichimoku spans and explicit within-window chikou position", + input_type="HLC", +) +def finite_ichimoku( + high: str, + low: str, + close: str, + period: int = 52, + tenkan_length: int = 9, + kijun_length: int = 26, + chikou_position: int = 26, +) -> dict[str, pl.Expr]: + """Pipeline's unshifted fields, not a chart's displaced Ichimoku cloud. + + A zero tenkan/kijun length selects the whole window, matching Python's + legacy [-0:] slice. Chikou is an index from the oldest price in the window, + not a negative shift; its index must be in [0, period). + """ + _period(period) + for name, value in (("tenkan_length", tenkan_length), ("kijun_length", kijun_length)): + if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= period: + raise ValueError(f"{name} must be an integer in [0, period]") + if ( + isinstance(chikou_position, bool) + or not isinstance(chikou_position, int) + or not 0 <= chikou_position < period + ): + raise ValueError("chikou_position must be an integer in [0, period)") + + def midpoint(width: int) -> pl.Expr: + return (_numeric(high).rolling_max(width) + _numeric(low).rolling_min(width)) / 2 + + tenkan = midpoint(tenkan_length or period) + kijun = midpoint(kijun_length or period) + outputs = { + "tenkan_sen": tenkan, + "kijun_sen": kijun, + "senkou_span_a": (tenkan + kijun) / 2, + "senkou_span_b": midpoint(period), + "chikou_span": pl.col(close).shift(period - 1 - chikou_position), + } + return {name: _ready(value, period) for name, value in outputs.items()} + + +@feature( + name="finite_macd_signal", + category="momentum", + description="MACD signal formed from three finite normalized exponential windows", + formula="finite_ewma(finite_ewma(fast)-finite_ewma(slow),signal)", +) +def finite_macd_signal( + close: str, fast_period: int = 12, slow_period: int = 26, signal_period: int = 9 +) -> pl.Expr: + """Finite MACD signal; length-one means use their sole observation.""" + for name, value in ( + ("fast_period", fast_period), + ("slow_period", slow_period), + ("signal_period", signal_period), + ): + try: + _period(value) + except ValueError as exc: + raise ValueError(f"{name}: {exc}") from exc + if slow_period <= fast_period: + raise ValueError("slow_period must be greater than fast_period") + + def weights(width: int) -> npt.NDArray[np.float64]: + if width == 1: + return np.ones(1) + return _exponential_weights(width, 1 - 2 / (1 + width)) + + difference = _weighted_expression(close, weights(fast_period)) - _weighted_expression( + close, weights(slow_period) + ) + return _weighted_expression(difference, weights(signal_period)) + + +@feature( + name="finite_true_range", + category="volatility", + description="Two-price true range with NaN-omitting horizontal components", + input_type="HLC", + formula="nanmax(high-low, abs(high-prev_close), abs(low-prev_close))", +) +def finite_true_range(high: str, low: str, close: str) -> pl.Expr: + """One-bar true range after its previous price; no arbitrary larger-window reduction.""" + upper, lower, previous = _numeric(high), _numeric(low), _numeric(close).shift(1) + return _ready( + pl.max_horizontal(upper - lower, (upper - previous).abs(), (lower - previous).abs()), 2 + ) diff --git a/tests/compat/test_zipline_factors.py b/tests/compat/test_zipline_factors.py new file mode 100644 index 00000000..80debbd3 --- /dev/null +++ b/tests/compat/test_zipline_factors.py @@ -0,0 +1,308 @@ +"""Finite Pipeline definitions must not silently use recursive or sample methods.""" + +from __future__ import annotations + +import math + +import numpy as np +import polars as pl +import pytest +from polars.testing import assert_frame_equal + +from ml4t.engineer import compute_features +from ml4t.engineer.core.registry import get_registry + + +def prices(close: list[float | None], volume: list[float | None] | None = None) -> pl.DataFrame: + return pl.DataFrame( + { + "open": close, + "high": [None if x is None else x + 2 for x in close], + "low": [None if x is None else x / 2 for x in close], + "close": close, + "volume": volume if volume is not None else [10.0] * len(close), + }, + schema_overrides=dict.fromkeys(("open", "high", "low", "close", "volume"), pl.Float64), + ) + + +def calculate(frame: pl.DataFrame, name: str, **params: object) -> pl.DataFrame: + return compute_features( + frame, [{"name": name, "params": params, "output": "factor"}], calendar=None + ) + + +@pytest.mark.parametrize("lazy", [False, True]) +def test_cutler_uses_finite_changes_instead_of_wilder_recursion(lazy: bool) -> None: + frame = prices([10.0, 13.0, 12.0, 14.0, 13.0, 17.0, 16.0]) + result = compute_features( + frame.lazy() if lazy else frame, + [{"name": "rsi_cutler", "params": {"period": 3}}], + calendar=None, + ) + assert isinstance(result, pl.LazyFrame if lazy else pl.DataFrame) + actual = result.collect() if lazy else result + np.testing.assert_allclose( + actual["rsi_cutler"].to_numpy(), + [np.nan, np.nan, np.nan, 100 * 5 / 6, 50, 100 * 6 / 7, 100 * 4 / 6], + rtol=1e-14, + equal_nan=True, + ) + native = calculate(frame, "rsi", period=3)["factor"].to_numpy() + assert not np.allclose(native[4:], actual["rsi_cutler"].to_numpy()[4:]) + + +@pytest.mark.parametrize("missing", [None, float("nan")]) +def test_cutler_omits_missing_changes_without_filling_quotes(missing: float | None) -> None: + frame = prices([1.0, 2.0, missing, 4.0, 5.0, 5.0, 3.0]) + result = calculate(frame, "rsi_cutler", period=3) + np.testing.assert_allclose( + result["factor"].to_numpy(), + [np.nan, np.nan, np.nan, 100, 100, 100, 100 / 3], + equal_nan=True, + ) + assert_frame_equal(result.select(frame.columns), frame) + + +@pytest.mark.parametrize( + ("name", "close", "volume", "params", "expected"), + [ + ("finite_mean", [1.0, np.nan, 5.0], None, {}, 3.0), + ("finite_weighted_mean", [1.0, np.nan, 5.0], [1.0, 2.0, 3.0], {}, 16 / 6), + ("finite_weighted_mean", [1.0, 2.0, 5.0], [0.0, 0.0, 0.0], {}, np.nan), + ("average_dollar_volume", [1.0, np.nan, 5.0], [1.0, 2.0, 3.0], {}, 16 / 3), + ("average_dollar_volume", [1.0, np.nan, 5.0], [0.0, 0.0, 0.0], {}, 0.0), + ("linear_weighted_mean", [1.0, np.nan, 5.0], None, {}, 16 / 6), + ("finite_ewma", [1.0, 3.0, 9.0], None, {"decay_rate": 0.5}, 43 / 7), + ("finite_ewma", [1.0, 3.0, 9.0], None, {"decay_rate": 1.0}, 13 / 3), + ("finite_ewma", [1.0, np.nan, 9.0], None, {"decay_rate": 0.5}, np.nan), + ("finite_ewm_std", [1.0, 3.0, 9.0], None, {"decay_rate": 1.0}, math.sqrt(52 / 3)), + ("finite_ewm_std", [1.0, 3.0, 9.0], None, {"decay_rate": 0.5}, math.sqrt(138 / 7)), + ("finite_ewm_std", [1.0, np.nan, 9.0], None, {"decay_rate": 0.5}, np.nan), + ( + "annualized_volatility", + [1.0, 3.0, 9.0], + None, + {"annualization_factor": 9.0}, + math.sqrt(104), + ), + ("finite_max_drawdown", [100.0, 50.0, 60.0], None, {}, 1.0), + ("finite_max_drawdown", [100.0, 60.0, 50.0], None, {}, 1.0), + ], +) +def test_finite_methods_preserve_nan_denominators_weights_and_scale( + name: str, + close: list[float], + volume: list[float] | None, + params: dict[str, object], + expected: float, +) -> None: + frame = prices(close, volume) + if name == "annualized_volatility": + frame = frame.with_columns(pl.col("close").alias("returns")) + actual = calculate(frame, name, period=3, **params)["factor"].to_numpy() + np.testing.assert_allclose(actual, [np.nan, np.nan, expected], equal_nan=True, rtol=1e-14) + + +@pytest.mark.parametrize("name", ["finite_ewma", "finite_ewm_std"]) +def test_finite_weights_do_not_overflow_or_underflow_with_legacy_common_power(name: str) -> None: + frame = prices([4.0, 4.0, 4.0, 4.0]) + result = calculate(frame, name, period=4, decay_rate=1e-100)["factor"][-1] + if name == "finite_ewma": + assert result == 4.0 + else: + assert math.isnan(result) + + +@pytest.mark.parametrize( + "name", ["finite_mean", "linear_weighted_mean", "finite_ewma", "rsi_cutler"] +) +def test_finite_window_never_crosses_asset_boundaries_and_lazy_matches_eager(name: str) -> None: + first = prices([10.0, 13.0, 12.0, 14.0]).with_columns(pl.lit("A").alias("asset")) + second = prices([100.0, 130.0, 120.0, 140.0]).with_columns(pl.lit("B").alias("asset")) + frame = pl.concat([first, second]).sort("close") + spec = [{"name": name, "params": {"period": 3}}] + result = compute_features(frame, spec, group_col="asset", calendar=None) + lazy = compute_features(frame.lazy(), spec, group_col="asset", calendar=None).collect() + assert_frame_equal(result, lazy) + for asset in ("A", "B"): + expected = compute_features(frame.filter(pl.col("asset") == asset), spec, calendar=None) + assert_frame_equal(result.filter(pl.col("asset") == asset), expected) + + +@pytest.mark.parametrize( + "name", + [ + "finite_mean", + "finite_weighted_mean", + "average_dollar_volume", + "linear_weighted_mean", + "finite_ewma", + "finite_ewm_std", + "annualized_volatility", + "finite_max_drawdown", + "rsi_cutler", + ], +) +@pytest.mark.parametrize("period", [0, -1, True, 1.5]) +def test_invalid_windows_fail_without_modifying_input(name: str, period: object) -> None: + frame = prices([1.0, 2.0, 3.0]).with_columns(pl.lit(0.1).alias("returns")) + before = frame.clone() + with pytest.raises((TypeError, ValueError), match="period"): + calculate(frame, name, period=period) + assert_frame_equal(frame, before) + + +@pytest.mark.parametrize("decay", [0.0, -0.1, 1.1, float("nan"), float("inf"), True]) +def test_invalid_decay_fails_before_output(decay: float) -> None: + with pytest.raises((TypeError, ValueError), match="decay_rate"): + calculate(prices([1.0, 2.0, 3.0]), "finite_ewma", period=3, decay_rate=decay) + + +def test_cutler_registry_metadata_declares_price_count_and_keeps_native_defaults() -> None: + cutler = get_registry().get("rsi_cutler") + native = get_registry().get("rsi") + assert cutler is not None and native is not None + assert cutler.lookback(period=5) == 5 + assert cutler.value_range == (0.0, 100.0) + assert cutler.normalized and not cutler.ta_lib_compatible + assert native.parameters["period"] == 14 + assert native.ta_lib_compatible + + +def test_finite_bands_use_population_std_and_nan_omission() -> None: + actual = calculate(prices([1.0, 3.0, 9.0]), "finite_bollinger_bands", period=3, k=2) + middle = 13 / 3 + distance = 2 * math.sqrt(104 / 9) + assert actual["factor_lower"][-1] == pytest.approx(middle - distance) + assert actual["factor_middle"][-1] == pytest.approx(middle) + assert actual["factor_upper"][-1] == pytest.approx(middle + distance) + missing = calculate(prices([1.0, np.nan, 9.0]), "finite_bollinger_bands", period=3, k=1) + assert missing.select("factor_lower", "factor_middle", "factor_upper").row(-1) == (1, 5, 9) + + +def test_aroon_first_extreme_tie_is_not_native_latest_tie() -> None: + frame = prices([3.5, 2.0, 4.0]).with_columns( + pl.Series("high", [4.0, 5.0, 5.0]), pl.Series("low", [3.0, 1.0, 1.0]) + ) + actual = calculate(frame, "finite_aroon", period=3) + assert actual["factor_down"][-1] == 50 + assert actual["factor_up"][-1] == 50 + assert ( + actual["factor_up"].head(2).is_null().all() + or np.isnan(actual["factor_up"].head(2).to_numpy()).all() + ) + + +def test_fast_stochastic_has_no_smoothing_or_zero_range_replacement() -> None: + frame = prices([3.5, 2.0, 4.0]).with_columns( + pl.Series("high", [4.0, 5.0, 5.0]), pl.Series("low", [3.0, 1.0, 1.0]) + ) + assert calculate(frame, "fast_stochastic", period=3)["factor"][-1] == 75 + flat = pl.DataFrame({name: [3.0] * 3 for name in ("open", "high", "low", "close", "volume")}) + assert math.isnan(calculate(flat, "fast_stochastic", period=3)["factor"][-1]) + + +def test_ichimoku_fields_use_unshifted_spans_and_position_within_window() -> None: + frame = prices([10.0, 14.0, 11.0, 20.0, 16.0, 14.0]).with_columns( + pl.Series("high", [12.0, 18.0, 14.0, 22.0, 20.0, 16.0]), + pl.Series("low", [8.0, 10.0, 9.0, 16.0, 13.0, 12.0]), + ) + actual = calculate( + frame, "finite_ichimoku", period=6, tenkan_length=2, kijun_length=3, chikou_position=2 + ) + assert actual.select(pl.col("^factor_.*$")).row(-1) == (16, 17, 16.5, 15, 11) + assert all( + actual[c].head(5).null_count() == 5 for c in actual.columns if c.startswith("factor_") + ) + + +def test_finite_macd_signal_uses_normalized_windows_not_recursive_ema() -> None: + actual = calculate( + prices([1.0, 2.0, 4.0, 8.0, 16.0]), + "finite_macd_signal", + fast_period=2, + slow_period=3, + signal_period=2, + ) + np.testing.assert_allclose( + actual["factor"].to_numpy(), [np.nan] * 3 + [0.875, 1.75], equal_nan=True + ) + + +def test_single_observation_macd_means_are_defined_instead_of_zero_weight_division() -> None: + actual = calculate( + prices([1.0, 3.0, 9.0]), + "finite_macd_signal", + fast_period=1, + slow_period=2, + signal_period=1, + ) + np.testing.assert_allclose(actual["factor"].to_numpy(), [np.nan, 0.5, 1.5], equal_nan=True) + + +def test_finite_true_range_uses_two_price_rows_and_nanmax_components() -> None: + frame = prices([10.0, 12.0, 13.0]).with_columns( + pl.Series("high", [12.0, 15.0, 14.0]), pl.Series("low", [8.0, 11.0, 12.0]) + ) + np.testing.assert_allclose( + calculate(frame, "finite_true_range")["factor"].to_numpy(), [np.nan, 5, 2], equal_nan=True + ) + + +@pytest.mark.parametrize( + ("name", "params"), + [ + ("finite_aroon", {"period": 1}), + ("finite_bollinger_bands", {"k": float("nan")}), + ("finite_ichimoku", {"period": 6, "tenkan_length": 7}), + ("finite_ichimoku", {"period": 6, "chikou_position": 6}), + ("finite_macd_signal", {"fast_period": 3, "slow_period": 3}), + ("finite_macd_signal", {"signal_period": 0}), + ], +) +def test_technical_options_reject_undefined_windows(name: str, params: dict[str, object]) -> None: + with pytest.raises(ValueError): + calculate(prices([10.0] * 60), name, **params) + + +@pytest.mark.parametrize("dtype,base", [(pl.Int64, 2**53), (pl.UInt64, 2**63), (pl.Int128, 2**126)]) +def test_cutler_preserves_integer_tick_differences_before_float_conversion(dtype, base) -> None: + frame = pl.DataFrame({"close": pl.Series([base, base + 3, base + 2, base + 4], dtype=dtype)}) + from ml4t.engineer.features.rolling import rsi_cutler + + assert frame.select(rsi_cutler("close", period=3)).item(-1, 0) == pytest.approx(100 * 5 / 6) + + +@pytest.mark.parametrize( + "name", + [ + "finite_mean", + "finite_weighted_mean", + "average_dollar_volume", + "linear_weighted_mean", + "finite_ewma", + "finite_ewm_std", + "rsi_cutler", + "finite_max_drawdown", + ], +) +def test_all_missing_window_does_not_create_an_observed_quote(name: str) -> None: + actual = calculate(prices([None] * 4), name, period=3)["factor"].to_numpy() + expected = ( + 0 + if name in {"average_dollar_volume", "linear_weighted_mean", "finite_weighted_mean"} + else np.nan + ) + np.testing.assert_allclose(actual[-1], expected, equal_nan=True) + + +@pytest.mark.parametrize("factor", [0.0, -1.0, float("nan"), float("inf"), True]) +def test_population_volatility_requires_explicit_positive_scale(factor: float) -> None: + with pytest.raises(ValueError, match="annualization_factor"): + calculate( + prices([10.0] * 4).with_columns(pl.lit(0.1).alias("returns")), + "annualized_volatility", + period=3, + annualization_factor=factor, + ) diff --git a/tests/core/test_registry.py b/tests/core/test_registry.py index 243de103..7d3aad4d 100644 --- a/tests/core/test_registry.py +++ b/tests/core/test_registry.py @@ -555,10 +555,10 @@ def test_registry_with_real_features(): # Test category queries work momentum = registry.list_by_category("momentum") - assert len(momentum) == 31 + assert len(momentum) == 35 volatility = registry.list_by_category("volatility") - assert len(volatility) == 15 + assert len(volatility) == 18 # Test stationarity queries stationary = registry.list_normalized() diff --git a/tests/test_all_features_integration.py b/tests/test_all_features_integration.py index d17b9497..d8cfb867 100644 --- a/tests/test_all_features_integration.py +++ b/tests/test_all_features_integration.py @@ -46,7 +46,7 @@ def sample_ohlcv_data(): def test_all_features_registered(): """Verify that features are actually registered.""" registry = get_registry() - assert len(registry) == 120, f"Expected 120 features, got {len(registry)}" + assert len(registry) == 135, f"Expected 135 features, got {len(registry)}" def test_all_features_compute_individually(sample_ohlcv_data): From ee91e0af75a541bd0d5405a5be4588f0420f78cb Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Tue, 20 Oct 2026 18:56:38 -0400 Subject: [PATCH 05/19] feat(features): add aligned rolling relationships and known event counts Keep missing pairs on their original clock, expose actual regression fields through the optional stats extra, and separate civil weekdays from publication instants. Reproduce and repair integer-tick and overflow errors in finite variance and range methods without changing existing native defaults. Refs #108. --- docs/user-guide/features.md | 68 ++++++- pyproject.toml | 1 + src/ml4t/engineer/features/AGENTS.md | 2 + src/ml4t/engineer/features/__init__.py | 4 + src/ml4t/engineer/features/events.py | 55 ++++++ src/ml4t/engineer/features/paired.py | 176 ++++++++++++++++++ src/ml4t/engineer/features/rolling.py | 174 ++++++++++++++--- src/ml4t/engineer/utils/dependencies.py | 8 + tests/compat/test_zipline_events.py | 101 ++++++++++ tests/compat/test_zipline_factors.py | 49 +++++ .../test_zipline_relationship_features.py | 158 ++++++++++++++++ uv.lock | 2 + 12 files changed, 771 insertions(+), 27 deletions(-) create mode 100644 src/ml4t/engineer/features/events.py create mode 100644 src/ml4t/engineer/features/paired.py create mode 100644 tests/compat/test_zipline_events.py create mode 100644 tests/compat/test_zipline_relationship_features.py diff --git a/docs/user-guide/features.md b/docs/user-guide/features.md index 886a0364..b6ea1ed2 100644 --- a/docs/user-guide/features.md +++ b/docs/user-guide/features.md @@ -150,8 +150,9 @@ defaults. These additions do not provide a Pipeline graph interpreter. Except for Cutler RSI and the composed MACD signal, `period` counts input rows and the first structurally usable row is `period - 1`. Missing observations still occupy their -original row positions. All methods return Float64 expressions and retain every input -row. Multi-output features use the usual `_` column names. Ichimoku +original row positions. Numeric results are Float64 and retain every input row; +Ichimoku's selected `chikou_span` price retains its original dtype. Multi-output +features use the usual `_` column names. Ichimoku propagates missing high/low values within each field's subwindow; all fields require the declared full window. Its zero tenkan/kijun lengths select that full window, matching the legacy `[-0:]` slice, while `chikou_position` must be in `[0, period)`. @@ -182,7 +183,68 @@ call counts the supplied observation rows without checking schedules. With a nam calendar, declare the complete grid before computing any feature. The finite methods do not infer frequency, select source revisions, fill gaps or make future information eligible. They currently support observation/expected-bar windows; do not reinterpret -their kernels as session or elapsed windows. +their kernels as session or elapsed windows. Variance methods subtract an original +window-local typed price anchor before floating conversion, so adjacent large integer ticks do +not collapse to zero variance. Output remains Float64. + +### Paired relationships and supplied event dates + +`features.paired.rolling_pearson`, `rolling_spearman`, `rolling_beta` and +`rolling_regression` take explicit dependent/independent columns or expressions. +Align their actual time/entity keys before computing and apply `.over(entity_col)` +for asset-isolated windows. `period` counts original rows. `allowed_missing` is a +count, not a percentage; a missing pair is removed from both sides without +compressing the timeline. At least two valid pairs remain necessary. Spearman +recomputes average tie ranks inside each window. Pearson/beta/regression propagate +undefined nonmissing infinities. Constant independent regression windows produce +NaN fields instead of aborting the whole asset panel. + +Regression returns a Struct with `alpha`, `beta`, `r_value`, `p_value`, `stderr`. +Its inferential fields require the optional `stats` extra (`uv add +'ml4t-engineer[stats]'`). The other paired methods and base imports do not need SciPy. +This is predictive rolling regression on supplied observations, not trend-on-time +regression or portfolio attribution. + + +```python +import math +import polars as pl +from ml4t.engineer.features.paired import rolling_regression + +paired = pl.DataFrame({"market": [1.0, 2.0, 3.0, 4.0], "feature": [1.0, 2.0, 2.0, 4.0]}) +result = paired.lazy().select( + rolling_regression("feature", "market", period=4).alias("regression") +).unnest("regression").collect() +assert math.isclose(result["beta"][3], 0.9, rel_tol=1e-14) +assert math.isclose(result["stderr"][3], math.sqrt(0.07), rel_tol=1e-14) +assert math.isclose(result["p_value"][3], 1 - 9 / math.sqrt(95), rel_tol=1e-14) +``` + +`features.events.event_business_days` counts Monday through Friday, including +exchange holidays, with the end date excluded. Supply separate event/reference +Date labels and timezone-aware publication/decision instants. Unknown publication +or publication after the decision yields null. This method neither selects an +event provider nor infers exchange sessions. `direction="until"` permits a known +future event; `direction="since"` uses the same supplied date with reversed endpoints. + + +```python +from datetime import date, datetime, timezone +import polars as pl +from ml4t.engineer.features.events import event_business_days + +events = pl.DataFrame({ + "event_date": [date(2025, 7, 4)], + "session": [date(2025, 7, 7)], + "known_at": [datetime(2025, 7, 1, tzinfo=timezone.utc)], + "decision": [datetime(2025, 7, 7, 14, tzinfo=timezone.utc)], +}) +result = events.select(event_business_days( + "event_date", reference_date_col="session", known_at_col="known_at", + decision_time_col="decision", direction="since", +).alias("weekdays")) +assert result["weekdays"][0] == 1.0 # July 4 is a weekday, despite the NYSE closure. +``` ### Calendar and bar grid diff --git a/pyproject.toml b/pyproject.toml index b593d2e4..f5baa985 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,6 +112,7 @@ calendars = [ # Statistical diagnostics stats = [ + "scipy>=1.14.0", "statsmodels>=0.14.0", ] diff --git a/src/ml4t/engineer/features/AGENTS.md b/src/ml4t/engineer/features/AGENTS.md index c2df0f09..5209d195 100644 --- a/src/ml4t/engineer/features/AGENTS.md +++ b/src/ml4t/engineer/features/AGENTS.md @@ -26,6 +26,8 @@ These are not registry entries but remain important: | File | Purpose | |------|---------| | `cross_asset.py` | multi-asset and panel features such as rolling correlation and beta | +| `paired.py` | full-window paired correlation/beta/regression with explicit missing-count limits | +| `events.py` | supplied known event weekday counts, separate from exchange-session counts | | `fdiff.py` | fractional differencing utilities such as `ffdiff()` and `find_optimal_d()` | | `composite.py` | composite indicators and z-score combinations | diff --git a/src/ml4t/engineer/features/__init__.py b/src/ml4t/engineer/features/__init__.py index 9506a7fc..94b6876f 100644 --- a/src/ml4t/engineer/features/__init__.py +++ b/src/ml4t/engineer/features/__init__.py @@ -29,11 +29,13 @@ from ml4t.engineer.features import ( composite, cross_asset, + events, fdiff, math, # noqa: F401 microstructure, # noqa: F401 ml, # noqa: F401 momentum, # noqa: F401 + paired, price_transform, # noqa: F401 regime, risk, @@ -48,6 +50,8 @@ # Advanced feature modules (single files) "composite", "cross_asset", + "events", + "paired", "fdiff", "regime", "risk", diff --git a/src/ml4t/engineer/features/events.py b/src/ml4t/engineer/features/events.py new file mode 100644 index 00000000..a12c3c12 --- /dev/null +++ b/src/ml4t/engineer/features/events.py @@ -0,0 +1,55 @@ +"""Predictive weekday counts from supplied event dates and publication instants.""" + +from typing import Literal + +import polars as pl + + +def _date(column: str) -> pl.Expr: + def check(value: pl.Expr, dtype: pl.DataType) -> pl.Expr: + if dtype != pl.Date: + raise ValueError( + f"{column!r} must be a civil/session Date, not an inferred instant date" + ) + return value + + return pl.col(column).pipe_with_dtype(check) + + +def _instant_ticks(column: str) -> pl.Expr: + def check(value: pl.Expr, dtype: pl.DataType) -> pl.Expr: + if not isinstance(dtype, pl.Datetime) or dtype.time_zone is None: + raise ValueError( + f"{column!r} must declare a timezone-aware publication/decision instant" + ) + scale = {"ms": 1_000_000, "us": 1_000, "ns": 1}[dtype.time_unit] + # Compare physical instants without truncating ns or overflowing a + # Datetime(ns) conversion for otherwise valid wider-range us/ms dates. + return value.cast(pl.Int64).cast(pl.Int128) * scale + + return pl.col(column).pipe_with_dtype(check) + + +def event_business_days( + event_date_col: str, + *, + reference_date_col: str, + known_at_col: str, + decision_time_col: str, + direction: Literal["since", "until"] = "since", +) -> pl.Expr: + """Weekdays since/until a supplied event known by the decision instant. + + Event/reference columns are civil or session Date labels. Publication and + decision columns are separate timezone-aware instants. Monday through Friday + count, including exchange holidays; the end date is excluded. No exchange + calendar, date provider, latest-event lookup or observation-spacing inference + is used. Missing or not-yet-known event dates yield null. Signed negative + counts remain possible if the caller supplies a date on the opposite side. + """ + if direction not in {"since", "until"}: + raise ValueError("direction must be 'since' or 'until'") + event, reference = _date(event_date_col), _date(reference_date_col) + start, end = (event, reference) if direction == "since" else (reference, event) + eligible = _instant_ticks(known_at_col) <= _instant_ticks(decision_time_col) + return pl.when(eligible).then(pl.business_day_count(start, end).cast(pl.Float64)) diff --git a/src/ml4t/engineer/features/paired.py b/src/ml4t/engineer/features/paired.py new file mode 100644 index 00000000..2b3209c1 --- /dev/null +++ b/src/ml4t/engineer/features/paired.py @@ -0,0 +1,176 @@ +"""Finite paired-observation correlations and regression features. + +Inputs must already be aligned by their actual keys. Apply expressions within +asset groups. Missing pairs occupy their original window positions and are +removed from both sides together, never by compressing the input timeline. +""" + +from collections.abc import Sequence + +import numpy as np +import numpy.typing as npt +import polars as pl + +from ml4t.engineer._numba import jit +from ml4t.engineer.features.cross_asset import beta_to_market, rolling_correlation +from ml4t.engineer.features.rolling import _period, _ready +from ml4t.engineer.utils.dependencies import require_dependency + + +def _validate(period: int, allowed_missing: int) -> None: + _period(period, minimum=2) + if ( + isinstance(allowed_missing, bool) + or not isinstance(allowed_missing, int) + or not 0 <= allowed_missing <= period + ): + raise ValueError("allowed_missing must be an integer in [0, period]") + + +def _expressions( + dependent: str | pl.Expr, independent: str | pl.Expr +) -> tuple[pl.Expr, pl.Expr, pl.Expr]: + y = pl.col(dependent) if isinstance(dependent, str) else dependent + x = pl.col(independent) if isinstance(independent, str) else independent + y, x = y.cast(pl.Float64), x.cast(pl.Float64) + missing = y.is_null() | x.is_null() | y.is_nan() | x.is_nan() + return pl.when(~missing).then(y), pl.when(~missing).then(x), y.is_infinite() | x.is_infinite() + + +def rolling_pearson( + dependent: str | pl.Expr, + independent: str | pl.Expr, + period: int = 20, + *, + allowed_missing: int = 0, +) -> pl.Expr: + """Pearson correlation over a full row window with a paired missing-count limit.""" + _validate(period, allowed_missing) + y, x, infinite = _expressions(dependent, independent) + result = rolling_correlation(y, x, window=period, min_periods=max(2, period - allowed_missing)) + return _ready( + pl.when(infinite.fill_null(False).rolling_max(period)).then(float("nan")).otherwise(result), + period, + ) + + +def rolling_beta( + dependent: str | pl.Expr, + independent: str | pl.Expr, + period: int = 60, + *, + allowed_missing: int = 0, +) -> pl.Expr: + """Slope Cov(independent,dependent)/Var(independent) using identical paired rows.""" + _validate(period, allowed_missing) + y, x, infinite = _expressions(dependent, independent) + result = beta_to_market(y, x, window=period, min_periods=max(2, period - allowed_missing)) + return _ready( + pl.when(infinite.fill_null(False).rolling_max(period)).then(float("nan")).otherwise(result), + period, + ) + + +@jit(nopython=True, cache=True) # type: ignore[misc] +def _ranks(values: npt.NDArray[np.float64]) -> npt.NDArray[np.float64]: + order = np.argsort(values) + result = np.empty(len(values)) + start = 0 + while start < len(order): + stop = start + 1 + while stop < len(order) and values[order[stop]] == values[order[start]]: + stop += 1 + rank = (start + 1 + stop) / 2 + for j in range(start, stop): + result[order[j]] = rank + start = stop + return result + + +@jit(nopython=True, cache=True) # type: ignore[misc] +def _spearman_windows( + y: npt.NDArray[np.float64], x: npt.NDArray[np.float64], period: int, allowed_missing: int +) -> npt.NDArray[np.float64]: + result = np.full(len(y), np.nan) + for end in range(period - 1, len(y)): + yy, xx = y[end - period + 1 : end + 1], x[end - period + 1 : end + 1] + valid = ~np.isnan(yy) & ~np.isnan(xx) + count = valid.sum() + if count < 2 or period - count > allowed_missing: + continue + yr, xr = _ranks(yy[valid]), _ranks(xx[valid]) + yr, xr = yr - yr.mean(), xr - xr.mean() + denominator = np.sqrt((yr * yr).sum() * (xr * xr).sum()) + if denominator > 0: + result[end] = (yr * xr).sum() / denominator + return result + + +def rolling_spearman( + dependent: str | pl.Expr, + independent: str | pl.Expr, + period: int = 20, + *, + allowed_missing: int = 0, +) -> pl.Expr: + """Spearman correlation with average ties recomputed inside each paired window.""" + _validate(period, allowed_missing) + y, x, _ = _expressions(dependent, independent) + return pl.map_batches( + [y, x], + lambda values: pl.Series( + _spearman_windows(values[0].to_numpy(), values[1].to_numpy(), period, allowed_missing) + ), + return_dtype=pl.Float64, + ) + + +_REGRESSION_FIELDS = ("alpha", "beta", "r_value", "p_value", "stderr") +_REGRESSION_DTYPE = pl.Struct(dict.fromkeys(_REGRESSION_FIELDS, pl.Float64)) + + +def rolling_regression( + dependent: str | pl.Expr, + independent: str | pl.Expr, + period: int = 20, + *, + allowed_missing: int = 0, +) -> pl.Expr: + """OLS of dependent on independent, returning a five-field Struct expression. + + Requires the stats extra for SciPy's inferential outputs. Constant independent + windows are undefined and yield NaN fields instead of failing the whole panel. + Nonfinite nonmissing inputs propagate undefined fields. No trend-on-time or + portfolio attribution is substituted for the supplied independent observations. + """ + _validate(period, allowed_missing) + require_dependency("scipy", "rolling regression inferential fields") + from scipy.stats import linregress + + y, x, _ = _expressions(dependent, independent) + + def compute(values: Sequence[pl.Series]) -> pl.Series: + yy, xx = values[0].to_numpy(), values[1].to_numpy() + output = np.full((len(yy), 5), np.nan) + for end in range(period - 1, len(yy)): + ys, xs = yy[end - period + 1 : end + 1], xx[end - period + 1 : end + 1] + valid = ~np.isnan(ys) & ~np.isnan(xs) + count = np.count_nonzero(valid) + if count < 2 or period - count > allowed_missing: + continue + ys, xs = ys[valid], xs[valid] + if not np.isfinite(ys).all() or not np.isfinite(xs).all() or np.ptp(xs) == 0: + continue + result = linregress(xs, ys) + output[end] = ( + result.intercept, + result.slope, + result.rvalue, + result.pvalue, + result.stderr, + ) + return pl.DataFrame( + {name: output[:, i] for i, name in enumerate(_REGRESSION_FIELDS)} + ).to_struct() + + return pl.map_batches([y, x], compute, return_dtype=_REGRESSION_DTYPE) diff --git a/src/ml4t/engineer/features/rolling.py b/src/ml4t/engineer/features/rolling.py index 5b588819..e7538b48 100644 --- a/src/ml4t/engineer/features/rolling.py +++ b/src/ml4t/engineer/features/rolling.py @@ -6,6 +6,7 @@ """ import math +from typing import Any import numpy as np import numpy.typing as npt @@ -29,6 +30,38 @@ def _numeric(column: str) -> pl.Expr: return pl.col(column).cast(pl.Float64).fill_nan(None) +def _difference(price: pl.Expr, reference: pl.Expr, dtype: pl.DataType) -> pl.Expr: + if dtype.is_integer(): + price, reference = price.cast(pl.Int128), reference.cast(pl.Int128) + if dtype == pl.Int128 or isinstance(dtype, pl.Decimal): + same_sign = ((price >= 0) & (reference >= 0)) | ((price <= 0) & (reference <= 0)) + safe_price = pl.when(same_sign).then(price).otherwise(0) + safe_reference = pl.when(same_sign).then(reference).otherwise(0) + exact = (safe_price - safe_reference).cast(pl.Float64) + return ( + pl.when(same_sign) + .then(exact) + .otherwise(price.cast(pl.Float64) - reference.cast(pl.Float64)) + ) + if dtype.is_integer(): + return (price - reference).cast(pl.Float64) + return price.cast(pl.Float64) - reference.cast(pl.Float64) + + +def _paired_difference(left: pl.Expr, right: pl.Expr) -> pl.Expr: + def left_type(value: pl.Expr, dtype: pl.DataType) -> pl.Expr: + def right_type(reference: pl.Expr, reference_dtype: pl.DataType) -> pl.Expr: + if dtype.is_integer() and reference_dtype.is_integer(): + return _difference(value, reference, pl.Int128()) + if isinstance(dtype, pl.Decimal) and isinstance(reference_dtype, pl.Decimal): + return _difference(value, reference, dtype) + return value.cast(pl.Float64) - reference.cast(pl.Float64) + + return right.pipe_with_dtype(right_type) + + return left.pipe_with_dtype(left_type) + + @feature( name="rsi_cutler", category="momentum", @@ -47,9 +80,7 @@ def rsi_cutler(close: str, period: int = 14) -> pl.Expr: _period(period) def change(price: pl.Expr, dtype: pl.DataType) -> pl.Expr: - if dtype.is_integer() and dtype != pl.Int128: - return price.cast(pl.Int128).diff().cast(pl.Float64) - return price.diff().cast(pl.Float64) + return _difference(price, price.shift(1), dtype) delta = pl.col(close).pipe_with_dtype(change).fill_nan(None) gain = delta.clip(lower_bound=0).rolling_mean(period, min_samples=1) @@ -106,13 +137,11 @@ def average_dollar_volume(close: str, volume: str, period: int = 20) -> pl.Expr: def _weighted_windows( values: npt.NDArray[np.float64], weights: npt.NDArray[np.float64], - variance: bool, omit: bool, ) -> npt.NDArray[np.float64]: width = len(weights) result = np.full(len(values), np.nan) weight_sum = weights.sum() - correction_denominator = weight_sum * weight_sum - (weights * weights).sum() for end in range(width - 1, len(values)): total = 0.0 missing = False @@ -121,21 +150,106 @@ def _weighted_windows( if np.isnan(value): missing = True else: - total += weights[j] * value + total += (weights[j] / weight_sum) * value if missing and not omit: continue - mean = total / weight_sum - if not variance: - result[end] = mean - elif correction_denominator > 0: - centered = 0.0 - for j in range(width): - delta = values[end - width + 1 + j] - mean - centered += weights[j] * delta * delta - result[end] = np.sqrt(centered * weight_sum / correction_denominator) + result[end] = total return result +def _variance_windows( + values: Any, + missing: npt.NDArray[np.bool_], + weights: npt.NDArray[np.float64], + omit: bool, + bias_corrected: bool, +) -> npt.NDArray[np.float64]: + width = len(weights) + result = np.full(len(values), np.nan) + for end in range(width - 1, len(values)): + start = end - width + 1 + first = -1 + weight_sum, weight_squares = 0.0, 0.0 + positive, negative, has_missing = False, False, False + for j in range(width): + if missing[start + j]: + has_missing = True + continue + if first < 0: + first = start + j + value = values[start + j] + positive = positive or value > 0 + negative = negative or value < 0 + weight_sum += weights[j] + weight_squares += weights[j] * weights[j] + if first < 0 or (has_missing and not omit): + continue + denominator = weight_sum * weight_sum - weight_squares + if bias_corrected and denominator <= 0: + continue + anchor = values[first] + deltas = np.zeros(width) + scale = 0.0 + for j in range(width): + if missing[start + j]: + continue + value = values[start + j] + if positive and negative: + delta = float(value) + elif value >= anchor: + delta = float(value - anchor) + else: + delta = -float(anchor - value) + deltas[j] = delta + scale = max(scale, abs(delta)) + if not np.isfinite(scale): + continue + if scale == 0: + result[end] = 0.0 + continue + mean = 0.0 + for j in range(width): + if not missing[start + j]: + mean += (weights[j] / weight_sum) * (deltas[j] / scale) + variance = 0.0 + for j in range(width): + if not missing[start + j]: + residual = deltas[j] / scale - mean + variance += (weights[j] / weight_sum) * residual * residual + correction = weight_sum * weight_sum / denominator if bias_corrected else 1.0 + result[end] = scale * np.sqrt(variance * correction) + return result + + +_variance_numba = jit(nopython=True, cache=True)(_variance_windows) + + +def _variance_expression( + close: str | pl.Expr, + weights: npt.NDArray[np.float64], + *, + omit: bool = False, + bias_corrected: bool = True, +) -> pl.Expr: + expression = pl.col(close) if isinstance(close, str) else close + + def compute(values: pl.Series) -> pl.Series: + missing = values.is_null().to_numpy() + if values.dtype in (pl.Float32, pl.Float64): + missing = missing | values.is_nan().fill_null(False).to_numpy() + if values.dtype in (pl.Int128, pl.Null) or isinstance(values.dtype, pl.Decimal): + result = _variance_windows(values.to_list(), missing, weights, omit, bias_corrected) + else: + result = _variance_numba( + values.fill_null(0).to_numpy(), missing, weights, omit, bias_corrected + ) + return pl.Series(result) + + # Keep raw integer/Decimal values through window-local subtraction. A global + # float anchor loses later one-tick variation after a large price-level change. + return expression.map_batches(compute, return_dtype=pl.Float64) + + def _weighted_expression( close: str | pl.Expr, weights: npt.NDArray[np.float64], @@ -144,8 +258,10 @@ def _weighted_expression( omit: bool = False, ) -> pl.Expr: expression = pl.col(close) if isinstance(close, str) else close + if variance: + return _variance_expression(expression, weights) return expression.cast(pl.Float64).map_batches( - lambda values: pl.Series(_weighted_windows(values.to_numpy(), weights, variance, omit)), + lambda values: pl.Series(_weighted_windows(values.to_numpy(), weights, omit)), return_dtype=pl.Float64, ) @@ -217,8 +333,8 @@ def annualized_volatility( or annualization_factor <= 0 ): raise ValueError("annualization_factor must be positive and finite") - volatility = _numeric(returns).rolling_std(period, min_samples=1, ddof=0) - return _ready(volatility * math.sqrt(annualization_factor), period) + volatility = _variance_expression(returns, np.ones(period), omit=True, bias_corrected=False) + return volatility * math.sqrt(annualization_factor) @jit(nopython=True, cache=True) # type: ignore[misc] @@ -276,7 +392,7 @@ def finite_bollinger_bands(close: str, period: int = 20, k: float = 2.0) -> dict if isinstance(k, bool) or not isinstance(k, int | float) or not math.isfinite(k) or k < 0: raise ValueError("k must be finite and nonnegative") middle = finite_mean(close, period) - distance = _ready(_numeric(close).rolling_std(period, min_samples=1, ddof=0), period) * k + distance = _variance_expression(close, np.ones(period), omit=True, bias_corrected=False) * k return {"lower": middle - distance, "middle": middle, "upper": middle + distance} @@ -315,6 +431,8 @@ def finite_aroon(high: str, low: str, period: int = 25) -> dict[str, pl.Expr]: def position(column: str, maximum: bool) -> pl.Expr: return ( pl.col(column) + .fill_nan(None) + .rank("dense") .cast(pl.Float64) .map_batches( lambda values: pl.Series(_extreme_position(values.to_numpy(), period, maximum)), @@ -337,9 +455,11 @@ def position(column: str, maximum: bool) -> pl.Expr: def fast_stochastic(high: str, low: str, close: str, period: int = 14) -> pl.Expr: """Unsmoothed K; zero ranges stay undefined rather than being replaced by zero.""" _period(period) - upper = _numeric(high).rolling_max(period, min_samples=1) - lower = _numeric(low).rolling_min(period, min_samples=1) - return _ready(100 * (_numeric(close) - lower) / (upper - lower), period) + upper = pl.col(high).fill_nan(None).rolling_max(period, min_samples=1) + lower = pl.col(low).fill_nan(None).rolling_min(period, min_samples=1) + numerator = _paired_difference(pl.col(close).fill_nan(None), lower) + denominator = _paired_difference(upper, lower) + return _ready(100 * numerator / denominator, period) @feature( @@ -431,7 +551,13 @@ def weights(width: int) -> npt.NDArray[np.float64]: ) def finite_true_range(high: str, low: str, close: str) -> pl.Expr: """One-bar true range after its previous price; no arbitrary larger-window reduction.""" - upper, lower, previous = _numeric(high), _numeric(low), _numeric(close).shift(1) + upper, lower = pl.col(high).fill_nan(None), pl.col(low).fill_nan(None) + previous = pl.col(close).fill_nan(None).shift(1) return _ready( - pl.max_horizontal(upper - lower, (upper - previous).abs(), (lower - previous).abs()), 2 + pl.max_horizontal( + _paired_difference(upper, lower), + _paired_difference(upper, previous).abs(), + _paired_difference(lower, previous).abs(), + ), + 2, ) diff --git a/src/ml4t/engineer/utils/dependencies.py b/src/ml4t/engineer/utils/dependencies.py index 7389c44a..a0201a7f 100644 --- a/src/ml4t/engineer/utils/dependencies.py +++ b/src/ml4t/engineer/utils/dependencies.py @@ -115,6 +115,14 @@ def __init__(self): def _register_dependencies(self): """Register all known optional dependencies.""" + self._deps["scipy"] = DependencyInfo( + name="SciPy", + import_name="scipy", + install_cmd="uv add 'ml4t-engineer[stats]'", + purpose="Rolling paired regression inferential fields", + features=["features.paired.rolling_regression"], + ) + # ML Libraries self._deps["lightgbm"] = DependencyInfo( name="LightGBM", diff --git a/tests/compat/test_zipline_events.py b/tests/compat/test_zipline_events.py new file mode 100644 index 00000000..2fe070ae --- /dev/null +++ b/tests/compat/test_zipline_events.py @@ -0,0 +1,101 @@ +"""Known event weekday counts must not infer exchange closures or future knowledge.""" + +import importlib +from datetime import UTC, date, datetime + +import numpy as np +import polars as pl +import pytest + + +def expression(direction: str = "since") -> pl.Expr: + module = importlib.import_module("ml4t.engineer.features.events") + return module.event_business_days( + "event", + reference_date_col="session", + known_at_col="known", + decision_time_col="decision", + direction=direction, + ).alias("days") + + +@pytest.mark.parametrize("direction", ["since", "until"]) +@pytest.mark.parametrize("lazy", [False, True]) +def test_event_counts_use_weekdays_including_nyse_holidays(direction: str, lazy: bool) -> None: + frame = pl.DataFrame( + { + "event": [date(2025, 7, 4), date(2025, 7, 5), None, date(2025, 7, 7)], + "session": [date(2025, 7, 7)] * 4, + "known": [datetime(2025, 7, 1, tzinfo=UTC)] * 4, + "decision": [datetime(2025, 7, 7, 14, tzinfo=UTC)] * 4, + } + ) + output = (frame.lazy() if lazy else frame).select(expression(direction)) + if lazy: + output = output.collect() + events = np.array(["2025-07-04", "2025-07-05", "NaT", "2025-07-07"], dtype="datetime64[D]") + valid = ~np.isnat(events) + expected = np.full(4, np.nan) + reference = np.array("2025-07-07", dtype="datetime64[D]") + expected[valid] = ( + np.busday_count(events[valid], reference) + if direction == "since" + else np.busday_count(reference, events[valid]) + ) + np.testing.assert_allclose(output["days"].to_numpy(), expected, equal_nan=True) + assert abs(output["days"][0]) == 1 + + +def test_event_dates_are_ineligible_before_announcement_at_exact_ns_cutoff() -> None: + instant = 1751896800000000000 + frame = pl.DataFrame( + { + "event": [date(2025, 7, 10)] * 4, + "session": [date(2025, 7, 7)] * 4, + "known": pl.Series([instant - 1, instant, instant + 1, None], dtype=pl.Int64).cast( + pl.Datetime("ns", "UTC") + ), + "decision": pl.Series([instant] * 4).cast(pl.Datetime("ns", "UTC")), + } + ) + actual = frame.select(expression("until"))["days"].to_list() + assert actual == [3.0, 3.0, None, None] + + +def test_event_knowledge_comparison_preserves_timezone_and_out_of_ns_instants() -> None: + frame = pl.DataFrame( + { + "event": [date(2500, 1, 2)], + "session": [date(2500, 1, 5)], + "known": [datetime(2499, 12, 1, tzinfo=UTC)], + "decision": [datetime(2500, 1, 5, tzinfo=UTC)], + } + ) + original = frame.select(expression()) + changed = frame.with_columns(pl.col("known").dt.convert_time_zone("America/New_York")).select( + expression() + ) + assert original.equals(changed) + assert original["days"][0] == np.busday_count("2500-01-02", "2500-01-05") + + +@pytest.mark.parametrize( + "invalid", ["reference_datetime", "naive_knowledge", "naive_decision", "direction"] +) +def test_event_contract_rejects_ambiguous_time_roles(invalid: str) -> None: + frame = pl.DataFrame( + { + "event": [date(2025, 7, 4)], + "session": [date(2025, 7, 7)], + "known": [datetime(2025, 7, 1, tzinfo=UTC)], + "decision": [datetime(2025, 7, 7, tzinfo=UTC)], + } + ) + if invalid == "reference_datetime": + frame = frame.with_columns(pl.col("session").cast(pl.Datetime("us", "UTC"))) + elif invalid == "naive_knowledge": + frame = frame.with_columns(pl.col("known").dt.replace_time_zone(None)) + elif invalid == "naive_decision": + frame = frame.with_columns(pl.col("decision").dt.replace_time_zone(None)) + with pytest.raises(ValueError): + frame.select(expression("invalid" if invalid == "direction" else "since")) diff --git a/tests/compat/test_zipline_factors.py b/tests/compat/test_zipline_factors.py index 80debbd3..94b84b2f 100644 --- a/tests/compat/test_zipline_factors.py +++ b/tests/compat/test_zipline_factors.py @@ -306,3 +306,52 @@ def test_population_volatility_requires_explicit_positive_scale(factor: float) - period=3, annualization_factor=factor, ) + + +@pytest.mark.parametrize("dtype,base", [(pl.Int64, 2**53), (pl.UInt64, 2**63), (pl.Int128, 2**126)]) +def test_finite_weighted_std_preserves_tiny_integer_price_variation(dtype, base) -> None: + frame = pl.DataFrame({"close": pl.Series([base, base + 1, base + 2], dtype=dtype)}) + frame = frame.with_columns( + *(pl.col("close").alias(c) for c in ("open", "high", "low")), pl.lit(1.0).alias("volume") + ) + assert calculate(frame, "finite_ewm_std", period=3, decay_rate=1.0)["factor"][-1] == 1.0 + + +def test_local_variance_does_not_lose_ticks_after_an_earlier_price_level_change() -> None: + frame = pl.DataFrame({"close": pl.Series([1, 2**63, 2**63 + 1, 2**63 + 2], dtype=pl.UInt64)}) + from ml4t.engineer.features.rolling import finite_ewm_std + + assert frame.select(finite_ewm_std("close", period=3, decay_rate=1.0)).item(-1, 0) == 1.0 + + +def test_finite_weighted_moments_do_not_overflow_representable_float_results() -> None: + from ml4t.engineer.features.rolling import finite_ewm_std, finite_ewma + + constant = pl.DataFrame({"close": [1e308] * 3}) + assert constant.select(finite_ewma("close", period=3, decay_rate=1)).item( + -1, 0 + ) == pytest.approx(1e308) + spread = pl.DataFrame({"close": [-1e300, 1e300]}) + assert spread.select(finite_ewm_std("close", period=2, decay_rate=1)).item( + -1, 0 + ) == pytest.approx(math.sqrt(2) * 1e300) + + +@pytest.mark.parametrize("name", ["finite_true_range", "fast_stochastic", "finite_aroon"]) +def test_normalized_and_range_methods_do_not_collapse_integer_price_ticks(name: str) -> None: + base = 2**63 + frame = pl.DataFrame( + { + "close": pl.Series([base + 2, base + 3, base + 4], dtype=pl.UInt64), + "high": pl.Series([base + 3, base + 4, base + 5], dtype=pl.UInt64), + "low": pl.Series([base, base + 1, base + 2], dtype=pl.UInt64), + } + ).with_columns(pl.col("close").alias("open"), pl.lit(1.0).alias("volume")) + result = calculate(frame, name, **({} if name == "finite_true_range" else {"period": 3})) + if name == "finite_true_range": + np.testing.assert_allclose(result["factor"].to_numpy(), [np.nan, 3, 3], equal_nan=True) + elif name == "fast_stochastic": + assert result["factor"][-1] == 80 + else: + assert result["factor_up"][-1] == 100 + assert result["factor_down"][-1] == 0 diff --git a/tests/compat/test_zipline_relationship_features.py b/tests/compat/test_zipline_relationship_features.py new file mode 100644 index 00000000..bd0cfd10 --- /dev/null +++ b/tests/compat/test_zipline_relationship_features.py @@ -0,0 +1,158 @@ +"""Paired rolling factors must use aligned target rows, not time-trend regression.""" + +from __future__ import annotations + +import importlib + +import numpy as np +import polars as pl +import pytest +from polars.testing import assert_frame_equal +from scipy import stats + + +def paired(): + return importlib.import_module("ml4t.engineer.features.paired") + + +@pytest.mark.parametrize("method", ["pearson", "spearman", "beta"]) +@pytest.mark.parametrize("lazy", [False, True]) +def test_paired_methods_match_independent_complete_window_oracles(method: str, lazy: bool) -> None: + frame = pl.DataFrame({"x": [1.0, 3.0, 2.0, 4.0, 6.0], "y": [2.0, 8.0, 7.0, 9.0, 10.0]}) + expression = getattr(paired(), f"rolling_{method}")("y", "x", period=3).alias("result") + result = (frame.lazy() if lazy else frame).select(expression) + if lazy: + assert isinstance(result, pl.LazyFrame) + result = result.collect() + expected = [np.nan, np.nan] + for end in range(2, len(frame)): + x, y = frame["x"].to_numpy()[end - 2 : end + 1], frame["y"].to_numpy()[end - 2 : end + 1] + if method == "pearson": + expected.append(stats.pearsonr(x, y).statistic) + elif method == "spearman": + expected.append(stats.spearmanr(x, y).statistic) + else: + expected.append(stats.linregress(x, y).slope) + np.testing.assert_allclose(result["result"].to_numpy(), expected, atol=1e-14, equal_nan=True) + + +@pytest.mark.parametrize("method", ["pearson", "spearman", "beta"]) +def test_missing_pair_is_removed_from_both_sides_and_full_row_window_remains(method: str) -> None: + frame = pl.DataFrame({"x": [1.0, 90.0, 3.0, 4.0], "y": [2.0, None, 6.0, 8.0]}) + function = getattr(paired(), f"rolling_{method}") + actual = frame.select(function("y", "x", period=4, allowed_missing=1).alias("result")) + np.testing.assert_allclose( + actual["result"].to_numpy(), [np.nan] * 3 + [2 if method == "beta" else 1], equal_nan=True + ) + strict = frame.select(function("y", "x", period=4).alias("result")).item(-1, 0) + assert strict is None or np.isnan(strict) + other_side = pl.DataFrame({"x": [1.0, None, 3.0, 4.0], "y": [2.0, 90.0, 6.0, 8.0]}) + assert other_side.select(function("y", "x", period=4, allowed_missing=1)).item( + -1, 0 + ) == pytest.approx(2 if method == "beta" else 1) + + +def test_spearman_recomputes_average_tie_ranks_in_each_window() -> None: + frame = pl.DataFrame({"x": [1.0, 1.0, 4.0, 2.0, 3.0], "y": [1.0, 2.0, 2.0, 4.0, 1.0]}) + actual = frame.select(paired().rolling_spearman("y", "x", period=3))["y"].to_numpy() + np.testing.assert_allclose(actual[2:], [0.5, 0.0, -0.5], atol=1e-14) + + +@pytest.mark.parametrize("lazy", [False, True]) +def test_rolling_regression_returns_all_five_actual_paired_fields(lazy: bool) -> None: + frame = pl.DataFrame({"x": [1.0, 3.0, 2.0, 4.0, 6.0], "y": [2.0, 8.0, 7.0, 9.0, 10.0]}) + result = ( + (frame.lazy() if lazy else frame) + .select(paired().rolling_regression("y", "x", period=3).alias("regression")) + .unnest("regression") + ) + if lazy: + result = result.collect() + assert result.columns == ["alpha", "beta", "r_value", "p_value", "stderr"] + assert all(result[c].head(2).is_nan().all() for c in result.columns) + for end in range(2, frame.height): + reference = stats.linregress(frame["x"].slice(end - 2, 3), frame["y"].slice(end - 2, 3)) + np.testing.assert_allclose( + result.row(end), + [ + reference.intercept, + reference.slope, + reference.rvalue, + reference.pvalue, + reference.stderr, + ], + rtol=1e-13, + atol=0, + ) + + +def test_regression_missing_pairs_and_constant_target_are_explicit() -> None: + frame = pl.DataFrame({"x": [1.0, None, 3.0, 4.0], "y": [3.0, 100.0, 7.0, 9.0]}) + result = frame.select( + paired().rolling_regression("y", "x", period=4, allowed_missing=1).alias("r") + ).unnest("r") + assert result["alpha"][-1] == pytest.approx(1) + assert result["beta"][-1] == pytest.approx(2) + constant = frame.with_columns(pl.lit(1.0).alias("x")) + result = constant.select(paired().rolling_regression("y", "x", period=3).alias("r")).unnest("r") + assert all(np.isnan(result[c][-1]) for c in result.columns) + + +def test_regression_has_independent_slope_intercept_pvalue_and_stderr_oracle() -> None: + frame = pl.DataFrame({"x": [1.0, 2.0, 3.0, 4.0], "y": [1.0, 2.0, 2.0, 4.0]}) + actual = frame.select(paired().rolling_regression("y", "x", period=4).alias("r")).unnest("r") + # Sxx=5, Sxy=4.5, Syy=4.75, residual SS=.7, df=2. + # The two-sided Student-t df=2 tail is 1-|t|/sqrt(t*t+2). + expected = [0.0, 0.9, 9 / np.sqrt(95), 1 - 9 / np.sqrt(95), np.sqrt(0.07)] + np.testing.assert_allclose(actual.row(-1), expected, rtol=1e-13, atol=1e-15) + + +@pytest.mark.parametrize("method", ["pearson", "spearman", "beta", "regression"]) +def test_empty_and_singleton_pair_series_remain_empty_or_undefined(method: str) -> None: + function = getattr(paired(), f"rolling_{method}") + for count in (0, 1): + frame = pl.DataFrame({"x": [1.0] * count, "y": [2.0] * count}) + actual = frame.select(function("y", "x", period=3).alias("r")) + assert actual.height == count + if count: + if method == "regression": + assert all(np.isnan(v) for v in actual["r"][0].values()) + else: + assert actual["r"][0] is None or np.isnan(actual["r"][0]) + + +@pytest.mark.parametrize("method", ["pearson", "spearman", "beta", "regression"]) +def test_paired_methods_do_not_cross_assets_or_mutate_inputs(method: str) -> None: + frame = pl.DataFrame( + { + "asset": ["A", "B"] * 4, + "x": [1.0, 10.0, 3.0, 20.0, 2.0, 40.0, 4.0, 30.0], + "y": [2.0, 30.0, 8.0, 10.0, 7.0, 50.0, 9.0, 20.0], + } + ) + before = frame.clone() + expression = getattr(paired(), f"rolling_{method}")("y", "x", period=3) + combined = frame.with_columns(expression.over("asset").alias("result")) + for asset in ("A", "B"): + separate = frame.filter(pl.col("asset") == asset).with_columns(expression.alias("result")) + assert_frame_equal(combined.filter(pl.col("asset") == asset), separate) + assert_frame_equal(frame, before) + + +@pytest.mark.parametrize("method", ["pearson", "spearman", "beta", "regression"]) +@pytest.mark.parametrize( + "params", + [ + {"period": 0}, + {"period": 1}, + {"period": True}, + {"period": 3, "allowed_missing": -1}, + {"period": 3, "allowed_missing": 4}, + {"period": 3, "allowed_missing": True}, + ], +) +def test_invalid_pair_windows_fail_before_evaluation( + method: str, params: dict[str, object] +) -> None: + with pytest.raises(ValueError): + getattr(paired(), f"rolling_{method}")("y", "x", **params) diff --git a/uv.lock b/uv.lock index 6525bd3a..6e203305 100644 --- a/uv.lock +++ b/uv.lock @@ -1656,6 +1656,7 @@ ml = [ { name = "shap" }, ] stats = [ + { name = "scipy" }, { name = "statsmodels" }, ] store = [ @@ -1756,6 +1757,7 @@ requires-dist = [ { name = "pyyaml", specifier = ">=6.0.0" }, { name = "ruff", marker = "extra == 'all'", specifier = ">=0.1.0" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.1.0" }, + { name = "scipy", marker = "extra == 'stats'", specifier = ">=1.14.0" }, { name = "seaborn", marker = "extra == 'all'", specifier = ">=0.12.0" }, { name = "seaborn", marker = "extra == 'viz'", specifier = ">=0.12.0" }, { name = "shap", marker = "extra == 'all'", specifier = ">=0.44.0" }, From b607e2fb4d0e4e6089974d1904c2ec7e71d0bbde Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Tue, 20 Oct 2026 19:11:39 -0400 Subject: [PATCH 06/19] fix(features): preserve weighted variance degrees under dominant weights Two positive weights imply unbiased variance (x1-x0)^2/2 regardless of their ratio. Reproduce cancellation and false NaN results before accumulating the equivalent cross-weight denominator directly. Correct the earlier constant-price test assumption to zero variance. Refs #108. --- src/ml4t/engineer/features/rolling.py | 11 ++++++----- tests/compat/test_zipline_factors.py | 12 ++++++++---- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/ml4t/engineer/features/rolling.py b/src/ml4t/engineer/features/rolling.py index e7538b48..1dab211b 100644 --- a/src/ml4t/engineer/features/rolling.py +++ b/src/ml4t/engineer/features/rolling.py @@ -169,7 +169,7 @@ def _variance_windows( for end in range(width - 1, len(values)): start = end - width + 1 first = -1 - weight_sum, weight_squares = 0.0, 0.0 + weight_sum, denominator = 0.0, 0.0 positive, negative, has_missing = False, False, False for j in range(width): if missing[start + j]: @@ -180,11 +180,11 @@ def _variance_windows( value = values[start + j] positive = positive or value > 0 negative = negative or value < 0 + # This equals W**2 - sum(w**2), without subtracting nearly equal numbers. + denominator += 2 * weight_sum * weights[j] weight_sum += weights[j] - weight_squares += weights[j] * weights[j] if first < 0 or (has_missing and not omit): continue - denominator = weight_sum * weight_sum - weight_squares if bias_corrected and denominator <= 0: continue anchor = values[first] @@ -216,8 +216,9 @@ def _variance_windows( if not missing[start + j]: residual = deltas[j] / scale - mean variance += (weights[j] / weight_sum) * residual * residual - correction = weight_sum * weight_sum / denominator if bias_corrected else 1.0 - result[end] = scale * np.sqrt(variance * correction) + if bias_corrected: + variance = variance * weight_sum * weight_sum / denominator + result[end] = scale * np.sqrt(variance) return result diff --git a/tests/compat/test_zipline_factors.py b/tests/compat/test_zipline_factors.py index 94b84b2f..09a873fc 100644 --- a/tests/compat/test_zipline_factors.py +++ b/tests/compat/test_zipline_factors.py @@ -108,10 +108,7 @@ def test_finite_methods_preserve_nan_denominators_weights_and_scale( def test_finite_weights_do_not_overflow_or_underflow_with_legacy_common_power(name: str) -> None: frame = prices([4.0, 4.0, 4.0, 4.0]) result = calculate(frame, name, period=4, decay_rate=1e-100)["factor"][-1] - if name == "finite_ewma": - assert result == 4.0 - else: - assert math.isnan(result) + assert result == (4.0 if name == "finite_ewma" else 0.0) @pytest.mark.parametrize( @@ -355,3 +352,10 @@ def test_normalized_and_range_methods_do_not_collapse_integer_price_ticks(name: else: assert result["factor_up"][-1] == 100 assert result["factor_down"][-1] == 0 + + +@pytest.mark.parametrize("decay", [1e-10, 1e-50, 1e-100]) +def test_two_observation_weighted_std_retains_exact_degrees_of_freedom(decay: float) -> None: + # For any two positive weights, the unbiased variance is (x1-x0)**2/2. + actual = calculate(prices([1.0, 5.0]), "finite_ewm_std", period=2, decay_rate=decay) + assert actual["factor"][-1] == pytest.approx(math.sqrt(8), rel=1e-14, abs=0) From 08f2dba8bd0e066fb395087ed7dbc53e7e470e80 Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Tue, 20 Oct 2026 19:33:07 -0400 Subject: [PATCH 07/19] feat(compat): map Zipline numeric factor names to native expressions Translate finite window counts, exponential constructors, named fields, paired statistics and known event roles without a legacy runtime or graph interpreter. Preserve output masks and price-to-return dependency warmup; reject unrelated options and unordered input roles. Refs #108, #109. --- docs/user-guide/features.md | 42 ++- src/ml4t/engineer/compat/__init__.py | 1 + src/ml4t/engineer/compat/zipline.py | 238 +++++++++++++++++ src/ml4t/engineer/core/lookbacks.py | 1 + src/ml4t/engineer/features/AGENTS.md | 4 +- src/ml4t/engineer/features/rolling.py | 59 ++++- tests/compat/test_zipline_factor_recipes.py | 271 ++++++++++++++++++++ tests/core/test_registry.py | 2 +- tests/test_all_features_integration.py | 2 +- 9 files changed, 614 insertions(+), 6 deletions(-) create mode 100644 src/ml4t/engineer/compat/__init__.py create mode 100644 src/ml4t/engineer/compat/zipline.py create mode 100644 tests/compat/test_zipline_factor_recipes.py diff --git a/docs/user-guide/features.md b/docs/user-guide/features.md index b6ea1ed2..7d00a585 100644 --- a/docs/user-guide/features.md +++ b/docs/user-guide/features.md @@ -133,6 +133,7 @@ defaults. These additions do not provide a Pipeline graph interpreter. | Feature | Definition and missingness | |---|---| | `rsi_cutler` | Simple gains/losses over `period` changes (`period + 1` prices). Omit changes adjacent to missing quotes; a flat window is undefined. | +| `absolute_percent_change` | Fractional change over `period` changes with an absolute reference denominator. This accepts signed values and does not multiply by 100. | | `finite_mean` | Require `period` rows, then divide by the count of nonmissing prices. | | `finite_weighted_mean` | Sum valid price-times-volume products, divided by the independently summed volume. Missing prices do not remove denominator weights. | | `average_dollar_volume` | Sum valid price-times-volume products, divided by the fixed `period` row count. Input price, share and currency basis must already agree. | @@ -148,7 +149,7 @@ defaults. These additions do not provide a Pipeline graph interpreter. | `finite_macd_signal` | Finite normalized fast/slow exponential means, followed by a finite signal mean. Length-one means use their sole observation. | | `finite_true_range` | Two-price true range with NaN-omitting high/low/previous-close components. There is no implicit larger-window reduction. | -Except for Cutler RSI and the composed MACD signal, `period` counts input rows and the +Except for Cutler RSI, absolute percent change and the composed MACD signal, `period` counts input rows and the first structurally usable row is `period - 1`. Missing observations still occupy their original row positions. Numeric results are Float64 and retain every input row; Ichimoku's selected `chikou_span` price retains its original dtype. Multi-output @@ -187,6 +188,45 @@ their kernels as session or elapsed windows. Variance methods subtract an origin window-local typed price anchor before floating conversion, so adjacent large integer ticks do not collapse to zero variance. Output remains Float64. +### Factor-name expression recipes + +`compat.zipline.factor_expression` translates supported built-in names into the +native methods above. It returns a Polars expression or a dictionary of named output +expressions, not a Pipeline Term. It never imports Zipline or evaluates an opaque +legacy graph. Use native column operators for arithmetic/comparisons and the existing +feature decorator for custom calculations. + +Recipe `window_length` counts prices/inputs as in Pipeline. Return and Cutler recipes +translate N prices into N-1 changes. The default annualized-volatility recipe derives +daily returns from `close` and keeps that extra dependency row of warmup. Explicit +`inputs=["returns"]` instead consumes supplied returns. EWMA/EWMSTD accept one explicit +`decay_rate`, `span`, `halflife` or `center_of_mass`, including the three legacy +constructor suffixes. Statistical recipes can consume already aligned column pairs; +their legacy correlation/regression lengths and beta missing percentage are explicit. + + +```python +import math +import polars as pl +from ml4t.engineer.compat.zipline import factor_expression + +frame = pl.DataFrame({"close": [1.0, 3.0, 9.0], "eligible": [True, False, True]}) +expression = factor_expression( + "EWMA.from_span", inputs=["close"], window_length=3, span=3, mask="eligible", +) +result = frame.lazy().with_columns(expression.alias("finite_mean")).collect() +assert math.isclose(result["finite_mean"][2], 43 / 7, rel_tol=1e-14) +assert result.height == frame.height +``` + +The output mask in this example does not erase the middle observation from history. +Apply `.over(entity_col)` for independent asset histories. These expression recipes +do not validate a market schedule. For bar data, validate the declared grid once with +`compute_features(data, [], feed_spec=...)` before deriving returns/features, or +explicitly choose `calendar=None` for observation rows. Recipe output masks are not +dated universe/knowledge selection or final output screens. Named fields stay explicit; +for example, select `factor_expression("BollingerBands", window_length=20)["middle"]`. + ### Paired relationships and supplied event dates `features.paired.rolling_pearson`, `rolling_spearman`, `rolling_beta` and diff --git a/src/ml4t/engineer/compat/__init__.py b/src/ml4t/engineer/compat/__init__.py new file mode 100644 index 00000000..326f3ee5 --- /dev/null +++ b/src/ml4t/engineer/compat/__init__.py @@ -0,0 +1 @@ +"""Explicit migration recipes; no legacy graph runtime or analytical facade.""" diff --git a/src/ml4t/engineer/compat/zipline.py b/src/ml4t/engineer/compat/zipline.py new file mode 100644 index 00000000..903a8988 --- /dev/null +++ b/src/ml4t/engineer/compat/zipline.py @@ -0,0 +1,238 @@ +"""Zipline factor-name recipes over native expressions, without importing Zipline. + +These return Polars expressions or named output expressions, not Pipeline Terms. +Validate the input feed/grid once, keep historical inputs, then apply output +screens. Dated source eligibility belongs to native feature preparation. +""" + +import inspect +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from typing import Any, Literal + +import polars as pl + +from ml4t.engineer.features import events, paired, rolling +from ml4t.engineer.features.utils import arithmetic + +Expression = pl.Expr | dict[str, pl.Expr] + +__all__ = ["factor_expression"] + + +@dataclass(frozen=True) +class _Recipe: + function: Callable[..., Expression] + columns: tuple[str, ...] + window: int | None + required_inputs: bool = False + order: tuple[int, ...] | None = None + window_arg: str | None = "period" + adjustment: int = 0 + + +def _clip(value: str, min_bound: float, max_bound: float) -> pl.Expr: + return pl.col(value).clip(min_bound, max_bound) + + +def _latest(value: str) -> pl.Expr: + return pl.col(value) + + +_RECIPES = { + "Returns": _Recipe(arithmetic.returns, ("close",), None, window_arg="periods", adjustment=-1), + "DailyReturns": _Recipe(arithmetic.returns, ("close",), 2, window_arg="periods", adjustment=-1), + "PercentChange": _Recipe( + rolling.absolute_percent_change, ("close",), None, True, adjustment=-1 + ), + "RateOfChangePercentage": _Recipe( + arithmetic.percentage_change, ("close",), None, True, window_arg="periods", adjustment=-1 + ), + "SimpleMovingAverage": _Recipe(rolling.finite_mean, ("close",), None, True), + "WeightedAverageValue": _Recipe(rolling.finite_weighted_mean, ("close", "volume"), None, True), + "VWAP": _Recipe(rolling.finite_weighted_mean, ("close", "volume"), None), + "AverageDollarVolume": _Recipe(rolling.average_dollar_volume, ("close", "volume"), None), + "LinearWeightedMovingAverage": _Recipe(rolling.linear_weighted_mean, ("close",), None, True), + "MaxDrawdown": _Recipe(rolling.finite_max_drawdown, ("close",), None, True), + "AnnualizedVolatility": _Recipe(rolling.annualized_volatility, ("returns",), 252), + "RSI": _Recipe(rolling.rsi_cutler, ("close",), 15, adjustment=-1), + "BollingerBands": _Recipe(rolling.finite_bollinger_bands, ("close",), None), + "Aroon": _Recipe(rolling.finite_aroon, ("low", "high"), None, order=(1, 0)), + "FastStochasticOscillator": _Recipe( + rolling.fast_stochastic, ("close", "low", "high"), 14, order=(2, 1, 0) + ), + "IchimokuKinkoHyo": _Recipe(rolling.finite_ichimoku, ("high", "low", "close"), 52), + "TrueRange": _Recipe(rolling.finite_true_range, ("high", "low", "close"), 2, window_arg=None), + "Clip": _Recipe(_clip, ("close",), 1, True, window_arg=None), + "Latest": _Recipe(_latest, ("close",), 1, True, window_arg=None), + "RollingPearson": _Recipe(paired.rolling_pearson, ("value", "target"), None, True), + "RollingSpearman": _Recipe(paired.rolling_spearman, ("value", "target"), None, True), + "RollingLinearRegression": _Recipe(paired.rolling_regression, ("value", "target"), None, True), + "SimpleBeta": _Recipe(paired.rolling_beta, ("returns", "target_returns"), None, True), + "ExponentialWeightedMovingAverage": _Recipe(rolling.finite_ewma, ("close",), None, True), + "ExponentialWeightedMovingStdDev": _Recipe(rolling.finite_ewm_std, ("close",), None, True), + "MovingAverageConvergenceDivergenceSignal": _Recipe( + rolling.finite_macd_signal, ("close",), None, window_arg=None + ), +} +_ALIASES = { + "EWMA": "ExponentialWeightedMovingAverage", + "EWMSTD": "ExponentialWeightedMovingStdDev", + "MACDSignal": "MovingAverageConvergenceDivergenceSignal", +} +_FACTORIES = { + "from_span": "span", + "from_halflife": "halflife", + "from_center_of_mass": "center_of_mass", +} +_EVENTS: dict[str, Literal["since", "until"]] = { + "BusinessDaysSincePreviousEvent": "since", + "BusinessDaysUntilNextEvent": "until", +} + + +def _inputs(inputs: Sequence[str] | None, columns: tuple[str, ...], required: bool) -> list[str]: + if inputs is None: + if required: + raise ValueError("This factor requires explicit input column names") + return list(columns) + if ( + not isinstance(inputs, Sequence) + or isinstance(inputs, str) + or len(inputs) != len(columns) + or any(not isinstance(c, str) or not c for c in inputs) + ): + raise ValueError(f"inputs must contain {len(columns)} nonempty column names") + return list(inputs) + + +def factor_expression( + name: str, + *, + inputs: Sequence[str] | None = None, + window_length: int | None = None, + mask: str | pl.Expr | None = None, + **params: Any, +) -> Expression: + """Map a supported factor name to one native calculation and explicit field names. + + window_length counts prices/inputs as in Pipeline; return/RSI recipes translate + N prices to N-1 changes. A mask controls output only and never removes warmup + history. False/null predicate values yield native null outputs; mask=None + leaves outputs unmasked. Factory suffixes support + the three EWMA/EWMSTD constructors. Opaque Terms/Assets are not executed. + """ + if not isinstance(name, str): + raise TypeError("name must identify a supported factor recipe") + pieces = name.split(".") + if len(pieces) > 2: + raise ValueError("Unsupported factor constructor") + canonical = _ALIASES.get(pieces[0], pieces[0]) + factory = pieces[1] if len(pieces) == 2 else None + if factory is not None and ( + factory not in _FACTORIES + or canonical not in {"ExponentialWeightedMovingAverage", "ExponentialWeightedMovingStdDev"} + ): + raise ValueError("Only the declared finite exponential constructors are supported") + if mask is not None and not isinstance(mask, str | pl.Expr): + raise TypeError("mask must be an input column name or expression") + if canonical in _EVENTS: + if isinstance(window_length, bool) or window_length not in (None, 0): + raise ValueError("Event weekday counts do not use rolling windows") + column = _inputs(inputs, ("event_date",), True)[0] + output: Expression = events.event_business_days( + column, direction=_EVENTS[canonical], **params + ) + else: + if canonical not in _RECIPES: + raise ValueError( + f"Unsupported numeric recipe {name!r}; port custom calculations to native expressions/registered features" + ) + route = _RECIPES[canonical] + columns = _inputs(inputs, route.columns, route.required_inputs) + if route.order is not None: + columns = [columns[index] for index in route.order] + options = dict(params) + if canonical.startswith("ExponentialWeightedMoving"): + if factory is not None and _FACTORIES[factory] not in options: + raise ValueError(f"{factory} requires {_FACTORIES[factory]}") + selectors = { + key: options.pop(key) + for key in ("decay_rate", "span", "halflife", "center_of_mass") + if key in options + } + options["decay_rate"] = rolling.exponential_decay(**selectors) + if canonical == "IchimokuKinkoHyo": + for old, new in ( + ("tenkan_sen_length", "tenkan_length"), + ("kijun_sen_length", "kijun_length"), + ("chikou_span_length", "chikou_position"), + ): + if old in options: + if new in options: + raise ValueError(f"Conflicting {old}/{new}") + options[new] = options.pop(old) + lengths = ( + ("correlation_length",) + if canonical in {"RollingPearson", "RollingSpearman"} + else ( + ("regression_length",) + if canonical in {"RollingLinearRegression", "SimpleBeta"} + else () + ) + ) + for legacy in lengths: + if legacy in options: + if window_length is not None: + raise ValueError("Declare one window_length or legacy statistical length") + window_length = options.pop(legacy) + width = window_length if window_length is not None else route.window + if route.window_arg is not None: + if width is None: + raise ValueError("Declare window_length") + rolling._period(width) + if route.window_arg in options: + raise ValueError("Use window_length rather than a second native window argument") + options[route.window_arg] = width + route.adjustment + elif canonical == "MovingAverageConvergenceDivergenceSignal": + if window_length is not None: + raise ValueError("MACDSignal derives its window from the three periods") + else: + assert width is not None + rolling._period(width) + if canonical == "TrueRange" and width != 2: + raise ValueError("TrueRange requires its two-price definition") + if canonical == "SimpleBeta": + assert width is not None + if width < 3: + raise ValueError("SimpleBeta regression_length must be >=3") + fraction = options.pop("allowed_missing_percentage", 0.25) + if ( + isinstance(fraction, bool) + or not isinstance(fraction, int | float) + or not 0 <= fraction <= 1 + ): + raise ValueError("allowed_missing_percentage must be in [0,1]") + if "allowed_missing" in options: + raise ValueError("Declare allowed_missing_percentage only once") + options["allowed_missing"] = int(width * fraction) + signature = inspect.signature(route.function) + column_params = list(signature.parameters)[: len(columns)] + unknown = set(options) - (set(signature.parameters) - set(column_params)) + if unknown: + raise ValueError(f"Unknown factor options: {sorted(unknown)}") + if canonical == "AnnualizedVolatility" and inputs is None: + output = route.function(arithmetic.returns("close"), **options) + assert isinstance(output, pl.Expr) and width is not None + output = rolling._ready(output, width + 1) + else: + output = route.function(*columns, **options) + if canonical in {"Clip", "Latest"}: + assert isinstance(output, pl.Expr) and width is not None + output = rolling._ready(output, width) + if mask is None: + return output + predicate = pl.col(mask) if isinstance(mask, str) else mask + if isinstance(output, dict): + return {key: pl.when(predicate).then(value) for key, value in output.items()} + return pl.when(predicate).then(output) diff --git a/src/ml4t/engineer/core/lookbacks.py b/src/ml4t/engineer/core/lookbacks.py index 543d18dd..c8c1ff1f 100644 --- a/src/ml4t/engineer/core/lookbacks.py +++ b/src/ml4t/engineer/core/lookbacks.py @@ -70,6 +70,7 @@ def _ulcer_index(params: Mapping[str, Any]) -> int: FEATURE_LOOKBACKS: dict[str, LookbackCalculator] = { + "absolute_percent_change": _parameter("period"), # Explicit finite-window methods, independent of the native TA-Lib defaults. "rsi_cutler": _parameter("period"), "finite_mean": _parameter("period", -1), diff --git a/src/ml4t/engineer/features/AGENTS.md b/src/ml4t/engineer/features/AGENTS.md index 5209d195..e821542e 100644 --- a/src/ml4t/engineer/features/AGENTS.md +++ b/src/ml4t/engineer/features/AGENTS.md @@ -7,7 +7,7 @@ This directory contains the registry-backed indicator implementations used by | Category | Count | Location | Examples | |----------|-------|----------|----------| -| momentum | 35 | `momentum/`, `rolling.py` | `rsi`, `rsi_cutler`, `macd`, `finite_macd_signal` | +| momentum | 36 | `momentum/`, `rolling.py` | `rsi`, `rsi_cutler`, `absolute_percent_change`, `finite_macd_signal` | | trend | 15 | `trend/`, `rolling.py` | `sma`, `ema`, `finite_mean`, `finite_ewma` | | volatility | 18 | `volatility/`, `rolling.py` | `atr`, `bollinger_bands`, `annualized_volatility` | | microstructure | 15 | `microstructure/` | `kyle_lambda`, `amihud_illiquidity`, `realized_spread` | @@ -72,6 +72,6 @@ durations. Do not resample, infer missing sessions or reinterpret recursive indi ## Trust Signals -- 135 total features across 11 categories +- 136 total features across 11 categories - 60 TA-Lib validated indicators at `1e-6` tolerance - registry metadata supports discovery by category, normalization, and TA-Lib parity diff --git a/src/ml4t/engineer/features/rolling.py b/src/ml4t/engineer/features/rolling.py index 1dab211b..de2b92fe 100644 --- a/src/ml4t/engineer/features/rolling.py +++ b/src/ml4t/engineer/features/rolling.py @@ -6,6 +6,7 @@ """ import math +from numbers import Real from typing import Any import numpy as np @@ -62,6 +63,62 @@ def right_type(reference: pl.Expr, reference_dtype: pl.DataType) -> pl.Expr: return left.pipe_with_dtype(left_type) +@feature( + name="absolute_percent_change", + category="momentum", + description="Fractional change with absolute reference denominator", + normalized=True, + value_range=(float("-inf"), float("inf")), + formula="(value - previous) / abs(previous)", +) +def absolute_percent_change(close: str, period: int = 1) -> pl.Expr: + """Fractional change of signed values; not percentage points or signed-denominator returns.""" + _period(period) + value = pl.col(close) + previous = value.shift(period) + return _paired_difference(value, previous) / previous.cast(pl.Float64).abs() + + +def exponential_decay( + *, + decay_rate: float | None = None, + span: float | None = None, + halflife: float | None = None, + center_of_mass: float | None = None, +) -> float: + """Resolve exactly one finite-weight selector, with no implicit recursive-EMA substitution.""" + choices = { + "decay_rate": decay_rate, + "span": span, + "halflife": halflife, + "center_of_mass": center_of_mass, + } + supplied = [(name, value) for name, value in choices.items() if value is not None] + if len(supplied) != 1: + raise ValueError("Declare exactly one of decay_rate, span, halflife, center_of_mass") + name, value = supplied[0] + if isinstance(value, bool) or not isinstance(value, Real): + raise ValueError(f"{name} must be a finite real scalar") + value = float(value) + if not math.isfinite(value): + raise ValueError(f"{name} must be a finite real scalar") + if name == "span": + if value <= 1: + raise ValueError("span must be > 1") + value = 1 - 2 / (1 + value) + elif name == "halflife": + if value <= 0: + raise ValueError("halflife must be positive") + value = math.exp(math.log(0.5) / value) + elif name == "center_of_mass": + if value <= 0: + raise ValueError("center_of_mass must be positive") + value = 1 - 1 / (1 + value) + if not 0 < value <= 1: + raise ValueError("Resolved decay_rate must be in (0, 1]") + return value + + @feature( name="rsi_cutler", category="momentum", @@ -323,7 +380,7 @@ def finite_ewm_std(close: str, period: int = 20, decay_rate: float = 0.9) -> pl. formula="nanstd(returns, ddof=0) * sqrt(annualization_factor)", ) def annualized_volatility( - returns: str, period: int = 252, annualization_factor: float = 252.0 + returns: str | pl.Expr, period: int = 252, annualization_factor: float = 252.0 ) -> pl.Expr: """Full-window population volatility; period counts supplied return observations.""" _period(period) diff --git a/tests/compat/test_zipline_factor_recipes.py b/tests/compat/test_zipline_factor_recipes.py new file mode 100644 index 00000000..a69473cc --- /dev/null +++ b/tests/compat/test_zipline_factor_recipes.py @@ -0,0 +1,271 @@ +"""Thin legacy-name recipes must preserve methods, input ordering and output masks.""" + +import importlib +import math +from datetime import UTC + +import numpy as np +import polars as pl +import pytest + + +def recipe(name: str, **options): + return importlib.import_module("ml4t.engineer.compat.zipline").factor_expression( + name, **options + ) + + +@pytest.mark.parametrize( + "name,window,expected", + [ + ("Returns", 3, [np.nan, np.nan, 0.21, 0.21]), + ("DailyReturns", None, [np.nan, 0.1, 0.1, 0.1]), + ("DailyReturns", 3, [np.nan, np.nan, 0.21, 0.21]), + ("RateOfChangePercentage", 3, [np.nan, np.nan, 21, 21]), + ], +) +def test_return_recipes_translate_price_count_to_changes(name, window, expected) -> None: + frame = pl.DataFrame({"close": [100.0, 110.0, 121.0, 133.1]}) + options = {"window_length": window} if window is not None else {} + if name == "RateOfChangePercentage": + options["inputs"] = ["close"] + actual = frame.lazy().select(recipe(name, **options).alias("result")).collect() + np.testing.assert_allclose( + actual["result"].to_numpy(), expected, rtol=1e-14, atol=0, equal_nan=True + ) + + +def test_absolute_percent_change_does_not_use_signed_denominator_or_percent_scale() -> None: + frame = pl.DataFrame({"value": [-100.0, -90.0, -110.0]}) + actual = frame.select( + recipe("PercentChange", inputs=["value"], window_length=2).alias("result") + ) + np.testing.assert_allclose( + actual["result"].to_numpy(), [np.nan, 0.1, -2 / 9], rtol=1e-14, equal_nan=True + ) + zero = pl.DataFrame({"value": [-0.0, 1.0, -0.0, 2.0]}) + np.testing.assert_allclose( + zero.select(recipe("PercentChange", inputs=["value"], window_length=2)) + .to_series() + .to_numpy(), + [np.nan, np.inf, -1, np.inf], + equal_nan=True, + ) + + +@pytest.mark.parametrize( + "name", + ["EWMA", "ExponentialWeightedMovingAverage", "EWMSTD", "ExponentialWeightedMovingStdDev"], +) +@pytest.mark.parametrize( + "constructor,params", + [ + ("from_span", {"span": 3.0}), + ("from_halflife", {"halflife": 1.0}), + ("from_center_of_mass", {"center_of_mass": 1.0}), + ], +) +def test_exponential_factories_select_the_same_declared_finite_weights( + name, constructor, params +) -> None: + frame = pl.DataFrame({"close": [1.0, 3.0, 9.0]}) + actual = frame.select( + recipe(f"{name}.{constructor}", inputs=["close"], window_length=3, **params) + ).item(-1, 0) + expected = ( + math.sqrt(138 / 7) if name in {"EWMSTD", "ExponentialWeightedMovingStdDev"} else 43 / 7 + ) + assert actual == pytest.approx(expected, rel=1e-14, abs=0) + + +@pytest.mark.parametrize( + "name", + [ + "SimpleMovingAverage", + "WeightedAverageValue", + "VWAP", + "AverageDollarVolume", + "LinearWeightedMovingAverage", + "MaxDrawdown", + "RSI", + "BollingerBands", + "Aroon", + "FastStochasticOscillator", + "IchimokuKinkoHyo", + "TrueRange", + "AnnualizedVolatility", + "MACDSignal", + "MovingAverageConvergenceDivergenceSignal", + ], +) +def test_numeric_recipes_produce_lazy_expressions_without_importing_legacy_runtime(name) -> None: + import sys + + frame = pl.DataFrame( + {"close": np.linspace(10.0, 20.0, 60), "volume": np.arange(1.0, 61.0)} + ).with_columns( + (pl.col("close") + 2).alias("high"), + (pl.col("close") - 2).alias("low"), + ) + options = {} + if name in {"SimpleMovingAverage", "MaxDrawdown", "LinearWeightedMovingAverage"}: + options.update(inputs=["close"], window_length=3) + elif name == "WeightedAverageValue": + options.update(inputs=["close", "volume"], window_length=3) + elif name in {"VWAP", "AverageDollarVolume", "BollingerBands", "Aroon"}: + options["window_length"] = 3 + output = recipe(name, **options) + if isinstance(output, dict): + expressions = [value.alias(key) for key, value in output.items()] + else: + assert isinstance(output, pl.Expr) + expressions = [output.alias("result")] + result = frame.lazy().select(expressions).collect() + assert result.height == frame.height + assert not any(key == "zipline" or key.startswith("zipline.") for key in sys.modules) + + +def test_aroon_recipe_translates_legacy_low_high_input_order() -> None: + frame = pl.DataFrame({"l": [3.0, 1.0, 1.0], "h": [4.0, 5.0, 5.0]}) + output = recipe("Aroon", inputs=["l", "h"], window_length=3) + result = frame.select([v.alias(k) for k, v in output.items()]) + assert result.row(-1) == (50.0, 50.0) + + +def test_output_mask_does_not_erase_prior_input_history() -> None: + frame = pl.DataFrame({"close": [1.0, 2.0, 3.0, 4.0], "mask": [False, True, False, True]}) + result = frame.with_columns( + recipe("SimpleMovingAverage", inputs=["close"], window_length=3, mask="mask").alias( + "result" + ) + ) + assert result["result"].to_list() == [None, None, None, 3.0] + assert result.height == 4 + assert result["close"].to_list() == [1.0, 2.0, 3.0, 4.0] + + +def test_clip_recipe_delegates_bounds_and_preserves_missing_values() -> None: + frame = pl.DataFrame({"value": [float("nan"), -3.0, -1.0, 4.0]}) + result = frame.select(recipe("Clip", inputs=["value"], min_bound=-2.0, max_bound=2.0)) + np.testing.assert_allclose(result.to_series().to_numpy(), [np.nan, -2, -1, 2], equal_nan=True) + + +def test_default_annualized_recipe_keeps_price_to_returns_dependency_warmup() -> None: + frame = pl.DataFrame({"close": [100.0, 110.0, 121.0, 133.1]}) + result = frame.select(recipe("AnnualizedVolatility", window_length=3).alias("result")) + initial = result["result"].head(3) + assert (initial.is_null() | initial.is_nan()).all() + assert result["result"][-1] == pytest.approx(0, abs=1e-14) + + +@pytest.mark.parametrize( + "name,options", + [ + ("RSI", {"regression_length": 3}), + ("VWAP", {"correlation_length": 3}), + ("VWAP", {"window_length": 3, "inputs": {"close", "volume"}}), + ( + "BusinessDaysSincePreviousEvent", + { + "inputs": ["event"], + "window_length": False, + "reference_date_col": "session", + "known_at_col": "known", + "decision_time_col": "decision", + }, + ), + ], +) +def test_options_cannot_change_unrelated_factor_meaning_or_input_order(name, options) -> None: + with pytest.raises((TypeError, ValueError)): + recipe(name, **options) + + +@pytest.mark.parametrize( + "name,length,expected", + [ + ("RollingPearson", "correlation_length", 9 / math.sqrt(95)), + ("RollingSpearman", "correlation_length", 3 / math.sqrt(10)), + ], +) +def test_paired_recipes_preserve_actual_paired_series(name, length, expected) -> None: + frame = pl.DataFrame({"x": [1.0, 2.0, 3.0, 4.0], "y": [1.0, 2.0, 2.0, 4.0]}) + result = frame.select(recipe(name, inputs=["y", "x"], **{length: 4})) + assert result.item(-1, 0) == pytest.approx(expected, rel=1e-14, abs=0) + + +def test_regression_recipe_exposes_actual_named_fields_and_beta_missing_fraction() -> None: + frame = pl.DataFrame({"x": [1.0, 2.0, 3.0, 4.0], "y": [1.0, 2.0, 2.0, 4.0]}) + result = frame.select( + recipe("RollingLinearRegression", inputs=["y", "x"], regression_length=4).alias( + "regression" + ) + ).unnest("regression") + np.testing.assert_allclose( + result.row(-1), + [0, 0.9, 9 / math.sqrt(95), 1 - 9 / math.sqrt(95), math.sqrt(0.07)], + rtol=1e-13, + atol=1e-15, + ) + gapped = pl.DataFrame({"x": [0.0, 1.0, 2.0, 3.0], "y": [1.0, None, 5.0, 7.0]}) + result = gapped.select(recipe("SimpleBeta", inputs=["y", "x"], regression_length=4)) + assert result.item(-1, 0) == pytest.approx(2.0, rel=1e-14) + + +@pytest.mark.parametrize( + "values,dtype", [(["A", "B"], pl.String), ([True, False], pl.Boolean), ([1, 2], pl.Int64)] +) +def test_latest_recipe_preserves_nonprice_input_dtypes(values, dtype) -> None: + frame = pl.DataFrame({"value": pl.Series(values, dtype=dtype)}) + result = frame.select(recipe("Latest", inputs=["value"]).alias("value")) + assert result.equals(frame) + + +@pytest.mark.parametrize( + "name,expected", [("BusinessDaysSincePreviousEvent", 1.0), ("BusinessDaysUntilNextEvent", -1.0)] +) +def test_event_recipes_use_explicit_date_and_knowledge_roles(name, expected) -> None: + from datetime import date, datetime + + frame = pl.DataFrame( + { + "event": [date(2025, 7, 4)], + "session": [date(2025, 7, 7)], + "known": [datetime(2025, 7, 1, tzinfo=UTC)], + "decision": [datetime(2025, 7, 7, tzinfo=UTC)], + } + ) + result = frame.select( + recipe( + name, + inputs=["event"], + reference_date_col="session", + known_at_col="known", + decision_time_col="decision", + ) + ) + assert result.item() == expected + + +@pytest.mark.parametrize( + "name,options", + [ + ("RSI", {"unknown": 1}), + ("RSI", {"window_length": True}), + ("RSI", {"inputs": "close"}), + ("VWAP", {"inputs": ["close"], "window_length": 3}), + ("EWMA", {"inputs": ["close"], "window_length": 3}), + ("EWMA", {"inputs": ["close"], "window_length": 3, "span": 3, "decay_rate": 0.5}), + ("SimpleMovingAverage", {"window_length": 3}), + ("SimpleMovingAverage", {"inputs": ["close"]}), + ("TrueRange", {"window_length": 3}), + ("RSI.from_span", {"span": 3}), + ("MACDSignal", {"window_length": 34}), + ("Factor", {}), + ], +) +def test_recipe_rejects_ambiguous_or_nonfunctional_requests_before_evaluation( + name, options +) -> None: + with pytest.raises((TypeError, ValueError)): + recipe(name, **options) diff --git a/tests/core/test_registry.py b/tests/core/test_registry.py index 7d3aad4d..8addbc5b 100644 --- a/tests/core/test_registry.py +++ b/tests/core/test_registry.py @@ -555,7 +555,7 @@ def test_registry_with_real_features(): # Test category queries work momentum = registry.list_by_category("momentum") - assert len(momentum) == 35 + assert len(momentum) == 36 volatility = registry.list_by_category("volatility") assert len(volatility) == 18 diff --git a/tests/test_all_features_integration.py b/tests/test_all_features_integration.py index d8cfb867..13d60d93 100644 --- a/tests/test_all_features_integration.py +++ b/tests/test_all_features_integration.py @@ -46,7 +46,7 @@ def sample_ohlcv_data(): def test_all_features_registered(): """Verify that features are actually registered.""" registry = get_registry() - assert len(registry) == 135, f"Expected 135 features, got {len(registry)}" + assert len(registry) == 136, f"Expected 136 features, got {len(registry)}" def test_all_features_compute_individually(sample_ohlcv_data): From b216d88c355b447bc4db14d758698d290b01eee8 Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Tue, 20 Oct 2026 19:50:53 -0400 Subject: [PATCH 08/19] feat(features): prepare keyed target-return factors without compressing gaps --- docs/user-guide/features.md | 50 +++ scripts/check_coverage.py | 1 + src/ml4t/engineer/compat/zipline.py | 71 ++- src/ml4t/engineer/features/paired.py | 209 ++++++++- .../test_zipline_return_factor_alignment.py | 422 ++++++++++++++++++ 5 files changed, 750 insertions(+), 3 deletions(-) create mode 100644 tests/compat/test_zipline_return_factor_alignment.py diff --git a/docs/user-guide/features.md b/docs/user-guide/features.md index 7d00a585..18933186 100644 --- a/docs/user-guide/features.md +++ b/docs/user-guide/features.md @@ -245,6 +245,56 @@ Its inferential fields require the optional `stats` extra (`uv add This is predictive rolling regression on supplied observations, not trend-on-time regression or portfolio attribution. +For price panels, `features.paired.target_return_features` performs the exact +time/identity join and derives returns before applying the paired statistic. +Declare the target as a mapping of physical identity columns to stable values, +including all components for a compound identity. `price_basis` records the +supplied Data/caller declaration; this function does not adjust prices. + +The default `index_mode="common_grid"` temporarily retains absent quotes on the +supplied union clock through both return derivation and rolling calculation. +`per_asset` instead counts each asset's own observations. Neither mode fills +quotes, compresses missing pairs out of rolling windows, or emits synthetic rows. +A named calendar validates the original declared grid before calculating any +return. Explicit `calendar=None` selects observation rows. These finite statistics +support observation/expected-bar units, not session or elapsed windows. + + +```python +from datetime import date +import math +import polars as pl +from ml4t.specs import FeedSpec +from ml4t.engineer.features.paired import target_return_features + +days = [date(2025, 7, day) for day in (7, 8, 9, 10)] +prices = pl.DataFrame({ + "timestamp": days * 2, "asset": ["A"] * 4 + ["B"] * 4, + "close": [100., 120., 80., 112., 100., 110., 99., 118.8], +}) +result = target_return_features( + prices.lazy(), target={"asset": "B"}, period=3, method="beta", + price_basis="raw", feed_spec=FeedSpec(time_contract_version=2, calendar=None), +).collect() +assert math.isclose(result.filter(pl.col("asset") == "A")["target_beta"][-1], 52 / 21) +assert result.select(prices.columns).equals(prices) +assert result["target_available_at"].null_count() == prices.height +``` + +Added return columns expose the actual paired inputs. `_available_at` is the latest +publication instant across both assets' price dependencies in the full window. +Without complete supplied timezone-aware `available_at_col` values it stays +unknown. Observation timestamps are not fabricated publication times. The caller +must still enforce decision-time eligibility before using these features. + +`compat.zipline.return_factor_features` maps `RollingPearsonOfReturns`, +`RollingSpearmanOfReturns`, `RollingLinearRegressionOfReturns` and `SimpleBeta` +onto this native preparation. It translates `returns_length=N` to an N-1-price +lag, accepts the corresponding correlation/regression length, and converts +SimpleBeta's allowed-missing percentage to a count. Its optional output mask +does not remove the target's prices or warmup history. Pass stable target IDs, +not opaque Zipline Asset objects. This recipe returns a native frame, not a Term. + ```python import math diff --git a/scripts/check_coverage.py b/scripts/check_coverage.py index b58bdb62..2435f4bd 100644 --- a/scripts/check_coverage.py +++ b/scripts/check_coverage.py @@ -38,6 +38,7 @@ "src/ml4t/engineer/core/validation.py", "src/ml4t/engineer/dataset.py", "src/ml4t/engineer/discovery/catalog.py", + "src/ml4t/engineer/features/paired.py", "src/ml4t/engineer/labeling/atr_barriers.py", "src/ml4t/engineer/labeling/calendar.py", "src/ml4t/engineer/labeling/horizon_labels.py", diff --git a/src/ml4t/engineer/compat/zipline.py b/src/ml4t/engineer/compat/zipline.py index 903a8988..b0b98003 100644 --- a/src/ml4t/engineer/compat/zipline.py +++ b/src/ml4t/engineer/compat/zipline.py @@ -6,7 +6,7 @@ """ import inspect -from collections.abc import Callable, Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from typing import Any, Literal @@ -17,7 +17,74 @@ Expression = pl.Expr | dict[str, pl.Expr] -__all__ = ["factor_expression"] +__all__ = ["factor_expression", "return_factor_features"] + + +def return_factor_features( + data: pl.DataFrame | pl.LazyFrame, + name: str, + *, + target: Mapping[str, Any], + returns_length: int = 2, + correlation_length: int | None = None, + regression_length: int | None = None, + allowed_missing_percentage: float | None = None, + mask: str | pl.Expr | None = None, + **native_options: Any, +) -> pl.DataFrame | pl.LazyFrame: + """Map return-factor constructors to native keyed price preparation. + + target contains stable physical IDs, not an opaque Zipline Asset. Declare + price_basis, feed_spec/calendar and column mappings through native_options. + The mask changes the statistic output only, retaining target input history. + This is a dataframe recipe, not a Pipeline interpreter or legacy constructor. + """ + methods = { + "RollingPearsonOfReturns": "pearson", + "RollingSpearmanOfReturns": "spearman", + "RollingLinearRegressionOfReturns": "regression", + "SimpleBeta": "beta", + } + if name not in methods: + raise ValueError(f"Unsupported return-factor recipe {name!r}") + method = methods[name] + rolling._period(returns_length, minimum=2) + if mask is not None and not isinstance(mask, str | pl.Expr): + raise TypeError("mask must be an input column name or expression") + correlation = method in {"pearson", "spearman"} + width = correlation_length if correlation else regression_length + wrong_length = regression_length if correlation else correlation_length + if width is None or wrong_length is not None: + raise ValueError("Declare only the correlation_length or regression_length for this recipe") + rolling._period(width, minimum=3 if method == "beta" else 2) + allowed_missing = 0 + if method == "beta": + if returns_length != 2: + raise ValueError("SimpleBeta uses daily (two-price) returns") + fraction = 0.25 if allowed_missing_percentage is None else allowed_missing_percentage + if ( + isinstance(fraction, bool) + or not isinstance(fraction, int | float) + or not 0 <= fraction <= 1 + ): + raise ValueError("allowed_missing_percentage must be in [0,1]") + allowed_missing = int(width * fraction) + elif allowed_missing_percentage is not None: + raise ValueError("Only SimpleBeta has allowed_missing_percentage") + output = paired.target_return_features( + data, + target=target, + period=width, + returns_periods=returns_length - 1, + method=method, + allowed_missing=allowed_missing, + **native_options, + ) + if mask is None: + return output + predicate = pl.col(mask) if isinstance(mask, str) else mask + column = f"{native_options.get('output', 'target')}_{method}" + return output.with_columns(pl.when(predicate).then(pl.col(column)).alias(column)) @dataclass(frozen=True) diff --git a/src/ml4t/engineer/features/paired.py b/src/ml4t/engineer/features/paired.py index 2b3209c1..96adb27f 100644 --- a/src/ml4t/engineer/features/paired.py +++ b/src/ml4t/engineer/features/paired.py @@ -5,11 +5,13 @@ removed from both sides together, never by compressing the input timeline. """ -from collections.abc import Sequence +from collections.abc import Mapping, Sequence +from typing import Any import numpy as np import numpy.typing as npt import polars as pl +from ml4t.specs import UNSET_CALENDAR, FeedSpec, WindowUnit from ml4t.engineer._numba import jit from ml4t.engineer.features.cross_asset import beta_to_market, rolling_correlation @@ -17,6 +19,211 @@ from ml4t.engineer.utils.dependencies import require_dependency +def target_return_features( + data: pl.DataFrame | pl.LazyFrame, + *, + target: Mapping[str, Any], + period: int, + price_basis: str, + returns_periods: int = 1, + method: str = "pearson", + feed_spec: FeedSpec | None = None, + timestamp_col: str | None = None, + entity_col: str | list[str] | None = None, + price_col: str | None = None, + calendar: Any = UNSET_CALENDAR, + window_unit: str | WindowUnit | None = None, + index_mode: str = "common_grid", + allowed_missing: int = 0, + missing_price_policy: str = "raise", + available_at_col: str | None = None, + output: str = "target", +) -> pl.DataFrame | pl.LazyFrame: + """Prepare keyed price returns and a rolling statistic against one stable target. + + The default common_grid uses the supplied union clock, retaining absent quotes + through both return derivation and the rolling window. per_asset instead counts + each asset's observations before joining target returns on exact timestamps. + Neither mode fills prices. A named calendar validates the original declared grid + before calculation. calendar=None selects observation rows, not a 24/7 schedule. + + Original columns, keys, order and row count survive. Added columns are + _asset_return, _target_return, _, _available_at and _price_basis. + Availability is unknown unless all price dependencies in the full window have + supplied timezone-aware publication instants. It is not a decision-time screen. + price_basis records a caller declaration; this function does not adjust prices. + """ + from ml4t.engineer.core.bar_grid import validate_declared_grid + from ml4t.engineer.features.events import _instant_ticks + from ml4t.engineer.features.utils.arithmetic import returns + + _validate(period, allowed_missing) + _period(returns_periods) + if period > 2**63 - 1 or returns_periods > 2**63 - 1: + raise ValueError("Window counts must fit signed 64-bit integers") + methods = { + "pearson": rolling_pearson, + "spearman": rolling_spearman, + "beta": rolling_beta, + "regression": rolling_regression, + } + if method not in methods: + raise ValueError(f"method must be one of {tuple(methods)}") + if not isinstance(price_basis, str) or not price_basis.strip(): + raise ValueError("Declare a nonempty price_basis") + if not isinstance(output, str) or not output: + raise ValueError("output must be a nonempty prefix") + if index_mode not in {"common_grid", "per_asset"}: + raise ValueError("index_mode must be common_grid or per_asset") + if missing_price_policy not in {"raise", "null"}: + raise ValueError("missing_price_policy must be raise or null") + contract = ( + FeedSpec(time_contract_version=2) if feed_spec is None else FeedSpec.from_any(feed_spec) + ) + + def bind(explicit: Any, declared: Any, default: Any, role: str) -> Any: + if explicit is not None and feed_spec is not None and declared is not None: + left = [explicit] if isinstance(explicit, str) else list(explicit) + right = [declared] if isinstance(declared, str) else list(declared) + if left != right: + raise ValueError(f"Conflicting {role} and feed_spec declaration") + return explicit if explicit is not None else declared if declared is not None else default + + timestamp = bind(timestamp_col, contract.timestamp_col, "timestamp", "timestamp_col") + identity = bind(entity_col, contract.entity_col, "asset", "entity_col") + entities = [identity] if isinstance(identity, str) else list(identity) + price = bind(price_col, contract.price_col, "close", "price_col") + if not entities or len(set(entities)) != len(entities): + raise ValueError("Declare distinct entity columns") + if not isinstance(target, Mapping) or set(target) != set(entities): + raise ValueError("target must supply exactly the declared stable entity columns") + if any(value is None for value in target.values()): + raise ValueError("target identity cannot be null") + if len({timestamp, *entities, price}) != len(entities) + 2: + raise ValueError("Timestamp, entity and price roles must be distinct") + resolved_calendar = contract.resolve_calendar(calendar=calendar) + unit = ( + WindowUnit(window_unit) + if window_unit is not None + else (WindowUnit.OBSERVATION if resolved_calendar is None else WindowUnit.EXPECTED_BAR) + ) + expected_unit = WindowUnit.OBSERVATION if resolved_calendar is None else WindowUnit.EXPECTED_BAR + if unit != expected_unit: + raise ValueError(f"This finite-row statistic requires window_unit={expected_unit.value!r}") + schema = data.collect_schema() if isinstance(data, pl.LazyFrame) else data.schema + required = [timestamp, *entities, price, *([available_at_col] if available_at_col else [])] + if set(required) - set(schema): + raise ValueError(f"Missing input columns: {sorted(set(required) - set(schema))}") + if schema[timestamp].base_type() not in {pl.Date, pl.Datetime}: + raise ValueError("timestamp_col must contain Date labels or Datetime instants") + if not schema[price].is_numeric(): + raise ValueError("price_col must be numeric") + names = [ + f"{output}_{name}" + for name in ("asset_return", "target_return", method, "available_at", "price_basis") + ] + if set(names) & set(schema) or any(c.startswith("__ml4t_target_") for c in schema): + raise ValueError("Output or reserved preparation columns already exist") + result_schema = dict(schema) + result_schema.update(dict.fromkeys(names[:2], pl.Float64())) + result_schema[names[2]] = _REGRESSION_DTYPE if method == "regression" else pl.Float64() + result_schema[names[3]] = pl.Datetime("ns", "UTC") + result_schema[names[4]] = pl.String() + keys = [*entities, timestamp] + prefix = "__ml4t_target_" + quote, known, pair_known, row_id = (prefix + s for s in ("quote", "known", "pair", "row")) + + def calculate(frame: pl.DataFrame) -> pl.DataFrame: + if frame.is_empty(): + return frame.with_columns( + [pl.lit(None, dtype=result_schema[c]).alias(c) for c in names] + ) + if frame.select(pl.any_horizontal(pl.col(c).is_null() for c in keys).any()).item(): + raise ValueError("Timestamp and entity keys cannot be null") + if frame.select(pl.struct(keys).is_duplicated().any()).item(): + raise ValueError("Duplicate timestamp/entity keys") + predicate = pl.all_horizontal( + pl.col(c) == pl.lit(target[c], dtype=schema[c]) for c in entities + ) + if frame.filter(predicate).is_empty(): + raise ValueError("target identity is absent from supplied input") + valid_price = (pl.col(price) > 0) & pl.col(price).cast(pl.Float64).is_finite() + if missing_price_policy == "raise" and frame.filter(~valid_price.fill_null(False)).height: + raise ValueError("Prices must be finite, positive and nonmissing") + if resolved_calendar is not None: + validate_declared_grid( + frame, + feed_spec=contract, + calendar=resolved_calendar, + timestamp_col=timestamp, + group_cols=entities, + ) + known_expr = ( + _instant_ticks(available_at_col) if available_at_col else pl.lit(None, pl.Int128) + ) + prepared = frame.select( + *keys, pl.when(valid_price).then(pl.col(price)).alias(quote), known_expr.alias(known) + ) + if ( + available_at_col + and prepared.filter((pl.col(known) < -(2**63)) | (pl.col(known) > 2**63 - 1)).height + ): + raise ValueError("Publication instants exceed Datetime(ns) range") + if index_mode == "common_grid": + grid = ( + frame.select(entities).unique().join(frame.select(timestamp).unique(), how="cross") + ) + prepared = grid.join(prepared, on=keys, how="left", validate="1:1") + prepared = prepared.sort(keys) + lag = min(returns_periods, prepared.height) + return_known = pl.when( + pl.col(known).is_not_null() & pl.col(known).shift(lag).is_not_null() + ).then(pl.max_horizontal(pl.col(known), pl.col(known).shift(lag))) + prepared = prepared.with_columns( + returns(quote, lag).cast(pl.Float64).over(entities).alias(names[0]), + return_known.over(entities).alias(known), + ) + benchmark = prepared.filter(predicate).select( + timestamp, pl.col(names[0]).alias(names[1]), pl.col(known).alias(pair_known) + ) + prepared = prepared.join(benchmark, on=timestamp, how="left", validate="m:1") + prepared = prepared.with_columns( + pl.when(pl.col(known).is_not_null() & pl.col(pair_known).is_not_null()) + .then(pl.max_horizontal(known, pair_known)) + .cast(pl.Int64) + .alias(pair_known) + ) + statistic = methods[method](names[0], names[1], period, allowed_missing=allowed_missing) + warmup = returns_periods + period + availability = ( + pl.when(pl.col(pair_known).is_not_null().cast(pl.Int64).rolling_sum(period) == period) + .then(pl.col(pair_known).rolling_max(period)) + .cast(pl.Datetime("ns", "UTC")) + ) + prepared = prepared.with_columns( + _ready(statistic, warmup).over(entities).alias(names[2]), + _ready(availability, warmup).over(entities).alias(names[3]), + pl.lit(price_basis).alias(names[4]), + ) + return ( + frame.with_row_index(row_id) + .join(prepared.select(*keys, *names), on=keys, how="left", validate="1:1") + .sort(row_id) + .drop(row_id) + ) + + if isinstance(data, pl.LazyFrame): + return data.map_batches( + calculate, + schema=result_schema, + predicate_pushdown=False, + projection_pushdown=False, + slice_pushdown=False, + streamable=False, + ) + return calculate(data) + + def _validate(period: int, allowed_missing: int) -> None: _period(period, minimum=2) if ( diff --git a/tests/compat/test_zipline_return_factor_alignment.py b/tests/compat/test_zipline_return_factor_alignment.py new file mode 100644 index 00000000..31255043 --- /dev/null +++ b/tests/compat/test_zipline_return_factor_alignment.py @@ -0,0 +1,422 @@ +"""Actual price->target return->rolling factor preparation preserves keys and clocks.""" + +import importlib +from datetime import UTC, date, datetime, timedelta + +import numpy as np +import polars as pl +import pytest +from ml4t.specs import FeedSpec +from polars.testing import assert_frame_equal + +DAYS = [date(2025, 7, day) for day in (7, 8, 9, 10, 11, 14, 15, 16)] + + +def feed(**options) -> FeedSpec: + defaults = { + "time_contract_version": 2, + "calendar": None, + "entity_col": "asset", + "data_frequency": "1d", + "bar_type": "time", + "timestamp_semantics": "session_label", + } + return FeedSpec(**{**defaults, **options}) + + +def run(frame, **options): + function = importlib.import_module("ml4t.engineer.features.paired").target_return_features + defaults = { + "target": {"asset": "B"}, + "period": 3, + "method": "beta", + "price_basis": "raw", + "feed_spec": feed(), + "output": "market", + } + return function(frame, **{**defaults, **options}) + + +def panel() -> pl.DataFrame: + return pl.DataFrame( + { + "timestamp": DAYS[:4] * 2, + "asset": ["A"] * 4 + ["B"] * 4, + "close": [100.0, 120.0, 80.0, 112.0, 100.0, 110.0, 99.0, 118.8], + } + ) + + +@pytest.mark.parametrize("lazy", [False, True]) +def test_actual_price_pipeline_preserves_exact_target_keys_and_independent_beta(lazy: bool) -> None: + frame = panel().sort("timestamp", "asset", descending=[False, True]) + result = run(frame.lazy() if lazy else frame) + assert isinstance(result, pl.LazyFrame if lazy else pl.DataFrame) + if lazy: + result = result.collect() + assert_frame_equal(result.select(frame.columns), frame) + a = result.filter(pl.col("asset") == "A") + np.testing.assert_allclose( + a["market_asset_return"].to_numpy(), + [np.nan, 0.2, -1 / 3, 0.4], + rtol=1e-14, + atol=0, + equal_nan=True, + ) + np.testing.assert_allclose( + a["market_target_return"].to_numpy(), + [np.nan, 0.1, -0.1, 0.2], + rtol=1e-14, + atol=0, + equal_nan=True, + ) + assert a["market_beta"][-1] == pytest.approx(52 / 21, rel=1e-13) + assert result.filter(pl.col("asset") == "B")["market_beta"][-1] == pytest.approx(1.0, rel=1e-13) + assert result["market_available_at"].null_count() == result.height + assert result["market_price_basis"].unique().to_list() == ["raw"] + + +def test_common_clock_is_not_compressed_before_missing_tolerant_statistics() -> None: + close = [100.0, 110.0, 121.0, 108.9, 130.68, 143.748, 129.3732, 155.24784] + b = pl.DataFrame({"timestamp": DAYS, "asset": "B", "close": close}) + a = b.filter(~pl.col("timestamp").is_in(DAYS[3:5])).with_columns(pl.lit("A").alias("asset")) + frame = pl.concat([a, b]).sort("timestamp", "asset") + common = run(frame, period=4, allowed_missing=1, index_mode="common_grid") + per_asset = run(frame, period=4, allowed_missing=1, index_mode="per_asset") + at = (pl.col("asset") == "A") & (pl.col("timestamp") == DAYS[6]) + value = common.filter(at)["market_beta"][0] + assert value is None or np.isnan(value) + assert np.isfinite(per_asset.filter(at)["market_beta"][0]) + missing_exit = common.filter((pl.col("asset") == "A") & (pl.col("timestamp") == DAYS[5])) + assert missing_exit["market_asset_return"][0] is None + assert per_asset.filter((pl.col("asset") == "A") & (pl.col("timestamp") == DAYS[5]))[ + "market_asset_return" + ][0] == pytest.approx(0.188, rel=1e-14) + assert common.height == frame.height + + +def test_named_grid_fails_before_a_gap_can_become_a_one_bar_return() -> None: + frame = panel().filter(pl.col("timestamp") != DAYS[1]) + before = frame.clone() + with pytest.raises(ValueError, match="missing scheduled session"): + run(frame, calendar="NYSE") + assert_frame_equal(frame, before) + + +def test_feature_availability_uses_both_assets_and_the_actual_price_dependencies() -> None: + frame = ( + panel() + .with_columns( + (pl.col("timestamp").cast(pl.Datetime("ns", "UTC")) + timedelta(hours=16)).alias( + "known" + ) + ) + .with_columns( + pl.when((pl.col("asset") == "B") & (pl.col("timestamp") == DAYS[3])) + .then(pl.col("known") + timedelta(hours=4)) + .otherwise(pl.col("known")) + .alias("known") + ) + ) + result = run(frame, available_at_col="known") + assert result.filter(pl.col("asset") == "A")["market_available_at"][-1] == datetime( + 2025, 7, 10, 20, tzinfo=UTC + ) + earlier = run(frame.filter(pl.col("timestamp") <= DAYS[2]), method="pearson") + assert earlier["market_available_at"].null_count() == earlier.height + + +def test_custom_roles_compound_identity_and_regression_fields_survive_the_pipeline() -> None: + frame = ( + panel() + .with_columns( + pl.struct(pl.lit("bundle").alias("namespace"), pl.col("asset").alias("sid")).alias( + "asset" + ) + ) + .rename({"timestamp": "when", "asset": "identity", "close": "quote"}) + ) + contract = FeedSpec( + time_contract_version=2, + calendar=None, + entity_col="identity", + timestamp_col="when", + price_col="quote", + timestamp_semantics="session_label", + data_frequency="1d", + ) + output = run( + frame, + feed_spec=contract, + target={"identity": {"namespace": "bundle", "sid": "B"}}, + method="regression", + ) + assert output.schema["identity"] == frame.schema["identity"] + row = output.filter(pl.col("identity").struct.field("sid") == "A")["market_regression"][-1] + assert row["beta"] == pytest.approx(52 / 21, rel=1e-13) + assert row["alpha"] == pytest.approx(-8 / 105, rel=1e-13) + + +@pytest.mark.parametrize( + "options", + [ + {"target": {"asset": "missing"}}, + {"target": {}}, + {"period": True}, + {"method": "time_trend"}, + {"price_basis": ""}, + {"calendar": "NYSE", "feed_spec": FeedSpec()}, + {"entity_col": "another"}, + {"timestamp_col": "another"}, + {"price_col": "another"}, + {"index_mode": "ffill"}, + {"window_unit": "elapsed"}, + ], +) +def test_target_contract_errors_are_atomic(options) -> None: + frame = panel() + before = frame.clone() + with pytest.raises((TypeError, ValueError)): + run(frame, **options) + assert_frame_equal(frame, before) + + +@pytest.mark.parametrize("method", ["pearson", "spearman", "beta", "regression"]) +def test_empty_frame_retains_declared_added_schema(method: str) -> None: + for frame in (panel().clear(), panel().clear().lazy()): + result = run(frame, method=method) + result = result.collect() if isinstance(result, pl.LazyFrame) else result + assert result.height == 0 + assert result.schema["market_available_at"] == pl.Datetime("ns", "UTC") + assert result.schema["market_" + method].base_type() == ( + pl.Struct if method == "regression" else pl.Float64 + ) + + +def test_appended_future_prices_do_not_change_prior_prepared_values() -> None: + before = panel() + later = pl.DataFrame({"timestamp": [DAYS[4]] * 2, "asset": ["A", "B"], "close": [1e9, 0.001]}) + expected = run(before) + actual = run(pl.concat([before, later])).filter(pl.col("timestamp") <= DAYS[3]) + assert_frame_equal(actual, expected) + # Lazy filtering must not remove warmup or the target before preparation. + selected = ( + run(before.lazy()) + .filter((pl.col("asset") == "A") & (pl.col("timestamp") == DAYS[3])) + .collect() + ) + assert selected["market_beta"][0] == pytest.approx(52 / 21) + + +def test_adjacent_uint64_ticks_survive_actual_price_to_factor_preparation() -> None: + frame = panel().with_columns( + pl.Series("close", [2**63 + v for v in (0, 2, 6, 12, 0, 1, 3, 6)], dtype=pl.UInt64) + ) + result = run(frame) + a = result.filter(pl.col("asset") == "A") + np.testing.assert_allclose( + a["market_asset_return"].tail(3), [2 / 2**63, 4 / (2**63 + 2), 6 / (2**63 + 6)], rtol=1e-14 + ) + assert a["market_beta"][-1] == pytest.approx(2.0, rel=1e-13) + + +@pytest.mark.parametrize("bad", [None, float("nan"), float("inf"), 0.0, -1.0]) +def test_invalid_supplied_prices_raise_or_become_explicit_missing_pairs(bad) -> None: + frame = panel().with_columns( + pl.when((pl.col("asset") == "A") & (pl.col("timestamp") == DAYS[1])) + .then(pl.lit(bad)) + .otherwise(pl.col("close")) + .alias("close") + ) + with pytest.raises(ValueError, match="finite, positive"): + run(frame) + actual = run(frame, missing_price_policy="null") + a = actual.filter(pl.col("asset") == "A") + assert a["market_asset_return"][1] is None + assert a["market_asset_return"][2] is None + assert not np.isfinite(a["market_beta"][-1]) if a["market_beta"][-1] is not None else True + + +def test_multicolumn_identity_does_not_join_reused_tickers_across_namespaces() -> None: + frame = pl.concat( + [ + panel().with_columns(pl.lit("one").alias("venue")), + panel().with_columns(pl.lit("two").alias("venue")), + ] + ) + contract = FeedSpec(time_contract_version=2, calendar=None, entity_col=["venue", "asset"]) + output = run(frame, feed_spec=contract, target={"venue": "one", "asset": "B"}) + assert_frame_equal(output.select(frame.columns), frame) + assert output.filter((pl.col("asset") == "A") & (pl.col("venue") == "two"))["market_beta"][ + -1 + ] == pytest.approx(52 / 21) + + +@pytest.mark.parametrize( + "options", + [ + {"returns_periods": 0}, + {"period": 2**64}, + {"returns_periods": 2**64}, + {"output": ""}, + {"missing_price_policy": "ffill"}, + {"entity_col": []}, + {"target": {"asset": None}}, + {"target": {"asset": "B", "other": 1}}, + {"window_unit": "expected_bar"}, + {"allowed_missing": -1}, + {"feed_spec": None, "entity_col": []}, + {"feed_spec": None, "price_col": "timestamp"}, + ], +) +def test_invalid_preparation_options_fail_before_mutating_input(options) -> None: + before = panel() + with pytest.raises((TypeError, ValueError)): + run(before, **options) + assert_frame_equal(before, panel()) + + +@pytest.mark.parametrize( + "kind", + [ + "duplicate", + "null_key", + "missing_column", + "string_time", + "string_price", + "collision", + "internal", + ], +) +def test_invalid_physical_frames_fail_with_explicit_errors(kind: str) -> None: + frame = panel() + if kind == "duplicate": + frame = pl.concat([frame, frame.head(1)]) + elif kind == "null_key": + frame = frame.with_columns(pl.lit(None).alias("asset")) + elif kind == "missing_column": + frame = frame.drop("close") + elif kind == "string_time": + frame = frame.with_columns(pl.col("timestamp").cast(pl.String)) + elif kind == "string_price": + frame = frame.with_columns(pl.col("close").cast(pl.String)) + else: + frame = frame.with_columns( + pl.lit(1).alias("market_beta" if kind == "collision" else "__ml4t_target_quote") + ) + with pytest.raises(ValueError): + run(frame) + + +def test_unknown_and_out_of_range_publication_instants_are_not_fabricated_or_wrapped() -> None: + frame = panel().with_columns(pl.lit(None, pl.Datetime("ns", "UTC")).alias("known")) + assert run(frame, available_at_col="known")["market_available_at"].null_count() == frame.height + naive = frame.with_columns(pl.lit(datetime(2025, 7, 7)).alias("known")) + with pytest.raises(ValueError, match="timezone-aware"): + run(naive, available_at_col="known") + wide = frame.with_columns(pl.lit(datetime(2500, 1, 1, tzinfo=UTC)).alias("known")) + with pytest.raises(ValueError, match="range"): + run(wide, available_at_col="known") + + +def test_named_calendar_and_explicit_none_precedence_reuse_the_actual_grid_validator() -> None: + frame = panel() + contract = feed(calendar="NYSE") + actual = run(frame, feed_spec=contract) + assert actual["market_beta"][-1] == pytest.approx(1.0) + sparse = frame.filter(pl.col("timestamp") != DAYS[1]) + with pytest.raises(ValueError, match="missing scheduled session"): + run(sparse, feed_spec=contract) + result = run(sparse, feed_spec=contract, calendar=None) + assert result.height == sparse.height + result = run(frame, feed_spec=None, calendar=None, entity_col="asset") + assert result.filter(pl.col("asset") == "A")["market_beta"][-1] == pytest.approx(52 / 21) + + +@pytest.mark.parametrize( + "name,method,length", + [ + ("RollingPearsonOfReturns", "pearson", "correlation_length"), + ("RollingSpearmanOfReturns", "spearman", "correlation_length"), + ("RollingLinearRegressionOfReturns", "regression", "regression_length"), + ("SimpleBeta", "beta", "regression_length"), + ], +) +@pytest.mark.parametrize("lazy", [False, True]) +def test_return_factor_recipe_uses_native_keyed_preparation_without_masking_target( + name, method, length, lazy +) -> None: + function = importlib.import_module("ml4t.engineer.compat.zipline").return_factor_features + frame = panel() + output = function( + frame.lazy() if lazy else frame, + name, + target={"asset": "B"}, + price_basis="raw", + feed_spec=feed(), + output="market", + mask=pl.col("asset") == "A", + **{length: 3}, + ) + assert isinstance(output, pl.LazyFrame if lazy else pl.DataFrame) + output = output.collect() if lazy else output + expected = run(frame, method=method) + assert_frame_equal( + output.filter(pl.col("asset") == "A"), expected.filter(pl.col("asset") == "A") + ) + assert output.filter(pl.col("asset") == "B")["market_" + method].null_count() == 4 + assert_frame_equal(output.select(frame.columns), frame) + + +def test_return_recipe_translates_price_lookback_and_retains_structural_warmup() -> None: + function = importlib.import_module("ml4t.engineer.compat.zipline").return_factor_features + close = [100.0, 110.0, 121.0, 108.9, 130.68, 143.748, 129.3732, 155.24784] + frame = pl.DataFrame( + {"timestamp": DAYS * 2, "asset": ["A"] * 8 + ["B"] * 8, "close": close + close} + ) + result = function( + frame, + "RollingPearsonOfReturns", + target={"asset": "B"}, + returns_length=3, + correlation_length=3, + price_basis="raw", + feed_spec=feed(), + output="market", + ) + a = result.filter(pl.col("asset") == "A") + assert a["market_asset_return"][2] == pytest.approx(0.21) + assert a["market_pearson"].head(4).is_null().all() + np.testing.assert_allclose(a["market_pearson"].tail(4), 1.0, atol=1e-14) + beta = function( + frame, + "SimpleBeta", + target={"asset": "B"}, + regression_length=4, + allowed_missing_percentage=1.0, + price_basis="raw", + feed_spec=feed(), + ) + assert beta.filter(pl.col("asset") == "A")["target_beta"].head(4).is_null().all() + + +@pytest.mark.parametrize( + "name,options", + [ + ("unknown", {}), + ("SimpleBeta", {"regression_length": 3, "returns_length": 3}), + ("SimpleBeta", {"regression_length": 2}), + ("SimpleBeta", {"regression_length": 3, "allowed_missing_percentage": -1}), + ("RollingPearsonOfReturns", {"correlation_length": 3, "returns_length": 1}), + ("RollingPearsonOfReturns", {"regression_length": 3}), + ("RollingSpearmanOfReturns", {"correlation_length": 3, "allowed_missing_percentage": 0.2}), + ("RollingLinearRegressionOfReturns", {"regression_length": 3, "correlation_length": 3}), + ("SimpleBeta", {"regression_length": 3, "mask": 17}), + ], +) +def test_invalid_legacy_return_options_fail_explicitly(name, options) -> None: + function = importlib.import_module("ml4t.engineer.compat.zipline").return_factor_features + with pytest.raises((TypeError, ValueError)): + function( + panel(), name, target={"asset": "B"}, price_basis="raw", feed_spec=feed(), **options + ) From 49abc733b33a10ec4ef8d43efbd2f60dbe8f8bf1 Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Tue, 20 Oct 2026 20:03:50 -0400 Subject: [PATCH 09/19] feat(features): add eligible cross-sectional selectors and boolean windows --- docs/user-guide/features.md | 49 ++++ src/ml4t/engineer/features/AGENTS.md | 1 + src/ml4t/engineer/features/__init__.py | 2 + src/ml4t/engineer/features/selection.py | 193 +++++++++++++++ tests/compat/test_zipline_selection.py | 306 ++++++++++++++++++++++++ 5 files changed, 551 insertions(+) create mode 100644 src/ml4t/engineer/features/selection.py create mode 100644 tests/compat/test_zipline_selection.py diff --git a/docs/user-guide/features.md b/docs/user-guide/features.md index 18933186..e8a9d70c 100644 --- a/docs/user-guide/features.md +++ b/docs/user-guide/features.md @@ -227,6 +227,55 @@ explicitly choose `calendar=None` for observation rows. Recipe output masks are dated universe/knowledge selection or final output screens. Named fields stay explicit; for example, select `factor_expression("BollingerBands", window_length=20)["middle"]`. +### Eligible cross-sectional transforms and boolean windows + +`features.selection.cross_section` computes `rank`, `top`, `bottom`, `demean`, +`zscore`, `winsorize`, `quantiles`, `percentile_between` or `peer_count` separately +at each supplied timestamp and optional classification group. Its `mask` selects +the input population before the transform. A later dataframe filter is a final +screen and does not change those inputs. Unknown groups, nulls and NaNs are +excluded. Map legacy classifier sentinels such as -1 to null explicitly. + +Ranks support ordinal/min/max/dense/average ties. Ordinal ties and top/bottom +selection use ascending stable entity IDs, independently of input row order. +Supply unique timestamp/entity keys; display symbols are not stable identities. +Z-scores use population standard deviation by default (`ddof=0`), not a fitted +training scaler. Constant groups have undefined z-scores. Typed local differences +retain adjacent large integer ticks, and scaled moments avoid variance overflow +or underflow for representable outputs. + + +```python +from datetime import date +import polars as pl +from ml4t.engineer.features.selection import cross_section + +signals = pl.DataFrame({ + "timestamp": [date(2025, 7, 7)] * 4, "asset": ["D", "B", "A", "C"], + "value": [3., 1., 1., 999.], "eligible": [True, True, True, False], +}) +ranked = signals.with_columns( + cross_section("value", method="rank", mask="eligible").alias("rank"), + cross_section("value", method="bottom", n=1, mask="eligible").alias("selected"), +) +assert ranked["rank"].to_list() == [3., 2., 1., None] +assert ranked.filter("selected")["asset"].to_list() == ["A"] +``` + +Winsorization uses the legacy floor/ceil order-statistic cutoffs, not linear +percentile clipping. Quantile bins and inclusive percentile filters use linear +edges. Duplicate quantile edges raise instead of silently changing the number of +bins. Native quantile labels are zero-based integers with null missing values. +These feature selectors do not invoke Diagnostic's analytical factor cleaner. + +`features.selection.rolling_filter` supplies full-row `any`, `all` and `at_least` +windows. Apply `.over(entity_col)` to isolate histories. Missing predicates count +false; leading incomplete windows are null. `is_missing` recognizes native null +and floating NaN, without treating infinity or a classifier sentinel as missing. +Ordinary Polars arithmetic, comparisons, boolean operators and `is_in` cover the +corresponding elementwise factor/filter/classifier operations. None of these +expressions selects Data revisions or establishes availability by itself. + ### Paired relationships and supplied event dates `features.paired.rolling_pearson`, `rolling_spearman`, `rolling_beta` and diff --git a/src/ml4t/engineer/features/AGENTS.md b/src/ml4t/engineer/features/AGENTS.md index e821542e..79582537 100644 --- a/src/ml4t/engineer/features/AGENTS.md +++ b/src/ml4t/engineer/features/AGENTS.md @@ -28,6 +28,7 @@ These are not registry entries but remain important: | `cross_asset.py` | multi-asset and panel features such as rolling correlation and beta | | `paired.py` | full-window paired correlation/beta/regression with explicit missing-count limits | | `events.py` | supplied known event weekday counts, separate from exchange-session counts | +| `selection.py` | eligible contemporary group transforms, stable selectors and boolean windows | | `fdiff.py` | fractional differencing utilities such as `ffdiff()` and `find_optimal_d()` | | `composite.py` | composite indicators and z-score combinations | diff --git a/src/ml4t/engineer/features/__init__.py b/src/ml4t/engineer/features/__init__.py index 94b6876f..72e62de4 100644 --- a/src/ml4t/engineer/features/__init__.py +++ b/src/ml4t/engineer/features/__init__.py @@ -40,6 +40,7 @@ regime, risk, rolling, + selection, statistics, # noqa: F401 trend, # noqa: F401 volatility, # noqa: F401 @@ -56,4 +57,5 @@ "regime", "risk", "rolling", + "selection", ] diff --git a/src/ml4t/engineer/features/selection.py b/src/ml4t/engineer/features/selection.py new file mode 100644 index 00000000..a78077af --- /dev/null +++ b/src/ml4t/engineer/features/selection.py @@ -0,0 +1,193 @@ +"""Contemporary eligible-group transforms and full-window boolean filters.""" + +import math + +import numpy as np +import polars as pl + +from ml4t.engineer.features.rolling import _paired_difference, _period + + +def _value(value: str | pl.Expr) -> pl.Expr: + return pl.col(value) if isinstance(value, str) else value + + +def is_missing(value: str | pl.Expr) -> pl.Expr: + """True for native null or floating NaN, without sentinel or infinity inference.""" + + def build(expression: pl.Expr, dtype: pl.DataType) -> pl.Expr: + return ( + expression.is_null() | expression.is_nan().fill_null(False) + if dtype.is_float() + else expression.is_null() + ) + + return _value(value).pipe_with_dtype(build) + + +def cross_section( + value: str | pl.Expr, + *, + method: str, + timestamp_col: str = "timestamp", + entity_col: str | list[str] = "asset", + groupby: str | list[str] | None = None, + mask: str | pl.Expr | None = None, + ascending: bool = True, + ties: str = "ordinal", + n: int | None = None, + bins: int | None = None, + min_percentile: float = 0.0, + max_percentile: float = 1.0, + ddof: int = 0, +) -> pl.Expr: + """Transform the eligible contemporary cross-section, independently per group. + + mask determines the input population, not a final screen. Null group labels, + null values and NaNs are excluded; legacy classifier sentinels must be mapped + explicitly. Ordinal ties use ascending stable entity IDs, not input row order. + Input keys must identify at most one observation per timestamp/entity. This + expression does not select source revisions or establish causal availability. + + Winsorization uses floor(n*lower)/ceil(n*upper)-1 order statistics. Quantile + edges and percentile bounds use linear interpolation. Duplicate quantile edges + raise rather than silently reducing bins. Percentile bounds are inclusive. + """ + methods = { + "rank", + "top", + "bottom", + "demean", + "zscore", + "winsorize", + "quantiles", + "percentile_between", + "peer_count", + } + if method not in methods: + raise ValueError(f"method must be one of {sorted(methods)}") + if not isinstance(ascending, bool) or ties not in {"ordinal", "min", "max", "dense", "average"}: + raise ValueError("Declare boolean ascending and a supported tie method") + if ties != "ordinal" and method != "rank": + raise ValueError("Only rank accepts a nonordinal tie method") + for name, count, applicable, minimum in ( + ("n", n, method in {"top", "bottom"}, 0), + ("bins", bins, method == "quantiles", 1), + ): + if applicable: + if isinstance(count, bool) or not isinstance(count, int) or count < minimum: + raise ValueError(f"{name} must be an integer >= {minimum}") + elif count is not None: + raise ValueError(f"{name} is not applicable to {method}") + if isinstance(ddof, bool) or not isinstance(ddof, int) or ddof < 0: + raise ValueError("ddof must be a nonnegative integer") + if ddof and method != "zscore": + raise ValueError("ddof applies only to zscore") + if ( + any( + isinstance(v, bool) or not isinstance(v, int | float) or not math.isfinite(v) + for v in (min_percentile, max_percentile) + ) + or not 0 <= min_percentile < max_percentile <= 1 + ): + raise ValueError("Declare 0 <= min_percentile < max_percentile <= 1") + if (min_percentile, max_percentile) != (0.0, 1.0) and method not in { + "winsorize", + "percentile_between", + }: + raise ValueError("Percentile bounds do not apply to this method") + entities = [entity_col] if isinstance(entity_col, str) else list(entity_col) + groups = [] if groupby is None else [groupby] if isinstance(groupby, str) else list(groupby) + if not entities or any( + not isinstance(c, str) or not c for c in [timestamp_col, *entities, *groups] + ): + raise ValueError("Declare nonempty timestamp, entity and grouping column names") + if mask is not None and not isinstance(mask, str | pl.Expr): + raise TypeError("mask must be a column name or expression") + original = _value(value) + eligible = ~is_missing(original) + if groups: + eligible &= pl.all_horizontal(pl.col(c).is_not_null() for c in groups) + if mask is not None: + eligible &= _value(mask).fill_null(False) + x = pl.when(eligible).then(original) + partition = list(dict.fromkeys([timestamp_col, *groups])) + if method == "peer_count": + return pl.when(eligible).then(x.count().over([*partition, x]).cast(pl.Float64)) + if method in {"rank", "top", "bottom"}: + descending = method == "top" or (method == "rank" and not ascending) + rank = x.rank(method=ties, descending=descending).over(partition, order_by=entities) + return rank.cast(pl.Float64) if method == "rank" else (rank <= n).fill_null(False) + centered = _paired_difference(x, x.drop_nulls().first()) + if method in {"demean", "zscore"}: + # Retain typed ticks near a large offset, then scale residuals before + # moments. Opposite extreme floats need scaling before subtraction too. + fallback = centered.abs().max().is_infinite().fill_null(False) + raw_scale = x.abs().max().cast(pl.Float64) + working = pl.when(fallback).then(x / raw_scale).otherwise(centered) + scale = working.abs().max() + normalized = working / pl.when(scale > 0).then(scale).otherwise(1.0) + result = normalized - normalized.mean() + if method == "zscore": + result = result / normalized.std(ddof=ddof) + else: + result = result * scale * pl.when(fallback).then(raw_scale).otherwise(1.0) + elif method == "winsorize": + ordered = x.drop_nulls().sort() + lower = ordered.get((x.count() * min_percentile).floor().cast(pl.Int64), null_on_oob=True) + upper = ordered.get( + (x.count() * max_percentile).ceil().cast(pl.Int64) - 1, null_on_oob=True + ) + result = x.clip(lower, upper) + elif method == "quantiles": + assert bins is not None + + def quantile_labels(series: pl.Series) -> pl.Series: + values = series.to_numpy() + valid = ~np.isnan(values) + if not valid.any(): + return pl.Series([None] * len(values), dtype=pl.Int64) + edges = np.quantile(values[valid], np.linspace(0.0, 1.0, bins + 1), method="linear") + if bins > 1 and len(np.unique(edges)) != len(edges): + raise ValueError("Duplicate quantile edges; declare a different bin count") + labels = np.searchsorted(edges, values, side="left") - 1 + labels[values == edges[0]] = 0 + return pl.Series(labels, dtype=pl.Int64).set(pl.Series(~valid), None) + + result = centered.map_batches(quantile_labels, return_dtype=pl.Int64) + else: + lower = centered.quantile(min_percentile, interpolation="linear") + upper = centered.quantile(max_percentile, interpolation="linear") + result = centered.is_between(lower, upper).fill_null(False) + return result.over(partition, order_by=entities) + + +def rolling_filter( + value: str | pl.Expr, + *, + period: int, + method: str, + minimum: int | None = None, +) -> pl.Expr: + """Any/all/at_least over full observation rows; missing predicates count false. + + Apply over(entity_col) to isolate histories. Leading incomplete windows are + null. Keep input eligibility and final screens separate from this history. + """ + _period(period) + if method not in {"any", "all", "at_least"}: + raise ValueError("method must be any, all or at_least") + if method == "at_least": + if isinstance(minimum, bool) or not isinstance(minimum, int) or not 0 <= minimum <= period: + raise ValueError("minimum must be an integer in [0,period]") + elif minimum is not None: + raise ValueError("minimum applies only to at_least") + + def build(expression: pl.Expr, dtype: pl.DataType) -> pl.Expr: + if dtype != pl.Boolean: + raise ValueError("rolling_filter requires boolean predicates") + return expression.fill_null(False).cast(pl.Int64) + + count = _value(value).pipe_with_dtype(build).rolling_sum(period) + threshold = 1 if method == "any" else period if method == "all" else minimum + return count >= threshold diff --git a/tests/compat/test_zipline_selection.py b/tests/compat/test_zipline_selection.py new file mode 100644 index 00000000..02d9a051 --- /dev/null +++ b/tests/compat/test_zipline_selection.py @@ -0,0 +1,306 @@ +"""Dated eligible cross-sections preserve stable ties and original missingness.""" + +import importlib +from datetime import date + +import numpy as np +import polars as pl +import pytest +from polars.testing import assert_frame_equal + + +def selection(): + return importlib.import_module("ml4t.engineer.features.selection") + + +def panel(): + return pl.DataFrame( + { + "timestamp": [date(2025, 7, 7)] * 6, + "asset": ["D", "B", "A", "C", "E", "F"], + "value": [3.0, 1.0, 1.0, 2.0, float("nan"), 999.0], + "sector": ["x", "x", "x", "y", "x", "x"], + "eligible": [True] * 5 + [False], + } + ) + + +def calculate(frame, method, **options): + return frame.with_columns( + selection() + .cross_section("value", method=method, groupby="sector", mask="eligible", **options) + .alias("result") + ) + + +@pytest.mark.parametrize("lazy", [False, True]) +@pytest.mark.parametrize( + "ties,expected", + [ + ("ordinal", [3.0, 2.0, 1.0, 1.0]), + ("min", [3.0, 1.0, 1.0, 1.0]), + ("max", [3.0, 2.0, 2.0, 1.0]), + ("dense", [2.0, 1.0, 1.0, 1.0]), + ("average", [3.0, 1.5, 1.5, 1.0]), + ], +) +def test_rank_ties_use_stable_ids_inside_the_actual_eligible_group(lazy, ties, expected): + frame = panel() + actual = calculate(frame.lazy() if lazy else frame, "rank", ties=ties) + assert isinstance(actual, pl.LazyFrame if lazy else pl.DataFrame) + actual = actual.collect() if lazy else actual + np.testing.assert_allclose(actual["result"].head(4), expected) + assert actual["result"].tail(2).null_count() == 2 + assert_frame_equal(actual.select(frame.columns), frame) + shuffled = calculate(frame.reverse(), "rank", ties=ties).sort("asset") + assert_frame_equal(shuffled, actual.sort("asset")) + + +@pytest.mark.parametrize( + "method,expected", + [ + ("demean", [4 / 3, -2 / 3, -2 / 3, 0.0]), + ("zscore", [np.sqrt(2), -1 / np.sqrt(2), -1 / np.sqrt(2), np.nan]), + ], +) +def test_group_transforms_use_only_contemporary_eligible_values(method, expected): + output = calculate(panel(), method) + np.testing.assert_allclose(output["result"].head(4), expected, rtol=1e-14, equal_nan=True) + assert output["result"].tail(2).null_count() == 2 + future = panel().with_columns( + pl.lit(date(2025, 7, 8)).alias("timestamp"), (pl.col("value") * 100).alias("value") + ) + combined = calculate(pl.concat([panel(), future]), method).head(6) + assert_frame_equal(output, combined) + + +@pytest.mark.parametrize( + "method,expected", + [ + ("top", [True, False, False, True, False, False]), + ("bottom", [False, False, True, True, False, False]), + ], +) +def test_top_bottom_use_exact_cardinality_and_stable_tie_selection(method, expected): + output = calculate(panel(), method, n=1) + assert output["result"].to_list() == expected + assert calculate(panel(), method, n=0)["result"].to_list() == [False] * 6 + + +def test_winsorization_uses_legacy_order_statistic_cutoffs_not_linear_clipping(): + frame = pl.DataFrame( + { + "timestamp": [date(2025, 7, 7)] * 5, + "asset": list("ABCDE"), + "value": [0.0, 1.0, 2.0, 100.0, np.nan], + } + ) + result = frame.select( + selection().cross_section( + "value", method="winsorize", min_percentile=0.25, max_percentile=0.75 + ) + ) + np.testing.assert_allclose(result.to_series(), [1.0, 1.0, 2.0, 2.0, np.nan], equal_nan=True) + + +def test_quantiles_use_linear_edges_and_percentile_filter_includes_both_bounds(): + frame = pl.DataFrame( + { + "timestamp": [date(2025, 7, 7)] * 6, + "asset": list("ABCDEF"), + "value": [0.0, 1.0, 2.0, 3.0, np.nan, None], + } + ) + result = frame.with_columns( + selection().cross_section("value", method="quantiles", bins=2).alias("quantile"), + selection() + .cross_section( + "value", method="percentile_between", min_percentile=0.25, max_percentile=0.75 + ) + .alias("between"), + ) + assert result["quantile"].to_list() == [0, 0, 1, 1, None, None] + assert result["between"].to_list() == [False, True, True, False, False, False] + tied = frame.with_columns(pl.lit(1.0).alias("value")) + with pytest.raises((ValueError, pl.exceptions.DuplicateError)): + tied.select(selection().cross_section("value", method="quantiles", bins=4)) + + +def test_peer_counts_keep_supplied_classifier_nulls_and_exclude_ineligible_members(): + frame = panel().with_columns( + pl.when(pl.col("asset") == "E").then(None).otherwise(pl.col("sector")).alias("sector") + ) + output = frame.select(selection().cross_section("sector", method="peer_count", mask="eligible")) + np.testing.assert_allclose( + output.to_series(), [3.0, 3.0, 3.0, 1.0, np.nan, np.nan], equal_nan=True + ) + + +@pytest.mark.parametrize("method", ["demean", "zscore"]) +def test_adjacent_large_integer_cross_sections_do_not_lose_tick_dispersion(method): + frame = pl.DataFrame( + { + "timestamp": [date(2025, 7, 7)] * 3, + "asset": list("ABC"), + "value": pl.Series([2**63, 2**63 + 1, 2**63 + 2], dtype=pl.UInt64), + } + ) + result = frame.select(selection().cross_section("value", method=method)).to_series() + expected = [-1.0, 0.0, 1.0] if method == "demean" else [-np.sqrt(1.5), 0.0, np.sqrt(1.5)] + np.testing.assert_allclose(result, expected, rtol=1e-14, atol=0) + + +@pytest.mark.parametrize("method", ["any", "all", "at_least"]) +def test_rolling_boolean_filters_keep_full_window_and_explicit_missing_false(method): + frame = pl.DataFrame({"value": [True, None, False, True, True]}) + kwargs = {"minimum": 2} if method == "at_least" else {} + result = frame.select( + selection().rolling_filter("value", period=3, method=method, **kwargs) + ).to_series() + expected = ( + [None, None, True, True, True] + if method == "any" + else ( + [None, None, False, False, False] + if method == "all" + else [None, None, False, False, True] + ) + ) + assert result.to_list() == expected + + +@pytest.mark.parametrize( + "options", + [ + {"method": "unknown"}, + {"method": "rank", "ties": "random"}, + {"method": "top", "n": -1}, + {"method": "top", "n": True}, + {"method": "quantiles", "bins": 0}, + {"method": "quantiles", "bins": True}, + {"method": "winsorize", "min_percentile": 0.8, "max_percentile": 0.2}, + ], +) +def test_invalid_selection_contracts_fail_before_output(options): + with pytest.raises((ValueError, TypeError)): + panel().select(selection().cross_section("value", **options)) + + +@pytest.mark.parametrize("scale", [np.ldexp(1.0, -1000), np.ldexp(1.0, 1000)]) +def test_representable_group_zscores_do_not_underflow_or_overflow_in_variance(scale): + frame = pl.DataFrame( + { + "timestamp": [date(2025, 7, 7)] * 3, + "asset": list("ABC"), + "value": [scale, 2 * scale, 3 * scale], + } + ) + actual = frame.select(selection().cross_section("value", method="zscore")).to_series() + np.testing.assert_allclose(actual, [-np.sqrt(1.5), 0.0, np.sqrt(1.5)], rtol=1e-14, atol=0) + + +@pytest.mark.parametrize( + "method,expected", + [ + ("demean", [-1e308, 1e308]), + ("zscore", [-1.0, 1.0]), + ], +) +def test_opposite_extreme_group_values_do_not_overflow_a_representable_transform(method, expected): + frame = pl.DataFrame( + {"timestamp": [date(2025, 7, 7)] * 2, "asset": list("AB"), "value": [-1e308, 1e308]} + ) + actual = frame.select(selection().cross_section("value", method=method)).to_series() + np.testing.assert_allclose(actual, expected, rtol=1e-14, atol=0) + + +@pytest.mark.parametrize( + "dtype,values,expected", + [ + (pl.Float64, [1.0, np.nan, None, np.inf], [False, True, True, False]), + (pl.Int64, [1, -1, None, 0], [False, False, True, False]), + (pl.String, ["a", "", None, "-1"], [False, False, True, False]), + ], +) +def test_native_missing_predicate_does_not_invent_classifier_sentinels(dtype, values, expected): + actual = pl.DataFrame({"x": pl.Series(values, dtype=dtype)}).select(selection().is_missing("x")) + assert actual.to_series().to_list() == expected + + +@pytest.mark.parametrize( + "method", + ["rank", "demean", "zscore", "winsorize", "quantiles", "percentile_between", "peer_count"], +) +def test_empty_all_missing_and_singleton_cross_sections_keep_defined_shapes(method): + kwargs = {"bins": 1} if method == "quantiles" else {} + frame = pl.DataFrame( + {"timestamp": [date(2025, 7, 7)], "asset": ["A"], "value": [None]}, + schema_overrides={"value": pl.Float64}, + ) + for source in (frame, frame.clear()): + result = source.select(selection().cross_section("value", method=method, **kwargs)) + assert result.height == source.height + if source.height: + assert ( + result.item() is None if method != "percentile_between" else result.item() is False + ) + present = frame.with_columns(pl.lit(3.0).alias("value")) + result = present.select(selection().cross_section("value", method=method, **kwargs)).item() + if method == "zscore": + assert np.isnan(result) + else: + assert ( + result + == { + "rank": 1.0, + "demean": 0.0, + "winsorize": 3.0, + "quantiles": 0, + "percentile_between": True, + "peer_count": 1.0, + }[method] + ) + + +@pytest.mark.parametrize( + "options", + [ + {"method": "demean", "n": 1}, + {"method": "demean", "bins": 2}, + {"method": "top", "n": 1, "ties": "dense"}, + {"method": "rank", "ascending": 1}, + {"method": "zscore", "ddof": -1}, + {"method": "rank", "ddof": 1}, + {"method": "rank", "min_percentile": 0.1}, + {"method": "winsorize", "max_percentile": np.nan}, + {"method": "rank", "entity_col": []}, + {"method": "rank", "groupby": [""]}, + {"method": "rank", "mask": 42}, + ], +) +def test_inapplicable_or_invalid_options_are_not_silently_ignored(options): + with pytest.raises((ValueError, TypeError)): + panel().select(selection().cross_section("value", **options)) + + +@pytest.mark.parametrize( + "options", + [ + {"period": 0, "method": "any"}, + {"period": 2, "method": "unknown"}, + {"period": 2, "method": "all", "minimum": 1}, + {"period": 2, "method": "at_least", "minimum": 3}, + {"period": 2, "method": "at_least", "minimum": True}, + ], +) +def test_invalid_boolean_window_contracts_fail_explicitly(options): + with pytest.raises(ValueError): + pl.DataFrame({"x": [True, False]}).select(selection().rolling_filter("x", **options)) + + +def test_boolean_window_rejects_numeric_predicates_and_respects_asset_groups(): + with pytest.raises(ValueError, match="boolean"): + pl.DataFrame({"x": [1, 0]}).select(selection().rolling_filter("x", period=2, method="any")) + frame = pl.DataFrame({"asset": ["A", "B"] * 3, "x": [True, False, True, False, True, False]}) + result = frame.select(selection().rolling_filter("x", period=2, method="all").over("asset")) + assert result.to_series().to_list() == [None, None, True, False, True, False] From bcf11b46070eda004851c6c2621721c693ae066d Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Tue, 20 Oct 2026 20:19:13 -0400 Subject: [PATCH 10/19] feat(features): prepare causal decision panels from declared source histories --- docs/user-guide/features.md | 82 +++ scripts/check_coverage.py | 1 + src/ml4t/engineer/artifacts/features.py | 56 +- src/ml4t/engineer/features/AGENTS.md | 1 + src/ml4t/engineer/features/__init__.py | 2 + src/ml4t/engineer/features/paired.py | 6 +- src/ml4t/engineer/features/preparation.py | 373 +++++++++++++ .../test_zipline_return_factor_alignment.py | 1 + tests/test_feature_panel_preparation.py | 510 ++++++++++++++++++ 9 files changed, 1028 insertions(+), 4 deletions(-) create mode 100644 src/ml4t/engineer/features/preparation.py create mode 100644 tests/test_feature_panel_preparation.py diff --git a/docs/user-guide/features.md b/docs/user-guide/features.md index e8a9d70c..e024f616 100644 --- a/docs/user-guide/features.md +++ b/docs/user-guide/features.md @@ -276,6 +276,88 @@ Ordinary Polars arithmetic, comparisons, boolean operators and `is_in` cover the corresponding elementwise factor/filter/classifier operations. None of these expressions selects Data revisions or establishes availability by itself. +### Decision-time feature preparation + +`features.preparation.prepare_feature_panel` consumes already prepared native +histories and supplied universe facts. Data or the caller selects source revisions +and adjustment perspective first. Engineer validates keys, preserves pre-entry +warmup and computes feature eligibility at each requested decision. It does not +choose the latest revision, apply corporate actions or fetch provider history. + +Supply a canonical `FeedSpec` instance. For a canonical serialized handoff, restore +it with `FeedSpec.from_mapping(...)` once. Unresolved config objects are rejected; +explicit column arguments cannot conflict with the declaration. A named calendar +requires the existing complete bar-grid declaration before any return/feature is +derived. Explicit `calendar=None` keeps observation-row windows. + +`decisions` has the declared observation timestamp column and a separate aware +`decision_time`. Universe rows have stable entity keys, `effective_from`, exclusive +`effective_to` (null means open), aware `available_at`, nullable boolean `member` +and optional classification columns. Effective intervals use the observation +clock, not an inferred midnight instant. Missing/unknown facts never imply +membership. Overlapping known active facts fail with an upstream-resolution error. + + +```python +from datetime import UTC, date, datetime +import polars as pl +from ml4t.specs import FeedSpec +from ml4t.engineer.features.preparation import prepare_feature_panel +from ml4t.engineer.features.rolling import finite_mean + +days = [date(2025, 7, d) for d in (7, 8, 9)] +history = pl.DataFrame({ + "timestamp": days, "asset": ["A"] * 3, "close": [1., 2., 3.], + "available_at": [datetime(2025, 7, d.day, 16, tzinfo=UTC) for d in days], +}) +decisions = pl.DataFrame({ + "timestamp": [days[-1]], "decision_time": [datetime(2025, 7, 9, 17, tzinfo=UTC)], +}) +universe = pl.DataFrame({ + "asset": ["A"], "effective_from": [days[-1]], + "effective_to": pl.Series([None], dtype=pl.Date), "member": [True], + "available_at": [datetime(2025, 7, 8, 12, tzinfo=UTC)], +}) +result = prepare_feature_panel( + history.lazy(), features={"average": finite_mean("close", period=3)}, + lookbacks={"average": 3}, input_columns=["close"], + decisions=decisions, universe=universe, price_basis="raw", + feed_spec=FeedSpec(time_contract_version=2, calendar=None, entity_col="asset"), +).collect() +assert result["average"].to_list() == [2.] +assert result["eligible"].to_list() == [True] +``` + +This asset enters the universe on July 9 but retains July 7-8 warmup. Expressions +are bare per-asset expressions; the preparer applies entity grouping. Declare each +finite row lookback and all input columns. Only complete, known, usable required +windows become eligible. Unavailable or input-masked rows remain as null inputs, +not shorter windows. A missing endpoint does not carry an earlier feature forward. +Undefined/nonfinite outputs, including undefined fields in regression Structs, +cannot become eligible. The output retains the observation/decision clocks, +conservative contributing availability, membership, eligibility and price basis. + +For per-decision restated histories, Data prepares each perspective and the caller +adds an aware snapshot column. Pass its name as `history_decision_time_col`; the +preparer requires an exact matching snapshot for each decision. Without that +column, duplicate history keys fail. It never selects or forward-fills a source +revision/perspective implicitly. Preserve the Data snapshot's provenance alongside +the resulting table. + +`input_mask` affects history before computation. Membership does not erase +pre-entry history. `screen` runs after all features are computed. Alternatively, +apply `cross_section(..., mask="eligible")` to the output, then filter final +selections. Cross-sectional masks determine the contemporary population and are +not a substitute for information-time validation. + +Owned `FeatureSchema` records optional `available_at_col`, `decision_time_col` and +`eligible_col`. `FeatureDefinition` records the canonical `feed_spec`, +`window_unit`, `price_basis` and finite `lookbacks`, with existing source-artifact +lineage and join keys. `FeatureSpec.to_dict()` preserves omitted calendar versus +explicit null; unused fields leave the old wire keys unchanged. Store actual +typed frames separately, for example with Parquet. These records do not select +source revisions or add a mandatory dataframe wrapper. + ### Paired relationships and supplied event dates `features.paired.rolling_pearson`, `rolling_spearman`, `rolling_beta` and diff --git a/scripts/check_coverage.py b/scripts/check_coverage.py index 2435f4bd..d4b3afd9 100644 --- a/scripts/check_coverage.py +++ b/scripts/check_coverage.py @@ -39,6 +39,7 @@ "src/ml4t/engineer/dataset.py", "src/ml4t/engineer/discovery/catalog.py", "src/ml4t/engineer/features/paired.py", + "src/ml4t/engineer/features/preparation.py", "src/ml4t/engineer/labeling/atr_barriers.py", "src/ml4t/engineer/labeling/calendar.py", "src/ml4t/engineer/labeling/horizon_labels.py", diff --git a/src/ml4t/engineer/artifacts/features.py b/src/ml4t/engineer/artifacts/features.py index fbe1b99f..9cbe7952 100644 --- a/src/ml4t/engineer/artifacts/features.py +++ b/src/ml4t/engineer/artifacts/features.py @@ -5,7 +5,14 @@ from dataclasses import dataclass, field from typing import Any -from ml4t.specs.base import ArtifactKind, ArtifactProvenance, ArtifactSpec, ArtifactStorage +from ml4t.specs import FeedSpec, WindowUnit +from ml4t.specs.base import ( + ArtifactKind, + ArtifactProvenance, + ArtifactSpec, + ArtifactStorage, + optional_str, +) @dataclass(frozen=True, slots=True) @@ -15,6 +22,9 @@ class FeatureSchema: timestamp_col: str = "timestamp" entity_col: str = "asset" feature_columns: tuple[str, ...] = () + available_at_col: str | None = None + decision_time_col: str | None = None + eligible_col: str | None = None @classmethod def from_mapping(cls, mapping: dict[str, Any] | None) -> FeatureSchema: @@ -25,6 +35,9 @@ def from_mapping(cls, mapping: dict[str, Any] | None) -> FeatureSchema: timestamp_col=str(mapping.get("timestamp_col", "timestamp")), entity_col=str(mapping.get("entity_col", "asset")), feature_columns=tuple(str(item) for item in feature_columns), + available_at_col=optional_str(mapping.get("available_at_col")), + decision_time_col=optional_str(mapping.get("decision_time_col")), + eligible_col=optional_str(mapping.get("eligible_col")), ) @@ -35,6 +48,29 @@ class FeatureDefinition: family: str = "financial" join_keys: tuple[str, ...] = () source_artifacts: tuple[str, ...] = () + feed_spec: FeedSpec | None = None + window_unit: WindowUnit | None = None + price_basis: str | None = None + lookbacks: dict[str, int] | None = None + + def __post_init__(self) -> None: + if self.feed_spec is not None: + object.__setattr__(self, "feed_spec", FeedSpec.from_any(self.feed_spec)) + if self.window_unit is not None: + object.__setattr__(self, "window_unit", WindowUnit(self.window_unit)) + if self.price_basis is not None and not self.price_basis.strip(): + raise ValueError("price_basis must be a nonempty declaration") + if self.lookbacks is not None: + if any( + not isinstance(name, str) + or not name + or isinstance(value, bool) + or not isinstance(value, int) + or value < 1 + for name, value in self.lookbacks.items() + ): + raise ValueError("lookbacks must map feature names to positive integer row counts") + object.__setattr__(self, "lookbacks", dict(self.lookbacks)) @classmethod def from_mapping(cls, mapping: dict[str, Any] | None) -> FeatureDefinition: @@ -46,6 +82,12 @@ def from_mapping(cls, mapping: dict[str, Any] | None) -> FeatureDefinition: family=str(mapping.get("family", "financial")), join_keys=tuple(str(item) for item in join_keys), source_artifacts=tuple(str(item) for item in source_artifacts), + feed_spec=FeedSpec.from_any(mapping["feed_spec"]) + if mapping.get("feed_spec") is not None + else None, + window_unit=mapping.get("window_unit"), + price_basis=optional_str(mapping.get("price_basis")), + lookbacks=mapping.get("lookbacks"), ) @@ -57,6 +99,18 @@ class FeatureSpec(ArtifactSpec): schema: FeatureSchema = field(default_factory=FeatureSchema) definition: FeatureDefinition = field(default_factory=FeatureDefinition) + def to_dict(self) -> dict[str, Any]: + payload = ArtifactSpec.to_dict(self) + for name in ("available_at_col", "decision_time_col", "eligible_col"): + if payload["schema"][name] is None: + del payload["schema"][name] + for name in ("feed_spec", "window_unit", "price_basis", "lookbacks"): + if payload["definition"][name] is None: + del payload["definition"][name] + if self.definition.feed_spec is not None: + payload["definition"]["feed_spec"] = self.definition.feed_spec.to_dict() + return payload + @classmethod def from_mapping(cls, mapping: dict[str, Any]) -> FeatureSpec: return cls( diff --git a/src/ml4t/engineer/features/AGENTS.md b/src/ml4t/engineer/features/AGENTS.md index 79582537..d440217c 100644 --- a/src/ml4t/engineer/features/AGENTS.md +++ b/src/ml4t/engineer/features/AGENTS.md @@ -29,6 +29,7 @@ These are not registry entries but remain important: | `paired.py` | full-window paired correlation/beta/regression with explicit missing-count limits | | `events.py` | supplied known event weekday counts, separate from exchange-session counts | | `selection.py` | eligible contemporary group transforms, stable selectors and boolean windows | +| `preparation.py` | decision-time eligibility over supplied prepared histories and universe facts | | `fdiff.py` | fractional differencing utilities such as `ffdiff()` and `find_optimal_d()` | | `composite.py` | composite indicators and z-score combinations | diff --git a/src/ml4t/engineer/features/__init__.py b/src/ml4t/engineer/features/__init__.py index 72e62de4..bf28eebd 100644 --- a/src/ml4t/engineer/features/__init__.py +++ b/src/ml4t/engineer/features/__init__.py @@ -36,6 +36,7 @@ ml, # noqa: F401 momentum, # noqa: F401 paired, + preparation, price_transform, # noqa: F401 regime, risk, @@ -53,6 +54,7 @@ "cross_asset", "events", "paired", + "preparation", "fdiff", "regime", "risk", diff --git a/src/ml4t/engineer/features/paired.py b/src/ml4t/engineer/features/paired.py index 96adb27f..7ee98c37 100644 --- a/src/ml4t/engineer/features/paired.py +++ b/src/ml4t/engineer/features/paired.py @@ -77,9 +77,9 @@ def target_return_features( raise ValueError("index_mode must be common_grid or per_asset") if missing_price_policy not in {"raise", "null"}: raise ValueError("missing_price_policy must be raise or null") - contract = ( - FeedSpec(time_contract_version=2) if feed_spec is None else FeedSpec.from_any(feed_spec) - ) + if feed_spec is not None and not isinstance(feed_spec, FeedSpec): + raise TypeError("feed_spec must be a canonical FeedSpec instance, not an unresolved config") + contract = FeedSpec(time_contract_version=2) if feed_spec is None else feed_spec def bind(explicit: Any, declared: Any, default: Any, role: str) -> Any: if explicit is not None and feed_spec is not None and declared is not None: diff --git a/src/ml4t/engineer/features/preparation.py b/src/ml4t/engineer/features/preparation.py new file mode 100644 index 00000000..e24e400b --- /dev/null +++ b/src/ml4t/engineer/features/preparation.py @@ -0,0 +1,373 @@ +"""Decision-time feature eligibility over caller/Data-prepared native histories.""" + +from collections.abc import Mapping, Sequence +from typing import Any + +import polars as pl +from ml4t.specs import UNSET_CALENDAR, FeedSpec, TimestampSemantics, WindowUnit + +from ml4t.engineer.core.bar_grid import validate_declared_grid +from ml4t.engineer.features.events import _instant_ticks +from ml4t.engineer.features.rolling import _period + + +def _usable_value(value: pl.Expr, dtype: pl.DataType) -> pl.Expr: + valid = value.is_not_null() + if dtype.is_float(): + valid &= value.is_finite() + elif isinstance(dtype, pl.Struct): + valid &= pl.all_horizontal( + _usable_value( + value.struct.field(field.name), + field.dtype if isinstance(field.dtype, pl.DataType) else field.dtype(), + ) + for field in dtype.fields + ) + return valid + + +def prepare_feature_panel( + data: pl.DataFrame | pl.LazyFrame, + *, + features: Mapping[str, pl.Expr], + lookbacks: Mapping[str, int], + input_columns: Sequence[str], + decisions: pl.DataFrame | pl.LazyFrame, + universe: pl.DataFrame | pl.LazyFrame, + price_basis: str, + feed_spec: FeedSpec | None = None, + timestamp_col: str | None = None, + entity_col: str | list[str] | None = None, + calendar: Any = UNSET_CALENDAR, + window_unit: str | WindowUnit | None = None, + available_at_col: str = "available_at", + decision_time_col: str = "decision_time", + history_decision_time_col: str | None = None, + input_mask: str | pl.Expr | None = None, + screen: str | pl.Expr | None = None, +) -> pl.DataFrame | pl.LazyFrame: + """Calculate finite per-asset features with explicit causal eligibility. + + decisions supplies an observation cutoff and a separate aware decision instant. + universe supplies stable IDs, effective_from/inclusive, effective_to/exclusive + (null means open), available_at and nullable boolean member, plus optional + classifications. Known active overlaps are ambiguous and raise. Missing facts + never imply membership. Pre-entry price history is retained for warmup. + + Data/callers select revisions and adjustment perspective before this boundary. + For different prepared histories at different decisions, label those snapshots + with history_decision_time_col; no snapshot/revision is implicitly selected or + carried forward. The declared price_basis is retained, not calculated. + + Supply bare per-asset expressions, their finite row lookbacks, and every input + column they require. Full required windows must contain known, usable inputs. + Unavailable/masked rows stay in history with null inputs, never compressed or + filled. A final screen runs only after all values and eligibility are computed. + Cross-sectional transforms operate on the resulting eligible population. + """ + if not features or set(features) != set(lookbacks): + raise ValueError("Declare the same nonempty feature names and lookbacks") + if any( + not isinstance(name, str) or not name or not isinstance(expr, pl.Expr) + for name, expr in features.items() + ): + raise ValueError("features must map nonempty names to native expressions") + for count in lookbacks.values(): + _period(count) + if count > 2**63 - 1: + raise ValueError("Feature lookbacks must fit signed 64-bit row counts") + if ( + isinstance(input_columns, str) + or not input_columns + or len(set(input_columns)) != len(input_columns) + ): + raise ValueError("Declare distinct input_columns for the supplied expressions") + if not isinstance(price_basis, str) or not price_basis.strip(): + raise ValueError("Declare a nonempty price_basis") + if any( + value is not None and not isinstance(value, str | pl.Expr) for value in (input_mask, screen) + ): + raise TypeError("input_mask and screen must be column names or expressions") + if feed_spec is not None and not isinstance(feed_spec, FeedSpec): + raise TypeError("feed_spec must be a canonical FeedSpec instance, not an unresolved config") + contract = FeedSpec(time_contract_version=2) if feed_spec is None else feed_spec + for name, explicit, declared in ( + ("timestamp_col", timestamp_col, contract.timestamp_col), + ("entity_col", entity_col, contract.entity_col), + ): + if ( + explicit is not None + and declared is not None + and feed_spec is not None + and ([explicit] if isinstance(explicit, str) else list(explicit)) + != ([declared] if isinstance(declared, str) else list(declared)) + ): + raise ValueError(f"Conflicting {name} and feed_spec declaration") + timestamp = timestamp_col or contract.timestamp_col + identity = entity_col if entity_col is not None else contract.entity_col + entities = ( + [identity] + if isinstance(identity, str) + else list(identity) + if identity is not None + else ["asset"] + ) + if not entities or len(set(entities)) != len(entities): + raise ValueError("Declare distinct stable entity columns") + keys = [timestamp, *entities] + if len(set(keys)) != len(keys) or set(input_columns) & set(keys): + raise ValueError("Input columns and timestamp/entity roles must be distinct") + resolved = contract.resolve_calendar(calendar=calendar) + unit = ( + WindowUnit(window_unit) + if window_unit is not None + else (WindowUnit.OBSERVATION if resolved is None else WindowUnit.EXPECTED_BAR) + ) + if unit != (WindowUnit.OBSERVATION if resolved is None else WindowUnit.EXPECTED_BAR): + raise ValueError( + "Finite feature preparation requires observation or declared expected_bar rows" + ) + decision_frame = ( + decisions.collect() if isinstance(decisions, pl.LazyFrame) else decisions.clone() + ) + universe_frame = universe.collect() if isinstance(universe, pl.LazyFrame) else universe.clone() + schema = data.collect_schema() if isinstance(data, pl.LazyFrame) else data.schema + for label, columns, required in ( + ( + "history", + schema, + [ + *keys, + *input_columns, + available_at_col, + *([history_decision_time_col] if history_decision_time_col else []), + ], + ), + ("decisions", decision_frame.schema, [timestamp, decision_time_col]), + ( + "universe", + universe_frame.schema, + [*entities, "effective_from", "effective_to", available_at_col, "member"], + ), + ): + if set(required) - set(columns): + raise ValueError(f"Missing {label} columns: {sorted(set(required) - set(columns))}") + if schema[timestamp].base_type() not in {pl.Date, pl.Datetime}: + raise ValueError("Observation keys require Date labels or declared Datetime instants") + if resolved is not None: + validate_declared_grid( + pl.DataFrame(schema=schema), + feed_spec=contract, + calendar=resolved, + timestamp_col=timestamp, + group_cols=entities, + ) + if schema[timestamp] != decision_frame.schema[timestamp]: + raise ValueError("Decision observation cutoffs must preserve the input timestamp dtype") + if any( + universe_frame.schema[c] != schema[timestamp] for c in ("effective_from", "effective_to") + ): + raise ValueError( + "Universe effective intervals must use the declared observation clock dtype" + ) + if universe_frame.schema["member"] != pl.Boolean: + raise ValueError("Universe member must be nullable boolean, not an inferred classifier") + roles = [*keys, decision_time_col, available_at_col, "member", "eligible", "price_basis"] + if len(set(roles)) != len(roles) or set(features) & set(roles): + raise ValueError("Feature and output roles collide") + extra = [ + c + for c in universe_frame.columns + if c not in [*entities, "effective_from", "effective_to", available_at_col, "member"] + ] + if set(extra) & (set(roles) | set(features)): + raise ValueError("Universe classification names collide with feature/output roles") + prefix = "__ml4t_feature_" + if any( + c.startswith(prefix) + for c in [*schema, *universe_frame.columns, *decision_frame.columns, *features] + ): + raise ValueError("Input columns collide with reserved feature preparation names") + known, cutoff, source_usable, feature_known = ( + prefix + c for c in ("known", "cutoff", "usable", "time") + ) + + def clock( + frame: pl.DataFrame, column: str, output: str, *, required: bool = False + ) -> pl.DataFrame: + expr = ( + pl.lit(None, pl.Int128) if frame.schema[column] == pl.Null else _instant_ticks(column) + ) + result = frame.with_columns(expr.alias(output)) + if required and result[output].null_count(): + raise ValueError(f"{column} must contain known decision/snapshot instants") + if result.filter((pl.col(output) < -(2**63)) | (pl.col(output) > 2**63 - 1)).height: + raise ValueError(f"{column} exceeds Datetime(ns) output range") + return result + + decision_frame = clock(decision_frame, decision_time_col, cutoff, required=True) + timestamp_dtype = schema[timestamp] + if ( + isinstance(timestamp_dtype, pl.Datetime) + and contract.timestamp_semantics != TimestampSemantics.SESSION_LABEL + ): + observation = pl.col(timestamp) + if timestamp_dtype.time_zone is None: + if contract.timezone is None: + raise ValueError("Naive observation instants require a declared timezone") + observation = observation.dt.replace_time_zone(contract.timezone) + decision_frame = decision_frame.with_columns(observation.alias(prefix + "observation")) + decision_frame = clock( + decision_frame, prefix + "observation", prefix + "observation_ticks", required=True + ) + if decision_frame.filter(pl.col(prefix + "observation_ticks") > pl.col(cutoff)).height: + raise ValueError("Observation cutoff cannot follow its decision instant") + universe_frame = clock(universe_frame, available_at_col, known) + if ( + decision_frame[timestamp].null_count() + or decision_frame.select(pl.struct(timestamp, cutoff).is_duplicated().any()).item() + ): + raise ValueError("Decision observation/instant keys must be nonnull and unique") + if universe_frame.select( + pl.any_horizontal(pl.col(c).is_null() for c in [*entities, "effective_from"]).any() + ).item(): + raise ValueError("Universe identity/effective start cannot be unknown") + if universe_frame.filter(pl.col("effective_to") <= pl.col("effective_from")).height: + raise ValueError("Universe effective intervals must be nonempty and end-exclusive") + template = ( + pl.DataFrame(schema=schema) + .select(*keys, *input_columns) + .lazy() + .with_columns(expr.over(entities).alias(name) for name, expr in features.items()) + .collect_schema() + ) + result_schema = { + timestamp: schema[timestamp], + **{c: schema[c] for c in entities}, + decision_time_col: pl.Datetime("ns", "UTC"), + **{name: template[name] for name in features}, + available_at_col: pl.Datetime("ns", "UTC"), + "member": pl.Boolean, + "eligible": pl.Boolean, + "price_basis": pl.String, + **{c: universe_frame.schema[c] for c in extra}, + } + + def calculate(frame: pl.DataFrame) -> pl.DataFrame: + source = clock(frame, available_at_col, known) + unique_keys = [*keys] + if history_decision_time_col: + source = clock(source, history_decision_time_col, prefix + "snapshot", required=True) + unique_keys.append(prefix + "snapshot") + if source.select(pl.any_horizontal(pl.col(c).is_null() for c in keys).any()).item(): + raise ValueError("History timestamp/entity keys cannot be unknown") + if source.select(pl.struct(unique_keys).is_duplicated().any()).item(): + raise ValueError( + "Duplicate history keys; select source revisions/perspectives upstream" + ) + output = [] + for i in range(decision_frame.height): + decision = decision_frame.slice(i, 1) + bound = pl.lit(decision[timestamp]).first() + instant = decision[cutoff].item() + history = source + if history_decision_time_col: + history = history.filter(pl.col(prefix + "snapshot") == instant) + if history.is_empty(): + raise ValueError("Missing prepared history snapshot for a requested decision") + history = history.filter(pl.col(timestamp) <= bound).sort([*entities, timestamp]) + facts = universe_frame.filter( + (pl.col("effective_from") <= bound) + & (pl.col("effective_to").is_null() | (pl.col("effective_to") > bound)) + & (pl.col(known) <= instant) + ) + if facts.select(pl.struct(entities).is_duplicated().any()).item(): + raise ValueError("Ambiguous overlapping known universe facts; resolve upstream") + if resolved is not None and not history.is_empty(): + validate_declared_grid( + history, + feed_spec=contract, + calendar=resolved, + timestamp_col=timestamp, + group_cols=entities, + ) + usable = pl.col(known).is_not_null() & (pl.col(known) <= instant) + for column in input_columns: + usable &= _usable_value(pl.col(column), schema[column]) + if input_mask is not None: + usable &= ( + pl.col(input_mask) if isinstance(input_mask, str) else input_mask + ).fill_null(False) + history = history.with_columns(usable.alias(source_usable)) + assets = pl.concat( + [history.filter(pl.col(known) <= instant).select(entities), facts.select(entities)] + ).unique() + history = history.with_columns( + pl.when(pl.col(source_usable)).then(pl.col(c)).alias(c) for c in input_columns + ) + available = [] + for name in features: + count = lookbacks[name] + full = pl.col(source_usable).cast(pl.Int64).rolling_sum(count) == count + available.append( + pl.when(full) + .then(pl.col(known).cast(pl.Int64).rolling_max(count)) + .over(entities) + .alias(prefix + name) + ) + history = ( + history.with_columns(available) + .select(*keys, *input_columns, *[prefix + name for name in features]) + .with_columns(expr.over(entities).alias(name) for name, expr in features.items()) + ) + history = history.with_columns( + pl.when(pl.col(prefix + name).is_not_null()).then(pl.col(name)).alias(name) + for name in features + ) + knowledge = pl.when( + pl.all_horizontal(pl.col(prefix + name).is_not_null() for name in features) + ).then(pl.max_horizontal(pl.col(prefix + name) for name in features)) + latest = history.filter(pl.col(timestamp) == bound).select( + *entities, *features, knowledge.alias(feature_known) + ) + result = assets.join(latest, on=entities, how="left", validate="1:1").join( + facts.select( + *entities, "member", *extra, pl.col(known).cast(pl.Int64).alias(known) + ), + on=entities, + how="left", + validate="1:1", + ) + result = result.with_columns( + bound.alias(timestamp), + pl.lit(instant).cast(pl.Datetime("ns", "UTC")).alias(decision_time_col), + pl.when(pl.col(feature_known).is_not_null()) + .then(pl.max_horizontal(feature_known, known)) + .cast(pl.Datetime("ns", "UTC")) + .alias(available_at_col), + ( + pl.col("member").fill_null(False) + & pl.col(feature_known).is_not_null() + & pl.all_horizontal( + _usable_value(pl.col(name), template[name]) for name in features + ) + ).alias("eligible"), + pl.lit(price_basis).alias("price_basis"), + ) + output.append(result.select(*result_schema)) + result = pl.concat(output) if output else pl.DataFrame(schema=result_schema) + result = result.sort([decision_time_col, timestamp, *entities]) + if screen is not None: + result = result.filter(pl.col(screen) if isinstance(screen, str) else screen) + return result + + if isinstance(data, pl.LazyFrame): + return data.map_batches( + calculate, + schema=result_schema, + predicate_pushdown=False, + projection_pushdown=False, + slice_pushdown=False, + streamable=False, + ) + return calculate(data) diff --git a/tests/compat/test_zipline_return_factor_alignment.py b/tests/compat/test_zipline_return_factor_alignment.py index 31255043..4a5db892 100644 --- a/tests/compat/test_zipline_return_factor_alignment.py +++ b/tests/compat/test_zipline_return_factor_alignment.py @@ -265,6 +265,7 @@ def test_multicolumn_identity_does_not_join_reused_tickers_across_namespaces() - {"target": {"asset": "B", "other": 1}}, {"window_unit": "expected_bar"}, {"allowed_missing": -1}, + {"feed_spec": {"timestamp_col": "timestamp", "time_col": "other"}}, {"feed_spec": None, "entity_col": []}, {"feed_spec": None, "price_col": "timestamp"}, ], diff --git a/tests/test_feature_panel_preparation.py b/tests/test_feature_panel_preparation.py new file mode 100644 index 00000000..f201cb52 --- /dev/null +++ b/tests/test_feature_panel_preparation.py @@ -0,0 +1,510 @@ +"""Actual prepared histories become causal features without revision/adjustment selection.""" + +import importlib +import json +from datetime import UTC, date, datetime + +import polars as pl +import pytest +from ml4t.specs import FeedSpec +from polars.testing import assert_frame_equal + +from ml4t.engineer.artifacts import FeatureSpec +from ml4t.engineer.features.paired import rolling_regression +from ml4t.engineer.features.rolling import finite_mean + +DAYS = [date(2025, 7, d) for d in (7, 8, 9, 10)] + + +def prepare(data, **options): + function = importlib.import_module("ml4t.engineer.features.preparation").prepare_feature_panel + defaults = { + "features": {"average": finite_mean("close", period=3)}, + "lookbacks": {"average": 3}, + "input_columns": ["close"], + "decisions": decisions(), + "universe": universe(), + "price_basis": "raw", + "feed_spec": FeedSpec( + time_contract_version=2, + calendar=None, + entity_col="asset", + timestamp_semantics="session_label", + ), + } + return function(data, **{**defaults, **options}) + + +def history(): + return pl.DataFrame( + { + "timestamp": DAYS * 2, + "asset": ["A"] * 4 + ["B"] * 4, + "close": [1.0, 2.0, 3.0, 4.0, 10.0, 20.0, 30.0, 40.0], + "available_at": [datetime(2025, 7, day.day, 16, tzinfo=UTC) for day in DAYS] * 2, + } + ) + + +def decisions(): + return pl.DataFrame( + { + "timestamp": DAYS[2:], + "decision_time": [datetime(2025, 7, d.day, 17, tzinfo=UTC) for d in DAYS[2:]], + } + ) + + +def universe(): + return pl.DataFrame( + { + "asset": ["A", "B"], + "effective_from": [DAYS[2]] * 2, + "effective_to": pl.Series([None, None], dtype=pl.Date), + "available_at": [datetime(2025, 7, 8, 12, tzinfo=UTC)] * 2, + "member": [True, False], + "sector": ["tech", "bank"], + } + ) + + +@pytest.mark.parametrize("lazy", [False, True]) +def test_dated_membership_keeps_preentry_warmup_and_original_feature_clock(lazy): + frame = history() + result = prepare(frame.lazy() if lazy else frame) + assert isinstance(result, pl.LazyFrame if lazy else pl.DataFrame) + result = result.collect() if lazy else result + assert result.select("timestamp", "asset").rows() == [ + (DAYS[2], "A"), + (DAYS[2], "B"), + (DAYS[3], "A"), + (DAYS[3], "B"), + ] + assert result["average"].to_list() == [2.0, 20.0, 3.0, 30.0] + assert result["eligible"].to_list() == [True, False, True, False] + assert result["member"].to_list() == [True, False, True, False] + assert result["sector"].to_list() == ["tech", "bank", "tech", "bank"] + assert ( + result["available_at"].to_list() + == [datetime(2025, 7, 9, 16, tzinfo=UTC)] * 2 + [datetime(2025, 7, 10, 16, tzinfo=UTC)] * 2 + ) + assert_frame_equal(frame, history()) + + +def test_future_publication_cannot_fill_or_compress_a_required_window(): + delayed = history().with_columns( + pl.when((pl.col("asset") == "A") & (pl.col("timestamp") == DAYS[1])) + .then(pl.lit(datetime(2025, 7, 10, 18, tzinfo=UTC))) + .otherwise(pl.col("available_at")) + .alias("available_at") + ) + result = prepare(delayed) + a = result.filter(pl.col("asset") == "A") + assert a["average"].null_count() == 2 + assert a["eligible"].to_list() == [False, False] + assert a["available_at"].null_count() == 2 + + +def test_changing_only_final_screen_does_not_change_warmup_or_group_values(): + before = prepare(history()) + screened = prepare(history(), screen=pl.col("asset") == "A") + assert_frame_equal(screened, before.filter(pl.col("asset") == "A")) + masked = prepare(history(), input_mask=pl.col("close") > 1.0) + assert masked.filter((pl.col("timestamp") == DAYS[2]) & (pl.col("asset") == "A"))[ + "eligible" + ].to_list() == [False] + + +def test_unavailable_and_expired_universe_facts_do_not_default_to_membership(): + facts = universe().with_columns( + pl.when(pl.col("asset") == "A") + .then(pl.lit(datetime(2025, 7, 10, 18, tzinfo=UTC))) + .otherwise(pl.col("available_at")) + .alias("available_at") + ) + result = prepare(history(), universe=facts) + assert result.filter(pl.col("asset") == "A")["member"].null_count() == 2 + assert not result["eligible"].any() + expired = universe().with_columns(pl.lit(DAYS[3]).alias("effective_to")) + result = prepare(history(), universe=expired) + assert result.filter(pl.col("timestamp") == DAYS[3])["member"].null_count() == 2 + + +def test_appended_future_quote_and_unknown_universe_change_do_not_revise_prior_features(): + expected = prepare(history()) + future = ( + history() + .with_columns( + pl.lit(date(2025, 7, 11)).alias("timestamp"), + pl.lit(1e9).alias("close"), + pl.lit(datetime(2025, 7, 11, 16, tzinfo=UTC)).alias("available_at"), + ) + .unique(subset=["asset", "timestamp"]) + ) + future_fact = universe().with_columns( + pl.lit(date(2025, 7, 11)).alias("effective_from"), + pl.lit(datetime(2025, 7, 11, 12, tzinfo=UTC)).alias("available_at"), + pl.lit("future").alias("sector"), + ) + actual = prepare(pl.concat([history(), future]), universe=pl.concat([universe(), future_fact])) + assert_frame_equal(actual, expected) + + +def test_decision_specific_Data_snapshots_are_consumed_without_selecting_revisions(): + early = history().with_columns( + pl.lit(datetime(2025, 7, 9, 17, tzinfo=UTC)).alias("snapshot_at") + ) + late = history().with_columns( + (pl.col("close") * 2).alias("close"), + pl.lit(datetime(2025, 7, 10, 17, tzinfo=UTC)).alias("snapshot_at"), + ) + result = prepare(pl.concat([early, late]), history_decision_time_col="snapshot_at") + assert result["average"].to_list() == [2.0, 20.0, 6.0, 60.0] + with pytest.raises(ValueError, match="[Dd]uplicate"): + prepare(pl.concat([early, late])) + + +def test_ambiguous_overlapping_universe_records_require_upstream_resolution(): + facts = pl.concat( + [ + universe(), + universe().filter(pl.col("asset") == "A").with_columns(pl.lit("other").alias("sector")), + ] + ) + with pytest.raises(ValueError, match="[Aa]mbiguous|[Dd]uplicate"): + prepare(history(), universe=facts) + + +@pytest.mark.parametrize("calendar", ["omitted", None, "NYSE"]) +def test_feature_artifact_public_roundtrip_preserves_required_feed_and_eligibility_declarations( + calendar, +): + feed = FeedSpec( + time_contract_version=2, **({} if calendar == "omitted" else {"calendar": calendar}) + ) + payload = { + "artifact_id": "causal-features", + "schema": { + "timestamp_col": "when", + "entity_col": "identity", + "feature_columns": ["average"], + "available_at_col": "known", + "decision_time_col": "decision", + "eligible_col": "eligible", + }, + "definition": { + "family": "financial", + "feed_spec": feed.to_dict(), + "window_unit": "observation" if calendar in (None, "omitted") else "expected_bar", + "price_basis": "split_adjusted", + "lookbacks": {"average": 3}, + }, + } + spec = FeatureSpec.from_mapping(json.loads(json.dumps(payload))) + output = json.loads(json.dumps(spec.to_dict())) + for name, value in payload["schema"].items(): + assert output["schema"][name] == value + for name, value in payload["definition"].items(): + assert output["definition"][name] == value + + +def test_adjacent_ns_observation_decision_and_publication_keys_do_not_round(): + frame = pl.DataFrame( + { + "timestamp": pl.Series([0, 1, 2], dtype=pl.Datetime("ns", "UTC")), + "asset": ["A"] * 3, + "close": [1.0, 2.0, 3.0], + "available_at": pl.Series([1, 2, 3], dtype=pl.Datetime("ns", "UTC")), + } + ) + requested = pl.DataFrame( + { + "timestamp": pl.Series([1, 2], dtype=pl.Datetime("ns", "UTC")), + "decision_time": pl.Series([2, 3], dtype=pl.Datetime("ns", "UTC")), + } + ) + facts = pl.DataFrame( + { + "asset": ["A"], + "effective_from": pl.Series([0], dtype=pl.Datetime("ns", "UTC")), + "effective_to": pl.Series([None], dtype=pl.Datetime("ns", "UTC")), + "available_at": pl.Series([0], dtype=pl.Datetime("ns", "UTC")), + "member": [True], + } + ) + result = prepare( + frame, + features={"average": finite_mean("close", period=2)}, + lookbacks={"average": 2}, + decisions=requested, + universe=facts, + feed_spec=FeedSpec( + time_contract_version=2, + calendar=None, + entity_col="asset", + timestamp_semantics="event_time", + ), + ) + assert result["timestamp"].cast(pl.Int64).to_list() == [1, 2] + assert result["decision_time"].cast(pl.Int64).to_list() == [2, 3] + assert result["available_at"].cast(pl.Int64).to_list() == [2, 3] + assert result["average"].to_list() == [1.5, 2.5] + + +def test_instant_observation_cutoff_cannot_follow_the_actual_decision(): + frame = history().with_columns( + pl.col("timestamp").cast(pl.Datetime("ns", "UTC")).dt.offset_by("20h") + ) + requested = decisions().with_columns( + pl.col("timestamp").cast(pl.Datetime("ns", "UTC")).dt.offset_by("20h") + ) + facts = universe().with_columns( + pl.col("effective_from").cast(pl.Datetime("ns", "UTC")), + pl.col("effective_to").cast(pl.Datetime("ns", "UTC")), + ) + with pytest.raises(ValueError, match="cutoff|decision"): + prepare( + frame, + decisions=requested, + universe=facts, + feed_spec=FeedSpec( + time_contract_version=2, calendar=None, timestamp_semantics="event_time" + ), + ) + + +def test_custom_roles_compound_ids_and_lazy_output_filter_preserve_the_real_clock(): + frame = ( + history() + .with_columns( + pl.struct(pl.lit("bundle").alias("namespace"), pl.col("asset").alias("sid")).alias( + "asset" + ) + ) + .rename({"timestamp": "when", "asset": "identity"}) + ) + facts = ( + universe() + .with_columns( + pl.struct(pl.lit("bundle").alias("namespace"), pl.col("asset").alias("sid")).alias( + "asset" + ) + ) + .rename({"asset": "identity"}) + ) + requested = decisions().rename({"timestamp": "when"}) + result = prepare( + frame.lazy(), + decisions=requested.lazy(), + universe=facts.lazy(), + feed_spec=FeedSpec( + time_contract_version=2, calendar=None, timestamp_col="when", entity_col="identity" + ), + ) + selected = result.filter( + (pl.col("identity").struct.field("sid") == "A") & (pl.col("when") == DAYS[3]) + ).collect() + assert selected["average"].to_list() == [3.0] + assert selected.schema["identity"] == frame.schema["identity"] + + +def test_missing_endpoint_or_market_coverage_does_not_carry_stale_features_forward(): + result = prepare( + history().filter(~((pl.col("asset") == "A") & (pl.col("timestamp") == DAYS[3]))) + ) + a = result.filter((pl.col("asset") == "A") & (pl.col("timestamp") == DAYS[3])) + assert a["member"][0] is True + assert a["average"][0] is None + assert a["eligible"][0] is False + empty = prepare(history().clear()) + assert empty.height == 4 + assert not empty["eligible"].any() + assert prepare(history(), decisions=decisions().clear()).is_empty() + + +@pytest.mark.parametrize( + "options", + [ + {"features": {}}, + {"lookbacks": {"other": 3}}, + {"lookbacks": {"average": True}}, + {"input_columns": "close"}, + {"input_columns": ["close", "close"]}, + {"input_columns": ["asset"]}, + {"price_basis": ""}, + {"input_mask": 1}, + {"screen": 1}, + {"timestamp_col": "other"}, + {"entity_col": "other"}, + {"entity_col": []}, + {"window_unit": "elapsed"}, + {"lookbacks": {"average": 2**64}}, + {"feed_spec": {"timestamp_col": "timestamp", "time_col": "other"}}, + {"features": {"average": "close"}}, + {"features": {"eligible": pl.col("close")}, "lookbacks": {"eligible": 1}}, + ], +) +def test_invalid_feature_contracts_fail_atomically(options): + before = history() + with pytest.raises((TypeError, ValueError)): + prepare(before, **options) + assert_frame_equal(before, history()) + + +@pytest.mark.parametrize( + "kind", + [ + "missing_history", + "missing_decision", + "missing_universe", + "bad_time", + "null_key", + "duplicate_key", + "duplicate_decision", + "bad_member", + "bad_interval", + "null_effective", + "classification_collision", + "reserved", + "naive_known", + "null_decision", + "range", + "missing_snapshot", + ], +) +def test_invalid_physical_inputs_do_not_produce_partial_panels(kind): + frame, requested, facts = history(), decisions(), universe() + options = {} + if kind == "missing_history": + frame = frame.drop("close") + elif kind == "missing_decision": + requested = requested.drop("decision_time") + elif kind == "missing_universe": + facts = facts.drop("member") + elif kind == "bad_time": + frame = frame.with_columns(pl.col("timestamp").cast(pl.String)) + elif kind == "null_key": + frame = frame.with_columns(pl.lit(None).alias("asset")) + elif kind == "duplicate_key": + frame = pl.concat([frame, frame.head(1)]) + elif kind == "duplicate_decision": + requested = pl.concat([requested, requested.head(1)]) + elif kind == "bad_member": + facts = facts.with_columns(pl.col("member").cast(pl.Int64)) + elif kind == "bad_interval": + facts = facts.with_columns(pl.col("effective_from").alias("effective_to")) + elif kind == "null_effective": + facts = facts.with_columns(pl.lit(None, pl.Date).alias("effective_from")) + elif kind == "classification_collision": + facts = facts.with_columns(pl.lit(1).alias("average")) + elif kind == "reserved": + frame = frame.with_columns(pl.lit(1).alias("__ml4t_feature_bad")) + elif kind == "naive_known": + frame = frame.with_columns(pl.col("available_at").dt.replace_time_zone(None)) + elif kind == "null_decision": + requested = requested.with_columns( + pl.lit(None, pl.Datetime("ns", "UTC")).alias("decision_time") + ) + elif kind == "range": + requested = requested.with_columns( + pl.lit(datetime(2500, 1, 1, tzinfo=UTC)).alias("decision_time") + ) + else: + frame = frame.with_columns(pl.lit(datetime(2025, 7, 11, tzinfo=UTC)).alias("snapshot_at")) + options["history_decision_time_col"] = "snapshot_at" + before = frame.clone() + with pytest.raises((TypeError, ValueError)): + prepare(frame, decisions=requested, universe=facts, **options) + assert_frame_equal(frame, before) + + +def test_named_schedule_checks_original_rows_and_explicit_none_remains_row_based(): + frame = history().filter(pl.col("timestamp") != DAYS[1]) + contract = FeedSpec( + time_contract_version=2, + calendar="NYSE", + entity_col="asset", + timestamp_semantics="session_label", + data_frequency="1d", + bar_type="time", + ) + with pytest.raises(ValueError, match="missing scheduled session"): + prepare(frame, feed_spec=contract) + row = prepare(frame, feed_spec=contract, calendar=None) + a = row.filter(pl.col("asset") == "A") + assert a["average"][0] is None + assert a["average"][1] == pytest.approx(8 / 3) + + +def test_unknown_availability_and_undefined_struct_outputs_cannot_become_eligible(): + unknown = history().with_columns(pl.lit(None).alias("available_at")) + result = prepare(unknown) + assert not result["eligible"].any() + assert result["available_at"].null_count() == result.height + expression = rolling_regression("close", pl.lit(1.0), period=2) + result = prepare(history(), features={"estimate": expression}, lookbacks={"estimate": 2}) + assert not result["eligible"].any() + + +def test_empty_history_does_not_waive_an_undeclared_default_calendar_grid(): + with pytest.raises(ValueError, match="declared bar grid"): + prepare(history().clear(), feed_spec=FeedSpec()) + + +@pytest.mark.parametrize("timezone", [None, "America/New_York"]) +def test_naive_observation_instants_require_and_use_declared_timezone(timezone): + frame = history().with_columns(pl.col("timestamp").cast(pl.Datetime("us")).dt.offset_by("12h")) + requested = decisions().with_columns( + pl.col("timestamp").cast(pl.Datetime("us")).dt.offset_by("12h") + ) + facts = universe().with_columns( + pl.col("effective_from").cast(pl.Datetime("us")), + pl.col("effective_to").cast(pl.Datetime("us")), + ) + options = { + "decisions": requested, + "universe": facts, + "feed_spec": FeedSpec( + time_contract_version=2, + calendar=None, + timestamp_semantics="event_time", + timezone=timezone, + ), + } + if timezone is None: + with pytest.raises(ValueError, match="timezone"): + prepare(frame, **options) + else: + assert prepare(frame, **options)["eligible"].to_list() == [True, False, True, False] + + +@pytest.mark.parametrize( + "definition", + [ + {"lookbacks": {"average": 0}}, + {"lookbacks": {"average": True}}, + {"lookbacks": {"": 3}}, + {"lookbacks": {"average": "3"}}, + {"price_basis": " "}, + {"window_unit": "median_spacing"}, + ], +) +def test_invalid_artifact_declarations_do_not_survive_public_loading(definition): + with pytest.raises((TypeError, ValueError)): + FeatureSpec.from_mapping({"artifact_id": "invalid", "definition": definition}) + + +def test_minimal_artifacts_keep_old_wire_keys_and_source_lineage(): + spec = FeatureSpec.from_mapping( + { + "artifact_id": "old", + "definition": { + "source_artifacts": ["Data:snapshot:abc"], + "join_keys": ["timestamp", "venue", "asset"], + }, + } + ) + output = spec.to_dict() + assert set(output["schema"]) == {"timestamp_col", "entity_col", "feature_columns"} + assert set(output["definition"]) == {"family", "join_keys", "source_artifacts"} + assert output["definition"]["source_artifacts"] == ["Data:snapshot:abc"] From 1e094245fca90a13a294a4ef3f86e48614211ac1 Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Tue, 20 Oct 2026 20:51:57 -0400 Subject: [PATCH 11/19] fix(features): select universe facts on the decision-effective clock --- docs/user-guide/features.md | 26 ++++-- src/ml4t/engineer/artifacts/features.py | 4 +- src/ml4t/engineer/features/preparation.py | 96 ++++++++++++++++++++--- tests/test_feature_panel_preparation.py | 73 ++++++++++++++++- 4 files changed, 176 insertions(+), 23 deletions(-) diff --git a/docs/user-guide/features.md b/docs/user-guide/features.md index f7230eeb..3bc7b563 100644 --- a/docs/user-guide/features.md +++ b/docs/user-guide/features.md @@ -305,11 +305,17 @@ explicit column arguments cannot conflict with the declaration. A named calendar requires the existing complete bar-grid declaration before any return/feature is derived. Explicit `calendar=None` keeps observation-row windows. -`decisions` has the declared observation timestamp column and a separate aware -`decision_time`. Universe rows have stable entity keys, `effective_from`, exclusive +`decisions` has the observation cutoff, a separate aware `decision_time` and an +explicit decision-effective universe key. The default `universe_time_col` selects +`effective_session`, a supplied session identifier. It need not equal the prior +price session used for a morning decision. For instant-effective universes, set +`universe_time_col` to the explicitly supplied instant column, such as +`decision_time`. No effective session is inferred from a UTC date or from prices. +Universe rows have stable entity keys, `effective_from`, exclusive `effective_to` (null means open), aware `available_at`, nullable boolean `member` -and optional classification columns. Effective intervals use the observation -clock, not an inferred midnight instant. Missing/unknown facts never imply +and optional classification columns. Effective intervals use that declared universe +clock. Date identifiers stay dates; instant comparisons preserve physical timezone +equivalence and ns boundaries. Missing/unknown facts never imply membership. Overlapping known active facts fail with an upstream-resolution error. @@ -327,6 +333,7 @@ history = pl.DataFrame({ }) decisions = pl.DataFrame({ "timestamp": [days[-1]], "decision_time": [datetime(2025, 7, 9, 17, tzinfo=UTC)], + "effective_session": [days[-1]], }) universe = pl.DataFrame({ "asset": ["A"], "effective_from": [days[-1]], @@ -350,7 +357,9 @@ windows become eligible. Unavailable or input-masked rows remain as null inputs, not shorter windows. A missing endpoint does not carry an earlier feature forward. Undefined/nonfinite outputs, including undefined fields in regression Structs, cannot become eligible. The output retains the observation/decision clocks, -conservative contributing availability, membership, eligibility and price basis. +conservative contributing availability, membership, eligibility, the declared +universe key and price basis. Membership is evaluated at that explicit key using +only facts known at the decision, not at the historical price cutoff. For per-decision restated histories, Data prepares each perspective and the caller adds an aware snapshot column. Pass its name as `history_decision_time_col`; the @@ -361,12 +370,13 @@ the resulting table. `input_mask` affects history before computation. Membership does not erase pre-entry history. `screen` runs after all features are computed. Alternatively, -apply `cross_section(..., mask="eligible")` to the output, then filter final -selections. Cross-sectional masks determine the contemporary population and are +apply `cross_section(..., timestamp_col="decision_time", mask="eligible")` to the +output, then filter final selections. If one instant has multiple declared universe +contexts, include their universe key in the group keys too. Cross-sectional masks determine the contemporary population and are not a substitute for information-time validation. Owned `FeatureSchema` records optional `available_at_col`, `decision_time_col` and -`eligible_col`. `FeatureDefinition` records the canonical `feed_spec`, +`eligible_col`, plus `universe_time_col`. `FeatureDefinition` records the canonical `feed_spec`, `window_unit`, `price_basis` and finite `lookbacks`, with existing source-artifact lineage and join keys. `FeatureSpec.to_dict()` preserves omitted calendar versus explicit null; unused fields leave the old wire keys unchanged. Store actual diff --git a/src/ml4t/engineer/artifacts/features.py b/src/ml4t/engineer/artifacts/features.py index 9cbe7952..2d5e827f 100644 --- a/src/ml4t/engineer/artifacts/features.py +++ b/src/ml4t/engineer/artifacts/features.py @@ -25,6 +25,7 @@ class FeatureSchema: available_at_col: str | None = None decision_time_col: str | None = None eligible_col: str | None = None + universe_time_col: str | None = None @classmethod def from_mapping(cls, mapping: dict[str, Any] | None) -> FeatureSchema: @@ -38,6 +39,7 @@ def from_mapping(cls, mapping: dict[str, Any] | None) -> FeatureSchema: available_at_col=optional_str(mapping.get("available_at_col")), decision_time_col=optional_str(mapping.get("decision_time_col")), eligible_col=optional_str(mapping.get("eligible_col")), + universe_time_col=optional_str(mapping.get("universe_time_col")), ) @@ -101,7 +103,7 @@ class FeatureSpec(ArtifactSpec): def to_dict(self) -> dict[str, Any]: payload = ArtifactSpec.to_dict(self) - for name in ("available_at_col", "decision_time_col", "eligible_col"): + for name in ("available_at_col", "decision_time_col", "eligible_col", "universe_time_col"): if payload["schema"][name] is None: del payload["schema"][name] for name in ("feed_spec", "window_unit", "price_basis", "lookbacks"): diff --git a/src/ml4t/engineer/features/preparation.py b/src/ml4t/engineer/features/preparation.py index e24e400b..9ff63cf3 100644 --- a/src/ml4t/engineer/features/preparation.py +++ b/src/ml4t/engineer/features/preparation.py @@ -42,13 +42,17 @@ def prepare_feature_panel( window_unit: str | WindowUnit | None = None, available_at_col: str = "available_at", decision_time_col: str = "decision_time", + universe_time_col: str = "effective_session", history_decision_time_col: str | None = None, input_mask: str | pl.Expr | None = None, screen: str | pl.Expr | None = None, ) -> pl.DataFrame | pl.LazyFrame: """Calculate finite per-asset features with explicit causal eligibility. - decisions supplies an observation cutoff and a separate aware decision instant. + decisions supplies an observation cutoff, an aware decision instant and the + universe's decision-effective key (effective_session by default). These need + not identify the same session. universe_time_col can select an explicit + instant column for instant-effective facts; no session is inferred from dates. universe supplies stable IDs, effective_from/inclusive, effective_to/exclusive (null means open), available_at and nullable boolean member, plus optional classifications. Known active overlaps are ambiguous and raise. Missing facts @@ -143,7 +147,7 @@ def prepare_feature_panel( *([history_decision_time_col] if history_decision_time_col else []), ], ), - ("decisions", decision_frame.schema, [timestamp, decision_time_col]), + ("decisions", decision_frame.schema, [timestamp, decision_time_col, universe_time_col]), ( "universe", universe_frame.schema, @@ -164,17 +168,32 @@ def prepare_feature_panel( ) if schema[timestamp] != decision_frame.schema[timestamp]: raise ValueError("Decision observation cutoffs must preserve the input timestamp dtype") - if any( - universe_frame.schema[c] != schema[timestamp] for c in ("effective_from", "effective_to") + universe_dtype = decision_frame.schema[universe_time_col] + if universe_frame.schema["effective_to"] == pl.Null: + universe_frame = universe_frame.with_columns(pl.col("effective_to").cast(universe_dtype)) + if universe_dtype.base_type() not in {pl.Date, pl.Datetime} or any( + universe_frame.schema[c].base_type() != universe_dtype.base_type() + for c in ("effective_from", "effective_to") ): raise ValueError( - "Universe effective intervals must use the declared observation clock dtype" + "Universe effective intervals must match the declared decision-effective clock" ) if universe_frame.schema["member"] != pl.Boolean: raise ValueError("Universe member must be nullable boolean, not an inferred classifier") roles = [*keys, decision_time_col, available_at_col, "member", "eligible", "price_basis"] if len(set(roles)) != len(roles) or set(features) & set(roles): raise ValueError("Feature and output roles collide") + if universe_time_col in { + *entities, + available_at_col, + "member", + "eligible", + "price_basis", + *features, + }: + raise ValueError("The universe clock collides with a different output role") + if universe_time_col not in roles: + roles.append(universe_time_col) extra = [ c for c in universe_frame.columns @@ -193,7 +212,12 @@ def prepare_feature_panel( ) def clock( - frame: pl.DataFrame, column: str, output: str, *, required: bool = False + frame: pl.DataFrame, + column: str, + output: str, + *, + required: bool = False, + range_guard: bool = True, ) -> pl.DataFrame: expr = ( pl.lit(None, pl.Int128) if frame.schema[column] == pl.Null else _instant_ticks(column) @@ -201,7 +225,10 @@ def clock( result = frame.with_columns(expr.alias(output)) if required and result[output].null_count(): raise ValueError(f"{column} must contain known decision/snapshot instants") - if result.filter((pl.col(output) < -(2**63)) | (pl.col(output) > 2**63 - 1)).height: + if ( + range_guard + and result.filter((pl.col(output) < -(2**63)) | (pl.col(output) > 2**63 - 1)).height + ): raise ValueError(f"{column} exceeds Datetime(ns) output range") return result @@ -223,16 +250,48 @@ def clock( if decision_frame.filter(pl.col(prefix + "observation_ticks") > pl.col(cutoff)).height: raise ValueError("Observation cutoff cannot follow its decision instant") universe_frame = clock(universe_frame, available_at_col, known) + universe_from, universe_to, universe_target = ( + "effective_from", + "effective_to", + universe_time_col, + ) + if isinstance(universe_dtype, pl.Datetime): + + def effective_clock(frame: pl.DataFrame, column: str, target: str) -> pl.DataFrame: + dtype = frame.schema[column] + assert isinstance(dtype, pl.Datetime) + value = pl.col(column) + if dtype.time_zone is None: + if contract.timezone is None: + raise ValueError("Naive universe instants require a declared timezone") + value = value.dt.replace_time_zone(contract.timezone) + temporary = target + "_aware" + return clock( + frame.with_columns(value.alias(temporary)), temporary, target, range_guard=False + ).drop(temporary) + + universe_from, universe_to, universe_target = ( + prefix + name for name in ("from", "to", "target") + ) + universe_frame = effective_clock(universe_frame, "effective_from", universe_from) + universe_frame = effective_clock(universe_frame, "effective_to", universe_to) + decision_frame = effective_clock(decision_frame, universe_time_col, universe_target) + if decision_frame[universe_target].null_count(): + raise ValueError("Decision-effective universe keys cannot be unknown") if ( decision_frame[timestamp].null_count() - or decision_frame.select(pl.struct(timestamp, cutoff).is_duplicated().any()).item() + or decision_frame.select( + pl.struct(list(dict.fromkeys([timestamp, cutoff, universe_target]))) + .is_duplicated() + .any() + ).item() ): raise ValueError("Decision observation/instant keys must be nonnull and unique") if universe_frame.select( pl.any_horizontal(pl.col(c).is_null() for c in [*entities, "effective_from"]).any() ).item(): raise ValueError("Universe identity/effective start cannot be unknown") - if universe_frame.filter(pl.col("effective_to") <= pl.col("effective_from")).height: + if universe_frame.filter(pl.col(universe_to) <= pl.col(universe_from)).height: raise ValueError("Universe effective intervals must be nonempty and end-exclusive") template = ( pl.DataFrame(schema=schema) @@ -252,6 +311,8 @@ def clock( "price_basis": pl.String, **{c: universe_frame.schema[c] for c in extra}, } + if universe_time_col not in result_schema: + result_schema[universe_time_col] = decision_frame.schema[universe_time_col] def calculate(frame: pl.DataFrame) -> pl.DataFrame: source = clock(frame, available_at_col, known) @@ -270,6 +331,11 @@ def calculate(frame: pl.DataFrame) -> pl.DataFrame: decision = decision_frame.slice(i, 1) bound = pl.lit(decision[timestamp]).first() instant = decision[cutoff].item() + effective = ( + pl.lit(decision[universe_target].item(), dtype=pl.Int128) + if isinstance(universe_dtype, pl.Datetime) + else pl.lit(decision[universe_time_col]).first() + ) history = source if history_decision_time_col: history = history.filter(pl.col(prefix + "snapshot") == instant) @@ -277,8 +343,8 @@ def calculate(frame: pl.DataFrame) -> pl.DataFrame: raise ValueError("Missing prepared history snapshot for a requested decision") history = history.filter(pl.col(timestamp) <= bound).sort([*entities, timestamp]) facts = universe_frame.filter( - (pl.col("effective_from") <= bound) - & (pl.col("effective_to").is_null() | (pl.col("effective_to") > bound)) + (pl.col(universe_from) <= effective) + & (pl.col(universe_to).is_null() | (pl.col(universe_to) > effective)) & (pl.col(known) <= instant) ) if facts.select(pl.struct(entities).is_duplicated().any()).item(): @@ -354,9 +420,15 @@ def calculate(frame: pl.DataFrame) -> pl.DataFrame: ).alias("eligible"), pl.lit(price_basis).alias("price_basis"), ) + if universe_time_col not in {*keys, decision_time_col}: + result = result.with_columns( + pl.lit(decision[universe_time_col]).first().alias(universe_time_col) + ) output.append(result.select(*result_schema)) result = pl.concat(output) if output else pl.DataFrame(schema=result_schema) - result = result.sort([decision_time_col, timestamp, *entities]) + result = result.sort( + list(dict.fromkeys([decision_time_col, timestamp, universe_time_col, *entities])) + ) if screen is not None: result = result.filter(pl.col(screen) if isinstance(screen, str) else screen) return result diff --git a/tests/test_feature_panel_preparation.py b/tests/test_feature_panel_preparation.py index f201cb52..d8a1b698 100644 --- a/tests/test_feature_panel_preparation.py +++ b/tests/test_feature_panel_preparation.py @@ -51,6 +51,7 @@ def decisions(): { "timestamp": DAYS[2:], "decision_time": [datetime(2025, 7, d.day, 17, tzinfo=UTC) for d in DAYS[2:]], + "effective_session": DAYS[2:], } ) @@ -91,6 +92,68 @@ def test_dated_membership_keeps_preentry_warmup_and_original_feature_clock(lazy) assert_frame_equal(frame, history()) +def test_next_session_decision_uses_current_membership_not_the_prior_price_session(): + requested = pl.DataFrame( + { + "timestamp": [DAYS[2]], + "decision_time": [datetime(2025, 7, 10, 13, tzinfo=UTC)], + "effective_session": [DAYS[3]], + } + ) + facts = universe().with_columns( + pl.when(pl.col("asset") == "A") + .then(pl.lit(DAYS[3])) + .otherwise(pl.col("effective_to")) + .alias("effective_to"), + pl.when(pl.col("asset") == "B") + .then(pl.lit(DAYS[3])) + .otherwise(pl.col("effective_from")) + .alias("effective_from"), + pl.lit(True).alias("member"), + ) + result = prepare(history(), decisions=requested, universe=facts) + assert result["eligible"].to_list() == [False, True] + assert result["average"].to_list() == [2.0, 20.0] + assert result["effective_session"].to_list() == [DAYS[3], DAYS[3]] + + +def test_explicit_instant_universe_clock_preserves_timezone_equivalence_and_exclusive_ns_end(): + base = int( + pl.Series([datetime(2025, 7, 10, 13, tzinfo=UTC)]) + .cast(pl.Datetime("ns", "UTC")) + .cast(pl.Int64)[0] + ) + requested = pl.DataFrame( + { + "timestamp": [DAYS[2]] * 2, + "decision_time": pl.Series([base + 2, base + 3], dtype=pl.Datetime("ns", "UTC")), + } + ) + facts = pl.DataFrame( + { + "asset": ["A"], + "effective_from": pl.Series([base + 1], dtype=pl.Datetime("ns", "America/New_York")), + "effective_to": pl.Series([base + 3], dtype=pl.Datetime("ns", "UTC")), + "available_at": [datetime(2025, 7, 9, tzinfo=UTC)], + "member": [True], + } + ) + result = prepare( + history(), decisions=requested, universe=facts, universe_time_col="decision_time" + ) + a = result.filter(pl.col("asset") == "A") + assert a["decision_time"].cast(pl.Int64).to_list() == [base + 2, base + 3] + assert a["eligible"].to_list() == [True, False] + + +def test_missing_universe_decision_key_is_not_inferred_from_the_price_or_utc_date(): + with pytest.raises(ValueError, match="effective_session"): + prepare(history(), decisions=decisions().drop("effective_session")) + assert prepare(history(), universe=universe().with_columns(pl.lit(None).alias("effective_to")))[ + "eligible" + ].to_list() == [True, False, True, False] + + def test_future_publication_cannot_fill_or_compress_a_required_window(): delayed = history().with_columns( pl.when((pl.col("asset") == "A") & (pl.col("timestamp") == DAYS[1])) @@ -191,6 +254,7 @@ def test_feature_artifact_public_roundtrip_preserves_required_feed_and_eligibili "available_at_col": "known", "decision_time_col": "decision", "eligible_col": "eligible", + "universe_time_col": "effective_session", }, "definition": { "family": "financial", @@ -238,6 +302,7 @@ def test_adjacent_ns_observation_decision_and_publication_keys_do_not_round(): lookbacks={"average": 2}, decisions=requested, universe=facts, + universe_time_col="decision_time", feed_spec=FeedSpec( time_contract_version=2, calendar=None, @@ -267,6 +332,7 @@ def test_instant_observation_cutoff_cannot_follow_the_actual_decision(): frame, decisions=requested, universe=facts, + universe_time_col="decision_time", feed_spec=FeedSpec( time_contract_version=2, calendar=None, timestamp_semantics="event_time" ), @@ -454,8 +520,10 @@ def test_empty_history_does_not_waive_an_undeclared_default_calendar_grid(): @pytest.mark.parametrize("timezone", [None, "America/New_York"]) def test_naive_observation_instants_require_and_use_declared_timezone(timezone): frame = history().with_columns(pl.col("timestamp").cast(pl.Datetime("us")).dt.offset_by("12h")) - requested = decisions().with_columns( - pl.col("timestamp").cast(pl.Datetime("us")).dt.offset_by("12h") + requested = ( + decisions() + .with_columns(pl.col("timestamp").cast(pl.Datetime("us")).dt.offset_by("12h")) + .with_columns(pl.col("timestamp").dt.offset_by("1h").alias("effective_at")) ) facts = universe().with_columns( pl.col("effective_from").cast(pl.Datetime("us")), @@ -464,6 +532,7 @@ def test_naive_observation_instants_require_and_use_declared_timezone(timezone): options = { "decisions": requested, "universe": facts, + "universe_time_col": "effective_at", "feed_spec": FeedSpec( time_contract_version=2, calendar=None, From 0483f8385e1a8ceba0a36dc4e9047df193012084 Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Tue, 20 Oct 2026 21:08:54 -0400 Subject: [PATCH 12/19] fix(calendar): distinguish invalid units from dependency deprecations --- src/ml4t/engineer/core/bar_grid.py | 9 +- .../compat/test_calendar_duration_runtime.py | 90 +++++++++++++++++++ 2 files changed, 96 insertions(+), 3 deletions(-) create mode 100644 tests/compat/test_calendar_duration_runtime.py diff --git a/src/ml4t/engineer/core/bar_grid.py b/src/ml4t/engineer/core/bar_grid.py index 31c4d031..78d789b9 100644 --- a/src/ml4t/engineer/core/bar_grid.py +++ b/src/ml4t/engineer/core/bar_grid.py @@ -51,14 +51,17 @@ def validate_declared_grid( if isinstance(frequency, str): import pandas as pd + duration_warning = getattr(pd.errors, "PandasDeprecationWarning", FutureWarning) try: with warnings.catch_warnings(): + # Deprecated pandas unit spellings are invalid declarations; + # unrelated dependency deprecations do not invalidate a duration. warnings.simplefilter("error", FutureWarning) - warnings.simplefilter("error", DeprecationWarning) + warnings.simplefilter("error", duration_warning) parsed = pd.Timedelta("1D" if frequency == "1d" else frequency) - except (ValueError, TypeError, FutureWarning, DeprecationWarning) as exc: + except (ValueError, TypeError, FutureWarning, duration_warning) as exc: raise ValueError("declared grid data_frequency must be a fixed duration") from exc - if not isinstance(parsed, pd.Timedelta) or parsed <= pd.Timedelta(0): + if not isinstance(parsed, pd.Timedelta) or parsed <= timedelta(0): raise ValueError("declared grid data_frequency requires a positive finite duration") frequency = parsed if not isinstance(frequency, timedelta): diff --git a/tests/compat/test_calendar_duration_runtime.py b/tests/compat/test_calendar_duration_runtime.py new file mode 100644 index 00000000..e1e4073a --- /dev/null +++ b/tests/compat/test_calendar_duration_runtime.py @@ -0,0 +1,90 @@ +"""Valid public grids remain valid in pandas2/NumPy2 legacy-coexistence environments.""" + +from datetime import date + +import polars as pl +import pytest +from ml4t.specs import FeedSpec + +from ml4t.engineer import compute_features + + +def bars(timestamp: pl.Series) -> pl.DataFrame: + return pl.DataFrame( + {"timestamp": timestamp, "asset": ["A"] * 4, "close": [100.0, 110.0, 120.0, 130.0]} + ).with_columns( + pl.col("close").alias("open"), + pl.col("close").alias("high"), + pl.col("close").alias("low"), + pl.lit(100).alias("volume"), + ) + + +def test_daily_public_feature_grid_does_not_misclassify_a_dependency_warning_as_bad_frequency(): + frame = bars(pl.Series([date(2025, 7, d) for d in (7, 8, 9, 10)])) + result = compute_features( + frame.lazy(), + [{"name": "sma", "params": {"period": 2}}], + feed_spec=FeedSpec( + time_contract_version=2, + calendar="NYSE", + entity_col="asset", + timestamp_semantics="session_label", + bar_type="time", + data_frequency="1d", + ), + ).collect() + assert result["sma"].to_list() == [None, 105.0, 115.0, 125.0] + assert result["timestamp"].equals(frame["timestamp"]) + + +@pytest.mark.parametrize( + "frequency,step", + [("1h", 3_600_000_000_000), ("30m", 1_800_000_000_000), ("1ms", 1_000_000), ("1ns", 1)], +) +def test_intraday_public_declared_grid_keeps_exact_frequency_and_rows_across_dependency_versions( + frequency, step +): + opened = 1751846400000000000 + frame = bars(pl.Series([opened + i * step for i in range(4)], dtype=pl.Datetime("ns", "UTC"))) + contract = FeedSpec( + time_contract_version=2, + calendar="24/7", + entity_col="asset", + timezone="UTC", + timestamp_semantics="event_time", + bar_type="time", + data_frequency=frequency, + bar_grid_anchor="session_open", + bar_label="left", + partial_bar_policy="exclude", + ) + result = compute_features(frame, [{"name": "sma", "params": {"period": 2}}], feed_spec=contract) + assert result["sma"].to_list() == [None, 105.0, 115.0, 125.0] + assert result["timestamp"].equals(frame["timestamp"]) + + +@pytest.mark.parametrize("frequency", ["NaT", "0h", "1month", "1M", "1Y", "1T", "1H"]) +def test_invalid_or_deprecated_ambiguous_frequencies_still_fail_before_calculation(frequency): + step = 60_000_000_000 if frequency == "1T" else 3_600_000_000_000 + frame = bars( + pl.Series( + [1751846400000000000 + i * step for i in range(4)], dtype=pl.Datetime("ns", "UTC") + ) + ) + with pytest.raises(ValueError, match="frequency"): + compute_features( + frame, + ["sma"], + feed_spec=FeedSpec( + time_contract_version=2, + calendar="24/7", + timestamp_semantics="event_time", + timezone="UTC", + bar_grid_anchor="session_open", + bar_label="left", + partial_bar_policy="exclude", + bar_type="time", + data_frequency=frequency, + ), + ) From cce590132846dbc7d558971b2d5a7f703d5cd9a2 Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Tue, 20 Oct 2026 21:19:38 -0400 Subject: [PATCH 13/19] feat(features): distinguish confirmed missing inputs from unknown history --- docs/user-guide/features.md | 8 ++- src/ml4t/engineer/artifacts/features.py | 12 +++- src/ml4t/engineer/features/preparation.py | 30 ++++++--- tests/test_feature_panel_preparation.py | 74 +++++++++++++++++++++++ 4 files changed, 115 insertions(+), 9 deletions(-) diff --git a/docs/user-guide/features.md b/docs/user-guide/features.md index 3bc7b563..c488dd23 100644 --- a/docs/user-guide/features.md +++ b/docs/user-guide/features.md @@ -355,6 +355,12 @@ are bare per-asset expressions; the preparer applies entity grouping. Declare ea finite row lookback and all input columns. Only complete, known, usable required windows become eligible. Unavailable or input-masked rows remain as null inputs, not shorter windows. A missing endpoint does not carry an earlier feature forward. +For kernels with deliberate omission rules, select +`missing_input_policy="known_missing"` to allow confirmed null/NaN or input-masked +values within the full original window. Every row still needs a publication time +known by the decision. Unknown/future rows and infinite inputs remain ineligible. +This option neither adds pre-listing rows nor fills absent scheduled bars; consume +an explicitly supplied complete panel when that is what the original loader emits. Undefined/nonfinite outputs, including undefined fields in regression Structs, cannot become eligible. The output retains the observation/decision clocks, conservative contributing availability, membership, eligibility, the declared @@ -377,7 +383,7 @@ not a substitute for information-time validation. Owned `FeatureSchema` records optional `available_at_col`, `decision_time_col` and `eligible_col`, plus `universe_time_col`. `FeatureDefinition` records the canonical `feed_spec`, -`window_unit`, `price_basis` and finite `lookbacks`, with existing source-artifact +`window_unit`, `price_basis`, finite `lookbacks` and optional `missing_input_policy`, with existing source-artifact lineage and join keys. `FeatureSpec.to_dict()` preserves omitted calendar versus explicit null; unused fields leave the old wire keys unchanged. Store actual typed frames separately, for example with Parquet. These records do not select diff --git a/src/ml4t/engineer/artifacts/features.py b/src/ml4t/engineer/artifacts/features.py index 2d5e827f..ff50dd2f 100644 --- a/src/ml4t/engineer/artifacts/features.py +++ b/src/ml4t/engineer/artifacts/features.py @@ -54,6 +54,7 @@ class FeatureDefinition: window_unit: WindowUnit | None = None price_basis: str | None = None lookbacks: dict[str, int] | None = None + missing_input_policy: str | None = None def __post_init__(self) -> None: if self.feed_spec is not None: @@ -62,6 +63,8 @@ def __post_init__(self) -> None: object.__setattr__(self, "window_unit", WindowUnit(self.window_unit)) if self.price_basis is not None and not self.price_basis.strip(): raise ValueError("price_basis must be a nonempty declaration") + if self.missing_input_policy not in {None, "complete", "known_missing"}: + raise ValueError("missing_input_policy must be complete or known_missing") if self.lookbacks is not None: if any( not isinstance(name, str) @@ -90,6 +93,7 @@ def from_mapping(cls, mapping: dict[str, Any] | None) -> FeatureDefinition: window_unit=mapping.get("window_unit"), price_basis=optional_str(mapping.get("price_basis")), lookbacks=mapping.get("lookbacks"), + missing_input_policy=optional_str(mapping.get("missing_input_policy")), ) @@ -106,7 +110,13 @@ def to_dict(self) -> dict[str, Any]: for name in ("available_at_col", "decision_time_col", "eligible_col", "universe_time_col"): if payload["schema"][name] is None: del payload["schema"][name] - for name in ("feed_spec", "window_unit", "price_basis", "lookbacks"): + for name in ( + "feed_spec", + "window_unit", + "price_basis", + "lookbacks", + "missing_input_policy", + ): if payload["definition"][name] is None: del payload["definition"][name] if self.definition.feed_spec is not None: diff --git a/src/ml4t/engineer/features/preparation.py b/src/ml4t/engineer/features/preparation.py index 9ff63cf3..abdb846b 100644 --- a/src/ml4t/engineer/features/preparation.py +++ b/src/ml4t/engineer/features/preparation.py @@ -1,7 +1,7 @@ """Decision-time feature eligibility over caller/Data-prepared native histories.""" from collections.abc import Mapping, Sequence -from typing import Any +from typing import Any, Literal import polars as pl from ml4t.specs import UNSET_CALENDAR, FeedSpec, TimestampSemantics, WindowUnit @@ -44,6 +44,7 @@ def prepare_feature_panel( decision_time_col: str = "decision_time", universe_time_col: str = "effective_session", history_decision_time_col: str | None = None, + missing_input_policy: Literal["complete", "known_missing"] = "complete", input_mask: str | pl.Expr | None = None, screen: str | pl.Expr | None = None, ) -> pl.DataFrame | pl.LazyFrame: @@ -64,7 +65,10 @@ def prepare_feature_panel( carried forward. The declared price_basis is retained, not calculated. Supply bare per-asset expressions, their finite row lookbacks, and every input - column they require. Full required windows must contain known, usable inputs. + column they require. Full required windows must contain known, usable inputs + by default. known_missing permits confirmed null/NaN or input-masked values + to follow the supplied kernel's missing-value definition. All original rows + must still have known publication times; infinite inputs remain invalid. Unavailable/masked rows stay in history with null inputs, never compressed or filled. A final screen runs only after all values and eligibility are computed. Cross-sectional transforms operate on the resulting eligible population. @@ -88,6 +92,8 @@ def prepare_feature_panel( raise ValueError("Declare distinct input_columns for the supplied expressions") if not isinstance(price_basis, str) or not price_basis.strip(): raise ValueError("Declare a nonempty price_basis") + if missing_input_policy not in {"complete", "known_missing"}: + raise ValueError("missing_input_policy must be complete or known_missing") if any( value is not None and not isinstance(value, str | pl.Expr) for value in (input_mask, screen) ): @@ -207,8 +213,8 @@ def prepare_feature_panel( for c in [*schema, *universe_frame.columns, *decision_frame.columns, *features] ): raise ValueError("Input columns collide with reserved feature preparation names") - known, cutoff, source_usable, feature_known = ( - prefix + c for c in ("known", "cutoff", "usable", "time") + known, cutoff, source_usable, source_ready, feature_known = ( + prefix + c for c in ("known", "cutoff", "usable", "ready", "time") ) def clock( @@ -358,13 +364,23 @@ def calculate(frame: pl.DataFrame) -> pl.DataFrame: group_cols=entities, ) usable = pl.col(known).is_not_null() & (pl.col(known) <= instant) + confirmed = usable for column in input_columns: - usable &= _usable_value(pl.col(column), schema[column]) + value = pl.col(column) + valid = _usable_value(value, schema[column]) + missing = value.is_null() + if schema[column].is_float(): + missing |= value.is_nan() + confirmed &= valid | missing + usable &= valid if input_mask is not None: usable &= ( pl.col(input_mask) if isinstance(input_mask, str) else input_mask ).fill_null(False) - history = history.with_columns(usable.alias(source_usable)) + history = history.with_columns( + usable.alias(source_usable), + (usable if missing_input_policy == "complete" else confirmed).alias(source_ready), + ) assets = pl.concat( [history.filter(pl.col(known) <= instant).select(entities), facts.select(entities)] ).unique() @@ -374,7 +390,7 @@ def calculate(frame: pl.DataFrame) -> pl.DataFrame: available = [] for name in features: count = lookbacks[name] - full = pl.col(source_usable).cast(pl.Int64).rolling_sum(count) == count + full = pl.col(source_ready).cast(pl.Int64).rolling_sum(count) == count available.append( pl.when(full) .then(pl.col(known).cast(pl.Int64).rolling_max(count)) diff --git a/tests/test_feature_panel_preparation.py b/tests/test_feature_panel_preparation.py index d8a1b698..fea9d024 100644 --- a/tests/test_feature_panel_preparation.py +++ b/tests/test_feature_panel_preparation.py @@ -168,6 +168,78 @@ def test_future_publication_cannot_fill_or_compress_a_required_window(): assert a["available_at"].null_count() == 2 +@pytest.mark.parametrize("lazy", [False, True]) +def test_explicit_known_missing_profile_uses_kernel_definition_without_changing_rows(lazy): + frame = history().with_columns( + pl.when((pl.col("asset") == "A") & (pl.col("timestamp") == DAYS[1])) + .then(None) + .otherwise(pl.col("close")) + .alias("close") + ) + strict = prepare(frame) + result = prepare(frame.lazy() if lazy else frame, missing_input_policy="known_missing") + result = result.collect() if lazy else result + assert result.select("timestamp", "asset").rows() == strict.select("timestamp", "asset").rows() + a = result.filter(pl.col("asset") == "A") + assert strict.filter(pl.col("asset") == "A")["eligible"].to_list() == [False, False] + assert a["average"].to_list() == [2.0, 3.5] + assert a["eligible"].to_list() == [True, True] + assert a["available_at"].to_list() == [ + datetime(2025, 7, 9, 16, tzinfo=UTC), + datetime(2025, 7, 10, 16, tzinfo=UTC), + ] + + +@pytest.mark.parametrize("value", [None, float("nan"), float("inf"), float("-inf")]) +def test_known_missing_profile_never_omits_unknown_future_or_invalid_infinite_inputs(value): + frame = history().with_columns( + pl.when((pl.col("asset") == "A") & (pl.col("timestamp") == DAYS[1])) + .then(pl.lit(value, dtype=pl.Float64)) + .otherwise(pl.col("close")) + .alias("close") + ) + if value is None or value != value: + frame = frame.with_columns( + pl.when((pl.col("asset") == "A") & (pl.col("timestamp") == DAYS[1])) + .then(None) + .otherwise(pl.col("available_at")) + .alias("available_at") + ) + result = prepare(frame, missing_input_policy="known_missing") + assert result.filter(pl.col("asset") == "A")["eligible"].to_list() == [False, False] + future = history().with_columns( + pl.when(pl.col("timestamp") == DAYS[1]) + .then(pl.lit(datetime(2025, 7, 10, 18, tzinfo=UTC))) + .otherwise(pl.col("available_at")) + .alias("available_at") + ) + assert not prepare(future, missing_input_policy="known_missing")["eligible"].any() + + +def test_known_missing_profile_preserves_masks_full_window_and_undefined_outputs(): + result = prepare( + history(), input_mask=pl.col("close") > 1, missing_input_policy="known_missing" + ) + assert result.filter(pl.col("asset") == "A")["average"].to_list() == [2.5, 3.0] + missing = history().with_columns(pl.lit(float("nan")).alias("close")) + result = prepare(missing, missing_input_policy="known_missing") + assert not result["eligible"].any() + short = history().filter(pl.col("timestamp") >= DAYS[1]) + result = prepare(short, missing_input_policy="known_missing") + a = result.filter(pl.col("asset") == "A") + assert a["average"].to_list() == [None, 3.0] + assert a["eligible"].to_list() == [False, True] + + +def test_custom_finite_expression_is_evaluated_per_entity_not_over_the_sorted_panel(): + result = prepare( + history(), + features={"tail_sum": pl.col("close").tail(3).sum()}, + lookbacks={"tail_sum": 3}, + ) + assert result["tail_sum"].to_list() == [6.0, 60.0, 9.0, 90.0] + + def test_changing_only_final_screen_does_not_change_warmup_or_group_values(): before = prepare(history()) screened = prepare(history(), screen=pl.col("asset") == "A") @@ -262,6 +334,7 @@ def test_feature_artifact_public_roundtrip_preserves_required_feed_and_eligibili "window_unit": "observation" if calendar in (None, "omitted") else "expected_bar", "price_basis": "split_adjusted", "lookbacks": {"average": 3}, + "missing_input_policy": "known_missing", }, } spec = FeatureSpec.from_mapping(json.loads(json.dumps(payload))) @@ -398,6 +471,7 @@ def test_missing_endpoint_or_market_coverage_does_not_carry_stale_features_forwa {"input_columns": ["close", "close"]}, {"input_columns": ["asset"]}, {"price_basis": ""}, + {"missing_input_policy": "ignore"}, {"input_mask": 1}, {"screen": 1}, {"timestamp_col": "other"}, From 3c5665cd2d08bad2dc5dc25b4a97573a84a7de88 Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Tue, 20 Oct 2026 21:27:47 -0400 Subject: [PATCH 14/19] docs(migration): map Pipeline methods and provide executable native ports --- docs/api/pipeline-ports.md | 87 +++++++++++++++ docs/user-guide/pipeline-migration.md | 146 ++++++++++++++++++++++++++ examples/pipeline_expression_ports.py | 65 ++++++++++++ examples/pipeline_migration.py | 98 +++++++++++++++++ mkdocs.yml | 2 + 5 files changed, 398 insertions(+) create mode 100644 docs/api/pipeline-ports.md create mode 100644 docs/user-guide/pipeline-migration.md create mode 100644 examples/pipeline_expression_ports.py create mode 100644 examples/pipeline_migration.py diff --git a/docs/api/pipeline-ports.md b/docs/api/pipeline-ports.md new file mode 100644 index 00000000..5edd7a2a --- /dev/null +++ b/docs/api/pipeline-ports.md @@ -0,0 +1,87 @@ +# Pipeline operation ports + +This reference maps the factor, filter and classifier surface inspected in +Zipline Reloaded 3.2.0 source `cf649ac06fddce76f6caa2403cd8494ac84f72ad`. +Routes are native Polars operations or explicit recipes, not legacy graph +objects. `factor_expression` and `return_factor_features` live under +`ml4t.engineer.compat.zipline`; rolling methods live under `features.rolling`, +relationships under `features.paired`, and selections under `features.selection`. +Use the [migration how-to](../user-guide/pipeline-migration.md) to prepare inputs. + +## Named factors and aliases + +| Legacy name | Native route | Definition and choices | +|---|---|---| +| `Returns` | `factor_expression` or precise `features.utils.arithmetic.returns` | N prices become N-1 changes; signed reference denominator; fraction, not percentage points. | +| `DailyReturns` | Same recipe with two prices | One change; retain both endpoint quote masks. | +| `PercentChange`, `RateOfChangePercentage` | `absolute_percent_change` or recipes | Absolute reference denominator; fractional result, despite legacy naming. | +| `SimpleMovingAverage` | `finite_mean` or recipe | Full row window, omit null/NaN values from mean denominator. | +| `WeightedAverageValue`, `VWAP` | `finite_weighted_mean` or recipes | Independent NaN sums of value*weight and weights; a missing value does not erase its weight. | +| `AverageDollarVolume` | `average_dollar_volume` or recipe | Sum of price*volume divided by fixed N; distinguish quote units and volume basis. | +| `LinearWeightedMovingAverage` | `linear_weighted_mean` or recipe | Weights 1..N and fixed weight denominator; omit missing numerator terms. | +| `ExponentialWeightedMovingAverage`, `EWMA` | `finite_ewma` or recipes | Finite normalized weights, missing values propagate. Not native recursive EMA. | +| `ExponentialWeightedMovingStdDev`, `EWMSTD` | `finite_ewm_std` or recipes | Same finite weights plus unbiased weighted variance correction. | +| EWMA/EWMSTD `.from_span`, `.from_halflife`, `.from_center_of_mass` | Recipe suffix or `exponential_decay` | Exactly one weight selector; finite-window semantics remain unchanged. | +| `AnnualizedVolatility` | `annualized_volatility` or recipe | Population stddev, default annualization factor 252 independent of calendar; default price recipe derives daily returns with extra warmup. Supplied returns avoid that dependency. | +| `MaxDrawdown` | `finite_max_drawdown` or recipe | Legacy selected largest absolute fall and trough denominator; not ordinary peak-denominator portfolio drawdown. | +| `RSI` | `rsi_cutler` or recipe | 15-price default, 14 finite changes, omit changes adjacent to missing prices; flat windows undefined. Native `rsi` is Wilder. | +| `BollingerBands` | `finite_bollinger_bands` or recipe | Population stddev, default multiplier 2, upper/middle/lower named outputs. | +| `Aroon` | `finite_aroon` or recipe | First extrema ties; indices divided by N-1. Recipe preserves legacy low/high input order. | +| `FastStochasticOscillator` | `fast_stochastic` or recipe | Unsmoothed K, no D output; full price window. | +| `IchimokuKinkoHyo` | `finite_ichimoku` or recipe | Unshifted named outputs; legacy zero Tenkan/Kijun options select the entire window, not an inferred default. | +| `MovingAverageConvergenceDivergenceSignal`, `MACDSignal` | `finite_macd_signal` or recipes | Finite normalized exponential windows; not recursively seeded TA-Lib MACD. | +| `TrueRange` | `finite_true_range` or recipe | Two-price dependency, NaN-max of high-low and gaps; not ATR. | +| `RollingPearson`, `RollingSpearman` | Recipes or paired rolling functions | Full paired windows, explicit missing-count allowance; Spearman averages ties inside each window. | +| Factor `.pearsonr`, `.spearmanr`, `.linear_regression`; `RollingLinearRegression` | Paired functions or recipes | Explicit target alignment; regression Struct contains alpha, beta, r_value, p_value, stderr and requires the optional `stats` extra. | +| `RollingPearsonOfReturns`, `RollingSpearmanOfReturns`, `RollingLinearRegressionOfReturns` | `return_factor_features` | Align common grid before deriving both price returns; `returns_length` counts prices, correlation/regression length counts return rows. | +| `SimpleBeta` | `return_factor_features` | Two-price daily returns, default maximum missing fraction .25 converted to a row count. Target is declared stable ID, not opaque Asset. | +| `BusinessDaysSincePreviousEvent`, `BusinessDaysUntilNextEvent` | `event_business_days` or recipes | Supplied known event Dates, end-exclusive Monday-Friday arithmetic including exchange holidays. Not exchange-session windows. | +| `PeerCount` | `cross_section(method="peer_count", groupby=...)` | Eligible peers with known classifier; explicit missing sentinel to null first. | +| Numeric, categorical or Boolean `Latest` | `pl.col(...)` or `Latest` recipe | Preserve dtype and missingness; no implicit historical fill. | +| `Clip` and Factor `.clip` | `pl.col(...).clip(...)` or recipe | Explicit bounds, nulls preserved; apply output mask after history. | +| `Factor`, `CustomFactor`, `RecarrayField` | Native expressions/feature registry, Struct field selection | Supporting graph roles are ports, not automatically executable legacy objects. | + +## Factor methods and expression operators + +| Operation | Native route | Material choices | +|---|---|---| +| Arithmetic `+`, `-`, `*`, `/`, `**`, `%`, reflected forms, unary minus | Corresponding `pl.Expr` operations | Native null and integer overflow rules; do not cast large tick prices before differencing. | +| Comparisons `<`, `<=`, `>`, `>=`, `.eq`, `!=`; Boolean `&`, `|`, `~` | Native expressions, explicit `.fill_null(False)` for legacy filters | Do not use Python `and`/`or` on expressions. | +| `sin`, `cos`, `tan`, `arcsin`, `arccos`, `arctan`, `sinh`, `cosh`, `tanh`, `arcsinh`, `arccosh`, `arctanh` | Same-named Polars expression functions | Domain-invalid real values remain undefined. | +| `log`, `log10`, `log1p`, `exp`, `expm1`, `sqrt`, `abs` | `.log()`, `.log(10)`, `.log1p()`, `.exp()`, `numpy.expm1(expr)`, `.sqrt()`, `.abs()` | Use expm1 directly for tiny inputs, not subtraction from a rounded exponential. | +| `.isnan`, `.notnan`, `.isfinite` | `is_missing`, inverse `is_missing`, `.is_finite().fill_null(False)` | Native nulls join NaNs in missingness; infinities are not missing but are nonfinite. | +| `.rank` | `cross_section(method="rank")` | ordinal/min/max/dense/average, ascending direction; ordinal ties use stable ascending IDs, not input row order. | +| `.top`, `.bottom` | Same-named cross-section methods | Exact N, stable ordinal ties; mask determines input population. | +| `.demean`, `.zscore` | Same-named cross-section methods | Contemporary date/group population, population ddof=0 default; zero variance undefined. Not a train-fitted scaler. | +| `.winsorize` | Same-named cross-section method | Floor/ceil legacy order-statistic bounds, not linear quantile clipping. | +| `.quantiles`, `.quartiles`, `.quintiles`, `.deciles` | `cross_section(method="quantiles", bins=...)` | 0-based labels, linear edges, duplicate-edge error, null exclusions. Aliases use 4/5/10 bins. | +| `.percentile_between` | Same-named cross-section method | Inclusive linear bounds; convert legacy percent values 0..100 to fractions 0..1. | +| `.mean`, `.stddev`, `.max`, `.min`, `.median`, `.sum`, `.notnull_count` daily summaries | Mask input then aggregate with native expressions over decision-time keys | Omit null/NaN, population stddev; sum of all missing values is zero, count is zero. Return one row per decision, not one per asset. | +| Asset slicing and named multi-output fields | Exact keyed filter/join; `.struct.field(name)` | Stable keys and explicit output fields replace opaque Asset lookup/record wrappers. | + +## Filters and classifiers + +| Legacy surface | Native route | Material choices | +|---|---|---| +| `All`, `Any`, `AtLeastN` | `rolling_filter` with all/any/at_least | Full original windows, missing predicates false; apply `.over(entity_col)`. | +| `AllPresent` | `rolling_filter(~is_missing(value), method="all")` | Known row presence is separate from actual value usability. | +| `MaximumFilter` | Eligible value equals its group's maximum | Retain all maximal ties; this differs from exact-count top(1). | +| `NullFilter`, `NotNullFilter` | `is_missing`, inverse | Normalize legacy sentinels explicitly; no automatic -1 interpretation. | +| `PercentileFilter` | Cross-section percentile_between | Inclusive bounds and eligible population. | +| `SingleAsset`, `StaticAssets`, `StaticSids` | Stable-ID equality or `.is_in(...)` | Explicitly resolve legacy Asset.sid through its owner; static membership is not a historical universe. | +| `ArrayPredicate`, `NumExprFilter` | Native elementwise Boolean expressions | Preserve declared null behavior; do not execute arbitrary legacy expression strings. | +| `Filter.if_else` | `pl.when(predicate).then(left).otherwise(right)` | Declare unknown-predicate handling explicitly. | +| `CustomFilter`, base `Filter` | Boolean expressions or reusable registered features | No graph interpreter; supply required history and knowledge clocks. | +| Classifier `eq`, `ne`, `element_of` | Equality, inequality, `.is_in(...)` | Map missing sentinel to null; fill null Boolean outputs as intended. | +| `startswith`, `endswith`, `has_substring` | `.str.starts_with`, `.str.ends_with`, `.str.contains(..., literal=True)` | Literal substring, not accidental regex interpretation. | +| `matches` | Explicit `re.compile(pattern).match(value)` mapping | Legacy Python re.match starts at the beginning and supports Python regex features. Polars unanchored Rust regex is not a general equivalent. | +| `relabel` | `.replace_strict(mapping, default=None)` or explicit mapping function | Preserve nulls and declare behavior for unknown labels. | +| `Quantiles` | Cross-section quantiles | Same bins/interpolation/missing choices as Factor method. | +| `Everything` | A declared constant valid group | Missing classifications require a separate policy, not automatic inclusion. | +| `Classifier.peer_count` | Cross-section peer_count | Contemporary eligible group count, not full static universe size. | +| `CustomClassifier`, base `Classifier` | Supplied categories or typed native expression | Retain dtype, nulls and known-at information; encode numeric sentinels only at an explicit receiver boundary. | + +Daily summaries, elementary operators and custom ports use existing Polars +functionality rather than duplicate Engineer kernels. The reference identifies +routes and material differences; listing a route alone is not a claim that every +possible legacy graph or regular expression has been qualified. diff --git a/docs/user-guide/pipeline-migration.md b/docs/user-guide/pipeline-migration.md new file mode 100644 index 00000000..f0200604 --- /dev/null +++ b/docs/user-guide/pipeline-migration.md @@ -0,0 +1,146 @@ +# Migrate Pipeline feature selection + +Port finite factors and dated screens to native Polars expressions. Keep source +history preparation in ML4T Data and order/account processing in ML4T Backtest. +Engineer does not run a Zipline graph or import Zipline in production. + +## Replace the calculation and retain its input contract + +The original momentum example constructs `RSI().top(3)` and `RSI().bottom(3)` in a +Pipeline. Its pricing loader supplies the previous session's close before the +current session opens. Preserve those different clocks when replacing the graph. + +Supply a long price panel with stable IDs, an observation timestamp and actual +publication times. Supply decisions with an observation cutoff, an aware +`decision_time` and an `effective_session` for membership. Supply dated universe +facts with `effective_from`, exclusive `effective_to`, `available_at`, `member` +and any classifications. Unknown membership is not membership. Data selects +source revisions, identities and adjustment perspectives before this boundary. + + +```python +from datetime import UTC, date, datetime +import polars as pl +from ml4t.specs import FeedSpec +from ml4t.engineer.compat.zipline import factor_expression +from ml4t.engineer.features.preparation import prepare_feature_panel +from ml4t.engineer.features.selection import cross_section + +days = [date(2025, 7, d) for d in (7, 8, 9)] +prices = pl.DataFrame({ + "timestamp": days * 3, + "asset": [17] * 3 + [23] * 3 + [31] * 3, + "close": [10., 11., 10., 10., 12., 13., 10., 9., 8.], + "available_at": [datetime(2025, 7, d.day, 20, tzinfo=UTC) for d in days] * 3, +}) +decisions = pl.DataFrame({ + "timestamp": [days[-1]], + "decision_time": [datetime(2025, 7, 10, 13, tzinfo=UTC)], + "effective_session": [date(2025, 7, 10)], +}) +universe = pl.DataFrame({ + "asset": [17, 23, 31], "effective_from": [date(2025, 7, 10)] * 3, + "effective_to": pl.Series([None] * 3, dtype=pl.Date), + "available_at": [datetime(2025, 7, 1, tzinfo=UTC)] * 3, + "member": [True] * 3, +}) +# This small example deliberately counts observation rows, not scheduled bars. +feed = FeedSpec(time_contract_version=2, calendar=None, entity_col="asset") +rsi = factor_expression("RSI", inputs=["close"], window_length=3) +assert isinstance(rsi, pl.Expr) +signals = prepare_feature_panel( + prices.lazy(), features={"rsi": rsi}, lookbacks={"rsi": 3}, + input_columns=["close"], decisions=decisions, universe=universe, + price_basis="raw", feed_spec=feed, missing_input_policy="known_missing", +).with_columns( + cross_section("rsi", method="top", n=1, timestamp_col="decision_time", + mask="eligible").alias("longs"), + cross_section("rsi", method="bottom", n=1, timestamp_col="decision_time", + mask="eligible").alias("shorts"), +).filter(pl.col("member").fill_null(False)).collect() +assert signals["rsi"].to_list() == [50., 100., 0.] +assert signals.filter("longs")["asset"].to_list() == [23] +assert signals.filter("shorts")["asset"].to_list() == [31] +assert signals["timestamp"].to_list() == [date(2025, 7, 9)] * 3 +assert signals["effective_session"].to_list() == [date(2025, 7, 10)] * 3 +``` + +The default legacy RSI uses 15 prices, so use `window_length=15` and +`lookbacks={"rsi": 15}` for that workflow. Native `rsi` remains Wilder RSI; +the recipe selects the separately named `rsi_cutler` definition. Select `n=3` +to reproduce the original selection count. + +For scheduled daily inputs, install `ml4t-engineer[calendars]` and declare +`calendar="NYSE"`, `timestamp_semantics="session_label"`, +`data_frequency="1d"`, and `bar_type="time"` in the feed instead of selecting +row mode. Omitted calendar follows feed metadata, then NYSE. Explicit `None` +disables calendars and does not mean a continuous schedule. Intraday bars also +need timezone, anchor, label side and partial-bar policy. Do not infer them from +the observations. Missing scheduled rows fail before a return or feature is +interpreted as one-bar data. + +The `known_missing` option permits a kernel's declared null/NaN omission rules +over a full, supplied row window. It does not invent pre-listing rows or treat +unknown publication as confirmed absence. A loader that emits a complete panel +with explicit pre-listing null cells must hand over those actual cells. Keep the +strict default when every input must be usable. Apply an input mask before the +calculation and a final screen after it. Filtering history to current membership +loses pre-entry warmup. Partition cross-sections by decision time, not the prior +price session; add the universe key if one instant has multiple contexts. + +## Port custom terms + +Run `uv run python examples/pipeline_migration.py` in the source checkout. The +example includes a finite custom mean, a positive-value custom filter and an +explicit sector-code classifier alongside the momentum screen. Replace a +`CustomFactor.compute` loop with a bare per-entity expression and declare its +finite lookback and all input columns. The preparer supplies entity grouping. +For reusable calculations, use the existing feature decorator and registry. +Replace `CustomFilter` with a Boolean expression, and `CustomClassifier` with a +supplied classification or an explicit native mapping. Retain missing categories +as null; do not convert them into a real category. Select multi-output Struct +fields explicitly. Opaque legacy objects, loaders and graph execution are not +accepted as expressions. + +## Adapt presentation only where needed + +Keep the native table for subsequent feature or label preparation. For a caller +that still expects a pandas MultiIndex, select `effective_session` and stable +asset IDs as its keys, then convert at this explicit eager boundary. With +PyArrow installed, `signals.select("effective_session", "asset", "longs", +"shorts").to_pandas(date_as_object=True).set_index(["effective_session", +"asset"])` preserves civil Date labels rather than claiming they are instants. +Rename the index levels to `date` and `asset` if that is the caller's contract. +For instant keys, preserve their timezone and nanoseconds. Never go through +Python `datetime` values to reconstruct a nanosecond index. + +If a retained caller needs Zipline Asset objects, map stable IDs through that +caller's AssetFinder explicitly. Do not stringify compound IDs or introduce a +Zipline dependency to native calculations. Native compound entity columns and +Struct IDs can remain unchanged; an old pandas caller must declare its own +hashable compound-key representation. This is a caller presentation edit, not +an automatic emulation of `Pipeline.run_pipeline`. + +## Preserve prepared-history metadata + +Consume `AdjustedHistory.prices` and the canonical feed in its +`metadata["feed_spec"]`. Use the declared price column and price basis, not a +hard-coded raw close. Data's daily frequency declaration is not by itself a +complete Engineer time-bar grid; add the fixture/feed owner's explicit bar-grid +declarations. For per-decision adjustment perspectives, retain the snapshot +decision key and pass `history_decision_time_col`. Do not reuse retrospectively +adjusted history as a causal snapshot. Store Data provenance alongside the +FeatureSpec and typed frame. + +The installed migration checks qualified Engineer development source +`cce590132846dbc7d558971b2d5a7f703d5cd9a2` against Zipline Reloaded 3.2.0 +(`cf649ac06fddce76f6caa2403cd8494ac84f72ad`), pandas 2.3.3 and NumPy 2.5.4. +Both original momentum PipelineEngine journeys used the real shifted pricing +loader and matched 135 keyed rows, RSI values and long/short selections each, +with complete quotes and a known-missing close, listing/delisting and supplied +lifetimes. This is development qualification, not an available release or proof +of order, fill, commission, leverage or portfolio-performance equivalence. + +See the [operation reference](../api/pipeline-ports.md) for individual methods, +defaults and material differences, and [native features](features.md) for the +preparation and expression contracts. diff --git a/examples/pipeline_expression_ports.py b/examples/pipeline_expression_ports.py new file mode 100644 index 00000000..69c5dace --- /dev/null +++ b/examples/pipeline_expression_ports.py @@ -0,0 +1,65 @@ +"""Native ports of elementwise math and daily summaries, not a graph interpreter.""" + +import numpy as np +import polars as pl + + +def elementary_features(frame: pl.DataFrame | pl.LazyFrame) -> pl.DataFrame | pl.LazyFrame: + value = pl.col("value") + functions = { + "sin": value.sin(), + "cos": value.cos(), + "tan": value.tan(), + "arcsin": value.arcsin(), + "arccos": value.arccos(), + "arctan": value.arctan(), + "sinh": value.sinh(), + "cosh": value.cosh(), + "tanh": value.tanh(), + "arcsinh": value.arcsinh(), + "arccosh": value.arccosh(), + "arctanh": value.arctanh(), + "log": value.log(), + "log10": value.log(10), + "log1p": value.log1p(), + "exp": value.exp(), + "expm1": np.expm1(value), + "sqrt": value.sqrt(), + "abs": value.abs(), + } + return frame.with_columns(expr.alias(name) for name, expr in functions.items()) + + +def daily_summary(frame: pl.DataFrame | pl.LazyFrame) -> pl.DataFrame | pl.LazyFrame: + value = pl.when(pl.col("eligible").fill_null(False)).then(pl.col("value").fill_nan(None)) + return ( + frame.group_by("decision_time") + .agg( + value.mean().alias("mean"), + value.std(ddof=0).alias("stddev"), + value.max().alias("max"), + value.min().alias("min"), + value.median().alias("median"), + value.sum().alias("sum"), + value.count().alias("notnull_count"), + ) + .sort("decision_time") + ) + + +if __name__ == "__main__": + values = pl.DataFrame( + { + "decision_time": [1] * 4 + [2] * 2, + "value": [1.0, 3.0, float("nan"), 999.0, None, 5.0], + "eligible": [True, True, True, False, True, False], + } + ) + summary = daily_summary(values) + assert summary["mean"].to_list() == [2.0, None] + assert summary["stddev"].to_list() == [1.0, None] + assert summary["sum"].to_list() == [4.0, 0.0] + assert summary["notnull_count"].to_list() == [2, 0] + elementary = elementary_features(pl.DataFrame({"value": [1e-20, None]})) + assert elementary["expm1"].to_list() == [1e-20, None] + print(summary) diff --git a/examples/pipeline_migration.py b/examples/pipeline_migration.py new file mode 100644 index 00000000..60af40bd --- /dev/null +++ b/examples/pipeline_migration.py @@ -0,0 +1,98 @@ +"""Port finite custom factors, classifications and screens without a legacy graph runtime.""" + +from datetime import UTC, date, datetime + +import polars as pl +from ml4t.specs import FeedSpec + +from ml4t.engineer.compat.zipline import factor_expression +from ml4t.engineer.features.preparation import prepare_feature_panel +from ml4t.engineer.features.selection import cross_section + + +def momentum_screen( + prices: pl.DataFrame | pl.LazyFrame, + decisions: pl.DataFrame, + universe: pl.DataFrame, + *, + feed_spec: FeedSpec, + price_basis: str, + window_length: int = 15, + n: int = 3, +) -> pl.DataFrame | pl.LazyFrame: + """Return supplied-member RSI selections plus a finite custom mean/classifier.""" + rsi = factor_expression("RSI", inputs=[feed_spec.price_col], window_length=window_length) + assert isinstance(rsi, pl.Expr) + panel = prepare_feature_panel( + prices, + features={ + "rsi": rsi, + "custom_mean": pl.col(feed_spec.price_col).rolling_mean(3, min_samples=1), + }, + lookbacks={"rsi": window_length, "custom_mean": 3}, + input_columns=[feed_spec.price_col], + decisions=decisions, + universe=universe, + price_basis=price_basis, + feed_spec=feed_spec, + missing_input_policy="known_missing", + ) + return panel.with_columns( + cross_section( + "rsi", method="top", n=n, timestamp_col="decision_time", mask="eligible" + ).alias("longs"), + cross_section( + "rsi", method="bottom", n=n, timestamp_col="decision_time", mask="eligible" + ).alias("shorts"), + pl.col("sector").replace_strict({"tech": 1, "bank": 2}, default=None).alias("sector_code"), + (pl.col("custom_mean") > 0).fill_null(False).alias("custom_filter"), + ).filter(pl.col("member").fill_null(False)) + + +def sample() -> pl.DataFrame: + days = [date(2025, 7, day) for day in (7, 8, 9)] + prices = pl.DataFrame( + { + "timestamp": days * 3, + "asset": [17] * 3 + [23] * 3 + [31] * 3, + "close": [10.0, 11.0, 10.0, 10.0, 12.0, 13.0, 10.0, 9.0, 8.0], + "available_at": [datetime(2025, 7, day.day, 20, tzinfo=UTC) for day in days] * 3, + } + ) + decisions = pl.DataFrame( + { + "timestamp": [days[-1]], + "decision_time": [datetime(2025, 7, 10, 13, tzinfo=UTC)], + "effective_session": [date(2025, 7, 10)], + } + ) + universe = pl.DataFrame( + { + "asset": [17, 23, 31], + "effective_from": [date(2025, 7, 10)] * 3, + "effective_to": pl.Series([None] * 3, dtype=pl.Date), + "available_at": [datetime(2025, 7, 1, tzinfo=UTC)] * 3, + "member": [True] * 3, + "sector": ["tech", "bank", None], + } + ) + result = momentum_screen( + prices.lazy(), + decisions, + universe, + feed_spec=FeedSpec(time_contract_version=2, calendar=None, entity_col="asset"), + price_basis="raw", + window_length=3, + n=1, + ) + assert isinstance(result, pl.LazyFrame) + return result.collect() + + +if __name__ == "__main__": + result = sample() + assert result["rsi"].to_list() == [50.0, 100.0, 0.0] + assert result.filter("longs")["asset"].to_list() == [23] + assert result.filter("shorts")["asset"].to_list() == [31] + assert result["sector_code"].to_list() == [1, 2, None] + print(result) diff --git a/mkdocs.yml b/mkdocs.yml index 827dec05..fbc29947 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -107,6 +107,7 @@ nav: - Quickstart: getting-started/quickstart.md - User Guide: - Features: user-guide/features.md + - Pipeline Migration: user-guide/pipeline-migration.md - Feature Discovery: user-guide/discovery.md - Labeling: user-guide/labeling.md - Alternative Bars: user-guide/bars.md @@ -116,6 +117,7 @@ nav: - ML Readiness: user-guide/ml-readiness.md - API Reference: - api/index.md + - Pipeline Operation Ports: api/pipeline-ports.md - Book Guide: - book-guide/index.md From ea9eeada8ca8e6f462f15790f6ebdcbee23273f8 Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Tue, 20 Oct 2026 21:31:26 -0400 Subject: [PATCH 15/19] fix(artifacts): preserve compound feature identity through serialization --- docs/user-guide/features.md | 3 +- src/ml4t/engineer/artifacts/features.py | 13 +++++-- tests/test_feature_panel_preparation.py | 47 +++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 3 deletions(-) diff --git a/docs/user-guide/features.md b/docs/user-guide/features.md index c488dd23..8bdc6e46 100644 --- a/docs/user-guide/features.md +++ b/docs/user-guide/features.md @@ -382,7 +382,8 @@ contexts, include their universe key in the group keys too. Cross-sectional mask not a substitute for information-time validation. Owned `FeatureSchema` records optional `available_at_col`, `decision_time_col` and -`eligible_col`, plus `universe_time_col`. `FeatureDefinition` records the canonical `feed_spec`, +`eligible_col`, plus `universe_time_col`; `entity_col` preserves one column name +or a list of compound identity columns. `FeatureDefinition` records the canonical `feed_spec`, `window_unit`, `price_basis`, finite `lookbacks` and optional `missing_input_policy`, with existing source-artifact lineage and join keys. `FeatureSpec.to_dict()` preserves omitted calendar versus explicit null; unused fields leave the old wire keys unchanged. Store actual diff --git a/src/ml4t/engineer/artifacts/features.py b/src/ml4t/engineer/artifacts/features.py index ff50dd2f..0b0fba2c 100644 --- a/src/ml4t/engineer/artifacts/features.py +++ b/src/ml4t/engineer/artifacts/features.py @@ -20,13 +20,22 @@ class FeatureSchema: """Column layout for a feature artifact.""" timestamp_col: str = "timestamp" - entity_col: str = "asset" + entity_col: str | tuple[str, ...] = "asset" feature_columns: tuple[str, ...] = () available_at_col: str | None = None decision_time_col: str | None = None eligible_col: str | None = None universe_time_col: str | None = None + def __post_init__(self) -> None: + columns = [self.entity_col] if isinstance(self.entity_col, str) else list(self.entity_col) + if not columns or any(not isinstance(c, str) or not c for c in columns): + raise ValueError("entity_col must declare nonempty stable entity columns") + if len(set(columns)) != len(columns): + raise ValueError("entity_col columns must be distinct") + if not isinstance(self.entity_col, str): + object.__setattr__(self, "entity_col", tuple(columns)) + @classmethod def from_mapping(cls, mapping: dict[str, Any] | None) -> FeatureSchema: if mapping is None: @@ -34,7 +43,7 @@ def from_mapping(cls, mapping: dict[str, Any] | None) -> FeatureSchema: feature_columns = mapping.get("feature_columns", ()) return cls( timestamp_col=str(mapping.get("timestamp_col", "timestamp")), - entity_col=str(mapping.get("entity_col", "asset")), + entity_col=mapping.get("entity_col", "asset"), feature_columns=tuple(str(item) for item in feature_columns), available_at_col=optional_str(mapping.get("available_at_col")), decision_time_col=optional_str(mapping.get("decision_time_col")), diff --git a/tests/test_feature_panel_preparation.py b/tests/test_feature_panel_preparation.py index fea9d024..97f55c04 100644 --- a/tests/test_feature_panel_preparation.py +++ b/tests/test_feature_panel_preparation.py @@ -6,6 +6,7 @@ import polars as pl import pytest +import yaml from ml4t.specs import FeedSpec from polars.testing import assert_frame_equal @@ -630,6 +631,7 @@ def test_naive_observation_instants_require_and_use_declared_timezone(timezone): {"lookbacks": {"average": "3"}}, {"price_basis": " "}, {"window_unit": "median_spacing"}, + {"missing_input_policy": "ignore"}, ], ) def test_invalid_artifact_declarations_do_not_survive_public_loading(definition): @@ -651,3 +653,48 @@ def test_minimal_artifacts_keep_old_wire_keys_and_source_lineage(): assert set(output["schema"]) == {"timestamp_col", "entity_col", "feature_columns"} assert set(output["definition"]) == {"family", "join_keys", "source_artifacts"} assert output["definition"]["source_artifacts"] == ["Data:snapshot:abc"] + + +def test_persisted_compound_entity_columns_keep_canonical_schema_and_physical_ns(tmp_path): + payload = { + "artifact_id": "compound-features", + "schema": {"entity_col": ["venue", "asset"], "feature_columns": ["average"]}, + "definition": { + "feed_spec": FeedSpec( + time_contract_version=2, calendar=None, entity_col=["venue", "asset"] + ).to_dict(), + "price_basis": "raw", + "missing_input_policy": "complete", + }, + } + original = history().with_columns(pl.lit("XNYS").alias("venue")) + facts = universe().with_columns(pl.lit("XNYS").alias("venue")) + prepared = prepare( + original, + universe=facts, + feed_spec=FeedSpec.from_mapping(payload["definition"]["feed_spec"]), + ) + prepared = prepared.with_columns( + (pl.col("available_at").cast(pl.Int64) + 1) + .cast(pl.Datetime("ns", "UTC")) + .alias("available_at") + ) + path = tmp_path / "features.parquet" + prepared.write_parquet(path) + restored = pl.read_parquet(path) + assert_frame_equal(restored, prepared) + for encoding in (json.loads(json.dumps(payload)), yaml.safe_load(yaml.safe_dump(payload))): + output = FeatureSpec.from_mapping(encoding).to_dict() + assert output["schema"]["entity_col"] == ["venue", "asset"] + feed = FeedSpec.from_mapping(output["definition"]["feed_spec"]) + assert feed.resolve_calendar() is None + assert ( + restored.select(feed.timestamp_col, *feed.entity_col).rows() + == prepared.select("timestamp", "venue", "asset").rows() + ) + + +@pytest.mark.parametrize("columns", [[], [""], ["asset", "asset"], [1]]) +def test_invalid_compound_entity_schema_fails_before_persistence(columns): + with pytest.raises(ValueError, match="entity_col"): + FeatureSpec.from_mapping({"artifact_id": "invalid", "schema": {"entity_col": columns}}) From 3ce735e5c9a54ea66ee6a9f87b4f2a68f1339a50 Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Tue, 20 Oct 2026 21:33:15 -0400 Subject: [PATCH 16/19] fix(selection): exclude unknown numeric groups and verify typed pandas presentation --- src/ml4t/engineer/features/selection.py | 4 ++-- tests/compat/test_zipline_selection.py | 10 ++++++++ tests/test_feature_panel_preparation.py | 31 +++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/ml4t/engineer/features/selection.py b/src/ml4t/engineer/features/selection.py index a78077af..381bc835 100644 --- a/src/ml4t/engineer/features/selection.py +++ b/src/ml4t/engineer/features/selection.py @@ -43,7 +43,7 @@ def cross_section( ) -> pl.Expr: """Transform the eligible contemporary cross-section, independently per group. - mask determines the input population, not a final screen. Null group labels, + mask determines the input population, not a final screen. Null/NaN group labels, null values and NaNs are excluded; legacy classifier sentinels must be mapped explicitly. Ordinal ties use ascending stable entity IDs, not input row order. Input keys must identify at most one observation per timestamp/entity. This @@ -107,7 +107,7 @@ def cross_section( original = _value(value) eligible = ~is_missing(original) if groups: - eligible &= pl.all_horizontal(pl.col(c).is_not_null() for c in groups) + eligible &= pl.all_horizontal(~is_missing(c) for c in groups) if mask is not None: eligible &= _value(mask).fill_null(False) x = pl.when(eligible).then(original) diff --git a/tests/compat/test_zipline_selection.py b/tests/compat/test_zipline_selection.py index 02d9a051..1233f680 100644 --- a/tests/compat/test_zipline_selection.py +++ b/tests/compat/test_zipline_selection.py @@ -74,6 +74,16 @@ def test_group_transforms_use_only_contemporary_eligible_values(method, expected assert_frame_equal(output, combined) +def test_undefined_numeric_classification_is_not_a_real_contemporary_group(): + frame = ( + panel() + .with_columns(pl.when(pl.col("sector") == "x").then(0.0).otherwise(1.0).alias("sector")) + .with_columns((pl.col("sector") / pl.col("sector")).alias("sector")) + ) + result = calculate(frame, "rank") + assert result["result"].to_list() == [None, None, None, 1.0, None, None] + + @pytest.mark.parametrize( "method,expected", [ diff --git a/tests/test_feature_panel_preparation.py b/tests/test_feature_panel_preparation.py index 97f55c04..c5ee4c18 100644 --- a/tests/test_feature_panel_preparation.py +++ b/tests/test_feature_panel_preparation.py @@ -698,3 +698,34 @@ def test_persisted_compound_entity_columns_keep_canonical_schema_and_physical_ns def test_invalid_compound_entity_schema_fails_before_persistence(columns): with pytest.raises(ValueError, match="entity_col"): FeatureSpec.from_mapping({"artifact_id": "invalid", "schema": {"entity_col": columns}}) + + +def test_explicit_pandas_presentation_keeps_effective_dates_exact_instants_and_stable_ids(): + pytest.importorskip("pyarrow", reason="Explicit pandas presentation requires PyArrow") + frame = prepare(history()).with_columns( + pl.when(pl.col("asset") == "A") + .then(2**63 + 1) + .otherwise(2**63 + 2) + .cast(pl.UInt64) + .alias("asset") + ) + presentation = ( + frame.select("effective_session", "asset", "average") + .to_pandas(date_as_object=True) + .set_index(["effective_session", "asset"]) + ) + assert list(presentation.index) == [ + (DAYS[2], 2**63 + 1), + (DAYS[2], 2**63 + 2), + (DAYS[3], 2**63 + 1), + (DAYS[3], 2**63 + 2), + ] + frame = frame.with_columns( + pl.Series("decision_time", [1, 2, 3, 4], dtype=pl.Datetime("ns", "UTC")) + ) + presentation = ( + frame.select("decision_time", "asset", "average") + .to_pandas() + .set_index(["decision_time", "asset"]) + ) + assert presentation.index.get_level_values("decision_time").asi8.tolist() == [1, 2, 3, 4] From ac096d1e180decebad87bcf00bff5bcae2d0a57d Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Tue, 20 Oct 2026 21:43:00 -0400 Subject: [PATCH 17/19] test(migration): qualify reference ports and document deliberate correctness differences --- docs/api/pipeline-ports.md | 9 +++++---- docs/user-guide/pipeline-migration.md | 13 +++++++++++++ tests/compat/test_zipline_selection.py | 6 ++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/docs/api/pipeline-ports.md b/docs/api/pipeline-ports.md index 5edd7a2a..a763c4ef 100644 --- a/docs/api/pipeline-ports.md +++ b/docs/api/pipeline-ports.md @@ -14,7 +14,8 @@ Use the [migration how-to](../user-guide/pipeline-migration.md) to prepare input |---|---|---| | `Returns` | `factor_expression` or precise `features.utils.arithmetic.returns` | N prices become N-1 changes; signed reference denominator; fraction, not percentage points. | | `DailyReturns` | Same recipe with two prices | One change; retain both endpoint quote masks. | -| `PercentChange`, `RateOfChangePercentage` | `absolute_percent_change` or recipes | Absolute reference denominator; fractional result, despite legacy naming. | +| `PercentChange` | `absolute_percent_change` or recipe | Absolute reference denominator; fractional result. | +| `RateOfChangePercentage` | `features.utils.arithmetic.percentage_change` or recipe | Signed reference denominator, percentage points (100 times the fractional return). | | `SimpleMovingAverage` | `finite_mean` or recipe | Full row window, omit null/NaN values from mean denominator. | | `WeightedAverageValue`, `VWAP` | `finite_weighted_mean` or recipes | Independent NaN sums of value*weight and weights; a missing value does not erase its weight. | | `AverageDollarVolume` | `average_dollar_volume` or recipe | Sum of price*volume divided by fixed N; distinguish quote units and volume basis. | @@ -46,7 +47,7 @@ Use the [migration how-to](../user-guide/pipeline-migration.md) to prepare input | Operation | Native route | Material choices | |---|---|---| | Arithmetic `+`, `-`, `*`, `/`, `**`, `%`, reflected forms, unary minus | Corresponding `pl.Expr` operations | Native null and integer overflow rules; do not cast large tick prices before differencing. | -| Comparisons `<`, `<=`, `>`, `>=`, `.eq`, `!=`; Boolean `&`, `|`, `~` | Native expressions, explicit `.fill_null(False)` for legacy filters | Do not use Python `and`/`or` on expressions. | +| Comparisons `<`, `<=`, `>`, `>=`, `.eq`, `!=`; Boolean `&`, `|`, `~` | Native expressions over explicitly normalized missing inputs | Do not use Python `and`/`or`. Polars orders NaNs differently from NumExpr comparisons; normalize NaN to null and declare the Boolean null policy, or preserve a specifically required legacy NaN comparison explicitly. Final eligibility still excludes undefined features. | | `sin`, `cos`, `tan`, `arcsin`, `arccos`, `arctan`, `sinh`, `cosh`, `tanh`, `arcsinh`, `arccosh`, `arctanh` | Same-named Polars expression functions | Domain-invalid real values remain undefined. | | `log`, `log10`, `log1p`, `exp`, `expm1`, `sqrt`, `abs` | `.log()`, `.log(10)`, `.log1p()`, `.exp()`, `numpy.expm1(expr)`, `.sqrt()`, `.abs()` | Use expm1 directly for tiny inputs, not subtraction from a rounded exponential. | | `.isnan`, `.notnan`, `.isfinite` | `is_missing`, inverse `is_missing`, `.is_finite().fill_null(False)` | Native nulls join NaNs in missingness; infinities are not missing but are nonfinite. | @@ -65,10 +66,10 @@ Use the [migration how-to](../user-guide/pipeline-migration.md) to prepare input |---|---|---| | `All`, `Any`, `AtLeastN` | `rolling_filter` with all/any/at_least | Full original windows, missing predicates false; apply `.over(entity_col)`. | | `AllPresent` | `rolling_filter(~is_missing(value), method="all")` | Known row presence is separate from actual value usability. | -| `MaximumFilter` | Eligible value equals its group's maximum | Retain all maximal ties; this differs from exact-count top(1). | +| `MaximumFilter` | `cross_section(method="top", n=1)` | One maximum per eligible group, stable-ID first tie. The pinned legacy implementation reinterprets float storage as signed integers and selects the wrong maximum for all-negative groups. Native numeric ordering deliberately corrects that defect. | | `NullFilter`, `NotNullFilter` | `is_missing`, inverse | Normalize legacy sentinels explicitly; no automatic -1 interpretation. | | `PercentileFilter` | Cross-section percentile_between | Inclusive bounds and eligible population. | -| `SingleAsset`, `StaticAssets`, `StaticSids` | Stable-ID equality or `.is_in(...)` | Explicitly resolve legacy Asset.sid through its owner; static membership is not a historical universe. | +| `SingleAsset`, `StaticAssets`, `StaticSids` | Stable-ID equality or `.is_in(...)` | Explicitly resolve legacy Asset.sid through its owner; static membership is not a historical universe. Legacy SingleAsset additionally raises unless its asset is alive throughout the requested range. Check that lifetime condition explicitly if retaining its strict behavior; a native equality filter alone does not raise. | | `ArrayPredicate`, `NumExprFilter` | Native elementwise Boolean expressions | Preserve declared null behavior; do not execute arbitrary legacy expression strings. | | `Filter.if_else` | `pl.when(predicate).then(left).otherwise(right)` | Declare unknown-predicate handling explicitly. | | `CustomFilter`, base `Filter` | Boolean expressions or reusable registered features | No graph interpreter; supply required history and knowledge clocks. | diff --git a/docs/user-guide/pipeline-migration.md b/docs/user-guide/pipeline-migration.md index f0200604..4b8c0e72 100644 --- a/docs/user-guide/pipeline-migration.md +++ b/docs/user-guide/pipeline-migration.md @@ -141,6 +141,19 @@ with complete quotes and a known-missing close, listing/delisting and supplied lifetimes. This is development qualification, not an available release or proof of order, fill, commission, leverage or portfolio-performance equivalence. +Development source `3c5665cd2d08bad2dc5dc25b4a97573a84a7de88` additionally +passed both missingness profiles with actual CustomFactor, CustomFilter and +CustomClassifier terms, and 33 installed operation comparisons covering all 19 +elementary math functions, seven daily summaries, five return-definition/window +cases and both weekday-event classes. `examples/pipeline_expression_ports.py` +contains the executable math and daily-summary ports. Data's unpublished +`0.2.2.dev41+gd48d849f7` artifact at +`d48d849f77d3ff75d05383cbaaed9b42f6ca95b7` fed actual known-at histories into +Engineer source `cce590132846dbc7d558971b2d5a7f703d5cd9a2`: an appended future +correction left the earlier mean 199 unchanged, and the later adjusted history +retained stable asset 17 across OLD/NEW symbols with a final two-price mean 99.5. +These artifacts are qualification inputs, not published install recommendations. + See the [operation reference](../api/pipeline-ports.md) for individual methods, defaults and material differences, and [native features](features.md) for the preparation and expression contracts. diff --git a/tests/compat/test_zipline_selection.py b/tests/compat/test_zipline_selection.py index 1233f680..bcb263d2 100644 --- a/tests/compat/test_zipline_selection.py +++ b/tests/compat/test_zipline_selection.py @@ -97,6 +97,12 @@ def test_top_bottom_use_exact_cardinality_and_stable_tie_selection(method, expec assert calculate(panel(), method, n=0)["result"].to_list() == [False] * 6 +def test_negative_group_maximum_uses_numeric_order_not_float_storage_bits(): + frame = panel().with_columns((-pl.col("value")).alias("value")) + result = calculate(frame, "top", n=1) + assert result["result"].to_list() == [False, False, True, True, False, False] + + def test_winsorization_uses_legacy_order_statistic_cutoffs_not_linear_clipping(): frame = pl.DataFrame( { From 4e20f54ff229fc566782dc30bf6818273a5b4a43 Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Tue, 20 Oct 2026 21:54:32 -0400 Subject: [PATCH 18/19] fix(selection): stabilize linear quantiles and keep core tests independent of stats extra --- docs/api/pipeline-ports.md | 2 +- src/ml4t/engineer/features/selection.py | 54 +++++++++++++++---- tests/compat/test_zipline_factor_recipes.py | 6 ++- .../test_zipline_relationship_features.py | 14 +++-- .../test_zipline_return_factor_alignment.py | 5 ++ tests/compat/test_zipline_selection.py | 44 +++++++++++++++ tests/test_feature_panel_preparation.py | 4 ++ 7 files changed, 113 insertions(+), 16 deletions(-) diff --git a/docs/api/pipeline-ports.md b/docs/api/pipeline-ports.md index a763c4ef..cd4057a9 100644 --- a/docs/api/pipeline-ports.md +++ b/docs/api/pipeline-ports.md @@ -55,7 +55,7 @@ Use the [migration how-to](../user-guide/pipeline-migration.md) to prepare input | `.top`, `.bottom` | Same-named cross-section methods | Exact N, stable ordinal ties; mask determines input population. | | `.demean`, `.zscore` | Same-named cross-section methods | Contemporary date/group population, population ddof=0 default; zero variance undefined. Not a train-fitted scaler. | | `.winsorize` | Same-named cross-section method | Floor/ceil legacy order-statistic bounds, not linear quantile clipping. | -| `.quantiles`, `.quartiles`, `.quintiles`, `.deciles` | `cross_section(method="quantiles", bins=...)` | 0-based labels, linear edges, duplicate-edge error, null exclusions. Aliases use 4/5/10 bins. | +| `.quantiles`, `.quartiles`, `.quintiles`, `.deciles` | `cross_section(method="quantiles", bins=...)` | 0-based labels, stable linear interpolation, duplicate-edge error, null exclusions. Nonmissing infinite values fail explicitly. Aliases use 4/5/10 bins. Floating values retain their local scale; integer/Decimal differences precede float conversion. | | `.percentile_between` | Same-named cross-section method | Inclusive linear bounds; convert legacy percent values 0..100 to fractions 0..1. | | `.mean`, `.stddev`, `.max`, `.min`, `.median`, `.sum`, `.notnull_count` daily summaries | Mask input then aggregate with native expressions over decision-time keys | Omit null/NaN, population stddev; sum of all missing values is zero, count is zero. Return one row per decision, not one per asset. | | Asset slicing and named multi-output fields | Exact keyed filter/join; `.struct.field(name)` | Stable keys and explicit output fields replace opaque Asset lookup/record wrappers. | diff --git a/src/ml4t/engineer/features/selection.py b/src/ml4t/engineer/features/selection.py index 381bc835..7608ad73 100644 --- a/src/ml4t/engineer/features/selection.py +++ b/src/ml4t/engineer/features/selection.py @@ -3,6 +3,7 @@ import math import numpy as np +import numpy.typing as npt import polars as pl from ml4t.engineer.features.rolling import _paired_difference, _period @@ -25,6 +26,25 @@ def build(expression: pl.Expr, dtype: pl.DataType) -> pl.Expr: return _value(value).pipe_with_dtype(build) +def _linear_edges( + values: npt.NDArray[np.float64], fractions: npt.NDArray[np.float64] +) -> npt.NDArray[np.float64]: + ordered = np.sort(values) + if not np.isfinite(ordered).all(): + raise ValueError("Quantile and percentile inputs require finite nonmissing values") + positions = (len(ordered) - 1) * np.asarray(fractions, dtype=np.float64) + a = ordered[np.floor(positions).astype(np.int64)] + b = ordered[np.ceil(positions).astype(np.int64)] + weight = positions - np.floor(positions) + opposite = np.signbit(a) != np.signbit(b) + edges = np.empty_like(positions) + # Avoid subtracting opposite finite extremes or losing small local values + # by centering every floating observation around a distant first value. + edges[opposite] = a[opposite] * (1 - weight[opposite]) + b[opposite] * weight[opposite] + edges[~opposite] = a[~opposite] + (b[~opposite] - a[~opposite]) * weight[~opposite] + return edges + + def cross_section( value: str | pl.Expr, *, @@ -139,26 +159,40 @@ def cross_section( (x.count() * max_percentile).ceil().cast(pl.Int64) - 1, null_on_oob=True ) result = x.clip(lower, upper) - elif method == "quantiles": - assert bins is not None + else: - def quantile_labels(series: pl.Series) -> pl.Series: + def coordinates(value: pl.Expr, dtype: pl.DataType) -> pl.Expr: + return ( + value.cast(pl.Float64) + if dtype.is_float() + else _paired_difference(value, value.drop_nulls().first()) + ) + + def select_quantiles(series: pl.Series) -> pl.Series: values = series.to_numpy() valid = ~np.isnan(values) if not valid.any(): - return pl.Series([None] * len(values), dtype=pl.Int64) - edges = np.quantile(values[valid], np.linspace(0.0, 1.0, bins + 1), method="linear") + return ( + pl.Series([None] * len(values), dtype=pl.Int64) + if method == "quantiles" + else pl.Series([False] * len(values), dtype=pl.Boolean) + ) + if method == "percentile_between": + lower, upper = _linear_edges( + values[valid], np.array([min_percentile, max_percentile]) + ) + return pl.Series(valid & (values >= lower) & (values <= upper)) + assert bins is not None + edges = _linear_edges(values[valid], np.linspace(0.0, 1.0, bins + 1)) if bins > 1 and len(np.unique(edges)) != len(edges): raise ValueError("Duplicate quantile edges; declare a different bin count") labels = np.searchsorted(edges, values, side="left") - 1 labels[values == edges[0]] = 0 return pl.Series(labels, dtype=pl.Int64).set(pl.Series(~valid), None) - result = centered.map_batches(quantile_labels, return_dtype=pl.Int64) - else: - lower = centered.quantile(min_percentile, interpolation="linear") - upper = centered.quantile(max_percentile, interpolation="linear") - result = centered.is_between(lower, upper).fill_null(False) + result = x.pipe_with_dtype(coordinates).map_batches( + select_quantiles, return_dtype=pl.Int64 if method == "quantiles" else pl.Boolean + ) return result.over(partition, order_by=entities) diff --git a/tests/compat/test_zipline_factor_recipes.py b/tests/compat/test_zipline_factor_recipes.py index a69473cc..79dc9c4f 100644 --- a/tests/compat/test_zipline_factor_recipes.py +++ b/tests/compat/test_zipline_factor_recipes.py @@ -194,7 +194,8 @@ def test_paired_recipes_preserve_actual_paired_series(name, length, expected) -> assert result.item(-1, 0) == pytest.approx(expected, rel=1e-14, abs=0) -def test_regression_recipe_exposes_actual_named_fields_and_beta_missing_fraction() -> None: +def test_regression_recipe_exposes_actual_named_fields() -> None: + pytest.importorskip("scipy", reason="Regression p-values require stats extra") frame = pl.DataFrame({"x": [1.0, 2.0, 3.0, 4.0], "y": [1.0, 2.0, 2.0, 4.0]}) result = frame.select( recipe("RollingLinearRegression", inputs=["y", "x"], regression_length=4).alias( @@ -207,6 +208,9 @@ def test_regression_recipe_exposes_actual_named_fields_and_beta_missing_fraction rtol=1e-13, atol=1e-15, ) + + +def test_beta_recipe_preserves_allowed_missing_fraction_without_stats_extra() -> None: gapped = pl.DataFrame({"x": [0.0, 1.0, 2.0, 3.0], "y": [1.0, None, 5.0, 7.0]}) result = gapped.select(recipe("SimpleBeta", inputs=["y", "x"], regression_length=4)) assert result.item(-1, 0) == pytest.approx(2.0, rel=1e-14) diff --git a/tests/compat/test_zipline_relationship_features.py b/tests/compat/test_zipline_relationship_features.py index bd0cfd10..3935878a 100644 --- a/tests/compat/test_zipline_relationship_features.py +++ b/tests/compat/test_zipline_relationship_features.py @@ -8,7 +8,6 @@ import polars as pl import pytest from polars.testing import assert_frame_equal -from scipy import stats def paired(): @@ -28,11 +27,11 @@ def test_paired_methods_match_independent_complete_window_oracles(method: str, l for end in range(2, len(frame)): x, y = frame["x"].to_numpy()[end - 2 : end + 1], frame["y"].to_numpy()[end - 2 : end + 1] if method == "pearson": - expected.append(stats.pearsonr(x, y).statistic) + expected.append(np.corrcoef(x, y)[0, 1]) elif method == "spearman": - expected.append(stats.spearmanr(x, y).statistic) + expected.append(np.corrcoef(np.argsort(np.argsort(x)), np.argsort(np.argsort(y)))[0, 1]) else: - expected.append(stats.linregress(x, y).slope) + expected.append(np.cov(x, y)[0, 1] / np.var(x, ddof=1)) np.testing.assert_allclose(result["result"].to_numpy(), expected, atol=1e-14, equal_nan=True) @@ -60,6 +59,7 @@ def test_spearman_recomputes_average_tie_ranks_in_each_window() -> None: @pytest.mark.parametrize("lazy", [False, True]) def test_rolling_regression_returns_all_five_actual_paired_fields(lazy: bool) -> None: + stats = pytest.importorskip("scipy.stats", reason="Regression p-values require stats extra") frame = pl.DataFrame({"x": [1.0, 3.0, 2.0, 4.0, 6.0], "y": [2.0, 8.0, 7.0, 9.0, 10.0]}) result = ( (frame.lazy() if lazy else frame) @@ -87,6 +87,7 @@ def test_rolling_regression_returns_all_five_actual_paired_fields(lazy: bool) -> def test_regression_missing_pairs_and_constant_target_are_explicit() -> None: + pytest.importorskip("scipy", reason="Regression p-values require stats extra") frame = pl.DataFrame({"x": [1.0, None, 3.0, 4.0], "y": [3.0, 100.0, 7.0, 9.0]}) result = frame.select( paired().rolling_regression("y", "x", period=4, allowed_missing=1).alias("r") @@ -99,6 +100,7 @@ def test_regression_missing_pairs_and_constant_target_are_explicit() -> None: def test_regression_has_independent_slope_intercept_pvalue_and_stderr_oracle() -> None: + pytest.importorskip("scipy", reason="Regression p-values require stats extra") frame = pl.DataFrame({"x": [1.0, 2.0, 3.0, 4.0], "y": [1.0, 2.0, 2.0, 4.0]}) actual = frame.select(paired().rolling_regression("y", "x", period=4).alias("r")).unnest("r") # Sxx=5, Sxy=4.5, Syy=4.75, residual SS=.7, df=2. @@ -109,6 +111,8 @@ def test_regression_has_independent_slope_intercept_pvalue_and_stderr_oracle() - @pytest.mark.parametrize("method", ["pearson", "spearman", "beta", "regression"]) def test_empty_and_singleton_pair_series_remain_empty_or_undefined(method: str) -> None: + if method == "regression": + pytest.importorskip("scipy", reason="Regression p-values require stats extra") function = getattr(paired(), f"rolling_{method}") for count in (0, 1): frame = pl.DataFrame({"x": [1.0] * count, "y": [2.0] * count}) @@ -123,6 +127,8 @@ def test_empty_and_singleton_pair_series_remain_empty_or_undefined(method: str) @pytest.mark.parametrize("method", ["pearson", "spearman", "beta", "regression"]) def test_paired_methods_do_not_cross_assets_or_mutate_inputs(method: str) -> None: + if method == "regression": + pytest.importorskip("scipy", reason="Regression p-values require stats extra") frame = pl.DataFrame( { "asset": ["A", "B"] * 4, diff --git a/tests/compat/test_zipline_return_factor_alignment.py b/tests/compat/test_zipline_return_factor_alignment.py index 4a5db892..b803ca9e 100644 --- a/tests/compat/test_zipline_return_factor_alignment.py +++ b/tests/compat/test_zipline_return_factor_alignment.py @@ -127,6 +127,7 @@ def test_feature_availability_uses_both_assets_and_the_actual_price_dependencies def test_custom_roles_compound_identity_and_regression_fields_survive_the_pipeline() -> None: + pytest.importorskip("scipy", reason="Regression p-values require stats extra") frame = ( panel() .with_columns( @@ -183,6 +184,8 @@ def test_target_contract_errors_are_atomic(options) -> None: @pytest.mark.parametrize("method", ["pearson", "spearman", "beta", "regression"]) def test_empty_frame_retains_declared_added_schema(method: str) -> None: + if method == "regression": + pytest.importorskip("scipy", reason="Regression p-values require stats extra") for frame in (panel().clear(), panel().clear().lazy()): result = run(frame, method=method) result = result.collect() if isinstance(result, pl.LazyFrame) else result @@ -347,6 +350,8 @@ def test_named_calendar_and_explicit_none_precedence_reuse_the_actual_grid_valid def test_return_factor_recipe_uses_native_keyed_preparation_without_masking_target( name, method, length, lazy ) -> None: + if method == "regression": + pytest.importorskip("scipy", reason="Regression p-values require stats extra") function = importlib.import_module("ml4t.engineer.compat.zipline").return_factor_features frame = panel() output = function( diff --git a/tests/compat/test_zipline_selection.py b/tests/compat/test_zipline_selection.py index bcb263d2..e103656c 100644 --- a/tests/compat/test_zipline_selection.py +++ b/tests/compat/test_zipline_selection.py @@ -137,6 +137,50 @@ def test_quantiles_use_linear_edges_and_percentile_filter_includes_both_bounds() ) assert result["quantile"].to_list() == [0, 0, 1, 1, None, None] assert result["between"].to_list() == [False, True, True, False, False, False] + + +@pytest.mark.parametrize("lazy", [False, True]) +@pytest.mark.parametrize( + "values,bins,selected", + [ + ([-1e308, 0.0, 1e308], [0, 0, 1], [False, True, False]), + ([-1e308, 1e308], [0, 1], [False, False]), + ([-1e100, 0.0, 1e-100, 1e100], [0, 0, 1, 1], [False, True, True, False]), + ], +) +def test_linear_quantile_selection_preserves_finite_opposite_extremes_and_local_scale( + lazy, values, bins, selected +): + frame = pl.DataFrame( + {"timestamp": [1] * len(values), "asset": range(len(values)), "value": values} + ) + result = (frame.lazy() if lazy else frame).with_columns( + selection().cross_section("value", method="quantiles", bins=2).alias("bin"), + selection() + .cross_section( + "value", method="percentile_between", min_percentile=0.25, max_percentile=0.75 + ) + .alias("selected"), + ) + result = result.collect() if lazy else result + assert result["bin"].to_list() == bins + assert result["selected"].to_list() == selected + + +def test_integer_percentile_bounds_do_not_truncate_fractional_value_edges(): + frame = pl.DataFrame({"timestamp": [1] * 3, "asset": [1, 2, 3], "value": [-0.5, 0.25, 0.75]}) + expression = selection().cross_section( + "value", method="percentile_between", min_percentile=0, max_percentile=1 + ) + assert frame.select(expression).to_series().to_list() == [True, True, True] + assert frame.clear().select(expression).schema["value"] == pl.Boolean + + +@pytest.mark.parametrize("method,options", [("quantiles", {"bins": 2}), ("percentile_between", {})]) +def test_nonfinite_quantile_inputs_fail_explicitly_without_partial_selection(method, options): + frame = pl.DataFrame({"timestamp": [1, 1], "asset": [1, 2], "value": [1.0, float("inf")]}) + with pytest.raises(ValueError, match="finite"): + frame.with_columns(selection().cross_section("value", method=method, **options)) tied = frame.with_columns(pl.lit(1.0).alias("value")) with pytest.raises((ValueError, pl.exceptions.DuplicateError)): tied.select(selection().cross_section("value", method="quantiles", bins=4)) diff --git a/tests/test_feature_panel_preparation.py b/tests/test_feature_panel_preparation.py index c5ee4c18..6df9cbb4 100644 --- a/tests/test_feature_panel_preparation.py +++ b/tests/test_feature_panel_preparation.py @@ -582,6 +582,10 @@ def test_unknown_availability_and_undefined_struct_outputs_cannot_become_eligibl result = prepare(unknown) assert not result["eligible"].any() assert result["available_at"].null_count() == result.height + + +def test_actual_undefined_regression_struct_fields_cannot_become_eligible(): + pytest.importorskip("scipy", reason="Regression p-values require stats extra") expression = rolling_regression("close", pl.lit(1.0), period=2) result = prepare(history(), features={"estimate": expression}, lookbacks={"estimate": 2}) assert not result["eligible"].any() From bc2042430ab134bde7fe725878028ad7de3bf224 Mon Sep 17 00:00:00 2001 From: Stefan Jansen Date: Tue, 20 Oct 2026 22:09:10 -0400 Subject: [PATCH 19/19] fix(examples): support legacy consoles and optional statistical documentation --- docs/user-guide/features.md | 21 +++++++++++++++------ examples/pipeline_expression_ports.py | 4 +++- examples/pipeline_migration.py | 3 ++- tests/test_examples.py | 14 ++++++++++++++ 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/docs/user-guide/features.md b/docs/user-guide/features.md index 8bdc6e46..53e742c2 100644 --- a/docs/user-guide/features.md +++ b/docs/user-guide/features.md @@ -461,16 +461,25 @@ not opaque Zipline Asset objects. This recipe returns a native frame, not a Term ```python import math +from importlib.util import find_spec import polars as pl from ml4t.engineer.features.paired import rolling_regression paired = pl.DataFrame({"market": [1.0, 2.0, 3.0, 4.0], "feature": [1.0, 2.0, 2.0, 4.0]}) -result = paired.lazy().select( - rolling_regression("feature", "market", period=4).alias("regression") -).unnest("regression").collect() -assert math.isclose(result["beta"][3], 0.9, rel_tol=1e-14) -assert math.isclose(result["stderr"][3], math.sqrt(0.07), rel_tol=1e-14) -assert math.isclose(result["p_value"][3], 1 - 9 / math.sqrt(95), rel_tol=1e-14) +if find_spec("scipy") is None: + try: + rolling_regression("feature", "market", period=4) + except ImportError as error: + assert "ml4t-engineer[stats]" in str(error) + else: + raise AssertionError("Inferential regression must require its optional dependency") +else: + result = paired.lazy().select( + rolling_regression("feature", "market", period=4).alias("regression") + ).unnest("regression").collect() + assert math.isclose(result["beta"][3], 0.9, rel_tol=1e-14) + assert math.isclose(result["stderr"][3], math.sqrt(0.07), rel_tol=1e-14) + assert math.isclose(result["p_value"][3], 1 - 9 / math.sqrt(95), rel_tol=1e-14) ``` `features.events.event_business_days` counts Monday through Friday, including diff --git a/examples/pipeline_expression_ports.py b/examples/pipeline_expression_ports.py index 69c5dace..da4da9f1 100644 --- a/examples/pipeline_expression_ports.py +++ b/examples/pipeline_expression_ports.py @@ -1,5 +1,7 @@ """Native ports of elementwise math and daily summaries, not a graph interpreter.""" +import json + import numpy as np import polars as pl @@ -62,4 +64,4 @@ def daily_summary(frame: pl.DataFrame | pl.LazyFrame) -> pl.DataFrame | pl.LazyF assert summary["notnull_count"].to_list() == [2, 0] elementary = elementary_features(pl.DataFrame({"value": [1e-20, None]})) assert elementary["expm1"].to_list() == [1e-20, None] - print(summary) + print(json.dumps(summary.to_dicts(), ensure_ascii=True, default=str)) diff --git a/examples/pipeline_migration.py b/examples/pipeline_migration.py index 60af40bd..ab69074c 100644 --- a/examples/pipeline_migration.py +++ b/examples/pipeline_migration.py @@ -1,5 +1,6 @@ """Port finite custom factors, classifications and screens without a legacy graph runtime.""" +import json from datetime import UTC, date, datetime import polars as pl @@ -95,4 +96,4 @@ def sample() -> pl.DataFrame: assert result.filter("longs")["asset"].to_list() == [23] assert result.filter("shorts")["asset"].to_list() == [31] assert result["sector_code"].to_list() == [1, 2, None] - print(result) + print(json.dumps(result.to_dicts(), ensure_ascii=True, default=str)) diff --git a/tests/test_examples.py b/tests/test_examples.py index d0fa5981..5286f6a8 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -38,3 +38,17 @@ def test_advertised_example_runs(script: Path) -> None: f"stdout:\n{result.stdout}\n" f"stderr:\n{result.stderr}" ) + + +@pytest.mark.parametrize("name", ["pipeline_expression_ports.py", "pipeline_migration.py"]) +def test_pipeline_examples_run_with_windows_legacy_console_encoding(name: str) -> None: + result = subprocess.run( + [sys.executable, str(EXAMPLE_DIR / name)], + cwd=EXAMPLE_DIR.parent, + env={**os.environ, "PYTHONIOENCODING": "cp1252"}, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + assert result.returncode == 0, result.stderr