Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 37 additions & 6 deletions docs/user-guide/features.md
Original file line number Diff line number Diff line change
Expand Up @@ -98,12 +98,43 @@ uses the explicit `calendar` argument, then declared feed metadata, then NYSE. E
`calendar=None` disables schedule interpretation. A named `"24/7"` calendar instead defines
expected bars and detects outages, including on weekends.

Calendar-aware execution requires a shared `FeedSpec(time_contract_version=2)` with an
explicit fixed intraday frequency and bar-grid policy. The initial implementation supports
time bars anchored at each tradable interval's open. It uses real holidays, early closes and
breaks from the selected schedule. Daily/session-label grids and session or elapsed-duration
windows are not supported by this validator; they fail rather than being treated as row counts.
Naive timestamps need a declared timezone. Aware timestamps already declare their timezone.
Calendar-aware execution requires a shared `FeedSpec(time_contract_version=2)` with declared
frequency and timestamp meaning. Fixed intraday time bars require an explicit bar-grid policy,
anchored at each tradable interval's open. The selected schedule supplies holidays, early closes
and breaks. Naive instants need a declared timezone; aware instants declare their timezone.

Daily bars instead declare `timestamp_semantics="session_label"`, `data_frequency="1d"` and
`bar_type="time"`. Do not supply intraday anchor, label-side or partial-bar fields for session
labels. Date values and datetime representations identify the displayed session date, without
timezone conversion. Validation checks every scheduled session between each asset's first and
last label, including completely missing sessions. A named `24/7` schedule includes weekends;
an NYSE schedule does not. No prices are filled or rows inserted. A daily frequency alone does
not turn `event_time` instants into session labels. Session-count and elapsed-duration feature
windows remain unsupported; they fail rather than being treated as expected-bar counts.

<!-- ml4t-exec -->
```python
from datetime import date

import polars as pl
from ml4t.engineer import compute_features
from ml4t.specs import FeedSpec

close = [100.0, 103.0, 106.0, 109.0]
daily_prices = pl.DataFrame({
"timestamp": [date(2025, 7, day) for day in (3, 7, 8, 9)],
"open": close, "high": close, "low": close, "close": close, "volume": 1.0,
})
daily_feed = FeedSpec(
time_contract_version=2, calendar="NYSE", data_frequency="1d",
timestamp_semantics="session_label", bar_type="time",
)
checked = compute_features(daily_prices, [], feed_spec=daily_feed)
returns = checked.with_columns(pl.col("close").log().diff().alias("returns"))
features = compute_features(returns, [{"name": "sma", "params": {"period": 2}}], feed_spec=daily_feed)
assert features["sma"].to_list() == [None, 101.5, 104.5, 107.5]
assert features["timestamp"].equals(daily_prices["timestamp"])
```

Validate prices before deriving returns. An empty feature list performs the same grid
validation and chronological sorting without adding feature columns:
Expand Down
97 changes: 73 additions & 24 deletions src/ml4t/engineer/core/bar_grid.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,15 @@ def validate_feature_grid(
timestamp_col: str | None,
group_cols: list[str],
) -> None:
"""Bind shared v2 declarations to the fixed-intraday schedule checker."""
"""Validate shared v2 daily session labels or fixed intraday bar timestamps."""
daily_labels = feed_spec.timestamp_semantics == TimestampSemantics.SESSION_LABEL
required = (
"data_frequency",
"timestamp_semantics",
"bar_type",
"bar_grid_anchor",
"bar_label",
"partial_bar_policy",
)
if not daily_labels:
required += ("bar_grid_anchor", "bar_label", "partial_bar_policy")
missing = [name for name in required if getattr(feed_spec, name) is None]
if feed_spec.time_contract_version != 2 or missing or timestamp_col is None:
raise ValueError(
Expand All @@ -41,16 +41,10 @@ def validate_feature_grid(
if feed_spec.timestamp_semantics not in {
TimestampSemantics.BAR_CLOSE,
TimestampSemantics.EVENT_TIME,
TimestampSemantics.SESSION_LABEL,
}:
raise ValueError("Fixed intraday bar grids require event_time or bar_close timestamps")
raise ValueError("Bar grids require event_time, bar_close or session_label timestamps")
schema = data.collect_schema() if isinstance(data, pl.LazyFrame) else data.schema
dtype = schema[timestamp_col]
timezone = feed_spec.timezone
if timezone is None and isinstance(dtype, pl.Datetime):
timezone = dtype.time_zone
if timezone is None:
raise ValueError("A declared bar grid requires timezone metadata for naive timestamps")

frequency = feed_spec.data_frequency
if isinstance(frequency, str):
import pandas as pd
Expand All @@ -59,25 +53,40 @@ def validate_feature_grid(
with warnings.catch_warnings():
warnings.simplefilter("error", FutureWarning)
warnings.simplefilter("error", DeprecationWarning)
parsed = pd.Timedelta(frequency)
parsed = pd.Timedelta("1D" if frequency == "1d" else frequency)
except (ValueError, TypeError, FutureWarning, DeprecationWarning) as exc:
raise ValueError("declared grid data_frequency must be a fixed duration") from exc
if not isinstance(parsed, pd.Timedelta) or parsed.value % 1000:
raise ValueError("declared grid data_frequency requires finite microsecond precision")
frequency = parsed.to_pytimedelta()
if not isinstance(frequency, timedelta):
raise ValueError("declared grid data_frequency must be a duration string or timedelta")
validate_bar_grid(
data,
calendar=calendar,
frequency=frequency,
timezone=timezone,
timestamp_col=timestamp_col,
group_cols=group_cols,
anchor=cast(Literal["session_open"], feed_spec.bar_grid_anchor),
label_side=cast(Literal["left", "right"], feed_spec.bar_label),
include_partial=feed_spec.partial_bar_policy == "include",
)
if daily_labels:
_validate_session_grid(
data,
calendar=calendar,
frequency=frequency,
timestamp_col=timestamp_col,
group_cols=group_cols,
)
else:
dtype = schema[timestamp_col]
timezone = feed_spec.timezone
if timezone is None and isinstance(dtype, pl.Datetime):
timezone = dtype.time_zone
if timezone is None:
raise ValueError("A declared bar grid requires timezone metadata for naive timestamps")
validate_bar_grid(
data,
calendar=calendar,
frequency=frequency,
timezone=timezone,
timestamp_col=timestamp_col,
group_cols=group_cols,
anchor=cast(Literal["session_open"], feed_spec.bar_grid_anchor),
label_side=cast(Literal["left", "right"], feed_spec.bar_label),
include_partial=feed_spec.partial_bar_policy == "include",
)
if "returns" in schema:
ordered = data.sort([*group_cols, timestamp_col])
missing_price = ~pl.col("close").cast(pl.Float64).is_finite().fill_null(False)
Expand All @@ -96,6 +105,46 @@ def validate_feature_grid(
)


def _validate_session_grid(
data: pl.DataFrame | pl.LazyFrame,
*,
calendar: str,
frequency: timedelta,
timestamp_col: str,
group_cols: list[str],
) -> None:
if frequency != timedelta(days=1):
raise ValueError("Daily session labels require declared data_frequency='1d'")
from ml4t.engineer.labeling.calendar import PandasMarketCalendar

adapter = PandasMarketCalendar(calendar)
labels = data.select([*group_cols, pl.col(timestamp_col).dt.date()])
frame = labels.collect() if isinstance(labels, pl.LazyFrame) else labels
if frame[timestamp_col].null_count():
raise ValueError("declared daily grid contains null session labels")
groups = frame.partition_by(group_cols, as_dict=True) if group_cols else {(): frame}
for group, rows in groups.items():
actual = sorted(rows[timestamp_col].to_list())
if not actual:
continue
observed = set(actual)
if len(actual) != len(observed):
raise ValueError(f"duplicate session label in group {group!r}")
schedule = adapter._calendar.schedule(start_date=actual[0], end_date=actual[-1])
expected = {label.date() for label in schedule.index}
missing = sorted(expected - observed)
if missing:
raise ValueError(
f"missing scheduled session in group {group!r} at {missing[0].isoformat()}; "
"validate daily prices before deriving returns, or explicitly select calendar=None"
)
off_schedule = sorted(observed - expected)
if off_schedule:
raise ValueError(
f"off-schedule session label in group {group!r} at {off_schedule[0].isoformat()}"
)


def validate_bar_grid(
data: pl.DataFrame | pl.LazyFrame,
*,
Expand Down
Loading
Loading