diff --git a/docs/design.md b/docs/design.md index 39e98e4..7de2587 100644 --- a/docs/design.md +++ b/docs/design.md @@ -308,8 +308,64 @@ Three consequences worth stating: - Storage duplicates for overlapping symbols. Irrelevant at daily resolution, where a full SPY history back to 1993 is a few thousand rows. +## Contract regimes + +`contract_specs` carries one row per symbol and no effective date. That is right for the +question it answers, "what is this contract today", and wrong for any consumer that +multiplies a HISTORICAL position or trade by it. Where an exchange re-denominated a +contract, dates before the change need the old multiplier. + +`contract_regimes.yaml` plus `marketdata.point_value_asof` / `tick_value_asof` are the +effective-dated companion. Two markets are declared today: **RTY**, where ICE halved the +Russell multiplier from $100 to $50 effective trade date 2016-12-05 and converted each +open lot into two, and **LBR**, carried for completeness because cotdata already bridges +the CME lumber replacement through its own `hist_codes` scale. + +Three decisions are worth stating, because each rules out an obvious alternative. + +**It is a separate table, not a `Valid_From` column on `contract_specs`.** Adding the +column would be harmless; adding the ROWS is not. Consumers index that table by `Symbol` +and npf's `validation/costs.py` does `specs.loc[sym]`, which silently returns a DataFrame +instead of a Series once a symbol has two rows. That is a wrong answer rather than an +error, in the repo whose numbers feed a gate verdict. Keeping `contract_specs` at one row +per symbol makes this change purely additive, so it needs no deprecation path despite the +package being public and on PyPI. + +**It is a packaged file, not a store table.** Every other table under `metadata/` is +written by a producer from a vendor. This one cannot be: Norgate and databento both +publish only the current specification, so a store artifact would be a producer writing a +hand-entered constant, and it would then need mirroring to every replica and a producer +run to change. A packaged file travels with the version, is byte-identical on the Windows +producer and every consumer, and resolves with no store configured. `registry.yaml` is +the precedent, for the reason written at the top of it: a curated fact belongs next to +the thing it governs, with its justification inline. + +**An undeclared symbol falls back to its current spec.** So a caller writes one code path +for every market and only the declared ones behave differently. The alternative, raising +or returning NaN for undeclared symbols, would push a branch into every consumer to +express "nothing re-denominated this", which is the common case. + +The file restates each declared symbol's CURRENT multiplier as its last regime, purely so +that value can be compared against the vendor-refreshed `contract_specs`. +`tests/test_contract_regimes.py` makes that comparison against the live store, skipping +when there is none. Without it the file has one silent failure mode and it is the bad +one: an exchange changes a multiplier again, the vendor picks it up, and this file keeps +back-dating the superseded value while every lookup still returns a plausible number. + ## Known holes +**The contract-regime list is bounded by what one audit could see.** The two declared +markets came from an audit of the 47-market cotmetrics universe +(`cotmetrics/docs/analysis/2026-08-22-effective-dated-contract-multipliers.md`), which used +two signals and neither is complete. CFTC market names are noisy: 72% of the boundaries +they produce are exchange-wide relabel dates, and they did not mark the Russell change at +all. The second signal, a one-week event where every reportable position column scales by +one factor, only catches an INSTANTANEOUS conversion; a multiplier change handled by +listing a new contract alongside the old and letting positions migrate over months leaves +no step. The audit also covered only symbols in that universe, so equities and any futures +market outside it were never checked. Absence from `contract_regimes.yaml` means nobody +established a change, not that none happened. + **Capital Gains looks unpopulated.** The column exists but fired zero times across TLT, VFINX, PRHSX, and FCNTX, including two funds with 11,735 rows each. Four tickers is not proof it never fires. `include_capital_gains` is off by default and diff --git a/pyproject.toml b/pyproject.toml index acfdf38..b1e6342 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ build-backend = "setuptools.build_meta" # COT-specific, and a name saying otherwise would undo the distinction the split # exists to draw. name = "crucible-marketdata" -version = "0.1.0" +version = "0.2.0" description = "Daily bars — equities, ETFs and futures — as a producer/consumer split over a file store, with adjustment derived on read." readme = "README.md" authors = [ diff --git a/src/marketdata/__init__.py b/src/marketdata/__init__.py index acf3e14..41d1204 100644 --- a/src/marketdata/__init__.py +++ b/src/marketdata/__init__.py @@ -29,6 +29,14 @@ provenance, require_coverage, ) +from .regimes import ( + REGIME_COLUMNS, + RegimeError, + declared_symbols, + point_value_asof, + read_contract_regimes, + tick_value_asof, +) from .registry import ( DOMAINS, REGISTRY, @@ -42,9 +50,13 @@ # read_metadata is public API rather than an internal reached for from outside: # contract specs (point value, tick size) are what turns a futures bar into # notional or risk units, so a package that reads bars reads specs too. +# +# `read_metadata` answers "what is this contract today" and `point_value_asof` answers +# "what was it worth on this date". Reach for the second whenever the position or trade +# being valued is historical: see regimes.py and contract_regimes.yaml. from .store import load_manifest, read_metadata, require_schema, schema_version -__version__ = "0.1.0" +__version__ = "0.2.0" __all__ = [ "get_bars", "available", "provenance", "Provenance", @@ -54,4 +66,6 @@ "symbol", "all_symbols", "by_asset_class", "domain_for", "DOMAINS", "REGISTRY", "Symbol", "load_manifest", "read_metadata", "schema_version", "require_schema", + "read_contract_regimes", "point_value_asof", "tick_value_asof", + "declared_symbols", "REGIME_COLUMNS", "RegimeError", ] diff --git a/src/marketdata/contract_regimes.yaml b/src/marketdata/contract_regimes.yaml new file mode 100644 index 0000000..92ca047 --- /dev/null +++ b/src/marketdata/contract_regimes.yaml @@ -0,0 +1,138 @@ +# Contract regimes: what a contract was worth BEFORE it was worth what it is now. +# +# `contract_specs` (metadata/contract_specs.parquet, refreshed by +# `marketdata-update --metadata`) answers "what is this contract today". It carries one +# row per symbol and no effective date, which is correct for its question and wrong for +# any consumer that multiplies a HISTORICAL position or trade by it. Where an exchange +# re-denominated a contract, every date before the change needs the old multiplier. +# +# This file is the effective-dated companion. It is deliberately NOT a column on +# contract_specs: adding rows there would give a symbol two entries, and consumers index +# that table by Symbol (npf's validation/costs.py does `specs.loc[sym]`), so a second row +# turns a Series into a DataFrame and produces a wrong answer rather than an error. +# +# WHY THIS IS A PACKAGED FILE AND NOT A STORE TABLE +# +# Every other table under metadata/ is written by a producer from a vendor. This one +# cannot be: Norgate and databento both publish only the CURRENT specification, so a +# store artifact would be a producer writing a hand-entered constant, and it would then +# have to be mirrored to every replica and re-run to change. A packaged file travels with +# the version, is byte-identical on the Windows producer and every consumer, and is +# readable with no store configured at all. registry.yaml is the precedent, and the +# reason it is the precedent is written at the top of it. +# +# SCHEMA +# +# : +# - valid_from: "YYYY-MM-DD" | null first date the regime applied, exchange local. +# null on the FIRST regime only, meaning "for all +# history before the next one". An explicit date +# on the first regime BOUNDS the series: earlier +# dates resolve to NaN rather than to a guess. +# point_value: USD per 1.00 of the quoted price. +# tick_value: | null USD per minimum tick. null where unverified. +# name: what the contract was called then. +# source: a citation a reader can check WITHOUT trusting +# this file. Required on every row: a multiplier +# with no citation is indistinguishable from a +# typo. +# +# INVARIANTS, both enforced by tests/test_contract_regimes.py +# +# 1. Every symbol lists ALL of its regimes, including the current one, sorted by +# valid_from. `point_value_asof` therefore never has to combine this file with +# contract_specs for a declared symbol. +# 2. The LAST regime here must equal contract_specs' current Point Value and Tick Value. +# That is a tripwire: if an exchange changes a multiplier again and the vendor picks +# it up, the test fails and names the symbol, rather than this file silently going +# stale and back-dating the new value over the old history. +# +# HOW TO ADD A REGIME +# +# Add the NEW regime as a row with its valid_from, and leave the old rows alone. +# +# Symbols absent from this file have had one regime for their whole stored history, as +# far as anyone has established. That is an assertion about what was checked, not a +# guarantee: see 'Contract regimes' in docs/design.md for what the audit behind +# this file could and could not see. + +RTY: + # ICE Futures U.S. halved the multiplier on every Russell index future, effective with + # the start of trading for trade date Monday 2016-12-05, and converted each open lot + # into two lots. The CFTC did NOT rename the market that week, so nothing in the COT + # data marks it; the rename in that series is 2017-08-15 and records the unrelated + # migration of the contract from ICE back to CME. Applying today's $50 to the whole + # history understates 59% of the Russell's priced weeks by exactly 2x. + # + # The contract was $100 on CME from its 2002 listing, moved to ICE in 2008 at $100, + # changed to $50 on ICE in 2016, and moved back to CME in 2017 still at $50. Both + # venue migrations transferred positions about 1:1 and neither changed the multiplier, + # so they are not regime boundaries and are not listed. + # + # The first regime is unbounded rather than dated at the contract's listing: what is + # established is that $100 was in force immediately before the change and that nothing + # re-denominated it across the whole span this stack can serve. The listing date itself + # was not checked, so asserting one would be decoration. + - valid_from: null + point_value: 100.0 + tick_value: 10.0 + name: "Russell 2000 index future, $100 multiplier (CME 2002-2008, ICE 2008-2016)" + source: >- + ICE FAQ 2016-10-31, 'Russell Index Contracts Price Multiplier Change' + (https://www.ice.com/publicdocs/futures_us/Russell_Multiplier_Change_FAQ.pdf). + It gives both prior values in the course of announcing the change: the multiplier + it was changing FROM ($100 per index point) and the tick it was changing from + ($10.00 per contract). + - valid_from: "2016-12-05" + point_value: 50.0 + tick_value: 5.0 + name: "E-mini Russell 2000, $50 multiplier" + source: >- + ICE FAQ 2016-10-31, same document: the multiplier 'will change to $50 per index + point, from the current $100 multiplier', effective with the start of trading for + trade date Monday 2016-12-05, with each open lot converted to two lots and the + minimum tick becoming $5.00 per contract. Independently confirmed in the cotdata + store: open interest 355,514 -> 691,904 on COT week 2016-12-06, ratio 1.946, with + the non-reportable buckets moving only 1.22-1.28x because the FAQ left the + reporting threshold at 200 lots. + +LBR: + # CME replaced Random Length Lumber (110,000 board feet) with Lumber (27,500 bf), + # listing the new contract 2022-08-08 and running the old one down to 2023-05. + # + # Listed for completeness rather than because it is broken. The CFTC gave the two + # contracts DIFFERENT market codes (058643 and 058644), so cotdata bridges them in its + # own registry with `hist_codes: [["058643", 4.0]]` and rescales predecessor contract + # counts into current-contract units before any consumer sees them. A consumer reading + # cotdata therefore already receives 27,500-bf-equivalent counts and must NOT also + # apply the 110.0 below: that would apply the conversion twice. The rows exist so a + # consumer working from a different positioning source, or from per-contract trade + # records, can reach the same answer. + # + # Norgate's &LBR series begins at the new contract's 2022-08 listing, so nothing in + # this package currently reads a price on the 110.0 regime at all. + # + # The first regime is DATED rather than unbounded, and that is the point of the + # distinction. Code 058643's CFTC names run 'RANDOM LENGTH LUMBER' (1995-09-26), + # 'RANDOM LENGTH LUMBER-NEW' (1995-12-12) and 'RANDOM LENGTH LUMBER-80/110000' + # (1999-12-21), which suggest the contract was resized before settling at 110,000 bf. + # Those earlier sizes and their effective dates could not be established from exchange + # records, so dates before 1995-12-12 resolve to NaN. A gap is visible; a guess is not. + - valid_from: "1995-12-12" + point_value: 110.0 + tick_value: null + name: "Random Length Lumber, 110,000 board feet" + source: >- + CME Random Length Lumber contract specification: 110,000 board feet, quoted in + $/1,000 bf, hence $110 per point. 110,000 / 27,500 = 4.0, which is the scale + cotdata's registry already carries for the predecessor code. valid_from is INFERRED + from the first COT week under the 'RANDOM LENGTH LUMBER-NEW' name rather than taken + from an exchange notice, so treat it as approximate. tick_value is null because the + old contract's minimum tick was not verified. + - valid_from: "2022-08-08" + point_value: 27.5 + tick_value: 13.75 + name: "Lumber, 27,500 board feet" + source: >- + CME Lumber futures contract specification, 27,500 board feet quoted in $/1,000 bf: + https://www.cmegroup.com/markets/agriculture/lumber-and-softs/lumber/specs diff --git a/src/marketdata/regimes.py b/src/marketdata/regimes.py new file mode 100644 index 0000000..312d4eb --- /dev/null +++ b/src/marketdata/regimes.py @@ -0,0 +1,230 @@ +"""Effective-dated contract multipliers: what a contract was worth at a past date. + +`store.read_metadata` answers "what is this contract today", one row per symbol with no +effective date. That is the right shape for its question and the wrong input for any +consumer that multiplies a HISTORICAL position or trade by it. Where an exchange +re-denominated a contract, every date before the change needs the old multiplier, and +today's table cannot express that. + +This module is the effective-dated companion, backed by the packaged +``contract_regimes.yaml``. Read that file first: it carries the schema, the two +invariants, and the reason each regime is believed, next to the regime itself. + +**This is additive.** ``read_metadata`` and the ``contract_specs`` table are untouched +and still return exactly one current row per symbol, so nothing that reads them changes +behaviour. That was the design constraint rather than an accident: consumers index specs +by ``Symbol`` (npf's ``validation/costs.py`` does ``specs.loc[sym]``), so giving a symbol +a second row there would turn a Series into a DataFrame and yield a wrong answer instead +of an error. + +The undeclared case is the one that decides whether callers have to branch, and they do +not. A symbol with no entry here has had one regime for its whole stored history as far +as anyone has established, so `point_value_asof` falls back to the current +``contract_specs`` value for every date. A caller therefore writes the same code for all +47 markets and only the two declared ones behave differently. +""" +from __future__ import annotations + +import os +from pathlib import Path +from typing import Dict, List, Optional + +import numpy as np +import pandas as pd +import yaml + +#: One row per regime. `Valid_From` is NaT on a first regime declared unbounded. +REGIME_COLUMNS = ("Symbol", "Valid_From", "Point_Value", "Tick_Value", "Name", "Source") + +_REQUIRED_KEYS = ("valid_from", "point_value", "name", "source") + + +class RegimeError(ValueError): + """The regime file cannot be trusted to answer a multiplier question.""" + + +def _regimes_path(path=None) -> Path: + """Explicit arg, else $MARKETDATA_CONTRACT_REGIMES, else the packaged YAML. + + Same three-step resolution as `registry.load_registry`, so a deployment that + overrides one can override the other the same way. + """ + return Path(path or os.environ.get( + "MARKETDATA_CONTRACT_REGIMES", Path(__file__).parent / "contract_regimes.yaml")) + + +def _parse_symbol(internal: str, rows) -> List[dict]: + if not isinstance(rows, list) or not rows: + raise RegimeError( + f"contract regimes: symbol '{internal}' must map to a non-empty list of " + f"regimes, got {type(rows).__name__}.") + + out = [] + for i, row in enumerate(rows): + if not isinstance(row, dict): + raise RegimeError( + f"contract regimes: {internal} regime {i} must be a mapping, " + f"got {type(row).__name__}.") + missing = [k for k in _REQUIRED_KEYS if k not in row] + if missing: + raise RegimeError( + f"contract regimes: {internal} regime {i} is missing {missing}. " + f"`source` is required on every row because a multiplier with no " + f"citation is indistinguishable from a typo.") + + # Only the first regime may be unbounded. A null valid_from on a later row would + # silently reorder the series and back-date the wrong multiplier. + if row["valid_from"] is None and i != 0: + raise RegimeError( + f"contract regimes: {internal} regime {i} has a null valid_from, which " + f"is allowed on the first regime only (it means 'all history before the " + f"next one'). Give this row the date the regime took effect.") + + point = row["point_value"] + if not isinstance(point, (int, float)) or not point > 0: + raise RegimeError( + f"contract regimes: {internal} regime {i} has point_value {point!r}; " + f"expected a positive number.") + tick = row.get("tick_value") + if tick is not None and (not isinstance(tick, (int, float)) or not tick > 0): + raise RegimeError( + f"contract regimes: {internal} regime {i} has tick_value {tick!r}; " + f"expected a positive number or null (null means unverified).") + + out.append({ + "Symbol": internal, + "Valid_From": pd.Timestamp(row["valid_from"]) if row["valid_from"] else pd.NaT, + "Point_Value": float(point), + "Tick_Value": float(tick) if tick is not None else float("nan"), + "Name": str(row["name"]), + "Source": str(row["source"]), + }) + + dated = [r["Valid_From"] for r in out if pd.notna(r["Valid_From"])] + if dated != sorted(dated): + raise RegimeError( + f"contract regimes: {internal} regimes are not in valid_from order. " + f"They are read in file order and the order is the meaning, so sort them.") + if len(set(dated)) != len(dated): + raise RegimeError( + f"contract regimes: {internal} has two regimes with the same valid_from. " + f"A date cannot resolve to two multipliers.") + return out + + +def load_regimes(path=None) -> pd.DataFrame: + """Parse the regime YAML into a frame of `REGIME_COLUMNS`, validating as it goes. + + A missing file is an EMPTY table rather than an error: no declared regimes means + every symbol uses its current spec, which is what the package did before this + existed. A malformed file is an error, because that is a claim that cannot be read + rather than an absence of claims. + """ + p = _regimes_path(path) + if not p.exists(): + return pd.DataFrame(columns=list(REGIME_COLUMNS)) + try: + with open(p, "r") as f: + data = yaml.safe_load(f) + except yaml.YAMLError as e: + raise RegimeError(f"contract regimes YAML is malformed ({p}): {e}") from e + + if data is None: + return pd.DataFrame(columns=list(REGIME_COLUMNS)) + if not isinstance(data, dict): + raise RegimeError( + f"contract regimes YAML must be a mapping of symbol -> regimes ({p}); " + f"got {type(data).__name__}.") + + rows: List[dict] = [] + for internal, regimes in data.items(): + rows.extend(_parse_symbol(str(internal), regimes)) + return pd.DataFrame(rows, columns=list(REGIME_COLUMNS)) + + +def read_contract_regimes(symbol: Optional[str] = None) -> pd.DataFrame: + """Every declared regime, or one symbol's, in effective order. + + An undeclared symbol returns an empty frame, which is a positive statement: nothing + here re-denominated it, so its current spec applies to its whole history. + """ + table = load_regimes() + if symbol is None: + return table + return table[table["Symbol"] == symbol].reset_index(drop=True) + + +def declared_symbols() -> Dict[str, int]: + """Symbol -> regime count, for callers that want to report what is covered.""" + table = load_regimes() + if table.empty: + return {} + return {str(s): int(n) for s, n in table["Symbol"].value_counts().items()} + + +def _current_spec(symbol: str, field: str) -> float: + """Today's value from contract_specs, for a symbol with no declared regimes.""" + from . import store + specs = store.read_metadata() + if specs is None or specs.empty or "Symbol" not in specs.columns: + return float("nan") + row = specs[specs["Symbol"].astype(str) == symbol] + if row.empty or field not in row.columns: + return float("nan") + value = pd.to_numeric(row[field], errors="coerce").iloc[0] + return float(value) if pd.notna(value) and value > 0 else float("nan") + + +def _normalize(dates) -> pd.DatetimeIndex: + """Whatever the caller passed, as naive ns timestamps in the order given. + + Timezone is dropped rather than rejected, matching how the rest of the package + normalizes a bar index. `valid_from` is an exchange-local calendar date, so a + tz-aware instant has more precision than the question can use anyway. + """ + index = pd.DatetimeIndex(pd.to_datetime(dates)) + if index.tz is not None: + index = index.tz_localize(None) + return index + + +def _asof(symbol: str, dates, column: str, spec_field: str) -> pd.Series: + index = _normalize(dates) + table = read_contract_regimes(symbol) + + if table.empty: + return pd.Series(_current_spec(symbol, spec_field), index=index, dtype="float64") + + # searchsorted rather than merge_asof, and the reason is a caller we have: a trade + # log has many rows on one date, and merge_asof's result has to be reindexed back + # onto the input, which raises on a duplicate label. This preserves input order, + # tolerates duplicates and needs no sort of the caller's dates. + # + # An unbounded first regime is the minimum representable timestamp, so no date can + # land before it. A BOUNDED first regime leaves earlier dates at position -1, which + # is where the NaN comes from: the multiplier was never established there. + values = table[column].to_numpy(dtype="float64") + starts = table["Valid_From"].fillna(pd.Timestamp.min).to_numpy(dtype="datetime64[ns]") + + pos = np.searchsorted(starts.astype("int64"), + index.to_numpy(dtype="datetime64[ns]").astype("int64"), + side="right") - 1 + out = np.where(pos >= 0, values[pos.clip(min=0)], np.nan) + return pd.Series(out, index=index, dtype="float64") + + +def point_value_asof(symbol: str, dates) -> pd.Series: + """USD per 1.00 of quoted price, for each date, indexed by `dates`. + + A declared symbol resolves against its regimes; an undeclared one gets its current + `contract_specs` value for every date, so a caller needs no branch. Dates before a + BOUNDED first regime return NaN: where the multiplier was never established, + returning nothing beats returning a guess, because a gap is visible downstream and a + guess is not. + """ + return _asof(symbol, dates, "Point_Value", "Point Value") + + +def tick_value_asof(symbol: str, dates) -> pd.Series: + """USD per minimum tick, for each date. NaN where a regime left it unverified.""" + return _asof(symbol, dates, "Tick_Value", "Tick Value") diff --git a/tests/test_contract_regimes.py b/tests/test_contract_regimes.py new file mode 100644 index 0000000..28eff50 --- /dev/null +++ b/tests/test_contract_regimes.py @@ -0,0 +1,237 @@ +"""Effective-dated multipliers: the parse, the lookup, and the staleness tripwire. + +The tripwire is the test worth reading. `contract_regimes.yaml` restates each declared +symbol's CURRENT multiplier as its last regime, so that value can be compared against the +vendor-refreshed `contract_specs`. Without that comparison the file has exactly one silent +failure mode, and it is the bad one: an exchange changes a multiplier again, the vendor +picks it up, and this file keeps back-dating the superseded value over the new history +while every lookup still returns a plausible number. +""" +import pandas as pd +import pytest +import yaml + +from marketdata import regimes, store +from marketdata.regimes import RegimeError + +# ── the packaged file ───────────────────────────────────────────────────── + + +def test_the_packaged_file_parses_and_declares_the_audited_symbols(): + table = regimes.load_regimes() + assert not table.empty + assert list(table.columns) == list(regimes.REGIME_COLUMNS) + # RTY is the confirmed defect, LBR is carried for completeness. Both are argued in + # cotmetrics/docs/analysis/2026-08-22-effective-dated-contract-multipliers.md. + assert regimes.declared_symbols() == {"RTY": 2, "LBR": 2} + + +def test_every_regime_cites_a_source(): + """A multiplier with no citation is indistinguishable from a typo.""" + table = regimes.load_regimes() + assert (table["Source"].str.len() > 40).all() + + +def test_the_russell_change_is_dated_to_the_exchange_notice(): + rty = regimes.read_contract_regimes("RTY") + assert list(rty["Point_Value"]) == [100.0, 50.0] + assert list(rty["Tick_Value"]) == [10.0, 5.0] + # ICE FAQ 2016-10-31: effective with the start of trading for trade date 2016-12-05. + assert pd.isna(rty["Valid_From"].iloc[0]) + assert rty["Valid_From"].iloc[1] == pd.Timestamp("2016-12-05") + + +# ── the lookup ──────────────────────────────────────────────────────────── + + +def test_lookup_returns_the_regime_in_force_on_each_date(): + dates = pd.to_datetime(["2002-08-13", "2016-11-29", "2016-12-05", "2026-08-18"]) + assert list(regimes.point_value_asof("RTY", dates)) == [100.0, 100.0, 50.0, 50.0] + assert list(regimes.tick_value_asof("RTY", dates)) == [10.0, 10.0, 5.0, 5.0] + + +def test_the_boundary_is_inclusive_of_its_own_date(): + """valid_from is the first date the new regime applies, not the last of the old.""" + assert regimes.point_value_asof("RTY", ["2016-12-04"]).iloc[0] == 100.0 + assert regimes.point_value_asof("RTY", ["2016-12-05"]).iloc[0] == 50.0 + + +def test_the_result_is_indexed_by_the_dates_asked_for_in_that_order(): + dates = pd.to_datetime(["2026-08-18", "2002-08-13", "2016-12-06"]) + out = regimes.point_value_asof("RTY", dates) + assert list(out.index) == list(dates) + assert list(out) == [50.0, 100.0, 50.0] + + +def test_repeated_dates_resolve_rather_than_raising(): + """A trade log has many rows on one date. An earlier merge_asof implementation + raised 'cannot reindex on an axis with duplicate labels' on exactly this input.""" + out = regimes.point_value_asof("RTY", ["2016-12-06", "2016-12-06", "2002-08-13"]) + assert list(out) == [50.0, 50.0, 100.0] + assert len(out) == 3 + + +def test_unsorted_dates_keep_the_caller_s_order(): + dates = ["2026-08-18", "2002-08-13", "2016-12-06"] + assert list(regimes.point_value_asof("RTY", dates)) == [50.0, 100.0, 50.0] + + +def test_timezone_aware_dates_are_accepted(): + """valid_from is an exchange-local calendar date, so tz is dropped, not rejected.""" + dates = pd.to_datetime(["2016-12-06", "2002-08-13"]).tz_localize("UTC") + out = regimes.point_value_asof("RTY", dates) + assert list(out) == [50.0, 100.0] + assert out.index.tz is None + + +def test_no_dates_gives_an_empty_series_not_an_error(): + assert regimes.point_value_asof("RTY", []).empty + + +def test_a_bounded_first_regime_returns_nan_before_it_rather_than_a_guess(): + """LBR's pre-1995 sizes were never established. A gap is visible; a guess is not.""" + out = regimes.point_value_asof("LBR", ["1995-09-26", "1995-12-12", "2022-08-08"]) + assert pd.isna(out.iloc[0]) + assert list(out.iloc[1:]) == [110.0, 27.5] + + +def test_an_unverified_tick_value_is_nan_while_its_point_value_still_resolves(): + when = ["2000-01-04"] + assert regimes.point_value_asof("LBR", when).iloc[0] == 110.0 + assert pd.isna(regimes.tick_value_asof("LBR", when).iloc[0]) + + +def test_an_undeclared_symbol_falls_back_to_its_current_spec(tmp_store): + """The whole point of the fallback: a caller writes one code path for every market.""" + store.write_metadata(pd.DataFrame([ + {"Symbol": "ES", "Point Value": 50.0, "Tick Value": 12.5}, + ]), source="test") + out = regimes.point_value_asof("ES", ["1997-09-16", "2026-08-18"]) + assert list(out) == [50.0, 50.0] + assert regimes.read_contract_regimes("ES").empty + + +def test_an_unknown_symbol_is_nan_rather_than_an_error(tmp_store): + """One unpriceable market must not take the other 46 down with it.""" + store.write_metadata(pd.DataFrame([{"Symbol": "ES", "Point Value": 50.0}]), + source="test") + assert regimes.point_value_asof("NOPE", ["2026-08-18"]).isna().all() + + +def test_a_declared_symbol_needs_no_store_at_all(tmp_store): + """Regimes are packaged, not stored, so the lookup works before any producer ran.""" + assert store.read_metadata().empty + assert regimes.point_value_asof("RTY", ["2010-01-05"]).iloc[0] == 100.0 + + +# ── the parse refuses what it cannot answer ─────────────────────────────── + + +def _write(tmp_path, monkeypatch, doc): + p = tmp_path / "regimes.yaml" + p.write_text(yaml.safe_dump(doc) if isinstance(doc, dict) else doc) + monkeypatch.setenv("MARKETDATA_CONTRACT_REGIMES", str(p)) + return p + + +def test_a_missing_file_is_an_empty_table_not_an_error(tmp_path, monkeypatch): + """No declared regimes means every symbol uses its current spec, as before.""" + monkeypatch.setenv("MARKETDATA_CONTRACT_REGIMES", str(tmp_path / "absent.yaml")) + assert regimes.load_regimes().empty + + +def test_a_malformed_file_raises(tmp_path, monkeypatch): + """An unreadable claim is not the same as an absence of claims.""" + _write(tmp_path, monkeypatch, "RTY: [oops\n") + with pytest.raises(RegimeError, match="malformed"): + regimes.load_regimes() + + +def test_a_regime_without_a_source_raises(tmp_path, monkeypatch): + _write(tmp_path, monkeypatch, + {"XX": [{"valid_from": None, "point_value": 5.0, "name": "x"}]}) + with pytest.raises(RegimeError, match="source"): + regimes.load_regimes() + + +def test_a_null_valid_from_on_a_later_regime_raises(tmp_path, monkeypatch): + """It would silently reorder the series and back-date the wrong multiplier.""" + _write(tmp_path, monkeypatch, {"XX": [ + {"valid_from": "2000-01-01", "point_value": 5.0, "name": "a", "source": "s" * 50}, + {"valid_from": None, "point_value": 10.0, "name": "b", "source": "s" * 50}, + ]}) + with pytest.raises(RegimeError, match="first regime only"): + regimes.load_regimes() + + +def test_out_of_order_regimes_raise(tmp_path, monkeypatch): + _write(tmp_path, monkeypatch, {"XX": [ + {"valid_from": "2010-01-01", "point_value": 5.0, "name": "a", "source": "s" * 50}, + {"valid_from": "2000-01-01", "point_value": 10.0, "name": "b", "source": "s" * 50}, + ]}) + with pytest.raises(RegimeError, match="valid_from order"): + regimes.load_regimes() + + +def test_two_regimes_on_one_date_raise(tmp_path, monkeypatch): + _write(tmp_path, monkeypatch, {"XX": [ + {"valid_from": "2000-01-01", "point_value": 5.0, "name": "a", "source": "s" * 50}, + {"valid_from": "2000-01-01", "point_value": 10.0, "name": "b", "source": "s" * 50}, + ]}) + with pytest.raises(RegimeError, match="same valid_from"): + regimes.load_regimes() + + +def test_a_non_positive_point_value_raises(tmp_path, monkeypatch): + _write(tmp_path, monkeypatch, + {"XX": [{"valid_from": None, "point_value": 0, "name": "x", "source": "s" * 50}]}) + with pytest.raises(RegimeError, match="point_value"): + regimes.load_regimes() + + +# ── the tripwire ────────────────────────────────────────────────────────── + + +def test_the_last_regime_must_match_the_current_spec(tmp_store): + """Invariant 2, on a fixture: this is the comparison, exercised where it can fail.""" + store.write_metadata(pd.DataFrame([ + {"Symbol": "RTY", "Point Value": 50.0, "Tick Value": 5.0}, + {"Symbol": "LBR", "Point Value": 27.5, "Tick Value": 13.75}, + ]), source="test") + for sym, table in regimes.load_regimes().groupby("Symbol"): + last = table.iloc[-1] + assert regimes._current_spec(sym, "Point Value") == last["Point_Value"] + + +def test_the_tripwire_fires_when_the_vendor_moves_under_the_file(tmp_store): + """A guard that has never fired is indistinguishable from one that is not wired in.""" + store.write_metadata(pd.DataFrame([ + {"Symbol": "RTY", "Point Value": 25.0, "Tick Value": 2.5}, # a third regime + ]), source="test") + last = regimes.read_contract_regimes("RTY").iloc[-1] + assert regimes._current_spec("RTY", "Point Value") != last["Point_Value"] + + +@pytest.mark.parametrize("field,column", [("Point Value", "Point_Value"), + ("Tick Value", "Tick_Value")]) +def test_live_store_agrees_with_the_last_declared_regime(field, column): + """The real tripwire, against whatever `MARKETDATA_STORE` currently holds. + + Skipped rather than failed when the store has no specs, because CI has no store and + a red suite there would say nothing about the regime file. + """ + specs = store.read_metadata() + if specs.empty or "Symbol" not in specs.columns: + pytest.skip("no contract_specs in MARKETDATA_STORE") + for sym, table in regimes.load_regimes().groupby("Symbol"): + if sym not in set(specs["Symbol"].astype(str)): + continue + current = regimes._current_spec(sym, field) + declared = table.iloc[-1][column] + if pd.isna(current) or pd.isna(declared): + continue + assert current == declared, ( + f"{sym}: contract_specs says {field}={current} but the last regime in " + f"contract_regimes.yaml says {declared}. Either the exchange changed the " + f"contract again (add a regime) or a regime is wrong. Do NOT edit the last " + f"regime to match without checking which one moved.")