diff --git a/pyproject.toml b/pyproject.toml index af1af9f..5209e1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "cotmetrics" -version = "0.3.1" +version = "0.4.0" description = "Turn raw COT data into positioning metrics and trading signals (positioning index, concentration/clustering/position-size, reversal signals)." readme = "README.md" license = { text = "MIT" } @@ -29,7 +29,13 @@ dependencies = [ # Daily bars (ADR-0007 moved them out of cotdata); editable: -e ../marketdata. # The DISTRIBUTION is crucible-marketdata because `marketdata` on PyPI is an # unrelated abandoned project; the IMPORT is still `marketdata`. - "crucible-marketdata>=0.1.0", + # + # FLOOR 0.2.0, and it is a correctness floor rather than a convenience one. + # `exposure.point_value_series` needs `marketdata.read_contract_regimes` / + # `point_value_asof` (added there in 0.2.0) to value a week at the multiplier that + # was in force THAT week. Against 0.1.0 it raises rather than falling back, because + # the flat answer is wrong by 2x on RTY before 2016-12-05 and wrong invisibly. + "crucible-marketdata>=0.2.0", ] [project.optional-dependencies] diff --git a/scripts/check_dep_floors.py b/scripts/check_dep_floors.py index f0db7ca..270165f 100644 --- a/scripts/check_dep_floors.py +++ b/scripts/check_dep_floors.py @@ -31,6 +31,12 @@ # rather than a package index, so only these need the extra check. INTERNAL = { "cotdata", + # Distribution name; the import is `marketdata`. ADR-0007 moved bars here. It was + # missing from this set while the floor was a nominal '>=0.1.0' and nothing noticed; + # it matters now that the floor is a CORRECTNESS one (effective-dated multipliers, + # marketdata 0.2.0), because an unchecked floor is not a floor. npf's and + # cot-analyzer's copies of this script already listed it. + "crucible-marketdata", "cotmetrics", "crucible", "crucible-stack", diff --git a/src/cotmetrics/exposure.py b/src/cotmetrics/exposure.py index e2f5e8b..96e650f 100644 --- a/src/cotmetrics/exposure.py +++ b/src/cotmetrics/exposure.py @@ -33,6 +33,15 @@ That is what `pct_rank` is for, and it is why `aggregate_exposure` returns it beside every level rather than leaving it to the caller to remember. +**The multiplier is effective-dated, and the middle rung is where that bites.** A +contract is not a fixed quantity of anything: ICE halved the Russell multiplier on +2016-12-05 and converted each open lot into two, with no CFTC rename to mark it. Today's +`contract_specs` row cannot say so, so `point_values` answers only whether a market can +be priced and `point_value_series` answers what each WEEK is worth. Getting this wrong +is quiet in a way the price-series mix-up below is not: the number stays plausible, and +because `expanding_pct_rank` ranks each week against its own past, an under-scaled first +half makes the second half read as more extreme rather than merely smaller. + **The two factors come from two different price series, deliberately.** Notional needs tradeable price LEVELS and takes only ``unadj``; volatility needs correct percentage RETURNS and takes only ``propadj``. Neither substitutes for the other and both failures @@ -207,6 +216,13 @@ def point_values() -> dict: a gap to paper over: MFS and MME are ICE MSCI futures priced off the EFA and EEM ETFs, and an ETF share is not a contract, so there is no multiplier that would make their contract count into dollars. `aggregate_exposure` names what it dropped. + + **This is the CURRENT multiplier, so it answers membership, not arithmetic.** Ask it + whether a market can be priced at all; ask `point_value_series` what to multiply a + given week by. The two differ wherever an exchange re-denominated a contract, and + the difference is not small: ICE halved the Russell multiplier on 2016-12-05, so + using this value for the whole history understates 59% of RTY's priced weeks by + exactly 2x. See `point_value_series`. """ from marketdata.store import read_metadata specs = read_metadata() @@ -220,6 +236,51 @@ def point_values() -> dict: if pd.notna(val) and val > 0} +def point_value_series(symbol: str, dates) -> pd.Series: + """USD per point FOR EACH WEEK, which is not the same as USD per point. + + `point_values` reads `contract_specs`, which carries one row per symbol and no + effective date. That is the right shape for "can this market be priced" and the + wrong input for "what was this position worth in 2010", because an exchange can + re-denominate a contract and the current table cannot say so. + + The live case is the Russell. ICE cut the multiplier from $100 to $50 per index + point effective 2016-12-05 and converted each open lot into two, with no CFTC rename + to mark it, so 740 of RTY's 1,247 priced weeks sit on the old contract. Multiplying + them by today's $50 halves both notional and dollar risk. Worse than the level error, + it compresses the first half of the history that `expanding_pct_rank` ranks the + second half against, so every post-2016 reading comes out more extreme than it is. + + marketdata owns the regime table (`contract_regimes.yaml`, added in 0.2.0) because a + contract's history is a property of the contract, not of this package, and because + npf's cost model needs the same answer. Here we only ask it. + + Undeclared symbols keep the current value for every date, which is a positive claim + and not a shrug: nothing established a re-denomination for them. Where a multiplier + was never established at all, marketdata returns NaN and the dollars go with it, + which is the outcome to want. A gap is visible on a chart; a guess is not. + """ + index = pd.DatetimeIndex(pd.to_datetime(dates)) + current = point_values().get(symbol) + + try: + import marketdata + declared = not marketdata.read_contract_regimes(symbol).empty + except (ImportError, AttributeError) as e: + # A marketdata too old to carry regimes. Refuse rather than silently returning + # the flat series: the whole point of this function is that the flat answer is + # wrong for some markets, and a wrong number here is invisible downstream. + raise ExposureError( + f"effective-dated contract multipliers need marketdata >= 0.2.0 " + f"(`marketdata.read_contract_regimes` is unavailable: {e}). Without it a " + f"re-denominated contract, such as RTY before 2016-12-05, would be valued " + f"at today's multiplier for its whole history.") from e + + if not declared: + return pd.Series(current, index=index, dtype="float64") + return marketdata.point_value_asof(symbol, index) + + # ── the two price series ────────────────────────────────────────────────────── @functools.lru_cache(maxsize=256) @@ -364,6 +425,7 @@ def market_exposure(name: str, *, leg: str = LEG_COMM, lookback: str = "Custom", out = pd.DataFrame({"net_contracts": net.to_numpy()}, index=dates) out.index.name = "Date" + # Membership only. The per-week values come from `point_value_series` below. pv = point_values().get(symbol) if pv is None: # Not an error. A market with no multiplier has no dollar value, and saying so @@ -375,9 +437,11 @@ def market_exposure(name: str, *, leg: str = LEG_COMM, lookback: str = "Custom", out["risk_usd"] = np.nan return out - out["point_value"] = pv + # Elementwise, not a scalar multiply. `pv` above answered whether this market has a + # multiplier at all; this answers what it was in each of these weeks. + out["point_value"] = point_value_series(symbol, out.index).to_numpy() out["price"] = _asof(price_levels(symbol), out.index, max_staleness_days).to_numpy() - out["notional_usd"] = out["net_contracts"] * pv * out["price"] + out["notional_usd"] = out["net_contracts"] * out["point_value"] * out["price"] out["sigma_daily"] = _asof( sigma_series(symbol, window=window, min_periods=min_periods), out.index, max_staleness_days).to_numpy() diff --git a/tests/test_exposure.py b/tests/test_exposure.py index 59c0a23..1c75d67 100644 --- a/tests/test_exposure.py +++ b/tests/test_exposure.py @@ -521,3 +521,136 @@ def test_the_gold_composite_still_starts_at_the_base(monkeypatch): got = ex.composite_price_index(["A"], numeraire=ex.NUMERAIRE_GOLD, frames={"A": {"symbol": "A"}}) assert got.iloc[0] == pytest.approx(100.0) + + +# ── effective-dated multipliers ─────────────────────────────────────────────── +# +# `point_values` reads contract_specs, one row per symbol with no effective date. It +# answers "can this market be priced". `point_value_series` answers "what was it worth +# that week", which differs wherever an exchange re-denominated a contract. The live +# case is RTY: ICE cut the Russell multiplier from $100 to $50 on 2016-12-05 with no +# CFTC rename to mark it, so 740 of its 1,247 priced weeks sit on the old contract. +# +# marketdata owns the regime data. These tests inject it rather than reading the +# packaged file, so they pin THIS package's behaviour (per-week multiply, refusal, +# fallback) and not marketdata's table, which has its own tests. + + +@pytest.fixture +def regimed(monkeypatch): + """Fake a marketdata whose only declared symbol is TEST: 200 before 2026-01-10, + 50 from it. Same shape as the real Russell change, an order of magnitude smaller.""" + def read_contract_regimes(symbol): + if symbol != "TEST": + return pd.DataFrame() + return pd.DataFrame({"Symbol": ["TEST", "TEST"], + "Valid_From": pd.to_datetime([None, "2026-01-10"]), + "Point_Value": [200.0, 50.0]}) + + def point_value_asof(symbol, dates): + idx = pd.DatetimeIndex(pd.to_datetime(dates)) + return pd.Series(np.where(idx < pd.Timestamp("2026-01-10"), 200.0, 50.0), + index=idx, dtype="float64") + + fake = type("M", (), {"read_contract_regimes": staticmethod(read_contract_regimes), + "point_value_asof": staticmethod(point_value_asof)}) + monkeypatch.setitem(__import__("sys").modules, "marketdata", fake) + return fake + + +def test_a_re_denominated_contract_uses_the_multiplier_of_its_own_week(priced, regimed): + frame = weekly(["2026-01-06", "2026-01-13"], comm=[-1000.0, -1000.0]) + out = ex.market_exposure("t", leg=ex.LEG_COMM, frame=frame, symbol="TEST") + assert list(out["point_value"]) == [200.0, 50.0] + # Same contract count, same price, four times the dollars before the change. + assert list(out["notional_usd"]) == [-1000 * 200 * 100, -1000 * 50 * 100] + + +def test_the_point_value_column_reports_the_week_not_today(priced, regimed): + """A reader checking why an old week is large must be able to see the multiplier + that produced it. A column carrying today's value would explain nothing.""" + frame = weekly(["2026-01-06", "2026-01-13"], comm=[0.0, 0.0]) + out = ex.market_exposure("t", leg=ex.LEG_COMM, frame=frame, symbol="TEST") + assert out["point_value"].nunique() == 2 + + +def test_risk_inherits_the_correction(priced, regimed): + """risk_usd is notional x sigma, so it must move with the multiplier, not beside it.""" + frame = weekly(["2026-01-06"], comm=[-1000.0]) + out = ex.market_exposure("t", leg=ex.LEG_COMM, frame=frame, symbol="TEST") + assert out["risk_usd"].iloc[0] == pytest.approx(-1000 * 200 * 100 * 0.02) + + +def test_an_undeclared_symbol_keeps_one_multiplier_for_its_whole_history(priced, monkeypatch): + """The common case, and it must stay a flat series: nothing established a + re-denomination for these markets, which is a claim, not a shrug.""" + fake = type("M", (), { + "read_contract_regimes": staticmethod(lambda s: pd.DataFrame()), + "point_value_asof": staticmethod(lambda s, d: pytest.fail( + "an undeclared symbol must not need a regime lookup"))}) + monkeypatch.setitem(__import__("sys").modules, "marketdata", fake) + dates = pd.to_datetime(["2026-01-06", "2026-01-13"]) + assert list(ex.point_value_series("TEST", dates)) == [50.0, 50.0] + + +def test_an_unestablished_multiplier_gives_no_dollars_rather_than_a_guess(priced, monkeypatch): + """marketdata returns NaN where a regime was never established (LBR before 1995). + The dollars must go with it: a gap is visible on a chart and a guess is not.""" + def point_value_asof(symbol, dates): + idx = pd.DatetimeIndex(pd.to_datetime(dates)) + return pd.Series([np.nan, 50.0], index=idx, dtype="float64") + + fake = type("M", (), { + "read_contract_regimes": staticmethod(lambda s: pd.DataFrame({"Symbol": ["TEST"]})), + "point_value_asof": staticmethod(point_value_asof)}) + monkeypatch.setitem(__import__("sys").modules, "marketdata", fake) + + frame = weekly(["2026-01-06", "2026-01-13"], comm=[-1000.0, -1000.0]) + out = ex.market_exposure("t", leg=ex.LEG_COMM, frame=frame, symbol="TEST") + assert np.isnan(out["notional_usd"].iloc[0]) + assert np.isnan(out["risk_usd"].iloc[0]) + assert out["notional_usd"].iloc[1] == -1000 * 50 * 100 + + +def test_a_marketdata_without_regimes_is_refused_not_silently_flattened(priced, monkeypatch): + """The failure this whole change exists to prevent. Falling back to the flat series + would return a plausible number that is wrong by 2x on RTY, and nothing downstream + could tell. Refuse loudly and name the version.""" + fake = type("M", (), {}) # marketdata < 0.2.0: no regime API at all + monkeypatch.setitem(__import__("sys").modules, "marketdata", fake) + with pytest.raises(ex.ExposureError, match="0.2.0"): + ex.point_value_series("TEST", pd.to_datetime(["2026-01-06"])) + + +def test_membership_still_comes_from_the_current_spec(priced, regimed): + """A market absent from contract_specs has no dollars at all, and that check must + stay on the CURRENT table: a regime table cannot say a market is unpriceable.""" + frame = weekly(["2026-01-06"], comm=[-1000.0]) + out = ex.market_exposure("t", leg=ex.LEG_COMM, frame=frame, symbol="ABSENT") + assert np.isnan(out["notional_usd"].iloc[0]) + assert np.isnan(out["point_value"].iloc[0]) + + +def test_weeks_with_no_multiplier_drop_out_of_the_aggregate(monkeypatch): + """An unestablished multiplier must not quietly become a zero in a sum.""" + monkeypatch.setattr(ex, "point_values", lambda: {"A": 1.0, "B": 1.0}) + monkeypatch.setattr(ex, "price_levels", lambda s, *a, **k: daily("2026-01-01", [10.0] * 40)) + monkeypatch.setattr(ex, "sigma_series", lambda s, **k: daily("2026-01-01", [0.01] * 40)) + + def point_value_asof(symbol, dates): + idx = pd.DatetimeIndex(pd.to_datetime(dates)) + vals = [np.nan, 1.0] if symbol == "A" else [1.0, 1.0] + return pd.Series(vals, index=idx, dtype="float64") + + fake = type("M", (), { + "read_contract_regimes": staticmethod(lambda s: pd.DataFrame({"Symbol": [s]})), + "point_value_asof": staticmethod(point_value_asof)}) + monkeypatch.setitem(__import__("sys").modules, "marketdata", fake) + + dates = ["2026-01-06", "2026-01-13"] + frames = {"a": {"frame": weekly(dates, comm=[-100.0, -100.0]), "symbol": "A"}, + "b": {"frame": weekly(dates, comm=[-100.0, -100.0]), "symbol": "B"}} + agg = ex.aggregate_exposure(["a", "b"], frames=frames) + # Week 1 is incomplete for A, so the total starts at week 2 rather than counting B alone. + assert list(agg.frame.index) == [pd.Timestamp("2026-01-13")] + assert agg.frame["notional_usd"].iloc[0] == -100 * 1.0 * 10 * 2