From 29599369c1f0a4ba1c92ae809bb155fc463859ef Mon Sep 17 00:00:00 2001 From: Matt Spinola Date: Sat, 22 Aug 2026 20:20:24 -0400 Subject: [PATCH] Value each week at the multiplier that was in force that week exposure applied ONE point value, today's, to a market's whole COT history. That is wrong wherever an exchange re-denominated a contract. ICE halved the Russell multiplier from $100 to $50 on 2016-12-05 and converted each open lot into two, with no CFTC rename to mark it, so 740 of RTY's 1,247 priced weeks were valued at half their true notional and dollar risk. Audited in #22, data added in marketdata#23. Splits the question in two rather than changing point_values in place: point_values() can this market be priced at all? Current spec, unchanged, still the membership gate, so a market with no multiplier still drops out naming itself. point_value_series() what was a given WEEK worth? Defers to marketdata's regime table for declared symbols and returns the flat current value otherwise. market_exposure multiplies elementwise and reports the week's multiplier in the point_value column, so a reader checking why an old week is large can see what produced it. Measured against the real store: RTY 740 weeks at 100.0 and 507 at 50.0, median pre-cut |notional| $1.33bn -> $2.66bn, post-cut unchanged, Russell share of Equities 11.9% -> 21.2%. All 46 other markets bit-identical. The percentile is the reason this mattered more than the level. expanding_pct_rank ranks each week against its own past, so an under-scaled first half made every post-2016 reading come out more extreme rather than merely smaller. Against a marketdata without the regime API this RAISES and names the version. Falling back to the flat series would return a plausible number that is wrong by 2x on RTY, and nothing downstream could tell. Same reasoning as returning NaN where a multiplier was never established (LBR before 1995) rather than a guess: a gap is visible on a chart and a guess is not. Floor: crucible-marketdata>=0.2.0, a correctness floor rather than a convenience one. crucible-marketdata was also missing from check_dep_floors.py's INTERNAL set, so that floor would not have been checked at all; npf's and cot-analyzer's copies already listed it. Verified the check now fails against 0.1.0. 352 passed (8 new), ruff clean. Co-Authored-By: Claude Opus 5 --- pyproject.toml | 10 ++- scripts/check_dep_floors.py | 6 ++ src/cotmetrics/exposure.py | 68 +++++++++++++++++- tests/test_exposure.py | 133 ++++++++++++++++++++++++++++++++++++ 4 files changed, 213 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index af1af9f..5209e1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "cotmetrics" -version = "0.3.1" +version = "0.4.0" description = "Turn raw COT data into positioning metrics and trading signals (positioning index, concentration/clustering/position-size, reversal signals)." readme = "README.md" license = { text = "MIT" } @@ -29,7 +29,13 @@ dependencies = [ # Daily bars (ADR-0007 moved them out of cotdata); editable: -e ../marketdata. # The DISTRIBUTION is crucible-marketdata because `marketdata` on PyPI is an # unrelated abandoned project; the IMPORT is still `marketdata`. - "crucible-marketdata>=0.1.0", + # + # FLOOR 0.2.0, and it is a correctness floor rather than a convenience one. + # `exposure.point_value_series` needs `marketdata.read_contract_regimes` / + # `point_value_asof` (added there in 0.2.0) to value a week at the multiplier that + # was in force THAT week. Against 0.1.0 it raises rather than falling back, because + # the flat answer is wrong by 2x on RTY before 2016-12-05 and wrong invisibly. + "crucible-marketdata>=0.2.0", ] [project.optional-dependencies] diff --git a/scripts/check_dep_floors.py b/scripts/check_dep_floors.py index f0db7ca..270165f 100644 --- a/scripts/check_dep_floors.py +++ b/scripts/check_dep_floors.py @@ -31,6 +31,12 @@ # rather than a package index, so only these need the extra check. INTERNAL = { "cotdata", + # Distribution name; the import is `marketdata`. ADR-0007 moved bars here. It was + # missing from this set while the floor was a nominal '>=0.1.0' and nothing noticed; + # it matters now that the floor is a CORRECTNESS one (effective-dated multipliers, + # marketdata 0.2.0), because an unchecked floor is not a floor. npf's and + # cot-analyzer's copies of this script already listed it. + "crucible-marketdata", "cotmetrics", "crucible", "crucible-stack", diff --git a/src/cotmetrics/exposure.py b/src/cotmetrics/exposure.py index e2f5e8b..96e650f 100644 --- a/src/cotmetrics/exposure.py +++ b/src/cotmetrics/exposure.py @@ -33,6 +33,15 @@ That is what `pct_rank` is for, and it is why `aggregate_exposure` returns it beside every level rather than leaving it to the caller to remember. +**The multiplier is effective-dated, and the middle rung is where that bites.** A +contract is not a fixed quantity of anything: ICE halved the Russell multiplier on +2016-12-05 and converted each open lot into two, with no CFTC rename to mark it. Today's +`contract_specs` row cannot say so, so `point_values` answers only whether a market can +be priced and `point_value_series` answers what each WEEK is worth. Getting this wrong +is quiet in a way the price-series mix-up below is not: the number stays plausible, and +because `expanding_pct_rank` ranks each week against its own past, an under-scaled first +half makes the second half read as more extreme rather than merely smaller. + **The two factors come from two different price series, deliberately.** Notional needs tradeable price LEVELS and takes only ``unadj``; volatility needs correct percentage RETURNS and takes only ``propadj``. Neither substitutes for the other and both failures @@ -207,6 +216,13 @@ def point_values() -> dict: a gap to paper over: MFS and MME are ICE MSCI futures priced off the EFA and EEM ETFs, and an ETF share is not a contract, so there is no multiplier that would make their contract count into dollars. `aggregate_exposure` names what it dropped. + + **This is the CURRENT multiplier, so it answers membership, not arithmetic.** Ask it + whether a market can be priced at all; ask `point_value_series` what to multiply a + given week by. The two differ wherever an exchange re-denominated a contract, and + the difference is not small: ICE halved the Russell multiplier on 2016-12-05, so + using this value for the whole history understates 59% of RTY's priced weeks by + exactly 2x. See `point_value_series`. """ from marketdata.store import read_metadata specs = read_metadata() @@ -220,6 +236,51 @@ def point_values() -> dict: if pd.notna(val) and val > 0} +def point_value_series(symbol: str, dates) -> pd.Series: + """USD per point FOR EACH WEEK, which is not the same as USD per point. + + `point_values` reads `contract_specs`, which carries one row per symbol and no + effective date. That is the right shape for "can this market be priced" and the + wrong input for "what was this position worth in 2010", because an exchange can + re-denominate a contract and the current table cannot say so. + + The live case is the Russell. ICE cut the multiplier from $100 to $50 per index + point effective 2016-12-05 and converted each open lot into two, with no CFTC rename + to mark it, so 740 of RTY's 1,247 priced weeks sit on the old contract. Multiplying + them by today's $50 halves both notional and dollar risk. Worse than the level error, + it compresses the first half of the history that `expanding_pct_rank` ranks the + second half against, so every post-2016 reading comes out more extreme than it is. + + marketdata owns the regime table (`contract_regimes.yaml`, added in 0.2.0) because a + contract's history is a property of the contract, not of this package, and because + npf's cost model needs the same answer. Here we only ask it. + + Undeclared symbols keep the current value for every date, which is a positive claim + and not a shrug: nothing established a re-denomination for them. Where a multiplier + was never established at all, marketdata returns NaN and the dollars go with it, + which is the outcome to want. A gap is visible on a chart; a guess is not. + """ + index = pd.DatetimeIndex(pd.to_datetime(dates)) + current = point_values().get(symbol) + + try: + import marketdata + declared = not marketdata.read_contract_regimes(symbol).empty + except (ImportError, AttributeError) as e: + # A marketdata too old to carry regimes. Refuse rather than silently returning + # the flat series: the whole point of this function is that the flat answer is + # wrong for some markets, and a wrong number here is invisible downstream. + raise ExposureError( + f"effective-dated contract multipliers need marketdata >= 0.2.0 " + f"(`marketdata.read_contract_regimes` is unavailable: {e}). Without it a " + f"re-denominated contract, such as RTY before 2016-12-05, would be valued " + f"at today's multiplier for its whole history.") from e + + if not declared: + return pd.Series(current, index=index, dtype="float64") + return marketdata.point_value_asof(symbol, index) + + # ── the two price series ────────────────────────────────────────────────────── @functools.lru_cache(maxsize=256) @@ -364,6 +425,7 @@ def market_exposure(name: str, *, leg: str = LEG_COMM, lookback: str = "Custom", out = pd.DataFrame({"net_contracts": net.to_numpy()}, index=dates) out.index.name = "Date" + # Membership only. The per-week values come from `point_value_series` below. pv = point_values().get(symbol) if pv is None: # Not an error. A market with no multiplier has no dollar value, and saying so @@ -375,9 +437,11 @@ def market_exposure(name: str, *, leg: str = LEG_COMM, lookback: str = "Custom", out["risk_usd"] = np.nan return out - out["point_value"] = pv + # Elementwise, not a scalar multiply. `pv` above answered whether this market has a + # multiplier at all; this answers what it was in each of these weeks. + out["point_value"] = point_value_series(symbol, out.index).to_numpy() out["price"] = _asof(price_levels(symbol), out.index, max_staleness_days).to_numpy() - out["notional_usd"] = out["net_contracts"] * pv * out["price"] + out["notional_usd"] = out["net_contracts"] * out["point_value"] * out["price"] out["sigma_daily"] = _asof( sigma_series(symbol, window=window, min_periods=min_periods), out.index, max_staleness_days).to_numpy() diff --git a/tests/test_exposure.py b/tests/test_exposure.py index 59c0a23..1c75d67 100644 --- a/tests/test_exposure.py +++ b/tests/test_exposure.py @@ -521,3 +521,136 @@ def test_the_gold_composite_still_starts_at_the_base(monkeypatch): got = ex.composite_price_index(["A"], numeraire=ex.NUMERAIRE_GOLD, frames={"A": {"symbol": "A"}}) assert got.iloc[0] == pytest.approx(100.0) + + +# ── effective-dated multipliers ─────────────────────────────────────────────── +# +# `point_values` reads contract_specs, one row per symbol with no effective date. It +# answers "can this market be priced". `point_value_series` answers "what was it worth +# that week", which differs wherever an exchange re-denominated a contract. The live +# case is RTY: ICE cut the Russell multiplier from $100 to $50 on 2016-12-05 with no +# CFTC rename to mark it, so 740 of its 1,247 priced weeks sit on the old contract. +# +# marketdata owns the regime data. These tests inject it rather than reading the +# packaged file, so they pin THIS package's behaviour (per-week multiply, refusal, +# fallback) and not marketdata's table, which has its own tests. + + +@pytest.fixture +def regimed(monkeypatch): + """Fake a marketdata whose only declared symbol is TEST: 200 before 2026-01-10, + 50 from it. Same shape as the real Russell change, an order of magnitude smaller.""" + def read_contract_regimes(symbol): + if symbol != "TEST": + return pd.DataFrame() + return pd.DataFrame({"Symbol": ["TEST", "TEST"], + "Valid_From": pd.to_datetime([None, "2026-01-10"]), + "Point_Value": [200.0, 50.0]}) + + def point_value_asof(symbol, dates): + idx = pd.DatetimeIndex(pd.to_datetime(dates)) + return pd.Series(np.where(idx < pd.Timestamp("2026-01-10"), 200.0, 50.0), + index=idx, dtype="float64") + + fake = type("M", (), {"read_contract_regimes": staticmethod(read_contract_regimes), + "point_value_asof": staticmethod(point_value_asof)}) + monkeypatch.setitem(__import__("sys").modules, "marketdata", fake) + return fake + + +def test_a_re_denominated_contract_uses_the_multiplier_of_its_own_week(priced, regimed): + frame = weekly(["2026-01-06", "2026-01-13"], comm=[-1000.0, -1000.0]) + out = ex.market_exposure("t", leg=ex.LEG_COMM, frame=frame, symbol="TEST") + assert list(out["point_value"]) == [200.0, 50.0] + # Same contract count, same price, four times the dollars before the change. + assert list(out["notional_usd"]) == [-1000 * 200 * 100, -1000 * 50 * 100] + + +def test_the_point_value_column_reports_the_week_not_today(priced, regimed): + """A reader checking why an old week is large must be able to see the multiplier + that produced it. A column carrying today's value would explain nothing.""" + frame = weekly(["2026-01-06", "2026-01-13"], comm=[0.0, 0.0]) + out = ex.market_exposure("t", leg=ex.LEG_COMM, frame=frame, symbol="TEST") + assert out["point_value"].nunique() == 2 + + +def test_risk_inherits_the_correction(priced, regimed): + """risk_usd is notional x sigma, so it must move with the multiplier, not beside it.""" + frame = weekly(["2026-01-06"], comm=[-1000.0]) + out = ex.market_exposure("t", leg=ex.LEG_COMM, frame=frame, symbol="TEST") + assert out["risk_usd"].iloc[0] == pytest.approx(-1000 * 200 * 100 * 0.02) + + +def test_an_undeclared_symbol_keeps_one_multiplier_for_its_whole_history(priced, monkeypatch): + """The common case, and it must stay a flat series: nothing established a + re-denomination for these markets, which is a claim, not a shrug.""" + fake = type("M", (), { + "read_contract_regimes": staticmethod(lambda s: pd.DataFrame()), + "point_value_asof": staticmethod(lambda s, d: pytest.fail( + "an undeclared symbol must not need a regime lookup"))}) + monkeypatch.setitem(__import__("sys").modules, "marketdata", fake) + dates = pd.to_datetime(["2026-01-06", "2026-01-13"]) + assert list(ex.point_value_series("TEST", dates)) == [50.0, 50.0] + + +def test_an_unestablished_multiplier_gives_no_dollars_rather_than_a_guess(priced, monkeypatch): + """marketdata returns NaN where a regime was never established (LBR before 1995). + The dollars must go with it: a gap is visible on a chart and a guess is not.""" + def point_value_asof(symbol, dates): + idx = pd.DatetimeIndex(pd.to_datetime(dates)) + return pd.Series([np.nan, 50.0], index=idx, dtype="float64") + + fake = type("M", (), { + "read_contract_regimes": staticmethod(lambda s: pd.DataFrame({"Symbol": ["TEST"]})), + "point_value_asof": staticmethod(point_value_asof)}) + monkeypatch.setitem(__import__("sys").modules, "marketdata", fake) + + frame = weekly(["2026-01-06", "2026-01-13"], comm=[-1000.0, -1000.0]) + out = ex.market_exposure("t", leg=ex.LEG_COMM, frame=frame, symbol="TEST") + assert np.isnan(out["notional_usd"].iloc[0]) + assert np.isnan(out["risk_usd"].iloc[0]) + assert out["notional_usd"].iloc[1] == -1000 * 50 * 100 + + +def test_a_marketdata_without_regimes_is_refused_not_silently_flattened(priced, monkeypatch): + """The failure this whole change exists to prevent. Falling back to the flat series + would return a plausible number that is wrong by 2x on RTY, and nothing downstream + could tell. Refuse loudly and name the version.""" + fake = type("M", (), {}) # marketdata < 0.2.0: no regime API at all + monkeypatch.setitem(__import__("sys").modules, "marketdata", fake) + with pytest.raises(ex.ExposureError, match="0.2.0"): + ex.point_value_series("TEST", pd.to_datetime(["2026-01-06"])) + + +def test_membership_still_comes_from_the_current_spec(priced, regimed): + """A market absent from contract_specs has no dollars at all, and that check must + stay on the CURRENT table: a regime table cannot say a market is unpriceable.""" + frame = weekly(["2026-01-06"], comm=[-1000.0]) + out = ex.market_exposure("t", leg=ex.LEG_COMM, frame=frame, symbol="ABSENT") + assert np.isnan(out["notional_usd"].iloc[0]) + assert np.isnan(out["point_value"].iloc[0]) + + +def test_weeks_with_no_multiplier_drop_out_of_the_aggregate(monkeypatch): + """An unestablished multiplier must not quietly become a zero in a sum.""" + monkeypatch.setattr(ex, "point_values", lambda: {"A": 1.0, "B": 1.0}) + monkeypatch.setattr(ex, "price_levels", lambda s, *a, **k: daily("2026-01-01", [10.0] * 40)) + monkeypatch.setattr(ex, "sigma_series", lambda s, **k: daily("2026-01-01", [0.01] * 40)) + + def point_value_asof(symbol, dates): + idx = pd.DatetimeIndex(pd.to_datetime(dates)) + vals = [np.nan, 1.0] if symbol == "A" else [1.0, 1.0] + return pd.Series(vals, index=idx, dtype="float64") + + fake = type("M", (), { + "read_contract_regimes": staticmethod(lambda s: pd.DataFrame({"Symbol": [s]})), + "point_value_asof": staticmethod(point_value_asof)}) + monkeypatch.setitem(__import__("sys").modules, "marketdata", fake) + + dates = ["2026-01-06", "2026-01-13"] + frames = {"a": {"frame": weekly(dates, comm=[-100.0, -100.0]), "symbol": "A"}, + "b": {"frame": weekly(dates, comm=[-100.0, -100.0]), "symbol": "B"}} + agg = ex.aggregate_exposure(["a", "b"], frames=frames) + # Week 1 is incomplete for A, so the total starts at week 2 rather than counting B alone. + assert list(agg.frame.index) == [pd.Timestamp("2026-01-13")] + assert agg.frame["notional_usd"].iloc[0] == -100 * 1.0 * 10 * 2