Value each week at the multiplier in force that week

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "cotmetrics"
-version = "0.3.1"
+version = "0.4.0"
 description = "Turn raw COT data into positioning metrics and trading signals (positioning index, concentration/clustering/position-size, reversal signals)."
 readme = "README.md"
 license = { text = "MIT" }
@@ -29,7 +29,13 @@ dependencies = [
     # Daily bars (ADR-0007 moved them out of cotdata); editable: -e ../marketdata.
     # The DISTRIBUTION is crucible-marketdata because `marketdata` on PyPI is an
     # unrelated abandoned project; the IMPORT is still `marketdata`.
-    "crucible-marketdata>=0.1.0",
+    #
+    # FLOOR 0.2.0, and it is a correctness floor rather than a convenience one.
+    # `exposure.point_value_series` needs `marketdata.read_contract_regimes` /
+    # `point_value_asof` (added there in 0.2.0) to value a week at the multiplier that
+    # was in force THAT week. Against 0.1.0 it raises rather than falling back, because
+    # the flat answer is wrong by 2x on RTY before 2016-12-05 and wrong invisibly.
+    "crucible-marketdata>=0.2.0",
 ]
 
 [project.optional-dependencies]

scripts/check_dep_floors.py

@@ -31,6 +31,12 @@
 # rather than a package index, so only these need the extra check.
 INTERNAL = {
     "cotdata",
+    # Distribution name; the import is `marketdata`. ADR-0007 moved bars here. It was
+    # missing from this set while the floor was a nominal '>=0.1.0' and nothing noticed;
+    # it matters now that the floor is a CORRECTNESS one (effective-dated multipliers,
+    # marketdata 0.2.0), because an unchecked floor is not a floor. npf's and
+    # cot-analyzer's copies of this script already listed it.
+    "crucible-marketdata",
     "cotmetrics",
     "crucible",
     "crucible-stack",

src/cotmetrics/exposure.py

@@ -33,6 +33,15 @@
 That is what `pct_rank` is for, and it is why `aggregate_exposure` returns it beside
 every level rather than leaving it to the caller to remember.
 
+**The multiplier is effective-dated, and the middle rung is where that bites.** A
+contract is not a fixed quantity of anything: ICE halved the Russell multiplier on
+2016-12-05 and converted each open lot into two, with no CFTC rename to mark it. Today's
+`contract_specs` row cannot say so, so `point_values` answers only whether a market can
+be priced and `point_value_series` answers what each WEEK is worth. Getting this wrong
+is quiet in a way the price-series mix-up below is not: the number stays plausible, and
+because `expanding_pct_rank` ranks each week against its own past, an under-scaled first
+half makes the second half read as more extreme rather than merely smaller.
+
 **The two factors come from two different price series, deliberately.** Notional needs
 tradeable price LEVELS and takes only ``unadj``; volatility needs correct percentage
 RETURNS and takes only ``propadj``. Neither substitutes for the other and both failures
@@ -207,6 +216,13 @@ def point_values() -> dict:
     a gap to paper over: MFS and MME are ICE MSCI futures priced off the EFA and EEM
     ETFs, and an ETF share is not a contract, so there is no multiplier that would make
     their contract count into dollars. `aggregate_exposure` names what it dropped.
+
+    **This is the CURRENT multiplier, so it answers membership, not arithmetic.** Ask it
+    whether a market can be priced at all; ask `point_value_series` what to multiply a
+    given week by. The two differ wherever an exchange re-denominated a contract, and
+    the difference is not small: ICE halved the Russell multiplier on 2016-12-05, so
+    using this value for the whole history understates 59% of RTY's priced weeks by
+    exactly 2x. See `point_value_series`.
     """
     from marketdata.store import read_metadata
     specs = read_metadata()
@@ -220,6 +236,51 @@ def point_values() -> dict:
             if pd.notna(val) and val > 0}
 
 
+def point_value_series(symbol: str, dates) -> pd.Series:
+    """USD per point FOR EACH WEEK, which is not the same as USD per point.
+
+    `point_values` reads `contract_specs`, which carries one row per symbol and no
+    effective date. That is the right shape for "can this market be priced" and the
+    wrong input for "what was this position worth in 2010", because an exchange can
+    re-denominate a contract and the current table cannot say so.
+
+    The live case is the Russell. ICE cut the multiplier from $100 to $50 per index
+    point effective 2016-12-05 and converted each open lot into two, with no CFTC rename
+    to mark it, so 740 of RTY's 1,247 priced weeks sit on the old contract. Multiplying
+    them by today's $50 halves both notional and dollar risk. Worse than the level error,
+    it compresses the first half of the history that `expanding_pct_rank` ranks the
+    second half against, so every post-2016 reading comes out more extreme than it is.
+
+    marketdata owns the regime table (`contract_regimes.yaml`, added in 0.2.0) because a
+    contract's history is a property of the contract, not of this package, and because
+    npf's cost model needs the same answer. Here we only ask it.
+
+    Undeclared symbols keep the current value for every date, which is a positive claim
+    and not a shrug: nothing established a re-denomination for them. Where a multiplier
+    was never established at all, marketdata returns NaN and the dollars go with it,
+    which is the outcome to want. A gap is visible on a chart; a guess is not.
+    """
+    index = pd.DatetimeIndex(pd.to_datetime(dates))
+    current = point_values().get(symbol)
+
+    try:
+        import marketdata
+        declared = not marketdata.read_contract_regimes(symbol).empty
+    except (ImportError, AttributeError) as e:
+        # A marketdata too old to carry regimes. Refuse rather than silently returning
+        # the flat series: the whole point of this function is that the flat answer is
+        # wrong for some markets, and a wrong number here is invisible downstream.
+        raise ExposureError(
+            f"effective-dated contract multipliers need marketdata >= 0.2.0 "
+            f"(`marketdata.read_contract_regimes` is unavailable: {e}). Without it a "
+            f"re-denominated contract, such as RTY before 2016-12-05, would be valued "
+            f"at today's multiplier for its whole history.") from e
+
+    if not declared:
+        return pd.Series(current, index=index, dtype="float64")
+    return marketdata.point_value_asof(symbol, index)
+
+
 # ── the two price series ──────────────────────────────────────────────────────
 
 @functools.lru_cache(maxsize=256)
@@ -364,6 +425,7 @@ def market_exposure(name: str, *, leg: str = LEG_COMM, lookback: str = "Custom",
     out = pd.DataFrame({"net_contracts": net.to_numpy()}, index=dates)
     out.index.name = "Date"
 
+    # Membership only. The per-week values come from `point_value_series` below.
     pv = point_values().get(symbol)
     if pv is None:
         # Not an error. A market with no multiplier has no dollar value, and saying so
@@ -375,9 +437,11 @@ def market_exposure(name: str, *, leg: str = LEG_COMM, lookback: str = "Custom",
         out["risk_usd"] = np.nan
         return out
 
-    out["point_value"] = pv
+    # Elementwise, not a scalar multiply. `pv` above answered whether this market has a
+    # multiplier at all; this answers what it was in each of these weeks.
+    out["point_value"] = point_value_series(symbol, out.index).to_numpy()
     out["price"] = _asof(price_levels(symbol), out.index, max_staleness_days).to_numpy()
-    out["notional_usd"] = out["net_contracts"] * pv * out["price"]
+    out["notional_usd"] = out["net_contracts"] * out["point_value"] * out["price"]
     out["sigma_daily"] = _asof(
         sigma_series(symbol, window=window, min_periods=min_periods),
         out.index, max_staleness_days).to_numpy()

tests/test_exposure.py

@@ -521,3 +521,136 @@ def test_the_gold_composite_still_starts_at_the_base(monkeypatch):
     got = ex.composite_price_index(["A"], numeraire=ex.NUMERAIRE_GOLD,
                                    frames={"A": {"symbol": "A"}})
     assert got.iloc[0] == pytest.approx(100.0)
+
+
+# ── effective-dated multipliers ───────────────────────────────────────────────
+#
+# `point_values` reads contract_specs, one row per symbol with no effective date. It
+# answers "can this market be priced". `point_value_series` answers "what was it worth
+# that week", which differs wherever an exchange re-denominated a contract. The live
+# case is RTY: ICE cut the Russell multiplier from $100 to $50 on 2016-12-05 with no
+# CFTC rename to mark it, so 740 of its 1,247 priced weeks sit on the old contract.
+#
+# marketdata owns the regime data. These tests inject it rather than reading the
+# packaged file, so they pin THIS package's behaviour (per-week multiply, refusal,
+# fallback) and not marketdata's table, which has its own tests.
+
+
+@pytest.fixture
+def regimed(monkeypatch):
+    """Fake a marketdata whose only declared symbol is TEST: 200 before 2026-01-10,
+    50 from it. Same shape as the real Russell change, an order of magnitude smaller."""
+    def read_contract_regimes(symbol):
+        if symbol != "TEST":
+            return pd.DataFrame()
+        return pd.DataFrame({"Symbol": ["TEST", "TEST"],
+                             "Valid_From": pd.to_datetime([None, "2026-01-10"]),
+                             "Point_Value": [200.0, 50.0]})
+
+    def point_value_asof(symbol, dates):
+        idx = pd.DatetimeIndex(pd.to_datetime(dates))
+        return pd.Series(np.where(idx < pd.Timestamp("2026-01-10"), 200.0, 50.0),
+                         index=idx, dtype="float64")
+
+    fake = type("M", (), {"read_contract_regimes": staticmethod(read_contract_regimes),
+                          "point_value_asof": staticmethod(point_value_asof)})
+    monkeypatch.setitem(__import__("sys").modules, "marketdata", fake)
+    return fake
+
+
+def test_a_re_denominated_contract_uses_the_multiplier_of_its_own_week(priced, regimed):
+    frame = weekly(["2026-01-06", "2026-01-13"], comm=[-1000.0, -1000.0])
+    out = ex.market_exposure("t", leg=ex.LEG_COMM, frame=frame, symbol="TEST")
+    assert list(out["point_value"]) == [200.0, 50.0]
+    # Same contract count, same price, four times the dollars before the change.
+    assert list(out["notional_usd"]) == [-1000 * 200 * 100, -1000 * 50 * 100]
+
+
+def test_the_point_value_column_reports_the_week_not_today(priced, regimed):
+    """A reader checking why an old week is large must be able to see the multiplier
+    that produced it. A column carrying today's value would explain nothing."""
+    frame = weekly(["2026-01-06", "2026-01-13"], comm=[0.0, 0.0])
+    out = ex.market_exposure("t", leg=ex.LEG_COMM, frame=frame, symbol="TEST")
+    assert out["point_value"].nunique() == 2
+
+
+def test_risk_inherits_the_correction(priced, regimed):
+    """risk_usd is notional x sigma, so it must move with the multiplier, not beside it."""
+    frame = weekly(["2026-01-06"], comm=[-1000.0])
+    out = ex.market_exposure("t", leg=ex.LEG_COMM, frame=frame, symbol="TEST")
+    assert out["risk_usd"].iloc[0] == pytest.approx(-1000 * 200 * 100 * 0.02)
+
+
+def test_an_undeclared_symbol_keeps_one_multiplier_for_its_whole_history(priced, monkeypatch):
+    """The common case, and it must stay a flat series: nothing established a
+    re-denomination for these markets, which is a claim, not a shrug."""
+    fake = type("M", (), {
+        "read_contract_regimes": staticmethod(lambda s: pd.DataFrame()),
+        "point_value_asof": staticmethod(lambda s, d: pytest.fail(
+            "an undeclared symbol must not need a regime lookup"))})
+    monkeypatch.setitem(__import__("sys").modules, "marketdata", fake)
+    dates = pd.to_datetime(["2026-01-06", "2026-01-13"])
+    assert list(ex.point_value_series("TEST", dates)) == [50.0, 50.0]
+
+
+def test_an_unestablished_multiplier_gives_no_dollars_rather_than_a_guess(priced, monkeypatch):
+    """marketdata returns NaN where a regime was never established (LBR before 1995).
+    The dollars must go with it: a gap is visible on a chart and a guess is not."""
+    def point_value_asof(symbol, dates):
+        idx = pd.DatetimeIndex(pd.to_datetime(dates))
+        return pd.Series([np.nan, 50.0], index=idx, dtype="float64")
+
+    fake = type("M", (), {
+        "read_contract_regimes": staticmethod(lambda s: pd.DataFrame({"Symbol": ["TEST"]})),
+        "point_value_asof": staticmethod(point_value_asof)})
+    monkeypatch.setitem(__import__("sys").modules, "marketdata", fake)
+
+    frame = weekly(["2026-01-06", "2026-01-13"], comm=[-1000.0, -1000.0])
+    out = ex.market_exposure("t", leg=ex.LEG_COMM, frame=frame, symbol="TEST")
+    assert np.isnan(out["notional_usd"].iloc[0])
+    assert np.isnan(out["risk_usd"].iloc[0])
+    assert out["notional_usd"].iloc[1] == -1000 * 50 * 100
+
+
+def test_a_marketdata_without_regimes_is_refused_not_silently_flattened(priced, monkeypatch):
+    """The failure this whole change exists to prevent. Falling back to the flat series
+    would return a plausible number that is wrong by 2x on RTY, and nothing downstream
+    could tell. Refuse loudly and name the version."""
+    fake = type("M", (), {})           # marketdata < 0.2.0: no regime API at all
+    monkeypatch.setitem(__import__("sys").modules, "marketdata", fake)
+    with pytest.raises(ex.ExposureError, match="0.2.0"):
+        ex.point_value_series("TEST", pd.to_datetime(["2026-01-06"]))
+
+
+def test_membership_still_comes_from_the_current_spec(priced, regimed):
+    """A market absent from contract_specs has no dollars at all, and that check must
+    stay on the CURRENT table: a regime table cannot say a market is unpriceable."""
+    frame = weekly(["2026-01-06"], comm=[-1000.0])
+    out = ex.market_exposure("t", leg=ex.LEG_COMM, frame=frame, symbol="ABSENT")
+    assert np.isnan(out["notional_usd"].iloc[0])
+    assert np.isnan(out["point_value"].iloc[0])
+
+
+def test_weeks_with_no_multiplier_drop_out_of_the_aggregate(monkeypatch):
+    """An unestablished multiplier must not quietly become a zero in a sum."""
+    monkeypatch.setattr(ex, "point_values", lambda: {"A": 1.0, "B": 1.0})
+    monkeypatch.setattr(ex, "price_levels", lambda s, *a, **k: daily("2026-01-01", [10.0] * 40))
+    monkeypatch.setattr(ex, "sigma_series", lambda s, **k: daily("2026-01-01", [0.01] * 40))
+
+    def point_value_asof(symbol, dates):
+        idx = pd.DatetimeIndex(pd.to_datetime(dates))
+        vals = [np.nan, 1.0] if symbol == "A" else [1.0, 1.0]
+        return pd.Series(vals, index=idx, dtype="float64")
+
+    fake = type("M", (), {
+        "read_contract_regimes": staticmethod(lambda s: pd.DataFrame({"Symbol": [s]})),
+        "point_value_asof": staticmethod(point_value_asof)})
+    monkeypatch.setitem(__import__("sys").modules, "marketdata", fake)
+
+    dates = ["2026-01-06", "2026-01-13"]
+    frames = {"a": {"frame": weekly(dates, comm=[-100.0, -100.0]), "symbol": "A"},
+              "b": {"frame": weekly(dates, comm=[-100.0, -100.0]), "symbol": "B"}}
+    agg = ex.aggregate_exposure(["a", "b"], frames=frames)
+    # Week 1 is incomplete for A, so the total starts at week 2 rather than counting B alone.
+    assert list(agg.frame.index) == [pd.Timestamp("2026-01-13")]
+    assert agg.frame["notional_usd"].iloc[0] == -100 * 1.0 * 10 * 2