Price MFS and MME off their ETF proxies

src/cotmetrics/CotIndexer.py

@@ -798,7 +798,15 @@ def retrieve_report_date_closing_prices(self, instrument, years, force_refresh=F
                 # 'split' for equities. Naming 'backadj' here asked for a futures
                 # adjustment on every symbol, so the two priced off ETF proxies
                 # (MFS, MME) raised and got no prices at all.
-                price_data = marketdata.get_bars(symbol, start=start_date)
+                # MFS/MME have no futures series of their own, so they read an
+                # ETF. See market_data.PRICE_PROXIES for why that is a separate
+                # map from the options one and what the substitution costs.
+                from cotmetrics.market_data import price_symbol
+                px_symbol = price_symbol(symbol)
+                if px_symbol != symbol:
+                    utils.cot_logger.info(
+                        f"{symbol}: no futures series, pricing off {px_symbol}.")
+                price_data = marketdata.get_bars(px_symbol, start=start_date)
                 if price_data.empty:
                     # Not an error, and NOT to be left silent either. A store that
                     # simply lacks a symbol returns an empty frame rather than
@@ -808,10 +816,9 @@ def retrieve_report_date_closing_prices(self, instrument, years, force_refresh=F
                     # and stop, rather than carrying an empty frame forward as though
                     # a read had succeeded.
                     utils.cot_logger.warning(
-                        f"{symbol}: no bars in the marketdata store, so every "
-                        f"price-derived column for it will be empty. Expected for a "
-                        f"market priced off an ETF proxy that has not been seeded "
-                        f"into the equities half (MFS -> EFA, MME -> EEM).")
+                        f"{symbol}: no bars in the marketdata store under "
+                        f"{px_symbol!r}, so every price-derived column for it will "
+                        f"be empty.")
             except Exception as e:
                 print(f"Error reading prices for {symbol} from the store: {e}")
                 utils.cot_logger.error(f"Error reading prices for {symbol} from the store: {e}")

src/cotmetrics/market_data.py

@@ -27,3 +27,31 @@
 except Exception as e:
     # Still best-effort (this runs at import), but no longer silent.
     utils.cot_logger.warning(f"symbol->name map unavailable ({config.params_path()}): {e}")
+
+
+# Markets whose PRICE comes from an ETF rather than from their own futures series.
+#
+# Deliberately NOT options_data.ETF_PROXIES, and the difference is the whole point.
+# That map exists because a futures options chain is illiquid, so it names a proxy for
+# markets that have perfectly good prices of their own: it maps ES to SPY. Reusing it
+# here would silently replace S&P futures prices with an ETF's across the entire book.
+#
+# This map is the narrow case: two ICE MSCI markets that Norgate carries no continuous
+# series for, so there is no futures price to prefer. cotdata still has their COT, which
+# is why they are in the universe at all. Both are Role: heldout, so a proxied price is
+# used for display and indexing rather than for anything selected or traded.
+#
+# THE SUBSTITUTION IS REAL AND IS NOT A DETAIL. An ETF tracks its index net of fees, in
+# USD, on US session hours, and the future prices a different thing: MSCI EAFE futures
+# carry basis, financing and a currency treatment the ETF does not. Levels are not
+# comparable and neither are returns over a dividend date. Anything comparing these two
+# markets against genuinely futures-priced ones has to know.
+PRICE_PROXIES = {
+    "MFS": "EFA",   # ICE MSCI EAFE future -> iShares MSCI EAFE
+    "MME": "EEM",   # ICE MSCI Emerging Markets future -> iShares MSCI EM
+}
+
+
+def price_symbol(symbol: str) -> str:
+    """The symbol to ask marketdata for. Its own, unless it is priced off a proxy."""
+    return PRICE_PROXIES.get(symbol, symbol)

src/cotmetrics/options_data.py

@@ -357,15 +357,15 @@ def update_all_daily_options():
     import marketdata
 
     import cotmetrics.utils as utils
-    from cotmetrics.market_data import _SYMBOL_TO_NAME
+    from cotmetrics.market_data import _SYMBOL_TO_NAME, price_symbol
 
     utils.cot_logger.info("Starting daily options Max Pain fetch for all instruments...")
     for symbol in _SYMBOL_TO_NAME.keys():
         try:
             # Fetch the latest prices to provide the live price for scaling the proxy ETF
             # Tier left to marketdata: this loop covers the whole universe,
             # including the ETF-proxy equities, and a futures tier raises on those.
-            price_df = marketdata.get_bars(symbol)
+            price_df = marketdata.get_bars(price_symbol(symbol))
             if price_df is not None and not price_df.empty:
                 live_price = price_df['Close'].iloc[-1]
             else:

src/cotmetrics/signals.py

@@ -1029,7 +1029,8 @@ def compute_weekly_rejection_scores(symbol: str, cot_dates: pd.DatetimeIndex, fo
     # positioning only; the tier name and the returned frame are unchanged.
     # Tier resolved from the symbol's domain rather than pinned, so this works
     # for the ETF-proxy equities as well as for futures.
-    daily_df = marketdata.get_bars(symbol)
+    from cotmetrics.market_data import price_symbol
+    daily_df = marketdata.get_bars(price_symbol(symbol))
     if daily_df is None or daily_df.empty:
         return pd.DataFrame()
 

tests/test_price_proxy.py

@@ -0,0 +1,95 @@
+"""Two COT markets are priced off an ETF, and only those two.
+
+MFS and MME are ICE MSCI futures that Norgate carries no continuous series for, so
+there is no futures price to read. cotdata still holds their COT, which is why they
+are in the universe at all. Until EFA/EEM were seeded into the equities half they had
+no prices whatsoever, and every price-derived column for them was empty.
+
+The dangerous failure here is not "no price". It is the WRONG price, silently, on a
+market that has a perfectly good one of its own. That is one careless dict entry away,
+so most of this file guards the blast radius rather than the feature.
+"""
+import pytest
+
+from cotmetrics.market_data import PRICE_PROXIES, price_symbol
+
+PROXIED = {"MFS": "EFA", "MME": "EEM"}
+
+
+def test_the_map_is_exactly_the_two_markets_without_their_own_series():
+    """A deliberately exact assertion. Growing this map is a modelling decision about
+    what a price MEANS for that market, not a config tweak, so it should fail here and
+    be argued for rather than pass quietly."""
+    assert PRICE_PROXIES == PROXIED
+
+
+@pytest.mark.parametrize("symbol, expected", sorted(PROXIED.items()))
+def test_a_proxied_market_resolves_to_its_etf(symbol, expected):
+    assert price_symbol(symbol) == expected
+
+
+@pytest.mark.parametrize("symbol", ["ES", "GC", "CL", "ZB", "6E", "BTC", "RTY", "NQ"])
+def test_a_market_with_its_own_series_is_never_proxied(symbol):
+    """The catastrophe this file exists to prevent.
+
+    options_data.ETF_PROXIES maps ES to SPY, GC to GLD and so on, because a futures
+    OPTIONS chain is illiquid. Reusing that map for prices would replace S&P futures
+    with an ETF across the whole book, and the result would look plausible.
+    """
+    assert price_symbol(symbol) == symbol
+
+
+def test_the_price_map_shares_no_keys_with_the_options_map():
+    """They answer different questions and must not converge. An overlap means some
+    market both has its own price and is being priced off something else."""
+    from cotmetrics.options_data import ETF_PROXIES
+
+    assert not (set(PRICE_PROXIES) & set(ETF_PROXIES))
+
+
+def test_an_unknown_symbol_passes_through():
+    assert price_symbol("ZZZ") == "ZZZ"
+
+
+@pytest.mark.parametrize("symbol, etf", sorted(PROXIED.items()))
+def test_the_proxy_is_a_symbol_marketdata_knows(symbol, etf):
+    """Half the seed: a map pointing at a ticker the registry has never heard of
+    resolves to nothing, and the market is exactly as priceless as before.
+
+    This is a property of the INSTALLED marketdata, not of a version pin, which is
+    why it is asserted rather than assumed. The siblings are editable installs, so
+    what is on disk is whatever that checkout is sitting at.
+    """
+    marketdata = pytest.importorskip("marketdata")
+
+    if not hasattr(marketdata, "all_symbols"):
+        pytest.skip("this marketdata checkout predates all_symbols()")
+    known = [s.internal for s in marketdata.all_symbols()]
+    if etf not in known:
+        pytest.skip(
+            f"{etf} is not in this marketdata checkout's registry, so {symbol} "
+            f"cannot be priced here. Pull the sibling past "
+            f"'Register EFA and EEM' (marketdata #17).")
+    assert marketdata.domain_for(etf) == "equities"
+
+
+@pytest.mark.parametrize("symbol, etf", sorted(PROXIED.items()))
+def test_the_proxy_has_bars_where_a_store_is_populated(symbol, etf):
+    """The other half, and it is a DEPLOYMENT fact rather than a code one.
+
+    CI points MARKETDATA_STORE at an empty /tmp directory, so this can only ever be
+    checked on a machine with a real store. Skipping keeps that honest instead of
+    either failing CI forever or quietly asserting nothing: run with `-rs` and the
+    skip says which it was. The bars themselves are seeded by
+    `marketdata-update --bars --domain equities --symbols EEM EFA`.
+    """
+    marketdata = pytest.importorskip("marketdata")
+
+    if etf not in [s.internal for s in marketdata.all_symbols()]:
+        pytest.skip(f"{etf} not in this marketdata checkout's registry")
+    df = marketdata.get_bars(etf)
+    if df.empty:
+        pytest.skip(
+            f"no {etf} bars in this store, so {symbol} has no price here. Seed with "
+            f"marketdata-update --bars --domain equities --symbols EEM EFA")
+    assert len(df) > 1000, f"{etf} has only {len(df)} bars, which is not a history"

tests/test_price_tier.py

@@ -47,7 +47,10 @@ def spy(symbol, adjustment=None, **kwargs):
     monkeypatch.setattr(signals.marketdata, "get_bars", spy)
     signals.compute_weekly_rejection_scores("MFS", pd.DatetimeIndex([]))
 
-    assert seen["symbol"] == "MFS"
+    # MFS resolves to its ETF proxy before the read (market_data.PRICE_PROXIES), so
+    # what reaches marketdata is EFA. Either way it is an equities-domain symbol, and
+    # the tier must be one an equity accepts.
+    assert seen["symbol"] == "EFA"
     assert seen["adjustment"] not in FUTURES_TIERS, (
         f"asked for {seen['adjustment']!r}, which raises on an equities symbol")