From a04058dcc6b86e36b70c8986ae7a7d288b8f92bf Mon Sep 17 00:00:00 2001 From: Matt Spinola Date: Thu, 20 Aug 2026 21:50:42 -0400 Subject: [PATCH 1/2] Price MFS and MME off their ETF proxies Both are ICE MSCI futures that Norgate carries no continuous series for, so there is no futures price to read. cotdata still holds their COT, which is why they are in the universe. Until now they had no prices at all and every price-derived column for them was empty. marketdata now registers EFA and EEM and the store holds their bars (marketdata claude/register-msci-proxies), so this is the half that uses them: a PRICE_PROXIES map in market_data, and a price_symbol() the three read paths go through. The map is deliberately NOT options_data.ETF_PROXIES, and the difference is the whole point. That one exists because a futures OPTIONS chain is illiquid, so it names a proxy for markets that have perfectly good prices of their own: it maps ES to SPY. Reusing it here would replace S&P futures prices with an ETF across the entire book, and the result would look plausible. Most of the new test file guards that blast radius rather than the feature: eight markets asserted never to proxy, and a check that the two maps share no keys. The substitution is real and the map says so. An ETF tracks its index net of fees, in USD, on US session hours; the future carries basis, financing and a currency treatment it does not. Levels are not comparable and neither are returns over a dividend date. Both markets are Role: heldout, so this feeds display and indexing rather than anything selected or traded. Verified on a cold cache against the real universe: MSCI EAFE and MSCI Emerging Mkts both go from zero priced weeks to 792 of 792, with Gold unchanged as a control. Co-Authored-By: Claude Opus 5 --- src/cotmetrics/CotIndexer.py | 17 ++++++--- src/cotmetrics/market_data.py | 28 +++++++++++++++ src/cotmetrics/options_data.py | 4 +-- src/cotmetrics/signals.py | 3 +- tests/test_price_proxy.py | 63 ++++++++++++++++++++++++++++++++++ tests/test_price_tier.py | 5 ++- 6 files changed, 111 insertions(+), 9 deletions(-) create mode 100644 tests/test_price_proxy.py diff --git a/src/cotmetrics/CotIndexer.py b/src/cotmetrics/CotIndexer.py index 8c00641..f295ad6 100644 --- a/src/cotmetrics/CotIndexer.py +++ b/src/cotmetrics/CotIndexer.py @@ -798,7 +798,15 @@ def retrieve_report_date_closing_prices(self, instrument, years, force_refresh=F # 'split' for equities. Naming 'backadj' here asked for a futures # adjustment on every symbol, so the two priced off ETF proxies # (MFS, MME) raised and got no prices at all. - price_data = marketdata.get_bars(symbol, start=start_date) + # MFS/MME have no futures series of their own, so they read an + # ETF. See market_data.PRICE_PROXIES for why that is a separate + # map from the options one and what the substitution costs. + from cotmetrics.market_data import price_symbol + px_symbol = price_symbol(symbol) + if px_symbol != symbol: + utils.cot_logger.info( + f"{symbol}: no futures series, pricing off {px_symbol}.") + price_data = marketdata.get_bars(px_symbol, start=start_date) if price_data.empty: # Not an error, and NOT to be left silent either. A store that # simply lacks a symbol returns an empty frame rather than @@ -808,10 +816,9 @@ def retrieve_report_date_closing_prices(self, instrument, years, force_refresh=F # and stop, rather than carrying an empty frame forward as though # a read had succeeded. utils.cot_logger.warning( - f"{symbol}: no bars in the marketdata store, so every " - f"price-derived column for it will be empty. Expected for a " - f"market priced off an ETF proxy that has not been seeded " - f"into the equities half (MFS -> EFA, MME -> EEM).") + f"{symbol}: no bars in the marketdata store under " + f"{px_symbol!r}, so every price-derived column for it will " + f"be empty.") except Exception as e: print(f"Error reading prices for {symbol} from the store: {e}") utils.cot_logger.error(f"Error reading prices for {symbol} from the store: {e}") diff --git a/src/cotmetrics/market_data.py b/src/cotmetrics/market_data.py index 36e643a..a1bca28 100644 --- a/src/cotmetrics/market_data.py +++ b/src/cotmetrics/market_data.py @@ -27,3 +27,31 @@ except Exception as e: # Still best-effort (this runs at import), but no longer silent. utils.cot_logger.warning(f"symbol->name map unavailable ({config.params_path()}): {e}") + + +# Markets whose PRICE comes from an ETF rather than from their own futures series. +# +# Deliberately NOT options_data.ETF_PROXIES, and the difference is the whole point. +# That map exists because a futures options chain is illiquid, so it names a proxy for +# markets that have perfectly good prices of their own: it maps ES to SPY. Reusing it +# here would silently replace S&P futures prices with an ETF's across the entire book. +# +# This map is the narrow case: two ICE MSCI markets that Norgate carries no continuous +# series for, so there is no futures price to prefer. cotdata still has their COT, which +# is why they are in the universe at all. Both are Role: heldout, so a proxied price is +# used for display and indexing rather than for anything selected or traded. +# +# THE SUBSTITUTION IS REAL AND IS NOT A DETAIL. An ETF tracks its index net of fees, in +# USD, on US session hours, and the future prices a different thing: MSCI EAFE futures +# carry basis, financing and a currency treatment the ETF does not. Levels are not +# comparable and neither are returns over a dividend date. Anything comparing these two +# markets against genuinely futures-priced ones has to know. +PRICE_PROXIES = { + "MFS": "EFA", # ICE MSCI EAFE future -> iShares MSCI EAFE + "MME": "EEM", # ICE MSCI Emerging Markets future -> iShares MSCI EM +} + + +def price_symbol(symbol: str) -> str: + """The symbol to ask marketdata for. Its own, unless it is priced off a proxy.""" + return PRICE_PROXIES.get(symbol, symbol) diff --git a/src/cotmetrics/options_data.py b/src/cotmetrics/options_data.py index 6410120..bfeea7a 100644 --- a/src/cotmetrics/options_data.py +++ b/src/cotmetrics/options_data.py @@ -357,7 +357,7 @@ def update_all_daily_options(): import marketdata import cotmetrics.utils as utils - from cotmetrics.market_data import _SYMBOL_TO_NAME + from cotmetrics.market_data import _SYMBOL_TO_NAME, price_symbol utils.cot_logger.info("Starting daily options Max Pain fetch for all instruments...") for symbol in _SYMBOL_TO_NAME.keys(): @@ -365,7 +365,7 @@ def update_all_daily_options(): # Fetch the latest prices to provide the live price for scaling the proxy ETF # Tier left to marketdata: this loop covers the whole universe, # including the ETF-proxy equities, and a futures tier raises on those. - price_df = marketdata.get_bars(symbol) + price_df = marketdata.get_bars(price_symbol(symbol)) if price_df is not None and not price_df.empty: live_price = price_df['Close'].iloc[-1] else: diff --git a/src/cotmetrics/signals.py b/src/cotmetrics/signals.py index e52b20f..b9f1fd0 100644 --- a/src/cotmetrics/signals.py +++ b/src/cotmetrics/signals.py @@ -1029,7 +1029,8 @@ def compute_weekly_rejection_scores(symbol: str, cot_dates: pd.DatetimeIndex, fo # positioning only; the tier name and the returned frame are unchanged. # Tier resolved from the symbol's domain rather than pinned, so this works # for the ETF-proxy equities as well as for futures. - daily_df = marketdata.get_bars(symbol) + from cotmetrics.market_data import price_symbol + daily_df = marketdata.get_bars(price_symbol(symbol)) if daily_df is None or daily_df.empty: return pd.DataFrame() diff --git a/tests/test_price_proxy.py b/tests/test_price_proxy.py new file mode 100644 index 0000000..9e85d9d --- /dev/null +++ b/tests/test_price_proxy.py @@ -0,0 +1,63 @@ +"""Two COT markets are priced off an ETF, and only those two. + +MFS and MME are ICE MSCI futures that Norgate carries no continuous series for, so +there is no futures price to read. cotdata still holds their COT, which is why they +are in the universe at all. Until EFA/EEM were seeded into the equities half they had +no prices whatsoever, and every price-derived column for them was empty. + +The dangerous failure here is not "no price". It is the WRONG price, silently, on a +market that has a perfectly good one of its own. That is one careless dict entry away, +so most of this file guards the blast radius rather than the feature. +""" +import pytest + +from cotmetrics.market_data import PRICE_PROXIES, price_symbol + +PROXIED = {"MFS": "EFA", "MME": "EEM"} + + +def test_the_map_is_exactly_the_two_markets_without_their_own_series(): + """A deliberately exact assertion. Growing this map is a modelling decision about + what a price MEANS for that market, not a config tweak, so it should fail here and + be argued for rather than pass quietly.""" + assert PRICE_PROXIES == PROXIED + + +@pytest.mark.parametrize("symbol, expected", sorted(PROXIED.items())) +def test_a_proxied_market_resolves_to_its_etf(symbol, expected): + assert price_symbol(symbol) == expected + + +@pytest.mark.parametrize("symbol", ["ES", "GC", "CL", "ZB", "6E", "BTC", "RTY", "NQ"]) +def test_a_market_with_its_own_series_is_never_proxied(symbol): + """The catastrophe this file exists to prevent. + + options_data.ETF_PROXIES maps ES to SPY, GC to GLD and so on, because a futures + OPTIONS chain is illiquid. Reusing that map for prices would replace S&P futures + with an ETF across the whole book, and the result would look plausible. + """ + assert price_symbol(symbol) == symbol + + +def test_the_price_map_shares_no_keys_with_the_options_map(): + """They answer different questions and must not converge. An overlap means some + market both has its own price and is being priced off something else.""" + from cotmetrics.options_data import ETF_PROXIES + + assert not (set(PRICE_PROXIES) & set(ETF_PROXIES)) + + +def test_an_unknown_symbol_passes_through(): + assert price_symbol("ZZZ") == "ZZZ" + + +def test_the_proxies_are_actually_readable(): + """The seed half. Without EFA/EEM in the store this map resolves to nothing and + the markets are exactly as priceless as before.""" + marketdata = pytest.importorskip("marketdata") + + for symbol, etf in PROXIED.items(): + assert etf in [s.internal for s in marketdata.all_symbols()], ( + f"{etf} is not in marketdata's registry, so {symbol} cannot be priced") + df = marketdata.get_bars(etf) + assert not df.empty, f"{etf} has no bars, so {symbol} still has no price" diff --git a/tests/test_price_tier.py b/tests/test_price_tier.py index 128be8d..c370723 100644 --- a/tests/test_price_tier.py +++ b/tests/test_price_tier.py @@ -47,7 +47,10 @@ def spy(symbol, adjustment=None, **kwargs): monkeypatch.setattr(signals.marketdata, "get_bars", spy) signals.compute_weekly_rejection_scores("MFS", pd.DatetimeIndex([])) - assert seen["symbol"] == "MFS" + # MFS resolves to its ETF proxy before the read (market_data.PRICE_PROXIES), so + # what reaches marketdata is EFA. Either way it is an equities-domain symbol, and + # the tier must be one an equity accepts. + assert seen["symbol"] == "EFA" assert seen["adjustment"] not in FUTURES_TIERS, ( f"asked for {seen['adjustment']!r}, which raises on an equities symbol") From 90e8a896cb45ceb39f4d1a46bbbc581ebcef29ad Mon Sep 17 00:00:00 2001 From: Matt Spinola Date: Thu, 20 Aug 2026 21:56:46 -0400 Subject: [PATCH 2/2] Skip the proxy's store check where there is no store The bars half of test_the_proxies_are_actually_readable could only ever pass on a machine with a populated store. CI points MARKETDATA_STORE at an empty /tmp directory, so it failed on all four Python versions, and no amount of merging would have fixed it: the registry half needed marketdata #17, but the bars half needed a deployment. Split in two, because they are two different kinds of fact. The registry check is about the INSTALLED marketdata and skips with a message naming the PR to pull. The bars check is about this machine's store and skips with the exact seeding command. Run with -rs and the skip says which, rather than the test either failing forever in CI or quietly asserting nothing. Verified both ways: against the real store 17 pass, and against an empty store 15 pass with 2 skips naming EFA and EEM. Co-Authored-By: Claude Opus 5 --- tests/test_price_proxy.py | 48 ++++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/tests/test_price_proxy.py b/tests/test_price_proxy.py index 9e85d9d..2f91f55 100644 --- a/tests/test_price_proxy.py +++ b/tests/test_price_proxy.py @@ -51,13 +51,45 @@ def test_an_unknown_symbol_passes_through(): assert price_symbol("ZZZ") == "ZZZ" -def test_the_proxies_are_actually_readable(): - """The seed half. Without EFA/EEM in the store this map resolves to nothing and - the markets are exactly as priceless as before.""" +@pytest.mark.parametrize("symbol, etf", sorted(PROXIED.items())) +def test_the_proxy_is_a_symbol_marketdata_knows(symbol, etf): + """Half the seed: a map pointing at a ticker the registry has never heard of + resolves to nothing, and the market is exactly as priceless as before. + + This is a property of the INSTALLED marketdata, not of a version pin, which is + why it is asserted rather than assumed. The siblings are editable installs, so + what is on disk is whatever that checkout is sitting at. + """ + marketdata = pytest.importorskip("marketdata") + + if not hasattr(marketdata, "all_symbols"): + pytest.skip("this marketdata checkout predates all_symbols()") + known = [s.internal for s in marketdata.all_symbols()] + if etf not in known: + pytest.skip( + f"{etf} is not in this marketdata checkout's registry, so {symbol} " + f"cannot be priced here. Pull the sibling past " + f"'Register EFA and EEM' (marketdata #17).") + assert marketdata.domain_for(etf) == "equities" + + +@pytest.mark.parametrize("symbol, etf", sorted(PROXIED.items())) +def test_the_proxy_has_bars_where_a_store_is_populated(symbol, etf): + """The other half, and it is a DEPLOYMENT fact rather than a code one. + + CI points MARKETDATA_STORE at an empty /tmp directory, so this can only ever be + checked on a machine with a real store. Skipping keeps that honest instead of + either failing CI forever or quietly asserting nothing: run with `-rs` and the + skip says which it was. The bars themselves are seeded by + `marketdata-update --bars --domain equities --symbols EEM EFA`. + """ marketdata = pytest.importorskip("marketdata") - for symbol, etf in PROXIED.items(): - assert etf in [s.internal for s in marketdata.all_symbols()], ( - f"{etf} is not in marketdata's registry, so {symbol} cannot be priced") - df = marketdata.get_bars(etf) - assert not df.empty, f"{etf} has no bars, so {symbol} still has no price" + if etf not in [s.internal for s in marketdata.all_symbols()]: + pytest.skip(f"{etf} not in this marketdata checkout's registry") + df = marketdata.get_bars(etf) + if df.empty: + pytest.skip( + f"no {etf} bars in this store, so {symbol} has no price here. Seed with " + f"marketdata-update --bars --domain equities --symbols EEM EFA") + assert len(df) > 1000, f"{etf} has only {len(df)} bars, which is not a history"