diff --git a/src/cotmetrics/CotIndexer.py b/src/cotmetrics/CotIndexer.py index 8c00641..f295ad6 100644 --- a/src/cotmetrics/CotIndexer.py +++ b/src/cotmetrics/CotIndexer.py @@ -798,7 +798,15 @@ def retrieve_report_date_closing_prices(self, instrument, years, force_refresh=F # 'split' for equities. Naming 'backadj' here asked for a futures # adjustment on every symbol, so the two priced off ETF proxies # (MFS, MME) raised and got no prices at all. - price_data = marketdata.get_bars(symbol, start=start_date) + # MFS/MME have no futures series of their own, so they read an + # ETF. See market_data.PRICE_PROXIES for why that is a separate + # map from the options one and what the substitution costs. + from cotmetrics.market_data import price_symbol + px_symbol = price_symbol(symbol) + if px_symbol != symbol: + utils.cot_logger.info( + f"{symbol}: no futures series, pricing off {px_symbol}.") + price_data = marketdata.get_bars(px_symbol, start=start_date) if price_data.empty: # Not an error, and NOT to be left silent either. A store that # simply lacks a symbol returns an empty frame rather than @@ -808,10 +816,9 @@ def retrieve_report_date_closing_prices(self, instrument, years, force_refresh=F # and stop, rather than carrying an empty frame forward as though # a read had succeeded. utils.cot_logger.warning( - f"{symbol}: no bars in the marketdata store, so every " - f"price-derived column for it will be empty. Expected for a " - f"market priced off an ETF proxy that has not been seeded " - f"into the equities half (MFS -> EFA, MME -> EEM).") + f"{symbol}: no bars in the marketdata store under " + f"{px_symbol!r}, so every price-derived column for it will " + f"be empty.") except Exception as e: print(f"Error reading prices for {symbol} from the store: {e}") utils.cot_logger.error(f"Error reading prices for {symbol} from the store: {e}") diff --git a/src/cotmetrics/market_data.py b/src/cotmetrics/market_data.py index 36e643a..a1bca28 100644 --- a/src/cotmetrics/market_data.py +++ b/src/cotmetrics/market_data.py @@ -27,3 +27,31 @@ except Exception as e: # Still best-effort (this runs at import), but no longer silent. utils.cot_logger.warning(f"symbol->name map unavailable ({config.params_path()}): {e}") + + +# Markets whose PRICE comes from an ETF rather than from their own futures series. +# +# Deliberately NOT options_data.ETF_PROXIES, and the difference is the whole point. +# That map exists because a futures options chain is illiquid, so it names a proxy for +# markets that have perfectly good prices of their own: it maps ES to SPY. Reusing it +# here would silently replace S&P futures prices with an ETF's across the entire book. +# +# This map is the narrow case: two ICE MSCI markets that Norgate carries no continuous +# series for, so there is no futures price to prefer. cotdata still has their COT, which +# is why they are in the universe at all. Both are Role: heldout, so a proxied price is +# used for display and indexing rather than for anything selected or traded. +# +# THE SUBSTITUTION IS REAL AND IS NOT A DETAIL. An ETF tracks its index net of fees, in +# USD, on US session hours, and the future prices a different thing: MSCI EAFE futures +# carry basis, financing and a currency treatment the ETF does not. Levels are not +# comparable and neither are returns over a dividend date. Anything comparing these two +# markets against genuinely futures-priced ones has to know. +PRICE_PROXIES = { + "MFS": "EFA", # ICE MSCI EAFE future -> iShares MSCI EAFE + "MME": "EEM", # ICE MSCI Emerging Markets future -> iShares MSCI EM +} + + +def price_symbol(symbol: str) -> str: + """The symbol to ask marketdata for. Its own, unless it is priced off a proxy.""" + return PRICE_PROXIES.get(symbol, symbol) diff --git a/src/cotmetrics/options_data.py b/src/cotmetrics/options_data.py index 6410120..bfeea7a 100644 --- a/src/cotmetrics/options_data.py +++ b/src/cotmetrics/options_data.py @@ -357,7 +357,7 @@ def update_all_daily_options(): import marketdata import cotmetrics.utils as utils - from cotmetrics.market_data import _SYMBOL_TO_NAME + from cotmetrics.market_data import _SYMBOL_TO_NAME, price_symbol utils.cot_logger.info("Starting daily options Max Pain fetch for all instruments...") for symbol in _SYMBOL_TO_NAME.keys(): @@ -365,7 +365,7 @@ def update_all_daily_options(): # Fetch the latest prices to provide the live price for scaling the proxy ETF # Tier left to marketdata: this loop covers the whole universe, # including the ETF-proxy equities, and a futures tier raises on those. - price_df = marketdata.get_bars(symbol) + price_df = marketdata.get_bars(price_symbol(symbol)) if price_df is not None and not price_df.empty: live_price = price_df['Close'].iloc[-1] else: diff --git a/src/cotmetrics/signals.py b/src/cotmetrics/signals.py index e52b20f..b9f1fd0 100644 --- a/src/cotmetrics/signals.py +++ b/src/cotmetrics/signals.py @@ -1029,7 +1029,8 @@ def compute_weekly_rejection_scores(symbol: str, cot_dates: pd.DatetimeIndex, fo # positioning only; the tier name and the returned frame are unchanged. # Tier resolved from the symbol's domain rather than pinned, so this works # for the ETF-proxy equities as well as for futures. - daily_df = marketdata.get_bars(symbol) + from cotmetrics.market_data import price_symbol + daily_df = marketdata.get_bars(price_symbol(symbol)) if daily_df is None or daily_df.empty: return pd.DataFrame() diff --git a/tests/test_price_proxy.py b/tests/test_price_proxy.py new file mode 100644 index 0000000..2f91f55 --- /dev/null +++ b/tests/test_price_proxy.py @@ -0,0 +1,95 @@ +"""Two COT markets are priced off an ETF, and only those two. + +MFS and MME are ICE MSCI futures that Norgate carries no continuous series for, so +there is no futures price to read. cotdata still holds their COT, which is why they +are in the universe at all. Until EFA/EEM were seeded into the equities half they had +no prices whatsoever, and every price-derived column for them was empty. + +The dangerous failure here is not "no price". It is the WRONG price, silently, on a +market that has a perfectly good one of its own. That is one careless dict entry away, +so most of this file guards the blast radius rather than the feature. +""" +import pytest + +from cotmetrics.market_data import PRICE_PROXIES, price_symbol + +PROXIED = {"MFS": "EFA", "MME": "EEM"} + + +def test_the_map_is_exactly_the_two_markets_without_their_own_series(): + """A deliberately exact assertion. Growing this map is a modelling decision about + what a price MEANS for that market, not a config tweak, so it should fail here and + be argued for rather than pass quietly.""" + assert PRICE_PROXIES == PROXIED + + +@pytest.mark.parametrize("symbol, expected", sorted(PROXIED.items())) +def test_a_proxied_market_resolves_to_its_etf(symbol, expected): + assert price_symbol(symbol) == expected + + +@pytest.mark.parametrize("symbol", ["ES", "GC", "CL", "ZB", "6E", "BTC", "RTY", "NQ"]) +def test_a_market_with_its_own_series_is_never_proxied(symbol): + """The catastrophe this file exists to prevent. + + options_data.ETF_PROXIES maps ES to SPY, GC to GLD and so on, because a futures + OPTIONS chain is illiquid. Reusing that map for prices would replace S&P futures + with an ETF across the whole book, and the result would look plausible. + """ + assert price_symbol(symbol) == symbol + + +def test_the_price_map_shares_no_keys_with_the_options_map(): + """They answer different questions and must not converge. An overlap means some + market both has its own price and is being priced off something else.""" + from cotmetrics.options_data import ETF_PROXIES + + assert not (set(PRICE_PROXIES) & set(ETF_PROXIES)) + + +def test_an_unknown_symbol_passes_through(): + assert price_symbol("ZZZ") == "ZZZ" + + +@pytest.mark.parametrize("symbol, etf", sorted(PROXIED.items())) +def test_the_proxy_is_a_symbol_marketdata_knows(symbol, etf): + """Half the seed: a map pointing at a ticker the registry has never heard of + resolves to nothing, and the market is exactly as priceless as before. + + This is a property of the INSTALLED marketdata, not of a version pin, which is + why it is asserted rather than assumed. The siblings are editable installs, so + what is on disk is whatever that checkout is sitting at. + """ + marketdata = pytest.importorskip("marketdata") + + if not hasattr(marketdata, "all_symbols"): + pytest.skip("this marketdata checkout predates all_symbols()") + known = [s.internal for s in marketdata.all_symbols()] + if etf not in known: + pytest.skip( + f"{etf} is not in this marketdata checkout's registry, so {symbol} " + f"cannot be priced here. Pull the sibling past " + f"'Register EFA and EEM' (marketdata #17).") + assert marketdata.domain_for(etf) == "equities" + + +@pytest.mark.parametrize("symbol, etf", sorted(PROXIED.items())) +def test_the_proxy_has_bars_where_a_store_is_populated(symbol, etf): + """The other half, and it is a DEPLOYMENT fact rather than a code one. + + CI points MARKETDATA_STORE at an empty /tmp directory, so this can only ever be + checked on a machine with a real store. Skipping keeps that honest instead of + either failing CI forever or quietly asserting nothing: run with `-rs` and the + skip says which it was. The bars themselves are seeded by + `marketdata-update --bars --domain equities --symbols EEM EFA`. + """ + marketdata = pytest.importorskip("marketdata") + + if etf not in [s.internal for s in marketdata.all_symbols()]: + pytest.skip(f"{etf} not in this marketdata checkout's registry") + df = marketdata.get_bars(etf) + if df.empty: + pytest.skip( + f"no {etf} bars in this store, so {symbol} has no price here. Seed with " + f"marketdata-update --bars --domain equities --symbols EEM EFA") + assert len(df) > 1000, f"{etf} has only {len(df)} bars, which is not a history" diff --git a/tests/test_price_tier.py b/tests/test_price_tier.py index 128be8d..c370723 100644 --- a/tests/test_price_tier.py +++ b/tests/test_price_tier.py @@ -47,7 +47,10 @@ def spy(symbol, adjustment=None, **kwargs): monkeypatch.setattr(signals.marketdata, "get_bars", spy) signals.compute_weekly_rejection_scores("MFS", pd.DatetimeIndex([])) - assert seen["symbol"] == "MFS" + # MFS resolves to its ETF proxy before the read (market_data.PRICE_PROXIES), so + # what reaches marketdata is EFA. Either way it is an equities-domain symbol, and + # the tier must be one an equity accepts. + assert seen["symbol"] == "EFA" assert seen["adjustment"] not in FUTURES_TIERS, ( f"asked for {seen['adjustment']!r}, which raises on an equities symbol")