diff --git a/pyproject.toml b/pyproject.toml index 9ce681a..1585fdc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,12 @@ dependencies = [ # 0.2.0 introduced. Against 0.1.0 cotmetrics raises rather than silently # pricing the Russell's pre-2016 weeks at half their true notional. "crucible-marketdata>=0.2.0", - # Floor is 0.6.0 because /exposure ranks over a trailing window with + # Floor is 0.7.0 because the heatmap's Offside column imports + # `cotmetrics.offside`. Against 0.6.0 the module does not exist, and since + # `use_pages` imports every page module at startup, that ImportError takes + # down the whole route registry rather than one column. + # + # 0.6.0 was the floor because /exposure ranks over a trailing window with # `windowed_pct_rank` / `windowed_quantile` and passes `rank_window` to the # aggregate. Against 0.5.0 those do not exist, so the Lookback control # raises AttributeError the moment anything but All history is chosen. @@ -50,8 +55,8 @@ dependencies = [ # 0.5.0 was itself a correctness floor, for the `sigma_weighted` column the # volatility panel draws: absent, the figure falls back to three panels and # nothing says why, which is the quiet kind of wrong a floor exists to - # prevent. Both reasons stand; the higher number covers both. - "cotmetrics[options]>=0.6.0", + # prevent. All three reasons stand; the higher number covers them. + "cotmetrics[options]>=0.7.0", ] [project.optional-dependencies] diff --git a/src/pages/analytics/heatmap.py b/src/pages/analytics/heatmap.py index ed97805..836dd88 100644 --- a/src/pages/analytics/heatmap.py +++ b/src/pages/analytics/heatmap.py @@ -9,7 +9,7 @@ import dash_ag_grid as dag import dash_bootstrap_components as dbc import pandas as pd -from cotmetrics import exposure +from cotmetrics import exposure, offside from cotmetrics.indexer import get_indexer from cotmetrics.reports import get_matrix_data from dash import ( @@ -357,6 +357,102 @@ def attach_spec_risk(df, newest_date): return df +#: How far under water is worth lighting, in the market's own weekly sigma. A DISPLAY +#: threshold, deliberately rounder than any figure in the study behind the measure: the +#: pooled tenth percentile of Large Spec readings is about -1.7 and the per-market median +#: cutoff about -1.4, so -2 lights a genuinely unusual reading without implying the grid +#: reproduces a statistic. Only the losing tail is lit; a cohort deep in PROFIT is not +#: distress, and the measure is not symmetric in what it says. +OFFSIDE_DEEP = -2.0 + +#: The cohort this column reads. Large Specs alone, NOT the large+small `LEG_SPEC` the +#: dollar-risk column uses, and the difference is not cosmetic: a basis computed on the +#: summed net describes a trader who is both cohorts at once, and the two have different +#: average costs and behave differently when under water (measured in +#: `npf/docs/handoffs/2026-08-23-offside-capitulation-prereg.md`). Large Specs is also +#: the cohort every published figure for this measure is quoted on. +OFFSIDE_LEG = exposure.LEG_LARGE + + +@functools.lru_cache(maxsize=256) +def _leg_offside(asset, newest_date): + """One market's weekly offside reading, and the cost basis behind it. + + Keyed by the store's newest date purely as a cache-buster, exactly as `_spec_risk` + is: a Friday release must invalidate this and nothing else does. Lookback is not a + key and the computation always passes "Custom", because a cost basis reads net + contracts and prices, none of which the index-window control touches. + + No percentile here, unlike the dollar-risk column, and the asymmetry is the point. + Dollar risk is incomparable across markets, so it needs ranking against a market's + own history before it means anything. Offside is ALREADY comparable: dividing by the + market's own weekly sigma is what the measure does, and 0 means "at the cohort's + average cost" in every market. Ranking it would throw that away and replace a + readable quantity with a percentile of one. + + Returns {date_str: (offside, basis, price)} with NaNs already turned into None, or + None when the market cannot be marked at all. Broad catch by design: this is a + display join, and one market without prices must not take the other rows down. + """ + try: + r = offside.market_offside(asset, leg=OFFSIDE_LEG, lookback="Custom") + except Exception as e: + utils.cot_logger.warning(f"heatmap: no offside reading for {asset}: {e}") + return None + return {ts.strftime('%Y-%m-%d'): (float(o) if o == o else None, + float(b) if b == b else None, + float(p) if p == p else None) + for ts, o, b, p in zip(r.index, r["offside"].to_numpy(), + r["basis"].to_numpy(), r["price"].to_numpy())} + + +def attach_offside(df, newest_date): + """Join the offside reading onto the matrix frame, by asset and week. + + Row-by-row on the row's OWN date rather than the page's target date, matching + `attach_spec_risk`: with no target selected each market shows its latest week, and + those can differ. + + Three columns ride the rowData and only one is a grid column: the basis and the mark + exist for the cell's tooltipValueGetter, which reads them off params.data. Dropping + them here would blank the tooltip, not raise. + """ + reads, bases, prices = [], [], [] + for asset, date in zip(df["Asset"], df["Date"]): + table = _leg_offside(asset, newest_date) or {} + o, b, p = table.get(date, (None, None, None)) + reads.append(o) + bases.append(b) + prices.append(p) + # Object dtype on purpose: a float column would coerce every None to NaN, and the + # grid's null guards ('params.value != null') key on null, not NaN. + df["Offside"] = pd.Series(reads, index=df.index, dtype=object) + df["Offside Basis"] = pd.Series(bases, index=df.index, dtype=object) + df["Offside Mark"] = pd.Series(prices, index=df.index, dtype=object) + return df + + +def offside_styles_for(colors, highlight=None): + """Cell styling for the Offside column. + + Lights the LOSING tail only, and uses the bear colour for it. That is a P&L + statement rather than a market-direction verdict: the number is the sign of the + cohort's own mark-to-market, so red means "these holders are down", not "this market + goes lower". The distinction matters more here than anywhere else on the page, + because the intuitive next step (they are trapped, so they must fold) was + pre-registered, tested, and did not hold. + + The null guard is load-bearing for the same reason it is on the risk column: JS + coerces null to 0, so without it a market with no basis yet would read as deeply + offside rather than as blank. + """ + return [ + {"condition": f"params.value != null && params.value <= {OFFSIDE_DEEP}", + "style": {"color": highlight or colors.bear}}, + {"condition": "true", "style": {"color": colors.dim}}, + ] + + def risk_rank_styles_for(colors, highlight=None): """Cell styling for the Risk %ile column. @@ -393,6 +489,7 @@ def render_heatmap_layout(assest_classes, lookback, palette_name, target_date): available = get_indexer().get_available_dates() df = attach_spec_risk(df, available[0] if available else None) + df = attach_offside(df, available[0] if available else None) matrix_date = "" if not df.empty: @@ -405,6 +502,7 @@ def render_heatmap_layout(assest_classes, lookback, palette_name, target_date): oi_styles = oi_styles_for(colors, highlight=color_palette[2]) risk_rank_styles = risk_rank_styles_for(colors, highlight=color_palette[2]) + offside_styles = offside_styles_for(colors) _RAW = models.RAW_PF.band _NORM = models.NPF.band @@ -561,6 +659,45 @@ def with_bg(styles, bg="rgba(255, 255, 255, 0.04)"): }, ] }, + { + # Its own group rather than a third column under Exposure, because it reads a + # different cohort (Large Specs, not Large+Small) and answers the opposite + # question. Exposure is about SIZE; this is about P&L per contract, and the + # two move independently: a cohort can be at a record position and in profit, + # which is in fact the common case. + "headerName": "Cost Basis · Large Specs", + "children": [ + { + "field": "Offside", + "minWidth": 100, + "headerTooltip": ( + f"How far {exposure.LEG_LABELS[OFFSIDE_LEG]} sit from their own " + f"average cost, in this market's weekly standard deviations. " + f"Negative is under water. Per CONTRACT, so position size does " + f"not enter: -3 means the cohort is three typical weekly moves " + f"below what it paid, whether it holds 400 lots or 400,000. " + f"Basis is average-cost on the weekly net, marked on " + f"ratio-adjusted prices. Lit at <= {OFFSIDE_DEEP:.0f}. This is a " + f"reading of who is LOSING, not a forecast: deep readings were " + f"tested for predicting capitulation and did not, so a lit cell " + f"is not a signal that the position is about to be cut. Hover a " + f"cell for the basis. Blank until the market has half a year of " + f"priced history"), + "valueFormatter": {"function": "params.value != null ? d3.format('+.1f')(params.value) : '–'"}, + # The two prices behind the ratio, on hover rather than in columns, + # on the same argument as the dollar-risk level one group over: a + # reader who wants the level wants it once, not in every row. + "tooltipValueGetter": {"function": ( + "params.data['Offside Basis'] != null ? " + "'cost ' + d3.format(',.2f')(params.data['Offside Basis'])" + " + ' vs mark ' + d3.format(',.2f')(params.data['Offside Mark'])" + " : null")}, + "cellStyle": {"styleConditions": offside_styles}, + "headerClass": "group-border-right", + "cellClass": "group-border-right", + }, + ] + }, { "headerName": "Open Interest", "children": [ diff --git a/tests/test_heatmap_offside.py b/tests/test_heatmap_offside.py new file mode 100644 index 0000000..885c4d8 --- /dev/null +++ b/tests/test_heatmap_offside.py @@ -0,0 +1,157 @@ +"""The Offside column the heatmap joins from cotmetrics.offside. + +Store-free, matching test_heatmap_exposure: `attach_offside` is fed through a +monkeypatched `_leg_offside`, and the styling is evaluated the way test_heatmap_styles +evaluates every other condition string. The arithmetic behind the number (the cost-basis +recurrence, the sigma division) is cotmetrics' to test, not this repo's. +""" +import pandas as pd +import pytest + +import viz_config +from pages.analytics import heatmap +from tests.test_heatmap_styles import _evaluate + + +@pytest.fixture(scope="module") +def colors(): + return heatmap.grid_colors(viz_config.get_palette(None)) + + +def _matrix(rows): + return pd.DataFrame(rows, columns=["Asset", "Date"]) + + +# ── the join ────────────────────────────────────────────────────────────────── + +def test_rows_join_on_their_own_week(monkeypatch): + """Each row reads its OWN date, not the page's, matching the exposure join.""" + tables = { + "Euro": {"2026-08-18": (-2.5, 1.12, 1.05), "2026-08-11": (-1.0, 1.12, 1.09)}, + "Crude Oil": {"2026-08-18": (0.8, 60.0, 64.0)}, + } + monkeypatch.setattr(heatmap, "_leg_offside", lambda asset, newest: tables.get(asset)) + df = _matrix([("Euro", "2026-08-11"), ("Crude Oil", "2026-08-18")]) + out = heatmap.attach_offside(df, "2026-08-18") + assert list(out["Offside"]) == [-1.0, 0.8] + assert list(out["Offside Basis"]) == [1.12, 60.0] + assert list(out["Offside Mark"]) == [1.09, 64.0] + + +def test_a_market_that_cannot_be_marked_stays_a_row(monkeypatch): + """A market with no priced basis gets None in all three columns rather than + dropping the row or raising. MSCI EAFE has no futures price series at all.""" + monkeypatch.setattr( + heatmap, "_leg_offside", + lambda asset, newest: {"2026-08-18": (-3.0, 10.0, 9.0)} if asset == "Euro" else None) + df = _matrix([("Euro", "2026-08-18"), ("MSCI EAFE", "2026-08-18"), + ("Euro", "1999-01-05")]) + out = heatmap.attach_offside(df, "2026-08-18") + assert list(out["Offside"]) == [-3.0, None, None] + assert list(out["Offside Basis"]) == [10.0, None, None] + + +def test_the_computation_failing_returns_none_not_a_traceback(monkeypatch): + """The lru-cached fetch turns ANY failure into None. One market with a broken + price read must not take the rest of the matrix down with it.""" + heatmap._leg_offside.cache_clear() + + def boom(*a, **k): + raise RuntimeError("no bars") + + monkeypatch.setattr(heatmap.offside, "market_offside", boom) + assert heatmap._leg_offside("Euro", "2026-08-18") is None + heatmap._leg_offside.cache_clear() + + +def test_the_columns_survive_as_object_dtype(monkeypatch): + """A float column coerces None to NaN, and the grid's null guards key on null. + + This is the same trap the exposure columns document: the styling condition reads + `params.value != null`, and NaN is not null in JS. + """ + monkeypatch.setattr(heatmap, "_leg_offside", lambda asset, newest: None) + out = heatmap.attach_offside(_matrix([("Euro", "2026-08-18")]), "2026-08-18") + assert out["Offside"].dtype == object + assert out["Offside"].iloc[0] is None + + +# ── the styling ─────────────────────────────────────────────────────────────── + +def test_a_deeply_underwater_cell_is_lit(colors): + conds = heatmap.offside_styles_for(colors) + assert _evaluate(conds, heatmap.OFFSIDE_DEEP - 1.0, {})["color"] == colors.bear + assert _evaluate(conds, heatmap.OFFSIDE_DEEP, {})["color"] == colors.bear + + +def test_a_cohort_in_profit_is_not_lit(colors): + """Only the losing tail lights. Being deep in PROFIT is not distress, so this + column is deliberately not symmetric the way a z-score column would be.""" + conds = heatmap.offside_styles_for(colors) + assert _evaluate(conds, 5.0, {})["color"] == colors.dim + assert _evaluate(conds, 0.0, {})["color"] == colors.dim + + +def test_a_market_with_no_basis_yet_is_not_lit(colors): + """JS coerces null to 0, so without the null guard an unpriced market would read + as deeply offside. It is the set with the LEAST history behind it.""" + conds = heatmap.offside_styles_for(colors) + assert _evaluate(conds, None, {})["color"] == colors.dim + + +def test_the_highlight_is_overridable(colors): + conds = heatmap.offside_styles_for(colors, highlight="#123456") + assert _evaluate(conds, -9.0, {})["color"] == "#123456" + + +# ── what the column says ────────────────────────────────────────────────────── + +def test_the_column_reads_large_specs_alone(): + """NOT the large+small LEG_SPEC the dollar-risk column uses: a basis on the summed + net describes a trader who is both cohorts at once, and they differ.""" + assert heatmap.OFFSIDE_LEG == heatmap.exposure.LEG_LARGE + + +def test_the_tooltip_refuses_to_promise_capitulation(): + """The measure's pre-registered test returned 'adverse-move proxy'. A tooltip that + let a reader infer a forecast from a lit cell would be asserting the thing that was + tested and did not hold, so the copy says so explicitly.""" + col = _offside_col() + tip = col["headerTooltip"] + assert "not a forecast" in tip + assert "did not" in tip + + +def test_the_tooltip_says_size_does_not_enter(): + """The most likely misreading is that this is an exposure. It is per contract.""" + tip = _offside_col()["headerTooltip"] + assert "per CONTRACT" in tip or "Per CONTRACT" in tip + + +def _offside_col(): + """The Offside column def, read out of the page's source. + + The column is built inside the render callback, which needs a store and a palette + to run, so this parses it instead. Only the literal parts of an f-string survive, + which is enough: every phrase asserted above is literal text, and a phrase that got + moved into an interpolated expression would fail here rather than pass silently. + """ + import ast + import inspect + tree = ast.parse(inspect.getsource(heatmap)) + for node in ast.walk(tree): + if isinstance(node, ast.Dict): + keys = [k.value for k in node.keys if isinstance(k, ast.Constant)] + if "field" in keys: + idx = keys.index("field") + field = node.values[idx] + if isinstance(field, ast.Constant) and field.value == "Offside": + out = {} + for k, v in zip(node.keys, node.values): + if isinstance(k, ast.Constant) and isinstance(v, ast.JoinedStr): + out[k.value] = "".join( + p.value for p in v.values if isinstance(p, ast.Constant)) + elif isinstance(k, ast.Constant) and isinstance(v, ast.Constant): + out[k.value] = v.value + return out + raise AssertionError("no Offside column def found in heatmap.py")