From 3d3f1eef005900521ca24e08df44ab67c8b64b47 Mon Sep 17 00:00:00 2001 From: Matt Spinola Date: Sun, 23 Aug 2026 21:21:22 -0400 Subject: [PATCH] Heatmap: an Offside column, so the grid says who is LOSING Every positioning column on this page is about size: how unusual it is, how extreme, how many dollars of risk. None of them says whether the people holding the position are up or down. This adds that, reading cotmetrics.offside: how far Large Specs sit from their own average cost, in the market's weekly sigma. Its own group rather than a third Exposure column, because it reads a different cohort (LEG_LARGE, not the LEG_SPEC dollar-risk column) and answers the opposite question. A basis on the summed large+small net describes a trader who is both cohorts at once, and the two have different costs and behave differently when under water. Large Specs is also the cohort every published figure for this measure is quoted on. No percentile, unlike Risk %ile, and the asymmetry is deliberate: dollar risk is incomparable across markets so it must be ranked against a market's own history first, while offside is already comparable by construction (dividing by the market's own sigma is what the measure does) and 0 means "at cost" everywhere. Ranking it would replace a readable quantity with a percentile of one. Only the losing tail lights, at <= -2 sigma, in the bear colour. That is a P&L statement, not a direction verdict: the number is the sign of the cohort's own mark, so red means "these holders are down", not "this market goes lower". The tooltip says so explicitly, and says the deep readings do NOT forecast a position being cut, because that exact inference was pre-registered, tested against 44 markets and did not hold (npf #214, verdict "adverse-move proxy"). A tooltip that let a reader draw it would be asserting the thing that failed. Verified against the live store: all 18 rows in Energies/Metals/Currencies populate with consistent basis/mark pairs, and the page registry still loads all 16 routes, which is the check that matters when use_pages imports every page at startup. Floor raised to cotmetrics>=0.7.0 for the same reason: against 0.6.0 the import takes down the whole registry, not one column. Co-Authored-By: Claude Fable 5 --- pyproject.toml | 11 ++- src/pages/analytics/heatmap.py | 139 ++++++++++++++++++++++++++++- tests/test_heatmap_offside.py | 157 +++++++++++++++++++++++++++++++++ 3 files changed, 303 insertions(+), 4 deletions(-) create mode 100644 tests/test_heatmap_offside.py diff --git a/pyproject.toml b/pyproject.toml index 9ce681a..1585fdc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,12 @@ dependencies = [ # 0.2.0 introduced. Against 0.1.0 cotmetrics raises rather than silently # pricing the Russell's pre-2016 weeks at half their true notional. "crucible-marketdata>=0.2.0", - # Floor is 0.6.0 because /exposure ranks over a trailing window with + # Floor is 0.7.0 because the heatmap's Offside column imports + # `cotmetrics.offside`. Against 0.6.0 the module does not exist, and since + # `use_pages` imports every page module at startup, that ImportError takes + # down the whole route registry rather than one column. + # + # 0.6.0 was the floor because /exposure ranks over a trailing window with # `windowed_pct_rank` / `windowed_quantile` and passes `rank_window` to the # aggregate. Against 0.5.0 those do not exist, so the Lookback control # raises AttributeError the moment anything but All history is chosen. @@ -50,8 +55,8 @@ dependencies = [ # 0.5.0 was itself a correctness floor, for the `sigma_weighted` column the # volatility panel draws: absent, the figure falls back to three panels and # nothing says why, which is the quiet kind of wrong a floor exists to - # prevent. Both reasons stand; the higher number covers both. - "cotmetrics[options]>=0.6.0", + # prevent. All three reasons stand; the higher number covers them. + "cotmetrics[options]>=0.7.0", ] [project.optional-dependencies] diff --git a/src/pages/analytics/heatmap.py b/src/pages/analytics/heatmap.py index ed97805..836dd88 100644 --- a/src/pages/analytics/heatmap.py +++ b/src/pages/analytics/heatmap.py @@ -9,7 +9,7 @@ import dash_ag_grid as dag import dash_bootstrap_components as dbc import pandas as pd -from cotmetrics import exposure +from cotmetrics import exposure, offside from cotmetrics.indexer import get_indexer from cotmetrics.reports import get_matrix_data from dash import ( @@ -357,6 +357,102 @@ def attach_spec_risk(df, newest_date): return df +#: How far under water is worth lighting, in the market's own weekly sigma. A DISPLAY +#: threshold, deliberately rounder than any figure in the study behind the measure: the +#: pooled tenth percentile of Large Spec readings is about -1.7 and the per-market median +#: cutoff about -1.4, so -2 lights a genuinely unusual reading without implying the grid +#: reproduces a statistic. Only the losing tail is lit; a cohort deep in PROFIT is not +#: distress, and the measure is not symmetric in what it says. +OFFSIDE_DEEP = -2.0 + +#: The cohort this column reads. Large Specs alone, NOT the large+small `LEG_SPEC` the +#: dollar-risk column uses, and the difference is not cosmetic: a basis computed on the +#: summed net describes a trader who is both cohorts at once, and the two have different +#: average costs and behave differently when under water (measured in +#: `npf/docs/handoffs/2026-08-23-offside-capitulation-prereg.md`). Large Specs is also +#: the cohort every published figure for this measure is quoted on. +OFFSIDE_LEG = exposure.LEG_LARGE + + +@functools.lru_cache(maxsize=256) +def _leg_offside(asset, newest_date): + """One market's weekly offside reading, and the cost basis behind it. + + Keyed by the store's newest date purely as a cache-buster, exactly as `_spec_risk` + is: a Friday release must invalidate this and nothing else does. Lookback is not a + key and the computation always passes "Custom", because a cost basis reads net + contracts and prices, none of which the index-window control touches. + + No percentile here, unlike the dollar-risk column, and the asymmetry is the point. + Dollar risk is incomparable across markets, so it needs ranking against a market's + own history before it means anything. Offside is ALREADY comparable: dividing by the + market's own weekly sigma is what the measure does, and 0 means "at the cohort's + average cost" in every market. Ranking it would throw that away and replace a + readable quantity with a percentile of one. + + Returns {date_str: (offside, basis, price)} with NaNs already turned into None, or + None when the market cannot be marked at all. Broad catch by design: this is a + display join, and one market without prices must not take the other rows down. + """ + try: + r = offside.market_offside(asset, leg=OFFSIDE_LEG, lookback="Custom") + except Exception as e: + utils.cot_logger.warning(f"heatmap: no offside reading for {asset}: {e}") + return None + return {ts.strftime('%Y-%m-%d'): (float(o) if o == o else None, + float(b) if b == b else None, + float(p) if p == p else None) + for ts, o, b, p in zip(r.index, r["offside"].to_numpy(), + r["basis"].to_numpy(), r["price"].to_numpy())} + + +def attach_offside(df, newest_date): + """Join the offside reading onto the matrix frame, by asset and week. + + Row-by-row on the row's OWN date rather than the page's target date, matching + `attach_spec_risk`: with no target selected each market shows its latest week, and + those can differ. + + Three columns ride the rowData and only one is a grid column: the basis and the mark + exist for the cell's tooltipValueGetter, which reads them off params.data. Dropping + them here would blank the tooltip, not raise. + """ + reads, bases, prices = [], [], [] + for asset, date in zip(df["Asset"], df["Date"]): + table = _leg_offside(asset, newest_date) or {} + o, b, p = table.get(date, (None, None, None)) + reads.append(o) + bases.append(b) + prices.append(p) + # Object dtype on purpose: a float column would coerce every None to NaN, and the + # grid's null guards ('params.value != null') key on null, not NaN. + df["Offside"] = pd.Series(reads, index=df.index, dtype=object) + df["Offside Basis"] = pd.Series(bases, index=df.index, dtype=object) + df["Offside Mark"] = pd.Series(prices, index=df.index, dtype=object) + return df + + +def offside_styles_for(colors, highlight=None): + """Cell styling for the Offside column. + + Lights the LOSING tail only, and uses the bear colour for it. That is a P&L + statement rather than a market-direction verdict: the number is the sign of the + cohort's own mark-to-market, so red means "these holders are down", not "this market + goes lower". The distinction matters more here than anywhere else on the page, + because the intuitive next step (they are trapped, so they must fold) was + pre-registered, tested, and did not hold. + + The null guard is load-bearing for the same reason it is on the risk column: JS + coerces null to 0, so without it a market with no basis yet would read as deeply + offside rather than as blank. + """ + return [ + {"condition": f"params.value != null && params.value <= {OFFSIDE_DEEP}", + "style": {"color": highlight or colors.bear}}, + {"condition": "true", "style": {"color": colors.dim}}, + ] + + def risk_rank_styles_for(colors, highlight=None): """Cell styling for the Risk %ile column. @@ -393,6 +489,7 @@ def render_heatmap_layout(assest_classes, lookback, palette_name, target_date): available = get_indexer().get_available_dates() df = attach_spec_risk(df, available[0] if available else None) + df = attach_offside(df, available[0] if available else None) matrix_date = "" if not df.empty: @@ -405,6 +502,7 @@ def render_heatmap_layout(assest_classes, lookback, palette_name, target_date): oi_styles = oi_styles_for(colors, highlight=color_palette[2]) risk_rank_styles = risk_rank_styles_for(colors, highlight=color_palette[2]) + offside_styles = offside_styles_for(colors) _RAW = models.RAW_PF.band _NORM = models.NPF.band @@ -561,6 +659,45 @@ def with_bg(styles, bg="rgba(255, 255, 255, 0.04)"): }, ] }, + { + # Its own group rather than a third column under Exposure, because it reads a + # different cohort (Large Specs, not Large+Small) and answers the opposite + # question. Exposure is about SIZE; this is about P&L per contract, and the + # two move independently: a cohort can be at a record position and in profit, + # which is in fact the common case. + "headerName": "Cost Basis · Large Specs", + "children": [ + { + "field": "Offside", + "minWidth": 100, + "headerTooltip": ( + f"How far {exposure.LEG_LABELS[OFFSIDE_LEG]} sit from their own " + f"average cost, in this market's weekly standard deviations. " + f"Negative is under water. Per CONTRACT, so position size does " + f"not enter: -3 means the cohort is three typical weekly moves " + f"below what it paid, whether it holds 400 lots or 400,000. " + f"Basis is average-cost on the weekly net, marked on " + f"ratio-adjusted prices. Lit at <= {OFFSIDE_DEEP:.0f}. This is a " + f"reading of who is LOSING, not a forecast: deep readings were " + f"tested for predicting capitulation and did not, so a lit cell " + f"is not a signal that the position is about to be cut. Hover a " + f"cell for the basis. Blank until the market has half a year of " + f"priced history"), + "valueFormatter": {"function": "params.value != null ? d3.format('+.1f')(params.value) : '–'"}, + # The two prices behind the ratio, on hover rather than in columns, + # on the same argument as the dollar-risk level one group over: a + # reader who wants the level wants it once, not in every row. + "tooltipValueGetter": {"function": ( + "params.data['Offside Basis'] != null ? " + "'cost ' + d3.format(',.2f')(params.data['Offside Basis'])" + " + ' vs mark ' + d3.format(',.2f')(params.data['Offside Mark'])" + " : null")}, + "cellStyle": {"styleConditions": offside_styles}, + "headerClass": "group-border-right", + "cellClass": "group-border-right", + }, + ] + }, { "headerName": "Open Interest", "children": [ diff --git a/tests/test_heatmap_offside.py b/tests/test_heatmap_offside.py new file mode 100644 index 0000000..885c4d8 --- /dev/null +++ b/tests/test_heatmap_offside.py @@ -0,0 +1,157 @@ +"""The Offside column the heatmap joins from cotmetrics.offside. + +Store-free, matching test_heatmap_exposure: `attach_offside` is fed through a +monkeypatched `_leg_offside`, and the styling is evaluated the way test_heatmap_styles +evaluates every other condition string. The arithmetic behind the number (the cost-basis +recurrence, the sigma division) is cotmetrics' to test, not this repo's. +""" +import pandas as pd +import pytest + +import viz_config +from pages.analytics import heatmap +from tests.test_heatmap_styles import _evaluate + + +@pytest.fixture(scope="module") +def colors(): + return heatmap.grid_colors(viz_config.get_palette(None)) + + +def _matrix(rows): + return pd.DataFrame(rows, columns=["Asset", "Date"]) + + +# ── the join ────────────────────────────────────────────────────────────────── + +def test_rows_join_on_their_own_week(monkeypatch): + """Each row reads its OWN date, not the page's, matching the exposure join.""" + tables = { + "Euro": {"2026-08-18": (-2.5, 1.12, 1.05), "2026-08-11": (-1.0, 1.12, 1.09)}, + "Crude Oil": {"2026-08-18": (0.8, 60.0, 64.0)}, + } + monkeypatch.setattr(heatmap, "_leg_offside", lambda asset, newest: tables.get(asset)) + df = _matrix([("Euro", "2026-08-11"), ("Crude Oil", "2026-08-18")]) + out = heatmap.attach_offside(df, "2026-08-18") + assert list(out["Offside"]) == [-1.0, 0.8] + assert list(out["Offside Basis"]) == [1.12, 60.0] + assert list(out["Offside Mark"]) == [1.09, 64.0] + + +def test_a_market_that_cannot_be_marked_stays_a_row(monkeypatch): + """A market with no priced basis gets None in all three columns rather than + dropping the row or raising. MSCI EAFE has no futures price series at all.""" + monkeypatch.setattr( + heatmap, "_leg_offside", + lambda asset, newest: {"2026-08-18": (-3.0, 10.0, 9.0)} if asset == "Euro" else None) + df = _matrix([("Euro", "2026-08-18"), ("MSCI EAFE", "2026-08-18"), + ("Euro", "1999-01-05")]) + out = heatmap.attach_offside(df, "2026-08-18") + assert list(out["Offside"]) == [-3.0, None, None] + assert list(out["Offside Basis"]) == [10.0, None, None] + + +def test_the_computation_failing_returns_none_not_a_traceback(monkeypatch): + """The lru-cached fetch turns ANY failure into None. One market with a broken + price read must not take the rest of the matrix down with it.""" + heatmap._leg_offside.cache_clear() + + def boom(*a, **k): + raise RuntimeError("no bars") + + monkeypatch.setattr(heatmap.offside, "market_offside", boom) + assert heatmap._leg_offside("Euro", "2026-08-18") is None + heatmap._leg_offside.cache_clear() + + +def test_the_columns_survive_as_object_dtype(monkeypatch): + """A float column coerces None to NaN, and the grid's null guards key on null. + + This is the same trap the exposure columns document: the styling condition reads + `params.value != null`, and NaN is not null in JS. + """ + monkeypatch.setattr(heatmap, "_leg_offside", lambda asset, newest: None) + out = heatmap.attach_offside(_matrix([("Euro", "2026-08-18")]), "2026-08-18") + assert out["Offside"].dtype == object + assert out["Offside"].iloc[0] is None + + +# ── the styling ─────────────────────────────────────────────────────────────── + +def test_a_deeply_underwater_cell_is_lit(colors): + conds = heatmap.offside_styles_for(colors) + assert _evaluate(conds, heatmap.OFFSIDE_DEEP - 1.0, {})["color"] == colors.bear + assert _evaluate(conds, heatmap.OFFSIDE_DEEP, {})["color"] == colors.bear + + +def test_a_cohort_in_profit_is_not_lit(colors): + """Only the losing tail lights. Being deep in PROFIT is not distress, so this + column is deliberately not symmetric the way a z-score column would be.""" + conds = heatmap.offside_styles_for(colors) + assert _evaluate(conds, 5.0, {})["color"] == colors.dim + assert _evaluate(conds, 0.0, {})["color"] == colors.dim + + +def test_a_market_with_no_basis_yet_is_not_lit(colors): + """JS coerces null to 0, so without the null guard an unpriced market would read + as deeply offside. It is the set with the LEAST history behind it.""" + conds = heatmap.offside_styles_for(colors) + assert _evaluate(conds, None, {})["color"] == colors.dim + + +def test_the_highlight_is_overridable(colors): + conds = heatmap.offside_styles_for(colors, highlight="#123456") + assert _evaluate(conds, -9.0, {})["color"] == "#123456" + + +# ── what the column says ────────────────────────────────────────────────────── + +def test_the_column_reads_large_specs_alone(): + """NOT the large+small LEG_SPEC the dollar-risk column uses: a basis on the summed + net describes a trader who is both cohorts at once, and they differ.""" + assert heatmap.OFFSIDE_LEG == heatmap.exposure.LEG_LARGE + + +def test_the_tooltip_refuses_to_promise_capitulation(): + """The measure's pre-registered test returned 'adverse-move proxy'. A tooltip that + let a reader infer a forecast from a lit cell would be asserting the thing that was + tested and did not hold, so the copy says so explicitly.""" + col = _offside_col() + tip = col["headerTooltip"] + assert "not a forecast" in tip + assert "did not" in tip + + +def test_the_tooltip_says_size_does_not_enter(): + """The most likely misreading is that this is an exposure. It is per contract.""" + tip = _offside_col()["headerTooltip"] + assert "per CONTRACT" in tip or "Per CONTRACT" in tip + + +def _offside_col(): + """The Offside column def, read out of the page's source. + + The column is built inside the render callback, which needs a store and a palette + to run, so this parses it instead. Only the literal parts of an f-string survive, + which is enough: every phrase asserted above is literal text, and a phrase that got + moved into an interpolated expression would fail here rather than pass silently. + """ + import ast + import inspect + tree = ast.parse(inspect.getsource(heatmap)) + for node in ast.walk(tree): + if isinstance(node, ast.Dict): + keys = [k.value for k in node.keys if isinstance(k, ast.Constant)] + if "field" in keys: + idx = keys.index("field") + field = node.values[idx] + if isinstance(field, ast.Constant) and field.value == "Offside": + out = {} + for k, v in zip(node.keys, node.values): + if isinstance(k, ast.Constant) and isinstance(v, ast.JoinedStr): + out[k.value] = "".join( + p.value for p in v.values if isinstance(p, ast.Constant)) + elif isinstance(k, ast.Constant) and isinstance(v, ast.Constant): + out[k.value] = v.value + return out + raise AssertionError("no Offside column def found in heatmap.py")