Heatmap: an Offside column, so the grid says who is losing

pyproject.toml

@@ -42,16 +42,21 @@ dependencies = [
     # 0.2.0 introduced. Against 0.1.0 cotmetrics raises rather than silently
     # pricing the Russell's pre-2016 weeks at half their true notional.
     "crucible-marketdata>=0.2.0",
-    # Floor is 0.6.0 because /exposure ranks over a trailing window with
+    # Floor is 0.7.0 because the heatmap's Offside column imports
+    # `cotmetrics.offside`. Against 0.6.0 the module does not exist, and since
+    # `use_pages` imports every page module at startup, that ImportError takes
+    # down the whole route registry rather than one column.
+    #
+    # 0.6.0 was the floor because /exposure ranks over a trailing window with
     # `windowed_pct_rank` / `windowed_quantile` and passes `rank_window` to the
     # aggregate. Against 0.5.0 those do not exist, so the Lookback control
     # raises AttributeError the moment anything but All history is chosen.
     #
     # 0.5.0 was itself a correctness floor, for the `sigma_weighted` column the
     # volatility panel draws: absent, the figure falls back to three panels and
     # nothing says why, which is the quiet kind of wrong a floor exists to
-    # prevent. Both reasons stand; the higher number covers both.
-    "cotmetrics[options]>=0.6.0",
+    # prevent. All three reasons stand; the higher number covers them.
+    "cotmetrics[options]>=0.7.0",
 ]
 
 [project.optional-dependencies]

src/pages/analytics/heatmap.py

@@ -9,7 +9,7 @@
 import dash_ag_grid as dag
 import dash_bootstrap_components as dbc
 import pandas as pd
-from cotmetrics import exposure
+from cotmetrics import exposure, offside
 from cotmetrics.indexer import get_indexer
 from cotmetrics.reports import get_matrix_data
 from dash import (
@@ -357,6 +357,102 @@ def attach_spec_risk(df, newest_date):
     return df
 
 
+#: How far under water is worth lighting, in the market's own weekly sigma. A DISPLAY
+#: threshold, deliberately rounder than any figure in the study behind the measure: the
+#: pooled tenth percentile of Large Spec readings is about -1.7 and the per-market median
+#: cutoff about -1.4, so -2 lights a genuinely unusual reading without implying the grid
+#: reproduces a statistic. Only the losing tail is lit; a cohort deep in PROFIT is not
+#: distress, and the measure is not symmetric in what it says.
+OFFSIDE_DEEP = -2.0
+
+#: The cohort this column reads. Large Specs alone, NOT the large+small `LEG_SPEC` the
+#: dollar-risk column uses, and the difference is not cosmetic: a basis computed on the
+#: summed net describes a trader who is both cohorts at once, and the two have different
+#: average costs and behave differently when under water (measured in
+#: `npf/docs/handoffs/2026-08-23-offside-capitulation-prereg.md`). Large Specs is also
+#: the cohort every published figure for this measure is quoted on.
+OFFSIDE_LEG = exposure.LEG_LARGE
+
+
+@functools.lru_cache(maxsize=256)
+def _leg_offside(asset, newest_date):
+    """One market's weekly offside reading, and the cost basis behind it.
+
+    Keyed by the store's newest date purely as a cache-buster, exactly as `_spec_risk`
+    is: a Friday release must invalidate this and nothing else does. Lookback is not a
+    key and the computation always passes "Custom", because a cost basis reads net
+    contracts and prices, none of which the index-window control touches.
+
+    No percentile here, unlike the dollar-risk column, and the asymmetry is the point.
+    Dollar risk is incomparable across markets, so it needs ranking against a market's
+    own history before it means anything. Offside is ALREADY comparable: dividing by the
+    market's own weekly sigma is what the measure does, and 0 means "at the cohort's
+    average cost" in every market. Ranking it would throw that away and replace a
+    readable quantity with a percentile of one.
+
+    Returns {date_str: (offside, basis, price)} with NaNs already turned into None, or
+    None when the market cannot be marked at all. Broad catch by design: this is a
+    display join, and one market without prices must not take the other rows down.
+    """
+    try:
+        r = offside.market_offside(asset, leg=OFFSIDE_LEG, lookback="Custom")
+    except Exception as e:
+        utils.cot_logger.warning(f"heatmap: no offside reading for {asset}: {e}")
+        return None
+    return {ts.strftime('%Y-%m-%d'): (float(o) if o == o else None,
+                                      float(b) if b == b else None,
+                                      float(p) if p == p else None)
+            for ts, o, b, p in zip(r.index, r["offside"].to_numpy(),
+                                   r["basis"].to_numpy(), r["price"].to_numpy())}
+
+
+def attach_offside(df, newest_date):
+    """Join the offside reading onto the matrix frame, by asset and week.
+
+    Row-by-row on the row's OWN date rather than the page's target date, matching
+    `attach_spec_risk`: with no target selected each market shows its latest week, and
+    those can differ.
+
+    Three columns ride the rowData and only one is a grid column: the basis and the mark
+    exist for the cell's tooltipValueGetter, which reads them off params.data. Dropping
+    them here would blank the tooltip, not raise.
+    """
+    reads, bases, prices = [], [], []
+    for asset, date in zip(df["Asset"], df["Date"]):
+        table = _leg_offside(asset, newest_date) or {}
+        o, b, p = table.get(date, (None, None, None))
+        reads.append(o)
+        bases.append(b)
+        prices.append(p)
+    # Object dtype on purpose: a float column would coerce every None to NaN, and the
+    # grid's null guards ('params.value != null') key on null, not NaN.
+    df["Offside"] = pd.Series(reads, index=df.index, dtype=object)
+    df["Offside Basis"] = pd.Series(bases, index=df.index, dtype=object)
+    df["Offside Mark"] = pd.Series(prices, index=df.index, dtype=object)
+    return df
+
+
+def offside_styles_for(colors, highlight=None):
+    """Cell styling for the Offside column.
+
+    Lights the LOSING tail only, and uses the bear colour for it. That is a P&L
+    statement rather than a market-direction verdict: the number is the sign of the
+    cohort's own mark-to-market, so red means "these holders are down", not "this market
+    goes lower". The distinction matters more here than anywhere else on the page,
+    because the intuitive next step (they are trapped, so they must fold) was
+    pre-registered, tested, and did not hold.
+
+    The null guard is load-bearing for the same reason it is on the risk column: JS
+    coerces null to 0, so without it a market with no basis yet would read as deeply
+    offside rather than as blank.
+    """
+    return [
+        {"condition": f"params.value != null && params.value <= {OFFSIDE_DEEP}",
+         "style": {"color": highlight or colors.bear}},
+        {"condition": "true", "style": {"color": colors.dim}},
+    ]
+
+
 def risk_rank_styles_for(colors, highlight=None):
     """Cell styling for the Risk %ile column.
 
@@ -393,6 +489,7 @@ def render_heatmap_layout(assest_classes, lookback, palette_name, target_date):
 
     available = get_indexer().get_available_dates()
     df = attach_spec_risk(df, available[0] if available else None)
+    df = attach_offside(df, available[0] if available else None)
 
     matrix_date = ""
     if not df.empty:
@@ -405,6 +502,7 @@ def render_heatmap_layout(assest_classes, lookback, palette_name, target_date):
 
     oi_styles = oi_styles_for(colors, highlight=color_palette[2])
     risk_rank_styles = risk_rank_styles_for(colors, highlight=color_palette[2])
+    offside_styles = offside_styles_for(colors)
 
     _RAW = models.RAW_PF.band
     _NORM = models.NPF.band
@@ -561,6 +659,45 @@ def with_bg(styles, bg="rgba(255, 255, 255, 0.04)"):
                 },
             ]
         },
+        {
+            # Its own group rather than a third column under Exposure, because it reads a
+            # different cohort (Large Specs, not Large+Small) and answers the opposite
+            # question. Exposure is about SIZE; this is about P&L per contract, and the
+            # two move independently: a cohort can be at a record position and in profit,
+            # which is in fact the common case.
+            "headerName": "Cost Basis · Large Specs",
+            "children": [
+                {
+                    "field": "Offside",
+                    "minWidth": 100,
+                    "headerTooltip": (
+                        f"How far {exposure.LEG_LABELS[OFFSIDE_LEG]} sit from their own "
+                        f"average cost, in this market's weekly standard deviations. "
+                        f"Negative is under water. Per CONTRACT, so position size does "
+                        f"not enter: -3 means the cohort is three typical weekly moves "
+                        f"below what it paid, whether it holds 400 lots or 400,000. "
+                        f"Basis is average-cost on the weekly net, marked on "
+                        f"ratio-adjusted prices. Lit at <= {OFFSIDE_DEEP:.0f}. This is a "
+                        f"reading of who is LOSING, not a forecast: deep readings were "
+                        f"tested for predicting capitulation and did not, so a lit cell "
+                        f"is not a signal that the position is about to be cut. Hover a "
+                        f"cell for the basis. Blank until the market has half a year of "
+                        f"priced history"),
+                    "valueFormatter": {"function": "params.value != null ? d3.format('+.1f')(params.value) : '–'"},
+                    # The two prices behind the ratio, on hover rather than in columns,
+                    # on the same argument as the dollar-risk level one group over: a
+                    # reader who wants the level wants it once, not in every row.
+                    "tooltipValueGetter": {"function": (
+                        "params.data['Offside Basis'] != null ? "
+                        "'cost ' + d3.format(',.2f')(params.data['Offside Basis'])"
+                        " + ' vs mark ' + d3.format(',.2f')(params.data['Offside Mark'])"
+                        " : null")},
+                    "cellStyle": {"styleConditions": offside_styles},
+                    "headerClass": "group-border-right",
+                    "cellClass": "group-border-right",
+                },
+            ]
+        },
         {
             "headerName": "Open Interest",
             "children": [

tests/test_heatmap_offside.py

@@ -0,0 +1,157 @@
+"""The Offside column the heatmap joins from cotmetrics.offside.
+
+Store-free, matching test_heatmap_exposure: `attach_offside` is fed through a
+monkeypatched `_leg_offside`, and the styling is evaluated the way test_heatmap_styles
+evaluates every other condition string. The arithmetic behind the number (the cost-basis
+recurrence, the sigma division) is cotmetrics' to test, not this repo's.
+"""
+import pandas as pd
+import pytest
+
+import viz_config
+from pages.analytics import heatmap
+from tests.test_heatmap_styles import _evaluate
+
+
+@pytest.fixture(scope="module")
+def colors():
+    return heatmap.grid_colors(viz_config.get_palette(None))
+
+
+def _matrix(rows):
+    return pd.DataFrame(rows, columns=["Asset", "Date"])
+
+
+# ── the join ──────────────────────────────────────────────────────────────────
+
+def test_rows_join_on_their_own_week(monkeypatch):
+    """Each row reads its OWN date, not the page's, matching the exposure join."""
+    tables = {
+        "Euro": {"2026-08-18": (-2.5, 1.12, 1.05), "2026-08-11": (-1.0, 1.12, 1.09)},
+        "Crude Oil": {"2026-08-18": (0.8, 60.0, 64.0)},
+    }
+    monkeypatch.setattr(heatmap, "_leg_offside", lambda asset, newest: tables.get(asset))
+    df = _matrix([("Euro", "2026-08-11"), ("Crude Oil", "2026-08-18")])
+    out = heatmap.attach_offside(df, "2026-08-18")
+    assert list(out["Offside"]) == [-1.0, 0.8]
+    assert list(out["Offside Basis"]) == [1.12, 60.0]
+    assert list(out["Offside Mark"]) == [1.09, 64.0]
+
+
+def test_a_market_that_cannot_be_marked_stays_a_row(monkeypatch):
+    """A market with no priced basis gets None in all three columns rather than
+    dropping the row or raising. MSCI EAFE has no futures price series at all."""
+    monkeypatch.setattr(
+        heatmap, "_leg_offside",
+        lambda asset, newest: {"2026-08-18": (-3.0, 10.0, 9.0)} if asset == "Euro" else None)
+    df = _matrix([("Euro", "2026-08-18"), ("MSCI EAFE", "2026-08-18"),
+                  ("Euro", "1999-01-05")])
+    out = heatmap.attach_offside(df, "2026-08-18")
+    assert list(out["Offside"]) == [-3.0, None, None]
+    assert list(out["Offside Basis"]) == [10.0, None, None]
+
+
+def test_the_computation_failing_returns_none_not_a_traceback(monkeypatch):
+    """The lru-cached fetch turns ANY failure into None. One market with a broken
+    price read must not take the rest of the matrix down with it."""
+    heatmap._leg_offside.cache_clear()
+
+    def boom(*a, **k):
+        raise RuntimeError("no bars")
+
+    monkeypatch.setattr(heatmap.offside, "market_offside", boom)
+    assert heatmap._leg_offside("Euro", "2026-08-18") is None
+    heatmap._leg_offside.cache_clear()
+
+
+def test_the_columns_survive_as_object_dtype(monkeypatch):
+    """A float column coerces None to NaN, and the grid's null guards key on null.
+
+    This is the same trap the exposure columns document: the styling condition reads
+    `params.value != null`, and NaN is not null in JS.
+    """
+    monkeypatch.setattr(heatmap, "_leg_offside", lambda asset, newest: None)
+    out = heatmap.attach_offside(_matrix([("Euro", "2026-08-18")]), "2026-08-18")
+    assert out["Offside"].dtype == object
+    assert out["Offside"].iloc[0] is None
+
+
+# ── the styling ───────────────────────────────────────────────────────────────
+
+def test_a_deeply_underwater_cell_is_lit(colors):
+    conds = heatmap.offside_styles_for(colors)
+    assert _evaluate(conds, heatmap.OFFSIDE_DEEP - 1.0, {})["color"] == colors.bear
+    assert _evaluate(conds, heatmap.OFFSIDE_DEEP, {})["color"] == colors.bear
+
+
+def test_a_cohort_in_profit_is_not_lit(colors):
+    """Only the losing tail lights. Being deep in PROFIT is not distress, so this
+    column is deliberately not symmetric the way a z-score column would be."""
+    conds = heatmap.offside_styles_for(colors)
+    assert _evaluate(conds, 5.0, {})["color"] == colors.dim
+    assert _evaluate(conds, 0.0, {})["color"] == colors.dim
+
+
+def test_a_market_with_no_basis_yet_is_not_lit(colors):
+    """JS coerces null to 0, so without the null guard an unpriced market would read
+    as deeply offside. It is the set with the LEAST history behind it."""
+    conds = heatmap.offside_styles_for(colors)
+    assert _evaluate(conds, None, {})["color"] == colors.dim
+
+
+def test_the_highlight_is_overridable(colors):
+    conds = heatmap.offside_styles_for(colors, highlight="#123456")
+    assert _evaluate(conds, -9.0, {})["color"] == "#123456"
+
+
+# ── what the column says ──────────────────────────────────────────────────────
+
+def test_the_column_reads_large_specs_alone():
+    """NOT the large+small LEG_SPEC the dollar-risk column uses: a basis on the summed
+    net describes a trader who is both cohorts at once, and they differ."""
+    assert heatmap.OFFSIDE_LEG == heatmap.exposure.LEG_LARGE
+
+
+def test_the_tooltip_refuses_to_promise_capitulation():
+    """The measure's pre-registered test returned 'adverse-move proxy'. A tooltip that
+    let a reader infer a forecast from a lit cell would be asserting the thing that was
+    tested and did not hold, so the copy says so explicitly."""
+    col = _offside_col()
+    tip = col["headerTooltip"]
+    assert "not a forecast" in tip
+    assert "did not" in tip
+
+
+def test_the_tooltip_says_size_does_not_enter():
+    """The most likely misreading is that this is an exposure. It is per contract."""
+    tip = _offside_col()["headerTooltip"]
+    assert "per CONTRACT" in tip or "Per CONTRACT" in tip
+
+
+def _offside_col():
+    """The Offside column def, read out of the page's source.
+
+    The column is built inside the render callback, which needs a store and a palette
+    to run, so this parses it instead. Only the literal parts of an f-string survive,
+    which is enough: every phrase asserted above is literal text, and a phrase that got
+    moved into an interpolated expression would fail here rather than pass silently.
+    """
+    import ast
+    import inspect
+    tree = ast.parse(inspect.getsource(heatmap))
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Dict):
+            keys = [k.value for k in node.keys if isinstance(k, ast.Constant)]
+            if "field" in keys:
+                idx = keys.index("field")
+                field = node.values[idx]
+                if isinstance(field, ast.Constant) and field.value == "Offside":
+                    out = {}
+                    for k, v in zip(node.keys, node.values):
+                        if isinstance(k, ast.Constant) and isinstance(v, ast.JoinedStr):
+                            out[k.value] = "".join(
+                                p.value for p in v.values if isinstance(p, ast.Constant))
+                        elif isinstance(k, ast.Constant) and isinstance(v, ast.Constant):
+                            out[k.value] = v.value
+                    return out
+    raise AssertionError("no Offside column def found in heatmap.py")