plot_splitgap.py

"""Temporal vs random split AUC — how much a random split flatters each model.

A random 70/30 split puts future lots in the training set. Every model gains from that,
and the ones with the most fitting freedom gain most. The spread bars matter as much as
the gap: the single 70/30 temporal AUC is itself noisy, and the rolling-origin range
(cuts at 50/60/70/80%) shows how noisy.

Reads phase4_splits.csv, written by phase4.py.
"""
import numpy as np, pandas as pd
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt

TEMP, RAND, INK, MUTED, SURF = "#4a3aa7", "#8a8a85", "#0b0b0b", "#52514e", "#fcfcfb"
SHORT = {
    "A  all 590 signals": "A · all 590 signals",
    "B  shortlist-5 (FDR ∩ stability)": "B · shortlist-5",
    "B' tier-1 only (Bonferroni ∩ stab)": "B' · tier-1 (3 signals)",
    "C  train-fold-only shortlist": "C · train-only reselection",
    "D  BH q=0.05 set (28 signals)": "D · BH q=0.05 (28)",
}

D = pd.read_csv("phase4_splits.csv")
t = D[D.split == "temporal"].set_index("model").auc
r = D[D.split == "random"].groupby("model").auc.agg(["mean", "std"])
g = D[D.split == "rolling"].groupby("model").auc.agg(["min", "max"])
order = t.sort_values().index.tolist()
ypos = np.arange(len(order))

fig, ax = plt.subplots(figsize=(10.5, 4.8))
fig.patch.set_facecolor(SURF)

for i, mdl in enumerate(order):
    ax.plot([t[mdl], r.loc[mdl, "mean"]], [i, i], color=MUTED, lw=1.3, zorder=1)
    ax.plot([g.loc[mdl, "min"], g.loc[mdl, "max"]], [i, i], color=TEMP, lw=5,
            alpha=.22, solid_capstyle="butt", zorder=2)
    ax.errorbar(r.loc[mdl, "mean"], i, xerr=r.loc[mdl, "std"], fmt="o", ms=8,
                color=RAND, ecolor=RAND, elinewidth=1.6, capsize=3,
                markeredgecolor=SURF, markeredgewidth=1.2, zorder=3)
    ax.plot(t[mdl], i, "o", ms=9, color=TEMP, markeredgecolor=SURF,
            markeredgewidth=1.2, zorder=4)
    ax.annotate(f"{r.loc[mdl,'mean']-t[mdl]:+.3f}", xy=(1.015, i),
                xycoords=("axes fraction", "data"), ha="left", va="center", fontsize=9,
                color=INK if r.loc[mdl, "mean"]-t[mdl] > .07 else MUTED)

ax.axvline(.5, color=INK, lw=1.1, ls=":")
ax.annotate("chance", xy=(.5, -.55), xytext=(4, 0), textcoords="offset points",
            fontsize=8.5, color=INK, va="center")
ax.set_yticks(ypos)
ax.set_yticklabels([SHORT.get(m, m) for m in order], fontsize=9.5)
ax.set_ylim(-.7, len(order)-.3)
ax.set_xlim(.44, max(.82, g["max"].max()+.02))
ax.set_xlabel("test-set AUC", color=MUTED)
ax.set_title("A random split flatters every model — most where there is most to fit",
             fontsize=11.5, color=INK, loc="left", pad=34)
ax.annotate("● temporal 70/30 (the honest number)     ● random 70/30, mean ± sd over 20 seeds"
            "     ▬ rolling-origin range, cuts at 50/60/70/80%",
            xy=(0, 1.012), xycoords="axes fraction", ha="left", va="bottom",
            fontsize=8.5, color=MUTED)
ax.annotate("● ", xy=(0, 1.012), xycoords="axes fraction", ha="left", va="bottom",
            fontsize=8.5, color=TEMP)
ax.annotate("gap", xy=(1.015, len(order)-.55), xycoords=("axes fraction", "data"),
            ha="left", va="center", fontsize=8.5, color=MUTED)

ax.set_facecolor(SURF)
ax.grid(axis="x", color="#e6e5e1", lw=.8)
ax.set_axisbelow(True)
for s in ("top", "right", "left"):
    ax.spines[s].set_visible(False)
ax.spines["bottom"].set_color("#d5d4cf")
ax.tick_params(colors=MUTED, labelsize=9)

fig.suptitle("Temporal vs random split — the gap the naive protocol hides",
             fontsize=13, color=INK, x=.008, ha="left", y=.975)
fig.text(.008, .012,
         "Random splits train on future lots. Model C re-derives its own shortlist inside the training "
         "fold, so its gap (+0.135) is\nselection leakage rather than split leakage — and its rolling range "
         "never reaches the full-record shortlist's. Model A's rolling\nrange spans 0.466–0.715: with 590 "
         "features and 78 training failures, the temporal number is barely a number. See REPORT.md.",
         fontsize=7.5, color=MUTED, ha="left", linespacing=1.5)
fig.tight_layout(rect=[0, .105, .93, .93])
fig.savefig("figures/split_gap.png", dpi=200, facecolor=fig.get_facecolor())

out = pd.DataFrame({"temporal": t, "random_mean": r["mean"], "random_sd": r["std"],
                    "gap": r["mean"]-t, "rolling_min": g["min"], "rolling_max": g["max"]})
print(out.round(3).to_string())
print("  -> figures/split_gap.png")