diff --git a/docs/autopilot/decisions.md b/docs/autopilot/decisions.md index 290e923..4f0f438 100644 --- a/docs/autopilot/decisions.md +++ b/docs/autopilot/decisions.md @@ -190,3 +190,15 @@ Question: Does `--season-level` run alongside the race-level backtest or inst Chosen: Instead of. `gp backtest --season-level` runs only the season harness and writes `season_level` into `backtest_summary.json`. If that file already exists in `--out-dir` (a race-level run pointed at the same dir), the block is merged in and the race-level keys are kept. A cutoff past a season's last round, a failed fit, or a failed chain is a named failed cutoff, and any failed or unattempted cutoff, or a scored cutoff with no legacy number, makes `v2_beats_legacy_spearman` null. Rejected: Running both in one invocation. The race-level grid is the slow part (~2.7 h with `--post-quali`) and the season harness is 12 fits for 2022-2025; coupling them makes the cheap one wait on the expensive one. Blast radius: `cli.cmd_backtest` (a new branch, the old path untouched) and `backtest.run_season_backtest`. + +## 2026-10-25 — slice-10 +Question: The queue asks for "an expected calibration error (or similar single number, defined in the PR)" for both arms, but not which bins, which rounds, or what file shape. +Chosen: Ten equal-width bins on [0, 1] (p = 1.0 in the top bin), the same 0.1 bins `docs/phase2-status.md` already reads the pre-weekend file with. ECE = n-weighted mean of |mean predicted - observed podium rate| over the non-empty bins, in probability units. The summary gets `calibration_podium.pre` (every scored round), and with `--post-quali` also `postquali` and `pre_paired` (pre restricted to the rounds post-quali scored), so the two ECEs compare the same races. Both CSVs carry `season, round` after the original two columns, which is what makes `pre_paired` computable. +Rejected: Equal-count (quantile) bins. They would put different bin edges on each arm and make the two tables impossible to read side by side. Also rejected: comparing post-quali against the all-rounds pre table only, which would let a failed post-quali round change one side and not the other. +Blast radius: `backtest.calibration_summary`, `_podium_calibration`, `run_backtest`'s summary. `calibration_podium.csv` gains two trailing columns; the existing two keep their names and order. No forecast moves. + +## 2026-10-25 — slice-10 +Question: The queue asks for rounds where post-quali log loss is worse than pre-weekend "by more than a stated threshold", and leaves the threshold to the PR. +Chosen: 1.0 nats (`POSTQUALI_WORSE_THRESHOLD_NATS`), strictly greater. That is post-quali putting about 2.7x less probability on the actual winner than pre-weekend did. On the Oct 4 run it would include the three rounds `phase2-status.md` names (deltas 2.39, 2.24, 1.57). How many of the other 10 worse rounds clear it is unknown from the sandbox, which has no `backtest_metrics.csv`. The grid-pole driver and the actual winner are read off the race rows (`grid_position == 1`, `finish_position == 1`) and added to every row of `backtest_metrics.csv`. +Rejected: Any positive delta. It would list all 13 worse rounds, most of them noise. Also rejected: a threshold relative to the round's pre log loss, which is harder to read and not what the queue asked for. +Blast radius: One constant and `backtest.postquali_worse_rounds`; listing only, feeds no gate. diff --git a/docs/autopilot/queue.md b/docs/autopilot/queue.md index 288979d..405f873 100644 --- a/docs/autopilot/queue.md +++ b/docs/autopilot/queue.md @@ -464,7 +464,7 @@ Acceptance: ## slice-10: post-quali calibration -Status: todo +Status: in-review — PR #23 opened by the build routine 2026-10-25. Spec: `docs/phase2-status.md` caveat 1 ("Calibration was not re-checked for post-quali"). diff --git a/src/backtest.py b/src/backtest.py index 8a2c5d9..86673e0 100644 --- a/src/backtest.py +++ b/src/backtest.py @@ -49,6 +49,13 @@ COARSE_ROUNDS = [2, 4, 6, 9, 12, 16, 20] EPS = 1e-3 +# Podium calibration bins: ten equal-width bins on [0, 1], the same 0.1 bins +# docs/phase2-status.md reads the pre-weekend file with. +CALIBRATION_N_BINS = 10 +# A round is listed as a post-quali "confident miss" when its winner log loss is +# worse than pre-weekend's by more than this many nats (~2.7x less probability +# on the actual winner). Listing only; it feeds no verdict. +POSTQUALI_WORSE_THRESHOLD_NATS = 1.0 def _log_loss_winner(p_win: dict, actual_winner: str) -> float: @@ -139,8 +146,11 @@ def score_round(forecast: pd.DataFrame, driver_rounds: pd.DataFrame, season: int p_win_pole = _pole_wins_baseline(actual) p_win_pace = _pace_rating_baseline(driver_rounds, actual, season, round_) + pole = actual[actual["grid_position"] == 1] result = { "season": season, "round": round_, "n_drivers": len(actual), + "actual_winner": actual_winner, + "grid_pole_driver": pole.iloc[0]["abbreviation"] if not pole.empty else None, "log_loss_winner_model": _log_loss_winner(p_win_model, actual_winner), "log_loss_winner_pole": _log_loss_winner(p_win_pole, actual_winner), "log_loss_winner_pace_rating": _log_loss_winner(p_win_pace, actual_winner), @@ -249,6 +259,64 @@ def _calibration_bins(forecast: pd.DataFrame, actual: pd.DataFrame, col: str, th return out +CALIBRATION_COLUMNS = ["p_podium_pred", "podium_actual", "season", "round"] + + +def _podium_calibration(forecast: pd.DataFrame, actual: pd.DataFrame, season: int, + round_: int) -> list[tuple]: + """One (p_podium_pred, podium_actual, season, round) row per scored driver. + Both arms write this shape, so their files can be compared row for row.""" + return [(p, hit, season, round_) for p, hit in _calibration_bins( + forecast, actual, "p_podium", lambda a: set(a[a["finish_position"] <= 3]["abbreviation"]))] + + +def calibration_summary(calib: pd.DataFrame, n_bins: int = CALIBRATION_N_BINS) -> dict: + """Binned reliability table and expected calibration error for one arm. + + Bins are equal-width on [0, 1] (p = 1.0 falls in the top bin); empty bins + are left out of the table. ECE is the n-weighted mean of + |mean predicted - observed rate| over the non-empty bins: 0 is perfectly + calibrated, and it is in probability units, so 0.02 means the forecast is + off by two points on average where its predictions actually fall.""" + if calib.empty: + return {"n": 0, "ece": None, "bins": []} + p = calib["p_podium_pred"].astype(float).to_numpy() + hit = calib["podium_actual"].astype(bool).to_numpy() + idx = np.clip(np.floor(p * n_bins).astype(int), 0, n_bins - 1) + bins, ece = [], 0.0 + for b in range(n_bins): + mask = idx == b + n = int(mask.sum()) + if not n: + continue + mean_pred, rate = float(p[mask].mean()), float(hit[mask].mean()) + ece += n / len(p) * abs(mean_pred - rate) + bins.append({"bin_lo": b / n_bins, "bin_hi": (b + 1) / n_bins, "n": n, + "mean_pred": mean_pred, "observed_rate": rate}) + return {"n": len(p), "ece": float(ece), "bins": bins} + + +def postquali_worse_rounds(metrics_df: pd.DataFrame, + threshold: float = POSTQUALI_WORSE_THRESHOLD_NATS) -> list[dict]: + """Rounds where post-quali winner log loss is worse than pre-weekend's by + more than `threshold` nats, worst first, with the grid-pole driver and the + actual winner. Raw material for "do the confident misses share a cause"; + no cause is inferred here.""" + cols = {"log_loss_winner_model", "log_loss_winner_postquali", "season", "round"} + if not cols <= set(metrics_df.columns): + return [] + both = metrics_df.dropna(subset=["log_loss_winner_postquali"]) + delta = both["log_loss_winner_postquali"] - both["log_loss_winner_model"] + worse = both.assign(delta=delta)[delta > threshold].sort_values("delta", ascending=False) + return [{"season": int(r["season"]), "round": int(r["round"]), + "log_loss_pre": float(r["log_loss_winner_model"]), + "log_loss_postquali": float(r["log_loss_winner_postquali"]), + "delta": float(r["delta"]), + "grid_pole_driver": r.get("grid_pole_driver"), + "actual_winner": r.get("actual_winner")} + for _, r in worse.iterrows()] + + POST_QUALI_METRICS = ("log_loss_winner", "brier_podium", "brier_points", "spearman") @@ -299,6 +367,8 @@ def post_quali_summary(metrics_df: pd.DataFrame, conditioning_log: pd.DataFrame, out["gate_invalid_reasons"] = reasons out["postquali_beats_pre_log_loss"] = (bool(out["postquali_log_loss_delta"] < 0) if not reasons else None) + out["postquali_worse_threshold_nats"] = POSTQUALI_WORSE_THRESHOLD_NATS + out["postquali_worse_rounds"] = postquali_worse_rounds(both) if len(conditioning_log): from model.conditioning import fallback_rate, regressions out["fallback_rate"] = fallback_rate(conditioning_log) @@ -310,13 +380,15 @@ def post_quali_summary(metrics_df: pd.DataFrame, conditioning_log: pd.DataFrame, def _post_quali_arm(prior_idata, driver_rounds, driver_meta, circuits, reliability_data, season: int, round_: int, n_trials: int, chains: int, - conditioning_records: list, outdir: str) -> dict: + conditioning_records: list, outdir: str, + calibration_rows: list | None = None) -> dict: """Condition on round `round_`'s quali (warm-started NUTS, spec sec 6), forecast from the real grid, score it. Always returns a `postquali_status` -- "ok" with the `*_postquali` metric keys, or "conditioning_failed" / "forecast_failed" / "unscored" with nothing else. A failure is printed and counted by `post_quali_summary`; the round has no post-quali score, never a - silent zero. Only a round that conditioned writes a conditioning record.""" + silent zero. Only a round that conditioned writes a conditioning record, and + only a scored round adds rows to `calibration_rows`.""" from dataclasses import asdict from cli import forecast_race from model.conditioning import append_record, condition_on_quali @@ -342,6 +414,9 @@ def _post_quali_arm(prior_idata, driver_rounds, driver_meta, circuits, reliabili if scored is None: print(f" WARNING: post-quali forecast for {season}:{round_} could not be scored") return {"postquali_status": "unscored"} + if calibration_rows is not None: + calibration_rows.extend(_podium_calibration( + forecast, _actual_outcomes(driver_rounds, season, round_), season, round_)) out = {f"{m}_postquali": scored[f"{m}_model"] for m in POST_QUALI_METRICS} out["postquali_status"] = "ok" return out @@ -362,6 +437,7 @@ def run_backtest(seasons: list[int], n_trials: int = 5000, method: str = "advi", results = [] calibration_rows = [] + calibration_rows_postquali = [] conditioning_records = [] t_start = time.time() @@ -447,22 +523,19 @@ def run_backtest(seasons: list[int], n_trials: int = 5000, method: str = "advi", metrics.update(_post_quali_arm( idata, driver_rounds, driver_meta, circuits, reliability_data, season, rr, n_trials, post_quali_chains, conditioning_records, outdir, + calibration_rows=calibration_rows_postquali, )) if metrics is not None: metrics["fit_elapsed_s"] = fit_elapsed results.append(metrics) - actual = _actual_outcomes(driver_rounds, season, rr) - calibration_rows.extend( - (p, hit) for p, hit in _calibration_bins( - forecast, actual, "p_podium", lambda a: set(a[a["finish_position"] <= 3]["abbreviation"]) - ) - ) + calibration_rows.extend(_podium_calibration( + forecast, _actual_outcomes(driver_rounds, season, rr), season, rr)) total_elapsed = time.time() - t_start print(f"[backtest] total elapsed: {total_elapsed:.1f}s ({total_elapsed / 60:.1f} min)") metrics_df = pd.DataFrame(results) - calib_df = pd.DataFrame(calibration_rows, columns=["p_podium_pred", "podium_actual"]) + calib_df = pd.DataFrame(calibration_rows, columns=CALIBRATION_COLUMNS) os.makedirs(outdir, exist_ok=True) metrics_df.to_csv(os.path.join(outdir, "backtest_metrics.csv"), index=False) @@ -471,6 +544,7 @@ def run_backtest(seasons: list[int], n_trials: int = 5000, method: str = "advi", summary = { "seasons": seasons, "n_rounds_scored": len(metrics_df), "rounds_mode": rounds, "method": method, "n_trials": n_trials, "total_elapsed_s": total_elapsed, + "calibration_podium": {"pre": calibration_summary(calib_df)}, } if not metrics_df.empty: summary.update({ @@ -501,6 +575,16 @@ def run_backtest(seasons: list[int], n_trials: int = 5000, method: str = "advi", summary["post_quali"] = True summary.update(post_quali_summary(metrics_df, pd.DataFrame(conditioning_records), rounds_mode=rounds)) + calib_post = pd.DataFrame(calibration_rows_postquali, columns=CALIBRATION_COLUMNS) + calib_post.to_csv(os.path.join(outdir, "calibration_podium_postquali.csv"), index=False) + # pre restricted to the rounds post-quali scored, so the two ECEs are + # over the same races and differ only in the forecast + paired = calib_df.merge(calib_post[["season", "round"]].drop_duplicates(), + on=["season", "round"]) + summary["calibration_podium"].update({ + "pre_paired": calibration_summary(paired), + "postquali": calibration_summary(calib_post), + }) with open(os.path.join(outdir, "backtest_summary.json"), "w") as f: json.dump(summary, f, indent=2) diff --git a/tests/test_postquali_calibration.py b/tests/test_postquali_calibration.py new file mode 100644 index 0000000..cff4d3a --- /dev/null +++ b/tests/test_postquali_calibration.py @@ -0,0 +1,146 @@ +"""Post-quali calibration (autopilot slice-10; docs/phase2-status.md caveat 1): +a podium calibration file per arm, a binned table and an ECE for both arms in +the summary, and the rounds where post-quali did much worse than pre-weekend.""" +import json +import os +import shutil + +import numpy as np +import pandas as pd +import pytest + +import backtest +import model.pace +from backtest import ( + CALIBRATION_COLUMNS, calibration_summary, post_quali_summary, postquali_worse_rounds, + score_round, +) + +FIXTURE = os.path.join(os.path.dirname(__file__), "..", "data", "driver_rounds_fixture.csv") + + +def _calib(pairs): + return pd.DataFrame([(p, h, 2024, 1) for p, h in pairs], columns=CALIBRATION_COLUMNS) + + +# --- calibration_summary ---------------------------------------------------- + +def test_binned_table_and_ece_match_a_hand_computation(): + # bin [0.0, 0.1): preds 0.05, 0.05, hits 0/2 -> |0.05 - 0| = 0.05, weight 2/5 + # bin [0.5, 0.6): preds 0.55 x3, hits 3/3 -> 0.45, weight 3/5 + out = calibration_summary(_calib([(0.05, False), (0.05, False), + (0.55, True), (0.55, True), (0.55, True)])) + assert out["n"] == 5 + assert [(b["bin_lo"], b["n"]) for b in out["bins"]] == [(0.0, 2), (0.5, 3)] + assert out["bins"][1]["mean_pred"] == pytest.approx(0.55) + assert out["bins"][1]["observed_rate"] == 1.0 + assert out["ece"] == pytest.approx(2 / 5 * 0.05 + 3 / 5 * 0.45) + + +def test_a_perfectly_calibrated_arm_has_zero_ece(): + pairs = [(0.25, True)] + [(0.25, False)] * 3 + [(0.75, True)] * 3 + [(0.75, False)] + assert calibration_summary(_calib(pairs))["ece"] == pytest.approx(0.0) + + +def test_p_of_one_falls_in_the_top_bin_and_empty_bins_are_omitted(): + out = calibration_summary(_calib([(1.0, True), (0.0, False)])) + assert [(b["bin_lo"], b["bin_hi"]) for b in out["bins"]] == [(0.0, 0.1), (0.9, 1.0)] + + +def test_an_empty_arm_reports_no_ece_rather_than_zero(): + assert calibration_summary(_calib([])) == {"n": 0, "ece": None, "bins": []} + + +# --- the confident-miss listing -------------------------------------------- + +def _metrics(pre, post): + n = len(pre) + return pd.DataFrame({ + "season": [2022] * n, "round": list(range(1, n + 1)), + "log_loss_winner_model": pre, "log_loss_winner_postquali": post, + "grid_pole_driver": [f"P{i}" for i in range(n)], + "actual_winner": [f"W{i}" for i in range(n)], + "brier_podium_model": 0.1, "brier_points_model": 0.2, "spearman_model": 0.5, + "brier_podium_postquali": 0.1, "brier_points_postquali": 0.2, "spearman_postquali": 0.5, + }) + + +def test_only_rounds_worse_by_more_than_the_threshold_are_listed_worst_first(): + # deltas: +2.4, +1.0 (at threshold: not listed), -0.5, +1.5, NaN (no post score) + m = _metrics([1.5, 1.0, 1.0, 1.0, 1.0], [3.9, 2.0, 0.5, 2.5, float("nan")]) + out = postquali_worse_rounds(m, threshold=1.0) + assert [(r["round"], r["grid_pole_driver"], r["actual_winner"]) for r in out] == [ + (1, "P0", "W0"), (4, "P3", "W3")] + assert out[0]["delta"] == pytest.approx(2.4) + assert out[0]["log_loss_pre"] == 1.5 and out[0]["log_loss_postquali"] == 3.9 + + +def test_post_quali_summary_carries_the_listing_and_its_threshold(): + out = post_quali_summary(_metrics([1.0, 1.0], [3.0, 0.5]), + pd.DataFrame({"fell_back": [False, False], "regression": [False, False]})) + assert out["postquali_worse_threshold_nats"] == backtest.POSTQUALI_WORSE_THRESHOLD_NATS + assert [r["round"] for r in out["postquali_worse_rounds"]] == [1] + json.dumps(out) # the listing must survive into backtest_summary.json + + +def test_score_round_names_the_pole_sitter_and_the_winner(): + dr = pd.read_csv(FIXTURE) + race = dr[(dr["season"] == 2024) & (dr["round"] == 1) & (dr["session_type"] == "R")] + forecast = pd.DataFrame({"driver": race["abbreviation"], "p_win": 0.05, "p_podium": 0.15, + "p_points": 0.5, "exp_points": np.arange(len(race), 0, -1)}) + out = score_round(forecast, dr, 2024, 1) + assert out["grid_pole_driver"] == race.loc[race["grid_position"] == 1, "abbreviation"].iloc[0] + assert out["actual_winner"] == race.loc[race["finish_position"] == 1, "abbreviation"].iloc[0] + + +# --- the arm's calibration rows ---------------------------------------------- + +def test_the_post_quali_arm_adds_one_calibration_row_per_scored_driver(monkeypatch, tmp_path): + import cli + import model.conditioning as conditioning + from model.conditioning import ConditioningRecord + + dr = pd.read_csv(FIXTURE) + race = dr[(dr["season"] == 2024) & (dr["round"] == 2) & (dr["session_type"] == "R")] + forecast = pd.DataFrame({"driver": race["abbreviation"], "p_win": 0.05, "p_podium": 0.2, + "p_points": 0.5, "exp_points": np.arange(len(race), 0, -1)}) + record = ConditioningRecord(season=2024, round=2, path="warm_start", fell_back=False, + early_season=True, warm_start_s=1.0, fallback_s=0.0, total_s=1.0, + warm_target_s=120.0, applied_target_s=300.0, regression=False) + monkeypatch.setattr(conditioning, "condition_on_quali", lambda *a, **kw: ("I", "D", record)) + monkeypatch.setattr(cli, "forecast_race", lambda *a, **kw: forecast) + rows = [] + out = backtest._post_quali_arm("PRIOR", dr, None, None, "REL", 2024, 2, 100, 1, [], + str(tmp_path), calibration_rows=rows) + assert out["postquali_status"] == "ok" + assert len(rows) == len(race) + assert sum(hit for _, hit, _, _ in rows) == 3 + assert {(s, r) for _, _, s, r in rows} == {(2024, 2)} + + +# --- end to end on the fixture --------------------------------------------- + +def test_end_to_end_writes_both_calibration_files_and_both_summaries(monkeypatch, tmp_path): + data_dir = tmp_path / "data" + data_dir.mkdir() + shutil.copy(FIXTURE, data_dir / "driver_rounds.csv") + monkeypatch.setattr(backtest, "DATA_DIR", str(data_dir)) + # conditioning is NUTS; swap in a tiny ADVI fit so this runs in seconds + real_advi = model.pace.fit_advi + monkeypatch.setattr(model.pace, "fit_nuts", + lambda data, **kw: real_advi(data, n=200, draws=20)) + out_dir = tmp_path / "out" + backtest.run_backtest([2024], n_trials=200, method="advi", rounds="all", advi_steps=200, + out_dir=str(out_dir), post_quali=True, post_quali_chains=1) + + pre = pd.read_csv(out_dir / "calibration_podium.csv") + post = pd.read_csv(out_dir / "calibration_podium_postquali.csv") + assert list(pre.columns) == list(post.columns) == CALIBRATION_COLUMNS + assert len(post) == len(pre) > 0 + summary = json.loads((out_dir / "backtest_summary.json").read_text()) + calib = summary["calibration_podium"] + assert set(calib) == {"pre", "pre_paired", "postquali"} + for arm in calib.values(): + assert arm["n"] == len(pre) and 0.0 <= arm["ece"] <= 1.0 + assert sum(b["n"] for b in arm["bins"]) == arm["n"] + assert isinstance(summary["postquali_worse_rounds"], list)