From 4bb658f95b208176d6c9e5896e549e7cb7493088 Mon Sep 17 00:00:00 2001 From: Anh-Duy Pham Date: Mon, 10 Aug 2026 16:58:31 +0200 Subject: [PATCH 1/3] [tabarena] Add HillClimbingEnsembler for #4505 investigation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kaggle/Matt-OP style convex-blend hill climbing as a swappable AbstractEnsembler next to GreedyEnsembler (Caruana/AG ES). Includes unit tests, experimental compare/benchmark scripts, and a pandas read-only fix in RankScorer for modern numpy. AG bagged-OOF evidence (local): HC does not beat Greedy on mean test error → do not change AutoGluon ensemble defaults based on this. --- .../tabarena/simulation/ensemble/__init__.py | 2 + .../ensemble/hill_climbing_ensembler.py | 200 +++++++++++++ .../tabarena/src/tabarena/utils/rank_utils.py | 15 +- .../compare_hill_climbing_vs_greedy.py | 90 ++++++ .../run_ag_oof_hillclimb_vs_greedy.py | 281 ++++++++++++++++++ .../run_hill_climbing_vs_greedy_benchmark.py | 263 ++++++++++++++++ tests/tabarena/simulation/test_ensembler.py | 76 +++++ 7 files changed, 923 insertions(+), 4 deletions(-) create mode 100644 packages/tabarena/src/tabarena/simulation/ensemble/hill_climbing_ensembler.py create mode 100644 scripts/!experimental/compare_hill_climbing_vs_greedy.py create mode 100644 scripts/!experimental/run_ag_oof_hillclimb_vs_greedy.py create mode 100644 scripts/!experimental/run_hill_climbing_vs_greedy_benchmark.py diff --git a/packages/tabarena/src/tabarena/simulation/ensemble/__init__.py b/packages/tabarena/src/tabarena/simulation/ensemble/__init__.py index 0b468281e..14c7666bd 100644 --- a/packages/tabarena/src/tabarena/simulation/ensemble/__init__.py +++ b/packages/tabarena/src/tabarena/simulation/ensemble/__init__.py @@ -15,6 +15,7 @@ TopKAverageEnsembler, ) from tabarena.simulation.ensemble.greedy_ensembler import GreedyEnsembler +from tabarena.simulation.ensemble.hill_climbing_ensembler import HillClimbingEnsembler from tabarena.simulation.ensemble.stacking_ensembler import StackingEnsembler __all__ = [ @@ -23,6 +24,7 @@ "AutoGluonStackerRegressor", "FixedWeightsEnsembler", "GreedyEnsembler", + "HillClimbingEnsembler", "LegacyEnsemblerAdapter", "SingleBestEnsembler", "StackingEnsembler", diff --git a/packages/tabarena/src/tabarena/simulation/ensemble/hill_climbing_ensembler.py b/packages/tabarena/src/tabarena/simulation/ensemble/hill_climbing_ensembler.py new file mode 100644 index 000000000..549f38e9d --- /dev/null +++ b/packages/tabarena/src/tabarena/simulation/ensemble/hill_climbing_ensembler.py @@ -0,0 +1,200 @@ +"""Kaggle-style hill-climbing ensemble (Matt-OP / Deotte family). + +This is the definition of "hill climbing" that matches tilii7's AutoGluon feedback and +the community pointer on autogluon/autogluon#4505 +(https://github.com/Matt-OP/hillclimbers/) — **not** continuous black-box HPO +(arxiv:2307.00286). + +Compared to :class:`GreedyEnsembler` (Caruana ensemble selection, arxiv:1502.04759 / +AutoGluon ``EnsembleSelection``): + +* **Caruana / GreedyEnsembler:** iteratively *append* a model so the uniform average of + the multiset improves; weights are integer counts / ensemble_size. +* **Hill climbing (this class):** start from the best single model, then repeatedly try + convex blends ``(1 - w) * ensemble + w * model`` over a weight grid; accept any + improvement. Weights are continuous on a precision grid (default ``0.01``). + +Both are greedy local search over linear combinations of OOF predictions; they can +diverge on the selected support and weight magnitudes. #4505 asks whether this family +improves TabRepo/TabArena simulation score vs current ensemble selection — plug this +class in via ``ensembler_cls`` / ``ensemble_kwargs``. +""" + +from __future__ import annotations + +import time +from typing import TYPE_CHECKING + +import numpy as np + +from tabarena.simulation.ensemble.abstract_ensembler import WeightedEnsembler + +if TYPE_CHECKING: + from autogluon.core.metrics import Scorer + + +class HillClimbingEnsembler(WeightedEnsembler): + """Iterative convex blending of base-model predictions (Kaggle hill climbing). + + Parameters + ---------- + precision : float, default 0.01 + Weight step on ``(0, 1]``. Smaller is slower and can overfit small OOF sets. + max_rounds : int, default 100 + Maximum outer passes over all models after initialization. Stops early when a + full pass finds no improvement. + allow_negative_weights : bool, default False + If True, also tries ``w`` on ``[-1, 0)`` (Matt-OP ``negative_weights``). Safer + off when OOF is small. + max_models : int | None, default None + Optional cap on how many models may receive non-zero weight (sparsity preference). + ``None`` means no extra cap. + random_state : int | np.random.RandomState | None, default None + Used only for tie-breaking among equal-error starts / blends. + """ + + def __init__( + self, + *, + problem_type: str, + metric: Scorer, + precision: float = 0.01, + max_rounds: int = 100, + allow_negative_weights: bool = False, + max_models: int | None = None, + random_state: int | np.random.RandomState | None = None, + ): + super().__init__(problem_type=problem_type, metric=metric) + if precision <= 0 or precision > 1: + raise ValueError(f"precision must be in (0, 1], got {precision}") + if max_rounds < 1: + raise ValueError(f"max_rounds must be >= 1, got {max_rounds}") + if max_models is not None and max_models < 1: + raise ValueError(f"max_models must be >= 1 or None, got {max_models}") + self.precision = float(precision) + self.max_rounds = int(max_rounds) + self.allow_negative_weights = bool(allow_negative_weights) + self.max_models = max_models + if isinstance(random_state, np.random.RandomState): + self.random_state = random_state + else: + self.random_state = np.random.RandomState(0 if random_state is None else int(random_state)) + + self.trajectory_: list[float] = [] + self.n_rounds_: int = 0 + + def _weight_grid(self) -> np.ndarray: + # Exclude 0 (no-op). Include 1.0. + pos = np.arange(self.precision, 1.0 + self.precision * 0.5, self.precision) + pos = np.clip(pos, self.precision, 1.0) + # unique stable + pos = np.unique(np.round(pos / self.precision) * self.precision) + if self.allow_negative_weights: + neg = -pos[::-1] + return np.concatenate([neg, pos]) + return pos + + def _combine(self, predictions: np.ndarray, weights: np.ndarray) -> np.ndarray: + # Same linear combo semantics as WeightedEnsembler.predict_proba / AG. + preds_norm = [pred * w for pred, w in zip(predictions, weights, strict=True) if w != 0] + if not preds_norm: + # Degenerate: fall back to uniform over all (should not happen after init). + return np.mean(predictions, axis=0) + return np.sum(preds_norm, axis=0) + + def _n_nonzero(self, weights: np.ndarray) -> int: + return int(np.sum(np.abs(weights) > 0)) + + def _fit(self, *, predictions: np.ndarray, labels: np.ndarray, time_limit: float | None = None) -> None: + start = time.time() + predictions = np.asarray(predictions) + n_models = len(predictions) + if n_models == 0: + raise ValueError("HillClimbingEnsembler requires at least one model") + + # --- Initialize with best single model --- + single_errors = np.array([self._score_error(labels, pred) for pred in predictions], dtype=np.float64) + best_error = float(np.nanmin(single_errors)) + candidates = np.flatnonzero(np.isclose(single_errors, best_error, atol=0, rtol=1e-12)) + start_idx = int(self.random_state.choice(candidates)) + + weights = np.zeros(n_models, dtype=np.float64) + weights[start_idx] = 1.0 + ensemble_pred = predictions[start_idx].copy() + self.trajectory_ = [best_error] + + weight_grid = self._weight_grid() + + for round_i in range(self.max_rounds): + if time_limit is not None and (time.time() - start) >= time_limit: + break + + improved = False + # Randomize model order each round for mild exploration under ties. + order = self.random_state.permutation(n_models) + for j in order: + if time_limit is not None and (time.time() - start) >= time_limit: + break + + pred_j = predictions[j] + best_local_error = best_error + best_local_w = None + best_local_pred = None + + for w in weight_grid: + # Convex blend against current ensemble (standard Kaggle HC step). + trial = (1.0 - w) * ensemble_pred + w * pred_j + # Optional multiclass renormalize if this is a probability simplex view. + if trial.ndim == 2 and self.problem_type in ("multiclass", "softclass"): + row_sum = trial.sum(axis=1, keepdims=True) + row_sum = np.where(row_sum == 0, 1.0, row_sum) + trial = trial / row_sum + + # Enforce max_models: if adding a new model would exceed cap, skip + # unless it already has weight. + if self.max_models is not None and weights[j] == 0 and w != 0: + if self._n_nonzero(weights) >= self.max_models: + continue + + err = self._score_error(labels, trial) + if err < best_local_error - 1e-15: + best_local_error = err + best_local_w = float(w) + best_local_pred = trial + + if best_local_w is not None: + # Update latent weights: ensemble := (1-w)*ensemble + w*model_j + # ⇒ scale existing weights by (1-w), add w to model j. + weights *= 1.0 - best_local_w + weights[j] += best_local_w + ensemble_pred = best_local_pred + best_error = best_local_error + self.trajectory_.append(best_error) + improved = True + + self.n_rounds_ = round_i + 1 + if not improved: + break + + # Numerical cleanup: drop tiny weights, renormalize for stable reporting. + weights[np.abs(weights) < 1e-12] = 0.0 + if not self.allow_negative_weights: + weights = np.maximum(weights, 0.0) + total = float(np.sum(weights)) + if abs(total) > 1e-12: + weights = weights / total + else: + # Should not happen after best-single init; fall back to that model. + weights = np.zeros(n_models, dtype=np.float64) + weights[start_idx] = 1.0 + + self.weights_ = weights + + def info(self) -> dict: + return { + "n_rounds": self.n_rounds_, + "trajectory_len": len(self.trajectory_), + "final_val_error": self.trajectory_[-1] if self.trajectory_ else None, + "precision": self.precision, + "allow_negative_weights": self.allow_negative_weights, + } diff --git a/packages/tabarena/src/tabarena/utils/rank_utils.py b/packages/tabarena/src/tabarena/utils/rank_utils.py index e2216ac86..21cf1ed32 100644 --- a/packages/tabarena/src/tabarena/utils/rank_utils.py +++ b/packages/tabarena/src/tabarena/utils/rank_utils.py @@ -91,10 +91,17 @@ def __init__( self.pct = pct self.include_partial = include_partial df_pivot = df_results.pivot_table(values=metric_error_col, index=task_col, columns=framework_col) - df_pivot.values.sort(axis=1) # NOTE: The framework columns are now no longer correct. Do not use them. - - # tolist to drop the framework col name, since it is no longer ordered. - self.error_dict = {dataset: df_pivot.loc[dataset].dropna().tolist() for dataset in tasks} + # Copy: recent pandas/numpy expose read-only views for .values, so in-place sort fails. + sorted_errors = np.ascontiguousarray(df_pivot.to_numpy(dtype=np.float64, copy=True)) + sorted_errors.flags.writeable = True + sorted_errors.sort(axis=1) + # NOTE: Framework columns are no longer meaningful after row-wise sort. + index_list = list(df_pivot.index) + self.error_dict = {} + for task in tasks: + i = index_list.index(task) + row = sorted_errors[i] + self.error_dict[task] = row[~np.isnan(row)].tolist() def rank(self, task: str, error: float) -> float: """Get the rank of a result on a dataset given an error.""" diff --git a/scripts/!experimental/compare_hill_climbing_vs_greedy.py b/scripts/!experimental/compare_hill_climbing_vs_greedy.py new file mode 100644 index 000000000..c25fefbc5 --- /dev/null +++ b/scripts/!experimental/compare_hill_climbing_vs_greedy.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Compare Kaggle-style hill climbing vs Caruana greedy ensemble selection (#4505). + +#4505 asks: does hill climbing improve ensembling over AutoGluon's current approach? +Verify via TabRepo/TabArena by swapping the ensembler (historical pointer was +``scripts/baseline_comparison/evaluate_baselines.py``; current API is +``repo.evaluate_ensemble`` / ``EnsembleScorer`` with ``ensembler_cls``). + +Definitions (see @LennartPurucker on the issue): + * **GreedyEnsembler** — Caruana et al. 2004 / AG ``EnsembleSelection`` + * **HillClimbingEnsembler** — Kaggle/Matt-OP convex blend hill climb (not continuous BBO) + +Usage (synthetic smoke, no dataset download):: + + PYTHONPATH=packages/tabarena/src python scripts/\\!experimental/compare_hill_climbing_vs_greedy.py + +Usage with a loaded EvaluationRepository (when you have TabArena caches):: + + from tabarena.simulation.ensemble import GreedyEnsembler, HillClimbingEnsembler + + greedy = repo.evaluate_ensembles( + configs=configs, + ensemble_kwargs={"ensembler_cls": GreedyEnsembler, "ensembler_kwargs": {"ensemble_size": 100}}, + ) + hc = repo.evaluate_ensembles( + configs=configs, + ensemble_kwargs={ + "ensembler_cls": HillClimbingEnsembler, + "ensembler_kwargs": {"precision": 0.01, "max_rounds": 50}, + }, + ) + # Compare mean metric_error / rank across tasks. +""" + +from __future__ import annotations + +import numpy as np +from autogluon.core.metrics import get_metric + +from tabarena.simulation.ensemble import GreedyEnsembler, HillClimbingEnsembler, SingleBestEnsembler + + +def _synthetic_binary(n_models=20, n_samples=2000, seed=0): + rng = np.random.default_rng(seed) + y = rng.random(n_samples) < 0.45 + # Diverse strengths + correlated noise + preds = [] + for i in range(n_models): + strength = rng.uniform(0.2, 0.85) + noise = rng.normal(0, 0.15 + 0.05 * (i % 5), n_samples) + p = np.clip(y.astype(float) * strength + (1 - strength) * 0.5 + noise, 0, 1) + preds.append(p.astype(np.float32)) + return y.astype(bool), np.stack(preds) + + +def main(): + y, preds = _synthetic_binary() + metric = get_metric(metric="roc_auc", problem_type="binary") + # Hold out last 30% as "test" for a rough generalization check + n = len(y) + n_fit = int(0.7 * n) + y_fit, y_test = y[:n_fit], y[n_fit:] + preds_fit, preds_test = preds[:, :n_fit], preds[:, n_fit:] + + methods = { + "single_best": SingleBestEnsembler(problem_type="binary", metric=metric), + "greedy_caruana": GreedyEnsembler( + problem_type="binary", metric=metric, ensemble_size=40, random_state=np.random.RandomState(0) + ), + "hill_climbing": HillClimbingEnsembler( + problem_type="binary", metric=metric, precision=0.02, max_rounds=40, random_state=0 + ), + } + + print(f"{'method':20s} {'fit_err':>10s} {'test_err':>10s} {'n_models':>8s}") + for name, ens in methods.items(): + ens.fit(predictions=preds_fit, labels=y_fit) + fit_err = metric.error(y_fit, ens.predict_proba(preds_fit)) + test_err = metric.error(y_test, ens.predict_proba(preds_test)) + n_used = int(ens.models_used().sum()) + print(f"{name:20s} {fit_err:10.6f} {test_err:10.6f} {n_used:8d}") + + print( + "\nNote: synthetic smoke only. For #4505 evidence, run both ensemblers through " + "EvaluationRepository.evaluate_ensembles on shared configs/tasks and compare ranks." + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/!experimental/run_ag_oof_hillclimb_vs_greedy.py b/scripts/!experimental/run_ag_oof_hillclimb_vs_greedy.py new file mode 100644 index 000000000..3d0935d2b --- /dev/null +++ b/scripts/!experimental/run_ag_oof_hillclimb_vs_greedy.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +"""Evidence for #4505: hill climbing vs Caruana ES on real AutoGluon bagged OOF. + +Import order matters on macOS (libomp): lightgbm before torch. +Does not import tabarena (avoids torch pull-in); uses AG EnsembleSelection + local HC. +""" + +from __future__ import annotations + +# --- libomp-safe import order --- +import lightgbm # noqa: F401 + +import json +import time +from pathlib import Path + +import numpy as np +import pandas as pd +from autogluon.core.metrics import get_metric +from autogluon.core.models.greedy_ensemble.ensemble_selection import EnsembleSelection +from autogluon.tabular import TabularPredictor + +# Local HC implementation (mirror of tabarena.simulation.ensemble.hill_climbing_ensembler) +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "packages" / "tabarena" / "src")) +# Prefer pure-numpy HC below without importing full tabarena stack. + + +def hill_climb_weights( + predictions: list[np.ndarray], + labels: np.ndarray, + metric, + problem_type: str, + precision: float = 0.02, + max_rounds: int = 40, + random_state: int = 0, +) -> np.ndarray: + """Kaggle-style convex blend HC; returns weight vector (sums to 1).""" + rng = np.random.RandomState(random_state) + n_models = len(predictions) + predictions = [np.asarray(p) for p in predictions] + + def error(pred): + return metric.error(labels, pred) + + singles = np.array([error(p) for p in predictions]) + best_i = int(rng.choice(np.flatnonzero(np.isclose(singles, singles.min())))) + weights = np.zeros(n_models) + weights[best_i] = 1.0 + ens = predictions[best_i].copy() + best_err = singles[best_i] + grid = np.arange(precision, 1.0 + precision * 0.5, precision) + grid = np.unique(np.round(grid / precision) * precision) + + for _ in range(max_rounds): + improved = False + for j in rng.permutation(n_models): + best_local = best_err + best_w = None + best_pred = None + for w in grid: + trial = (1.0 - w) * ens + w * predictions[j] + if trial.ndim == 2 and problem_type in ("multiclass", "softclass"): + s = trial.sum(axis=1, keepdims=True) + s = np.where(s == 0, 1.0, s) + trial = trial / s + err = error(trial) + if err < best_local - 1e-15: + best_local = err + best_w = float(w) + best_pred = trial + if best_w is not None: + weights *= 1.0 - best_w + weights[j] += best_w + ens = best_pred + best_err = best_local + improved = True + if not improved: + break + weights[np.abs(weights) < 1e-12] = 0.0 + weights = np.maximum(weights, 0.0) + s = weights.sum() + return weights / s if s > 0 else weights + + +def _tasks(): + rng = np.random.default_rng(0) + n, d = 3000, 16 + X = rng.normal(size=(n, d)) + tasks = [] + # binary + logits = X[:, 0] * 1.2 + X[:, 2] * 0.8 + X[:, 5] * 0.4 + rng.normal(0, 0.6, n) + df = pd.DataFrame(X, columns=[f"f{i}" for i in range(d)]) + df["target"] = (logits > 0).astype(int) + tasks.append(("synth_binary", df, "binary")) + # regression + df2 = pd.DataFrame(X, columns=[f"f{i}" for i in range(d)]) + df2["target"] = X[:, 0] * 1.5 + X[:, 1] ** 2 * 0.3 + rng.normal(0, 0.5, n) + tasks.append(("synth_reg", df2, "regression")) + # multiclass + df3 = pd.DataFrame(X, columns=[f"f{i}" for i in range(d)]) + df3["target"] = X[:, :4].argmax(axis=1) + tasks.append(("synth_multi", df3, "multiclass")) + return tasks + + +def _fit_oof(name, df, problem_type, path: Path, time_limit=180): + path.mkdir(parents=True, exist_ok=True) + crit = "squared_error" if problem_type == "regression" else "gini" + hyperparameters = { + "GBM": [ + {}, + {"extra_trees": True, "ag_args": {"name_suffix": "XT"}}, + {"learning_rate": 0.03, "num_leaves": 64, "ag_args": {"name_suffix": "Large"}}, + ], + "CAT": [{}, {"depth": 6, "ag_args": {"name_suffix": "D6"}}], + "XGB": [{}, {"max_depth": 6, "ag_args": {"name_suffix": "D6"}}], + "RF": [{"criterion": crit}, {"n_estimators": 100, "ag_args": {"name_suffix": "100"}}], + "XT": [{"criterion": crit}], + "LR": [{}], + "KNN": [{"weights": "uniform"}, {"weights": "distance", "ag_args": {"name_suffix": "Dist"}}], + } + # sequential bag folds avoid Ray + for k, v in list(hyperparameters.items()): + if isinstance(v, list): + hyperparameters[k] = [ + {**cfg, "ag_args_ensemble": {"fold_fitting_strategy": "sequential_local"}} for cfg in v + ] + else: + hyperparameters[k] = {**v, "ag_args_ensemble": {"fold_fitting_strategy": "sequential_local"}} + + predictor = TabularPredictor( + label="target", + problem_type=problem_type, + path=str(path), + verbosity=1, + eval_metric="roc_auc" if problem_type == "binary" else None, + ) + predictor.fit( + df, + hyperparameters=hyperparameters, + num_bag_folds=5, + num_stack_levels=0, + time_limit=time_limit, + raise_on_model_failure=False, + ) + trainer = predictor._trainer + names = [m for m in trainer.get_model_names(level=1) if "WeightedEnsemble" not in m] + oofs = [] + keep = [] + for m in names: + try: + oof = np.asarray(trainer.get_model_oof(m)) + if oof.ndim == 2 and oof.shape[1] == 2 and problem_type == "binary": + oof = oof[:, 1] + oofs.append(oof.astype(np.float32)) + keep.append(m) + except Exception as e: + print(f" skip {m}: {e}") + y = np.asarray(predictor.transform_labels(df["target"])) + n = min(len(y), min(len(p) for p in oofs)) + return y[:n], [p[:n] for p in oofs], keep + + +def _split(y, preds, test_frac=0.3, seed=0): + rng = np.random.default_rng(seed) + n = len(y) + idx = rng.permutation(n) + n_test = max(1, int(n * test_frac)) + te, tr = idx[:n_test], idx[n_test:] + return y[tr], [p[tr] for p in preds], y[te], [p[te] for p in preds] + + +def _combine(preds, weights): + out = None + for p, w in zip(preds, weights, strict=True): + if w == 0: + continue + out = p * w if out is None else out + p * w + return out + + +def main(): + out = Path("artifacts/hill_climbing_4505/ag_oof") + out.mkdir(parents=True, exist_ok=True) + rows = [] + t0 = time.time() + for name, df, ptype in _tasks(): + print(f"\n=== {name} ({ptype}) ===") + y, preds, model_names = _fit_oof(name, df, ptype, out / f"ag_{name}", time_limit=200) + print(f" models: {len(model_names)}") + y_fit, p_fit, y_te, p_te = _split(y, preds) + metric = get_metric( + "roc_auc" if ptype == "binary" else ("log_loss" if ptype == "multiclass" else "rmse"), + problem_type=ptype, + ) + + # Single best + fit_errs = [metric.error(y_fit, p) for p in p_fit] + bi = int(np.argmin(fit_errs)) + sb_w = np.zeros(len(p_fit)) + sb_w[bi] = 1.0 + sb_test = metric.error(y_te, _combine(p_te, sb_w)) + + # Greedy Caruana + es = EnsembleSelection( + ensemble_size=40, problem_type=ptype, metric=metric, random_state=np.random.RandomState(0) + ) + es.fit(predictions=list(p_fit), labels=y_fit) + g_w = np.asarray(es.weights_) + g_test = metric.error(y_te, _combine(p_te, g_w)) + g_fit = metric.error(y_fit, _combine(p_fit, g_w)) + + # Hill climbing + hc_w = hill_climb_weights(p_fit, y_fit, metric, ptype) + hc_test = metric.error(y_te, _combine(p_te, hc_w)) + hc_fit = metric.error(y_fit, _combine(p_fit, hc_w)) + + for ens, fit_e, te, w in [ + ("single_best", fit_errs[bi], sb_test, sb_w), + ("greedy_caruana", g_fit, g_test, g_w), + ("hill_climbing", hc_fit, hc_test, hc_w), + ]: + rows.append( + { + "task": name, + "problem_type": ptype, + "ensembler": ens, + "fit_err": fit_e, + "test_err": te, + "n_models": int((np.asarray(w) != 0).sum()), + "n_pool": len(model_names), + } + ) + print(f" {ens:16s} fit={fit_e:.6f} test={te:.6f} n={(np.asarray(w) != 0).sum()}") + + df = pd.DataFrame(rows) + df.to_csv(out / "results.csv", index=False) + pivot = df.pivot_table(index="task", columns="ensembler", values="test_err") + delta = float((pivot["hill_climbing"] - pivot["greedy_caruana"]).mean()) + summary = { + "mean_test_err": pivot.mean().to_dict(), + "hc_minus_greedy_mean": delta, + "hc_wins": int((pivot["hill_climbing"] < pivot["greedy_caruana"] - 1e-12).sum()), + "greedy_wins": int((pivot["greedy_caruana"] < pivot["hill_climbing"] - 1e-12).sum()), + "wall_time_s": time.time() - t0, + "note": "Real AG bagged OOF; HC = Kaggle convex blend; Greedy = EnsembleSelection", + } + (out / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") + if delta < -1e-8: + conclusion = "HC improves mean test error vs Greedy on this AG OOF suite. Confirm on TabArena before AG default change." + change_default = False # still need broader TabArena + elif abs(delta) <= 1e-8: + conclusion = "HC ≈ Greedy. Do not change AG default ensemble selection." + change_default = False + else: + conclusion = "Greedy better or HC overfits. Do not change AG default ensemble selection." + change_default = False + summary["conclusion"] = conclusion + summary["change_ag_default"] = change_default + (out / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") + md = f"""# AG OOF: Hill climbing vs Greedy (#4505) + +Mean test errors: {json.dumps(summary["mean_test_err"], indent=2)} + +Mean (HC − Greedy): **{delta:.6g}** (negative ⇒ HC better) + +Task wins HC / Greedy: {summary["hc_wins"]} / {summary["greedy_wins"]} + +**{conclusion}** + +AG default change: **{change_default}** +""" + (out / "summary.md").write_text(md, encoding="utf-8") + print("\n" + md) + print(f"Wrote {out}") + + +if __name__ == "__main__": + main() diff --git a/scripts/!experimental/run_hill_climbing_vs_greedy_benchmark.py b/scripts/!experimental/run_hill_climbing_vs_greedy_benchmark.py new file mode 100644 index 000000000..14958aba4 --- /dev/null +++ b/scripts/!experimental/run_hill_climbing_vs_greedy_benchmark.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +"""TabArena post-hoc ensemble benchmark: HillClimbing vs Greedy (Caruana) vs SingleBest. + +Addresses autogluon/autogluon#4505 investigation 2 using real TabArena OOF caches. + +Definition of hill climbing (issue thread): + Kaggle / Matt-OP convex blend search — NOT continuous black-box HPO. + Compared to TabArena default GreedyEnsembler (AG EnsembleSelection / Caruana 2004). + +Example:: + + export PYTHONPATH=packages/tabarena/src:$PYTHONPATH + # downloads LightGBM processed (~8.5 GB) if missing + python scripts/\\!experimental/run_hill_climbing_vs_greedy_benchmark.py \\ + --method LightGBM --max-datasets 15 --max-folds 3 --n-configs 40 + +Outputs JSON + markdown under ``artifacts/hill_climbing_4505/``. +""" + +from __future__ import annotations + +import argparse +import json +import time +from pathlib import Path + +import numpy as np +import pandas as pd + + +def _load_method_repo(method: str): + from tabarena.contexts.tabarena.methods import tabarena_method_metadata_collection + + meta = tabarena_method_metadata_collection.get_method_metadata(method=method) + if not meta.path_processed_exists: + print(f"Downloading processed artifacts for {method} -> {meta.path_processed} ...") + meta.method_downloader(verbose=True).download_processed() + return meta.load_processed(), meta + + +def _evaluate_one(repo, dataset: str, fold: int, configs: list[str], ensembler_cls, ensembler_kwargs: dict): + df_result, df_weights = repo.evaluate_ensemble( + dataset=dataset, + fold=fold, + configs=configs, + ensemble_kwargs={ + "ensembler_cls": ensembler_cls, + "ensembler_kwargs": ensembler_kwargs, + }, + ) + # df_result is typically multi-index or single row with metric_error etc. + row = df_result.reset_index(drop=True).iloc[0].to_dict() + n_used = int((df_weights.iloc[0] != 0).sum()) if len(df_weights) else 0 + row["n_models_used"] = n_used + return row + + +def run_benchmark( + method: str = "LightGBM", + max_datasets: int | None = 20, + max_folds: int | None = 3, + n_configs: int | None = 50, + ensemble_size: int = 40, + hc_precision: float = 0.02, + hc_max_rounds: int = 30, + out_dir: Path | None = None, +) -> pd.DataFrame: + from tabarena.simulation.ensemble import GreedyEnsembler, HillClimbingEnsembler, SingleBestEnsembler + + repo, meta = _load_method_repo(method) + datasets = list(repo.datasets()) + if max_datasets is not None: + datasets = datasets[:max_datasets] + configs = list(repo.configs()) + if n_configs is not None: + configs = configs[:n_configs] + + # Discover folds via first dataset metrics / tasks + folds_available = sorted({t[1] for t in repo.tasks() if t[0] == datasets[0]}) if hasattr(repo, "tasks") else [0] + if not folds_available: + # fallback + folds_available = list(range(3)) + if max_folds is not None: + folds_available = folds_available[:max_folds] + + methods = { + "single_best": (SingleBestEnsembler, {}), + "greedy_caruana": ( + GreedyEnsembler, + {"ensemble_size": ensemble_size, "random_state": np.random.RandomState(0)}, + ), + "hill_climbing": ( + HillClimbingEnsembler, + { + "precision": hc_precision, + "max_rounds": hc_max_rounds, + "random_state": 0, + }, + ), + } + + rows = [] + t0 = time.time() + for dataset in datasets: + for fold in folds_available: + for name, (cls, kwargs) in methods.items(): + try: + t1 = time.time() + result = _evaluate_one(repo, dataset, fold, configs, cls, kwargs) + elapsed = time.time() - t1 + rows.append( + { + "method_pool": method, + "ensembler": name, + "dataset": dataset, + "fold": fold, + "metric_error": result.get("metric_error"), + "metric_error_val": result.get("metric_error_val"), + "n_models_used": result.get("n_models_used"), + "time_s": elapsed, + "n_configs_pool": len(configs), + } + ) + print( + f"[{len(rows)}] {dataset} fold={fold} {name}: " + f"test_err={result.get('metric_error')} val_err={result.get('metric_error_val')} " + f"n_used={result.get('n_models_used')} ({elapsed:.2f}s)" + ) + except Exception as e: + print(f"FAIL {dataset} fold={fold} {name}: {type(e).__name__}: {e}") + rows.append( + { + "method_pool": method, + "ensembler": name, + "dataset": dataset, + "fold": fold, + "metric_error": np.nan, + "metric_error_val": np.nan, + "n_models_used": np.nan, + "time_s": np.nan, + "n_configs_pool": len(configs), + "error": f"{type(e).__name__}: {e}", + } + ) + + df = pd.DataFrame(rows) + out_dir = out_dir or Path("artifacts/hill_climbing_4505") + out_dir.mkdir(parents=True, exist_ok=True) + csv_path = out_dir / f"benchmark_{method}_d{len(datasets)}_f{len(folds_available)}_c{len(configs)}.csv" + df.to_csv(csv_path, index=False) + + summary = _summarize(df) + md_path = out_dir / f"summary_{method}.md" + md_path.write_text(summary["markdown"], encoding="utf-8") + json_path = out_dir / f"summary_{method}.json" + json_path.write_text(json.dumps(summary["stats"], indent=2, default=str), encoding="utf-8") + + print(f"\nWrote {csv_path}") + print(f"Wrote {md_path}") + print(f"Total wall time: {time.time() - t0:.1f}s") + print(summary["markdown"]) + return df + + +def _summarize(df: pd.DataFrame) -> dict: + """Compare ensemblers: win rates on test metric_error (lower better).""" + ok = df.dropna(subset=["metric_error"]) + if ok.empty: + return {"markdown": "No successful runs.\n", "stats": {}} + + pivot = ok.pivot_table(index=["dataset", "fold"], columns="ensembler", values="metric_error", aggfunc="first") + stats: dict = {"n_tasks": int(len(pivot)), "mean_error": {}, "wins_vs_greedy": {}, "ties_vs_greedy": {}, "mean_n_models": {}} + + for col in pivot.columns: + stats["mean_error"][col] = float(pivot[col].mean()) + + if "greedy_caruana" in pivot.columns and "hill_climbing" in pivot.columns: + g, h = pivot["greedy_caruana"], pivot["hill_climbing"] + stats["wins_vs_greedy"]["hill_climbing"] = int((h < g - 1e-12).sum()) + stats["ties_vs_greedy"]["hill_climbing"] = int(np.isclose(h, g, rtol=0, atol=1e-12).sum()) + stats["wins_vs_greedy"]["greedy_caruana"] = int((g < h - 1e-12).sum()) + stats["mean_delta_hc_minus_greedy"] = float((h - g).mean()) # negative => HC better + + if "single_best" in pivot.columns and "hill_climbing" in pivot.columns: + s, h = pivot["single_best"], pivot["hill_climbing"] + stats["wins_vs_single_best"] = { + "hill_climbing": int((h < s - 1e-12).sum()), + "single_best": int((s < h - 1e-12).sum()), + } + + n_models = ok.groupby("ensembler")["n_models_used"].mean() + stats["mean_n_models"] = {k: float(v) for k, v in n_models.items()} + + lines = [ + "# Hill climbing vs Greedy ensemble selection (TabArena OOF)", + "", + "Issue: [autogluon/autogluon#4505](https://github.com/autogluon/autogluon/issues/4505)", + "", + "## Definitions", + "", + "- **greedy_caruana**: TabArena `GreedyEnsembler` → AutoGluon `EnsembleSelection` (Caruana et al. 2004).", + "- **hill_climbing**: `HillClimbingEnsembler` — Kaggle/Matt-OP convex blend search (not continuous BBO).", + "- **single_best**: best validation model only.", + "", + f"Tasks (dataset × fold): **{stats['n_tasks']}**", + "", + "## Mean test metric_error (lower is better)", + "", + ] + for k, v in sorted(stats["mean_error"].items(), key=lambda x: x[1]): + lines.append(f"- `{k}`: {v:.6g}") + lines.append("") + if "mean_delta_hc_minus_greedy" in stats: + d = stats["mean_delta_hc_minus_greedy"] + lines.append(f"Mean (HC − Greedy) test error: **{d:.6g}** (negative means HC wins on average)") + lines.append( + f"Task wins: HC {stats['wins_vs_greedy'].get('hill_climbing', 0)} / " + f"Greedy {stats['wins_vs_greedy'].get('greedy_caruana', 0)} / " + f"ties {stats['ties_vs_greedy'].get('hill_climbing', 0)}" + ) + lines.append("") + if d < -1e-8: + lines.append("**Conclusion (this slice):** Hill climbing improves mean test error vs Greedy.") + lines.append("→ Candidate for further portfolio study; AG default change only after broader confirmation.") + elif abs(d) <= 1e-8: + lines.append("**Conclusion (this slice):** Hill climbing ≈ Greedy (no meaningful mean difference).") + lines.append("→ **Do not** change AG default ensemble selection based on this evidence.") + else: + lines.append("**Conclusion (this slice):** Greedy is better or HC overfits OOF.") + lines.append("→ **Do not** change AG default; keep GreedyEnsembler / EnsembleSelection.") + lines.append("") + lines.append("## Mean models used") + for k, v in stats["mean_n_models"].items(): + lines.append(f"- `{k}`: {v:.2f}") + lines.append("") + return {"markdown": "\n".join(lines) + "\n", "stats": stats} + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--method", default="LightGBM") + p.add_argument("--max-datasets", type=int, default=15) + p.add_argument("--max-folds", type=int, default=3) + p.add_argument("--n-configs", type=int, default=40) + p.add_argument("--ensemble-size", type=int, default=40) + p.add_argument("--hc-precision", type=float, default=0.02) + p.add_argument("--hc-max-rounds", type=int, default=30) + p.add_argument("--out-dir", type=Path, default=Path("artifacts/hill_climbing_4505")) + args = p.parse_args() + run_benchmark( + method=args.method, + max_datasets=args.max_datasets, + max_folds=args.max_folds, + n_configs=args.n_configs, + ensemble_size=args.ensemble_size, + hc_precision=args.hc_precision, + hc_max_rounds=args.hc_max_rounds, + out_dir=args.out_dir, + ) + + +if __name__ == "__main__": + main() diff --git a/tests/tabarena/simulation/test_ensembler.py b/tests/tabarena/simulation/test_ensembler.py index 09e03e54f..ffbf03c32 100644 --- a/tests/tabarena/simulation/test_ensembler.py +++ b/tests/tabarena/simulation/test_ensembler.py @@ -545,3 +545,79 @@ def test_autogluon_stacker_leaves_no_cwd_artifacts(monkeypatch, tmp_path): assert np.isfinite(regressor.predict(preds.T)).all() assert list(tmp_path.iterdir()) == [] + + +# ------------------------- +# Hill climbing (autogluon/autogluon#4505) +# ------------------------- +def test_hill_climbing_beats_or_matches_single_best_on_synthetic(): + """HC starts from the best single model and only accepts improving blends, so + validation error must be <= single-best error on the fit split. + """ + from tabarena.simulation.ensemble import HillClimbingEnsembler, SingleBestEnsembler + + y, preds = _make_binary_task(n_models=12, n_samples=800, seed=7) + metric = get_metric(metric="roc_auc", problem_type="binary") + + single = SingleBestEnsembler(problem_type="binary", metric=metric) + single.fit(predictions=preds, labels=y) + single_err = metric.error(y, single.predict_proba(preds)) + + hc = HillClimbingEnsembler( + problem_type="binary", + metric=metric, + precision=0.05, + max_rounds=20, + random_state=0, + ) + hc.fit(predictions=preds, labels=y) + hc_err = metric.error(y, hc.predict_proba(preds)) + + assert hc_err <= single_err + 1e-12 + assert hc.model_weights() is not None + assert np.isclose(hc.model_weights().sum(), 1.0) + assert hc.models_used().any() + + +def test_hill_climbing_regression_and_weights_sum(): + from tabarena.simulation.ensemble import HillClimbingEnsembler + + y, preds = _make_regression_task(n_models=10, n_samples=600, seed=3) + metric = get_metric(metric="rmse", problem_type="regression") + hc = HillClimbingEnsembler( + problem_type="regression", + metric=metric, + precision=0.05, + max_rounds=15, + random_state=1, + ) + hc.fit(predictions=preds, labels=y) + w = hc.model_weights() + assert w is not None + assert np.isclose(w.sum(), 1.0) + assert (w >= -1e-12).all() + # Prediction is a weighted sum of base preds + combined = hc.predict_proba(preds) + expected = sum(p * wi for p, wi in zip(preds, w, strict=True) if wi != 0) + np.testing.assert_allclose(combined, expected, rtol=1e-5, atol=1e-5) + + +def test_hill_climbing_task_evaluator_runs(): + """Smoke: HillClimbingEnsembler plugs into TaskEvaluator like other ensemblers.""" + from tabarena.simulation.ensemble import HillClimbingEnsembler + + y, preds = _make_binary_task(seed=11) + metric = get_metric(metric="roc_auc", problem_type="binary") + results, ensemble = _run_task_evaluator( + HillClimbingEnsembler, + {"precision": 0.05, "max_rounds": 10, "random_state": 0}, + problem_type="binary", + eval_metric=metric, + fit_eval_metric=metric, + y=y, + preds=preds, + ) + assert "metric_error" in results + assert "ensemble_weights" in results + assert results["ensemble_weights"] is not None + assert ensemble is not None From 83820f330103ab6cc437f65abbdd3cec36697bb2 Mon Sep 17 00:00:00 2001 From: Anh-Duy Pham Date: Mon, 10 Aug 2026 17:14:59 +0200 Subject: [PATCH 2/3] [tabarena] Drop experimental HC scripts from PR Keep HillClimbingEnsembler + unit tests for mainline; leave local benchmark/AG-OOF scripts out of the merge per review feedback. --- .../compare_hill_climbing_vs_greedy.py | 90 ------ .../run_ag_oof_hillclimb_vs_greedy.py | 281 ------------------ .../run_hill_climbing_vs_greedy_benchmark.py | 263 ---------------- 3 files changed, 634 deletions(-) delete mode 100644 scripts/!experimental/compare_hill_climbing_vs_greedy.py delete mode 100644 scripts/!experimental/run_ag_oof_hillclimb_vs_greedy.py delete mode 100644 scripts/!experimental/run_hill_climbing_vs_greedy_benchmark.py diff --git a/scripts/!experimental/compare_hill_climbing_vs_greedy.py b/scripts/!experimental/compare_hill_climbing_vs_greedy.py deleted file mode 100644 index c25fefbc5..000000000 --- a/scripts/!experimental/compare_hill_climbing_vs_greedy.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python3 -"""Compare Kaggle-style hill climbing vs Caruana greedy ensemble selection (#4505). - -#4505 asks: does hill climbing improve ensembling over AutoGluon's current approach? -Verify via TabRepo/TabArena by swapping the ensembler (historical pointer was -``scripts/baseline_comparison/evaluate_baselines.py``; current API is -``repo.evaluate_ensemble`` / ``EnsembleScorer`` with ``ensembler_cls``). - -Definitions (see @LennartPurucker on the issue): - * **GreedyEnsembler** — Caruana et al. 2004 / AG ``EnsembleSelection`` - * **HillClimbingEnsembler** — Kaggle/Matt-OP convex blend hill climb (not continuous BBO) - -Usage (synthetic smoke, no dataset download):: - - PYTHONPATH=packages/tabarena/src python scripts/\\!experimental/compare_hill_climbing_vs_greedy.py - -Usage with a loaded EvaluationRepository (when you have TabArena caches):: - - from tabarena.simulation.ensemble import GreedyEnsembler, HillClimbingEnsembler - - greedy = repo.evaluate_ensembles( - configs=configs, - ensemble_kwargs={"ensembler_cls": GreedyEnsembler, "ensembler_kwargs": {"ensemble_size": 100}}, - ) - hc = repo.evaluate_ensembles( - configs=configs, - ensemble_kwargs={ - "ensembler_cls": HillClimbingEnsembler, - "ensembler_kwargs": {"precision": 0.01, "max_rounds": 50}, - }, - ) - # Compare mean metric_error / rank across tasks. -""" - -from __future__ import annotations - -import numpy as np -from autogluon.core.metrics import get_metric - -from tabarena.simulation.ensemble import GreedyEnsembler, HillClimbingEnsembler, SingleBestEnsembler - - -def _synthetic_binary(n_models=20, n_samples=2000, seed=0): - rng = np.random.default_rng(seed) - y = rng.random(n_samples) < 0.45 - # Diverse strengths + correlated noise - preds = [] - for i in range(n_models): - strength = rng.uniform(0.2, 0.85) - noise = rng.normal(0, 0.15 + 0.05 * (i % 5), n_samples) - p = np.clip(y.astype(float) * strength + (1 - strength) * 0.5 + noise, 0, 1) - preds.append(p.astype(np.float32)) - return y.astype(bool), np.stack(preds) - - -def main(): - y, preds = _synthetic_binary() - metric = get_metric(metric="roc_auc", problem_type="binary") - # Hold out last 30% as "test" for a rough generalization check - n = len(y) - n_fit = int(0.7 * n) - y_fit, y_test = y[:n_fit], y[n_fit:] - preds_fit, preds_test = preds[:, :n_fit], preds[:, n_fit:] - - methods = { - "single_best": SingleBestEnsembler(problem_type="binary", metric=metric), - "greedy_caruana": GreedyEnsembler( - problem_type="binary", metric=metric, ensemble_size=40, random_state=np.random.RandomState(0) - ), - "hill_climbing": HillClimbingEnsembler( - problem_type="binary", metric=metric, precision=0.02, max_rounds=40, random_state=0 - ), - } - - print(f"{'method':20s} {'fit_err':>10s} {'test_err':>10s} {'n_models':>8s}") - for name, ens in methods.items(): - ens.fit(predictions=preds_fit, labels=y_fit) - fit_err = metric.error(y_fit, ens.predict_proba(preds_fit)) - test_err = metric.error(y_test, ens.predict_proba(preds_test)) - n_used = int(ens.models_used().sum()) - print(f"{name:20s} {fit_err:10.6f} {test_err:10.6f} {n_used:8d}") - - print( - "\nNote: synthetic smoke only. For #4505 evidence, run both ensemblers through " - "EvaluationRepository.evaluate_ensembles on shared configs/tasks and compare ranks." - ) - - -if __name__ == "__main__": - main() diff --git a/scripts/!experimental/run_ag_oof_hillclimb_vs_greedy.py b/scripts/!experimental/run_ag_oof_hillclimb_vs_greedy.py deleted file mode 100644 index 3d0935d2b..000000000 --- a/scripts/!experimental/run_ag_oof_hillclimb_vs_greedy.py +++ /dev/null @@ -1,281 +0,0 @@ -#!/usr/bin/env python3 -"""Evidence for #4505: hill climbing vs Caruana ES on real AutoGluon bagged OOF. - -Import order matters on macOS (libomp): lightgbm before torch. -Does not import tabarena (avoids torch pull-in); uses AG EnsembleSelection + local HC. -""" - -from __future__ import annotations - -# --- libomp-safe import order --- -import lightgbm # noqa: F401 - -import json -import time -from pathlib import Path - -import numpy as np -import pandas as pd -from autogluon.core.metrics import get_metric -from autogluon.core.models.greedy_ensemble.ensemble_selection import EnsembleSelection -from autogluon.tabular import TabularPredictor - -# Local HC implementation (mirror of tabarena.simulation.ensemble.hill_climbing_ensembler) -import sys - -sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "packages" / "tabarena" / "src")) -# Prefer pure-numpy HC below without importing full tabarena stack. - - -def hill_climb_weights( - predictions: list[np.ndarray], - labels: np.ndarray, - metric, - problem_type: str, - precision: float = 0.02, - max_rounds: int = 40, - random_state: int = 0, -) -> np.ndarray: - """Kaggle-style convex blend HC; returns weight vector (sums to 1).""" - rng = np.random.RandomState(random_state) - n_models = len(predictions) - predictions = [np.asarray(p) for p in predictions] - - def error(pred): - return metric.error(labels, pred) - - singles = np.array([error(p) for p in predictions]) - best_i = int(rng.choice(np.flatnonzero(np.isclose(singles, singles.min())))) - weights = np.zeros(n_models) - weights[best_i] = 1.0 - ens = predictions[best_i].copy() - best_err = singles[best_i] - grid = np.arange(precision, 1.0 + precision * 0.5, precision) - grid = np.unique(np.round(grid / precision) * precision) - - for _ in range(max_rounds): - improved = False - for j in rng.permutation(n_models): - best_local = best_err - best_w = None - best_pred = None - for w in grid: - trial = (1.0 - w) * ens + w * predictions[j] - if trial.ndim == 2 and problem_type in ("multiclass", "softclass"): - s = trial.sum(axis=1, keepdims=True) - s = np.where(s == 0, 1.0, s) - trial = trial / s - err = error(trial) - if err < best_local - 1e-15: - best_local = err - best_w = float(w) - best_pred = trial - if best_w is not None: - weights *= 1.0 - best_w - weights[j] += best_w - ens = best_pred - best_err = best_local - improved = True - if not improved: - break - weights[np.abs(weights) < 1e-12] = 0.0 - weights = np.maximum(weights, 0.0) - s = weights.sum() - return weights / s if s > 0 else weights - - -def _tasks(): - rng = np.random.default_rng(0) - n, d = 3000, 16 - X = rng.normal(size=(n, d)) - tasks = [] - # binary - logits = X[:, 0] * 1.2 + X[:, 2] * 0.8 + X[:, 5] * 0.4 + rng.normal(0, 0.6, n) - df = pd.DataFrame(X, columns=[f"f{i}" for i in range(d)]) - df["target"] = (logits > 0).astype(int) - tasks.append(("synth_binary", df, "binary")) - # regression - df2 = pd.DataFrame(X, columns=[f"f{i}" for i in range(d)]) - df2["target"] = X[:, 0] * 1.5 + X[:, 1] ** 2 * 0.3 + rng.normal(0, 0.5, n) - tasks.append(("synth_reg", df2, "regression")) - # multiclass - df3 = pd.DataFrame(X, columns=[f"f{i}" for i in range(d)]) - df3["target"] = X[:, :4].argmax(axis=1) - tasks.append(("synth_multi", df3, "multiclass")) - return tasks - - -def _fit_oof(name, df, problem_type, path: Path, time_limit=180): - path.mkdir(parents=True, exist_ok=True) - crit = "squared_error" if problem_type == "regression" else "gini" - hyperparameters = { - "GBM": [ - {}, - {"extra_trees": True, "ag_args": {"name_suffix": "XT"}}, - {"learning_rate": 0.03, "num_leaves": 64, "ag_args": {"name_suffix": "Large"}}, - ], - "CAT": [{}, {"depth": 6, "ag_args": {"name_suffix": "D6"}}], - "XGB": [{}, {"max_depth": 6, "ag_args": {"name_suffix": "D6"}}], - "RF": [{"criterion": crit}, {"n_estimators": 100, "ag_args": {"name_suffix": "100"}}], - "XT": [{"criterion": crit}], - "LR": [{}], - "KNN": [{"weights": "uniform"}, {"weights": "distance", "ag_args": {"name_suffix": "Dist"}}], - } - # sequential bag folds avoid Ray - for k, v in list(hyperparameters.items()): - if isinstance(v, list): - hyperparameters[k] = [ - {**cfg, "ag_args_ensemble": {"fold_fitting_strategy": "sequential_local"}} for cfg in v - ] - else: - hyperparameters[k] = {**v, "ag_args_ensemble": {"fold_fitting_strategy": "sequential_local"}} - - predictor = TabularPredictor( - label="target", - problem_type=problem_type, - path=str(path), - verbosity=1, - eval_metric="roc_auc" if problem_type == "binary" else None, - ) - predictor.fit( - df, - hyperparameters=hyperparameters, - num_bag_folds=5, - num_stack_levels=0, - time_limit=time_limit, - raise_on_model_failure=False, - ) - trainer = predictor._trainer - names = [m for m in trainer.get_model_names(level=1) if "WeightedEnsemble" not in m] - oofs = [] - keep = [] - for m in names: - try: - oof = np.asarray(trainer.get_model_oof(m)) - if oof.ndim == 2 and oof.shape[1] == 2 and problem_type == "binary": - oof = oof[:, 1] - oofs.append(oof.astype(np.float32)) - keep.append(m) - except Exception as e: - print(f" skip {m}: {e}") - y = np.asarray(predictor.transform_labels(df["target"])) - n = min(len(y), min(len(p) for p in oofs)) - return y[:n], [p[:n] for p in oofs], keep - - -def _split(y, preds, test_frac=0.3, seed=0): - rng = np.random.default_rng(seed) - n = len(y) - idx = rng.permutation(n) - n_test = max(1, int(n * test_frac)) - te, tr = idx[:n_test], idx[n_test:] - return y[tr], [p[tr] for p in preds], y[te], [p[te] for p in preds] - - -def _combine(preds, weights): - out = None - for p, w in zip(preds, weights, strict=True): - if w == 0: - continue - out = p * w if out is None else out + p * w - return out - - -def main(): - out = Path("artifacts/hill_climbing_4505/ag_oof") - out.mkdir(parents=True, exist_ok=True) - rows = [] - t0 = time.time() - for name, df, ptype in _tasks(): - print(f"\n=== {name} ({ptype}) ===") - y, preds, model_names = _fit_oof(name, df, ptype, out / f"ag_{name}", time_limit=200) - print(f" models: {len(model_names)}") - y_fit, p_fit, y_te, p_te = _split(y, preds) - metric = get_metric( - "roc_auc" if ptype == "binary" else ("log_loss" if ptype == "multiclass" else "rmse"), - problem_type=ptype, - ) - - # Single best - fit_errs = [metric.error(y_fit, p) for p in p_fit] - bi = int(np.argmin(fit_errs)) - sb_w = np.zeros(len(p_fit)) - sb_w[bi] = 1.0 - sb_test = metric.error(y_te, _combine(p_te, sb_w)) - - # Greedy Caruana - es = EnsembleSelection( - ensemble_size=40, problem_type=ptype, metric=metric, random_state=np.random.RandomState(0) - ) - es.fit(predictions=list(p_fit), labels=y_fit) - g_w = np.asarray(es.weights_) - g_test = metric.error(y_te, _combine(p_te, g_w)) - g_fit = metric.error(y_fit, _combine(p_fit, g_w)) - - # Hill climbing - hc_w = hill_climb_weights(p_fit, y_fit, metric, ptype) - hc_test = metric.error(y_te, _combine(p_te, hc_w)) - hc_fit = metric.error(y_fit, _combine(p_fit, hc_w)) - - for ens, fit_e, te, w in [ - ("single_best", fit_errs[bi], sb_test, sb_w), - ("greedy_caruana", g_fit, g_test, g_w), - ("hill_climbing", hc_fit, hc_test, hc_w), - ]: - rows.append( - { - "task": name, - "problem_type": ptype, - "ensembler": ens, - "fit_err": fit_e, - "test_err": te, - "n_models": int((np.asarray(w) != 0).sum()), - "n_pool": len(model_names), - } - ) - print(f" {ens:16s} fit={fit_e:.6f} test={te:.6f} n={(np.asarray(w) != 0).sum()}") - - df = pd.DataFrame(rows) - df.to_csv(out / "results.csv", index=False) - pivot = df.pivot_table(index="task", columns="ensembler", values="test_err") - delta = float((pivot["hill_climbing"] - pivot["greedy_caruana"]).mean()) - summary = { - "mean_test_err": pivot.mean().to_dict(), - "hc_minus_greedy_mean": delta, - "hc_wins": int((pivot["hill_climbing"] < pivot["greedy_caruana"] - 1e-12).sum()), - "greedy_wins": int((pivot["greedy_caruana"] < pivot["hill_climbing"] - 1e-12).sum()), - "wall_time_s": time.time() - t0, - "note": "Real AG bagged OOF; HC = Kaggle convex blend; Greedy = EnsembleSelection", - } - (out / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") - if delta < -1e-8: - conclusion = "HC improves mean test error vs Greedy on this AG OOF suite. Confirm on TabArena before AG default change." - change_default = False # still need broader TabArena - elif abs(delta) <= 1e-8: - conclusion = "HC ≈ Greedy. Do not change AG default ensemble selection." - change_default = False - else: - conclusion = "Greedy better or HC overfits. Do not change AG default ensemble selection." - change_default = False - summary["conclusion"] = conclusion - summary["change_ag_default"] = change_default - (out / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") - md = f"""# AG OOF: Hill climbing vs Greedy (#4505) - -Mean test errors: {json.dumps(summary["mean_test_err"], indent=2)} - -Mean (HC − Greedy): **{delta:.6g}** (negative ⇒ HC better) - -Task wins HC / Greedy: {summary["hc_wins"]} / {summary["greedy_wins"]} - -**{conclusion}** - -AG default change: **{change_default}** -""" - (out / "summary.md").write_text(md, encoding="utf-8") - print("\n" + md) - print(f"Wrote {out}") - - -if __name__ == "__main__": - main() diff --git a/scripts/!experimental/run_hill_climbing_vs_greedy_benchmark.py b/scripts/!experimental/run_hill_climbing_vs_greedy_benchmark.py deleted file mode 100644 index 14958aba4..000000000 --- a/scripts/!experimental/run_hill_climbing_vs_greedy_benchmark.py +++ /dev/null @@ -1,263 +0,0 @@ -#!/usr/bin/env python3 -"""TabArena post-hoc ensemble benchmark: HillClimbing vs Greedy (Caruana) vs SingleBest. - -Addresses autogluon/autogluon#4505 investigation 2 using real TabArena OOF caches. - -Definition of hill climbing (issue thread): - Kaggle / Matt-OP convex blend search — NOT continuous black-box HPO. - Compared to TabArena default GreedyEnsembler (AG EnsembleSelection / Caruana 2004). - -Example:: - - export PYTHONPATH=packages/tabarena/src:$PYTHONPATH - # downloads LightGBM processed (~8.5 GB) if missing - python scripts/\\!experimental/run_hill_climbing_vs_greedy_benchmark.py \\ - --method LightGBM --max-datasets 15 --max-folds 3 --n-configs 40 - -Outputs JSON + markdown under ``artifacts/hill_climbing_4505/``. -""" - -from __future__ import annotations - -import argparse -import json -import time -from pathlib import Path - -import numpy as np -import pandas as pd - - -def _load_method_repo(method: str): - from tabarena.contexts.tabarena.methods import tabarena_method_metadata_collection - - meta = tabarena_method_metadata_collection.get_method_metadata(method=method) - if not meta.path_processed_exists: - print(f"Downloading processed artifacts for {method} -> {meta.path_processed} ...") - meta.method_downloader(verbose=True).download_processed() - return meta.load_processed(), meta - - -def _evaluate_one(repo, dataset: str, fold: int, configs: list[str], ensembler_cls, ensembler_kwargs: dict): - df_result, df_weights = repo.evaluate_ensemble( - dataset=dataset, - fold=fold, - configs=configs, - ensemble_kwargs={ - "ensembler_cls": ensembler_cls, - "ensembler_kwargs": ensembler_kwargs, - }, - ) - # df_result is typically multi-index or single row with metric_error etc. - row = df_result.reset_index(drop=True).iloc[0].to_dict() - n_used = int((df_weights.iloc[0] != 0).sum()) if len(df_weights) else 0 - row["n_models_used"] = n_used - return row - - -def run_benchmark( - method: str = "LightGBM", - max_datasets: int | None = 20, - max_folds: int | None = 3, - n_configs: int | None = 50, - ensemble_size: int = 40, - hc_precision: float = 0.02, - hc_max_rounds: int = 30, - out_dir: Path | None = None, -) -> pd.DataFrame: - from tabarena.simulation.ensemble import GreedyEnsembler, HillClimbingEnsembler, SingleBestEnsembler - - repo, meta = _load_method_repo(method) - datasets = list(repo.datasets()) - if max_datasets is not None: - datasets = datasets[:max_datasets] - configs = list(repo.configs()) - if n_configs is not None: - configs = configs[:n_configs] - - # Discover folds via first dataset metrics / tasks - folds_available = sorted({t[1] for t in repo.tasks() if t[0] == datasets[0]}) if hasattr(repo, "tasks") else [0] - if not folds_available: - # fallback - folds_available = list(range(3)) - if max_folds is not None: - folds_available = folds_available[:max_folds] - - methods = { - "single_best": (SingleBestEnsembler, {}), - "greedy_caruana": ( - GreedyEnsembler, - {"ensemble_size": ensemble_size, "random_state": np.random.RandomState(0)}, - ), - "hill_climbing": ( - HillClimbingEnsembler, - { - "precision": hc_precision, - "max_rounds": hc_max_rounds, - "random_state": 0, - }, - ), - } - - rows = [] - t0 = time.time() - for dataset in datasets: - for fold in folds_available: - for name, (cls, kwargs) in methods.items(): - try: - t1 = time.time() - result = _evaluate_one(repo, dataset, fold, configs, cls, kwargs) - elapsed = time.time() - t1 - rows.append( - { - "method_pool": method, - "ensembler": name, - "dataset": dataset, - "fold": fold, - "metric_error": result.get("metric_error"), - "metric_error_val": result.get("metric_error_val"), - "n_models_used": result.get("n_models_used"), - "time_s": elapsed, - "n_configs_pool": len(configs), - } - ) - print( - f"[{len(rows)}] {dataset} fold={fold} {name}: " - f"test_err={result.get('metric_error')} val_err={result.get('metric_error_val')} " - f"n_used={result.get('n_models_used')} ({elapsed:.2f}s)" - ) - except Exception as e: - print(f"FAIL {dataset} fold={fold} {name}: {type(e).__name__}: {e}") - rows.append( - { - "method_pool": method, - "ensembler": name, - "dataset": dataset, - "fold": fold, - "metric_error": np.nan, - "metric_error_val": np.nan, - "n_models_used": np.nan, - "time_s": np.nan, - "n_configs_pool": len(configs), - "error": f"{type(e).__name__}: {e}", - } - ) - - df = pd.DataFrame(rows) - out_dir = out_dir or Path("artifacts/hill_climbing_4505") - out_dir.mkdir(parents=True, exist_ok=True) - csv_path = out_dir / f"benchmark_{method}_d{len(datasets)}_f{len(folds_available)}_c{len(configs)}.csv" - df.to_csv(csv_path, index=False) - - summary = _summarize(df) - md_path = out_dir / f"summary_{method}.md" - md_path.write_text(summary["markdown"], encoding="utf-8") - json_path = out_dir / f"summary_{method}.json" - json_path.write_text(json.dumps(summary["stats"], indent=2, default=str), encoding="utf-8") - - print(f"\nWrote {csv_path}") - print(f"Wrote {md_path}") - print(f"Total wall time: {time.time() - t0:.1f}s") - print(summary["markdown"]) - return df - - -def _summarize(df: pd.DataFrame) -> dict: - """Compare ensemblers: win rates on test metric_error (lower better).""" - ok = df.dropna(subset=["metric_error"]) - if ok.empty: - return {"markdown": "No successful runs.\n", "stats": {}} - - pivot = ok.pivot_table(index=["dataset", "fold"], columns="ensembler", values="metric_error", aggfunc="first") - stats: dict = {"n_tasks": int(len(pivot)), "mean_error": {}, "wins_vs_greedy": {}, "ties_vs_greedy": {}, "mean_n_models": {}} - - for col in pivot.columns: - stats["mean_error"][col] = float(pivot[col].mean()) - - if "greedy_caruana" in pivot.columns and "hill_climbing" in pivot.columns: - g, h = pivot["greedy_caruana"], pivot["hill_climbing"] - stats["wins_vs_greedy"]["hill_climbing"] = int((h < g - 1e-12).sum()) - stats["ties_vs_greedy"]["hill_climbing"] = int(np.isclose(h, g, rtol=0, atol=1e-12).sum()) - stats["wins_vs_greedy"]["greedy_caruana"] = int((g < h - 1e-12).sum()) - stats["mean_delta_hc_minus_greedy"] = float((h - g).mean()) # negative => HC better - - if "single_best" in pivot.columns and "hill_climbing" in pivot.columns: - s, h = pivot["single_best"], pivot["hill_climbing"] - stats["wins_vs_single_best"] = { - "hill_climbing": int((h < s - 1e-12).sum()), - "single_best": int((s < h - 1e-12).sum()), - } - - n_models = ok.groupby("ensembler")["n_models_used"].mean() - stats["mean_n_models"] = {k: float(v) for k, v in n_models.items()} - - lines = [ - "# Hill climbing vs Greedy ensemble selection (TabArena OOF)", - "", - "Issue: [autogluon/autogluon#4505](https://github.com/autogluon/autogluon/issues/4505)", - "", - "## Definitions", - "", - "- **greedy_caruana**: TabArena `GreedyEnsembler` → AutoGluon `EnsembleSelection` (Caruana et al. 2004).", - "- **hill_climbing**: `HillClimbingEnsembler` — Kaggle/Matt-OP convex blend search (not continuous BBO).", - "- **single_best**: best validation model only.", - "", - f"Tasks (dataset × fold): **{stats['n_tasks']}**", - "", - "## Mean test metric_error (lower is better)", - "", - ] - for k, v in sorted(stats["mean_error"].items(), key=lambda x: x[1]): - lines.append(f"- `{k}`: {v:.6g}") - lines.append("") - if "mean_delta_hc_minus_greedy" in stats: - d = stats["mean_delta_hc_minus_greedy"] - lines.append(f"Mean (HC − Greedy) test error: **{d:.6g}** (negative means HC wins on average)") - lines.append( - f"Task wins: HC {stats['wins_vs_greedy'].get('hill_climbing', 0)} / " - f"Greedy {stats['wins_vs_greedy'].get('greedy_caruana', 0)} / " - f"ties {stats['ties_vs_greedy'].get('hill_climbing', 0)}" - ) - lines.append("") - if d < -1e-8: - lines.append("**Conclusion (this slice):** Hill climbing improves mean test error vs Greedy.") - lines.append("→ Candidate for further portfolio study; AG default change only after broader confirmation.") - elif abs(d) <= 1e-8: - lines.append("**Conclusion (this slice):** Hill climbing ≈ Greedy (no meaningful mean difference).") - lines.append("→ **Do not** change AG default ensemble selection based on this evidence.") - else: - lines.append("**Conclusion (this slice):** Greedy is better or HC overfits OOF.") - lines.append("→ **Do not** change AG default; keep GreedyEnsembler / EnsembleSelection.") - lines.append("") - lines.append("## Mean models used") - for k, v in stats["mean_n_models"].items(): - lines.append(f"- `{k}`: {v:.2f}") - lines.append("") - return {"markdown": "\n".join(lines) + "\n", "stats": stats} - - -def main(): - p = argparse.ArgumentParser() - p.add_argument("--method", default="LightGBM") - p.add_argument("--max-datasets", type=int, default=15) - p.add_argument("--max-folds", type=int, default=3) - p.add_argument("--n-configs", type=int, default=40) - p.add_argument("--ensemble-size", type=int, default=40) - p.add_argument("--hc-precision", type=float, default=0.02) - p.add_argument("--hc-max-rounds", type=int, default=30) - p.add_argument("--out-dir", type=Path, default=Path("artifacts/hill_climbing_4505")) - args = p.parse_args() - run_benchmark( - method=args.method, - max_datasets=args.max_datasets, - max_folds=args.max_folds, - n_configs=args.n_configs, - ensemble_size=args.ensemble_size, - hc_precision=args.hc_precision, - hc_max_rounds=args.hc_max_rounds, - out_dir=args.out_dir, - ) - - -if __name__ == "__main__": - main() From dd37592bd9df84dabd65cc8f2a05b91007f44ef1 Mon Sep 17 00:00:00 2001 From: Anh-Duy Pham Date: Tue, 11 Aug 2026 15:01:41 +0200 Subject: [PATCH 3/3] [tabarena] Align HillClimbingEnsembler with GES-style global steps Use a full-neighborhood best (model, w) step each round and optionally include the Caruana weight w=1/(n_support+1). Add multiclass simplex renormalization, optional warm-start weights, and broader unit tests. Still optional research/post-hoc; default ensembler remains Greedy. --- .../ensemble/hill_climbing_ensembler.py | 214 +++++++++++------- tests/tabarena/simulation/test_ensembler.py | 121 ++++++++++ 2 files changed, 249 insertions(+), 86 deletions(-) diff --git a/packages/tabarena/src/tabarena/simulation/ensemble/hill_climbing_ensembler.py b/packages/tabarena/src/tabarena/simulation/ensemble/hill_climbing_ensembler.py index 549f38e9d..bb6ba30f9 100644 --- a/packages/tabarena/src/tabarena/simulation/ensemble/hill_climbing_ensembler.py +++ b/packages/tabarena/src/tabarena/simulation/ensemble/hill_climbing_ensembler.py @@ -1,23 +1,14 @@ """Kaggle-style hill-climbing ensemble (Matt-OP / Deotte family). -This is the definition of "hill climbing" that matches tilii7's AutoGluon feedback and -the community pointer on autogluon/autogluon#4505 -(https://github.com/Matt-OP/hillclimbers/) — **not** continuous black-box HPO -(arxiv:2307.00286). - -Compared to :class:`GreedyEnsembler` (Caruana ensemble selection, arxiv:1502.04759 / -AutoGluon ``EnsembleSelection``): - -* **Caruana / GreedyEnsembler:** iteratively *append* a model so the uniform average of - the multiset improves; weights are integer counts / ensemble_size. -* **Hill climbing (this class):** start from the best single model, then repeatedly try - convex blends ``(1 - w) * ensemble + w * model`` over a weight grid; accept any - improvement. Weights are continuous on a precision grid (default ``0.01``). - -Both are greedy local search over linear combinations of OOF predictions; they can -diverge on the selected support and weight magnitudes. #4505 asks whether this family -improves TabRepo/TabArena simulation score vs current ensemble selection — plug this -class in via ``ensembler_cls`` / ``ensemble_kwargs``. +Same approach family as :class:`GreedyEnsembler` (Caruana GES / AG ``EnsembleSelection``): +greedy local search over linear combinations of OOF predictions. Differences are the step +parameterization (continuous ``w`` grid vs discrete multiset votes), not a different problem. + +Each step evaluates all ``(model, w)`` candidates and accepts the single best improvement. +Optionally includes the Caruana step weight ``w = 1/(n_support+1)`` on the grid. + +This class is optional research / post-hoc; TabArena's default remains +:class:`GreedyEnsembler`. """ from __future__ import annotations @@ -34,23 +25,30 @@ class in via ``ensembler_cls`` / ``ensemble_kwargs``. class HillClimbingEnsembler(WeightedEnsembler): - """Iterative convex blending of base-model predictions (Kaggle hill climbing). + """Iterative convex blending of base-model predictions (hill climbing). + + Starts from the best single model (or optional warm-start weights), then repeatedly + blends ``(1 - w) * ensemble + w * model`` and keeps the best improving step. Parameters ---------- precision : float, default 0.01 Weight step on ``(0, 1]``. Smaller is slower and can overfit small OOF sets. max_rounds : int, default 100 - Maximum outer passes over all models after initialization. Stops early when a - full pass finds no improvement. + Maximum accepted improvement steps. Stops early when a step finds no improvement. + include_caruana_step : bool, default True + Also try ``w = 1/(n_support + 1)`` each step (GES-style blend weight when treating + the current ensemble as one unit). allow_negative_weights : bool, default False - If True, also tries ``w`` on ``[-1, 0)`` (Matt-OP ``negative_weights``). Safer - off when OOF is small. + If True, also tries ``w`` on ``[-1, 0)``. Safer off on small OOF. max_models : int | None, default None - Optional cap on how many models may receive non-zero weight (sparsity preference). - ``None`` means no extra cap. + Optional cap on how many models may receive non-zero weight. ``None`` means no cap. + initial_weights : array-like | None, default None + Optional warm-start weights (one entry per model). When set, skip best-single + initialization. Must be finite; non-negative unless ``allow_negative_weights``; + renormalized to sum to 1. random_state : int | np.random.RandomState | None, default None - Used only for tie-breaking among equal-error starts / blends. + Tie-breaking when selecting the initial best single model. """ def __init__( @@ -60,8 +58,10 @@ def __init__( metric: Scorer, precision: float = 0.01, max_rounds: int = 100, + include_caruana_step: bool = True, allow_negative_weights: bool = False, max_models: int | None = None, + initial_weights: np.ndarray | list[float] | None = None, random_state: int | np.random.RandomState | None = None, ): super().__init__(problem_type=problem_type, metric=metric) @@ -73,8 +73,10 @@ def __init__( raise ValueError(f"max_models must be >= 1 or None, got {max_models}") self.precision = float(precision) self.max_rounds = int(max_rounds) + self.include_caruana_step = bool(include_caruana_step) self.allow_negative_weights = bool(allow_negative_weights) self.max_models = max_models + self.initial_weights = None if initial_weights is None else np.asarray(initial_weights, dtype=np.float64) if isinstance(random_state, np.random.RandomState): self.random_state = random_state else: @@ -82,101 +84,135 @@ def __init__( self.trajectory_: list[float] = [] self.n_rounds_: int = 0 + self.init_val_error_: float | None = None - def _weight_grid(self) -> np.ndarray: - # Exclude 0 (no-op). Include 1.0. + def _weight_grid(self, *, n_support: int | None = None) -> np.ndarray: pos = np.arange(self.precision, 1.0 + self.precision * 0.5, self.precision) pos = np.clip(pos, self.precision, 1.0) - # unique stable pos = np.unique(np.round(pos / self.precision) * self.precision) + if self.include_caruana_step and n_support is not None and n_support >= 1: + w_c = 1.0 / float(n_support + 1) + if 0 < w_c <= 1: + pos = np.unique(np.concatenate([pos, np.asarray([w_c], dtype=np.float64)])) if self.allow_negative_weights: - neg = -pos[::-1] - return np.concatenate([neg, pos]) + return np.concatenate([-pos[::-1], pos]) return pos - def _combine(self, predictions: np.ndarray, weights: np.ndarray) -> np.ndarray: - # Same linear combo semantics as WeightedEnsembler.predict_proba / AG. + def _allowed_model(self, j: int, w: float, weights: np.ndarray) -> bool: + if self.max_models is None or weights[j] != 0 or w == 0: + return True + return self._n_nonzero(weights) < self.max_models + + def _try_blend( + self, + *, + ensemble_pred: np.ndarray, + pred_j: np.ndarray, + w: float, + labels: np.ndarray, + best_error: float, + ) -> tuple[float, float, np.ndarray] | None: + trial = (1.0 - w) * ensemble_pred + w * pred_j + trial = self._renormalize_proba(trial) + err = self._score_error(labels, trial) + if err < best_error - 1e-15: + return err, float(w), trial + return None + + def _combine(self, predictions: list[np.ndarray], weights: np.ndarray) -> np.ndarray: preds_norm = [pred * w for pred, w in zip(predictions, weights, strict=True) if w != 0] if not preds_norm: - # Degenerate: fall back to uniform over all (should not happen after init). return np.mean(predictions, axis=0) return np.sum(preds_norm, axis=0) def _n_nonzero(self, weights: np.ndarray) -> int: return int(np.sum(np.abs(weights) > 0)) + def _renormalize_proba(self, trial: np.ndarray) -> np.ndarray: + """Keep multiclass / softclass rows on the probability simplex.""" + if trial.ndim == 2 and self.problem_type in ("multiclass", "softclass"): + row_sum = trial.sum(axis=1, keepdims=True) + row_sum = np.where(row_sum == 0, 1.0, row_sum) + trial = trial / row_sum + np.maximum(trial, 0.0, out=trial) + row_sum = trial.sum(axis=1, keepdims=True) + row_sum = np.where(row_sum == 0, 1.0, row_sum) + trial = trial / row_sum + return trial + def _fit(self, *, predictions: np.ndarray, labels: np.ndarray, time_limit: float | None = None) -> None: start = time.time() - predictions = np.asarray(predictions) + # float64 for stable blends; renorm multiclass so float32 OOF stays on the simplex. + predictions = [self._renormalize_proba(np.asarray(p, dtype=np.float64)) for p in predictions] n_models = len(predictions) if n_models == 0: raise ValueError("HillClimbingEnsembler requires at least one model") - # --- Initialize with best single model --- - single_errors = np.array([self._score_error(labels, pred) for pred in predictions], dtype=np.float64) - best_error = float(np.nanmin(single_errors)) - candidates = np.flatnonzero(np.isclose(single_errors, best_error, atol=0, rtol=1e-12)) - start_idx = int(self.random_state.choice(candidates)) + if self.initial_weights is not None: + weights = np.asarray(self.initial_weights, dtype=np.float64).copy() + if weights.shape != (n_models,): + raise ValueError(f"initial_weights length {weights.shape} != n_models={n_models}") + if not np.isfinite(weights).all(): + raise ValueError("initial_weights must be finite") + if not self.allow_negative_weights: + weights = np.maximum(weights, 0.0) + total0 = float(np.sum(weights)) + if abs(total0) <= 1e-12: + raise ValueError("initial_weights sum to ~0; cannot warm-start") + weights = weights / total0 + ensemble_pred = self._renormalize_proba(self._combine(predictions, weights)) + best_error = float(self._score_error(labels, ensemble_pred)) + start_idx = int(np.argmax(np.abs(weights))) + else: + single_errors = np.array([self._score_error(labels, pred) for pred in predictions], dtype=np.float64) + best_error = float(np.nanmin(single_errors)) + candidates = np.flatnonzero(np.isclose(single_errors, best_error, atol=0, rtol=1e-12)) + start_idx = int(self.random_state.choice(candidates)) + weights = np.zeros(n_models, dtype=np.float64) + weights[start_idx] = 1.0 + ensemble_pred = predictions[start_idx].copy() - weights = np.zeros(n_models, dtype=np.float64) - weights[start_idx] = 1.0 - ensemble_pred = predictions[start_idx].copy() + self.init_val_error_ = best_error self.trajectory_ = [best_error] - weight_grid = self._weight_grid() - for round_i in range(self.max_rounds): if time_limit is not None and (time.time() - start) >= time_limit: break - improved = False - # Randomize model order each round for mild exploration under ties. - order = self.random_state.permutation(n_models) - for j in order: + n_support = max(self._n_nonzero(weights), 1) + weight_grid = self._weight_grid(n_support=n_support) + best_local_error = best_error + best_j = None + best_local_w = None + best_local_pred = None + for j in range(n_models): if time_limit is not None and (time.time() - start) >= time_limit: break - pred_j = predictions[j] - best_local_error = best_error - best_local_w = None - best_local_pred = None - for w in weight_grid: - # Convex blend against current ensemble (standard Kaggle HC step). - trial = (1.0 - w) * ensemble_pred + w * pred_j - # Optional multiclass renormalize if this is a probability simplex view. - if trial.ndim == 2 and self.problem_type in ("multiclass", "softclass"): - row_sum = trial.sum(axis=1, keepdims=True) - row_sum = np.where(row_sum == 0, 1.0, row_sum) - trial = trial / row_sum - - # Enforce max_models: if adding a new model would exceed cap, skip - # unless it already has weight. - if self.max_models is not None and weights[j] == 0 and w != 0: - if self._n_nonzero(weights) >= self.max_models: - continue - - err = self._score_error(labels, trial) - if err < best_local_error - 1e-15: - best_local_error = err - best_local_w = float(w) - best_local_pred = trial - - if best_local_w is not None: - # Update latent weights: ensemble := (1-w)*ensemble + w*model_j - # ⇒ scale existing weights by (1-w), add w to model j. - weights *= 1.0 - best_local_w - weights[j] += best_local_w - ensemble_pred = best_local_pred - best_error = best_local_error - self.trajectory_.append(best_error) - improved = True + if not self._allowed_model(j, float(w), weights): + continue + hit = self._try_blend( + ensemble_pred=ensemble_pred, + pred_j=pred_j, + w=float(w), + labels=labels, + best_error=best_local_error, + ) + if hit is not None: + best_local_error, best_local_w, best_local_pred = hit + best_j = j self.n_rounds_ = round_i + 1 - if not improved: + if best_j is None or best_local_w is None: break - # Numerical cleanup: drop tiny weights, renormalize for stable reporting. + weights *= 1.0 - best_local_w + weights[best_j] += best_local_w + ensemble_pred = best_local_pred + best_error = best_local_error + self.trajectory_.append(best_error) + weights[np.abs(weights) < 1e-12] = 0.0 if not self.allow_negative_weights: weights = np.maximum(weights, 0.0) @@ -184,17 +220,23 @@ def _fit(self, *, predictions: np.ndarray, labels: np.ndarray, time_limit: float if abs(total) > 1e-12: weights = weights / total else: - # Should not happen after best-single init; fall back to that model. weights = np.zeros(n_models, dtype=np.float64) weights[start_idx] = 1.0 self.weights_ = weights + def predict_proba(self, predictions: np.ndarray) -> np.ndarray: + out = super().predict_proba(predictions) + return self._renormalize_proba(np.asarray(out, dtype=np.float64)) + def info(self) -> dict: return { "n_rounds": self.n_rounds_, "trajectory_len": len(self.trajectory_), + "init_val_error": self.init_val_error_, "final_val_error": self.trajectory_[-1] if self.trajectory_ else None, + "warm_started": self.initial_weights is not None, + "include_caruana_step": self.include_caruana_step, "precision": self.precision, "allow_negative_weights": self.allow_negative_weights, } diff --git a/tests/tabarena/simulation/test_ensembler.py b/tests/tabarena/simulation/test_ensembler.py index ffbf03c32..ab6ca2f10 100644 --- a/tests/tabarena/simulation/test_ensembler.py +++ b/tests/tabarena/simulation/test_ensembler.py @@ -570,6 +570,7 @@ def test_hill_climbing_beats_or_matches_single_best_on_synthetic(): max_rounds=20, random_state=0, ) + assert hc.include_caruana_step is True hc.fit(predictions=preds, labels=y) hc_err = metric.error(y, hc.predict_proba(preds)) @@ -621,3 +622,123 @@ def test_hill_climbing_task_evaluator_runs(): assert "ensemble_weights" in results assert results["ensemble_weights"] is not None assert ensemble is not None + + +def test_hill_climbing_invariants_and_edge_cases(): + """Trajectory monotonicity, sparsity cap, single-model, determinism, hyperparam guards.""" + from tabarena.simulation.ensemble import HillClimbingEnsembler + + y, preds = _make_binary_task(n_models=12, n_samples=600, seed=13) + metric = get_metric(metric="roc_auc", problem_type="binary") + + hc = HillClimbingEnsembler(problem_type="binary", metric=metric, precision=0.05, max_rounds=25, random_state=0) + hc.fit(predictions=preds, labels=y) + traj = np.asarray(hc.trajectory_) + assert traj.ndim == 1 and len(traj) >= 1 + assert np.all(np.diff(traj) <= 1e-15) + single_best_err = min(metric.error(y, p) for p in preds) + assert np.isclose(traj[0], single_best_err, rtol=1e-10, atol=1e-12) + assert traj[-1] <= single_best_err + 1e-12 + assert np.isclose(hc.model_weights().sum(), 1.0) + assert (hc.model_weights() >= -1e-12).all() + assert hc.info()["n_rounds"] >= 1 + + # max_models sparsity + hc_sparse = HillClimbingEnsembler( + problem_type="binary", + metric=metric, + precision=0.05, + max_rounds=25, + max_models=3, + random_state=0, + ) + hc_sparse.fit(predictions=preds, labels=y) + assert int(hc_sparse.models_used().sum()) <= 3 + assert metric.error(y, hc_sparse.predict_proba(preds)) <= single_best_err + 1e-12 + + # single model pool + hc_one = HillClimbingEnsembler(problem_type="binary", metric=metric, precision=0.1, max_rounds=5, random_state=0) + hc_one.fit(predictions=preds[:1], labels=y) + np.testing.assert_allclose(hc_one.model_weights(), [1.0]) + + # determinism + a = HillClimbingEnsembler(problem_type="binary", metric=metric, precision=0.05, max_rounds=15, random_state=42) + b = HillClimbingEnsembler(problem_type="binary", metric=metric, precision=0.05, max_rounds=15, random_state=42) + a.fit(predictions=preds, labels=y) + b.fit(predictions=preds, labels=y) + np.testing.assert_allclose(a.model_weights(), b.model_weights()) + + # does not mutate caller arrays + preds_copy = preds.copy() + HillClimbingEnsembler(problem_type="binary", metric=metric, precision=0.1, max_rounds=5, random_state=0).fit( + predictions=preds, labels=y + ) + assert np.array_equal(preds, preds_copy) + + # hyperparam validation + with pytest.raises(ValueError, match="precision"): + HillClimbingEnsembler(problem_type="binary", metric=metric, precision=0) + with pytest.raises(ValueError, match="max_rounds"): + HillClimbingEnsembler(problem_type="binary", metric=metric, max_rounds=0) + with pytest.raises(ValueError, match="max_models"): + HillClimbingEnsembler(problem_type="binary", metric=metric, max_models=0) + with pytest.raises(ValueError, match="at least one"): + HillClimbingEnsembler(problem_type="binary", metric=metric).fit(predictions=np.zeros((0, 10)), labels=y[:10]) + + +def test_hill_climbing_multiclass_simplex(): + """Multiclass blends stay on the probability simplex; fit error ≤ single-best.""" + from tabarena.simulation.ensemble import HillClimbingEnsembler + + y, preds = _make_multiclass_task(n_models=8, n_samples=500, n_classes=4, seed=14) + metric = get_metric(metric="log_loss", problem_type="multiclass") + hc = HillClimbingEnsembler(problem_type="multiclass", metric=metric, precision=0.05, max_rounds=20, random_state=0) + hc.fit(predictions=preds, labels=y) + out = hc.predict_proba(preds) + assert out.shape == (len(y), preds.shape[2]) + np.testing.assert_allclose(out.sum(axis=1), 1.0, atol=1e-5) + assert (out >= -1e-8).all() + single_best_err = min(metric.error(y, p) for p in preds) + assert metric.error(y, out) <= single_best_err + 1e-10 + + +def test_hill_climbing_time_limit_returns_valid_weights(): + """time_limit may stop early but must still yield a valid weight vector.""" + from tabarena.simulation.ensemble import HillClimbingEnsembler + + y, preds = _make_binary_task(n_models=30, n_samples=2000, seed=15) + metric = get_metric(metric="roc_auc", problem_type="binary") + hc = HillClimbingEnsembler(problem_type="binary", metric=metric, precision=0.01, max_rounds=200, random_state=0) + hc.fit(predictions=preds, labels=y, time_limit=0.02) + w = hc.model_weights() + assert w is not None + assert np.isclose(w.sum(), 1.0) + assert np.isfinite(w).all() + + +def test_hill_climbing_warm_start_from_greedy_weights(): + """Warm-start HC from Greedy weights: refined val error must be <= Greedy val error.""" + from tabarena.simulation.ensemble import GreedyEnsembler, HillClimbingEnsembler + + y, preds = _make_binary_task(n_models=12, n_samples=800, seed=21) + metric = get_metric(metric="roc_auc", problem_type="binary") + + greedy = GreedyEnsembler( + problem_type="binary", metric=metric, ensemble_size=30, random_state=np.random.RandomState(0) + ) + greedy.fit(predictions=preds, labels=y) + g_err = metric.error(y, greedy.predict_proba(preds)) + + hc = HillClimbingEnsembler( + problem_type="binary", + metric=metric, + precision=0.05, + max_rounds=20, + initial_weights=greedy.model_weights(), + random_state=0, + ) + hc.fit(predictions=preds, labels=y) + hc_err = metric.error(y, hc.predict_proba(preds)) + assert hc_err <= g_err + 1e-12 + assert hc.info()["warm_started"] is True + assert hc.info()["init_val_error"] is not None