From 2fce8b274affd71448aab2d096551830ea24209c Mon Sep 17 00:00:00 2001 From: LennartPurucker Date: Mon, 10 Aug 2026 08:46:42 +0000 Subject: [PATCH 1/7] add: new chimeraboost version --- packages/tabarena/pyproject.toml | 2 +- packages/tabarena/src/tabarena/models/chimeraboost/info.py | 2 +- .../tabarena/src/tabarena/models/chimeraboost/model.py | 7 ++----- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/packages/tabarena/pyproject.toml b/packages/tabarena/pyproject.toml index dc0ab49bc..71fd9b88e 100644 --- a/packages/tabarena/pyproject.toml +++ b/packages/tabarena/pyproject.toml @@ -114,7 +114,7 @@ limix = [ ] tabpfnwide = ["tabpfnwide>=0.3.0"] iltm = ["iltm>=0.1.1"] -chimeraboost = ["chimeraboost>=0.14.1"] +chimeraboost = ["chimeraboost>=0.30.0"] nori = ["synthefy-nori>=0.10.0"] # EXAONE Tabular is not published on PyPI; pinned to a commit so the benchmarked code is fixed. # Keep in sync with `pip_extra` in models/exaone_tabular/info.py. diff --git a/packages/tabarena/src/tabarena/models/chimeraboost/info.py b/packages/tabarena/src/tabarena/models/chimeraboost/info.py index 20d01543a..6700ac8b3 100644 --- a/packages/tabarena/src/tabarena/models/chimeraboost/info.py +++ b/packages/tabarena/src/tabarena/models/chimeraboost/info.py @@ -50,5 +50,5 @@ model_cls=ChimeraBoostModel, search_space=gen_chimeraboost, method_metadata=chimeraboost_new_method_metadata, - pip_extra=("chimeraboost>=0.14.1",), + pip_extra=("chimeraboost>=0.30.0",), ) diff --git a/packages/tabarena/src/tabarena/models/chimeraboost/model.py b/packages/tabarena/src/tabarena/models/chimeraboost/model.py index 18a3dbbac..726e97875 100644 --- a/packages/tabarena/src/tabarena/models/chimeraboost/model.py +++ b/packages/tabarena/src/tabarena/models/chimeraboost/model.py @@ -24,6 +24,7 @@ class ChimeraBoostModel(AbstractModel): ag_key = "CHIMERA" ag_name = "ChimeraBoost" seed_name = "random_state" # AutoGluon injects the framework seed here + _supported_problem_types = ["binary", "multiclass", "regression"] def _preprocess(self, X: pd.DataFrame, is_train=False, **kwargs) -> pd.DataFrame: """Pass the frame straight to ChimeraBoost with categoricals marked by @@ -107,16 +108,12 @@ def _get_default_auxiliary_params(self) -> dict: default_auxiliary_params.update({"valid_raw_types": ["int", "float", "category"]}) return default_auxiliary_params - @classmethod - def supported_problem_types(cls) -> list[str] | None: - return ["binary", "multiclass", "regression"] - @classmethod def warmup(cls, **kwargs) -> None: """Pre-compile the numba kernels (~10s cold start, then disk-cached per environment).""" import chimeraboost - chimeraboost.warmup() # requires chimeraboost>=0.14.1 (the pinned extra) + chimeraboost.warmup() def _get_default_resources(self) -> tuple[int, int]: # Physical cores only (matches RealMLP/XRFM); ChimeraBoost is CPU-only. From c7d90a71cd4772c86aeb41736f1cc47ae94d59a1 Mon Sep 17 00:00:00 2001 From: LennartPurucker Date: Mon, 10 Aug 2026 09:21:33 +0000 Subject: [PATCH 2/7] add: cleaner API support across all models --- .../src/tabarena/models/chimeraboost/model.py | 26 +----- .../tabarena/models/exaone_tabular/model.py | 50 +++--------- .../src/tabarena/models/iltm/model.py | 24 ++---- .../src/tabarena/models/limix/model.py | 57 ++++--------- .../src/tabarena/models/modernnca/model.py | 19 +---- .../src/tabarena/models/nori/model.py | 46 ++--------- .../src/tabarena/models/orionmsp/model.py | 26 +----- .../models/perpetual_booster/model.py | 5 +- .../src/tabarena/models/realmlp/model.py | 30 +------ .../src/tabarena/models/sap_rpt_oss/model.py | 29 +------ .../src/tabarena/models/tabdpt/model.py | 26 +----- .../src/tabarena/models/tabfm/model.py | 56 ++++--------- .../src/tabarena/models/tabicl/model.py | 44 ++-------- .../src/tabarena/models/tabm/model.py | 30 +------ .../src/tabarena/models/tabpfn_3/model.py | 53 +++--------- .../src/tabarena/models/tabpfnv2_5/model.py | 81 ++++++------------- .../src/tabarena/models/tabpfnwide/model.py | 37 ++------- .../src/tabarena/models/tabstar/model.py | 29 +------ .../src/tabarena/models/tabswift/model.py | 49 +++-------- .../src/tabarena/models/xrfm/model.py | 30 +------ 20 files changed, 144 insertions(+), 603 deletions(-) diff --git a/packages/tabarena/src/tabarena/models/chimeraboost/model.py b/packages/tabarena/src/tabarena/models/chimeraboost/model.py index 726e97875..50b1235c9 100644 --- a/packages/tabarena/src/tabarena/models/chimeraboost/model.py +++ b/packages/tabarena/src/tabarena/models/chimeraboost/model.py @@ -11,7 +11,6 @@ import time from typing import TYPE_CHECKING -from autogluon.common.utils.resource_utils import ResourceManager from autogluon.core.models import AbstractModel if TYPE_CHECKING: @@ -25,6 +24,8 @@ class ChimeraBoostModel(AbstractModel): ag_name = "ChimeraBoost" seed_name = "random_state" # AutoGluon injects the framework seed here _supported_problem_types = ["binary", "multiclass", "regression"] + _default_auxiliary_params_extra = {"valid_raw_types": ["int", "float", "category"]} + default_resources_physical_cores_only = True def _preprocess(self, X: pd.DataFrame, is_train=False, **kwargs) -> pd.DataFrame: """Pass the frame straight to ChimeraBoost with categoricals marked by @@ -103,11 +104,6 @@ def _set_default_params(self): for param, val in default_params.items(): self._set_default_param_value(param, val) - def _get_default_auxiliary_params(self) -> dict: - default_auxiliary_params = super()._get_default_auxiliary_params() - default_auxiliary_params.update({"valid_raw_types": ["int", "float", "category"]}) - return default_auxiliary_params - @classmethod def warmup(cls, **kwargs) -> None: """Pre-compile the numba kernels (~10s cold start, then disk-cached per environment).""" @@ -115,20 +111,6 @@ def warmup(cls, **kwargs) -> None: chimeraboost.warmup() - def _get_default_resources(self) -> tuple[int, int]: - # Physical cores only (matches RealMLP/XRFM); ChimeraBoost is CPU-only. - num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True) - return num_cpus, 0 - - def _estimate_memory_usage(self, X: pd.DataFrame, **kwargs) -> int: - return self.estimate_memory_usage_static( - X=X, - problem_type=self.problem_type, - num_classes=self.num_classes, - hyperparameters=self._get_model_params(), - **kwargs, - ) - @classmethod def _estimate_memory_usage_static( cls, @@ -157,7 +139,3 @@ def _estimate_memory_usage_static( hist = p * 256 * 2 * cell # transient per-level histograms baseline = 1_000_000_000 # python + numba + autogluon overhead return int(baseline + 3 * data + binned + stats + hist) - - @classmethod - def _class_tags(cls) -> dict: - return {"can_estimate_memory_usage_static": True} diff --git a/packages/tabarena/src/tabarena/models/exaone_tabular/model.py b/packages/tabarena/src/tabarena/models/exaone_tabular/model.py index 33abd6ca8..99345c8c3 100644 --- a/packages/tabarena/src/tabarena/models/exaone_tabular/model.py +++ b/packages/tabarena/src/tabarena/models/exaone_tabular/model.py @@ -35,6 +35,18 @@ class EXAONETabularModel(AbstractTorchModel): ag_name = "TA-EXAONE-Tabular" ag_priority = 65 seed_name = "seed" + _supported_problem_types = ["binary", "multiclass", "regression"] + default_num_gpus = 1 + default_resources_physical_cores_only = True + minimum_num_gpus = 1 + # Sequential fold fitting avoids contention on the shared Hugging Face checkpoint cache. + # ``refit_folds=True`` matches the other TFM wrappers (TabICL, TabSwift, TabPFN-3, ...): for + # an in-context-learning model, refitting one model on all data gives faster inference at + # similar quality to the bagged ensemble. + _default_ag_args_ensemble_extra = { + "fold_fitting_strategy": "sequential_local", + "refit_folds": True, + } def __init__(self, **kwargs): super().__init__(**kwargs) @@ -124,10 +136,6 @@ def _set_default_params(self): for param, val in default_params.items(): self._set_default_param_value(param, val) - @classmethod - def supported_problem_types(cls) -> list[str] | None: - return ["binary", "multiclass", "regression"] - def get_device(self) -> str: return self.model.device.type @@ -136,40 +144,6 @@ def _set_device(self, device: str): self.model.device = device self.model.model = self.model.model.to(device) - def _get_default_resources(self) -> tuple[int, int]: - # Use only physical cores for better performance based on benchmarks - num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True) - num_gpus = min(1, ResourceManager.get_gpu_count_torch(cuda_only=True)) - return num_cpus, num_gpus - - def get_minimum_resources(self, is_gpu_available: bool = False) -> dict[str, int | float]: - return { - "num_cpus": 1, - "num_gpus": 1 if is_gpu_available else 0, - } - - @classmethod - def _get_default_ag_args_ensemble(cls, **kwargs) -> dict: - """Sequential fold fitting avoids contention on the shared Hugging Face checkpoint cache. - - ``refit_folds=True`` matches the other TFM wrappers (TabICL, TabSwift, TabPFN-3, ...): for - an in-context-learning model, refitting one model on all data gives faster inference at - similar quality to the bagged ensemble. - """ - default_ag_args_ensemble = super()._get_default_ag_args_ensemble(**kwargs) - default_ag_args_ensemble.update( - { - "fold_fitting_strategy": "sequential_local", - "refit_folds": True, - }, - ) - return default_ag_args_ensemble - - @classmethod - def _class_tags(cls) -> dict: - # TODO: implement memory estimation and set to True - return {"can_estimate_memory_usage_static": False} - def _more_tags(self) -> dict: return {"can_refit_full": True} diff --git a/packages/tabarena/src/tabarena/models/iltm/model.py b/packages/tabarena/src/tabarena/models/iltm/model.py index ab57f79df..2eca3c583 100644 --- a/packages/tabarena/src/tabarena/models/iltm/model.py +++ b/packages/tabarena/src/tabarena/models/iltm/model.py @@ -28,8 +28,13 @@ class ILTMModel(AbstractTorchModel): ag_priority = 65 seed_name = "seed" + _supported_problem_types = ["binary", "multiclass", "regression"] + _categorical_indices: list[int] | None """The indices of the categorical features, detected during preprocessing.""" + default_num_gpus = 1 + default_resources_physical_cores_only = True + minimum_num_gpus = 1 def _preprocess(self, X: pd.DataFrame, *, is_train: bool = False, **kwargs) -> pd.DataFrame: """Detect indices of pandas `category`-dtype columns for iLTM's `cat_features`. @@ -113,10 +118,6 @@ def _fit( fit_max_time=time_limit, ) - @classmethod - def supported_problem_types(cls) -> list[str] | None: - return ["binary", "multiclass", "regression"] - def _predict_proba(self, X, **kwargs): # See _ensure_iltm_logger_patched docstring: bagged child models are # unpickled in the parent process without running iLTM's __init__, so @@ -133,21 +134,6 @@ def _set_device(self, device: str): if getattr(self.model, "_model", None) is not None: self.model._model = self.model._model.to(device) - def _get_default_resources(self) -> tuple[int, int]: - num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True) - num_gpus = min(1, ResourceManager.get_gpu_count_torch(cuda_only=True)) - return num_cpus, num_gpus - - def get_minimum_resources(self, is_gpu_available: bool = False) -> dict[str, int | float]: - return { - "num_cpus": 1, - "num_gpus": 1 if is_gpu_available else 0, - } - - @classmethod - def _class_tags(cls) -> dict: - return {"can_estimate_memory_usage_static": False} - def _ensure_iltm_logger_patched() -> None: """Workaround for upstream bug in iltm==0.1.0. diff --git a/packages/tabarena/src/tabarena/models/limix/model.py b/packages/tabarena/src/tabarena/models/limix/model.py index c60fa2cc3..b28d00134 100644 --- a/packages/tabarena/src/tabarena/models/limix/model.py +++ b/packages/tabarena/src/tabarena/models/limix/model.py @@ -46,12 +46,28 @@ class LimiXModel(AbstractTorchModel): ag_priority = 100 seed_name = "random_state" + _supported_problem_types = ["binary", "multiclass", "regression"] + subsample_train_n_rows: int = 75_000 """Empirically, even with 140 GB of VRAM available we still hit OOM on LimiX's retrieval + clustering inference path on TabArena-scale datasets, so subsampling is the only reliable lever to keep it running. We-sub-sample datasets above 75k rows to 50k rows following the LimiX documentation examples.""" batch_test_n_rows: int = 5_000 """We batch forward passes with more than 10k test rows.""" + default_num_gpus = 1 + default_resources_physical_cores_only = True + minimum_num_gpus = 1 + # Sequential fold fitting avoids contention on the shared HF checkpoint cache. + _default_ag_args_ensemble_extra = { + "fold_fitting_strategy": "sequential_local", + "refit_folds": True, + } + # We set the default to 100k to try to run on all of TabArena. + # Note, all examples of LimiX code itself says one should skip above 50k. + _default_auxiliary_params_extra = { + # "max_rows": 50_000, # Technically from LimiX + "max_classes": 10, + } def __init__(self, **kwargs): super().__init__(**kwargs) @@ -220,10 +236,6 @@ def _predict_proba(self, X: pd.DataFrame, **kwargs) -> np.ndarray: return self._convert_proba_to_unified_form(y_pred_proba) - @classmethod - def supported_problem_types(cls) -> list[str] | None: - return ["binary", "multiclass", "regression"] - def get_device(self) -> str: return self.model.device.type if self.model is not None else "cpu" @@ -235,43 +247,6 @@ def _set_device(self, device: str): if self.model.model is not None: self.model.model.to(device) - def _get_default_resources(self) -> tuple[int, int]: - num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True) - num_gpus = min(1, ResourceManager.get_gpu_count_torch(cuda_only=True)) - return num_cpus, num_gpus - - def get_minimum_resources(self, is_gpu_available: bool = False) -> dict[str, int | float]: - return { - "num_cpus": 1, - "num_gpus": 1 if is_gpu_available else 0, - } - - @classmethod - def _get_default_ag_args_ensemble(cls, **kwargs) -> dict: - """Sequential fold fitting avoids contention on the shared HF checkpoint cache.""" - default_ag_args_ensemble = super()._get_default_ag_args_ensemble(**kwargs) - default_ag_args_ensemble.update( - { - "fold_fitting_strategy": "sequential_local", - "refit_folds": True, - }, - ) - return default_ag_args_ensemble - - def _get_default_auxiliary_params(self) -> dict: - """We set the default to 100k to try to run on all of TabArena. - - Note, all examples of LimiX code itself says one should skip above 50k. - """ - default_auxiliary_params = super()._get_default_auxiliary_params() - default_auxiliary_params.update( - { - # "max_rows": 50_000, # Technically from LimiX - "max_classes": 10, - }, - ) - return default_auxiliary_params - def _more_tags(self) -> dict: return {"can_refit_full": True} diff --git a/packages/tabarena/src/tabarena/models/modernnca/model.py b/packages/tabarena/src/tabarena/models/modernnca/model.py index d311c0a25..192f67a2c 100644 --- a/packages/tabarena/src/tabarena/models/modernnca/model.py +++ b/packages/tabarena/src/tabarena/models/modernnca/model.py @@ -307,6 +307,7 @@ def predict_proba(self, X: pd.DataFrame) -> np.ndarray: class ModernNCAModel(AbstractModel): ag_key = "MNCA" ag_name = "ModernNCA" + _supported_problem_types = ["binary", "multiclass", "regression"] def __init__(self, **kwargs): super().__init__(**kwargs) @@ -424,10 +425,6 @@ def _set_default_params(self): for param, val in default_params.items(): self._set_default_param_value(param, val) - @classmethod - def supported_problem_types(cls) -> list[str] | None: - return ["binary", "multiclass", "regression"] - @classmethod def warmup(cls, *, num_gpus: float | None = None, **kwargs) -> None: """Torch-backed despite subclassing ``AbstractModel``: warm torch + CUDA context.""" @@ -446,16 +443,6 @@ def _get_default_resources(self) -> tuple[int, int]: num_gpus = 1 if torch.cuda.is_available() else 0 return num_cpus, num_gpus - def _estimate_memory_usage(self, X: pd.DataFrame, **kwargs) -> int: - hyperparameters = self._get_model_params() - return self.estimate_memory_usage_static( - X=X, - problem_type=self.problem_type, - num_classes=self.num_classes, - hyperparameters=hyperparameters, - **kwargs, - ) - # FIXME: Find a better estimate for memory usage of TabM. Currently borrowed from FASTAI estimate. @classmethod def _estimate_memory_usage_static( @@ -466,10 +453,6 @@ def _estimate_memory_usage_static( ) -> int: return 10 * get_approximate_df_mem_usage(X).sum() - @classmethod - def _class_tags(cls): - return {"can_estimate_memory_usage_static": True} - def _more_tags(self) -> dict: # TODO: Need to add train params support, track best epoch # How to force stopping at a specific epoch? diff --git a/packages/tabarena/src/tabarena/models/nori/model.py b/packages/tabarena/src/tabarena/models/nori/model.py index 6bb0d33f8..57dfdff8f 100644 --- a/packages/tabarena/src/tabarena/models/nori/model.py +++ b/packages/tabarena/src/tabarena/models/nori/model.py @@ -37,6 +37,14 @@ class NoriModel(AbstractTorchModel): ag_key = "TA-NORI" ag_name = "TA-Nori" ag_priority = 65 + _supported_problem_types = ["regression"] + default_num_gpus = 1 + default_resources_physical_cores_only = True + minimum_num_gpus = 1 + # Cap context size at 100k rows; no feature or class limits (regression-only). + _default_auxiliary_params_extra = { + "max_rows": 100_000, + } def __init__(self, **kwargs): super().__init__(**kwargs) @@ -104,10 +112,6 @@ def _set_default_params(self): for param, val in default_params.items(): self._set_default_param_value(param, val) - @classmethod - def supported_problem_types(cls) -> list[str] | None: - return ["regression"] - # --- Resource and GPU management --- def get_device(self) -> str: device = self.model.device @@ -128,27 +132,6 @@ def _set_device(self, device: str): if getattr(predictor, "model", None) is not None: predictor.model.to(torch_device) - def _get_default_resources(self) -> tuple[int, int]: - num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True) - num_gpus = min(1, ResourceManager.get_gpu_count_torch(cuda_only=True)) - return num_cpus, num_gpus - - def get_minimum_resources(self, is_gpu_available: bool = False) -> dict[str, int | float]: - return { - "num_cpus": 1, - "num_gpus": 1 if is_gpu_available else 0, - } - - def _get_default_auxiliary_params(self) -> dict: - """Cap context size at 100k rows; no feature or class limits (regression-only).""" - default_auxiliary_params = super()._get_default_auxiliary_params() - default_auxiliary_params.update( - { - "max_rows": 100_000, - }, - ) - return default_auxiliary_params - @classmethod def _get_default_ag_args_ensemble(cls, **kwargs) -> dict: """Fit one fold at a time (avoids contention on the shared checkpoint cache) and @@ -163,15 +146,6 @@ def _get_default_ag_args_ensemble(cls, **kwargs) -> dict: ) return default_ag_args_ensemble - def _estimate_memory_usage(self, X: pd.DataFrame, **kwargs) -> int: - return self.estimate_memory_usage_static( - X=X, - problem_type=self.problem_type, - num_classes=self.num_classes, - hyperparameters=self._get_model_params(), - **kwargs, - ) - @classmethod def _estimate_memory_usage_static(cls, *, X: pd.DataFrame, **kwargs) -> int: """Assume a small-model baseline (weights + activations) plus the dataset footprint.""" @@ -179,10 +153,6 @@ def _estimate_memory_usage_static(cls, *, X: pd.DataFrame, **kwargs) -> int: dataset_mem_est = 5 * get_approximate_df_mem_usage(X).sum() return int(baseline_mem_est + dataset_mem_est) - @classmethod - def _class_tags(cls) -> dict: - return {"can_estimate_memory_usage_static": True} - def _more_tags(self) -> dict: return {"can_refit_full": True} diff --git a/packages/tabarena/src/tabarena/models/orionmsp/model.py b/packages/tabarena/src/tabarena/models/orionmsp/model.py index 0ed9cde87..af24f9483 100644 --- a/packages/tabarena/src/tabarena/models/orionmsp/model.py +++ b/packages/tabarena/src/tabarena/models/orionmsp/model.py @@ -36,6 +36,10 @@ class OrionMSPModel(AbstractTorchModel): ag_name = "TA-OrionMSP" ag_priority = 65 seed_name = "random_state" + _supported_problem_types = ["binary", "multiclass"] + default_num_gpus = 1 + default_resources_physical_cores_only = True + minimum_num_gpus = 1 def _fit( self, @@ -112,10 +116,6 @@ def _set_default_params(self): for param, val in default_params.items(): self._set_default_param_value(param, val) - @classmethod - def supported_problem_types(cls) -> list[str] | None: - return ["binary", "multiclass"] - def get_device(self) -> str: return self.model.device @@ -124,19 +124,6 @@ def _set_device(self, device: str): if hasattr(self.model, "to"): self.model.to(device) - def _get_default_resources(self) -> tuple[int, int]: - # Use only physical cores for better performance based on benchmarks - num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True) - - num_gpus = min(1, ResourceManager.get_gpu_count_torch(cuda_only=True)) - return num_cpus, num_gpus - - def get_minimum_resources(self, is_gpu_available: bool = False) -> dict[str, int | float]: - return { - "num_cpus": 1, - "num_gpus": 1 if is_gpu_available else 0, - } - @classmethod def _get_default_ag_args_ensemble(cls, **kwargs) -> dict: """Set fold_fitting_strategy to sequential_local, @@ -150,11 +137,6 @@ def _get_default_ag_args_ensemble(cls, **kwargs) -> dict: default_ag_args_ensemble.update(extra_ag_args_ensemble) return default_ag_args_ensemble - @classmethod - def _class_tags(cls) -> dict: - # TODO: support memory estimate! - return {"can_estimate_memory_usage_static": False} - def _more_tags(self) -> dict: return {"can_refit_full": True} diff --git a/packages/tabarena/src/tabarena/models/perpetual_booster/model.py b/packages/tabarena/src/tabarena/models/perpetual_booster/model.py index c2dcbbf3f..9b807d320 100644 --- a/packages/tabarena/src/tabarena/models/perpetual_booster/model.py +++ b/packages/tabarena/src/tabarena/models/perpetual_booster/model.py @@ -19,6 +19,7 @@ class PerpetualBoosterModel(AbstractModel): ag_key = "PB" ag_name = "PerpetualBooster" + _supported_problem_types = ["binary", "multiclass", "regression"] # FIXME: random seed not supported # seed_name = "random_state" @@ -90,10 +91,6 @@ def _set_default_params(self): for param, val in default_params.items(): self._set_default_param_value(param, val) - @classmethod - def supported_problem_types(cls) -> list[str] | None: - return ["binary", "multiclass", "regression"] - def _more_tags(self) -> dict: return {"can_refit_full": True} diff --git a/packages/tabarena/src/tabarena/models/realmlp/model.py b/packages/tabarena/src/tabarena/models/realmlp/model.py index 5ebd4e96e..0c2d1f626 100644 --- a/packages/tabarena/src/tabarena/models/realmlp/model.py +++ b/packages/tabarena/src/tabarena/models/realmlp/model.py @@ -8,7 +8,6 @@ import numpy as np import pandas as pd -from autogluon.common.utils.resource_utils import ResourceManager from autogluon.tabular import __version__ from autogluon.tabular.models.abstract.abstract_torch_model import AbstractTorchModel from sklearn.impute import SimpleImputer @@ -44,6 +43,9 @@ class RealMLPModel(AbstractTorchModel): ag_name = "TA-RealMLP" ag_priority = 75 seed_name = "random_state" + _supported_problem_types = ["binary", "multiclass", "regression"] + default_num_gpus = 1 + default_resources_physical_cores_only = True def __init__(self, **kwargs): super().__init__(**kwargs) @@ -328,31 +330,9 @@ def _set_default_params(self): for param, val in default_params.items(): self._set_default_param_value(param, val) - @classmethod - def supported_problem_types(cls) -> list[str] | None: - return ["binary", "multiclass", "regression"] - def _get_default_stopping_metric(self): return self.eval_metric - def _get_default_resources(self) -> tuple[int, int]: - # Use only physical cores for better performance based on benchmarks - num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True) - - num_gpus = min(1, ResourceManager.get_gpu_count_torch(cuda_only=True)) - - return num_cpus, num_gpus - - def _estimate_memory_usage(self, X: pd.DataFrame, **kwargs) -> int: - hyperparameters = self._get_model_params() - return self.estimate_memory_usage_static( - X=X, - problem_type=self.problem_type, - num_classes=self.num_classes, - hyperparameters=hyperparameters, - **kwargs, - ) - @classmethod def _estimate_memory_usage_static( cls, @@ -424,10 +404,6 @@ def _validate_fit_memory_usage(self, mem_error_threshold: float = 1, **kwargs): **kwargs, ) - @classmethod - def _class_tags(cls) -> dict: - return {"can_estimate_memory_usage_static": True} - def _more_tags(self) -> dict: # TODO: Need to add train params support, track best epoch # How to mirror RealMLP learning rate scheduler while forcing stopping at a specific epoch? diff --git a/packages/tabarena/src/tabarena/models/sap_rpt_oss/model.py b/packages/tabarena/src/tabarena/models/sap_rpt_oss/model.py index 18f02cf73..0fc2fb1b1 100644 --- a/packages/tabarena/src/tabarena/models/sap_rpt_oss/model.py +++ b/packages/tabarena/src/tabarena/models/sap_rpt_oss/model.py @@ -21,6 +21,10 @@ class SAPRPTOSSModel(AbstractTorchModel): ag_name = "SAP-RPT-OSS" ag_priority = 65 seed_name = "random_state" + _supported_problem_types = ["binary", "multiclass", "regression"] + default_num_gpus = 1 + default_resources_physical_cores_only = True + minimum_num_gpus = 0.5 # TODO: Figure out if num_cpus could be used somewhere # TODO: Pre-download the used LM checkpoint used for the embeddings @@ -80,26 +84,6 @@ def get_device(self) -> str: def _set_device(self, device: str): self.model.model.to(device) - @classmethod - def supported_problem_types(cls) -> list[str] | None: - return ["binary", "multiclass", "regression"] - - def _get_default_resources(self) -> tuple[int, int]: - # Use only physical cores for better performance based on benchmarks - num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True) - - num_gpus = min(1, ResourceManager.get_gpu_count_torch(cuda_only=True)) - return num_cpus, num_gpus - - def get_minimum_resources( - self, - is_gpu_available: bool = False, - ) -> dict[str, int | float]: - return { - "num_cpus": 1, - "num_gpus": 0.5 if is_gpu_available else 0, - } - @classmethod def _get_default_ag_args_ensemble(cls, **kwargs) -> dict: """Set fold_fitting_strategy to sequential_local, @@ -112,11 +96,6 @@ def _get_default_ag_args_ensemble(cls, **kwargs) -> dict: default_ag_args_ensemble.update(extra_ag_args_ensemble) return default_ag_args_ensemble - @classmethod - def _class_tags(cls) -> dict: - # TODO: support memory estimate! - return {"can_estimate_memory_usage_static": False} - def _more_tags(self) -> dict: return {"can_refit_full": True} diff --git a/packages/tabarena/src/tabarena/models/tabdpt/model.py b/packages/tabarena/src/tabarena/models/tabdpt/model.py index 00b5619a3..74af660c9 100644 --- a/packages/tabarena/src/tabarena/models/tabdpt/model.py +++ b/packages/tabarena/src/tabarena/models/tabdpt/model.py @@ -2,7 +2,6 @@ from typing import TYPE_CHECKING, ClassVar -from autogluon.common.utils.resource_utils import ResourceManager from autogluon.core.constants import BINARY, MULTICLASS, REGRESSION from autogluon.features.generators import LabelEncoderFeatureGenerator from autogluon.tabular.models.abstract.abstract_torch_model import AbstractTorchModel @@ -62,6 +61,10 @@ class TabDPTModelBase(AbstractTorchModel): #: Predict-time hyperparameters accepted by this version, split by task. ``temperature`` / #: ``permute_classes`` are classification-only. Overridden per concrete subclass. _predict_hp_names: ClassVar[dict[str, tuple[str, ...]]] = {"classifier": (), "regressor": ()} + _supported_problem_types = ["binary", "multiclass", "regression"] + default_num_gpus = 1 + default_resources_physical_cores_only = True + minimum_num_gpus = 0.5 def __init__(self, **kwargs): super().__init__(**kwargs) @@ -174,23 +177,6 @@ def _set_device(self, device: str): self.model.use_flash = self._use_flash_og self.model.model.use_flash = self._use_flash_og - def _get_default_resources(self) -> tuple[int, int]: - # Use only physical cores for better performance based on benchmarks - num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True) - - num_gpus = min(1, ResourceManager.get_gpu_count_torch(cuda_only=True)) - - return num_cpus, num_gpus - - def get_minimum_resources( - self, - is_gpu_available: bool = False, - ) -> dict[str, int | float]: - return { - "num_cpus": 1, - "num_gpus": 0.5 if is_gpu_available else 0, - } - def _predict_proba(self, X, **kwargs) -> np.ndarray: X = self.preprocess(X, **kwargs) @@ -213,10 +199,6 @@ def _preprocess(self, X: pd.DataFrame, **kwargs) -> pd.DataFrame: ) return X.to_numpy() - @classmethod - def supported_problem_types(cls) -> list[str] | None: - return ["binary", "multiclass", "regression"] - def _more_tags(self) -> dict: return {"can_refit_full": True} diff --git a/packages/tabarena/src/tabarena/models/tabfm/model.py b/packages/tabarena/src/tabarena/models/tabfm/model.py index ef624d77d..7e07822c9 100644 --- a/packages/tabarena/src/tabarena/models/tabfm/model.py +++ b/packages/tabarena/src/tabarena/models/tabfm/model.py @@ -3,7 +3,6 @@ import logging from typing import TYPE_CHECKING -from autogluon.common.utils.resource_utils import ResourceManager from autogluon.tabular.models.abstract.abstract_torch_model import AbstractTorchModel if TYPE_CHECKING: @@ -108,6 +107,18 @@ class TabFMModel(AbstractTorchModel): ag_name = "TA-TabFM" ag_priority = 65 seed_name = "random_state" + _supported_problem_types = ["binary", "multiclass", "regression"] + default_num_gpus = 1 + default_resources_physical_cores_only = True + minimum_num_gpus = 1 + # Set fold_fitting_strategy to sequential_local, + # as parallel folding crashes if model weights aren't pre-downloaded. + # refit_folds avoids storing one in-context model per fold (each carries the + # full training context), refitting a single model on all data instead. + _default_ag_args_ensemble_extra = { + "fold_fitting_strategy": "sequential_local", + "refit_folds": True, + } def _fit( self, @@ -148,47 +159,8 @@ def _set_device(self, device: str): if getattr(self.model, "model", None) is not None: self.model.model.to(device) - @classmethod - def supported_problem_types(cls) -> list[str] | None: - return ["binary", "multiclass", "regression"] - - def _get_default_resources(self) -> tuple[int, int]: - # Use only physical cores for better performance based on benchmarks. - num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True) - num_gpus = min(1, ResourceManager.get_gpu_count_torch(cuda_only=True)) - return num_cpus, num_gpus - - def get_minimum_resources( - self, - is_gpu_available: bool = False, - ) -> dict[str, int | float]: - return { - "num_cpus": 1, - "num_gpus": 1 if is_gpu_available else 0, - } - - @classmethod - def _get_default_ag_args_ensemble(cls, **kwargs) -> dict: - """Set fold_fitting_strategy to sequential_local, - as parallel folding crashes if model weights aren't pre-downloaded. - refit_folds avoids storing one in-context model per fold (each carries the - full training context), refitting a single model on all data instead. - """ - default_ag_args_ensemble = super()._get_default_ag_args_ensemble(**kwargs) - default_ag_args_ensemble.update( - { - "fold_fitting_strategy": "sequential_local", - "refit_folds": True, - }, - ) - return default_ag_args_ensemble - - @classmethod - def _class_tags(cls) -> dict: - # TODO: support memory estimate! - tags = super()._class_tags() - tags["can_estimate_memory_usage_static"] = False - return tags + # TODO: support memory estimate! Implementing `_estimate_memory_usage_static` is all it + # takes; AutoGluon derives the capability from its presence. def _more_tags(self) -> dict: return {"can_refit_full": True} diff --git a/packages/tabarena/src/tabarena/models/tabicl/model.py b/packages/tabarena/src/tabarena/models/tabicl/model.py index 7dde67d91..74a342ac6 100644 --- a/packages/tabarena/src/tabarena/models/tabicl/model.py +++ b/packages/tabarena/src/tabarena/models/tabicl/model.py @@ -5,7 +5,6 @@ from typing import TYPE_CHECKING from autogluon.common.utils.pandas_utils import get_approximate_df_mem_usage -from autogluon.common.utils.resource_utils import ResourceManager from autogluon.tabular import __version__ from autogluon.tabular.models.abstract.abstract_torch_model import AbstractTorchModel @@ -35,6 +34,9 @@ class TabICLModelBase(AbstractTorchModel): default_classification_model: str | None = None default_regression_model: str | None = None + default_num_gpus = 1 + default_resources_physical_cores_only = True + minimum_num_gpus = 1 def get_model_cls(self): if self.problem_type in ["binary", "multiclass"]: @@ -146,32 +148,6 @@ def _fit( y=y, ) - def _get_default_resources(self) -> tuple[int, int]: - # Use only physical cores for better performance based on benchmarks - num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True) - - num_gpus = min(1, ResourceManager.get_gpu_count_torch(cuda_only=True)) - return num_cpus, num_gpus - - def get_minimum_resources( - self, - is_gpu_available: bool = False, - ) -> dict[str, int | float]: - return { - "num_cpus": 1, - "num_gpus": 1 if is_gpu_available else 0, - } - - def _estimate_memory_usage(self, X: pd.DataFrame, **kwargs) -> int: - hyperparameters = self._get_model_params() - return self.estimate_memory_usage_static( - X=X, - problem_type=self.problem_type, - num_classes=self.num_classes, - hyperparameters=hyperparameters, - **kwargs, - ) - # TODO: move memory estimate to specific models below. @classmethod def _estimate_memory_usage_static( @@ -220,10 +196,6 @@ def _get_default_ag_args_ensemble(cls, **kwargs) -> dict: default_ag_args_ensemble.update(extra_ag_args_ensemble) return default_ag_args_ensemble - @classmethod - def _class_tags(cls) -> dict: - return {"can_estimate_memory_usage_static": True} - def _more_tags(self) -> dict: return {"can_refit_full": True} @@ -265,10 +237,7 @@ class TabICLModel(TabICLModelBase): ag_name = "TA-TabICL" default_classification_model: str | None = "tabicl-classifier-v1.1-20250506.ckpt" - - @classmethod - def supported_problem_types(cls) -> list[str] | None: - return ["binary", "multiclass"] + _supported_problem_types = ["binary", "multiclass"] @staticmethod def checkpoint_search_space() -> list[str]: @@ -293,10 +262,7 @@ class TabICLv2Model(TabICLModelBase): default_classification_model: str | None = "tabicl-classifier-v2-20260212.ckpt" default_regression_model: str | None = "tabicl-regressor-v2-20260212.ckpt" - - @classmethod - def supported_problem_types(cls) -> list[str] | None: - return ["binary", "multiclass", "regression"] + _supported_problem_types = ["binary", "multiclass", "regression"] # TODO: search over v1 checkpoints too? @staticmethod diff --git a/packages/tabarena/src/tabarena/models/tabm/model.py b/packages/tabarena/src/tabarena/models/tabm/model.py index a14272730..5b00191e1 100644 --- a/packages/tabarena/src/tabarena/models/tabm/model.py +++ b/packages/tabarena/src/tabarena/models/tabm/model.py @@ -11,7 +11,6 @@ import time import pandas as pd -from autogluon.common.utils.resource_utils import ResourceManager from autogluon.tabular.models.abstract.abstract_torch_model import AbstractTorchModel from ._internal.tabm_utils import get_tabm_auto_batch_size @@ -33,6 +32,9 @@ class TabMModel(AbstractTorchModel): ag_key = "TA-TABM" ag_name = "TA-TabM" ag_priority = 85 + _supported_problem_types = ["binary", "multiclass", "regression"] + default_num_gpus = 1 + default_resources_physical_cores_only = True def __init__(self, **kwargs): super().__init__(**kwargs) @@ -155,30 +157,9 @@ def _set_default_params(self): for param, val in default_params.items(): self._set_default_param_value(param, val) - @classmethod - def supported_problem_types(cls) -> list[str] | None: - return ["binary", "multiclass", "regression"] - def _get_default_stopping_metric(self): return self.eval_metric - def _get_default_resources(self) -> tuple[int, int]: - # Use only physical cores for better performance based on benchmarks - num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True) - - num_gpus = min(1, ResourceManager.get_gpu_count_torch(cuda_only=True)) - return num_cpus, num_gpus - - def _estimate_memory_usage(self, X: pd.DataFrame, **kwargs) -> int: - hyperparameters = self._get_model_params() - return self.estimate_memory_usage_static( - X=X, - problem_type=self.problem_type, - num_classes=self.num_classes, - hyperparameters=hyperparameters, - **kwargs, - ) - @classmethod def _estimate_memory_usage_static( cls, @@ -291,10 +272,7 @@ def _estimate_tabm_ram( @classmethod def _class_tags(cls): - return { - "can_estimate_memory_usage_static": True, - "reset_torch_threads": True, - } + return {"reset_torch_threads": True} def _more_tags(self) -> dict: # TODO: Need to add train params support, track best epoch diff --git a/packages/tabarena/src/tabarena/models/tabpfn_3/model.py b/packages/tabarena/src/tabarena/models/tabpfn_3/model.py index 16d3e1a14..8f9b91fff 100644 --- a/packages/tabarena/src/tabarena/models/tabpfn_3/model.py +++ b/packages/tabarena/src/tabarena/models/tabpfn_3/model.py @@ -3,7 +3,6 @@ from typing import TYPE_CHECKING from autogluon.common.utils.pandas_utils import get_approximate_df_mem_usage -from autogluon.common.utils.resource_utils import ResourceManager from autogluon.tabular.models.abstract.abstract_torch_model import AbstractTorchModel if TYPE_CHECKING: @@ -21,6 +20,8 @@ class TabPFN3Model(AbstractTorchModel): default_classification_model: str | None = "tabpfn-v3-classifier-v3_default.ckpt" default_regression_model: str | None = "tabpfn-v3-regressor-v3_default.ckpt" + _supported_problem_types = ["binary", "multiclass", "regression"] + checkpoint_param_name: str = "checkpoint_per_problem_type" """Name of the optional config hyperparameter that overrides the checkpoint per problem type. @@ -36,6 +37,14 @@ class TabPFN3Model(AbstractTorchModel): """The indices of the categorical features, detected during preprocessing.""" fixed_random_state: int = 0 """Using a fixed random seed, as in TabPFN-2.6.""" + _default_auxiliary_params_extra = { + "max_classes": 160, + # Batch inference once we exceed 150_000 samples (batching starts at 150_001). + "max_batch_size": 150_000, + } + default_num_gpus = 1 + default_resources_physical_cores_only = True + minimum_num_gpus = 1 def _preprocess(self, X: pd.DataFrame, *, is_train=False, **kwargs) -> pd.DataFrame: """Minimal model-specific preprocessing to detect the indices of categorical features.""" @@ -120,25 +129,9 @@ def _set_default_params(self): for param, val in default_params.items(): self._set_default_param_value(param, val) - @classmethod - def supported_problem_types(cls) -> list[str] | None: - """Default code here supports all problem types, but can be overridden if needed.""" - return ["binary", "multiclass", "regression"] - # TODO: # - add support for many-class wrapper to remove the limit fully # - add row/col limit? - def _get_default_auxiliary_params(self) -> dict: - default_auxiliary_params = super()._get_default_auxiliary_params() - default_auxiliary_params.update( - { - "max_classes": 160, - # Batch inference once we exceed 150_000 samples (batching starts at 150_001). - "max_batch_size": 150_000, - }, - ) - return default_auxiliary_params - @classmethod def _get_default_ag_args_ensemble(cls, **kwargs) -> dict: """Ensure one fold is fit at a time and refits is enabled by default.""" @@ -175,18 +168,6 @@ def _resolve_tabpfn_device(num_gpus: int) -> str | list[str]: return [f"cuda:{i}" for i in range(num_gpus)] - def _get_default_resources(self) -> tuple[int, int]: - # Use only physical cores for better performance based on benchmarks - num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True) - num_gpus = min(1, ResourceManager.get_gpu_count_torch(cuda_only=True)) - return num_cpus, num_gpus - - def get_minimum_resources(self, is_gpu_available: bool = False) -> dict[str, int | float]: - return { - "num_cpus": 1, - "num_gpus": 1 if is_gpu_available else 0, - } - def get_device(self) -> str: base = self.model if hasattr(base, "devices_"): @@ -203,20 +184,6 @@ def get_device(self) -> str: def _set_device(self, device: str): self.model.to(device) - def _estimate_memory_usage(self, X: pd.DataFrame, **kwargs) -> int: - hyperparameters = self._get_model_params() - return self.estimate_memory_usage_static( - X=X, - problem_type=self.problem_type, - num_classes=self.num_classes, - hyperparameters=hyperparameters, - **kwargs, - ) - - @classmethod - def _class_tags(cls): - return {"can_estimate_memory_usage_static": True} - # TODO: obtain memory estimation with/without chunking @classmethod def _estimate_memory_usage_static( diff --git a/packages/tabarena/src/tabarena/models/tabpfnv2_5/model.py b/packages/tabarena/src/tabarena/models/tabpfnv2_5/model.py index 5ec09b15f..17c383f6d 100644 --- a/packages/tabarena/src/tabarena/models/tabpfnv2_5/model.py +++ b/packages/tabarena/src/tabarena/models/tabpfnv2_5/model.py @@ -6,7 +6,6 @@ from typing import TYPE_CHECKING from autogluon.common.utils.pandas_utils import get_approximate_df_mem_usage -from autogluon.common.utils.resource_utils import ResourceManager from autogluon.features.generators import LabelEncoderFeatureGenerator from autogluon.tabular.models.abstract.abstract_torch_model import AbstractTorchModel @@ -33,6 +32,8 @@ class TabPFNModel(AbstractTorchModel): ag_name = "NOTSET" ag_priority = 105 seed_name = "random_state" + _supported_problem_types = ["binary", "multiclass", "regression"] + fixed_random_state: int | None = None """If not None, this fixes the random state to a static value to avoid that the validation score is misleading for the refit model.""" @@ -40,6 +41,9 @@ class TabPFNModel(AbstractTorchModel): custom_model_dir: str | None = None default_classification_model: str | None = "NOTSET" default_regression_model: str | None = "NOTSET" + default_num_gpus = 1 + default_resources_physical_cores_only = True + minimum_num_gpus = 1 def __init__(self, **kwargs): super().__init__(**kwargs) @@ -257,23 +261,6 @@ def _fit( output_dir=Path(self.path) / "tmp_model", ) - def _get_default_resources(self) -> tuple[int, int]: - # Use only physical cores for better performance based on benchmarks - num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True) - - num_gpus = min(1, ResourceManager.get_gpu_count_torch(cuda_only=True)) - - return num_cpus, num_gpus - - def get_minimum_resources( - self, - is_gpu_available: bool = False, - ) -> dict[str, int | float]: - return { - "num_cpus": 1, - "num_gpus": 1 if is_gpu_available else 0, - } - def _set_default_params(self): default_params = { "ignore_pretraining_limits": True, # to ignore warnings and size limits @@ -302,10 +289,6 @@ def get_device(self) -> str: def _set_device(self, device: str): self._get_base_tabpfn_model().to(device) - @classmethod - def supported_problem_types(cls) -> list[str] | None: - return ["binary", "multiclass", "regression"] - @classmethod def _get_default_ag_args_ensemble(cls, **kwargs) -> dict: """Set fold_fitting_strategy to sequential_local, @@ -320,16 +303,6 @@ def _get_default_ag_args_ensemble(cls, **kwargs) -> dict: default_ag_args_ensemble.update(extra_ag_args_ensemble) return default_ag_args_ensemble - def _estimate_memory_usage(self, X: pd.DataFrame, **kwargs) -> int: - hyperparameters = self._get_model_params() - return self.estimate_memory_usage_static( - X=X, - problem_type=self.problem_type, - num_classes=self.num_classes, - hyperparameters=hyperparameters, - **kwargs, - ) - @classmethod def _estimate_memory_usage_static( cls, @@ -364,10 +337,6 @@ def _estimate_memory_usage_static( model_mem + 4 * X_mem + 2 * activation_mem + baseline_overhead_mem_est, ) - @classmethod - def _class_tags(cls): - return {"can_estimate_memory_usage_static": True} - def _more_tags(self) -> dict: return {"can_refit_full": True} @@ -393,6 +362,11 @@ class RealTabPFNv25Model(TabPFNModel): default_classification_model: str | None = "tabpfn-v2.5-classifier-v2.5_default.ckpt" default_regression_model: str | None = "tabpfn-v2.5-regressor-v2.5_default.ckpt" + _default_auxiliary_params_extra = { + "max_rows": 100_000, + "max_features": 2000, + "max_classes": 10, + } @staticmethod def extra_checkpoints_for_tuning(problem_type: str) -> list[str]: @@ -418,17 +392,6 @@ def extra_checkpoints_for_tuning(problem_type: str) -> list[str]: "tabpfn-v2.5-regressor-v2.5_variant.ckpt", ] - def _get_default_auxiliary_params(self) -> dict: - default_auxiliary_params = super()._get_default_auxiliary_params() - default_auxiliary_params.update( - { - "max_rows": 100_000, - "max_features": 2000, - "max_classes": 10, - }, - ) - return default_auxiliary_params - class TabPFNv26Model(TabPFNModel): """TabPFN-2.6 version.""" @@ -445,6 +408,14 @@ class TabPFNv26Model(TabPFNModel): default_classification_model: str | None = "tabpfn-v2.6-classifier-v2.6_default.ckpt" default_regression_model: str | None = "tabpfn-v2.6-regressor-v2.6_default.ckpt" + _max_batch_size_resolved: int | None = None + """Prediction chunk size resolved during `_fit` for large data, or None to use the + declared `ag.max_batch_size`. Runtime state rather than a `params_aux` entry, since + `params_aux` is resolved configuration and is immutable after construction.""" + _default_auxiliary_params_extra = { + "max_rows": 100_000, + } + @staticmethod def extra_checkpoints_for_tuning(problem_type: str) -> list[str]: """The list of checkpoints to use for hyperparameter tuning.""" @@ -454,15 +425,6 @@ def extra_checkpoints_for_tuning(problem_type: str) -> list[str]: # We do not put a limit on number of classes or features anymore for # the sake of the benchmark. - def _get_default_auxiliary_params(self) -> dict: - default_auxiliary_params = super()._get_default_auxiliary_params() - default_auxiliary_params.update( - { - "max_rows": 100_000, - }, - ) - return default_auxiliary_params - @classmethod def _estimate_memory_usage_static( cls, @@ -479,10 +441,15 @@ def _estimate_memory_usage_static( baseline_overhead_mem_est = 1e9 # 1 GB generic overhead return dataset_size_mem_est + baseline_overhead_mem_est + def _get_max_batch_size(self) -> int | None: + if self._max_batch_size_resolved is not None: + return self._max_batch_size_resolved + return super()._get_max_batch_size() + def _adjust_hyperparameters_for_large_data(self, *, X: pd.DataFrame, hps: dict, is_classification: bool) -> dict: if (X.shape[0] > 70_000) and (X.shape[1] > 300): print("Adjust max_batch_size and MAX_NUMBER_OF_FEATURES for large data.") - self.params_aux["max_batch_size"] = 8192 + self._max_batch_size_resolved = 8192 if "inference_config" not in hps: hps["inference_config"] = {} diff --git a/packages/tabarena/src/tabarena/models/tabpfnwide/model.py b/packages/tabarena/src/tabarena/models/tabpfnwide/model.py index cbd7547d1..00f08a37e 100644 --- a/packages/tabarena/src/tabarena/models/tabpfnwide/model.py +++ b/packages/tabarena/src/tabarena/models/tabpfnwide/model.py @@ -29,6 +29,14 @@ class TabPFNWideModel(AbstractTorchModel): ag_name = "TA-TabPFN-Wide" ag_priority = 65 seed_name = "random_state" + _supported_problem_types = ["binary", "multiclass"] + default_num_gpus = 1 + default_resources_physical_cores_only = True + minimum_num_gpus = 1 + _default_auxiliary_params_extra = { + "max_rows": 10_000, + "max_classes": 10, + } def __init__(self, **kwargs): super().__init__(**kwargs) @@ -99,10 +107,6 @@ def _set_default_params(self): for param, val in default_params.items(): self._set_default_param_value(param, val) - @classmethod - def supported_problem_types(cls) -> list[str] | None: - return ["binary", "multiclass"] - def get_device(self) -> str: if hasattr(self.model, "device"): return self.model.device @@ -112,17 +116,6 @@ def _set_device(self, device: str): if hasattr(self.model, "to"): self.model.to(device) - def _get_default_resources(self) -> tuple[int, int]: - num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True) - num_gpus = min(1, ResourceManager.get_gpu_count_torch(cuda_only=True)) - return num_cpus, num_gpus - - def get_minimum_resources(self, is_gpu_available: bool = False) -> dict[str, int | float]: - return { - "num_cpus": 1, - "num_gpus": 1 if is_gpu_available else 0, - } - @classmethod def _get_default_ag_args_ensemble(cls, **kwargs) -> dict: """Ensure one fold is fit at a time and refits is enabled by default.""" @@ -134,19 +127,5 @@ def _get_default_ag_args_ensemble(cls, **kwargs) -> dict: default_ag_args_ensemble.update(extra_ag_args_ensemble) return default_ag_args_ensemble - @classmethod - def _class_tags(cls) -> dict: - return {"can_estimate_memory_usage_static": False} - def _more_tags(self) -> dict: return {"can_refit_full": True} - - def _get_default_auxiliary_params(self) -> dict: - default_auxiliary_params = super()._get_default_auxiliary_params() - default_auxiliary_params.update( - { - "max_rows": 10_000, - "max_classes": 10, - }, - ) - return default_auxiliary_params diff --git a/packages/tabarena/src/tabarena/models/tabstar/model.py b/packages/tabarena/src/tabarena/models/tabstar/model.py index 1867288ce..a502a08ff 100644 --- a/packages/tabarena/src/tabarena/models/tabstar/model.py +++ b/packages/tabarena/src/tabarena/models/tabstar/model.py @@ -22,6 +22,10 @@ class TabSTARModel(AbstractModel): ag_name = "TabSTAR" ag_priority = 65 seed_name = "random_state" + _supported_problem_types = ["binary", "multiclass", "regression"] + default_num_gpus = 1 + default_resources_physical_cores_only = True + minimum_num_gpus = 1 def _fit( self, @@ -125,10 +129,6 @@ def _set_default_params(self): for param, val in default_params.items(): self._set_default_param_value(param, val) - @classmethod - def supported_problem_types(cls) -> list[str] | None: - return ["binary", "multiclass", "regression"] - @classmethod def warmup(cls, *, num_gpus: float | None = None, **kwargs) -> None: """Warm torch (+ CUDA context) and the TabSTAR/transformers imports (untimed, data-independent).""" @@ -137,22 +137,6 @@ def warmup(cls, *, num_gpus: float | None = None, **kwargs) -> None: warmup_torch(cuda=None if num_gpus is None else num_gpus > 0) warmup_imports("tabstar.tabstar_model") - def _get_default_resources(self) -> tuple[int, int]: - # Use only physical cores for better performance based on benchmarks - num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True) - - num_gpus = min(1, ResourceManager.get_gpu_count_torch(cuda_only=True)) - return num_cpus, num_gpus - - def get_minimum_resources( - self, - is_gpu_available: bool = False, - ) -> dict[str, int | float]: - return { - "num_cpus": 1, - "num_gpus": 1 if is_gpu_available else 0, - } - @classmethod def _get_default_ag_args_ensemble(cls, **kwargs) -> dict: """Set fold_fitting_strategy to sequential_local, @@ -166,11 +150,6 @@ def _get_default_ag_args_ensemble(cls, **kwargs) -> dict: default_ag_args_ensemble.update(extra_ag_args_ensemble) return default_ag_args_ensemble - @classmethod - def _class_tags(cls) -> dict: - # TODO: support memory estimate! - return {"can_estimate_memory_usage_static": False} - def _more_tags(self) -> dict: return {"can_refit_full": True} diff --git a/packages/tabarena/src/tabarena/models/tabswift/model.py b/packages/tabarena/src/tabarena/models/tabswift/model.py index 228d6dfc7..a4bc55e37 100644 --- a/packages/tabarena/src/tabarena/models/tabswift/model.py +++ b/packages/tabarena/src/tabarena/models/tabswift/model.py @@ -33,6 +33,18 @@ class TabSwiftModel(AbstractTorchModel): ag_name = "TA-TabSwift" ag_priority = 65 seed_name = "random_state" + _supported_problem_types = ["binary", "multiclass", "regression"] + default_num_gpus = 1 + default_resources_physical_cores_only = True + minimum_num_gpus = 1 + # Sequential fold fitting avoids contention on the shared HF checkpoint cache. + # ``refit_folds=True`` matches the other TFM wrappers (TabICL, LimiX, TabPFN-3, ...): + # for an in-context-learning model, refitting one model on all data gives faster + # inference at similar quality to the bagged ensemble. + _default_ag_args_ensemble_extra = { + "fold_fitting_strategy": "sequential_local", + "refit_folds": True, + } def __init__(self, **kwargs): super().__init__(**kwargs) @@ -158,10 +170,6 @@ def _set_default_params(self): for param, val in default_params.items(): self._set_default_param_value(param, val) - @classmethod - def supported_problem_types(cls) -> list[str] | None: - return ["binary", "multiclass", "regression"] - def get_device(self) -> str: return self.model.device_.type if self.model is not None else "cpu" @@ -171,39 +179,6 @@ def _set_device(self, device: str): if self.model.model_ is not None: self.model.model_ = self.model.model_.to(device) - def _get_default_resources(self) -> tuple[int, int]: - num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True) - num_gpus = min(1, ResourceManager.get_gpu_count_torch(cuda_only=True)) - return num_cpus, num_gpus - - def get_minimum_resources(self, is_gpu_available: bool = False) -> dict[str, int | float]: - return { - "num_cpus": 1, - "num_gpus": 1 if is_gpu_available else 0, - } - - @classmethod - def _get_default_ag_args_ensemble(cls, **kwargs) -> dict: - """Sequential fold fitting avoids contention on the shared HF checkpoint cache. - - ``refit_folds=True`` matches the other TFM wrappers (TabICL, LimiX, TabPFN-3, ...): - for an in-context-learning model, refitting one model on all data gives faster - inference at similar quality to the bagged ensemble. - """ - default_ag_args_ensemble = super()._get_default_ag_args_ensemble(**kwargs) - default_ag_args_ensemble.update( - { - "fold_fitting_strategy": "sequential_local", - "refit_folds": True, - }, - ) - return default_ag_args_ensemble - - @classmethod - def _class_tags(cls) -> dict: - # TODO: implement memory estimation and set to True - return {"can_estimate_memory_usage_static": False} - def _more_tags(self) -> dict: return {"can_refit_full": True} diff --git a/packages/tabarena/src/tabarena/models/xrfm/model.py b/packages/tabarena/src/tabarena/models/xrfm/model.py index 719d62b16..87def4a71 100644 --- a/packages/tabarena/src/tabarena/models/xrfm/model.py +++ b/packages/tabarena/src/tabarena/models/xrfm/model.py @@ -8,7 +8,6 @@ import numpy as np import pandas as pd from autogluon.common.utils.pandas_utils import get_approximate_df_mem_usage -from autogluon.common.utils.resource_utils import ResourceManager from autogluon.core.constants import MULTICLASS, REGRESSION from autogluon.core.models import AbstractModel from sklearn.impute import SimpleImputer @@ -179,6 +178,9 @@ class XRFMModel(AbstractModel): ag_key = "XRFM" ag_name = "xRFM" seed_name = "random_state" + _supported_problem_types = ["binary", "multiclass", "regression"] + default_num_gpus = 1 + default_resources_physical_cores_only = True def __init__(self, **kwargs): super().__init__(**kwargs) @@ -383,10 +385,6 @@ def _set_default_params(self): for param, val in default_params.items(): self._set_default_param_value(param, val) - @classmethod - def supported_problem_types(cls) -> list[str] | None: - return ["binary", "multiclass", "regression"] - @classmethod def warmup(cls, *, num_gpus: float | None = None, **kwargs) -> None: """Warm torch (+ CUDA context) and the xRFM library import (untimed, data-independent).""" @@ -398,14 +396,6 @@ def warmup(cls, *, num_gpus: float | None = None, **kwargs) -> None: def _get_default_stopping_metric(self): return self.eval_metric - def _get_default_resources(self) -> tuple[int, int]: - # Use only physical cores for better performance based on benchmarks - num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True) - - num_gpus = min(1, ResourceManager.get_gpu_count_torch(cuda_only=True)) - - return num_cpus, num_gpus - def _validate_fit_memory_usage( self, mem_error_threshold: float = 1.0, @@ -416,16 +406,6 @@ def _validate_fit_memory_usage( **kwargs, ) - def _estimate_memory_usage(self, X: pd.DataFrame, **kwargs) -> int: - hyperparameters = self._get_model_params() - return self.estimate_memory_usage_static( - X=X, - problem_type=self.problem_type, - num_classes=self.num_classes, - hyperparameters=hyperparameters, - **kwargs, - ) - @classmethod def _estimate_memory_usage_static( cls, @@ -459,10 +439,6 @@ def _estimate_memory_usage_static( ) # using the tree strategy caps at <40 GB return model_mem_estimate + dataset_size_mem_est - @classmethod - def _class_tags(cls) -> dict: - return {"can_estimate_memory_usage_static": True} - def _more_tags(self) -> dict: # TODO: Need to add train params support, track best epoch # How to mirror RealMLP learning rate scheduler while forcing stopping at a specific epoch? From c305ab69ff88141f7b7b05997aab4119bed63a55 Mon Sep 17 00:00:00 2001 From: LennartPurucker Date: Mon, 10 Aug 2026 09:32:15 +0000 Subject: [PATCH 3/7] maint: new sill files for the AG code --- .claude/skills/add-model/SKILL.md | 17 +- .../add-model/references/model_patterns.md | 171 ++++++++++-------- .claude/skills/benchmark-model/SKILL.md | 10 +- 3 files changed, 109 insertions(+), 89 deletions(-) diff --git a/.claude/skills/add-model/SKILL.md b/.claude/skills/add-model/SKILL.md index 324cd8649..d80f1df4f 100644 --- a/.claude/skills/add-model/SKILL.md +++ b/.claude/skills/add-model/SKILL.md @@ -81,8 +81,8 @@ machinery — `get_device()` / `_set_device()` are abstract and the load path ca (so a non-torch device string like `"gpu"` would crash it). If the model is **not** torch (JAX/Flax, or any library that manages device placement itself at the process level, e.g. via `CUDA_VISIBLE_DEVICES` / `jax.devices()`), inherit **`AbstractModel`** even though it runs on GPU, and -just add the GPU resource methods (`_get_default_resources`, `get_minimum_resources`, -`_get_default_ag_args_ensemble` with `sequential_local`, `_class_tags`, `_more_tags`) — do **not** +just add the GPU resource attributes (`default_num_gpus`, `minimum_num_gpus`, +`_default_ag_args_ensemble_extra` with `sequential_local`, plus `_more_tags`) — do **not** implement `get_device`/`_set_device`. `tabstar/model.py` (a GPU foundation model on `AbstractModel`) is the reference; `tabfm/model.py` is the JAX example. @@ -110,10 +110,17 @@ __all__ = ["gen_{ModelKey}", "{ModelKey}_info", "{ModelKey}_method_metadata"] The AutoGluon wrapper class. Use the template in `references/model_patterns.md` section "Model wrapper template". Key points: - Start with `from __future__ import annotations` - Inherit from `AbstractTorchModel` (torch-based models) or `AbstractModel` (CPU models **and non-torch GPU models** — see Step 2: JAX/Flax etc. use `AbstractModel`) -- Set `ag_key`, `ag_name`, `ag_priority = 65`, `seed_name = "random_state"` -- Implement `_fit()`, `_set_default_params()`, `supported_problem_types()` +- Set `ag_key`, `ag_name`, `ag_priority = 65`, `seed_name = "random_state"`, and + `_supported_problem_types = [...]` +- Implement `_fit()` and `_set_default_params()` +- **Declare config as class attributes, not override methods** (AutoGluon 1.6). Read + `references/model_patterns.md` → "Declare config as class attributes". Overriding + `supported_problem_types()` is the one AutoGluon actively rejects: `verify_model` raises, so the + model's smoke test fails. The others (`_get_default_resources`, `get_minimum_resources`, + `_get_default_ag_args_ensemble`, `_get_default_auxiliary_params`) still work but are the old + style. Never mutate `self.params` / `self.params_aux` after construction — it raises in 1.7. - **Honor the `_fit` contract** (read `references/model_patterns.md` → "The `_fit` contract"). The most common review findings on new wrappers are: ignoring the provided `X_val`/`y_val` (and instead auto-splitting a second holdout), ignoring `time_limit`, hardcoding the thread count instead of wiring `num_cpus`, and label-encoding + `fillna(0)` categoricals when the library handles them natively. `models/realmlp/model.py` is the reference for all of these. (In-context-learning foundation models have no train loop / no eval set, so they legitimately ignore `time_limit` + `X_val` — see `sap_rpt_oss`/`tabstar`/`tabfm`.) -- For GPU models: also implement `_get_default_resources()`, `get_minimum_resources()`, `_get_default_ag_args_ensemble()` (with `fold_fitting_strategy: sequential_local` — **and `refit_folds: True` for foundation/pre-trained TFMs**; see the "Foundation models: set `refit_folds=True`" note in `references/model_patterns.md`. From-scratch NNs omit it), `_class_tags()` (with `can_estimate_memory_usage_static: False`), `_more_tags()` (with `can_refit_full: True`). **Only torch models** (`AbstractTorchModel`) additionally implement `get_device()` / `_set_device()`; non-torch GPU models on `AbstractModel` must NOT (they have no `.to(device)`). +- For GPU models: also set `default_resources_physical_cores_only = True`, `default_num_gpus = 1`, `minimum_num_gpus = 1`, and `_default_ag_args_ensemble_extra` (with `fold_fitting_strategy: sequential_local` — **and `refit_folds: True` for foundation/pre-trained TFMs**; see the "Foundation models: set `refit_folds=True`" note in `references/model_patterns.md`. From-scratch NNs omit it), plus `_more_tags()` (with `can_refit_full: True`). Do **not** declare a `can_estimate_memory_usage_static` tag: AutoGluon derives it from whether you implement `_estimate_memory_usage_static`. **Only torch models** (`AbstractTorchModel`) additionally implement `get_device()` / `_set_device()`; non-torch GPU models on `AbstractModel` must NOT (they have no `.to(device)`). - Docstring must include: description, paper title, authors, codebase URL, license - Keep optional third-party imports (the wrapped library itself) inside `_fit` / per-method scope so importing this module never requires the optional dep at top-level - Decide the model's untimed **warm-up** (Step 3g) while you have the library docs in hand diff --git a/.claude/skills/add-model/references/model_patterns.md b/.claude/skills/add-model/references/model_patterns.md index 2820f438a..ecd939cd1 100644 --- a/.claude/skills/add-model/references/model_patterns.md +++ b/.claude/skills/add-model/references/model_patterns.md @@ -51,6 +51,21 @@ class {ClassName}Model(AbstractTorchModel): ag_priority = 65 seed_name = "random_state" + # --- AutoGluon 1.6 declarative config: attributes, not method overrides --- + _supported_problem_types = ["binary", "multiclass", "regression"] + # GPU models only: count physical cores, take one CUDA GPU, and require a whole GPU + # per fit. Drop all three for a CPU model (0 is the inherited default). + default_resources_physical_cores_only = True + default_num_gpus = 1 + minimum_num_gpus = 1 + # sequential_local avoids crashes when weights are not pre-downloaded and folds fit in + # parallel. Foundation / pre-trained models ALSO set refit_folds (see the note below); + # from-scratch NNs (TabM, RealMLP) omit it. + _default_ag_args_ensemble_extra = { + "fold_fitting_strategy": "sequential_local", + "refit_folds": True, + } + def _fit( self, X: pd.DataFrame, @@ -114,55 +129,21 @@ class {ClassName}Model(AbstractTorchModel): for param, val in default_params.items(): self._set_default_param_value(param, val) - @classmethod - def supported_problem_types(cls) -> list[str] | None: - return ["binary", "multiclass", "regression"] - def get_device(self) -> str: return self.model.device def _set_device(self, device: str): self.model.to(device) - def _get_default_resources(self) -> tuple[int, int]: - num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True) - num_gpus = min(1, ResourceManager.get_gpu_count_torch(cuda_only=True)) - return num_cpus, num_gpus - - def get_minimum_resources(self, is_gpu_available: bool = False) -> dict[str, int | float]: - return { - "num_cpus": 1, - "num_gpus": 1 if is_gpu_available else 0, - } - - @classmethod - def _get_default_ag_args_ensemble(cls, **kwargs) -> dict: - """Set fold_fitting_strategy to sequential_local to avoid crashes - if model weights aren't pre-downloaded when fitting in parallel. - - Foundation / pre-trained (in-context-learning) models ALSO set ``refit_folds=True`` - here — see the note just below. From-scratch NNs (TabM, RealMLP) omit it. - """ - default_ag_args_ensemble = super()._get_default_ag_args_ensemble(**kwargs) - default_ag_args_ensemble.update( - { - "fold_fitting_strategy": "sequential_local", - # Foundation models only — drop this line for from-scratch NNs. - "refit_folds": True, - }, - ) - return default_ag_args_ensemble - - @classmethod - def _class_tags(cls) -> dict: - # TODO: implement memory estimation and set to True - return {"can_estimate_memory_usage_static": False} - def _more_tags(self) -> dict: return {"can_refit_full": True} ``` -> **Foundation models: set `refit_folds=True` in `_get_default_ag_args_ensemble`.** Every +Note what is *not* in that body. Problem types, resources, minimum resources and ensemble args +are declared as class attributes at the top of the class (see the next section), and the +memory-estimate capability is derived rather than declared. + +> **Foundation models: set `refit_folds=True` in `_default_ag_args_ensemble_extra`.** Every > pre-trained / in-context-learning wrapper (TabPFN, TabICL, LimiX, TabDPT, SAP-RPT-OSS, > OrionMSP, TabSwift, ...) sets `refit_folds=True` *alongside* > `fold_fitting_strategy: "sequential_local"`. A TFM has no train loop, so after bagging it @@ -170,6 +151,46 @@ class {ClassName}Model(AbstractTorchModel): > ensemble. **Do not ship a TFM wrapper with only `sequential_local`** (a recurring miss). > From-scratch NNs (TabM, RealMLP) intentionally omit it and set `can_refit_full=False`. +### Declare config as class attributes (AutoGluon 1.6) + +AutoGluon 1.6 replaced a set of override methods with class attributes. Declare the attribute; +do not override the method. Only `_supported_problem_types` is enforced (AutoGluon's +`FitHelper.verify_model` raises on the old override, so `pytest -m models -k ` fails), +but the whole table is the current convention and a new wrapper should follow all of it. + +| Do not override | Declare instead | +|---|---| +| `supported_problem_types()` | `_supported_problem_types = [...]` | +| `_get_default_auxiliary_params()` | `_default_auxiliary_params_extra = {...}` | +| `_get_default_ag_args_ensemble()` | `_default_ag_args_ensemble_extra = {...}` | +| `_get_default_resources()` | `default_resources_physical_cores_only` + `default_num_gpus` | +| `get_minimum_resources()` | `minimum_num_gpus` (+ `gpu_required` if the model cannot run on CPU) | + +The two `_extra` dicts are merged base-most class first, so a subclass wins over its parent. +That covers the common `super()` + `.update({...})` shape; keep the method only when the body +genuinely needs the parent's resolved value (for example +`refit_folds=parent.pop("refit_folds", True)`) or branches on state. Overriding still works at +runtime for every row except the first, so an inherited wrapper you have not converted is not +broken, just old. + +`_default_auxiliary_params_extra` gains a typo guard the override never had: `verify_model` +checks every declared key against the known auxiliary params and fails on an unknown one. A +misspelled key in an overridden `_get_default_auxiliary_params` is silently ignored instead. + +**Memory estimation is derived, not declared.** Do not write +`_class_tags() -> {"can_estimate_memory_usage_static": ...}`; AutoGluon reads whether the class +implements `_estimate_memory_usage_static`. And do not write an `_estimate_memory_usage` that +just forwards to the static estimate — that is the base-class default. So the whole memory story +for a new model is: implement `_estimate_memory_usage_static` (and it is on), or don't (and it is +off). Keep `_class_tags` only for other tags, e.g. TabM's `reset_torch_threads`. + +**Never mutate `self.params` or `self.params_aux` after construction.** They are resolved +configuration; mutation warns in AutoGluon 1.6 and raises in 1.7. If `_fit` computes a value that +a later call needs, store it on the instance and override the getter. The +`TabPFNv26Model._max_batch_size_resolved` + `_get_max_batch_size()` pair in +`models/tabpfnv2_5/model.py` is the in-repo example; AutoGluon's own pattern references are +`AbstractModel.temperature_scalar` and `AbstractModel._get_max_batch_size`. + ### Choosing `AbstractTorchModel` vs `AbstractModel` `AbstractTorchModel` exists **only** to provide torch device management — `get_device()` / @@ -200,31 +221,21 @@ class {ClassName}Model(AbstractModel): ag_priority = 65 seed_name = "random_state" + _supported_problem_types = ["binary", "multiclass", "regression"] + default_resources_physical_cores_only = True + default_num_gpus = 1 + minimum_num_gpus = 1 + # refit_folds=True for foundation models (see note above); drop it for from-scratch NNs. + _default_ag_args_ensemble_extra = { + "fold_fitting_strategy": "sequential_local", + "refit_folds": True, + } + def _fit(self, X, y, num_cpus=1, num_gpus=0, **kwargs): # Validate GPU availability against the actual backend (e.g. jax), not torch. # Load the (pre-trained) model, build the sklearn-style wrapper, fit. ... - @classmethod - def supported_problem_types(cls): return ["binary", "multiclass", "regression"] - - def _get_default_resources(self): - num_cpus = ResourceManager.get_cpu_count(only_physical_cores=True) - num_gpus = min(1, ResourceManager.get_gpu_count_torch(cuda_only=True)) - return num_cpus, num_gpus - - def get_minimum_resources(self, is_gpu_available=False): - return {"num_cpus": 1, "num_gpus": 1 if is_gpu_available else 0} - - @classmethod - def _get_default_ag_args_ensemble(cls, **kwargs): - d = super()._get_default_ag_args_ensemble(**kwargs) - # refit_folds=True for foundation models (see note above); drop it for from-scratch NNs. - d.update({"fold_fitting_strategy": "sequential_local", "refit_folds": True}) - return d - - @classmethod - def _class_tags(cls): return {"can_estimate_memory_usage_static": False} def _more_tags(self): return {"can_refit_full": True} # NOTE: no get_device / _set_device — those are AbstractTorchModel-only. ``` @@ -241,6 +252,10 @@ class {ClassName}Model(AbstractModel): ag_name = "TA-{ModelName}" ag_priority = 65 seed_name = "random_state" + _supported_problem_types = ["binary", "multiclass", "regression"] + # CPU model: no GPU attributes. Set this only if the library benchmarks better on + # physical cores (most GBDTs and NNs do); leave it off to count logical cores. + default_resources_physical_cores_only = True def _fit( self, @@ -267,14 +282,6 @@ class {ClassName}Model(AbstractModel): for param, val in default_params.items(): self._set_default_param_value(param, val) - @classmethod - def supported_problem_types(cls) -> list[str] | None: - return ["binary", "multiclass", "regression"] - - @classmethod - def _class_tags(cls) -> dict: - return {"can_estimate_memory_usage_static": False} - def _more_tags(self) -> dict: return {"can_refit_full": True} ``` @@ -437,13 +444,19 @@ Decision order: ## Memory estimation — implement it for CPU models that fan out across folds -`can_estimate_memory_usage_static: False` with a `# TODO` is fine to *ship*, but for CPU models a -real estimate is what lets the scheduler safely fit cross-validation folds in parallel — a big -usability win that reviewers will ask for. When you can estimate peak memory from -`(n_rows, n_features, n_classes, …)`, implement `_estimate_memory_usage` / a static -`_estimate_memory_usage_static` and flip the tag to `True`. Reference: +Shipping without an estimate is fine (leave `_estimate_memory_usage_static` unimplemented and add +a `# TODO`), but for CPU models a real estimate is what lets the scheduler safely fit +cross-validation folds in parallel — a big usability win that reviewers will ask for. When you can +estimate peak memory from `(n_rows, n_features, n_classes, …)`, implement the classmethod +`_estimate_memory_usage_static`. That single method is the whole opt-in: AutoGluon 1.6 derives +`can_estimate_memory_usage_static` from its presence and the base `_estimate_memory_usage` already +forwards to it, so there is no tag to flip and no instance wrapper to write. Reference: `autogluon/tabular/src/autogluon/tabular/models/ebm/ebm_model.py` (`_estimate_memory_usage_static`). +GPU models have a parallel hook, `_estimate_gpu_memory_usage_static`, which enables VRAM safety +checks the same way. Without it AutoGluon budgets parallel folds against node RAM, which is why +benchmark runs pass `fake_memory_for_estimates` (see the `benchmark-model` skill). + --- ## hpo.py template @@ -712,11 +725,11 @@ if self.fixed_random_state is not None: ### max_rows / max_features limits ```python -def _get_default_auxiliary_params(self) -> dict: - default_auxiliary_params = super()._get_default_auxiliary_params() - default_auxiliary_params.update({ - "max_rows": 100_000, - "max_features": 2000, - }) - return default_auxiliary_params +_default_auxiliary_params_extra = { + "max_rows": 100_000, + "max_features": 2000, +} ``` +AutoGluon 1.6 also offers `min_features` / `min_cells` / `max_cells`, and reports a constraint +miss as a skip rather than a failure. Keys are validated, so a typo fails `verify_model` +instead of being silently ignored. diff --git a/.claude/skills/benchmark-model/SKILL.md b/.claude/skills/benchmark-model/SKILL.md index 844d2c584..172af2772 100644 --- a/.claude/skills/benchmark-model/SKILL.md +++ b/.claude/skills/benchmark-model/SKILL.md @@ -38,11 +38,11 @@ Given `MODEL`, read the model's contribution under `packages/tabarena/src/tabare | Derived value | Where to read it | Drives | |---|---|---| | **compute** (`"cpu"`/`"gpu"`) | `info.py` → `ModelDescriptor(compute=...)` / `MethodMetadata.compute` | `resources={"num_gpus": 1}` + `name="gpu"` for GPU; drop the override + `name="cpu"` for CPU | -| **problem types** | `model.py` → `supported_problem_types()` (a classmethod returning a subset of `["binary","multiclass","regression"]`, or `None`/**absent** = all types) | the eval `subsets`: all-types → `[[], ["binary"], ["multiclass"], ["regression"]]` (`[]` = the full set / overall leaderboard); regression-only (e.g. Nori) → `[["regression"]]` and scope setup with `task_subset=TaskSubset(subset="regression")` | +| **problem types** | `model.py` → the `_supported_problem_types` class attribute (a subset of `["binary","multiclass","regression"]`; **absent** = all types). Read it via `model_cls.supported_problem_types()` | the eval `subsets`: all-types → `[[], ["binary"], ["multiclass"], ["regression"]]` (`[]` = the full set / overall leaderboard); regression-only (e.g. Nori) → `[["regression"]]` and scope setup with `task_subset=TaskSubset(subset="regression")` | | **HPO search space** | `info.py` → `search_space` (a `gen_` generator); empty/absent ⇒ no HPO | default `NUM_CONFIGS` (foundation models with no real search space → `0`) | | **pip extra** | `info.py` → `ModelInfo(pip_extra=...)` | the "install into the run venv" reminder in the docstring + Step 4 | | **weights prefetch** | `info.py` → `ModelInfo(prefetch_weights=...)` (not `None` ⇒ foundation model) | a docstring note that the checkpoint is fetched from HF by the registry before the fits (no per-script action) | -| **static memory estimate** | `model.py` → `_estimate_memory_usage_static` / `can_estimate_memory_usage_static` | whether `fake_memory_for_estimates` can actually cap fold-parallelism (Step 1a caveat) | +| **static memory estimate** | `model.py` → whether the class implements `_estimate_memory_usage_static` (AutoGluon 1.6 derives `can_estimate_memory_usage_static` from its presence) | whether `fake_memory_for_estimates` can actually cap fold-parallelism (Step 1a caveat) | Prefer **reading these files** over importing the model (no optional deps needed). If the venv already has the model installed, you may confirm quickly with: ` -c "from tabarena.models.utils import get_model_info_from_name as g; i=g(''); print(i.method_metadata.compute, i.pip_extra, i.prefetch_weights, i.model_cls.supported_problem_types())"` @@ -63,9 +63,9 @@ it only makes the budget more conservative, which is safe on the VRAM Date: Mon, 10 Aug 2026 10:27:24 +0000 Subject: [PATCH 4/7] fix: other model failures resolved --- .../src/tabarena/models/_model_info.py | 9 +++ .../src/tabarena/models/limix/model.py | 22 +++++++ .../src/tabarena/models/tabdpt/info.py | 7 ++- .../src/tabarena/models/tabpfnv2_5/model.py | 63 ++++++++++++------- .../tabarena/tools/sync_pyproject_extras.py | 4 ++ tests/tabarena/models/smoke_configs.py | 11 +++- tests/tabarena/models/test_all_models.py | 9 ++- 7 files changed, 99 insertions(+), 26 deletions(-) diff --git a/packages/tabarena/src/tabarena/models/_model_info.py b/packages/tabarena/src/tabarena/models/_model_info.py index 53dab4633..44e155e4d 100644 --- a/packages/tabarena/src/tabarena/models/_model_info.py +++ b/packages/tabarena/src/tabarena/models/_model_info.py @@ -39,6 +39,14 @@ class ModelInfo: ``None`` (the default) means the model has nothing to prefetch (tree / linear baselines). Consumed by :func:`tabarena.models.prefetch.prefetch_weights`, the single standardized entry point for warming weights before a benchmark runs. + superseded + True when a newer entry has replaced this one and the two cannot be installed together, + because `pip_extra` pins a version the newer entry excludes. The model stays registered so + its published results remain loadable, but it is left out of the installable extras (see + :mod:`tabarena.tools.sync_pyproject_extras`) and skipped by the model smoke tests, since the + environment necessarily carries the newer entry's dependency. `TabDPT_GPU` (needs + `tabdpt<1.2`, superseded by TabDPT-Turbo) is the reference case. Not a way to park a model + that merely fails. """ model_cls: type @@ -46,3 +54,4 @@ class ModelInfo: method_metadata: MethodMetadata pip_extra: tuple[str, ...] = field(default_factory=tuple) prefetch_weights: Callable[[], None] | None = None + superseded: bool = False diff --git a/packages/tabarena/src/tabarena/models/limix/model.py b/packages/tabarena/src/tabarena/models/limix/model.py index b28d00134..7a4747a78 100644 --- a/packages/tabarena/src/tabarena/models/limix/model.py +++ b/packages/tabarena/src/tabarena/models/limix/model.py @@ -293,6 +293,15 @@ def _nan_clean_encoder_cls() -> type: ``info.py``, which would otherwise transitively import ``torch``). ``functools.cache`` gives a stable class identity across calls, which the idempotency check relies on. + A class built inside a function is normally unpicklable: pickle stores a class by + ``__module__`` + ``__qualname__`` and re-looks it up on load, and the default qualname + here would be ``_nan_clean_encoder_cls.._NaNCleanEncoder``, which pickle rejects + outright. Since AutoGluon pickles every fitted model (bagging alone pickles each fold + child back to the parent), the qualname is rewritten to a plain module-level name and the + module ``__getattr__`` below resolves it, rebuilding the class on demand in a process that + has not called this factory yet. ``functools.cache`` is what makes that lookup return the + *same* object, which is the identity check pickle performs. + The wrapper itself: LimiX's bundled 16M checkpoint starts its preprocess pipeline with a ``NanEncoder`` (`_vendor/model/encoders.py:361`) that replaces NaN cells in ``x`` with the per-column mean computed over the *train portion only* @@ -328,4 +337,17 @@ def forward(self, x): out["data"] = torch.nan_to_num(out["data"], nan=0.0, posinf=0.0, neginf=0.0) return out + # Make the class reachable as `._NaNCleanEncoder` so pickle can find it. + _NaNCleanEncoder.__qualname__ = _NaNCleanEncoder.__name__ return _NaNCleanEncoder + + +def __getattr__(name: str) -> type: + """Resolve the lazily-built ``_NaNCleanEncoder`` for pickle (PEP 562). + + Only consulted for names missing from the module namespace, so it costs nothing on a + normal attribute access and never imports ``torch`` on its own. + """ + if name == "_NaNCleanEncoder": + return _nan_clean_encoder_cls() + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/packages/tabarena/src/tabarena/models/tabdpt/info.py b/packages/tabarena/src/tabarena/models/tabdpt/info.py index 38fd498cd..7513f03fc 100644 --- a/packages/tabarena/src/tabarena/models/tabdpt/info.py +++ b/packages/tabarena/src/tabarena/models/tabdpt/info.py @@ -46,12 +46,17 @@ ) +# Pinned below 1.2: this entry runs the `tabdpt1_1` checkpoint, whose architecture config +# (8 keys, no `enc_cell_dim`) the 1.2 loader cannot read — `TabDPTModel.load` reads v1.2-only keys +# with no legacy branch. TabDPT-Turbo below is the current default and needs >=1.2.0, so the two +# cannot be installed together, which is what `superseded` records. tabdpt_info = ModelInfo( model_cls=TabDPTModel, search_space=gen_tabdpt, method_metadata=tabdpt_method_metadata, - pip_extra=("tabdpt>=1.2.0",), + pip_extra=("tabdpt<1.2",), prefetch_weights=TabDPTModel.prefetch_weights, + superseded=True, ) diff --git a/packages/tabarena/src/tabarena/models/tabpfnv2_5/model.py b/packages/tabarena/src/tabarena/models/tabpfnv2_5/model.py index 17c383f6d..9dbe3443a 100644 --- a/packages/tabarena/src/tabarena/models/tabpfnv2_5/model.py +++ b/packages/tabarena/src/tabarena/models/tabpfnv2_5/model.py @@ -446,35 +446,54 @@ def _get_max_batch_size(self) -> int | None: return self._max_batch_size_resolved return super()._get_max_batch_size() + #: Per-estimator feature cap applied on wide, large data (the checkpoint ships 500-680). + large_data_max_features_per_estimator: int = 300 + def _adjust_hyperparameters_for_large_data(self, *, X: pd.DataFrame, hps: dict, is_classification: bool) -> dict: + """Trade some feature coverage for memory on data that is both large and wide. + + Caps every preprocessor's ``max_features_per_estimator`` and shrinks the prediction + chunk size. The cap is applied to the transforms *the checkpoint itself ships*: a v2.6 + checkpoint embeds its own ``InferenceConfig``, and tabpfn passes a dict override through + ``InferenceConfig.override_with_user_input_and_resolve_auto``, so every field we do not + name keeps the checkpoint's value. + """ if (X.shape[0] > 70_000) and (X.shape[1] > 300): - print("Adjust max_batch_size and MAX_NUMBER_OF_FEATURES for large data.") + print("Adjust max_batch_size and max_features_per_estimator for large data.") self._max_batch_size_resolved = 8192 - if "inference_config" not in hps: - hps["inference_config"] = {} + inference_config = hps.get("inference_config") or {} + hps["inference_config"] = { + **inference_config, + "PREPROCESS_TRANSFORMS": self._capped_preprocess_transforms( + model_path=hps.get("model_path"), + is_classification=is_classification, + ), + } - # More extreme heuristic to avoid OOM - import dataclasses + return hps - from tabpfn.inference_config import ( - _get_v2_6_config, - v2_6_classifier_preprocessor_configs, - v2_6_regressor_preprocessor_configs, - ) + def _capped_preprocess_transforms(self, *, model_path, is_classification: bool) -> list: + """The checkpoint's own preprocessor transforms, with the feature cap lowered. - task_type = "multiclass" if is_classification else "regression" - preprocessor_configs = ( - v2_6_classifier_preprocessor_configs() if is_classification else v2_6_regressor_preprocessor_configs() - ) - preprocessor_configs = [ - dataclasses.replace(cfg, max_features_per_estimator=300) for cfg in preprocessor_configs - ] - hps["inference_config"] = _get_v2_6_config( - preprocessor_configs=preprocessor_configs, - task_type=task_type, - ) + Reads them via ``get_inference_config()``, which loads the checkpoint without fit data + for exactly this purpose. Do not rebuild the config from ``tabpfn.inference_config`` + factories: the v2.6 factories were removed once v2.6 checkpoints started embedding + their config, so reconstructing it both breaks on import and discards whatever the + checkpoint actually shipped. + """ + import dataclasses - return hps + from tabpfn import TabPFNClassifier, TabPFNRegressor + + model_cls = TabPFNClassifier if is_classification else TabPFNRegressor + probe_kwargs = {"device": "cpu"} # config only; keep the probe off the GPU + if model_path is not None: + probe_kwargs["model_path"] = model_path + transforms = model_cls(**probe_kwargs).get_inference_config().PREPROCESS_TRANSFORMS + return [ + dataclasses.replace(transform, max_features_per_estimator=self.large_data_max_features_per_estimator) + for transform in transforms + ] def prefetch_weights() -> None: diff --git a/packages/tabarena/src/tabarena/tools/sync_pyproject_extras.py b/packages/tabarena/src/tabarena/tools/sync_pyproject_extras.py index 91484b37e..225a8856c 100644 --- a/packages/tabarena/src/tabarena/tools/sync_pyproject_extras.py +++ b/packages/tabarena/src/tabarena/tools/sync_pyproject_extras.py @@ -30,6 +30,10 @@ def _expected_extras() -> dict[str, list[str]]: """ extras: dict[str, set[str]] = defaultdict(set) for info in get_model_registry().values(): + if info.superseded: + # Its pin contradicts the entry that replaced it; unioning them would make the + # extra unresolvable. See `ModelInfo.superseded`. + continue # Resolve the package short name from the model class module path. # Example: tabarena.models.ebm.model → "ebm" # For multi-class folders (tabicl), this lumps both TabICL and diff --git a/tests/tabarena/models/smoke_configs.py b/tests/tabarena/models/smoke_configs.py index e32cb565b..ff374be78 100644 --- a/tests/tabarena/models/smoke_configs.py +++ b/tests/tabarena/models/smoke_configs.py @@ -16,10 +16,15 @@ class ModelSmokeTest: hyperparameters: passed to ``FitHelper.verify_model(model_hyperparameters=...)``. problem_types: restricts the tested problem types; ``None`` tests all of binary + multiclass + regression (AutoGluon's default). + verify_single_prediction_equivalent_to_multi: whether predicting one row must match + that row's value from a batched predict (AutoGluon's default is True, at + ``atol=1e-5``). Set False only for a model shown to satisfy it on CPU and to + miss it on GPU, and say so in a comment. """ hyperparameters: dict = field(default_factory=dict) problem_types: tuple[str, ...] | None = None + verify_single_prediction_equivalent_to_multi: bool = True # Keyed by the registry method name (``MethodMetadata.method`` -- the same key @@ -45,7 +50,11 @@ class ModelSmokeTest: "TabPFN-Wide": ModelSmokeTest({"device": "cpu"}), "TabICL_GPU": ModelSmokeTest({"n_estimators": 1}), "TabICLv2": ModelSmokeTest({"n_estimators": 1}), - "TabSwift": ModelSmokeTest({"n_estimators": 1}), + # Single-row vs batched predictions agree exactly on CPU but drift ~2.4e-4 on GPU (the + # tolerance is 1e-5), so the mismatch is float non-determinism in the CUDA kernels rather + # than batch-dependent preprocessing. Verified by running `FitHelper.verify_model` with + # CUDA_VISIBLE_DEVICES="" — it passes with the check on. + "TabSwift": ModelSmokeTest({"n_estimators": 1}, verify_single_prediction_equivalent_to_multi=False), "TabSTAR": ModelSmokeTest({"max_epochs": 1}), "TabFM": ModelSmokeTest({"n_estimators": 1}), "Nori": ModelSmokeTest(problem_types=("regression",)), diff --git a/tests/tabarena/models/test_all_models.py b/tests/tabarena/models/test_all_models.py index 668fb7294..dc4cef774 100644 --- a/tests/tabarena/models/test_all_models.py +++ b/tests/tabarena/models/test_all_models.py @@ -36,12 +36,15 @@ def test_model_smoke(method: str) -> None: Run a single model during development with ``-k`` (e.g. ``pytest -m models -k TabM``). Skips (rather than fails) when a model cannot run in the current environment: - its optional dependency is not installed (``ImportError``), or it is a - GPU-only model (``compute='gpu'``) and no CUDA device is available. + its optional dependency is not installed (``ImportError``), it is a + GPU-only model (``compute='gpu'``) and no CUDA device is available, or it is + ``superseded`` and so pins a version the installed one excludes. """ info = _REGISTRY[method] if info.method_metadata.compute == "gpu" and not _CUDA_AVAILABLE: pytest.skip(f"{method}: requires a GPU (compute='gpu') and no CUDA device is available") + if info.superseded: + pytest.skip(f"{method}: superseded; its pip_extra {info.pip_extra} conflicts with the installed version") cfg = smoke_for(method) try: @@ -53,6 +56,8 @@ def test_model_smoke(method: str) -> None: } if cfg.problem_types is not None: kwargs["problem_types"] = list(cfg.problem_types) + if not cfg.verify_single_prediction_equivalent_to_multi: + kwargs["verify_single_prediction_equivalent_to_multi"] = False FitHelper.verify_model(**kwargs) except ImportError as err: pytest.skip(f"{method}: optional dependency not installed ({err})") From 74372521025152f191ece6b1f2f1612d961c1d11 Mon Sep 17 00:00:00 2001 From: LennartPurucker Date: Mon, 10 Aug 2026 10:28:49 +0000 Subject: [PATCH 5/7] test: cover the superseded pip_extra exclusion `ModelInfo.superseded` keeps a replaced entry's pin out of the pyproject extras. Without it TabDPT_GPU's `tabdpt<1.2` unions with TabDPT-Turbo's `tabdpt>=1.2.0` into one unresolvable extra, which would break `pip install tabarena[benchmark]` in CI. Co-Authored-By: Claude Opus 5 --- tests/tabarena/tools/__init__.py | 1 + .../tools/test_sync_pyproject_extras.py | 77 +++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 tests/tabarena/tools/__init__.py create mode 100644 tests/tabarena/tools/test_sync_pyproject_extras.py diff --git a/tests/tabarena/tools/__init__.py b/tests/tabarena/tools/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/tests/tabarena/tools/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/tabarena/tools/test_sync_pyproject_extras.py b/tests/tabarena/tools/test_sync_pyproject_extras.py new file mode 100644 index 000000000..37153b715 --- /dev/null +++ b/tests/tabarena/tools/test_sync_pyproject_extras.py @@ -0,0 +1,77 @@ +"""`superseded` model entries stay out of the installable pyproject extras. + +A superseded entry pins a version the entry that replaced it excludes (TabDPT_GPU needs +`tabdpt<1.2`, TabDPT-Turbo needs `>=1.2.0`), so unioning the two into one extra would make it +unresolvable and break `pip install tabarena[...]`. +""" + +from __future__ import annotations + +import pytest + +from tabarena.models._method_metadata import MethodMetadata +from tabarena.models._model_info import ModelInfo +from tabarena.tools.sync_pyproject_extras import _expected_extras + + +class _DummyModel: + # The tool derives the extra's name from the model class's module path, so spell out a + # realistic one instead of inheriting this test module's. + __module__ = "tabarena.models.dummy.model" + + +def _info(*, pip_extra: tuple[str, ...], superseded: bool = False) -> ModelInfo: + return ModelInfo( + model_cls=_DummyModel, + search_space=lambda: None, + method_metadata=MethodMetadata(method="Dummy"), + pip_extra=pip_extra, + superseded=superseded, + ) + + +@pytest.fixture +def registry(monkeypatch): + """Point `_expected_extras` at a registry we control.""" + + def _install(entries: dict[str, ModelInfo]) -> None: + monkeypatch.setattr( + "tabarena.tools.sync_pyproject_extras.get_model_registry", + lambda: entries, + ) + + return _install + + +def test_superseded_pin_is_excluded(registry): + """The current entry's pin survives; the superseded entry's conflicting pin does not.""" + registry( + { + "Current": _info(pip_extra=("dummy>=1.2.0",)), + "Old": _info(pip_extra=("dummy<1.2",), superseded=True), + } + ) + + assert _expected_extras() == {"dummy": ["dummy>=1.2.0"]} + + +def test_non_superseded_pins_are_unioned(registry): + """Without the flag the two pins are unioned, which is what makes the extra unresolvable.""" + registry( + { + "Current": _info(pip_extra=("dummy>=1.2.0",)), + "Old": _info(pip_extra=("dummy<1.2",)), + } + ) + assert _expected_extras() == {"dummy": ["dummy<1.2", "dummy>=1.2.0"]} + + +def test_folder_with_only_superseded_entries_drops_out(registry): + """A folder whose every entry is superseded contributes no extra at all.""" + registry({"Old": _info(pip_extra=("dummy<1.2",), superseded=True)}) + assert _expected_extras() == {} + + +def test_real_registry_keeps_tabdpt_installable(): + """The shipped registry must not union TabDPT's mutually exclusive pins.""" + assert _expected_extras()["tabdpt"] == ["tabdpt>=1.2.0"] From bd10e2dfef4fbfd8590de912d40b380c6692b6b0 Mon Sep 17 00:00:00 2001 From: LennartPurucker Date: Tue, 11 Aug 2026 07:58:38 +0000 Subject: [PATCH 6/7] add: ChimeraBoost 0.30.0 results as suite tabarena-2026-08-10 Register the chimeraboost_10082026 rerun on ChimeraBoost 0.30.0, whose reworked algorithm improves regression and small-data accuracy and speeds up large data (#463). Same run shape as the 0.14.1 runs, so accuracy and timings stay comparable: rank 28 -> 21 on the default leaderboard. The default arena collection now carries only the new suite; both 0.14.1 runs stay reachable through the complete collection. The two per-date superseded lists collapse into one `methods_superseded`, since the date in `methods_superseded_2026_07_13` named the rerun that displaced those entries rather than the suite they ran in, so a second dated list would have used the same suffix for the opposite meaning. Behavior is unchanged (42 current / 76 complete methods). Co-Authored-By: Claude Opus 5 --- .../src/tabarena/contexts/tabarena/methods.py | 16 +++--- .../src/tabarena/models/chimeraboost/info.py | 32 ++++++++++-- packages/tabflow_slurm/BENCHMARK_LOG.md | 52 +++++++++++++++++++ 3 files changed, 89 insertions(+), 11 deletions(-) diff --git a/packages/tabarena/src/tabarena/contexts/tabarena/methods.py b/packages/tabarena/src/tabarena/contexts/tabarena/methods.py index 80f59e557..d323e1e14 100644 --- a/packages/tabarena/src/tabarena/contexts/tabarena/methods.py +++ b/packages/tabarena/src/tabarena/contexts/tabarena/methods.py @@ -44,6 +44,7 @@ from tabarena.models.chimeraboost.info import ( chimeraboost_method_metadata, chimeraboost_new_method_metadata, + chimeraboost_v030_method_metadata, ) from tabarena.models.ebm.info import ( ebm_method_metadata as ebm_metadata, @@ -143,7 +144,7 @@ ] + methods_2025_10_20 -# The latest results for each method — exactly the methods in the TabArena paper (one per method), +# The latest results for each method — exactly the methods on the TabArena LB (one per method), # each referencing its per-model `info.py` MethodMetadata directly. TabArenaContext uses this # collection as-is (no separate name allowlist). To add a newly processed/uploaded method, import its # `info.py` metadata above and add it here under the appropriate group. @@ -157,7 +158,7 @@ tabfm_plus_method_metadata, # Default tabular models (CPU) catboost_new_method_metadata, - chimeraboost_new_method_metadata, + chimeraboost_v030_method_metadata, ebm_new_method_metadata, extra_trees_new_method_metadata, knn_metadata, @@ -197,9 +198,12 @@ ], ) -# Superseded predecessors of the tabarena-2026-07-13 reruns that appear in no dated suite list above. -methods_superseded_2026_07_13: list[MethodMetadata] = [ - chimeraboost_method_metadata, +# Per-model `info.py` metadata that a rerun has replaced. These belong to no dated suite list above, +# so they are named here to keep their hosted artifacts reachable through the complete collection. +# Append a method here whenever a rerun takes over its slot in the collection. +methods_superseded: list[MethodMetadata] = [ + chimeraboost_method_metadata, # 0.14.1 before the untimed warm-up (suite tabarena-2026-06-30) + chimeraboost_new_method_metadata, # 0.14.1 with the warm-up (suite tabarena-2026-07-13) nori_method_metadata, tabfm_method_metadata, tabswift_method_metadata, @@ -220,7 +224,7 @@ *methods_2025_11_01_ag, *methods_2026_08_05_ag, *methods_misc, - *methods_superseded_2026_07_13, + *methods_superseded, ] if m not in _collection_methods ] diff --git a/packages/tabarena/src/tabarena/models/chimeraboost/info.py b/packages/tabarena/src/tabarena/models/chimeraboost/info.py index 6700ac8b3..196927c0a 100644 --- a/packages/tabarena/src/tabarena/models/chimeraboost/info.py +++ b/packages/tabarena/src/tabarena/models/chimeraboost/info.py @@ -26,10 +26,9 @@ cache_kwargs={"bucket": "tabarena", "prefix": "cache"}, # only if uploading (s3 adds "upload_as_public": True) ) -# Rerun with the untimed environment warm-up (``ChimeraBoostModel.warmup`` pre-compiles the numba -# kernels outside the timed fit) — the ChimeraBoost used by TabArena going forward once processed -# and uploaded. ``(method, suite)`` is the unique artifact key, so the new suite keeps these -# results separate from the superseded run above. +# Superseded ChimeraBoost 0.14.1 run with the untimed environment warm-up +# (``ChimeraBoostModel.warmup`` pre-compiles the numba kernels outside the timed fit). Kept so the +# hosted artifacts stay loadable; superseded by the 0.30.0 rerun below. chimeraboost_new_method_metadata = MethodMetadata.config( method="ChimeraBoost", ag_key="CHIMERA", @@ -46,9 +45,32 @@ cache_kwargs={"bucket": "tabarena", "prefix": "cache"}, # only if uploading (s3 adds "upload_as_public": True) ) +# ChimeraBoost 0.30.0 (https://github.com/autogluon/tabarena/issues/463), which reworks the +# algorithm for better regression and small-data accuracy and better speed on large data. Same run +# shape as the 0.14.1 runs above (default config + the full 200-config search space, all splits, the +# same CPU partition), so accuracy and timings stay comparable. The ChimeraBoost used by TabArena +# going forward; ``(method, suite)`` is the unique artifact key, so the new suite keeps these +# results separate from the superseded runs. +chimeraboost_v030_method_metadata = MethodMetadata.config( + method="ChimeraBoost", + ag_key="CHIMERA", + compute="cpu", + is_bag=True, + can_hpo=True, + config_default="ChimeraBoost_c1_default_BAG_L1", + suite="tabarena-2026-08-10", + date="2026-08-10", + date_introduced="2026-05-26", + reference_url="https://github.com/bbstats/chimeraboost", + display_name="ChimeraBoost", + verified=True, + cache_type="r2", # one of: "local", "r2", "s3" + cache_kwargs={"bucket": "tabarena", "prefix": "cache"}, # only if uploading (s3 adds "upload_as_public": True) +) + chimeraboost_info = ModelInfo( model_cls=ChimeraBoostModel, search_space=gen_chimeraboost, - method_metadata=chimeraboost_new_method_metadata, + method_metadata=chimeraboost_v030_method_metadata, pip_extra=("chimeraboost>=0.30.0",), ) diff --git a/packages/tabflow_slurm/BENCHMARK_LOG.md b/packages/tabflow_slurm/BENCHMARK_LOG.md index 317f7eb49..7abbd8603 100644 --- a/packages/tabflow_slurm/BENCHMARK_LOG.md +++ b/packages/tabflow_slurm/BENCHMARK_LOG.md @@ -33,6 +33,58 @@ run against `main`. To reproduce an entry, check out its recorded **git SHA**. --- +## 2026-08-10 — chimeraboost_10082026 + +- **Model(s):** ChimeraBoost (all configs) +- **Git SHA:** `c305ab69` +- **Purpose:** Full rerun on ChimeraBoost 0.30.0 (requested in + https://github.com/autogluon/tabarena/issues/463), which reworks the algorithm for better + regression and small-data accuracy and better speed on large data. The registered baseline + (suite `tabarena-2026-07-13`) is 0.14.1, so this repeats that run's shape to keep accuracy and + timings comparable. Processed and uploaded as suite `tabarena-2026-08-10` + (`chimeraboost_v030_method_metadata`), which replaces 0.14.1 in the arena collection. +- **Notes:** Same shape as both 0.14.1 runs: full task set (all splits), default config + the full + 200-config HPO space, CPU partition `cpun416mtspotinteractive` (16 vCPUs, 64 GB RAM, 0 GB VRAM), + `memory_limit`/`num_cpus` left `None` so node values are picked up, bundle size 10. Extra dep in + the run venv: `chimeraboost>=0.30.0`. Warm-up stays untimed (`ChimeraBoostModel.warmup` + pre-compiles the numba kernels outside the fit and numba's disk cache carries them into the fold + workers). 0.30.0's `refit_full="replay"` default is inert by design here: it only fires for fits + that use ChimeraBoost's own internal split, and the wrapper passes AutoGluon's bagging validation + fold as an explicit `eval_set` — refitting on that fold would train on the rows whose predictions + become the out-of-fold predictions used for scoring and ensembling. About 10 h of cluster time + (09:34 to 19:45). + +```python +from tabarena.benchmark.experiment import TabArenaV0pt1ExperimentBundle +from tabarena.benchmark.task.metadata import TaskSubset +from tabflow_slurm import ( + GCPSlurmSetup, + ModelJob, + PathSetup, + TabArenaV0pt1BenchmarkPlan, + TabArenaV0pt1ResourcesSetup, +) + +plan = TabArenaV0pt1BenchmarkPlan( + benchmark_name="chimeraboost_10082026", + model_jobs=[ + ModelJob(models=("ChimeraBoost", "all"), name="cpu"), + ], + task_subset=TaskSubset(), # all splits of every task, as in the 0.14.1 runs + path_setup=PathSetup( + workspace="/home/lennart_priorlabs_ai/workspace/benchmarking/tabarena_workspace", + python_path="/home/lennart_priorlabs_ai/.venvs/tabarena_10082026/bin/python", + ), + experiment_bundle=TabArenaV0pt1ExperimentBundle(model_verbosity=2), + resources_setup=TabArenaV0pt1ResourcesSetup(num_cpus=None, memory_limit=None), + # Same CPU partition as the 0.14.1 runs (16 vCPUs, 64 GB RAM) for comparable timings. + scheduler_setup=GCPSlurmSetup(bundle_size=10, cpu_partition="cpun416mtspotinteractive"), +) +plan.setup_jobs() +``` + +--- + ## 2026-07-10 — tabdptturbo_10072026 - **Model(s):** TabDPT-Turbo (0 — single default config, no HPO) From 8caf1e5494b5e588168ad9058850ca47fa813442 Mon Sep 17 00:00:00 2001 From: LennartPurucker Date: Tue, 11 Aug 2026 07:58:48 +0000 Subject: [PATCH 7/7] maint: upload-method records the run in the benchmark log BENCHMARK_LOG.md entries were only ever offered by benchmark-model, at launch time and before any results exist, so runs reached upload unlogged. Give upload-method a step that checks for the entry once the run is finished, including how to recover the setup-time SHA from the reflog and the run's own timestamps. Also document the rerun case: registering a rerun is a swap plus a `methods_superseded` append, not an addition. Co-Authored-By: Claude Opus 5 --- .claude/skills/upload-method/SKILL.md | 43 +++++++++++++++++++++++++-- CLAUDE.md | 2 +- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/.claude/skills/upload-method/SKILL.md b/.claude/skills/upload-method/SKILL.md index 9ac352aa0..1b19290e3 100644 --- a/.claude/skills/upload-method/SKILL.md +++ b/.claude/skills/upload-method/SKILL.md @@ -30,6 +30,7 @@ by default**, and makes the small deterministic code edits along the way. - The model's `info.py` `MethodMetadata` — fill in the manual upload fields, fix any raw-data mismatches the inspect diff surfaces. - The arena collection registration in `methods.py` so the method appears in the benchmark. + - The run's entry in `packages/tabflow_slurm/BENCHMARK_LOG.md`, if the launch stage never wrote one. 4. **A fallback command sheet** only for steps the environment can't run. Execution requirements (check before starting; hand off the affected step if missing): @@ -194,11 +195,44 @@ edit `packages/tabarena/src/tabarena/contexts/tabarena/methods.py` (read it firs It flows into `tabarena_method_metadata_complete_collection` automatically (no separate edit). Other arenas (e.g. `beyondarena`) register in their own collection's `methods.py`. +**When the upload is a rerun of an already-registered method** (a new library version, a remeasured +run), the entry is a *swap*, not an addition: point the collection at the new metadata and append the +one it replaced to `methods_superseded`, so the predecessor's hosted artifacts stay reachable through +the complete collection. Say in the report that the default leaderboard now carries only the new run. + **Caveat to surface in the report:** the collection entry only resolves to *downloadable* artifacts once the real upload (Step 4.5) has actually run. The code edit is safe to land now, but the method won't load for others until uploaded. -## Step 6: Lint touched files +## Step 6: Record the run in the benchmark log (Claude does now) + +`packages/tabflow_slurm/BENCHMARK_LOG.md` is the committed record of every cluster run (the +`tmp_scripts/run_.py` that launched it is gitignored, so this log is the only surviving copy of +the setup). The `benchmark-model` skill merely *offers* to write the entry at launch time, before any +results exist, so in practice runs reach upload unlogged. Check here, where the run is finished: + +1. `grep -n "^## " packages/tabflow_slurm/BENCHMARK_LOG.md | head` — if the run's `benchmark_name` + is already there, nothing to do. +2. Otherwise add an entry at the **top** of the log (append-only, newest-first), following the + template in the file's "Conventions" section: `## YYYY-MM-DD — `, then + **Model(s)** / **Git SHA** / **Purpose** / **Notes**, then the verbatim plan. + +Two fields need care: + +- **Git SHA** is HEAD at *setup* time, not HEAD now. `git reflog --date=iso` shows when HEAD sat on + which commit; bracket the launch with the run's own timestamps (the earliest `results.pkl` mtime + under the run's `data/`, or the earliest file in `/slurm_out//`) and pick + the commit that was HEAD then. +- **The python block** is the launch script's `setup()` body copied verbatim, with its module-level + constants (`BENCHMARK_NAME`, `WORKSPACE`, `PYTHON_PATH`, `MODEL`, `NUM_CONFIGS`) inlined as + literals and `_path_setup()` expanded, so the snippet stands alone against its SHA. Never refactor + a neighbouring entry to match the current API. + +The launch script's module docstring usually holds the *why* (issue link, what changed versus the +previous run, partition choice, model-specific caveats) — that is the Purpose/Notes material. Add the +processed suite id and the metadata variable name too, so the log ties the run to its artifacts. + +## Step 7: Lint touched files ```bash ruff check @@ -207,8 +241,9 @@ ruff format --check Touched files are the model's `info.py` and `contexts//methods.py`. Fix anything reported (the `from __future__ import annotations` import is already present in both — don't drop it). +`BENCHMARK_LOG.md` is markdown, so ruff does not apply to it. -## Step 7: Report +## Step 8: Report Tell the maintainer: @@ -216,7 +251,9 @@ Tell the maintainer: r2 destinations confirmed after the real upload — plus any data-quality warnings from processing (e.g. `Not close TEST` prediction-fidelity lines, with affected datasets and severity). - **Edits Claude made**: the `info.py` upload fields (suite / cache_type / cache_kwargs / date / - verified, plus any inspect-diff fixes) and the `methods.py` import + collection entry. + verified, plus any inspect-diff fixes), the `methods.py` import + collection entry (and the + `methods_superseded` append, for a rerun), and the `BENCHMARK_LOG.md` entry (with the SHA it + recorded and how it was determined, since that one is inferred). - **Steps handed off** (only if the env couldn't run one): the exact command(s) from Step 4. - **Open decisions / TODOs**: whether to flip `verified` to `True` (only after sign-off), committing the working-tree edits, and — if the method should appear on the website — that `update-leaderboard` diff --git a/CLAUDE.md b/CLAUDE.md index a87996a42..10f212212 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,7 +11,7 @@ This file only documents Claude-specific extensions. - **`add-model`** — Use whenever the user asks to add/integrate/wrap a new tabular ML model. It encodes the full change: a per-model folder (`model.py` wrapper, `hpo.py` search space, `info.py` registry entry) plus edits to `models/__init__.py`, `models/utils.py`, and the `pyproject.toml` extra — and points to reference implementations for each model class (foundation, torch, sklearn-like). It also decides the model's untimed environment `warmup` (classmethod convention in `models/warmup.py`), asking the user when the library's warm-up/pre-compile entry point is unknown and reporting warm-up's limitation (main process + disk-backed caches only; parallel-fold Ray workers stay cold). The model is auto-discovered from its `info.py` and fit-tested automatically by `tests/tabarena/models/test_all_models.py` (no per-model test file); only add a `smoke_configs.py` override if its toy fit needs faster hyperparameters. - **`add-system`** — The sibling of `add-model`, for a whole *pipeline* rather than a single model: AutoML frameworks (AutoGluon, LightAutoML, FLAML), LLM-driven agents, hosted prediction APIs, or a model run through a heavier self-managing interface (TabFM+). Use it whenever the thing being added does its own model selection, tuning and ensembling inside its own budget, so inventing a TabArena search space for it makes no sense. It creates `packages/tabarena/src/tabarena/systems//` (`system.py` = the `ExternalSystemModel` subclass, `hpo.py` = the `SystemConfigGenerator`, `info.py` = `SystemInfo` + `MethodMetadata.system(...)`), and its main judgment call is `tags`: `with-llm` and `closed-source-api` decide which leaderboard entrant pools the system competes in, so the skill asks rather than guesses. Sits before `benchmark-model` in the lifecycle. - **`benchmark-model`** — Use whenever a maintainer wants to *run* an already-integrated model on the benchmark cluster (e.g. "benchmark TabM", "run Nori on the cluster", "make a setup/eval script for DenseLight"). It scaffolds a single `tmp_scripts/run_.py` with `setup` and `eval` subcommands that share one `benchmark_name` + `PathSetup` (so they can't drift), auto-filling GPU/CPU, eval `subsets`, the pip-extra install reminder, foundation-model prefetch by introspecting the model's registry `info.py` + `supported_problem_types()`, and — mandatory for every GPU model — `fake_memory_for_estimates` set to the partition's VRAM in GB (asking the user when the VRAM isn't inferable from context) so AutoGluon caps parallel bagging folds by VRAM instead of node RAM. Sits in the lifecycle between `add-model` (integrate) and `upload-method` (publish). -- **`upload-method`** — Use whenever a maintainer points at a benchmark run's output dir and wants to process / upload / register a method's results (e.g. "upload this method", "host/publish ``'s results", "register `` in the leaderboard"). By default Claude runs the whole flow itself after stating the plan (`scripts/run_process_method.py` inspect → `--process`; `scripts/run_upload_results.py` dry-run → `--no-dry-run`, background + monitors, r2 verified after upload) and lands the edits: the model's `info.py` `MethodMetadata` (suite / date / `cache_type` / `cache_kwargs` / verified, plus inspect-diff fixes) and the arena-collection registration in `contexts//methods.py`. A command sheet is only handed off when the env can't run a step (no `tabarena[benchmark]` venv / no R2 creds). Mirrors AGENTS.md → "Processing & uploading method artifacts (maintainers)". +- **`upload-method`** — Use whenever a maintainer points at a benchmark run's output dir and wants to process / upload / register a method's results (e.g. "upload this method", "host/publish ``'s results", "register `` in the leaderboard"). By default Claude runs the whole flow itself after stating the plan (`scripts/run_process_method.py` inspect → `--process`; `scripts/run_upload_results.py` dry-run → `--no-dry-run`, background + monitors, r2 verified after upload) and lands the edits: the model's `info.py` `MethodMetadata` (suite / date / `cache_type` / `cache_kwargs` / verified, plus inspect-diff fixes), the arena-collection registration in `contexts//methods.py`, and the run's `BENCHMARK_LOG.md` entry when the launch stage never wrote one. A command sheet is only handed off when the env can't run a step (no `tabarena[benchmark]` venv / no R2 creds). Mirrors AGENTS.md → "Processing & uploading method artifacts (maintainers)". - **`update-leaderboard`** — Use whenever a maintainer wants to regenerate the website artifacts and refresh a leaderboard Space with the latest results (e.g. "update the leaderboard", "regenerate the artifacts and refresh `leaderboard-testing`", "push the new results to the leaderboard"). Takes the **path to the leaderboard Space repo** (`data/` + `main.py` + its own `.venv`) as its key input. It (1) pre-checks that newly added models classify correctly in `website/website_format.py` (Foundation Model / Tree-based / … not `❓ Other`) before generating, (2) runs `scripts/run_generate_website_artifacts.py` (background + monitor; the trailing ray SIGTERM traces are harmless), (3) refreshes the Space's `data/` via delete-then-copy to avoid the stale-unzipped-`.png` gotcha, (4) bumps the version history in the Space's `website_texts.py` (new dated entry + current-version line, verified/unverified from the model's `info.py`), and (5) optionally serves the Space locally (its own `.venv`, `127.0.0.1:7860`). Last stage after `upload-method`; the maintainer commits/pushes (Git LFS + Xet). When the user describes work that matches a skill's trigger criteria, invoke the skill via the Skill tool instead of recreating the steps manually.