diff --git a/packages/tabarena/src/tabarena/benchmark/exec_models/__init__.py b/packages/tabarena/src/tabarena/benchmark/exec_models/__init__.py index 22b225558..613d532f4 100644 --- a/packages/tabarena/src/tabarena/benchmark/exec_models/__init__.py +++ b/packages/tabarena/src/tabarena/benchmark/exec_models/__init__.py @@ -18,6 +18,11 @@ AGSingleWrapper, AGWrapper, ) +from tabarena.benchmark.exec_models.autogluon_v2 import ( + AGSingleBagWrapperV2, + AGSingleWrapperV2, + AGWrapperV2, +) from tabarena.benchmark.exec_models.base import AbstractExecModel from tabarena.benchmark.exec_models.external import ExternalSystemModel from tabarena.benchmark.exec_models.registry import infer_model_cls @@ -25,8 +30,11 @@ __all__ = [ "AGModelWrapper", "AGSingleBagWrapper", + "AGSingleBagWrapperV2", "AGSingleWrapper", + "AGSingleWrapperV2", "AGWrapper", + "AGWrapperV2", "AbstractExecModel", "ExternalSystemModel", "infer_model_cls", diff --git a/packages/tabarena/src/tabarena/benchmark/exec_models/autogluon_v2.py b/packages/tabarena/src/tabarena/benchmark/exec_models/autogluon_v2.py new file mode 100644 index 000000000..ddb8d56b2 --- /dev/null +++ b/packages/tabarena/src/tabarena/benchmark/exec_models/autogluon_v2.py @@ -0,0 +1,200 @@ +"""AutoGluon exec-model wrappers that delegate non-IID validation splitting to AutoGluon. + +The V1 wrappers in :mod:`.autogluon` resolve a task's grouped / temporal validation splits in +TabArena (``resolve_validation_splits`` -> explicit ``ag_args_ensemble['custom_splits']``, or +``resolve_holdout_split`` -> explicit ``tuning_data``), then hand AutoGluon finished index lists. +AutoGluon now understands the structure itself: ``TabularPredictor.fit(validation_structure=...)`` +takes the same declarative description (``group_on`` / ``time_on`` / ``stratify_on`` / +``group_time_on``) and builds the splits internally, for both the bagged and holdout paths. + +The V2 wrappers here pass that description through instead of resolving anything, so a run +exercises AutoGluon's native implementation. Everything else is inherited unchanged: the same +model, hyperparameters, preprocessing pipeline, feature generator, and resources. + +Two things the caller must get right for the two paths to agree: + +- **Split seed.** AutoGluon's learner seeds structure splits from its own ``random_state`` + (default 0), while TabArena's splits come from ``data_foundry``, which uses 4267 internally. + These wrappers therefore set the learner's ``random_state`` to :data:`SPLIT_RANDOM_STATE`, but + only on a task that actually declares a structure -- the same seed also drives AutoGluon's + default splitter, which unstructured tasks are left to use as-is. +- **Fold sizing.** TabArena's fold counts come from its own policy + (``ValidationMetadata.resolve_number_of_splits``: a tiny-data regime below a group-instance + threshold, fixed defaults above it), which these wrappers deliberately do NOT apply -- sizing + is AutoGluon's to own, via ``validation_size_curves``. Pass explicit ``num_bag_folds`` / + ``num_bag_sets`` to take sizing out of the comparison, or configure the curves to match the + policy. ``size_validation_on_groups`` is set from the task so a curve that opts into group + sizing reads the same count TabArena would. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from loguru import logger + +from tabarena.benchmark.exec_models.autogluon import AGSingleBagWrapper, AGSingleWrapper, AGWrapper + +if TYPE_CHECKING: + import pandas as pd + from autogluon.common.utils.validation_structure import ValidationStructure + +#: ``data_foundry``'s split seed (``SPLIT_RANDOM_STATE``), which TabArena's resolved splits use. +#: AutoGluon's learner defaults to 0, so matching TabArena means passing this instead. +SPLIT_RANDOM_STATE = 4267 + + +class AGWrapperV2(AGWrapper): + """An :class:`AGWrapper` that hands the task's structure to AutoGluon instead of resolving it. + + Parameters + ---------- + temporal_forward_only: bool, default False + Ask AutoGluon for forward-chaining temporal validation instead of the default + leave-one-block-out: fold *i* validates time block *i+1* and trains only on earlier + blocks, so no fold is trained on data from after the window it is scored on. Costs the + earliest block (never validated, hence no out-of-fold prediction for those rows) and + trains each fold on less data. No effect on a task without ``time_on`` / + ``group_time_on``. Cannot be combined with stacking (AutoGluon raises). + split_random_state: int, default :data:`SPLIT_RANDOM_STATE` + Seed for AutoGluon's structure-aware splitting, injected as the learner's + ``random_state`` when a structure is declared (see :meth:`_build_predictor_args`). + Defaults to ``data_foundry``'s seed so the folds match TabArena's; a different value + gives different (equally valid) folds. + **kwargs: + As :class:`AGWrapper`. + """ + + def __init__( + self, + split_random_state: int = SPLIT_RANDOM_STATE, + temporal_forward_only: bool = False, + **kwargs, + ): + super().__init__(**kwargs) + self.split_random_state = split_random_state + self.temporal_forward_only = temporal_forward_only + + def _build_predictor_args(self, **kwargs) -> tuple[pd.DataFrame, dict, dict]: + """Seed the learner for structure-aware splitting, but only when it does any. + + The learner's ``random_state`` seeds both ``ValidationStructure``'s splits and + AutoGluon's *default* splitter. Setting it unconditionally would therefore reseed the + default splitter on tasks that have no structure to honor -- which are exactly the tasks + where V1 leaves that splitter alone, so the two paths would build different (equally + valid) folds for a purely IID task. Injecting only alongside a declared + ``validation_structure`` keeps this seed scoped to what it names. + + An explicit ``init_kwargs["learner_kwargs"]["random_state"]`` still wins. + """ + train_data, init_kwargs, fit_kwargs = super()._build_predictor_args(**kwargs) + if fit_kwargs.get("validation_structure") is not None: + init_kwargs.setdefault("learner_kwargs", {}).setdefault("random_state", self.split_random_state) + return train_data, init_kwargs, fit_kwargs + + def validation_structure(self) -> ValidationStructure | None: + """The task's :class:`ValidationStructure`, projected from its validation metadata. + + Carries only the structure -- which columns group, order, and stratify the data. The + fold *counts* are not part of it: they stay ``num_bag_folds`` / ``num_bag_sets`` (or come + from AutoGluon's ``validation_size_curves``), unlike the V1 path where TabArena's policy + decided them. + + ``None`` for a task with no grouped or temporal structure, *even when it declares + ``stratify_on``*. Such a task has no leakage to prevent, and AutoGluon's built-in bagging + already stratifies classification folds by the label -- which is what these tasks + stratify on (``stratify_on`` equals the target). Declaring the structure anyway would + take over splitting to produce equally valid but differently seeded folds, so V1 leaves + AutoGluon's default splitter alone here (``resolve_validation_splits`` returns no + ``custom_splits`` when neither ``group_on`` nor ``time_on`` is set, and + ``resolve_holdout_split`` documents the same choice for the holdout path). Taking over + only where there is leakage to fix is both what matches V1 and the better default. + + ``group_time_on`` is deliberately NOT forwarded, despite both sides having a field by + that name. They mean different things: + + - TabArena's is time *within* a group, read only by the group-aware feature generator to + order rows inside a group. ``resolve_validation_splits`` ignores it, so it has no + bearing on how V1 builds folds. + - AutoGluon's is a *split* directive: whole groups blocked in time order, mutually + exclusive with ``group_on`` / ``time_on``. + + Forwarding it would change the split structure rather than reproduce it -- a task like + ``parkinsons_biomedical_voice_measurements`` (``group_on=patient_id``, + ``group_time_on=session_number``) would be blocked by session in time order instead of + held group-disjoint by patient. TabArena has no split regime that is both grouped and + temporal (``resolve_validation_splits`` raises ``NotImplementedError`` when ``group_on`` + and ``time_on`` are both set), so nothing here needs AutoGluon's ``group_time_on``. + """ + from autogluon.common.utils.validation_structure import ValidationStructure + + metadata = self.validation_metadata + if metadata.group_on is None and metadata.time_on is None: + return None + return ValidationStructure( + group_on=metadata.group_on, + time_on=metadata.time_on, + stratify_on=metadata.stratify_on, + temporal_forward_only=self.temporal_forward_only, + # TabArena counts group instances (rather than rows) exactly when its group labels + # are per-group; mirror that so group-based sizing reads the same count. + size_validation_on_groups=metadata.group_labels == "per_group", + ) + + def _apply_validation_splits(self, fit_kwargs: dict, *, X: pd.DataFrame, y: pd.Series) -> int | None: + """Declare the structure in ``fit_kwargs`` and leave the fold counts alone. + + Overrides the V1 behavior of popping ``num_bag_folds`` / ``num_bag_sets``, running them + through TabArena's resolver, and writing back adjusted counts plus ``custom_splits``. + Here AutoGluon reads the counts as given and resolves the splits itself, so any clamping + (fewer groups than folds, temporal blocks, repeats collapsed to 1) happens inside + ``ValidationStructure.custom_splits``. + """ + num_folds = fit_kwargs.get("num_bag_folds") + if not self.use_task_specific_validation: + return num_folds + + validation_structure = self.validation_structure() + if validation_structure is None: + logger.info("Task declares no validation structure; leaving AutoGluon's defaults in place.") + return num_folds + fit_kwargs["validation_structure"] = validation_structure + logger.info( + f"Delegating validation splitting to AutoGluon: {validation_structure} " + f"(num_bag_folds={num_folds}, num_bag_sets={fit_kwargs.get('num_bag_sets')}, " + f"split_random_state={self.split_random_state})", + ) + return num_folds + + def _apply_task_specific_holdout( + self, + *, + X: pd.DataFrame, + y: pd.Series, + num_folds: int | None, + ) -> tuple[pd.DataFrame, pd.Series, None, None]: + """Leave the data whole -- AutoGluon carves any structure-aware holdout itself. + + The V1 path resolves the holdout here and passes the rows as ``tuning_data``, because a + non-bagged ``TabularPredictor`` fit ignores ``custom_splits``. With + ``validation_structure`` declared, the predictor resolves that split internally (both for + a plain holdout fit and for ``use_bag_holdout``), so carving it here would double up. + """ + return X, y, None, None + + +class AGSingleWrapperV2(AGWrapperV2, AGSingleWrapper): + """:class:`AGSingleWrapper` (one model, no weighted ensemble) on the native-structure path. + + Inherits ``fit_weighted_ensemble=False`` and ``calibrate=False`` from + :class:`AGSingleWrapper`, so a fit here is the single configured model and nothing else. + """ + + +class AGSingleBagWrapperV2(AGWrapperV2, AGSingleBagWrapper): + """:class:`AGSingleBagWrapper` (bagged, with per-child artifacts) on the native-structure path. + + The bagged wrapper used for benchmarking: AutoGluon builds the group/time-aware folds, and + the per-child out-of-fold indices and test predictions are still exposed for ensemble + simulation -- which also makes the realized folds directly comparable to the V1 path's. + """ diff --git a/packages/tabarena/src/tabarena/benchmark/experiment/__init__.py b/packages/tabarena/src/tabarena/benchmark/experiment/__init__.py index 4f96e62a2..16b142286 100644 --- a/packages/tabarena/src/tabarena/benchmark/experiment/__init__.py +++ b/packages/tabarena/src/tabarena/benchmark/experiment/__init__.py @@ -7,7 +7,9 @@ ) from tabarena.benchmark.experiment.experiment_constructor import ( AGExperiment, + AGExperimentV2, AGModelBagExperiment, + AGModelBagExperimentV2, AGModelExperiment, AGModelOuterExperiment, Experiment, @@ -38,7 +40,9 @@ __all__ = [ "AGExperiment", + "AGExperimentV2", "AGModelBagExperiment", + "AGModelBagExperimentV2", "AGModelExperiment", "AGModelOuterExperiment", "BeyondArenaExperimentBundle", diff --git a/packages/tabarena/src/tabarena/benchmark/experiment/bundle.py b/packages/tabarena/src/tabarena/benchmark/experiment/bundle.py index 2e304cbd9..bff42a385 100644 --- a/packages/tabarena/src/tabarena/benchmark/experiment/bundle.py +++ b/packages/tabarena/src/tabarena/benchmark/experiment/bundle.py @@ -8,7 +8,7 @@ from pathlib import Path from typing import TYPE_CHECKING, ClassVar -from tabarena.benchmark.experiment.experiment_constructor import Experiment +from tabarena.benchmark.experiment.experiment_constructor import AGModelBagExperiment, Experiment from tabarena.benchmark.experiment.model_constraints import ( TABICL_CONSTRAINTS, TABPFNV2_CONSTRAINTS, @@ -193,6 +193,18 @@ class TabArenaExperimentBundle: """If True, experiments built by this bundle adapt their validation data dynamically based on the task at run time (handled by the run engine). WARNING: this can overwrite the configured validation of a configuration!""" + bag_experiment_cls: type[AGModelBagExperiment] = AGModelBagExperiment + """Which bagged experiment flavour to build, and with it *who* resolves the validation + splits when ``dynamic_tabarena_validation_protocol`` is on. + + - ``AGModelBagExperiment`` (default): TabArena resolves the task's grouped / temporal folds + and hands AutoGluon explicit ``custom_splits``. + - ``AGModelBagExperimentV2``: the structure is declared to + ``TabularPredictor.fit(validation_structure=...)`` and AutoGluon resolves the folds itself. + Note that TabArena's fold-count *policy* is not applied on this path (AutoGluon owns sizing + via ``validation_size_curves``), so pass explicit fold counts or configure those curves. + + Ignored for the holdout / outer / system flavours and for pre-built ``Experiment`` entries.""" text_cache_mode: TextCacheMode = "require" """How a text task's semantic-embedding cache is treated at fit time, enforced on *every* experiment this bundle builds (bagged, holdout, and outer alike): ``require`` (default) fails @@ -694,6 +706,7 @@ def _generate_model_configs( preprocessing_pipeline=preprocessing_pipeline, fold_fitting_strategy="sequential_local" if self.sequential_local_fold_fitting else None, dynamic_tabarena_validation_protocol=self.dynamic_tabarena_validation_protocol, + experiment_cls=self.bag_experiment_cls, ) diff --git a/packages/tabarena/src/tabarena/benchmark/experiment/experiment_constructor.py b/packages/tabarena/src/tabarena/benchmark/experiment/experiment_constructor.py index 92c297a76..5c5c082d6 100644 --- a/packages/tabarena/src/tabarena/benchmark/experiment/experiment_constructor.py +++ b/packages/tabarena/src/tabarena/benchmark/experiment/experiment_constructor.py @@ -17,6 +17,7 @@ AGSingleWrapper, AGWrapper, ) +from tabarena.benchmark.exec_models.autogluon_v2 import AGSingleBagWrapperV2, AGWrapperV2 from tabarena.benchmark.exec_models.registry import infer_model_cls from tabarena.benchmark.experiment.experiment_runner import ExperimentRunner, OOFExperimentRunner from tabarena.benchmark.experiment.model_constraints import ModelConstraints @@ -704,13 +705,59 @@ def _apply_model_specific_preprocessing(self, method_kwargs: dict, model_specifi A full ``TabularPredictor`` run has no single ``model_hyperparameters``; its ``fit_kwargs["hyperparameters"]`` maps each model key to a config dict *or a list of config dicts*. Every config is wrapped, so single- and multi-model AutoGluon experiments both get - the same model-specific preprocessing as the single-model config experiments. A no-op when - there is no ``hyperparameters`` dict (e.g. a preset-driven run) — the model-agnostic feature - generator still applies. + the same model-specific preprocessing as the single-model config experiments. + + The hyperparameters are first resolved to a config dict the same way + ``TabularPredictor.fit`` resolves them, so every run shape gets per-model preprocessing + identical to a dict-driven run: + + * a *named* config (``hyperparameters`` as a string, e.g. an AutoGluon preset's portfolio + ``"noncommercial_2026_08_05"``) is expanded via the same lookup ``fit`` performs; + * a bare ``presets=`` run takes the ``hyperparameters`` entry of the preset dict(s) + (first-to-last, the last preset that sets the key wins — and, like AutoGluon, an + explicit ``hyperparameters`` key blocks the presets' value even when it is ``None``); + * no hyperparameters from either source falls back to ``fit``'s own ``"default"``. + + Without the resolution the string / preset would pass through to AutoGluon untouched and + every model would silently miss the model-specific step (e.g. raw string columns would + reach models whose encoders only handle ``category`` dtype, like TabDPT). The resolved + dict is written back to ``fit_kwargs["hyperparameters"]``; explicit fit kwargs take + precedence over presets in AutoGluon, and the value equals what the preset would have + resolved to, so the fit itself is unchanged. """ - hyperparameters = method_kwargs.get("fit_kwargs", {}).get("hyperparameters") + fit_kwargs = method_kwargs.get("fit_kwargs", {}) + hyperparameters = fit_kwargs.get("hyperparameters") + if hyperparameters is None and "hyperparameters" not in fit_kwargs and "presets" in fit_kwargs: + # Mirrors autogluon.common's `_apply_presets`: string presets resolve through the + # preset dict / aliases (and YAML paths), inline dict presets are used as-is. + from autogluon.common.utils.decorators import _resolve_preset_str + from autogluon.tabular.configs.presets_configs import tabular_presets_alias, tabular_presets_dict + + presets = fit_kwargs["presets"] + for preset in presets if isinstance(presets, list) else [presets]: + if isinstance(preset, str): + preset = _resolve_preset_str(preset, tabular_presets_dict, tabular_presets_alias) + if isinstance(preset, dict) and "hyperparameters" in preset: + hyperparameters = preset["hyperparameters"] + if hyperparameters is None: + hyperparameters = "default" # TabularPredictor.fit's fallback when nothing sets them + if isinstance(hyperparameters, str): + # Raises ValueError (listing the valid names) at experiment-preprocessing time on a + # typo, instead of deeper inside the fit. + from autogluon.tabular.configs.hyperparameter_configs import get_hyperparameter_config + + hyperparameters = get_hyperparameter_config(hyperparameters) if not isinstance(hyperparameters, dict): + warnings.warn( + f"Experiment {self.name!r} uses the tabarena_default preprocessing pipeline but " + f"its fit_kwargs['hyperparameters'] resolved to {hyperparameters!r}, which the " + f"model-specific preprocessing cannot be injected into. Models that rely on it " + f"(e.g. ordinal encoding of string columns) will misbehave; pass hyperparameters " + f"as a config dict or a named AutoGluon config string.", + stacklevel=2, + ) return + fit_kwargs["hyperparameters"] = hyperparameters for model_key, configs in hyperparameters.items(): if isinstance(configs, list): hyperparameters[model_key] = [ @@ -967,6 +1014,36 @@ def _merge_model_hyperparameters(model_hyperparameters: dict, extra_model_hyperp return merged +class AGModelBagExperimentV2(AGModelBagExperiment): + """An :class:`AGModelBagExperiment` that lets AutoGluon build the validation splits. + + Identical to its parent except for the wrapper it fixes (``AGSingleBagWrapperV2``): the + task's grouped / temporal structure is declared to ``TabularPredictor.fit`` via + ``validation_structure`` rather than resolved in TabArena into ``custom_splits``. See + :mod:`tabarena.benchmark.exec_models.autogluon_v2` for what the caller still owns (the + split seed and the fold counts). + """ + + _method_cls = AGSingleBagWrapperV2 + + +class AGExperimentV2(AGExperiment): + """An :class:`AGExperiment` that lets AutoGluon build the validation splits. + + The multi-model counterpart of :class:`AGModelBagExperimentV2`: a full ``TabularPredictor`` + fit (any number of models, presets, stacking) whose grouped / temporal validation splits are + declared via ``validation_structure`` rather than resolved in TabArena. This is the path for + running AutoGluon presets on non-IID data, where TabArena cannot pre-resolve splits per model + because one fit trains many. + + Identical to its parent except for the wrapper it fixes (``AGWrapperV2``). See + :mod:`tabarena.benchmark.exec_models.autogluon_v2` for what the caller still owns (the split + seed and the fold counts). + """ + + _method_cls = AGWrapperV2 + + class AGModelOuterExperiment(Experiment): """Fit a single AutoGluon model on all data, with no train/val split. diff --git a/packages/tabarena/src/tabarena/contexts/abstract_arena_context.py b/packages/tabarena/src/tabarena/contexts/abstract_arena_context.py index d28e44be3..43ae7f5cc 100644 --- a/packages/tabarena/src/tabarena/contexts/abstract_arena_context.py +++ b/packages/tabarena/src/tabarena/contexts/abstract_arena_context.py @@ -1238,10 +1238,15 @@ def _generate_subset_figs( if collect_composite and lb_df is not None: # Always the compact format — the composite reads its metric # columns (Elo / Impro%) from it, regardless of what format - # `website_leaderboard_kwargs` chose for the saved CSVs. + # `website_leaderboard_kwargs` chose for the saved CSVs. The + # `[X% IMPUTED]` name suffix stays off: the composite aligns + # subsets by method name, and the imputation rate differs per + # subset, so a suffixed name fragments one method into + # per-subset rows that never join. lb_compact = self.leaderboard_to_website_format( leaderboard=lb_df, compact=True, + include_imputed_in_name=False, ) if plot_tuning_trajectories: self.plot_tuning_trajectories( diff --git a/packages/tabarena/src/tabarena/utils/config_utils.py b/packages/tabarena/src/tabarena/utils/config_utils.py index 471e3d966..81ea65cae 100644 --- a/packages/tabarena/src/tabarena/utils/config_utils.py +++ b/packages/tabarena/src/tabarena/utils/config_utils.py @@ -501,6 +501,7 @@ def generate_bag_experiments( add_name_suffix_to_params: bool = True, add_seed: AddSeed = "static", fold_fitting_strategy: Literal["sequential_local"] | None = None, + experiment_cls: type[AGModelBagExperiment] = AGModelBagExperiment, **kwargs, ) -> list[AGModelBagExperiment]: """Build a bagged :class:`AGModelBagExperiment` per config (``num_bag_folds`` x ``num_bag_sets`` children). @@ -508,15 +509,19 @@ def generate_bag_experiments( Each config is first tagged with its random seed (``add_seed``, see :func:`_apply_seed_to_bag_configs`) and any ``fold_fitting_strategy``; experiments are then named ``{ag_name}{name_suffix}{name_bag_suffix}`` and built. ``**kwargs`` are forwarded to - :class:`AGModelBagExperiment` (e.g. ``preprocessing_pipeline``, - ``dynamic_tabarena_validation_protocol``). + ``experiment_cls`` (e.g. ``preprocessing_pipeline``, ``dynamic_tabarena_validation_protocol``). + + ``experiment_cls`` selects the bagged experiment flavour, and with it which exec-model wrapper + fits the folds: :class:`AGModelBagExperiment` (TabArena resolves grouped / temporal splits) or + :class:`~tabarena.benchmark.experiment.AGModelBagExperimentV2` (AutoGluon resolves them from a + declared ``validation_structure``). Everything else about the build is identical. """ configs = _apply_seed_to_bag_configs(configs, add_seed, num_bag_folds=num_bag_folds, num_bag_sets=num_bag_sets) if fold_fitting_strategy is not None: configs = [add_fold_fitting_strategy(config, fold_fitting_strategy=fold_fitting_strategy) for config in configs] def build_experiment(name: str, config: dict) -> AGModelBagExperiment: - return AGModelBagExperiment( + return experiment_cls( name=name, model_cls=model_cls, model_hyperparameters=config, diff --git a/tests/tabarena/benchmark/experiment/test_build_config_pipeline.py b/tests/tabarena/benchmark/experiment/test_build_config_pipeline.py index 289ae33c1..728014625 100644 --- a/tests/tabarena/benchmark/experiment/test_build_config_pipeline.py +++ b/tests/tabarena/benchmark/experiment/test_build_config_pipeline.py @@ -11,6 +11,8 @@ import copy +import pytest + from tabarena.benchmark.experiment import ( ModelConstraints, TabArenaExperimentBundle, @@ -214,8 +216,92 @@ def test_autogluon_experiment_tabarena_default_multi_model(): assert _has_model_specific(hp["XGB"]) -def test_autogluon_experiment_tabarena_default_preset_only_applies_model_agnostic(): - """With no `hyperparameters` dict (e.g. preset-driven), only the model-agnostic step applies.""" +def _all_wrapped(hyperparameters: dict) -> bool: + return all( + _has_model_specific(config) + for configs in hyperparameters.values() + for config in (configs if isinstance(configs, list) else [configs]) + ) + + +def test_autogluon_experiment_tabarena_default_bare_preset_resolves_its_hyperparameters(): + """A bare preset run (alias included) takes the `hyperparameters` entry from the preset dict, + expands it, and wraps every config — same outcome as passing the portfolio explicitly. + """ + from autogluon.tabular.configs.hyperparameter_configs import get_hyperparameter_config + from autogluon.tabular.configs.presets_configs import tabular_presets_dict + + rmk = _apply_tabarena_default({"presets": "extreme"}) # alias of extreme_quality + hyperparameters = rmk["fit_kwargs"]["hyperparameters"] + expected = get_hyperparameter_config(tabular_presets_dict["extreme_quality"]["hyperparameters"]) + assert set(hyperparameters) == set(expected) + assert _all_wrapped(hyperparameters) + + +def test_autogluon_experiment_tabarena_default_preset_without_hyperparameters_uses_default(): + """A preset that sets no `hyperparameters` (e.g. medium_quality) falls back to AutoGluon's + `"default"` config, expanded and wrapped — matching what `TabularPredictor.fit` would run. + """ + from autogluon.tabular.configs.hyperparameter_configs import get_hyperparameter_config + rmk = _apply_tabarena_default({"presets": "medium_quality"}) + hyperparameters = rmk["fit_kwargs"]["hyperparameters"] + assert set(hyperparameters) == set(get_hyperparameter_config("default")) + assert _all_wrapped(hyperparameters) + + +def test_autogluon_experiment_tabarena_default_explicit_none_blocks_the_preset(): + """An explicit `hyperparameters=None` key beats the preset's value in AutoGluon + (`apply_presets` only fills missing keys), so it resolves to `"default"`, not the preset's + portfolio. + """ + from autogluon.tabular.configs.hyperparameter_configs import get_hyperparameter_config + + rmk = _apply_tabarena_default({"presets": "extreme", "hyperparameters": None}) + hyperparameters = rmk["fit_kwargs"]["hyperparameters"] + assert set(hyperparameters) == set(get_hyperparameter_config("default")) + assert _all_wrapped(hyperparameters) + + +def test_autogluon_experiment_tabarena_default_last_preset_wins(): + """With a preset list, the last preset that sets `hyperparameters` wins (AutoGluon's + first-to-last merge). + """ + from autogluon.tabular.configs.hyperparameter_configs import get_hyperparameter_config + from autogluon.tabular.configs.presets_configs import tabular_presets_dict + + rmk = _apply_tabarena_default({"presets": ["medium_quality", "extreme"]}) + expected = get_hyperparameter_config(tabular_presets_dict["extreme_quality"]["hyperparameters"]) + assert set(rmk["fit_kwargs"]["hyperparameters"]) == set(expected) + + +def test_autogluon_experiment_tabarena_default_unresolvable_hyperparameters_warns(): + """Hyperparameters of a type the injection cannot handle warn instead of passing silently.""" + with pytest.warns(UserWarning, match="model-specific"): + rmk = _apply_tabarena_default({"hyperparameters": 123}) assert rmk["fit_kwargs"]["feature_generator_cls"] is TabArenaModelAgnosticPreprocessing - assert "hyperparameters" not in rmk["fit_kwargs"] # nothing to wrap; no crash + assert rmk["fit_kwargs"]["hyperparameters"] == 123 # left untouched for AutoGluon to reject + + +def test_autogluon_experiment_tabarena_default_named_config_is_expanded(): + """A named AutoGluon config string (what the shipped presets carry, e.g. + `"noncommercial_2026_08_05"`) is expanded to its config dict and every config is wrapped, + so a preset-driven run gets the same model-specific preprocessing as a dict-driven run. + """ + from autogluon.tabular.configs.hyperparameter_configs import get_hyperparameter_config + + rmk = _apply_tabarena_default({"hyperparameters": "very_light"}) + hyperparameters = rmk["fit_kwargs"]["hyperparameters"] + assert isinstance(hyperparameters, dict) + assert set(hyperparameters) == set(get_hyperparameter_config("very_light")) + for configs in hyperparameters.values(): + for config in configs if isinstance(configs, list) else [configs]: + assert _has_model_specific(config) + + +def test_autogluon_experiment_tabarena_default_unknown_named_config_raises(): + """A typo in the named config fails at experiment-preprocessing time with the valid names, + not node-side inside the fit. + """ + with pytest.raises(ValueError, match="not_a_real_config"): + _apply_tabarena_default({"hyperparameters": "not_a_real_config"}) diff --git a/tests/tabarena/benchmark/experiment/test_bundle.py b/tests/tabarena/benchmark/experiment/test_bundle.py index 3f6c68c30..479a4b555 100644 --- a/tests/tabarena/benchmark/experiment/test_bundle.py +++ b/tests/tabarena/benchmark/experiment/test_bundle.py @@ -15,6 +15,7 @@ import pytest from tabarena.benchmark.experiment import ( + AGModelBagExperiment, BeyondArenaExperimentBundle, TabArenaExperimentBundle, TabArenaV0pt1ExperimentBundle, @@ -43,6 +44,7 @@ "adapt_num_folds_to_n_classes", "shuffle_features", "dynamic_tabarena_validation_protocol", + "bag_experiment_cls", "text_cache_mode", "custom_model_constraints", } @@ -61,6 +63,7 @@ "verbosity": 2, "model_verbosity": 4, "custom_model_constraints": {}, + "bag_experiment_cls": AGModelBagExperiment, } # Hardcoded full post-init state for each subclass instantiated with no args.