diff --git a/docs/genai-features.md b/docs/genai-features.md index 2fd0129..f2c5889 100644 --- a/docs/genai-features.md +++ b/docs/genai-features.md @@ -10,6 +10,14 @@ just generates candidates, and the data does the rest.

GenAI proposes feature code; classical cross-validation measures the lift and the gate decides what survives

+!!! firefly "Wired into AutoML" + + When a `FeatureEngineerPort` is present in the container (i.e. GenAI is enabled), + [`AutoML.from_context(app)`](automl.md) runs this propose → execute → measure → gate loop + **before** model selection and trains on the engineered features — the gate's accepted/rejected + audit is threaded into `result.extras["feature_engineering"]`. Classical-first stays the default: + with GenAI off, `AutoML` is unchanged. + ## The loop `GenAIFeatureEngineer` runs **propose → execute → measure → gate**: diff --git a/src/fireflyframework_datascience/automl/facade.py b/src/fireflyframework_datascience/automl/facade.py index 54e5624..dec7cb6 100644 --- a/src/fireflyframework_datascience/automl/facade.py +++ b/src/fireflyframework_datascience/automl/facade.py @@ -17,6 +17,7 @@ from fireflyframework_datascience.datasets import Dataset from fireflyframework_datascience.evaluation import MetricsEvaluatorPort from fireflyframework_datascience.explainability import ExplainerPort +from fireflyframework_datascience.features import FeatureEngineerPort from fireflyframework_datascience.models import Model, TrainerPort from fireflyframework_datascience.search import SearchPolicyPort from fireflyframework_datascience.tracking import TrackerPort @@ -37,6 +38,7 @@ def __init__( validator: ValidatorPort | None = None, tracker: TrackerPort | None = None, explainer: ExplainerPort | None = None, + feature_engineer: FeatureEngineerPort | None = None, cv: int = 5, n_trials: int = 20, random_state: int = 42, @@ -47,6 +49,7 @@ def __init__( self._validator = validator self._tracker = tracker self._explainer = explainer + self._feature_engineer = feature_engineer self._cv = cv self._n_trials = n_trials self._random_state = random_state @@ -63,6 +66,7 @@ def from_context(cls, context: Any, **overrides: Any) -> AutoML: validator=container.resolve_optional(ValidatorPort), tracker=container.resolve_optional(TrackerPort), explainer=container.resolve_optional(ExplainerPort), + feature_engineer=container.resolve_optional(FeatureEngineerPort), **overrides, ) @@ -74,6 +78,13 @@ def fit(self, dataset: Dataset, *, task: TaskType | None = None, metric: str | N if self._validator is not None: self._validator.validate(dataset.X, dataset.y).raise_if_failed() + # GenAI feature engineering (when wired): the LLM proposes, the gate keeps only measured wins, + # and the engineered dataset feeds model selection. Off unless a FeatureEngineerPort is present. + engineering = None + if self._feature_engineer is not None: + engineering = self._feature_engineer.engineer(dataset) + dataset = engineering.dataset + candidates = [t for t in self._trainers if t.supports(task)] if not candidates: from fireflyframework_datascience.core.exceptions import FireflyDataScienceError @@ -118,6 +129,7 @@ def fit(self, dataset: Dataset, *, task: TaskType | None = None, metric: str | N evaluator=self._evaluator, cv_scoring=scoring, explainer=self._explainer, + extras={"feature_engineering": engineering} if engineering is not None else {}, ) # -- internals -------------------------------------------------------- diff --git a/tests/test_automl_genai_wiring.py b/tests/test_automl_genai_wiring.py new file mode 100644 index 0000000..0233eff --- /dev/null +++ b/tests/test_automl_genai_wiring.py @@ -0,0 +1,63 @@ +# Copyright 2026 Firefly Software Foundation. +"""When a FeatureEngineerPort is wired, AutoML.fit must engineer features first and train on them. + +This closes the integrity gap where enabling GenAI did not actually change AutoML: the engineer was a +registered bean the facade never consumed. Real data (breast_cancer), LLM-free (StaticFeatureProposer). +""" + +from __future__ import annotations + + +def test_automl_applies_a_wired_feature_engineer() -> None: + from sklearn.datasets import load_breast_cancer + + from fireflyframework_datascience.automl import AutoML + from fireflyframework_datascience.core.types import TaskType + from fireflyframework_datascience.datasets import Dataset + from fireflyframework_datascience.features import CostBenefitGate, FeatureProposal, StaticFeatureProposer + from fireflyframework_datascience.features.genai import GenAIFeatureEngineer + + raw = load_breast_cancer(as_frame=True) + ds = Dataset( + name="breast_cancer", + X=raw.data, + y=raw.target, + task=TaskType.BINARY, + target_name="target", + feature_names=list(raw.data.columns), + ) + proposer = StaticFeatureProposer( + [FeatureProposal("rad_area", "df['rad_area'] = df['mean radius'] * df['mean area']", "interaction")] + ) + # A permissive gate so the test asserts the *wiring* (engineer runs, its dataset is used), not the + # lift magnitude — the accept/reject logic is covered by the feature-engineering tests. + engineer = GenAIFeatureEngineer(proposer, gate=CostBenefitGate(min_gain=-1.0), cv=3) + + result = AutoML(feature_engineer=engineer, cv=3, n_trials=1, random_state=0).fit(ds) + + # the engineered column was used to train the winning model + assert "rad_area" in result.best_model.feature_names + # and the audit trail is threaded into the result + engineering = result.extras["feature_engineering"] + assert any(a.proposal.name == "rad_area" for a in engineering.accepted) + + +def test_automl_without_feature_engineer_has_no_engineering_extra() -> None: + from sklearn.datasets import load_breast_cancer + + from fireflyframework_datascience.automl import AutoML + from fireflyframework_datascience.core.types import TaskType + from fireflyframework_datascience.datasets import Dataset + + raw = load_breast_cancer(as_frame=True) + ds = Dataset( + name="breast_cancer", + X=raw.data, + y=raw.target, + task=TaskType.BINARY, + target_name="target", + feature_names=list(raw.data.columns), + ) + result = AutoML(cv=3, n_trials=1, random_state=0).fit(ds) + assert "feature_engineering" not in result.extras + assert "rad_area" not in result.best_model.feature_names