Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions docs/genai-features.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,14 @@ just generates candidates, and the data does the rest.

<p align="center"><img src="img/genai-classical-fusion.svg" alt="GenAI proposes feature code; classical cross-validation measures the lift and the gate decides what survives" width="100%"></p>

!!! firefly "Wired into AutoML"

When a `FeatureEngineerPort` is present in the container (i.e. GenAI is enabled),
[`AutoML.from_context(app)`](automl.md) runs this propose → execute → measure → gate loop
**before** model selection and trains on the engineered features — the gate's accepted/rejected
audit is threaded into `result.extras["feature_engineering"]`. Classical-first stays the default:
with GenAI off, `AutoML` is unchanged.

## The loop

`GenAIFeatureEngineer` runs **propose → execute → measure → gate**:
Expand Down
12 changes: 12 additions & 0 deletions src/fireflyframework_datascience/automl/facade.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from fireflyframework_datascience.datasets import Dataset
from fireflyframework_datascience.evaluation import MetricsEvaluatorPort
from fireflyframework_datascience.explainability import ExplainerPort
from fireflyframework_datascience.features import FeatureEngineerPort
from fireflyframework_datascience.models import Model, TrainerPort
from fireflyframework_datascience.search import SearchPolicyPort
from fireflyframework_datascience.tracking import TrackerPort
Expand All @@ -37,6 +38,7 @@ def __init__(
validator: ValidatorPort | None = None,
tracker: TrackerPort | None = None,
explainer: ExplainerPort | None = None,
feature_engineer: FeatureEngineerPort | None = None,
cv: int = 5,
n_trials: int = 20,
random_state: int = 42,
Expand All @@ -47,6 +49,7 @@ def __init__(
self._validator = validator
self._tracker = tracker
self._explainer = explainer
self._feature_engineer = feature_engineer
self._cv = cv
self._n_trials = n_trials
self._random_state = random_state
Expand All @@ -63,6 +66,7 @@ def from_context(cls, context: Any, **overrides: Any) -> AutoML:
validator=container.resolve_optional(ValidatorPort),
tracker=container.resolve_optional(TrackerPort),
explainer=container.resolve_optional(ExplainerPort),
feature_engineer=container.resolve_optional(FeatureEngineerPort),
**overrides,
)

Expand All @@ -74,6 +78,13 @@ def fit(self, dataset: Dataset, *, task: TaskType | None = None, metric: str | N
if self._validator is not None:
self._validator.validate(dataset.X, dataset.y).raise_if_failed()

# GenAI feature engineering (when wired): the LLM proposes, the gate keeps only measured wins,
# and the engineered dataset feeds model selection. Off unless a FeatureEngineerPort is present.
engineering = None
if self._feature_engineer is not None:
engineering = self._feature_engineer.engineer(dataset)
dataset = engineering.dataset

candidates = [t for t in self._trainers if t.supports(task)]
if not candidates:
from fireflyframework_datascience.core.exceptions import FireflyDataScienceError
Expand Down Expand Up @@ -118,6 +129,7 @@ def fit(self, dataset: Dataset, *, task: TaskType | None = None, metric: str | N
evaluator=self._evaluator,
cv_scoring=scoring,
explainer=self._explainer,
extras={"feature_engineering": engineering} if engineering is not None else {},
)

# -- internals --------------------------------------------------------
Expand Down
63 changes: 63 additions & 0 deletions tests/test_automl_genai_wiring.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
# Copyright 2026 Firefly Software Foundation.
"""When a FeatureEngineerPort is wired, AutoML.fit must engineer features first and train on them.

This closes the integrity gap where enabling GenAI did not actually change AutoML: the engineer was a
registered bean the facade never consumed. Real data (breast_cancer), LLM-free (StaticFeatureProposer).
"""

from __future__ import annotations


def test_automl_applies_a_wired_feature_engineer() -> None:
from sklearn.datasets import load_breast_cancer

from fireflyframework_datascience.automl import AutoML
from fireflyframework_datascience.core.types import TaskType
from fireflyframework_datascience.datasets import Dataset
from fireflyframework_datascience.features import CostBenefitGate, FeatureProposal, StaticFeatureProposer
from fireflyframework_datascience.features.genai import GenAIFeatureEngineer

raw = load_breast_cancer(as_frame=True)
ds = Dataset(
name="breast_cancer",
X=raw.data,
y=raw.target,
task=TaskType.BINARY,
target_name="target",
feature_names=list(raw.data.columns),
)
proposer = StaticFeatureProposer(
[FeatureProposal("rad_area", "df['rad_area'] = df['mean radius'] * df['mean area']", "interaction")]
)
# A permissive gate so the test asserts the *wiring* (engineer runs, its dataset is used), not the
# lift magnitude — the accept/reject logic is covered by the feature-engineering tests.
engineer = GenAIFeatureEngineer(proposer, gate=CostBenefitGate(min_gain=-1.0), cv=3)

result = AutoML(feature_engineer=engineer, cv=3, n_trials=1, random_state=0).fit(ds)

# the engineered column was used to train the winning model
assert "rad_area" in result.best_model.feature_names
# and the audit trail is threaded into the result
engineering = result.extras["feature_engineering"]
assert any(a.proposal.name == "rad_area" for a in engineering.accepted)


def test_automl_without_feature_engineer_has_no_engineering_extra() -> None:
from sklearn.datasets import load_breast_cancer

from fireflyframework_datascience.automl import AutoML
from fireflyframework_datascience.core.types import TaskType
from fireflyframework_datascience.datasets import Dataset

raw = load_breast_cancer(as_frame=True)
ds = Dataset(
name="breast_cancer",
X=raw.data,
y=raw.target,
task=TaskType.BINARY,
target_name="target",
feature_names=list(raw.data.columns),
)
result = AutoML(cv=3, n_trials=1, random_state=0).fit(ds)
assert "feature_engineering" not in result.extras
assert "rad_area" not in result.best_model.feature_names
Loading