From ad17952367a3bf28bd43cc210fb4237dd257abac Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Tue, 25 Aug 2026 19:36:24 -0700 Subject: [PATCH 1/3] Update C++ APIs to accept missing_go_to_left --- include/treelite/c_api.h | 54 +++++++++++++--- include/treelite/model_loader.h | 24 +++++-- include/treelite/tree.h | 7 +- src/c_api/sklearn.cc | 47 ++++++++++++-- src/model_loader/sklearn.cc | 43 ------------ src/model_loader/sklearn_bulk.cc | 108 ++++++++++++++++++++++++++----- 6 files changed, 202 insertions(+), 81 deletions(-) diff --git a/include/treelite/c_api.h b/include/treelite/c_api.h index 0b6198e1..b1b7c053 100644 --- a/include/treelite/c_api.h +++ b/include/treelite/c_api.h @@ -177,6 +177,14 @@ TREELITE_DLL int TreeliteLoadLightGBMModelFromString( * \defgroup sklearn C API: Model loaders for scikit-learn * \{ */ +/*! + * \brief Deprecated. Please use \ref TreeliteLoadSKLearnRandomForestRegressorEx instead. + */ +TREELITE_DLL int TreeliteLoadSKLearnRandomForestRegressor(int n_estimators, int n_features, + int n_targets, int64_t const* node_count, int64_t const** children_left, + int64_t const** children_right, int64_t const** feature, double const** threshold, + double const** value, int64_t const** n_node_samples, double const** weighted_n_node_samples, + double const** impurity, TreeliteModelHandle* out); /*! * \brief Load a scikit-learn RandomForestRegressor model from a collection of arrays. Refer to * https://scikit-learn.org/stable/auto_examples/tree/plot_unveil_tree_structure.html to @@ -196,6 +204,9 @@ TREELITE_DLL int TreeliteLoadLightGBMModelFromString( * the i-th tree. This is only defined if node k is an internal (non-leaf) node. * \param value value[i][k] stores the leaf output of node k of the i-th tree. This is only defined * if node k is a leaf node. + * \param missing_go_to_left missing_go_to_left[i][k] stores the default direction for the missing + * value at node k of the i-th tree. This is only defined if node k is an + * internal (non-leaf) node. * \param n_node_samples n_node_samples[i][k] stores the number of data samples associated with * node k of the i-th tree. * \param weighted_n_node_samples weighted_n_node_samples[i][k] stores the sum of weighted data @@ -205,11 +216,19 @@ TREELITE_DLL int TreeliteLoadLightGBMModelFromString( * \param out Loaded model * \return 0 for success, -1 for failure */ -TREELITE_DLL int TreeliteLoadSKLearnRandomForestRegressor(int n_estimators, int n_features, +TREELITE_DLL int TreeliteLoadSKLearnRandomForestRegressorEx(int n_estimators, int n_features, int n_targets, int64_t const* node_count, int64_t const** children_left, int64_t const** children_right, int64_t const** feature, double const** threshold, - double const** value, int64_t const** n_node_samples, double const** weighted_n_node_samples, - double const** impurity, TreeliteModelHandle* out); + double const** value, uint8_t const** missing_go_to_left, int64_t const** n_node_samples, + double const** weighted_n_node_samples, double const** impurity, TreeliteModelHandle* out); +/*! + * \brief Deprecated. Please use \ref TreeliteLoadSKLearnIsolationForestEx instead. + */ +TREELITE_DLL int TreeliteLoadSKLearnIsolationForest(int n_estimators, int n_features, + int64_t const* node_count, int64_t const** children_left, int64_t const** children_right, + int64_t const** feature, double const** threshold, double const** value, + int64_t const** n_node_samples, double const** weighted_n_node_samples, double const** impurity, + double ratio_c, TreeliteModelHandle* out); /*! * \brief Load a scikit-learn IsolationForest model from a collection of arrays. Refer to * https://scikit-learn.org/stable/auto_examples/tree/plot_unveil_tree_structure.html to @@ -227,20 +246,33 @@ TREELITE_DLL int TreeliteLoadSKLearnRandomForestRegressor(int n_estimators, int * the i-th tree. This is only defined if node k is an internal (non-leaf) node. * \param value value[i][k] stores the expected isolation depth of node k of the i-th tree. This is * only defined if node k is a leaf node. + * \param missing_go_to_left missing_go_to_left[i][k] stores the default direction for the missing + * value at node k of the i-th tree. This is only defined if node k is an + * internal (non-leaf) node. * \param n_node_samples n_node_samples[i][k] stores the number of data samples associated with * node k of the i-th tree. * \param weighted_n_node_samples weighted_n_node_samples[i][k] stores the sum of weighted data * samples associated with node k of the i-th tree. * \param impurity Not used, but must be passed as array of arrays for each tree and node. * \param ratio_c Standardizing constant to use for calculation of the anomaly score. + * \param offset Offset used to define the decision function from the raw scores. * \param out Loaded model * \return 0 for success, -1 for failure */ -TREELITE_DLL int TreeliteLoadSKLearnIsolationForest(int n_estimators, int n_features, +TREELITE_DLL int TreeliteLoadSKLearnIsolationForestEx(int n_estimators, int n_features, int64_t const* node_count, int64_t const** children_left, int64_t const** children_right, int64_t const** feature, double const** threshold, double const** value, - int64_t const** n_node_samples, double const** weighted_n_node_samples, double const** impurity, - double ratio_c, TreeliteModelHandle* out); + uint8_t const** missing_go_to_left, int64_t const** n_node_samples, + double const** weighted_n_node_samples, double const** impurity, double ratio_c, double offset, + TreeliteModelHandle* out); +/*! + * \brief Deprecated. Please use \ref TreeliteLoadSKLearnRandomForestClassifierEx instead. + */ +TREELITE_DLL int TreeliteLoadSKLearnRandomForestClassifier(int n_estimators, int n_features, + int n_targets, int32_t const* n_classes, int64_t const* node_count, + int64_t const** children_left, int64_t const** children_right, int64_t const** feature, + double const** threshold, double const** value, int64_t const** n_node_samples, + double const** weighted_n_node_samples, double const** impurity, TreeliteModelHandle* out); /*! * \brief Load a scikit-learn RandomForestClassifier model from a collection of arrays. Refer to * https://scikit-learn.org/stable/auto_examples/tree/plot_unveil_tree_structure.html to @@ -261,6 +293,9 @@ TREELITE_DLL int TreeliteLoadSKLearnIsolationForest(int n_estimators, int n_feat * the i-th tree. This is only defined if node k is an internal (non-leaf) node. * \param value value[i][k] stores the leaf output of node k of the i-th tree. This is only defined * if node k is a leaf node. + * \param missing_go_to_left missing_go_to_left[i][k] stores the default direction for the missing + * value at node k of the i-th tree. This is only defined if node k is an + * internal (non-leaf) node. * \param n_node_samples n_node_samples[i][k] stores the number of data samples associated with * node k of the i-th tree. * \param weighted_n_node_samples weighted_n_node_samples[i][k] stores the sum of weighted data @@ -270,11 +305,12 @@ TREELITE_DLL int TreeliteLoadSKLearnIsolationForest(int n_estimators, int n_feat * \param out Loaded model * \return 0 for success, -1 for failure */ -TREELITE_DLL int TreeliteLoadSKLearnRandomForestClassifier(int n_estimators, int n_features, +TREELITE_DLL int TreeliteLoadSKLearnRandomForestClassifierEx(int n_estimators, int n_features, int n_targets, int32_t const* n_classes, int64_t const* node_count, int64_t const** children_left, int64_t const** children_right, int64_t const** feature, - double const** threshold, double const** value, int64_t const** n_node_samples, - double const** weighted_n_node_samples, double const** impurity, TreeliteModelHandle* out); + double const** threshold, double const** value, uint8_t const** missing_go_to_left, + int64_t const** n_node_samples, double const** weighted_n_node_samples, double const** impurity, + TreeliteModelHandle* out); /*! * \brief Load a scikit-learn GradientBoostingRegressor model from a collection of arrays. Refer * to https://scikit-learn.org/stable/auto_examples/tree/plot_unveil_tree_structure.html to diff --git a/include/treelite/model_loader.h b/include/treelite/model_loader.h index da20478e..4dfd426c 100644 --- a/include/treelite/model_loader.h +++ b/include/treelite/model_loader.h @@ -120,6 +120,9 @@ namespace sklearn { * the i-th tree. This is only defined if node k is an internal (non-leaf) node. * \param value value[i][k] stores the leaf output of node k of the i-th tree. This is only defined * if node k is a leaf node. + * \param missing_go_to_left missing_go_to_left[i][k] stores the default direction for the missing + * value at node k of the i-th tree. This is only defined if node k is an + * internal (non-leaf) node. * \param n_node_samples n_node_samples[i][k] stores the number of data samples associated with * node k of the i-th tree. * \param weighted_n_node_samples weighted_n_node_samples[i][k] stores the sum of weighted data @@ -131,8 +134,9 @@ namespace sklearn { std::unique_ptr LoadRandomForestRegressor(int n_estimators, int n_features, int n_targets, std::int64_t const* node_count, std::int64_t const** children_left, std::int64_t const** children_right, std::int64_t const** feature, double const** threshold, - double const** value, std::int64_t const** n_node_samples, - double const** weighted_n_node_samples, double const** impurity); + double const** value, std::uint8_t const** missing_go_to_left, + std::int64_t const** n_node_samples, double const** weighted_n_node_samples, + double const** impurity); /*! * \brief Load a scikit-learn IsolationForest model from a collection of arrays. Refer to * https://scikit-learn.org/stable/auto_examples/tree/plot_unveil_tree_structure.html to @@ -150,19 +154,24 @@ std::unique_ptr LoadRandomForestRegressor(int n_estimators, int * the i-th tree. This is only defined if node k is an internal (non-leaf) node. * \param value value[i][k] stores the expected isolation depth of node k of the i-th tree. This is * only defined if node k is a leaf node. + * \param missing_go_to_left missing_go_to_left[i][k] stores the default direction for the missing + * value at node k of the i-th tree. This is only defined if node k is an + * internal (non-leaf) node. * \param n_node_samples n_node_samples[i][k] stores the number of data samples associated with * node k of the i-th tree. * \param weighted_n_node_samples weighted_n_node_samples[i][k] stores the sum of weighted data * samples associated with node k of the i-th tree. * \param impurity Not used, but must be passed as array of arrays for each tree and node. * \param ratio_c Standardizing constant to use for calculation of the anomaly score. + * \param offset Offset used to define the decision function from the raw scores. * \return Loaded model */ std::unique_ptr LoadIsolationForest(int n_estimators, int n_features, std::int64_t const* node_count, std::int64_t const** children_left, std::int64_t const** children_right, std::int64_t const** feature, double const** threshold, - double const** value, std::int64_t const** n_node_samples, - double const** weighted_n_node_samples, double const** impurity, double ratio_c); + double const** value, std::uint8_t const** missing_go_to_left, + std::int64_t const** n_node_samples, double const** weighted_n_node_samples, + double const** impurity, double ratio_c, double offset); /*! * \brief Load a scikit-learn RandomForestClassifier model from a collection of arrays. Refer to * https://scikit-learn.org/stable/auto_examples/tree/plot_unveil_tree_structure.html to @@ -183,6 +192,9 @@ std::unique_ptr LoadIsolationForest(int n_estimators, int n_fea * the i-th tree. This is only defined if node k is an internal (non-leaf) node. * \param value value[i][k] stores the leaf output of node k of the i-th tree. This is only defined * if node k is a leaf node. + * \param missing_go_to_left missing_go_to_left[i][k] stores the default direction for the missing + * value at node k of the i-th tree. This is only defined if node k is an + * internal (non-leaf) node. * \param n_node_samples n_node_samples[i][k] stores the number of data samples associated with * node k of the i-th tree. * \param weighted_n_node_samples weighted_n_node_samples[i][k] stores the sum of weighted data @@ -195,8 +207,8 @@ std::unique_ptr LoadRandomForestClassifier(int n_estimators, in int n_targets, int32_t const* n_classes, std::int64_t const* node_count, std::int64_t const** children_left, std::int64_t const** children_right, std::int64_t const** feature, double const** threshold, double const** value, - std::int64_t const** n_node_samples, double const** weighted_n_node_samples, - double const** impurity); + std::uint8_t const** missing_go_to_left, std::int64_t const** n_node_samples, + double const** weighted_n_node_samples, double const** impurity); /*! * \brief Load a scikit-learn GradientBoostingRegressor model from a collection of arrays. Refer * to https://scikit-learn.org/stable/auto_examples/tree/plot_unveil_tree_structure.html to diff --git a/include/treelite/tree.h b/include/treelite/tree.h index 2c56dfcf..e20296e7 100644 --- a/include/treelite/tree.h +++ b/include/treelite/tree.h @@ -149,9 +149,10 @@ class Tree { template friend void BulkConstructTree(Tree& tree, int n_nodes, std::int64_t const* children_left, std::int64_t const* children_right, std::int64_t const* feature, double const* threshold, - double const* value, std::int64_t const* n_node_samples, - double const* weighted_n_node_samples, double const* impurity, std::int64_t total_sample_cnt, - int n_targets, int max_num_class, bool is_classifier); + double const* value, std::uint8_t const* missing_go_to_left, + std::int64_t const* n_node_samples, double const* weighted_n_node_samples, + double const* impurity, std::int64_t total_sample_cnt, int n_targets, int max_num_class, + bool is_classifier); public: /*! \brief Number of nodes */ diff --git a/src/c_api/sklearn.cc b/src/c_api/sklearn.cc index 641c78f7..ca5557d0 100644 --- a/src/c_api/sklearn.cc +++ b/src/c_api/sklearn.cc @@ -17,10 +17,23 @@ int TreeliteLoadSKLearnRandomForestRegressor(int n_estimators, int n_features, i std::int64_t const** children_right, std::int64_t const** feature, double const** threshold, double const** value, std::int64_t const** n_node_samples, double const** weighted_n_node_samples, double const** impurity, TreeliteModelHandle* out) { + TREELITE_LOG(WARNING) << "TreeliteLoadSKLearnRandomForestRegressor() is deprecated. Please use " + << "TreeliteLoadSKLearnRandomForestRegressorEx() instead."; + return TreeliteLoadSKLearnRandomForestRegressorEx(n_estimators, n_features, n_targets, node_count, + children_left, children_right, feature, threshold, value, nullptr, n_node_samples, + weighted_n_node_samples, impurity, out); +} + +int TreeliteLoadSKLearnRandomForestRegressorEx(int n_estimators, int n_features, int n_targets, + std::int64_t const* node_count, std::int64_t const** children_left, + std::int64_t const** children_right, std::int64_t const** feature, double const** threshold, + double const** value, std::uint8_t const** missing_go_to_left, + std::int64_t const** n_node_samples, double const** weighted_n_node_samples, + double const** impurity, TreeliteModelHandle* out) { API_BEGIN(); auto model = treelite::model_loader::sklearn::LoadRandomForestRegressor(n_estimators, n_features, n_targets, node_count, children_left, children_right, feature, threshold, value, - n_node_samples, weighted_n_node_samples, impurity); + missing_go_to_left, n_node_samples, weighted_n_node_samples, impurity); *out = static_cast(model.release()); API_END(); } @@ -31,10 +44,23 @@ int TreeliteLoadSKLearnIsolationForest(int n_estimators, int n_features, double const** value, std::int64_t const** n_node_samples, double const** weighted_n_node_samples, double const** impurity, double ratio_c, TreeliteModelHandle* out) { + TREELITE_LOG(WARNING) << "TreeliteLoadSKLearnIsolationForest() is deprecated. Please use " + << "TreeliteLoadSKLearnIsolationForestEx() instead."; + return TreeliteLoadSKLearnIsolationForestEx(n_estimators, n_features, node_count, children_left, + children_right, feature, threshold, value, nullptr, n_node_samples, weighted_n_node_samples, + impurity, ratio_c, -0.5, out); +} + +int TreeliteLoadSKLearnIsolationForestEx(int n_estimators, int n_features, + std::int64_t const* node_count, std::int64_t const** children_left, + std::int64_t const** children_right, std::int64_t const** feature, double const** threshold, + double const** value, std::uint8_t const** missing_go_to_left, + std::int64_t const** n_node_samples, double const** weighted_n_node_samples, + double const** impurity, double ratio_c, double offset, TreeliteModelHandle* out) { API_BEGIN(); auto model = treelite::model_loader::sklearn::LoadIsolationForest(n_estimators, n_features, - node_count, children_left, children_right, feature, threshold, value, n_node_samples, - weighted_n_node_samples, impurity, ratio_c); + node_count, children_left, children_right, feature, threshold, value, missing_go_to_left, + n_node_samples, weighted_n_node_samples, impurity, ratio_c, offset); *out = static_cast(model.release()); API_END(); } @@ -45,10 +71,23 @@ int TreeliteLoadSKLearnRandomForestClassifier(int n_estimators, int n_features, std::int64_t const** feature, double const** threshold, double const** value, std::int64_t const** n_node_samples, double const** weighted_n_node_samples, double const** impurity, TreeliteModelHandle* out) { + TREELITE_LOG(WARNING) << "TreeliteLoadSKLearnRandomForestClassifier() is deprecated. Please use " + << "TreeliteLoadSKLearnRandomForestClassifierEx() instead."; + return TreeliteLoadSKLearnRandomForestClassifierEx(n_estimators, n_features, n_targets, n_classes, + node_count, children_left, children_right, feature, threshold, value, nullptr, n_node_samples, + weighted_n_node_samples, impurity, out); +} + +int TreeliteLoadSKLearnRandomForestClassifierEx(int n_estimators, int n_features, int n_targets, + std::int32_t const* n_classes, std::int64_t const* node_count, + std::int64_t const** children_left, std::int64_t const** children_right, + std::int64_t const** feature, double const** threshold, double const** value, + std::uint8_t const** missing_go_to_left, std::int64_t const** n_node_samples, + double const** weighted_n_node_samples, double const** impurity, TreeliteModelHandle* out) { API_BEGIN(); auto model = treelite::model_loader::sklearn::LoadRandomForestClassifier(n_estimators, n_features, n_targets, n_classes, node_count, children_left, children_right, feature, threshold, value, - n_node_samples, weighted_n_node_samples, impurity); + missing_go_to_left, n_node_samples, weighted_n_node_samples, impurity); *out = static_cast(model.release()); API_END(); } diff --git a/src/model_loader/sklearn.cc b/src/model_loader/sklearn.cc index e91acedb..fe64f215 100644 --- a/src/model_loader/sklearn.cc +++ b/src/model_loader/sklearn.cc @@ -25,37 +25,6 @@ namespace treelite::model_loader::sklearn { namespace detail { -namespace stdex = std::experimental; -// Multidimensional array views. Use row-major (C) layout -template -using Array2DView = stdex::mdspan, stdex::layout_right>; - -class IsolationForestMixIn { - public: - explicit IsolationForestMixIn(double ratio_c) : ratio_c_{ratio_c} {} - - void HandleMetadata(model_builder::ModelBuilder& builder, int n_trees, int n_features, - [[maybe_unused]] int n_targets, [[maybe_unused]] std::int32_t const* n_classes) { - model_builder::Metadata metadata{n_features, TaskType::kIsolationForest, true, 1, {1}, {1, 1}}; - model_builder::TreeAnnotation tree_annotation{ - n_trees, std::vector(n_trees, 0), std::vector(n_trees, 0)}; - - std::ostringstream oss; - model_builder::PostProcessorFunc postprocessor{ - "exponential_standard_ratio", {{"ratio_c", ratio_c_}}}; - - builder.InitializeMetadata(metadata, tree_annotation, postprocessor, {0.0}, std::nullopt); - } - - void HandleLeafNode(model_builder::ModelBuilder& builder, int tree_id, int node_id, - double const** value, [[maybe_unused]] std::int32_t const* n_classes) const { - builder.LeafScalar(value[tree_id][node_id]); - } - - private: - double ratio_c_; -}; - class GradientBoostingRegressorMixIn { public: explicit GradientBoostingRegressorMixIn(double base_score) : base_score_{base_score} {} @@ -370,18 +339,6 @@ std::unique_ptr LoadHistGradientBoosting(MixIn& mixin, int n_tr } // namespace detail -std::unique_ptr LoadIsolationForest(int n_estimators, int n_features, - std::int64_t const* node_count, std::int64_t const** children_left, - std::int64_t const** children_right, std::int64_t const** feature, double const** threshold, - double const** value, std::int64_t const** n_node_samples, - double const** weighted_n_node_samples, double const** impurity, double ratio_c) { - detail::IsolationForestMixIn mixin{ratio_c}; - std::vector n_classes{1}; - return detail::LoadSKLearnModel(mixin, n_estimators, n_features, 1, n_classes.data(), node_count, - children_left, children_right, feature, threshold, value, n_node_samples, - weighted_n_node_samples, impurity); -} - std::unique_ptr LoadGradientBoostingRegressor(int n_iter, int n_features, std::int64_t const* node_count, std::int64_t const** children_left, std::int64_t const** children_right, std::int64_t const** feature, double const** threshold, diff --git a/src/model_loader/sklearn_bulk.cc b/src/model_loader/sklearn_bulk.cc index d486f67e..09e726a2 100644 --- a/src/model_loader/sklearn_bulk.cc +++ b/src/model_loader/sklearn_bulk.cc @@ -37,9 +37,9 @@ template void BulkConstructTree(Tree& tree, int n_nodes, std::int64_t const* children_left, std::int64_t const* children_right, std::int64_t const* feature, double const* threshold, double const* value, - std::int64_t const* n_node_samples, double const* weighted_n_node_samples, - double const* impurity, std::int64_t total_sample_cnt, int n_targets, int max_num_class, - bool is_classifier) { + std::uint8_t const* missing_go_to_left, std::int64_t const* n_node_samples, + double const* weighted_n_node_samples, double const* impurity, std::int64_t total_sample_cnt, + int n_targets, int max_num_class, bool is_classifier) { // Clear and pre-allocate all arrays at once - key optimization! tree.node_type_.Clear(); tree.cleft_.Clear(); @@ -133,7 +133,11 @@ void BulkConstructTree(Tree& tree, int n_nodes, tree.cmp_.PushBack(Operator::kLE); } - tree.default_left_.PushBack(true); + if (missing_go_to_left) { + tree.default_left_.PushBack(static_cast(missing_go_to_left[node_id])); + } else { + tree.default_left_.PushBack(true); + } tree.category_list_right_child_.PushBack(false); // Handle leaf values @@ -212,12 +216,12 @@ void BulkConstructTree(Tree& tree, int n_nodes, // Explicit instantiation template void BulkConstructTree(Tree&, int, std::int64_t const*, - std::int64_t const*, std::int64_t const*, double const*, double const*, std::int64_t const*, - double const*, double const*, std::int64_t, int, int, bool); + std::int64_t const*, std::int64_t const*, double const*, double const*, std::uint8_t const*, + std::int64_t const*, double const*, double const*, std::int64_t, int, int, bool); template void BulkConstructTree(Tree&, int, std::int64_t const*, - std::int64_t const*, std::int64_t const*, double const*, double const*, std::int64_t const*, - double const*, double const*, std::int64_t, int, int, bool); + std::int64_t const*, std::int64_t const*, double const*, double const*, std::uint8_t const*, + std::int64_t const*, double const*, double const*, std::int64_t, int, int, bool); } // namespace treelite @@ -233,8 +237,8 @@ std::unique_ptr LoadRandomForestClassifier(int n_estimators, in int n_targets, std::int32_t const* n_classes, std::int64_t const* node_count, std::int64_t const** children_left, std::int64_t const** children_right, std::int64_t const** feature, double const** threshold, double const** value, - std::int64_t const** n_node_samples, double const** weighted_n_node_samples, - double const** impurity) { + std::uint8_t const** missing_go_to_left, std::int64_t const** n_node_samples, + double const** weighted_n_node_samples, double const** impurity) { TREELITE_CHECK_GT(n_estimators, 0) << "n_estimators must be at least 1"; TREELITE_CHECK_GT(n_features, 0) << "n_features must be at least 1"; @@ -277,11 +281,13 @@ std::unique_ptr LoadRandomForestClassifier(int n_estimators, in for (int tree_id = 0; tree_id < n_estimators; ++tree_id) { int const n_nodes = static_cast(node_count[tree_id]); std::int64_t const total_sample_cnt = n_node_samples[tree_id][0]; + std::uint8_t const* missing_go_to_left_ptr + = missing_go_to_left ? missing_go_to_left[tree_id] : nullptr; BulkConstructTree(preset.trees[tree_id], n_nodes, children_left[tree_id], children_right[tree_id], feature[tree_id], threshold[tree_id], value[tree_id], - n_node_samples[tree_id], weighted_n_node_samples[tree_id], impurity[tree_id], - total_sample_cnt, n_targets, max_num_class, + missing_go_to_left_ptr, n_node_samples[tree_id], weighted_n_node_samples[tree_id], + impurity[tree_id], total_sample_cnt, n_targets, max_num_class, true); // is_classifier } @@ -297,8 +303,9 @@ std::unique_ptr LoadRandomForestClassifier(int n_estimators, in std::unique_ptr LoadRandomForestRegressor(int n_estimators, int n_features, int n_targets, std::int64_t const* node_count, std::int64_t const** children_left, std::int64_t const** children_right, std::int64_t const** feature, double const** threshold, - double const** value, std::int64_t const** n_node_samples, - double const** weighted_n_node_samples, double const** impurity) { + double const** value, std::uint8_t const** missing_go_to_left, + std::int64_t const** n_node_samples, double const** weighted_n_node_samples, + double const** impurity) { TREELITE_CHECK_GT(n_estimators, 0) << "n_estimators must be at least 1"; TREELITE_CHECK_GT(n_features, 0) << "n_features must be at least 1"; @@ -337,11 +344,13 @@ std::unique_ptr LoadRandomForestRegressor(int n_estimators, int for (int tree_id = 0; tree_id < n_estimators; ++tree_id) { int const n_nodes = static_cast(node_count[tree_id]); std::int64_t const total_sample_cnt = n_node_samples[tree_id][0]; + std::uint8_t const* missing_go_to_left_ptr + = missing_go_to_left ? missing_go_to_left[tree_id] : nullptr; BulkConstructTree(preset.trees[tree_id], n_nodes, children_left[tree_id], children_right[tree_id], feature[tree_id], threshold[tree_id], value[tree_id], - n_node_samples[tree_id], weighted_n_node_samples[tree_id], impurity[tree_id], - total_sample_cnt, n_targets, + missing_go_to_left_ptr, n_node_samples[tree_id], weighted_n_node_samples[tree_id], + impurity[tree_id], total_sample_cnt, n_targets, 1, // max_num_class = 1 for regressors false); // is_classifier } @@ -349,4 +358,71 @@ std::unique_ptr LoadRandomForestRegressor(int n_estimators, int return model; } +/** + * Load an IsolationForest using bulk construction + * + * This is an optimized version that constructs trees in bulk rather than + * going through the ModelBuilder node-by-node. + */ +std::unique_ptr LoadIsolationForest(int n_estimators, int n_features, + std::int64_t const* node_count, std::int64_t const** children_left, + std::int64_t const** children_right, std::int64_t const** feature, double const** threshold, + double const** value, std::uint8_t const** missing_go_to_left, + std::int64_t const** n_node_samples, double const** weighted_n_node_samples, + double const** impurity, double ratio_c, double offset) { + TREELITE_CHECK_GT(n_estimators, 0) << "n_estimators must be at least 1"; + TREELITE_CHECK_GT(n_features, 0) << "n_features must be at least 1"; + + // Create model with double precision + auto model = Model::Create(); + + // Set up model metadata + std::int32_t const n_targets = 1; + model->num_feature = n_features; + model->task_type = TaskType::kIsolationForest; + model->average_tree_output = true; + model->num_target = n_targets; + + // For isolation forests, num_class is always 1 + model->num_class = std::vector(n_targets, 1); + + // Set leaf vector shape + model->leaf_vector_shape = std::vector{n_targets, 1}; + + // Set up tree annotation arrays + model->target_id = std::vector(n_estimators, 0); + model->class_id = std::vector(n_estimators, 0); + + // Set postprocessor + model->postprocessor = "exponential_standard_ratio"; + model->ratio_c = static_cast(ratio_c); + + // Set offset + model->attributes = R"attr({"sklearn_iforest_offset": -0.5})attr"; + + // Set base scores + model->base_scores = std::vector{0.0}; + + // Get the typed model preset + auto& preset = std::get>(model->variant_); + preset.trees.resize(n_estimators); + + // Construct each tree using bulk operations + for (int tree_id = 0; tree_id < n_estimators; ++tree_id) { + int const n_nodes = static_cast(node_count[tree_id]); + std::int64_t const total_sample_cnt = n_node_samples[tree_id][0]; + std::uint8_t const* missing_go_to_left_ptr + = missing_go_to_left ? missing_go_to_left[tree_id] : nullptr; + + BulkConstructTree(preset.trees[tree_id], n_nodes, children_left[tree_id], + children_right[tree_id], feature[tree_id], threshold[tree_id], value[tree_id], + missing_go_to_left_ptr, n_node_samples[tree_id], weighted_n_node_samples[tree_id], + impurity[tree_id], total_sample_cnt, n_targets, + 1, // max_num_class = 1 for isolation forests + false); // is_classifier + } + + return model; +} + } // namespace treelite::model_loader::sklearn From f5caf0928361a027c8edc9c1569361ddcc4b75ee Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Tue, 25 Aug 2026 20:03:07 -0700 Subject: [PATCH 2/3] Update the Python layer --- python/treelite/sklearn/importer.py | 32 +++++++++++++---------------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/python/treelite/sklearn/importer.py b/python/treelite/sklearn/importer.py index 1d7f3d94..dd3030b6 100644 --- a/python/treelite/sklearn/importer.py +++ b/python/treelite/sklearn/importer.py @@ -20,6 +20,7 @@ class ArrayOfArrays: def __init__(self, *, dtype): int8_ptr_type = ctypes.POINTER(ctypes.c_int8) + uint8_ptr_type = ctypes.POINTER(ctypes.c_uint8) int64_ptr_type = ctypes.POINTER(ctypes.c_int64) uint32_ptr_type = ctypes.POINTER(ctypes.c_uint32) float64_ptr_type = ctypes.POINTER(ctypes.c_double) @@ -32,6 +33,8 @@ def __init__(self, *, dtype): self.ptr_type = uint32_ptr_type elif dtype == np.int8: self.ptr_type = int8_ptr_type + elif dtype == np.uint8: + self.ptr_type = uint8_ptr_type elif dtype == "void": self.ptr_type = void_ptr_type else: @@ -188,6 +191,7 @@ def import_model(sklearn_model) -> Model: feature = ArrayOfArrays(dtype=np.int64) threshold = ArrayOfArrays(dtype=np.float64) value = ArrayOfArrays(dtype=np.float64) + missing_go_to_left = ArrayOfArrays(dtype=np.uint8) n_node_samples = ArrayOfArrays(dtype=np.int64) weighted_n_node_samples = ArrayOfArrays(dtype=np.float64) impurity = ArrayOfArrays(dtype=np.float64) @@ -233,6 +237,9 @@ def import_model(sklearn_model) -> Model: expected_shape=leaf_value_expected_shape(tree.node_count), ) feature.add(tree.feature, expected_shape=(tree.node_count,)) + missing_go_to_left.add( + tree.missing_go_to_left, expected_shape=(tree.node_count,) + ) n_node_samples.add(tree.n_node_samples, expected_shape=(tree.node_count,)) weighted_n_node_samples.add( tree.weighted_n_node_samples, expected_shape=(tree.node_count,) @@ -242,7 +249,7 @@ def import_model(sklearn_model) -> Model: handle = ctypes.c_void_p() if isinstance(sklearn_model, (RandomForestR, ExtraTreesR)): _check_call( - _LIB.TreeliteLoadSKLearnRandomForestRegressor( + _LIB.TreeliteLoadSKLearnRandomForestRegressorEx( ctypes.c_int(sklearn_model.n_estimators), ctypes.c_int(sklearn_model.n_features_in_), ctypes.c_int(sklearn_model.n_outputs_), @@ -252,6 +259,7 @@ def import_model(sklearn_model) -> Model: feature.as_c_array(), threshold.as_c_array(), value.as_c_array(), + missing_go_to_left.as_c_array(), n_node_samples.as_c_array(), weighted_n_node_samples.as_c_array(), impurity.as_c_array(), @@ -259,9 +267,8 @@ def import_model(sklearn_model) -> Model: ) ) elif isinstance(sklearn_model, IsolationForest): - # TODO(chyunsu3): In Treelite 5.0, pass offset_ field via TreeliteLoadSKLearnIsolationForest() _check_call( - _LIB.TreeliteLoadSKLearnIsolationForest( + _LIB.TreeliteLoadSKLearnIsolationForestEx( ctypes.c_int(sklearn_model.n_estimators), ctypes.c_int(sklearn_model.n_features_in_), c_array(ctypes.c_int64, node_count), @@ -270,31 +277,19 @@ def import_model(sklearn_model) -> Model: feature.as_c_array(), threshold.as_c_array(), value.as_c_array(), + missing_go_to_left.as_c_array(), n_node_samples.as_c_array(), weighted_n_node_samples.as_c_array(), impurity.as_c_array(), ctypes.c_double(ratio_c), + ctypes.c_double(sklearn_model.offset_), ctypes.byref(handle), ) ) - # Store `offset_` field as a model attribute - attributes = { - "sklearn_iforest_offset": float(sklearn_model.offset_), - } - attributes_serialized = json.dumps(attributes) - _check_call( - _LIB.TreeliteSetHeaderField( - handle, - c_str("attributes"), - _numpy2pybuffer( - np.frombuffer(attributes_serialized.encode("utf-8"), dtype="S1") - ), - ) - ) elif isinstance(sklearn_model, (RandomForestC, ExtraTreesC)): n_classes = np.array(sklearn_model.n_classes_, dtype=np.int32) _check_call( - _LIB.TreeliteLoadSKLearnRandomForestClassifier( + _LIB.TreeliteLoadSKLearnRandomForestClassifierEx( ctypes.c_int(sklearn_model.n_estimators), ctypes.c_int(sklearn_model.n_features_in_), ctypes.c_int(sklearn_model.n_outputs_), @@ -305,6 +300,7 @@ def import_model(sklearn_model) -> Model: feature.as_c_array(), threshold.as_c_array(), value.as_c_array(), + missing_go_to_left.as_c_array(), n_node_samples.as_c_array(), weighted_n_node_samples.as_c_array(), impurity.as_c_array(), From 83ce7329beeff7155b54ff8440697a0f8c51586f Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Mon, 24 Aug 2026 16:54:51 -0700 Subject: [PATCH 3/3] Add test coverage for missing values --- .../test_sklearn_trees_missing_values.py | 197 ++++++++++++++++++ 1 file changed, 197 insertions(+) create mode 100644 tests/python/test_sklearn_trees_missing_values.py diff --git a/tests/python/test_sklearn_trees_missing_values.py b/tests/python/test_sklearn_trees_missing_values.py new file mode 100644 index 00000000..ee9e3b2b --- /dev/null +++ b/tests/python/test_sklearn_trees_missing_values.py @@ -0,0 +1,197 @@ +""" +Test coverage for scikit-learn tree models trained with data with missing values. +Adapted from https://github.com/scikit-learn/scikit-learn/blob/1.9.0/sklearn/tree/tests/test_tree.py +""" + +import numpy as np +import pytest + +import treelite + +try: + from sklearn.ensemble import RandomForestClassifier, RandomForestRegressor +except ImportError: + pytest.skip("scikit-learn not installed; skipping", allow_module_level=True) + + +@pytest.mark.parametrize("criterion", ["squared_error"]) +def test_heuristic_default_direction(criterion): + """ + When training data contains no missing values, scikit-learn uses the following + heuristic for predicting with a missing value. + For each binary test node, + * Choose the left or right child node, whichever has the higher sample count. + * If the two child nodes have identical sample counts, choose the right child node. + """ + X = np.array([[0, 1, 2, 3, 8, 9, 11, 12, 15]]).T + y = np.array([0.1, 0.2, 0.3, 0.2, 1.4, 1.4, 1.5, 1.6, 2.6]) + + regr = RandomForestRegressor( + random_state=42, + max_depth=1, + criterion=criterion, + n_estimators=1, + bootstrap=False, + ) + regr.fit(X, y) + tl_model = treelite.sklearn.import_model(regr) + + # There should be a single test node, with the test threshold being somewhere between 3 and 8. + threshold = regr.estimators_[0].tree_.threshold[0] + assert 3 < threshold < 8 + + # Goes to right node because it has the most data points + y_pred = regr.predict([[np.nan]]) + np.testing.assert_allclose(y_pred, [np.mean(y[-5:])]) + np.testing.assert_allclose( + treelite.gtil.predict(tl_model, np.array([[np.nan]])).squeeze(), + y_pred, + ) + + # Equal number of elements in both nodes + X_equal = X[:-1] + y_equal = y[:-1] + + regr = RandomForestRegressor( + random_state=42, + max_depth=1, + criterion=criterion, + n_estimators=1, + bootstrap=False, + ) + regr.fit(X_equal, y_equal) + tl_model = treelite.sklearn.import_model(regr) + + # Goes to right node because the implementation sets: + # missing_go_to_left = n_left > n_right, which is False + y_pred = regr.predict([[np.nan]]) + np.testing.assert_allclose(y_pred, [np.mean(y_equal[-4:])]) + np.testing.assert_allclose( + treelite.gtil.predict(tl_model, np.array([[np.nan]])).squeeze(), + y_pred, + ) + + +@pytest.mark.parametrize("criterion", ["entropy", "gini"]) +def test_best_splitter_three_classes(criterion): + """ + Test the following scenario with a tree stump: + Training data has three class labels: 0, 1, 2. + All missing values are assigned class 0; all non-missing values are assigned + either class 1 or 2. + At predict time, the model should classify missing values as class 0. + """ + missing_values_class = 0 + X = np.array([[np.nan] * 4 + [0, 1, 2, 3, 8, 9, 11, 12]]).T + y = np.array([missing_values_class] * 4 + [1] * 4 + [2] * 4) + clf = RandomForestClassifier( + random_state=42, + max_depth=2, + criterion=criterion, + n_estimators=1, + bootstrap=False, + ) + clf.fit(X, y) + tl_model = treelite.sklearn.import_model(clf) + + X_test = np.array([[np.nan, 3, 12]]).T + y_pred = clf.predict(X_test) + np.testing.assert_array_equal(y_pred, [missing_values_class, 1, 2]) + + tl_pred = treelite.gtil.predict(tl_model, X_test) + tl_pred = np.argmax(tl_pred[:, 0, :], axis=1) + np.testing.assert_array_equal(tl_pred, y_pred) + + +@pytest.mark.parametrize("criterion", ["entropy", "gini"]) +def test_best_splitter_to_left(criterion): + """ + Test the following scenario with a tree stump: + Training data has two class labels: 0, 1. + Class 0 gets only missing values. + Class 1 gets only non-missing values. + At predict time, the model should classify missing values as class 0. + """ + X = np.array([[np.nan] * 4 + [0, 1, 2, 3, 4, 5]]).T + y = np.array([0] * 4 + [1] * 6) + + clf = RandomForestClassifier( + random_state=42, + max_depth=2, + criterion=criterion, + n_estimators=1, + bootstrap=False, + ) + clf.fit(X, y) + tl_model = treelite.sklearn.import_model(clf) + + X_test = np.array([[np.nan, 5, np.nan]]).T + y_pred = clf.predict(X_test) + np.testing.assert_array_equal(y_pred, [0, 1, 0]) + + tl_pred = treelite.gtil.predict(tl_model, X_test) + tl_pred = np.argmax(tl_pred[:, 0, :], axis=1) + np.testing.assert_array_equal(tl_pred, y_pred) + + +@pytest.mark.parametrize("criterion", ["entropy", "gini"]) +def test_best_splitter_to_right(criterion): + """ + Test the following scenario with a tree stump: + Training data has two class labels: 0, 1. + Class 0 gets only non-missing values. + Class 1 gets a mix of missing values and non-missing values. + At predict time, the model should classify missing values as class 1. + """ + X = np.array([[np.nan] * 4 + [0, 1, 2, 3, 4, 5]]).T + y = np.array([1] * 4 + [0] * 4 + [1] * 2) + + clf = RandomForestClassifier( + random_state=42, + max_depth=2, + criterion=criterion, + n_estimators=1, + bootstrap=False, + ) + clf.fit(X, y) + tl_model = treelite.sklearn.import_model(clf) + + X_test = np.array([[np.nan, 1.2, 4.8]]).T + y_pred = clf.predict(X_test) + np.testing.assert_array_equal(y_pred, [1, 0, 1]) + + tl_pred = treelite.gtil.predict(tl_model, X_test) + tl_pred = np.argmax(tl_pred[:, 0, :], axis=1) + np.testing.assert_array_equal(tl_pred, y_pred) + + +@pytest.mark.parametrize("criterion", ["entropy", "gini"]) +def test_best_splitter_missing_both_classes_has_nan(criterion): + """ + Test the following scenario with a tree stump: + Training data has two class labels: 0, 1. + Each class gets 4 non-missing values and 1 missing value. + """ + X = np.array([[1, 2, 3, 5, np.nan, 10, 20, 30, 60, np.nan]]).T + y = np.array([0] * 5 + [1] * 5) + + clf = RandomForestClassifier( + random_state=42, + max_depth=1, + criterion=criterion, + n_estimators=1, + bootstrap=False, + ) + clf.fit(X, y) + tl_model = treelite.sklearn.import_model(clf) + + X_test = np.array([[np.nan, 2.3, 34.2]]).T + y_pred = clf.predict(X_test) + + # Missing value goes to the class at the right (here 1) because the implementation + # searches right first. + np.testing.assert_array_equal(y_pred, [1, 0, 1]) + + tl_pred = treelite.gtil.predict(tl_model, X_test) + tl_pred = np.argmax(tl_pred[:, 0, :], axis=1) + np.testing.assert_array_equal(tl_pred, y_pred)