Add missing value support in scikit-learn random forest models

include/treelite/c_api.h

@@ -177,6 +177,14 @@ TREELITE_DLL int TreeliteLoadLightGBMModelFromString(
  * \defgroup sklearn C API: Model loaders for scikit-learn
  * \{
  */
+/*!
+ * \brief Deprecated. Please use \ref TreeliteLoadSKLearnRandomForestRegressorEx instead.
+ */
+TREELITE_DLL int TreeliteLoadSKLearnRandomForestRegressor(int n_estimators, int n_features,
+    int n_targets, int64_t const* node_count, int64_t const** children_left,
+    int64_t const** children_right, int64_t const** feature, double const** threshold,
+    double const** value, int64_t const** n_node_samples, double const** weighted_n_node_samples,
+    double const** impurity, TreeliteModelHandle* out);
 /*!
  * \brief Load a scikit-learn RandomForestRegressor model from a collection of arrays. Refer to
  *        https://scikit-learn.org/stable/auto_examples/tree/plot_unveil_tree_structure.html to
@@ -196,6 +204,9 @@ TREELITE_DLL int TreeliteLoadLightGBMModelFromString(
  *                  the i-th tree. This is only defined if node k is an internal (non-leaf) node.
  * \param value value[i][k] stores the leaf output of node k of the i-th tree. This is only defined
  *              if node k is a leaf node.
+ * \param missing_go_to_left missing_go_to_left[i][k] stores the default direction for the missing
+ *                           value at node k of the i-th tree. This is only defined if node k is an
+ *                           internal (non-leaf) node.
  * \param n_node_samples n_node_samples[i][k] stores the number of data samples associated with
  *                       node k of the i-th tree.
  * \param weighted_n_node_samples weighted_n_node_samples[i][k] stores the sum of weighted data
@@ -205,11 +216,19 @@ TREELITE_DLL int TreeliteLoadLightGBMModelFromString(
  * \param out Loaded model
  * \return 0 for success, -1 for failure
  */
-TREELITE_DLL int TreeliteLoadSKLearnRandomForestRegressor(int n_estimators, int n_features,
+TREELITE_DLL int TreeliteLoadSKLearnRandomForestRegressorEx(int n_estimators, int n_features,
     int n_targets, int64_t const* node_count, int64_t const** children_left,
     int64_t const** children_right, int64_t const** feature, double const** threshold,
-    double const** value, int64_t const** n_node_samples, double const** weighted_n_node_samples,
-    double const** impurity, TreeliteModelHandle* out);
+    double const** value, uint8_t const** missing_go_to_left, int64_t const** n_node_samples,
+    double const** weighted_n_node_samples, double const** impurity, TreeliteModelHandle* out);
+/*!
+ * \brief Deprecated. Please use \ref TreeliteLoadSKLearnIsolationForestEx instead.
+ */
+TREELITE_DLL int TreeliteLoadSKLearnIsolationForest(int n_estimators, int n_features,
+    int64_t const* node_count, int64_t const** children_left, int64_t const** children_right,
+    int64_t const** feature, double const** threshold, double const** value,
+    int64_t const** n_node_samples, double const** weighted_n_node_samples, double const** impurity,
+    double ratio_c, TreeliteModelHandle* out);
 /*!
  * \brief Load a scikit-learn IsolationForest model from a collection of arrays. Refer to
  *        https://scikit-learn.org/stable/auto_examples/tree/plot_unveil_tree_structure.html to
@@ -227,20 +246,33 @@ TREELITE_DLL int TreeliteLoadSKLearnRandomForestRegressor(int n_estimators, int
  *                  the i-th tree. This is only defined if node k is an internal (non-leaf) node.
  * \param value value[i][k] stores the expected isolation depth of node k of the i-th tree. This is
  *              only defined if node k is a leaf node.
+ * \param missing_go_to_left missing_go_to_left[i][k] stores the default direction for the missing
+ *                           value at node k of the i-th tree. This is only defined if node k is an
+ *                           internal (non-leaf) node.
  * \param n_node_samples n_node_samples[i][k] stores the number of data samples associated with
  *                       node k of the i-th tree.
  * \param weighted_n_node_samples weighted_n_node_samples[i][k] stores the sum of weighted data
  *                                samples associated with node k of the i-th tree.
  * \param impurity Not used, but must be passed as array of arrays for each tree and node.
  * \param ratio_c Standardizing constant to use for calculation of the anomaly score.
+ * \param offset Offset used to define the decision function from the raw scores.
  * \param out Loaded model
  * \return 0 for success, -1 for failure
  */
-TREELITE_DLL int TreeliteLoadSKLearnIsolationForest(int n_estimators, int n_features,
+TREELITE_DLL int TreeliteLoadSKLearnIsolationForestEx(int n_estimators, int n_features,
     int64_t const* node_count, int64_t const** children_left, int64_t const** children_right,
     int64_t const** feature, double const** threshold, double const** value,
-    int64_t const** n_node_samples, double const** weighted_n_node_samples, double const** impurity,
-    double ratio_c, TreeliteModelHandle* out);
+    uint8_t const** missing_go_to_left, int64_t const** n_node_samples,
+    double const** weighted_n_node_samples, double const** impurity, double ratio_c, double offset,
+    TreeliteModelHandle* out);
+/*!
+ * \brief Deprecated. Please use \ref TreeliteLoadSKLearnRandomForestClassifierEx instead.
+ */
+TREELITE_DLL int TreeliteLoadSKLearnRandomForestClassifier(int n_estimators, int n_features,
+    int n_targets, int32_t const* n_classes, int64_t const* node_count,
+    int64_t const** children_left, int64_t const** children_right, int64_t const** feature,
+    double const** threshold, double const** value, int64_t const** n_node_samples,
+    double const** weighted_n_node_samples, double const** impurity, TreeliteModelHandle* out);
 /*!
  * \brief Load a scikit-learn RandomForestClassifier model from a collection of arrays. Refer to
  *        https://scikit-learn.org/stable/auto_examples/tree/plot_unveil_tree_structure.html to
@@ -261,6 +293,9 @@ TREELITE_DLL int TreeliteLoadSKLearnIsolationForest(int n_estimators, int n_feat
  *                  the i-th tree. This is only defined if node k is an internal (non-leaf) node.
  * \param value value[i][k] stores the leaf output of node k of the i-th tree. This is only defined
  *              if node k is a leaf node.
+ * \param missing_go_to_left missing_go_to_left[i][k] stores the default direction for the missing
+ *                           value at node k of the i-th tree. This is only defined if node k is an
+ *                           internal (non-leaf) node.
  * \param n_node_samples n_node_samples[i][k] stores the number of data samples associated with
  *                       node k of the i-th tree.
  * \param weighted_n_node_samples weighted_n_node_samples[i][k] stores the sum of weighted data
@@ -270,11 +305,12 @@ TREELITE_DLL int TreeliteLoadSKLearnIsolationForest(int n_estimators, int n_feat
  * \param out Loaded model
  * \return 0 for success, -1 for failure
  */
-TREELITE_DLL int TreeliteLoadSKLearnRandomForestClassifier(int n_estimators, int n_features,
+TREELITE_DLL int TreeliteLoadSKLearnRandomForestClassifierEx(int n_estimators, int n_features,
     int n_targets, int32_t const* n_classes, int64_t const* node_count,
     int64_t const** children_left, int64_t const** children_right, int64_t const** feature,
-    double const** threshold, double const** value, int64_t const** n_node_samples,
-    double const** weighted_n_node_samples, double const** impurity, TreeliteModelHandle* out);
+    double const** threshold, double const** value, uint8_t const** missing_go_to_left,
+    int64_t const** n_node_samples, double const** weighted_n_node_samples, double const** impurity,
+    TreeliteModelHandle* out);
 /*!
  * \brief Load a scikit-learn GradientBoostingRegressor model from a collection of arrays. Refer
  *        to https://scikit-learn.org/stable/auto_examples/tree/plot_unveil_tree_structure.html to

include/treelite/model_loader.h

@@ -120,6 +120,9 @@ namespace sklearn {
  *                  the i-th tree. This is only defined if node k is an internal (non-leaf) node.
  * \param value value[i][k] stores the leaf output of node k of the i-th tree. This is only defined
  *              if node k is a leaf node.
+ * \param missing_go_to_left missing_go_to_left[i][k] stores the default direction for the missing
+ *                           value at node k of the i-th tree. This is only defined if node k is an
+ *                           internal (non-leaf) node.
  * \param n_node_samples n_node_samples[i][k] stores the number of data samples associated with
  *                       node k of the i-th tree.
  * \param weighted_n_node_samples weighted_n_node_samples[i][k] stores the sum of weighted data
@@ -131,8 +134,9 @@ namespace sklearn {
 std::unique_ptr<treelite::Model> LoadRandomForestRegressor(int n_estimators, int n_features,
     int n_targets, std::int64_t const* node_count, std::int64_t const** children_left,
     std::int64_t const** children_right, std::int64_t const** feature, double const** threshold,
-    double const** value, std::int64_t const** n_node_samples,
-    double const** weighted_n_node_samples, double const** impurity);
+    double const** value, std::uint8_t const** missing_go_to_left,
+    std::int64_t const** n_node_samples, double const** weighted_n_node_samples,
+    double const** impurity);
 /*!
  * \brief Load a scikit-learn IsolationForest model from a collection of arrays. Refer to
  *        https://scikit-learn.org/stable/auto_examples/tree/plot_unveil_tree_structure.html to
@@ -150,19 +154,24 @@ std::unique_ptr<treelite::Model> LoadRandomForestRegressor(int n_estimators, int
  *                  the i-th tree. This is only defined if node k is an internal (non-leaf) node.
  * \param value value[i][k] stores the expected isolation depth of node k of the i-th tree. This is
  *              only defined if node k is a leaf node.
+ * \param missing_go_to_left missing_go_to_left[i][k] stores the default direction for the missing
+ *                           value at node k of the i-th tree. This is only defined if node k is an
+ *                           internal (non-leaf) node.
  * \param n_node_samples n_node_samples[i][k] stores the number of data samples associated with
  *                       node k of the i-th tree.
  * \param weighted_n_node_samples weighted_n_node_samples[i][k] stores the sum of weighted data
  *                                samples associated with node k of the i-th tree.
  * \param impurity Not used, but must be passed as array of arrays for each tree and node.
  * \param ratio_c Standardizing constant to use for calculation of the anomaly score.
+ * \param offset Offset used to define the decision function from the raw scores.
  * \return Loaded model
  */
 std::unique_ptr<treelite::Model> LoadIsolationForest(int n_estimators, int n_features,
     std::int64_t const* node_count, std::int64_t const** children_left,
     std::int64_t const** children_right, std::int64_t const** feature, double const** threshold,
-    double const** value, std::int64_t const** n_node_samples,
-    double const** weighted_n_node_samples, double const** impurity, double ratio_c);
+    double const** value, std::uint8_t const** missing_go_to_left,
+    std::int64_t const** n_node_samples, double const** weighted_n_node_samples,
+    double const** impurity, double ratio_c, double offset);
 /*!
  * \brief Load a scikit-learn RandomForestClassifier model from a collection of arrays. Refer to
  *        https://scikit-learn.org/stable/auto_examples/tree/plot_unveil_tree_structure.html to
@@ -183,6 +192,9 @@ std::unique_ptr<treelite::Model> LoadIsolationForest(int n_estimators, int n_fea
  *                  the i-th tree. This is only defined if node k is an internal (non-leaf) node.
  * \param value value[i][k] stores the leaf output of node k of the i-th tree. This is only defined
  *              if node k is a leaf node.
+ * \param missing_go_to_left missing_go_to_left[i][k] stores the default direction for the missing
+ *                           value at node k of the i-th tree. This is only defined if node k is an
+ *                           internal (non-leaf) node.
  * \param n_node_samples n_node_samples[i][k] stores the number of data samples associated with
  *                       node k of the i-th tree.
  * \param weighted_n_node_samples weighted_n_node_samples[i][k] stores the sum of weighted data
@@ -195,8 +207,8 @@ std::unique_ptr<treelite::Model> LoadRandomForestClassifier(int n_estimators, in
     int n_targets, int32_t const* n_classes, std::int64_t const* node_count,
     std::int64_t const** children_left, std::int64_t const** children_right,
     std::int64_t const** feature, double const** threshold, double const** value,
-    std::int64_t const** n_node_samples, double const** weighted_n_node_samples,
-    double const** impurity);
+    std::uint8_t const** missing_go_to_left, std::int64_t const** n_node_samples,
+    double const** weighted_n_node_samples, double const** impurity);
 /*!
  * \brief Load a scikit-learn GradientBoostingRegressor model from a collection of arrays. Refer
  *        to https://scikit-learn.org/stable/auto_examples/tree/plot_unveil_tree_structure.html to

include/treelite/tree.h

@@ -149,9 +149,10 @@ class Tree {
   template <typename X, typename Y>
   friend void BulkConstructTree(Tree<X, Y>& tree, int n_nodes, std::int64_t const* children_left,
       std::int64_t const* children_right, std::int64_t const* feature, double const* threshold,
-      double const* value, std::int64_t const* n_node_samples,
-      double const* weighted_n_node_samples, double const* impurity, std::int64_t total_sample_cnt,
-      int n_targets, int max_num_class, bool is_classifier);
+      double const* value, std::uint8_t const* missing_go_to_left,
+      std::int64_t const* n_node_samples, double const* weighted_n_node_samples,
+      double const* impurity, std::int64_t total_sample_cnt, int n_targets, int max_num_class,
+      bool is_classifier);
 
  public:
   /*! \brief Number of nodes */

python/treelite/sklearn/importer.py

@@ -20,6 +20,7 @@ class ArrayOfArrays:
 
     def __init__(self, *, dtype):
         int8_ptr_type = ctypes.POINTER(ctypes.c_int8)
+        uint8_ptr_type = ctypes.POINTER(ctypes.c_uint8)
         int64_ptr_type = ctypes.POINTER(ctypes.c_int64)
         uint32_ptr_type = ctypes.POINTER(ctypes.c_uint32)
         float64_ptr_type = ctypes.POINTER(ctypes.c_double)
@@ -32,6 +33,8 @@ def __init__(self, *, dtype):
             self.ptr_type = uint32_ptr_type
         elif dtype == np.int8:
             self.ptr_type = int8_ptr_type
+        elif dtype == np.uint8:
+            self.ptr_type = uint8_ptr_type
         elif dtype == "void":
             self.ptr_type = void_ptr_type
         else:
@@ -188,6 +191,7 @@ def import_model(sklearn_model) -> Model:
     feature = ArrayOfArrays(dtype=np.int64)
     threshold = ArrayOfArrays(dtype=np.float64)
     value = ArrayOfArrays(dtype=np.float64)
+    missing_go_to_left = ArrayOfArrays(dtype=np.uint8)
     n_node_samples = ArrayOfArrays(dtype=np.int64)
     weighted_n_node_samples = ArrayOfArrays(dtype=np.float64)
     impurity = ArrayOfArrays(dtype=np.float64)
@@ -233,6 +237,9 @@ def import_model(sklearn_model) -> Model:
                     expected_shape=leaf_value_expected_shape(tree.node_count),
                 )
                 feature.add(tree.feature, expected_shape=(tree.node_count,))
+            missing_go_to_left.add(
+                tree.missing_go_to_left, expected_shape=(tree.node_count,)
+            )
             n_node_samples.add(tree.n_node_samples, expected_shape=(tree.node_count,))
             weighted_n_node_samples.add(
                 tree.weighted_n_node_samples, expected_shape=(tree.node_count,)
@@ -242,7 +249,7 @@ def import_model(sklearn_model) -> Model:
     handle = ctypes.c_void_p()
     if isinstance(sklearn_model, (RandomForestR, ExtraTreesR)):
         _check_call(
-            _LIB.TreeliteLoadSKLearnRandomForestRegressor(
+            _LIB.TreeliteLoadSKLearnRandomForestRegressorEx(
                 ctypes.c_int(sklearn_model.n_estimators),
                 ctypes.c_int(sklearn_model.n_features_in_),
                 ctypes.c_int(sklearn_model.n_outputs_),
@@ -252,16 +259,16 @@ def import_model(sklearn_model) -> Model:
                 feature.as_c_array(),
                 threshold.as_c_array(),
                 value.as_c_array(),
+                missing_go_to_left.as_c_array(),
                 n_node_samples.as_c_array(),
                 weighted_n_node_samples.as_c_array(),
                 impurity.as_c_array(),
                 ctypes.byref(handle),
             )
         )
     elif isinstance(sklearn_model, IsolationForest):
-        # TODO(chyunsu3): In Treelite 5.0, pass offset_ field via TreeliteLoadSKLearnIsolationForest()
         _check_call(
-            _LIB.TreeliteLoadSKLearnIsolationForest(
+            _LIB.TreeliteLoadSKLearnIsolationForestEx(
                 ctypes.c_int(sklearn_model.n_estimators),
                 ctypes.c_int(sklearn_model.n_features_in_),
                 c_array(ctypes.c_int64, node_count),
@@ -270,31 +277,19 @@ def import_model(sklearn_model) -> Model:
                 feature.as_c_array(),
                 threshold.as_c_array(),
                 value.as_c_array(),
+                missing_go_to_left.as_c_array(),
                 n_node_samples.as_c_array(),
                 weighted_n_node_samples.as_c_array(),
                 impurity.as_c_array(),
                 ctypes.c_double(ratio_c),
+                ctypes.c_double(sklearn_model.offset_),
                 ctypes.byref(handle),
             )
         )
-        # Store `offset_` field as a model attribute
-        attributes = {
-            "sklearn_iforest_offset": float(sklearn_model.offset_),
-        }
-        attributes_serialized = json.dumps(attributes)
-        _check_call(
-            _LIB.TreeliteSetHeaderField(
-                handle,
-                c_str("attributes"),
-                _numpy2pybuffer(
-                    np.frombuffer(attributes_serialized.encode("utf-8"), dtype="S1")
-                ),
-            )
-        )
     elif isinstance(sklearn_model, (RandomForestC, ExtraTreesC)):
         n_classes = np.array(sklearn_model.n_classes_, dtype=np.int32)
         _check_call(
-            _LIB.TreeliteLoadSKLearnRandomForestClassifier(
+            _LIB.TreeliteLoadSKLearnRandomForestClassifierEx(
                 ctypes.c_int(sklearn_model.n_estimators),
                 ctypes.c_int(sklearn_model.n_features_in_),
                 ctypes.c_int(sklearn_model.n_outputs_),
@@ -305,6 +300,7 @@ def import_model(sklearn_model) -> Model:
                 feature.as_c_array(),
                 threshold.as_c_array(),
                 value.as_c_array(),
+                missing_go_to_left.as_c_array(),
                 n_node_samples.as_c_array(),
                 weighted_n_node_samples.as_c_array(),
                 impurity.as_c_array(),