Nocats

Author: adam2392

doc/whats_new/upcoming_changes/sklearn.tree/33354.enhancement.rst

@@ -0,0 +1,5 @@
+- :class:`tree.DecisionTreeRegressor`, :class:`tree.DecisionTreeClassifier`
+  now have native support for categorical features for binary classification and
+  single-output regression. Categorical features can be specified with the
+  `categorical_features` parameter. Up to 256 categories per features are supported.
+  By `:user:`Adam Li <adam2392>`, `:user:`Arthur Lacote <cakedev0>` and :user:`Christian Lorentzen <lorentzenchr>`

sklearn/ensemble/_forest.py

@@ -1512,6 +1512,7 @@ class labels (multi-output problem).
         ],
     }
     _parameter_constraints.pop("splitter")
+    _parameter_constraints.pop("categorical_features")
 
     def __init__(
         self,
@@ -1889,6 +1890,7 @@ class RandomForestRegressor(ForestRegressor):
         **DecisionTreeRegressor._parameter_constraints,
     }
     _parameter_constraints.pop("splitter")
+    _parameter_constraints.pop("categorical_features")
 
     def __init__(
         self,
@@ -2285,6 +2287,8 @@ class labels (multi-output problem).
         ],
     }
     _parameter_constraints.pop("splitter")
+    # TODO: remove once randomsplitter supports categorical features
+    _parameter_constraints.pop("categorical_features")
 
     def __init__(
         self,
@@ -2641,6 +2645,8 @@ class ExtraTreesRegressor(ForestRegressor):
         **DecisionTreeRegressor._parameter_constraints,
     }
     _parameter_constraints.pop("splitter")
+    # TODO: remove once randomsplitter supports categorical features
+    _parameter_constraints.pop("categorical_features")
 
     def __init__(
         self,
@@ -2904,7 +2910,13 @@ class RandomTreesEmbedding(TransformerMixin, BaseForest):
         **BaseDecisionTree._parameter_constraints,
         "sparse_output": ["boolean"],
     }
-    for param in ("max_features", "ccp_alpha", "splitter", "monotonic_cst"):
+    for param in (
+        "max_features",
+        "ccp_alpha",
+        "splitter",
+        "monotonic_cst",
+        "categorical_features",
+    ):
         _parameter_constraints.pop(param)
 
     criterion = "squared_error"

sklearn/ensemble/_gb.py

@@ -378,6 +378,7 @@ class BaseGradientBoosting(BaseEnsemble, metaclass=ABCMeta):
     }
     _parameter_constraints.pop("splitter")
     _parameter_constraints.pop("monotonic_cst")
+    _parameter_constraints.pop("categorical_features")
 
     @abstractmethod
     def __init__(

sklearn/ensemble/_gradient_boosting.pyx

@@ -66,7 +66,7 @@ cdef void _predict_regression_tree_inplace_fast_dense(
         node = root_node
         # While node not a leaf
         while node.left_child != TREE_LEAF:
-            if X[i, node.feature] <= node.threshold:
+            if X[i, node.feature] <= node.split_value.threshold:
                 node = root_node + node.left_child
             else:
                 node = root_node + node.right_child
@@ -148,7 +148,7 @@ def _predict_regression_tree_stages_sparse(
                     else:
                         feature_value = 0.
 
-                    if feature_value <= node.threshold:
+                    if feature_value <= node.split_value.threshold:
                         node = root_node + node.left_child
                     else:
                         node = root_node + node.right_child

sklearn/inspection/tests/test_partial_dependence.py

@@ -523,6 +523,57 @@ def test_partial_dependence_easy_target(est, power):
     assert r2 > 0.99
 
 
+def test_partial_dependence_recursion_decision_tree_categorical_target_feature():
+    # Non-ordinal categorical signal: categories {1, 2, 5} -> 1 and
+    # {0, 3, 4} -> 0. Recursion must route with the categorical bitset.
+    category_values = np.arange(6, dtype=np.float64)
+    X = np.repeat(category_values, 4).reshape(-1, 1)
+    y = np.isin(X.ravel(), [1, 2, 5]).astype(np.float64)
+
+    est = DecisionTreeRegressor(
+        categorical_features=[0], max_depth=1, random_state=0
+    ).fit(X, y)
+
+    recursion = partial_dependence(
+        est, X, features=[0], method="recursion", categorical_features=[0]
+    )
+    brute = partial_dependence(
+        est, X, features=[0], method="brute", categorical_features=[0]
+    )
+
+    grid = recursion["grid_values"][0].reshape(-1, 1)
+    expected = est.predict(grid)
+
+    assert_array_equal(recursion["grid_values"][0], category_values)
+    assert_allclose(recursion["average"][0], brute["average"][0])
+    assert_allclose(recursion["average"][0], expected)
+
+
+def test_partial_dependence_recursion_decision_tree_missing_target_feature():
+    # Missing values are isolated by the split. Recursion must route np.nan
+    # according to the fitted node's missing_go_to_left flag.
+    X = np.array([0.0, 0.0, np.nan, np.nan, 1.0, 1.0, 1.0]).reshape(-1, 1)
+    y = np.array([0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0])
+
+    est = DecisionTreeRegressor(max_depth=1, random_state=0).fit(X, y)
+    assert est.tree_.missing_go_to_left[0]
+    custom_values = {0: [0.0, np.nan, 1.0]}
+
+    recursion = partial_dependence(
+        est, X, features=[0], method="recursion", custom_values=custom_values
+    )
+    brute = partial_dependence(
+        est, X, features=[0], method="brute", custom_values=custom_values
+    )
+
+    grid = recursion["grid_values"][0].reshape(-1, 1)
+    expected = est.predict(grid)
+
+    assert_array_equal(recursion["grid_values"][0], custom_values[0])
+    assert_allclose(recursion["average"][0], brute["average"][0])
+    assert_allclose(recursion["average"][0], expected)
+
+
 @pytest.mark.parametrize(
     "Estimator",
     (