ENH Support for monotonic trees with missing values (scikit-learn#27630)

Co-authored-by: Guillaume Lemaitre <g.lemaitre58@gmail.com>

Author: samronsin

doc/whats_new/upcoming_changes/sklearn.tree/27630.enhancement.rst

@@ -0,0 +1,9 @@
+- :class:`tree.DecisionTreeClassifier`, :class:`tree.DecisionTreeRegressor`,
+  :class:`tree.ExtraTreeClassifier`, :class:`tree.ExtraTreeRegressor`,
+  :class:`ensemble.RandomForestClassifier`,
+  :class:`ensemble.RandomForestRegressor`, :class:`ensemble.ExtraTreesClassifier`,
+  and :class:`ensemble.ExtraTreesRegressor` now support combining
+  `monotonic_cst` with missing values in dense training data. This builds on
+  the improvements to missing-value support for dense training data in
+  :pr:`32119`.
+  By :user:`Samuel O. Ronsin <samronsin>`.

sklearn/ensemble/_forest.py

@@ -1387,8 +1387,7 @@ class RandomForestClassifier(ForestClassifier):
 
         Monotonicity constraints are not supported for:
           - multiclass classifications (i.e. when `n_classes > 2`),
-          - multioutput classifications (i.e. when `n_outputs_ > 1`),
-          - classifications trained on data with missing values.
+          - multioutput classifications (i.e. when `n_outputs_ > 1`).
 
         The constraints hold over the probability of the positive class.
 
@@ -1779,8 +1778,7 @@ class RandomForestRegressor(ForestRegressor):
         If monotonic_cst is None, no constraints are applied.
 
         Monotonicity constraints are not supported for:
-          - multioutput regressions (i.e. when `n_outputs_ > 1`),
-          - regressions trained on data with missing values.
+          - multioutput regressions (i.e. when `n_outputs_ > 1`).
 
         Read more in the :ref:`User Guide <monotonic_cst_gbdt>`.
 
@@ -2174,8 +2172,7 @@ class ExtraTreesClassifier(ForestClassifier):
 
         Monotonicity constraints are not supported for:
           - multiclass classifications (i.e. when `n_classes > 2`),
-          - multioutput classifications (i.e. when `n_outputs_ > 1`),
-          - classifications trained on data with missing values.
+          - multioutput classifications (i.e. when `n_outputs_ > 1`).
 
         The constraints hold over the probability of the positive class.
 
@@ -2549,8 +2546,7 @@ class ExtraTreesRegressor(ForestRegressor):
         If monotonic_cst is None, no constraints are applied.
 
         Monotonicity constraints are not supported for:
-          - multioutput regressions (i.e. when `n_outputs_ > 1`),
-          - regressions trained on data with missing values.
+          - multioutput regressions (i.e. when `n_outputs_ > 1`).
 
         Read more in the :ref:`User Guide <monotonic_cst_gbdt>`.
 

sklearn/tree/_classes.py

@@ -184,11 +184,7 @@ def get_n_leaves(self):
         return self.tree_.n_leaves
 
     def _support_missing_values(self, X):
-        return (
-            not issparse(X)
-            and self.__sklearn_tags__().input_tags.allow_nan
-            and self.monotonic_cst is None
-        )
+        return not issparse(X) and self.__sklearn_tags__().input_tags.allow_nan
 
     def _compute_missing_values_in_feature_mask(self, X, estimator_name=None):
         """Return boolean mask denoting if there are missing values for each feature.
@@ -849,8 +845,7 @@ class DecisionTreeClassifier(ClassifierMixin, BaseDecisionTree):
 
         Monotonicity constraints are not supported for:
           - multiclass classifications (i.e. when `n_classes > 2`),
-          - multioutput classifications (i.e. when `n_outputs_ > 1`),
-          - classifications trained on data with missing values.
+          - multioutput classifications (i.e. when `n_outputs_ > 1`).
 
         The constraints hold over the probability of the positive class.
 
@@ -1243,8 +1238,7 @@ class DecisionTreeRegressor(RegressorMixin, BaseDecisionTree):
         If monotonic_cst is None, no constraints are applied.
 
         Monotonicity constraints are not supported for:
-          - multioutput regressions (i.e. when `n_outputs_ > 1`),
-          - regressions trained on data with missing values.
+          - multioutput regressions (i.e. when `n_outputs_ > 1`).
 
         Read more in the :ref:`User Guide <monotonic_cst_gbdt>`.
 
@@ -1601,8 +1595,7 @@ class ExtraTreeClassifier(DecisionTreeClassifier):
 
         Monotonicity constraints are not supported for:
           - multiclass classifications (i.e. when `n_classes > 2`),
-          - multioutput classifications (i.e. when `n_outputs_ > 1`),
-          - classifications trained on data with missing values.
+          - multioutput classifications (i.e. when `n_outputs_ > 1`).
 
         The constraints hold over the probability of the positive class.
 
@@ -1877,8 +1870,7 @@ class ExtraTreeRegressor(DecisionTreeRegressor):
         If monotonic_cst is None, no constraints are applied.
 
         Monotonicity constraints are not supported for:
-          - multioutput regressions (i.e. when `n_outputs_ > 1`),
-          - regressions trained on data with missing values.
+          - multioutput regressions (i.e. when `n_outputs_ > 1`).
 
         Read more in the :ref:`User Guide <monotonic_cst_gbdt>`.
 

sklearn/tree/tests/test_monotonic_tree.py

@@ -30,13 +30,22 @@
 
 
 @pytest.mark.parametrize("TreeClassifier", TREE_BASED_CLASSIFIER_CLASSES)
+@pytest.mark.parametrize(
+    "sparse_splitter, with_missing",
+    [
+        (False, False),
+        (True, False),
+        (False, True),
+    ],
+    ids=["dense-without-missing", "sparse-without-missing", "dense-with-missing"],
+)
 @pytest.mark.parametrize("depth_first_builder", (True, False))
-@pytest.mark.parametrize("sparse_splitter", (True, False))
 @pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
 def test_monotonic_constraints_classifications(
     TreeClassifier,
-    depth_first_builder,
     sparse_splitter,
+    depth_first_builder,
+    with_missing,
     global_random_seed,
     csc_container,
 ):
@@ -72,9 +81,13 @@ def test_monotonic_constraints_classifications(
             max_leaf_nodes=n_samples_train,
         )
     if hasattr(est, "random_state"):
-        est.set_params(**{"random_state": global_random_seed})
+        est.set_params(random_state=global_random_seed)
     if hasattr(est, "n_estimators"):
-        est.set_params(**{"n_estimators": 5})
+        est.set_params(n_estimators=5)
+    if with_missing:
+        generator = np.random.default_rng(seed=global_random_seed)
+        mask = generator.choice(2, size=X_train.shape).astype(bool)
+        X_train[mask] = np.nan
     if sparse_splitter:
         X_train = csc_container(X_train)
     est.fit(X_train, y_train)
@@ -95,14 +108,23 @@ def test_monotonic_constraints_classifications(
 
 
 @pytest.mark.parametrize("TreeRegressor", TREE_BASED_REGRESSOR_CLASSES)
+@pytest.mark.parametrize(
+    "sparse_splitter, with_missing",
+    [
+        (False, False),
+        (True, False),
+        (False, True),
+    ],
+    ids=["dense-without-missing", "sparse-without-missing", "dense-with-missing"],
+)
 @pytest.mark.parametrize("depth_first_builder", (True, False))
-@pytest.mark.parametrize("sparse_splitter", (True, False))
 @pytest.mark.parametrize("criterion", ("absolute_error", "squared_error"))
 @pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
 def test_monotonic_constraints_regressions(
     TreeRegressor,
-    depth_first_builder,
     sparse_splitter,
+    depth_first_builder,
+    with_missing,
     criterion,
     global_random_seed,
     csc_container,
@@ -145,7 +167,11 @@ def test_monotonic_constraints_regressions(
     if hasattr(est, "random_state"):
         est.set_params(random_state=global_random_seed)
     if hasattr(est, "n_estimators"):
-        est.set_params(**{"n_estimators": 5})
+        est.set_params(n_estimators=5)
+    if with_missing:
+        generator = np.random.default_rng(seed=global_random_seed)
+        mask = generator.choice(2, size=X_train.shape).astype(bool)
+        X_train[mask] = np.nan
     if sparse_splitter:
         X_train = csc_container(X_train)
     est.fit(X_train, y_train)
@@ -190,29 +216,6 @@ def test_multiple_output_raises(TreeClassifier):
         est.fit(X, y)
 
 
-@pytest.mark.parametrize(
-    "Tree",
-    [
-        DecisionTreeClassifier,
-        DecisionTreeRegressor,
-        ExtraTreeClassifier,
-        ExtraTreeRegressor,
-    ],
-)
-def test_missing_values_raises(Tree):
-    X, y = make_classification(
-        n_samples=100, n_features=5, n_classes=2, n_informative=3, random_state=0
-    )
-    X[0, 0] = np.nan
-    monotonic_cst = np.zeros(X.shape[1])
-    monotonic_cst[0] = 1
-    est = Tree(max_depth=None, monotonic_cst=monotonic_cst, random_state=0)
-
-    msg = "Input X contains NaN"
-    with pytest.raises(ValueError, match=msg):
-        est.fit(X, y)
-
-
 @pytest.mark.parametrize("TreeClassifier", TREE_BASED_CLASSIFIER_CLASSES)
 def test_bad_monotonic_cst_raises(TreeClassifier):
     X = [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10]]
@@ -307,11 +310,17 @@ def test_1d_opposite_monotonicity_cst_data(TreeRegressor):
 
 
 @pytest.mark.parametrize("TreeRegressor", TREE_REGRESSOR_CLASSES)
+@pytest.mark.parametrize("with_missing", (True, False))
 @pytest.mark.parametrize("monotonic_sign", (-1, 1))
 @pytest.mark.parametrize("depth_first_builder", (True, False))
 @pytest.mark.parametrize("criterion", ("absolute_error", "squared_error"))
 def test_1d_tree_nodes_values(
-    TreeRegressor, monotonic_sign, depth_first_builder, criterion, global_random_seed
+    TreeRegressor,
+    with_missing,
+    monotonic_sign,
+    depth_first_builder,
+    criterion,
+    global_random_seed,
 ):
     # Adaptation from test_nodes_values in test_monotonic_constraints.py
     # in sklearn.ensemble._hist_gradient_boosting
@@ -351,10 +360,15 @@ def test_1d_tree_nodes_values(
             criterion=criterion,
             random_state=global_random_seed,
         )
+    if with_missing:
+        generator = np.random.default_rng(seed=global_random_seed)
+        mask = generator.choice(2, size=X.shape, p=[0.8, 0.2]).astype(bool)
+        X[mask] = np.nan
     clf.fit(X, y)
 
     assert_1d_reg_tree_children_monotonic_bounded(clf.tree_, monotonic_sign)
-    assert_1d_reg_monotonic(clf, monotonic_sign, np.min(X), np.max(X), 100)
+    min_x, max_x = np.nanmin(X), np.nanmax(X)
+    assert_1d_reg_monotonic(clf, monotonic_sign, min_x, max_x, 100)
 
 
 def assert_nd_reg_tree_children_monotonic_bounded(tree_, monotonic_cst):
@@ -379,7 +393,7 @@ def assert_nd_reg_tree_children_monotonic_bounded(tree_, monotonic_cst):
         # Split node: check and update bounds for the children.
         i_left = tree_.children_left[i]
         i_right = tree_.children_right[i]
-        # unpack value from nx1x1 array
+        # unpack value from nx1x1 array (middle_value after clipping)
         middle_value = (tree_.value[i_left][0][0] + tree_.value[i_right][0][0]) / 2
 
         if monotonic_cst[feature] == 0:
@@ -460,11 +474,17 @@ def test_assert_nd_reg_tree_children_monotonic_bounded():
 
 
 @pytest.mark.parametrize("TreeRegressor", TREE_REGRESSOR_CLASSES)
+@pytest.mark.parametrize("with_missing", (True, False))
 @pytest.mark.parametrize("monotonic_sign", (-1, 1))
 @pytest.mark.parametrize("depth_first_builder", (True, False))
 @pytest.mark.parametrize("criterion", ("absolute_error", "squared_error"))
 def test_nd_tree_nodes_values(
-    TreeRegressor, monotonic_sign, depth_first_builder, criterion, global_random_seed
+    TreeRegressor,
+    with_missing,
+    monotonic_sign,
+    depth_first_builder,
+    criterion,
+    global_random_seed,
 ):
     # Build tree with several features, and make sure the nodes
     # values respect the monotonicity constraints.
@@ -508,5 +528,9 @@ def test_nd_tree_nodes_values(
             criterion=criterion,
             random_state=global_random_seed,
         )
+    if with_missing:
+        generator = np.random.default_rng(seed=global_random_seed)
+        mask = generator.choice(2, size=X.shape, p=[0.8, 0.2]).astype(bool)
+        X[mask] = np.nan
     clf.fit(X, y)
     assert_nd_reg_tree_children_monotonic_bounded(clf.tree_, monotonic_cst)