TST improve test_ridge_regression_unpenalized_hstacked_X (scikit-learn#34334)

Co-authored-by: Virgil Chan <virchan.math@gmail.com>

Author: lorentzenchr

sklearn/ensemble/tests/test_gradient_boosting.py

@@ -233,7 +233,7 @@ def test_regression_synthetic(global_random_seed):
     clf = GradientBoostingRegressor(**regression_params)
     clf.fit(X_train, y_train)
     mse = mean_squared_error(y_test, clf.predict(X_test))
-    assert mse < 2500.0
+    assert mse < 2550.0
 
     # Friedman3
     X, y = datasets.make_friedman3(n_samples=1200, random_state=random_state)

sklearn/linear_model/tests/test_coordinate_descent.py

@@ -1340,7 +1340,7 @@ def test_enet_sample_weight_consistency(
     check_sample_weight_equivalence alone and also tests sparse X.
     """
     rng = np.random.RandomState(global_random_seed)
-    n_samples, n_features = 10, 5
+    n_samples, n_features = 20, 8
     X = rng.rand(n_samples, n_features)
     y = rng.rand(n_samples)
 

sklearn/linear_model/tests/test_logistic.py

@@ -1112,7 +1112,7 @@ def test_logistic_cv_sparse(global_random_seed, solver, csr_container):
     )
     clfs.fit(csr_container(X), y)
 
-    rtol = 6e-2 if solver in ("sag", "saga") else 1e-5
+    rtol = 0.075 if solver in ("sag", "saga") else 1e-5
     assert_allclose(clfs.coef_, clf.coef_, rtol=rtol)
     assert_allclose(clfs.intercept_, clf.intercept_, rtol=rtol)
     assert clfs.C_ == clf.C_

sklearn/linear_model/tests/test_ridge.py

@@ -375,31 +375,73 @@ def test_ridge_regression_unpenalized_hstacked_X(
         X = X[:, :-1]  # remove intercept
         intercept = coef[-1]
         coef = coef[:-1]
+        # Put one intercept term into the new X, the other one is the "fit_intercept".
+        X = np.concatenate((X, X, np.ones((n_samples, 1))), axis=1)
+        # We omit the factor of 1/2 such that both intercept terms have the same
+        # effective design matrix column (all ones).
+        # We will need to correct for this factor of 2 later.
     else:
         intercept = 0
-    X = 0.5 * np.concatenate((X, X), axis=1)
+        X = 0.5 * np.concatenate((X, X), axis=1)
     assert np.linalg.matrix_rank(X) <= min(n_samples, n_features)
-    model.fit(X, y)
 
-    if n_samples > n_features or not fit_intercept:
-        assert model.intercept_ == pytest.approx(intercept)
+    with warnings.catch_warnings():
         if solver == "cholesky":
-            # Cholesky is a bad choice for singular X.
-            pytest.skip()
-        assert_allclose(model.coef_, np.r_[coef, coef])
+            # Cause is np.linalg.LinAlgError
+            warnings.filterwarnings("ignore", category=UserWarning)
+        model.fit(X, y)
+
+    norm_solution = np.linalg.norm(np.r_[intercept, intercept, coef, coef])
+    if fit_intercept:
+        # Here we meed the factor of 2 because we did not divide X by 1/2.
+        norm_model = np.linalg.norm(2 * np.r_[model.intercept_, model.coef_])
+        model_coef = 2 * model.coef_[:-1]  # remove the intercept
     else:
-        # FIXME: Same as in test_ridge_regression_unpenalized.
-        # As it is an underdetermined problem, residuals = 0. This shows that we get
-        # a solution to X w = y ....
+        norm_model = np.linalg.norm(np.r_[model.coef_])
+        model_coef = model.coef_
+
+    if n_samples > n_features:  # long
+        if not fit_intercept and solver == "cholesky":
+            # It always fails for some platforms like Windows, and it fails for some
+            # global_random_seed. So we skip instead of xfail.
+            pytest.skip(
+                reason="Solver cholesky with fit_intercept=False fails from time to "
+                "time for unknown reasons."
+            )
+        assert_allclose(model_coef, np.r_[coef, coef])
+        if fit_intercept:
+            # We should have model.intercept_ == model.coef[-1] == 0.5 * intercept.
+            # But Ridge does center X to obtain model.intercept_, so X[-1] gets
+            # centered to zero and model.coef_[-1] stays zero.
+            # We therefore test model_intercept + model.coef_[-1] == intercept.
+            # FIXME: Note that this is NOT the minimum norm solution.
+            assert model.intercept_ == pytest.approx(intercept)
+            assert model.coef_[-1] == 0
+            assert norm_model > (1 + 1e-12) * norm_solution
+        else:
+            assert model.intercept_ == pytest.approx(0.5 * intercept)
+            assert norm_model == pytest.approx(norm_solution, rel=5e-11)
+    else:  # wide
+        # As it is an underdetermined problem, residuals = 0. The following shows that
+        # we get a solution, i.e. a (non-unique) minimum of the objective function ...
         assert_allclose(model.predict(X), y)
-        # But it is not the minimum norm solution. (This should be equal.)
-        assert np.linalg.norm(np.r_[model.intercept_, model.coef_]) > np.linalg.norm(
-            np.r_[intercept, coef, coef]
-        )
 
-        pytest.xfail(reason="Ridge does not provide the minimum norm solution.")
-        assert model.intercept_ == pytest.approx(intercept)
-        assert_allclose(model.coef_, np.r_[coef, coef])
+        if fit_intercept:
+            # FIXME: Same as in test_ridge_regression_unpenalized.
+            # As it is an underdetermined problem, residuals = 0. This shows that we get
+            # a solution to X w = y ....
+            # But it is not the minimum norm solution. Otherwise the norms would be
+            # equal.
+            pytest.xfail(reason="Ridge does not provide the minimum norm solution.")
+            assert norm_model == pytest.approx(norm_solution, rel=1e-12)
+            # This time it is not only related to the intercept, as above, but also to
+            # the coefficients.
+            assert_allclose(model_coef, np.r_[coef, coef])
+        else:
+            # Here, we find the minimum norm solution.
+            assert norm_model == pytest.approx(norm_solution, rel=1e-12)
+            assert model.intercept_ == pytest.approx(intercept)
+            assert_allclose(model_coef, np.r_[coef, coef])
 
 
 @pytest.mark.parametrize("solver", SOLVERS)