docs: Use skore for evaluation models

examples/applications/plot_impact_imbalanced_classes.py

@@ -45,7 +45,7 @@
 # %%
 from imblearn.datasets import make_imbalance
 
-ratio = 30
+ratio, pos_label = 30, ">50K"
 df_res, y_res = make_imbalance(
     df,
     y,
@@ -54,163 +54,67 @@
 y_res.value_counts()
 
 # %% [markdown]
-# We will perform a cross-validation evaluation to get an estimate of the test
-# score.
+# We will use `skore.evaluate` to get an estimate of the test scores using
+# cross-validation.
 #
 # As a baseline, we could use a classifier which will always predict the
 # majority class independently of the features provided.
 
-from sklearn.dummy import DummyClassifier
-
 # %%
-from sklearn.model_selection import cross_validate
+import skore
+from sklearn.dummy import DummyClassifier
 
 dummy_clf = DummyClassifier(strategy="most_frequent")
-scoring = ["accuracy", "balanced_accuracy"]
-cv_result = cross_validate(dummy_clf, df_res, y_res, scoring=scoring)
-print(f"Accuracy score of a dummy classifier: {cv_result['test_accuracy'].mean():.3f}")
-
-# %% [markdown]
-# Instead of using the accuracy, we can use the balanced accuracy which will
-# take into account the balancing issue.
-
-# %%
-print(
-    "Balanced accuracy score of a dummy classifier: "
-    f"{cv_result['test_balanced_accuracy'].mean():.3f}"
-)
+report = skore.evaluate(dummy_clf, df_res, y_res, splitter=5, pos_label=pos_label)
+report.metrics.summarize().frame()
 
 # %% [markdown]
 # Strategies to learn from an imbalanced dataset
-# ----------------------------------------------
-# We will use a dictionary and a list to continuously store the results of
-# our experiments and show them as a pandas dataframe.
-
-# %%
-index = []
-scores = {"Accuracy": [], "Balanced accuracy": []}
-
-# %% [markdown]
+# -----------------------------------------------
+#
+# We will compare different strategies to learn from an imbalanced dataset by
+# collecting all estimators and evaluating each using `skore.evaluate` to
+# compute cross-validated metrics.
+#
 # Dummy baseline
 # ..............
 #
 # Before to train a real machine learning model, we can store the results
 # obtained with our :class:`~sklearn.dummy.DummyClassifier`.
 
 # %%
-import pandas as pd
-
-index += ["Dummy classifier"]
-cv_result = cross_validate(dummy_clf, df_res, y_res, scoring=scoring)
-scores["Accuracy"].append(cv_result["test_accuracy"].mean())
-scores["Balanced accuracy"].append(cv_result["test_balanced_accuracy"].mean())
-
-df_scores = pd.DataFrame(scores, index=index)
-df_scores
+estimators = [("Dummy classifier", dummy_clf)]
 
 # %% [markdown]
 # Linear classifier baseline
 # ..........................
 #
-# We will create a machine learning pipeline using a
-# :class:`~sklearn.linear_model.LogisticRegression` classifier. In this regard,
-# we will need to one-hot encode the categorical columns and standardized the
-# numerical columns before to inject the data into the
-# :class:`~sklearn.linear_model.LogisticRegression` classifier.
-#
-# First, we define our numerical and categorical pipelines.
-
-# %%
-from sklearn.impute import SimpleImputer
-from sklearn.pipeline import make_pipeline
-from sklearn.preprocessing import OneHotEncoder, StandardScaler
-
-num_pipe = make_pipeline(
-    StandardScaler(), SimpleImputer(strategy="mean", add_indicator=True)
-)
-cat_pipe = make_pipeline(
-    SimpleImputer(strategy="constant", fill_value="missing"),
-    OneHotEncoder(handle_unknown="ignore"),
-)
-
-# %% [markdown]
-# Then, we can create a preprocessor which will dispatch the categorical
-# columns to the categorical pipeline and the numerical columns to the
-# numerical pipeline
-
-# %%
-from sklearn.compose import make_column_selector as selector
-from sklearn.compose import make_column_transformer
-
-preprocessor_linear = make_column_transformer(
-    (num_pipe, selector(dtype_include="number")),
-    (cat_pipe, selector(dtype_include="category")),
-    n_jobs=2,
-)
-
-# %% [markdown]
-# Finally, we connect our preprocessor with our
-# :class:`~sklearn.linear_model.LogisticRegression`. We can then evaluate our
-# model.
+# We use `skrub.tabular_pipeline` to create a machine learning pipeline with
+# proper preprocessing automatically adapted to the estimator. For a
+# :class:`~sklearn.linear_model.LogisticRegression`, it will automatically
+# handle missing values, encode categorical columns, and scale numerical
+# columns.
 
 # %%
 from sklearn.linear_model import LogisticRegression
+from skrub import tabular_pipeline
 
-lr_clf = make_pipeline(preprocessor_linear, LogisticRegression(max_iter=1000))
-
-# %%
-index += ["Logistic regression"]
-cv_result = cross_validate(lr_clf, df_res, y_res, scoring=scoring)
-scores["Accuracy"].append(cv_result["test_accuracy"].mean())
-scores["Balanced accuracy"].append(cv_result["test_balanced_accuracy"].mean())
-
-df_scores = pd.DataFrame(scores, index=index)
-df_scores
+lr_clf = tabular_pipeline(LogisticRegression(max_iter=1000))
+estimators.append(("Logistic regression", lr_clf))
 
 # %% [markdown]
-# We can see that our linear model is learning slightly better than our dummy
-# baseline. However, it is impacted by the class imbalance.
-#
 # We can verify that something similar is happening with a tree-based model
-# such as :class:`~sklearn.ensemble.RandomForestClassifier`. With this type of
-# classifier, we will not need to scale the numerical data, and we will only
-# need to ordinal encode the categorical data.
-
-from sklearn.ensemble import RandomForestClassifier
+# such as :class:`~sklearn.ensemble.RandomForestClassifier`. `tabular_pipeline`
+# will automatically adapt the preprocessing for tree-based models (e.g. no
+# scaling needed).
 
 # %%
-from sklearn.preprocessing import OrdinalEncoder
-
-num_pipe = SimpleImputer(strategy="mean", add_indicator=True)
-cat_pipe = make_pipeline(
-    SimpleImputer(strategy="constant", fill_value="missing"),
-    OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=-1),
-)
-
-preprocessor_tree = make_column_transformer(
-    (num_pipe, selector(dtype_include="number")),
-    (cat_pipe, selector(dtype_include="category")),
-    n_jobs=2,
-)
-
-rf_clf = make_pipeline(
-    preprocessor_tree, RandomForestClassifier(random_state=42, n_jobs=2)
-)
-
-# %%
-index += ["Random forest"]
-cv_result = cross_validate(rf_clf, df_res, y_res, scoring=scoring)
-scores["Accuracy"].append(cv_result["test_accuracy"].mean())
-scores["Balanced accuracy"].append(cv_result["test_balanced_accuracy"].mean())
+from sklearn.ensemble import RandomForestClassifier
 
-df_scores = pd.DataFrame(scores, index=index)
-df_scores
+rf_clf = tabular_pipeline(RandomForestClassifier(random_state=42, n_jobs=2))
+estimators.append(("Random forest", rf_clf))
 
 # %% [markdown]
-# The :class:`~sklearn.ensemble.RandomForestClassifier` is as well affected by
-# the class imbalanced, slightly less than the linear model. Now, we will
-# present different approach to improve the performance of these 2 models.
-#
 # Use `class_weight`
 # ..................
 #
@@ -223,88 +127,45 @@
 # linear model and tree-based model.
 
 # %%
-lr_clf.set_params(logisticregression__class_weight="balanced")
-
-index += ["Logistic regression with balanced class weights"]
-cv_result = cross_validate(lr_clf, df_res, y_res, scoring=scoring)
-scores["Accuracy"].append(cv_result["test_accuracy"].mean())
-scores["Balanced accuracy"].append(cv_result["test_balanced_accuracy"].mean())
-
-df_scores = pd.DataFrame(scores, index=index)
-df_scores
-
-# %%
-rf_clf.set_params(randomforestclassifier__class_weight="balanced")
-
-index += ["Random forest with balanced class weights"]
-cv_result = cross_validate(rf_clf, df_res, y_res, scoring=scoring)
-scores["Accuracy"].append(cv_result["test_accuracy"].mean())
-scores["Balanced accuracy"].append(cv_result["test_balanced_accuracy"].mean())
+lr_clf_balanced = tabular_pipeline(
+    LogisticRegression(max_iter=1000, class_weight="balanced")
+)
+estimators.append(("Logistic regression with balanced class weights", lr_clf_balanced))
 
-df_scores = pd.DataFrame(scores, index=index)
-df_scores
+rf_clf_balanced = tabular_pipeline(
+    RandomForestClassifier(random_state=42, n_jobs=2, class_weight="balanced")
+)
+estimators.append(("Random forest with balanced class weights", rf_clf_balanced))
 
 # %% [markdown]
-# We can see that using `class_weight` was really effective for the linear
-# model, alleviating the issue of learning from imbalanced classes. However,
-# the :class:`~sklearn.ensemble.RandomForestClassifier` is still biased toward
-# the majority class, mainly due to the criterion which is not suited enough to
-# fight the class imbalance.
-#
 # Resample the training set during learning
 # .........................................
 #
 # Another way is to resample the training set by under-sampling or
 # over-sampling some of the samples. `imbalanced-learn` provides some samplers
 # to do such processing.
+#
+# We need to use the `imbalanced-learn` pipeline to properly handle the
+# samplers within the pipeline. We insert the sampler before the final
+# estimator in the pipeline created by `skrub.tabular_pipeline`.
 
 # %%
 from imblearn.pipeline import make_pipeline as make_pipeline_with_sampler
 from imblearn.under_sampling import RandomUnderSampler
 
-lr_clf = make_pipeline_with_sampler(
-    preprocessor_linear,
-    RandomUnderSampler(random_state=42),
-    LogisticRegression(max_iter=1000),
+# We extract the preprocessing steps and the estimator from the tabular
+# pipeline and insert the sampler before the estimator.
+lr_clf_undersampled = make_pipeline_with_sampler(
+    *lr_clf[:-1], RandomUnderSampler(random_state=42), lr_clf[-1]
 )
+estimators.append(("Under-sampling + Logistic regression", lr_clf_undersampled))
 
-# %%
-index += ["Under-sampling + Logistic regression"]
-cv_result = cross_validate(lr_clf, df_res, y_res, scoring=scoring)
-scores["Accuracy"].append(cv_result["test_accuracy"].mean())
-scores["Balanced accuracy"].append(cv_result["test_balanced_accuracy"].mean())
-
-df_scores = pd.DataFrame(scores, index=index)
-df_scores
-
-# %%
-rf_clf = make_pipeline_with_sampler(
-    preprocessor_tree,
-    RandomUnderSampler(random_state=42),
-    RandomForestClassifier(random_state=42, n_jobs=2),
+rf_clf_undersampled = make_pipeline_with_sampler(
+    *rf_clf[:-1], RandomUnderSampler(random_state=42), rf_clf[-1]
 )
-
-# %%
-index += ["Under-sampling + Random forest"]
-cv_result = cross_validate(rf_clf, df_res, y_res, scoring=scoring)
-scores["Accuracy"].append(cv_result["test_accuracy"].mean())
-scores["Balanced accuracy"].append(cv_result["test_balanced_accuracy"].mean())
-
-df_scores = pd.DataFrame(scores, index=index)
-df_scores
+estimators.append(("Under-sampling + Random forest", rf_clf_undersampled))
 
 # %% [markdown]
-# Applying a random under-sampler before the training of the linear model or
-# random forest, allows to not focus on the majority class at the cost of
-# making more mistake for samples in the majority class (i.e. decreased
-# accuracy).
-#
-# We could apply any type of samplers and find which sampler is working best
-# on the current dataset.
-#
-# Instead, we will present another way by using classifiers which will apply
-# sampling internally.
-#
 # Use of specific balanced algorithms from imbalanced-learn
 # .........................................................
 #
@@ -317,53 +178,54 @@
 # %%
 from imblearn.ensemble import BalancedRandomForestClassifier
 
-rf_clf = make_pipeline(
-    preprocessor_tree,
+brf_clf = tabular_pipeline(
     BalancedRandomForestClassifier(
         sampling_strategy="all",
         replacement=True,
         bootstrap=False,
         random_state=42,
         n_jobs=2,
-    ),
+    )
 )
+estimators.append(("Balanced random forest", brf_clf))
 
 # %%
-index += ["Balanced random forest"]
-cv_result = cross_validate(rf_clf, df_res, y_res, scoring=scoring)
-scores["Accuracy"].append(cv_result["test_accuracy"].mean())
-scores["Balanced accuracy"].append(cv_result["test_balanced_accuracy"].mean())
-
-df_scores = pd.DataFrame(scores, index=index)
-df_scores
-
-# %% [markdown]
-# The performance with the
-# :class:`~imblearn.ensemble.BalancedRandomForestClassifier` is better than
-# applying a single random under-sampling. We will use a gradient-boosting
-# classifier within a :class:`~imblearn.ensemble.BalancedBaggingClassifier`.
-
 from sklearn.ensemble import HistGradientBoostingClassifier
 
 from imblearn.ensemble import BalancedBaggingClassifier
 
-bag_clf = make_pipeline(
-    preprocessor_tree,
+bag_clf = tabular_pipeline(
     BalancedBaggingClassifier(
         estimator=HistGradientBoostingClassifier(random_state=42),
         n_estimators=10,
         random_state=42,
         n_jobs=2,
-    ),
+    )
 )
+estimators.append(("Balanced bag of histogram gradient boosting", bag_clf))
 
-index += ["Balanced bag of histogram gradient boosting"]
-cv_result = cross_validate(bag_clf, df_res, y_res, scoring=scoring)
-scores["Accuracy"].append(cv_result["test_accuracy"].mean())
-scores["Balanced accuracy"].append(cv_result["test_balanced_accuracy"].mean())
+# %% [markdown]
+# Now, we can use `skore.evaluate` to evaluate each estimator with
+# cross-validation and collect all results in a single dataframe.
 
-df_scores = pd.DataFrame(scores, index=index)
-df_scores
+# %%
+report = skore.evaluate(
+    [est for _, est in estimators], df_res, y_res, splitter=5, pos_label=pos_label
+)
+report
+
+# %%
+results = report.metrics.summarize().frame(favorability=False)
+results.rename(
+    {
+        previous_name: new_name
+        for previous_name, (new_name, _) in zip(
+            results.columns.get_level_values(1), estimators
+        )
+    },
+    axis="columns",
+    level=1,
+)
 
 # %% [markdown]
 # This last approach is the most effective. The different under-sampling allows