From c67b8d0acb81a6de10cb34f05b73bda40e123496 Mon Sep 17 00:00:00 2001 From: Emil Hvitfeldt Date: Thu, 21 May 2026 16:54:48 -0700 Subject: [PATCH 1/3] add missing categories --- learn/models/conformal-regression/index.qmd | 1 + learn/models/pls/index.qmd | 1 + learn/work/bayes-opt/index.qmd | 1 + learn/work/case-weights/index.qmd | 1 + learn/work/fairness-detectors/index.qmd | 1 + learn/work/fairness-readmission/index.qmd | 1 + learn/work/nested-resampling/index.qmd | 1 + learn/work/sparse-matrix/index.qmd | 1 + learn/work/sparse-recipe/index.qmd | 1 + learn/work/tune-svm/index.qmd | 1 + learn/work/tune-text/index.qmd | 1 + 11 files changed, 11 insertions(+) diff --git a/learn/models/conformal-regression/index.qmd b/learn/models/conformal-regression/index.qmd index 0922a2ab..6e6a0d02 100644 --- a/learn/models/conformal-regression/index.qmd +++ b/learn/models/conformal-regression/index.qmd @@ -1,6 +1,7 @@ --- title: "Conformal inference for regression models" categories: + - model fitting - probably - parsnip - tuning diff --git a/learn/models/pls/index.qmd b/learn/models/pls/index.qmd index 65b14bdc..bc9b0402 100644 --- a/learn/models/pls/index.qmd +++ b/learn/models/pls/index.qmd @@ -1,6 +1,7 @@ --- title: "Multivariate analysis using partial least squares" categories: + - model fitting - pre-processing - multivariate analysis - correlation diff --git a/learn/work/bayes-opt/index.qmd b/learn/work/bayes-opt/index.qmd index a49b32e7..20382876 100644 --- a/learn/work/bayes-opt/index.qmd +++ b/learn/work/bayes-opt/index.qmd @@ -1,6 +1,7 @@ --- title: "Iterative Bayesian optimization of a classification model" categories: + - tuning and workflows - tuning - SVMs type: learn-subsection diff --git a/learn/work/case-weights/index.qmd b/learn/work/case-weights/index.qmd index b0c2766c..fd7fbc67 100644 --- a/learn/work/case-weights/index.qmd +++ b/learn/work/case-weights/index.qmd @@ -1,6 +1,7 @@ --- title: "Creating case weights based on time" categories: + - tuning and workflows - model fitting - time series type: learn-subsection diff --git a/learn/work/fairness-detectors/index.qmd b/learn/work/fairness-detectors/index.qmd index fb6fa899..71151c5a 100644 --- a/learn/work/fairness-detectors/index.qmd +++ b/learn/work/fairness-detectors/index.qmd @@ -1,6 +1,7 @@ --- title: "Are GPT detectors fair? A machine learning fairness case study" categories: + - tuning and workflows - yardstick - fairness - classification diff --git a/learn/work/fairness-readmission/index.qmd b/learn/work/fairness-readmission/index.qmd index 78cef182..e6d17539 100644 --- a/learn/work/fairness-readmission/index.qmd +++ b/learn/work/fairness-readmission/index.qmd @@ -1,6 +1,7 @@ --- title: "Fair prediction of hospital readmission: a machine learning fairness case study" categories: + - tuning and workflows - yardstick - fairness - tuning diff --git a/learn/work/nested-resampling/index.qmd b/learn/work/nested-resampling/index.qmd index 99344a4f..83afa528 100644 --- a/learn/work/nested-resampling/index.qmd +++ b/learn/work/nested-resampling/index.qmd @@ -1,6 +1,7 @@ --- title: "Nested resampling" categories: + - tuning and workflows - SVMs type: learn-subsection weight: 2 diff --git a/learn/work/sparse-matrix/index.qmd b/learn/work/sparse-matrix/index.qmd index 3d7e751d..43dd200e 100644 --- a/learn/work/sparse-matrix/index.qmd +++ b/learn/work/sparse-matrix/index.qmd @@ -1,6 +1,7 @@ --- title: "Model tuning using a sparse matrix" categories: + - tuning and workflows - tuning - classification - sparse data diff --git a/learn/work/sparse-recipe/index.qmd b/learn/work/sparse-recipe/index.qmd index 5c363e4e..bbf05a62 100644 --- a/learn/work/sparse-recipe/index.qmd +++ b/learn/work/sparse-recipe/index.qmd @@ -1,6 +1,7 @@ --- title: "Using recipes to create sparse data" categories: + - tuning and workflows - tuning - classification - sparse data diff --git a/learn/work/tune-svm/index.qmd b/learn/work/tune-svm/index.qmd index 7e354626..53cc82b8 100644 --- a/learn/work/tune-svm/index.qmd +++ b/learn/work/tune-svm/index.qmd @@ -1,6 +1,7 @@ --- title: "Model tuning via grid search" categories: + - tuning and workflows - tuning - SVMs - classification diff --git a/learn/work/tune-text/index.qmd b/learn/work/tune-text/index.qmd index 9c3095e7..27acb5c9 100644 --- a/learn/work/tune-text/index.qmd +++ b/learn/work/tune-text/index.qmd @@ -1,6 +1,7 @@ --- title: "Tuning text models" categories: + - tuning and workflows - tuning - regression - sparse data From 2a8813b9dbdbf2e1285fca945c53e314547fb6b5 Mon Sep 17 00:00:00 2001 From: Emil Hvitfeldt Date: Thu, 21 May 2026 16:58:55 -0700 Subject: [PATCH 2/3] group categories in learn --- learn/group-categories.html | 56 +++++++++++++++++++++++++++++++++++++ learn/index.html.md | 1 + learn/index.qmd | 1 + styles.scss | 8 ++++++ 4 files changed, 66 insertions(+) create mode 100644 learn/group-categories.html diff --git a/learn/group-categories.html b/learn/group-categories.html new file mode 100644 index 00000000..c0a92dca --- /dev/null +++ b/learn/group-categories.html @@ -0,0 +1,56 @@ + diff --git a/learn/index.html.md b/learn/index.html.md index fd771516..893b19a2 100644 --- a/learn/index.html.md +++ b/learn/index.html.md @@ -14,6 +14,7 @@ listing: - "../start/resampling/index.qmd" - "../start/tuning/index.qmd" - "../start/case-study/index.qmd" +include-after-body: group-categories.html --- After you know [what you need to get started](/start/) with tidymodels, you can learn more and go further. Find articles here to help you solve specific problems using the tidymodels framework. diff --git a/learn/index.qmd b/learn/index.qmd index fd771516..893b19a2 100644 --- a/learn/index.qmd +++ b/learn/index.qmd @@ -14,6 +14,7 @@ listing: - "../start/resampling/index.qmd" - "../start/tuning/index.qmd" - "../start/case-study/index.qmd" +include-after-body: group-categories.html --- After you know [what you need to get started](/start/) with tidymodels, you can learn more and go further. Find articles here to help you solve specific problems using the tidymodels framework. diff --git a/styles.scss b/styles.scss index 30bc9e5f..f31b665a 100644 --- a/styles.scss +++ b/styles.scss @@ -136,6 +136,14 @@ h3, h4, h5 { } } +.quarto-listing-category-subtitle { + font-size: 0.85rem; + text-transform: uppercase; + letter-spacing: 0.05em; + color: rgba($theme-cranberry, 0.7); + margin-top: 1rem; + margin-bottom: 0.4rem; +} // Bracketed TOC for get started pages ----------------------------------------- // :not(.toc-active) makes it so only get-started pages are selected From bdddeea8d1ba44e50dbfadab8b15ab2c62609b21 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 22 May 2026 00:21:12 +0000 Subject: [PATCH 3/3] render changed pages [skip ci] --- .../conformal-regression/index/execute-results/html.json | 4 ++-- _freeze/learn/models/pls/index/execute-results/html.json | 4 ++-- _freeze/learn/work/bayes-opt/index/execute-results/html.json | 4 ++-- .../learn/work/case-weights/index/execute-results/html.json | 4 ++-- .../work/fairness-detectors/index/execute-results/html.json | 4 ++-- .../work/fairness-readmission/index/execute-results/html.json | 4 ++-- .../work/nested-resampling/index/execute-results/html.json | 4 ++-- .../learn/work/sparse-matrix/index/execute-results/html.json | 4 ++-- .../learn/work/sparse-recipe/index/execute-results/html.json | 4 ++-- _freeze/learn/work/tune-svm/index/execute-results/html.json | 4 ++-- _freeze/learn/work/tune-text/index/execute-results/html.json | 4 ++-- learn/models/conformal-regression/index.html.md | 1 + learn/models/pls/index.html.md | 1 + learn/work/bayes-opt/index.html.md | 1 + learn/work/case-weights/index.html.md | 1 + learn/work/fairness-detectors/index.html.md | 1 + learn/work/fairness-readmission/index.html.md | 1 + learn/work/nested-resampling/index.html.md | 1 + learn/work/sparse-matrix/index.html.md | 1 + learn/work/sparse-recipe/index.html.md | 1 + learn/work/tune-svm/index.html.md | 1 + learn/work/tune-text/index.html.md | 1 + 22 files changed, 33 insertions(+), 22 deletions(-) diff --git a/_freeze/learn/models/conformal-regression/index/execute-results/html.json b/_freeze/learn/models/conformal-regression/index/execute-results/html.json index 604f6986..3696be95 100644 --- a/_freeze/learn/models/conformal-regression/index/execute-results/html.json +++ b/_freeze/learn/models/conformal-regression/index/execute-results/html.json @@ -1,8 +1,8 @@ { - "hash": "02ed69cbb4421557a2a584a7f9247475", + "hash": "a0967b9b3c74e1d33919c4b4de2c28fb", "result": { "engine": "knitr", - "markdown": "---\ntitle: \"Conformal inference for regression models\"\ncategories:\n - probably\n - parsnip\n - tuning\n - regression\ntype: learn-subsection\nweight: 5\ndescription: | \n The probably package has functions to create prediction intervals for regression models.\ntoc: true\ntoc-depth: 2\nr-packages:\n - tidymodels\n - probably\n - nnet\n - quantregForest\n - future\ninclude-after-body: ../../../html/resources.html\n---\n\n\n\n\n\n\nTo use code in this article, you will need to install the following packages: future, nnet, probably, quantregForest, and tidymodels. The probably package should be version 1.0.2 or greater.\n\nWhat is [conformal inference](https://en.wikipedia.org/wiki/Conformal_prediction)? It is a collection of statistical methods that are mostly used to construct prediction intervals (or prediction sets) for any type of regression or classification model. The basic idea revolves around some Frequentist theory on how to construct probability statements about whether a new sample could have been from an existing reference distribution.\n\nSince this article is focused on regression problems, we'll motivate the methodology in the context of a numeric outcome. For example, suppose we have collected a set of 1,000 standard normal data points. \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(tidymodels)\ntidymodels_prefer()\n\nset.seed(1)\nreference_data <- tibble(data = rnorm(1000))\n\nreference_data |> \n ggplot(aes(x = data)) +\n geom_line(stat = \"density\")\n```\n\n::: {.cell-output-display}\n![](figs/normal-dist-1.svg){fig-align='center' width=70%}\n:::\n:::\n\n\nIf we had a new observation that we thought might be from the same distribution, how would we say (probabilistically) whether we believe that it belongs to the original distribution? \n\nIf we thought that 1,000 were a sufficient sample size, we might compute some quantiles of these data to define \"the mainstream of the data.\" Let's use the 5th and 95th quantiles to set boundaries that define what we would expect to see most of the time: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nquants <- quantile(reference_data$data, probs = c(0.05, 0.95))\n\nreference_data |> \n ggplot(aes(x = data)) +\n geom_line(stat = \"density\") + \n geom_vline(xintercept = quants, col = \"red\", lty = 2)\n```\n\n::: {.cell-output-display}\n![](figs/normal-quant-1.svg){fig-align='center' width=70%}\n:::\n:::\n\n\nIf we were to get a new sample beyond these boundaries, we would be able to say that we are about 90% sure that the new sample does not conform to the original distribution. This works under the assumption that the data are exchangeable. \n\nWe can apply this relatively simple idea to model predictions. Suppose we have a model created on a numeric outcome. If we make predictions on a data set we can compute the model residuals and create a sort of reference error distribution. If we compute a prediction on a new unknown sample, we could center this reference distribution around its predicted value. For some significance level, we now know the range of sample values that \"conform\" to the variance seen in the reference distribution. That range can define our prediction interval.\n\nThere are a variety of ways to apply this concept (which is unsurprisingly more complex than the above description). The probably package has implemented a few of these.\n\nLet's make a simple example to illustrate the results. We'll simulate a data set with a single predictor along with some unknown samples: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nmake_data <- function(n, std_dev = 1 / 5) {\n tibble(x = runif(n, min = -1)) |>\n mutate(\n y = (x^3) + 2 * exp(-6 * (x - 0.3)^2),\n y = y + rnorm(n, sd = std_dev)\n )\n}\n\nn <- 1000\nset.seed(8383)\ntrain_data <- make_data(n)\n\ntrain_data |> \n ggplot(aes(x, y)) + \n geom_point(alpha = 1 / 10)\n```\n\n::: {.cell-output-display}\n![](figs/example-data-1.svg){fig-align='center' width=70%}\n:::\n:::\n\n\nWe'll use these data as a training set and fit a model: \n\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(484)\nnnet_wflow <- \n workflow(y ~ x, mlp(hidden_units = 4) |> set_mode(\"regression\"))\n\nnnet_fit <- nnet_wflow |> fit(train_data)\nnnet_pred <- augment(nnet_fit, train_data)\n\ntrain_data |> \n ggplot(aes(x)) + \n geom_point(aes(y = y), alpha = 1 / 10) +\n geom_line(data = nnet_pred, aes(y = .pred),\n linewidth = 1, col = \"blue\")\n```\n\n::: {.cell-output-display}\n![](figs/model-fit-1.svg){fig-align='center' width=70%}\n:::\n:::\n\n\nLet's examine three methods^[A fourth method is implemented for \"full conformal prediction.\" It was developed mainly as a proof of concept. While it is effective, it is _very_ computationally expensive.] to produce prediction intervals: \n\n## Split Conformal Inference\n\nThe most straightforward approach is reserving some data for estimating the residual distribution. We know that simply re-predicting the training set is **a bad idea**; the residuals would be smaller than they should be since the same data are used to create and evaluate the model. \n\nLet's simulate another data set containing 250 samples and call that the \"calibration set\". These data can be predicted and the corresponding residuals can be used to define what conforms to the model. We'll also create a large test set to see if we've done a good job. \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(7292)\ncal_data <- make_data(250)\ntest_data <- make_data(10000)\n```\n:::\n\n\nThe probably package has a set of functions with the prefix `int_conformal` that can be used to create prediction intervals. One is: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nsplit_int <- int_conformal_split(nnet_fit, cal_data)\nsplit_int\n#> Split Conformal inference\n#> preprocessor: formula \n#> model: mlp (engine = nnet) \n#> calibration set size: 250 \n#> \n#> Use `predict(object, new_data, level)` to compute prediction intervals\n```\n:::\n\n\nTo get predictions on new data, we use the standard `predict()` method on this object: \n\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\n# Produce 90% prediction intervals\ntest_split_res <- \n predict(split_int, test_data, level = 0.90) |> \n bind_cols(test_data)\n\ntest_split_res |> slice(1:5)\n#> # A tibble: 5 × 5\n#> .pred .pred_lower .pred_upper x y\n#> \n#> 1 1.30 0.933 1.67 0.621 1.17 \n#> 2 -0.298 -0.666 0.0691 -0.658 -0.302 \n#> 3 0.133 -0.234 0.501 -0.329 0.0970\n#> 4 1.33 0.964 1.70 0.611 1.21 \n#> 5 1.24 0.873 1.61 0.0145 1.22\n```\n:::\n\n\nThe results: \n\n\n::: {.cell layout-align=\"center\"}\n::: {.cell-output-display}\n![](figs/split-pred-plit-1.svg){fig-align='center' width=70%}\n:::\n:::\n\n\nSince we know the outcome values, we can compute the coverage for this particular data set. Since we created a 90% prediction interval, about 90% of the outcomes should be within our bounds. Let's create a function and apply it to these data:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ncoverage <- function(x) {\n x |> \n mutate(in_bound = .pred_lower <= y & .pred_upper >= y) |> \n summarise(coverage = mean(in_bound) * 100)\n}\ncoverage(test_split_res)\n#> # A tibble: 1 × 1\n#> coverage\n#> \n#> 1 93.0\n```\n:::\n\n\n## Using Resampling Results\n\nIn a way, the calibration set serves a similar role as a traditional validation set. Since we can't just re-predict our training set, we need to evaluate our model on a separate collection of labeled data. \n\nHowever, we also know that resampling methods can serve the same purpose. During resampling, we can compute a set of predictions that were not used to fit the model. For example, in 10-fold cross-validation, there are 10 collections of held-out predictions that do not suffer from the severe bias that occurs when we simple re-predict the training set. \n\nThe CV+ estimator ([Barber _et al._ (2021)](https://scholar.google.com/scholar?hl=en&as_sdt=0%2C5&q=%22Predictive+inference+with+the+jackknife%2B%22&btnG=)) can be used to assemble a reference distribution of residuals. If we use `fit_resamples()` or one of the `tune_*()` functions, we can gather those residuals and use them to create prediction intervals. We'll have to make sure that we save the out-of-sample predictions as well as the resasmpled models. \n\nThe `control_*()` functions can be used for this purpose. We can set `save_pred = TRUE` to save the predicted values. For the resampled models, we can use the tools to [extract the models](https://tidymodels.github.io/tidymodels_dot_org/learn/index.html#category=extracting%20results) via the `extract` argument. We'll use the `I()` function to return the fitted workflows from each resample: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nctrl <- control_resamples(save_pred = TRUE, extract = I)\n```\n:::\n\n\nLet's use 10-fold cross-validation to resample the neural network: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(493)\nfolds <- vfold_cv(train_data)\n\nnnet_rs <- \n nnet_wflow |> \n fit_resamples(folds, control = ctrl)\n\ncollect_metrics(nnet_rs)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> \n#> 1 rmse standard 0.199 10 0.00477 pre0_mod0_post0\n#> 2 rsq standard 0.951 10 0.00425 pre0_mod0_post0\n```\n:::\n\n\nThe model has an estimated root mean squared error of 0.199 and an estimated R2 of 0.951.\n\nWe can create another object for computing the intervals:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ncv_int <- int_conformal_cv(nnet_rs)\ncv_int\n#> Conformal inference via CV+\n#> preprocessor: formula \n#> model: mlp (engine = nnet) \n#> number of models: 10 \n#> training set size: 1,000 \n#> \n#> Use `predict(object, new_data, level)` to compute prediction intervals\n\n# Produce 90% prediction intervals\ntest_cv_res <- \n predict(cv_int, test_data, level = 0.90) |> \n bind_cols(test_data)\n\ntest_cv_res |> slice(1:5)\n#> # A tibble: 5 × 5\n#> .pred_lower .pred .pred_upper x y\n#> \n#> 1 0.960 1.29 1.61 0.621 1.17 \n#> 2 -0.630 -0.304 0.0226 -0.658 -0.302 \n#> 3 -0.189 0.137 0.464 -0.329 0.0970\n#> 4 0.991 1.32 1.64 0.611 1.21 \n#> 5 0.912 1.24 1.57 0.0145 1.22\n```\n:::\n\n\nThe results for this method are:\n\n\n::: {.cell layout-align=\"center\"}\n::: {.cell-output-display}\n![](figs/cv-pred-1.svg){fig-align='center' width=70%}\n:::\n:::\n\n\nAt each point, the interval length is 0.65; the previous split conformal interval width was 0.73. This is due to the training set being larger than the calibration set. \n\nNote that the coverage is a little better since it is near 90%:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ncoverage(test_cv_res)\n#> # A tibble: 1 × 1\n#> coverage\n#> \n#> 1 89.4\n```\n:::\n\n\n## Adaptive width intervals\n\nThe simulated data that we've been using has a constant error variance. That has worked for the two methods shown since their intervals are always the same width. Real data does not always have constant variation. \n\nTo demonstrate, we can take the previous simulation system and induce a variance that is dynamic over the predictor range:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nmake_variable_data <- function(n, std_dev = 1 / 5) {\n tibble(x = runif(n, min = -1)) |>\n mutate(\n y = (x^3) + 2 * exp(-6 * (x - 0.3)^2),\n y = y + rnorm(n, sd = std_dev * abs(x))\n )\n}\n\nmake_variable_data(1000) |> \n ggplot(aes(x, y)) + \n geom_point(alpha = 1 / 5)\n```\n\n::: {.cell-output-display}\n![](figs/variable-plot-1.svg){fig-align='center' width=70%}\n:::\n:::\n\n\nLet's create new data sets and re-fit our model: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(7292)\ntrain_variable_data <- make_variable_data(n)\ncal_variable_data <- make_variable_data(250)\ntest_variable_data <- make_variable_data(10000)\n\nnnet_variable_fit <- nnet_wflow |> fit(train_variable_data)\nnnet_variable_pred <- augment(nnet_variable_fit, train_variable_data)\n```\n:::\n\n\nWe can recreate the CV+ interval for this new version of the data:\n\n\n::: {.cell layout-align=\"center\"}\n::: {.cell-output-display}\n![](figs/cv-redux-1.svg){fig-align='center' width=70%}\n:::\n:::\n\n\nThe _average_ coverage is good: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ncoverage(test_cv_variable_res)\n#> # A tibble: 1 × 1\n#> coverage\n#> \n#> 1 90.1\n```\n:::\n\n\n\nHowever, there are obvious areas where it is either too wide or too narrow. \n\nConformalized quantile regression ([Romano _et al_ (2019)](https://scholar.google.com/scholar?hl=en&as_sdt=0%2C5&q=%22Conformalized+quantile+regression%22)) is a method to produce intervals that can properly scale the intervals based on what was observed in the training data. It fits an initial [quantile regression](https://en.wikipedia.org/wiki/Quantile_regression) model to do so and also required a split data set, such as our calibration data. \n\nThe original work used basic quantile regression models. For tidymodels, we'll use [quantile regression forests](https://scholar.google.com/scholar?hl=en&as_sdt=0%2C5&q=%22Quantile+Regression+Forests%22). The reason being that some models require nonlinear terms, interactions, and other features to model the data adequately. Users would have to have a parallel modeling process for that model. Random forests for quantile regression are very adaptable, low maintenance, and are robust to tuning parameter specification. There are both minor and major downsides, as seen below. \n\nThe function for this, `int_conformal_quantile()`, has a slightly different interface than the previous functions: \n\n* It requires both the training and calibration sets.\n* The confidence level must be set in advance. \n\nWe'll also pass an argument to the `quantregForest()` function that we'll use 2,000 trees to make the forest: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nquant_int <-\n int_conformal_quantile(\n nnet_variable_fit, \n train_data = train_variable_data,\n cal_data = cal_variable_data, \n level = 0.90,\n ntree = 2000)\nquant_int\n#> Split Conformal inference via Quantile Regression\n#> preprocessor: formula \n#> model: mlp (engine = nnet) \n#> calibration set size: 250 \n#> confidence level: 0.9 \n#> \n#> Use `predict(object, new_data)` to compute prediction intervals\n\ntest_quant_res <- \n predict(quant_int, test_variable_data) |> \n bind_cols(test_variable_data)\n\ntest_quant_res |> slice(1:5)\n#> # A tibble: 5 × 5\n#> .pred .pred_lower .pred_upper x y\n#> \n#> 1 0.995 0.574 1.14 0.776 1.10 \n#> 2 -0.328 -0.546 -0.0904 -0.686 -0.180\n#> 3 0.266 0.161 0.459 -0.274 0.364\n#> 4 1.01 0.666 1.23 0.759 1.21 \n#> 5 2.01 1.90 2.12 0.367 1.87\n```\n:::\n\n\nThe results: \n\n\n::: {.cell layout-align=\"center\"}\n::: {.cell-output-display}\n![](figs/quant-pred-split-1.svg){fig-align='center' width=70%}\n:::\n:::\n\n\nMinor downside: The bumpiness of the bound is the consequence of using a tree-based model for the quantiles. \n\nDespite that, the intervals do adapt their widths to suit the data. The coverage is also close to the target: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ncoverage(test_quant_res)\n#> # A tibble: 1 × 1\n#> coverage\n#> \n#> 1 91.8\n```\n:::\n\n### The major downside\n\nWhen our predictions are extrapolating, the intervals can be very poor. Tree-based models can behave very differently than other models when predicting beyond the training set; they follow a static trend even as the predictor values move towards infinity. \n\nTo illustrate this, we can simulate another point outside of the training set range: \n\n\n\n::: {.cell layout-align=\"center\"}\n::: {.cell-output-display}\n![](figs/extrapolation-1.svg){fig-align='center' width=70%}\n:::\n:::\n\n\nThe neural network does fairly well but the quantile regression forest carries the same extract trend forward. In this case, the interval doesn't even include the predicted value (since two different models are used to compute both quantities). \n\nTo protect against this, we suggest using the tools in the [applicable package](https://applicable.tidymodels.org). It can help quantify how much a prediction is outside of the mainstream of the training set.\n\n\n## Coverage\n\nSpeaking of coverage, there's a [GitHub repo](https://github.com/topepo/conformal_sim) where we did simulations for these methods to determine their average coverage. The repository's readme file has a summary. \n\n## Learning More\n\nIf you are interested and would like to learn more, try these resources: \n\n* [_Introduction To Conformal Prediction With Python_](https://christophmolnar.com/books/conformal-prediction/) by Christoph Molnar\n (highly recommended)\n * [`awesome-conformal-prediction`](https://github.com/valeman/awesome-conformal-prediction) on GitHub. \n* [Ryan Tibshirani's notes](https://www.stat.berkeley.edu/~ryantibs/statlearn-s23/lectures/conformal.pdf) (pdf)\n* Angelopoulos, Anastasios N., and Stephen Bates. \"[A gentle introduction to conformal prediction and distribution-free uncertainty quantification](https://arxiv.org/abs/2107.07511).\" arXiv preprint arXiv:2107.07511 (2021).\n\n\n## Session information\n\n\n::: {.cell layout-align=\"center\"}\n\n```\n#> ─ Session info ─────────────────────────────────────────────────────\n#> version R version 4.6.0 (2026-04-24)\n#> language (EN)\n#> pandoc 3.1.3\n#> quarto 1.9.37\n#> \n#> ─ Packages ─────────────────────────────────────────────────────────\n#> package version date (UTC)\n#> broom 1.0.13 2026-05-14\n#> dials 1.4.3 2026-04-11\n#> dplyr 1.2.1 2026-04-03\n#> future 1.70.0 2026-03-14\n#> ggplot2 4.0.3 2026-04-22\n#> infer 1.1.0 2025-12-18\n#> nnet 7.3-20 2025-01-01\n#> parsnip 1.6.0 2026-05-14\n#> probably 1.2.0 2025-10-16\n#> purrr 1.2.2 2026-04-10\n#> quantregForest 1.3-7.1 2024-10-07\n#> recipes 1.3.2 2026-04-02\n#> rlang 1.2.0 2026-04-06\n#> rsample 1.3.2 2026-01-30\n#> tibble 3.3.1 2026-01-11\n#> tidymodels 1.5.0 2026-04-23\n#> tune 2.1.0 2026-04-17\n#> workflows 1.3.0 2025-08-27\n#> yardstick 1.4.0 2026-04-07\n#> \n#> ────────────────────────────────────────────────────────────────────\n```\n:::\n\n", + "markdown": "---\ntitle: \"Conformal inference for regression models\"\ncategories:\n - model fitting\n - probably\n - parsnip\n - tuning\n - regression\ntype: learn-subsection\nweight: 5\ndescription: | \n The probably package has functions to create prediction intervals for regression models.\ntoc: true\ntoc-depth: 2\nr-packages:\n - tidymodels\n - probably\n - nnet\n - quantregForest\n - future\ninclude-after-body: ../../../html/resources.html\n---\n\n\n\n\n\n\nTo use code in this article, you will need to install the following packages: future, nnet, probably, quantregForest, and tidymodels. The probably package should be version 1.0.2 or greater.\n\nWhat is [conformal inference](https://en.wikipedia.org/wiki/Conformal_prediction)? It is a collection of statistical methods that are mostly used to construct prediction intervals (or prediction sets) for any type of regression or classification model. The basic idea revolves around some Frequentist theory on how to construct probability statements about whether a new sample could have been from an existing reference distribution.\n\nSince this article is focused on regression problems, we'll motivate the methodology in the context of a numeric outcome. For example, suppose we have collected a set of 1,000 standard normal data points. \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(tidymodels)\ntidymodels_prefer()\n\nset.seed(1)\nreference_data <- tibble(data = rnorm(1000))\n\nreference_data |> \n ggplot(aes(x = data)) +\n geom_line(stat = \"density\")\n```\n\n::: {.cell-output-display}\n![](figs/normal-dist-1.svg){fig-align='center' width=70%}\n:::\n:::\n\n\nIf we had a new observation that we thought might be from the same distribution, how would we say (probabilistically) whether we believe that it belongs to the original distribution? \n\nIf we thought that 1,000 were a sufficient sample size, we might compute some quantiles of these data to define \"the mainstream of the data.\" Let's use the 5th and 95th quantiles to set boundaries that define what we would expect to see most of the time: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nquants <- quantile(reference_data$data, probs = c(0.05, 0.95))\n\nreference_data |> \n ggplot(aes(x = data)) +\n geom_line(stat = \"density\") + \n geom_vline(xintercept = quants, col = \"red\", lty = 2)\n```\n\n::: {.cell-output-display}\n![](figs/normal-quant-1.svg){fig-align='center' width=70%}\n:::\n:::\n\n\nIf we were to get a new sample beyond these boundaries, we would be able to say that we are about 90% sure that the new sample does not conform to the original distribution. This works under the assumption that the data are exchangeable. \n\nWe can apply this relatively simple idea to model predictions. Suppose we have a model created on a numeric outcome. If we make predictions on a data set we can compute the model residuals and create a sort of reference error distribution. If we compute a prediction on a new unknown sample, we could center this reference distribution around its predicted value. For some significance level, we now know the range of sample values that \"conform\" to the variance seen in the reference distribution. That range can define our prediction interval.\n\nThere are a variety of ways to apply this concept (which is unsurprisingly more complex than the above description). The probably package has implemented a few of these.\n\nLet's make a simple example to illustrate the results. We'll simulate a data set with a single predictor along with some unknown samples: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nmake_data <- function(n, std_dev = 1 / 5) {\n tibble(x = runif(n, min = -1)) |>\n mutate(\n y = (x^3) + 2 * exp(-6 * (x - 0.3)^2),\n y = y + rnorm(n, sd = std_dev)\n )\n}\n\nn <- 1000\nset.seed(8383)\ntrain_data <- make_data(n)\n\ntrain_data |> \n ggplot(aes(x, y)) + \n geom_point(alpha = 1 / 10)\n```\n\n::: {.cell-output-display}\n![](figs/example-data-1.svg){fig-align='center' width=70%}\n:::\n:::\n\n\nWe'll use these data as a training set and fit a model: \n\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(484)\nnnet_wflow <- \n workflow(y ~ x, mlp(hidden_units = 4) |> set_mode(\"regression\"))\n\nnnet_fit <- nnet_wflow |> fit(train_data)\nnnet_pred <- augment(nnet_fit, train_data)\n\ntrain_data |> \n ggplot(aes(x)) + \n geom_point(aes(y = y), alpha = 1 / 10) +\n geom_line(data = nnet_pred, aes(y = .pred),\n linewidth = 1, col = \"blue\")\n```\n\n::: {.cell-output-display}\n![](figs/model-fit-1.svg){fig-align='center' width=70%}\n:::\n:::\n\n\nLet's examine three methods^[A fourth method is implemented for \"full conformal prediction.\" It was developed mainly as a proof of concept. While it is effective, it is _very_ computationally expensive.] to produce prediction intervals: \n\n## Split Conformal Inference\n\nThe most straightforward approach is reserving some data for estimating the residual distribution. We know that simply re-predicting the training set is **a bad idea**; the residuals would be smaller than they should be since the same data are used to create and evaluate the model. \n\nLet's simulate another data set containing 250 samples and call that the \"calibration set\". These data can be predicted and the corresponding residuals can be used to define what conforms to the model. We'll also create a large test set to see if we've done a good job. \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(7292)\ncal_data <- make_data(250)\ntest_data <- make_data(10000)\n```\n:::\n\n\nThe probably package has a set of functions with the prefix `int_conformal` that can be used to create prediction intervals. One is: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nsplit_int <- int_conformal_split(nnet_fit, cal_data)\nsplit_int\n#> Split Conformal inference\n#> preprocessor: formula \n#> model: mlp (engine = nnet) \n#> calibration set size: 250 \n#> \n#> Use `predict(object, new_data, level)` to compute prediction intervals\n```\n:::\n\n\nTo get predictions on new data, we use the standard `predict()` method on this object: \n\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\n# Produce 90% prediction intervals\ntest_split_res <- \n predict(split_int, test_data, level = 0.90) |> \n bind_cols(test_data)\n\ntest_split_res |> slice(1:5)\n#> # A tibble: 5 × 5\n#> .pred .pred_lower .pred_upper x y\n#> \n#> 1 1.30 0.933 1.67 0.621 1.17 \n#> 2 -0.298 -0.666 0.0691 -0.658 -0.302 \n#> 3 0.133 -0.234 0.501 -0.329 0.0970\n#> 4 1.33 0.964 1.70 0.611 1.21 \n#> 5 1.24 0.873 1.61 0.0145 1.22\n```\n:::\n\n\nThe results: \n\n\n::: {.cell layout-align=\"center\"}\n::: {.cell-output-display}\n![](figs/split-pred-plit-1.svg){fig-align='center' width=70%}\n:::\n:::\n\n\nSince we know the outcome values, we can compute the coverage for this particular data set. Since we created a 90% prediction interval, about 90% of the outcomes should be within our bounds. Let's create a function and apply it to these data:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ncoverage <- function(x) {\n x |> \n mutate(in_bound = .pred_lower <= y & .pred_upper >= y) |> \n summarise(coverage = mean(in_bound) * 100)\n}\ncoverage(test_split_res)\n#> # A tibble: 1 × 1\n#> coverage\n#> \n#> 1 93.0\n```\n:::\n\n\n## Using Resampling Results\n\nIn a way, the calibration set serves a similar role as a traditional validation set. Since we can't just re-predict our training set, we need to evaluate our model on a separate collection of labeled data. \n\nHowever, we also know that resampling methods can serve the same purpose. During resampling, we can compute a set of predictions that were not used to fit the model. For example, in 10-fold cross-validation, there are 10 collections of held-out predictions that do not suffer from the severe bias that occurs when we simple re-predict the training set. \n\nThe CV+ estimator ([Barber _et al._ (2021)](https://scholar.google.com/scholar?hl=en&as_sdt=0%2C5&q=%22Predictive+inference+with+the+jackknife%2B%22&btnG=)) can be used to assemble a reference distribution of residuals. If we use `fit_resamples()` or one of the `tune_*()` functions, we can gather those residuals and use them to create prediction intervals. We'll have to make sure that we save the out-of-sample predictions as well as the resasmpled models. \n\nThe `control_*()` functions can be used for this purpose. We can set `save_pred = TRUE` to save the predicted values. For the resampled models, we can use the tools to [extract the models](https://tidymodels.github.io/tidymodels_dot_org/learn/index.html#category=extracting%20results) via the `extract` argument. We'll use the `I()` function to return the fitted workflows from each resample: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nctrl <- control_resamples(save_pred = TRUE, extract = I)\n```\n:::\n\n\nLet's use 10-fold cross-validation to resample the neural network: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(493)\nfolds <- vfold_cv(train_data)\n\nnnet_rs <- \n nnet_wflow |> \n fit_resamples(folds, control = ctrl)\n\ncollect_metrics(nnet_rs)\n#> # A tibble: 2 × 6\n#> .metric .estimator mean n std_err .config \n#> \n#> 1 rmse standard 0.199 10 0.00477 pre0_mod0_post0\n#> 2 rsq standard 0.951 10 0.00425 pre0_mod0_post0\n```\n:::\n\n\nThe model has an estimated root mean squared error of 0.199 and an estimated R2 of 0.951.\n\nWe can create another object for computing the intervals:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ncv_int <- int_conformal_cv(nnet_rs)\ncv_int\n#> Conformal inference via CV+\n#> preprocessor: formula \n#> model: mlp (engine = nnet) \n#> number of models: 10 \n#> training set size: 1,000 \n#> \n#> Use `predict(object, new_data, level)` to compute prediction intervals\n\n# Produce 90% prediction intervals\ntest_cv_res <- \n predict(cv_int, test_data, level = 0.90) |> \n bind_cols(test_data)\n\ntest_cv_res |> slice(1:5)\n#> # A tibble: 5 × 5\n#> .pred_lower .pred .pred_upper x y\n#> \n#> 1 0.960 1.29 1.61 0.621 1.17 \n#> 2 -0.630 -0.304 0.0226 -0.658 -0.302 \n#> 3 -0.189 0.137 0.464 -0.329 0.0970\n#> 4 0.991 1.32 1.64 0.611 1.21 \n#> 5 0.912 1.24 1.57 0.0145 1.22\n```\n:::\n\n\nThe results for this method are:\n\n\n::: {.cell layout-align=\"center\"}\n::: {.cell-output-display}\n![](figs/cv-pred-1.svg){fig-align='center' width=70%}\n:::\n:::\n\n\nAt each point, the interval length is 0.65; the previous split conformal interval width was 0.73. This is due to the training set being larger than the calibration set. \n\nNote that the coverage is a little better since it is near 90%:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ncoverage(test_cv_res)\n#> # A tibble: 1 × 1\n#> coverage\n#> \n#> 1 89.4\n```\n:::\n\n\n## Adaptive width intervals\n\nThe simulated data that we've been using has a constant error variance. That has worked for the two methods shown since their intervals are always the same width. Real data does not always have constant variation. \n\nTo demonstrate, we can take the previous simulation system and induce a variance that is dynamic over the predictor range:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nmake_variable_data <- function(n, std_dev = 1 / 5) {\n tibble(x = runif(n, min = -1)) |>\n mutate(\n y = (x^3) + 2 * exp(-6 * (x - 0.3)^2),\n y = y + rnorm(n, sd = std_dev * abs(x))\n )\n}\n\nmake_variable_data(1000) |> \n ggplot(aes(x, y)) + \n geom_point(alpha = 1 / 5)\n```\n\n::: {.cell-output-display}\n![](figs/variable-plot-1.svg){fig-align='center' width=70%}\n:::\n:::\n\n\nLet's create new data sets and re-fit our model: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(7292)\ntrain_variable_data <- make_variable_data(n)\ncal_variable_data <- make_variable_data(250)\ntest_variable_data <- make_variable_data(10000)\n\nnnet_variable_fit <- nnet_wflow |> fit(train_variable_data)\nnnet_variable_pred <- augment(nnet_variable_fit, train_variable_data)\n```\n:::\n\n\nWe can recreate the CV+ interval for this new version of the data:\n\n\n::: {.cell layout-align=\"center\"}\n::: {.cell-output-display}\n![](figs/cv-redux-1.svg){fig-align='center' width=70%}\n:::\n:::\n\n\nThe _average_ coverage is good: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ncoverage(test_cv_variable_res)\n#> # A tibble: 1 × 1\n#> coverage\n#> \n#> 1 90.1\n```\n:::\n\n\n\nHowever, there are obvious areas where it is either too wide or too narrow. \n\nConformalized quantile regression ([Romano _et al_ (2019)](https://scholar.google.com/scholar?hl=en&as_sdt=0%2C5&q=%22Conformalized+quantile+regression%22)) is a method to produce intervals that can properly scale the intervals based on what was observed in the training data. It fits an initial [quantile regression](https://en.wikipedia.org/wiki/Quantile_regression) model to do so and also required a split data set, such as our calibration data. \n\nThe original work used basic quantile regression models. For tidymodels, we'll use [quantile regression forests](https://scholar.google.com/scholar?hl=en&as_sdt=0%2C5&q=%22Quantile+Regression+Forests%22). The reason being that some models require nonlinear terms, interactions, and other features to model the data adequately. Users would have to have a parallel modeling process for that model. Random forests for quantile regression are very adaptable, low maintenance, and are robust to tuning parameter specification. There are both minor and major downsides, as seen below. \n\nThe function for this, `int_conformal_quantile()`, has a slightly different interface than the previous functions: \n\n* It requires both the training and calibration sets.\n* The confidence level must be set in advance. \n\nWe'll also pass an argument to the `quantregForest()` function that we'll use 2,000 trees to make the forest: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nquant_int <-\n int_conformal_quantile(\n nnet_variable_fit, \n train_data = train_variable_data,\n cal_data = cal_variable_data, \n level = 0.90,\n ntree = 2000)\nquant_int\n#> Split Conformal inference via Quantile Regression\n#> preprocessor: formula \n#> model: mlp (engine = nnet) \n#> calibration set size: 250 \n#> confidence level: 0.9 \n#> \n#> Use `predict(object, new_data)` to compute prediction intervals\n\ntest_quant_res <- \n predict(quant_int, test_variable_data) |> \n bind_cols(test_variable_data)\n\ntest_quant_res |> slice(1:5)\n#> # A tibble: 5 × 5\n#> .pred .pred_lower .pred_upper x y\n#> \n#> 1 0.995 0.574 1.14 0.776 1.10 \n#> 2 -0.328 -0.546 -0.0904 -0.686 -0.180\n#> 3 0.266 0.161 0.459 -0.274 0.364\n#> 4 1.01 0.666 1.23 0.759 1.21 \n#> 5 2.01 1.90 2.12 0.367 1.87\n```\n:::\n\n\nThe results: \n\n\n::: {.cell layout-align=\"center\"}\n::: {.cell-output-display}\n![](figs/quant-pred-split-1.svg){fig-align='center' width=70%}\n:::\n:::\n\n\nMinor downside: The bumpiness of the bound is the consequence of using a tree-based model for the quantiles. \n\nDespite that, the intervals do adapt their widths to suit the data. The coverage is also close to the target: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ncoverage(test_quant_res)\n#> # A tibble: 1 × 1\n#> coverage\n#> \n#> 1 91.8\n```\n:::\n\n### The major downside\n\nWhen our predictions are extrapolating, the intervals can be very poor. Tree-based models can behave very differently than other models when predicting beyond the training set; they follow a static trend even as the predictor values move towards infinity. \n\nTo illustrate this, we can simulate another point outside of the training set range: \n\n\n\n::: {.cell layout-align=\"center\"}\n::: {.cell-output-display}\n![](figs/extrapolation-1.svg){fig-align='center' width=70%}\n:::\n:::\n\n\nThe neural network does fairly well but the quantile regression forest carries the same extract trend forward. In this case, the interval doesn't even include the predicted value (since two different models are used to compute both quantities). \n\nTo protect against this, we suggest using the tools in the [applicable package](https://applicable.tidymodels.org). It can help quantify how much a prediction is outside of the mainstream of the training set.\n\n\n## Coverage\n\nSpeaking of coverage, there's a [GitHub repo](https://github.com/topepo/conformal_sim) where we did simulations for these methods to determine their average coverage. The repository's readme file has a summary. \n\n## Learning More\n\nIf you are interested and would like to learn more, try these resources: \n\n* [_Introduction To Conformal Prediction With Python_](https://christophmolnar.com/books/conformal-prediction/) by Christoph Molnar\n (highly recommended)\n * [`awesome-conformal-prediction`](https://github.com/valeman/awesome-conformal-prediction) on GitHub. \n* [Ryan Tibshirani's notes](https://www.stat.berkeley.edu/~ryantibs/statlearn-s23/lectures/conformal.pdf) (pdf)\n* Angelopoulos, Anastasios N., and Stephen Bates. \"[A gentle introduction to conformal prediction and distribution-free uncertainty quantification](https://arxiv.org/abs/2107.07511).\" arXiv preprint arXiv:2107.07511 (2021).\n\n\n## Session information\n\n\n::: {.cell layout-align=\"center\"}\n\n```\n#> ─ Session info ─────────────────────────────────────────────────────\n#> version R version 4.6.0 (2026-04-24)\n#> language (EN)\n#> pandoc 3.1.3\n#> quarto 1.9.37\n#> \n#> ─ Packages ─────────────────────────────────────────────────────────\n#> package version date (UTC)\n#> broom 1.0.13 2026-05-14\n#> dials 1.4.3 2026-04-11\n#> dplyr 1.2.1 2026-04-03\n#> future 1.70.0 2026-03-14\n#> ggplot2 4.0.3 2026-04-22\n#> infer 1.1.0 2025-12-18\n#> nnet 7.3-20 2025-01-01\n#> parsnip 1.6.0 2026-05-14\n#> probably 1.2.0 2025-10-16\n#> purrr 1.2.2 2026-04-10\n#> quantregForest 1.3-7.1 2024-10-07\n#> recipes 1.3.2 2026-04-02\n#> rlang 1.2.0 2026-04-06\n#> rsample 1.3.2 2026-01-30\n#> tibble 3.3.1 2026-01-11\n#> tidymodels 1.5.0 2026-04-23\n#> tune 2.1.0 2026-04-17\n#> workflows 1.3.0 2025-08-27\n#> yardstick 1.4.0 2026-04-07\n#> \n#> ────────────────────────────────────────────────────────────────────\n```\n:::\n\n", "supporting": [], "filters": [ "rmarkdown/pagebreak.lua" diff --git a/_freeze/learn/models/pls/index/execute-results/html.json b/_freeze/learn/models/pls/index/execute-results/html.json index c8b4d2ae..7c894c83 100644 --- a/_freeze/learn/models/pls/index/execute-results/html.json +++ b/_freeze/learn/models/pls/index/execute-results/html.json @@ -1,8 +1,8 @@ { - "hash": "151055e6f230af993f45603d749fd946", + "hash": "d91f50819845f57655fa9bd91f0d9955", "result": { "engine": "knitr", - "markdown": "---\ntitle: \"Multivariate analysis using partial least squares\"\ncategories:\n - pre-processing\n - multivariate analysis\n - correlation\ntype: learn-subsection\nweight: 6\ndescription: | \n Build and fit a predictive model with more than one outcome.\ntoc: true\ntoc-depth: 2\nr-packages:\n - tidymodels\n - pls\n - sessioninfo\ninclude-after-body: ../../../html/resources.html\n---\n\n\n\n\n\n\n## Introduction\n\nTo use code in this article, you will need to install the following packages: pls, sessioninfo, and tidymodels.\n\n\"Multivariate analysis\" usually refers to multiple _outcomes_ being modeled, analyzed, and/or predicted. There are multivariate versions of many common statistical tools. For example, suppose there was a data set with columns `y1` and `y2` representing two outcomes to be predicted. The `lm()` function would look something like:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlm(cbind(y1, y2) ~ ., data = dat)\n```\n:::\n\n\nThis `cbind()` call is pretty awkward and is a consequence of how the traditional formula infrastructure works. The recipes package is a lot easier to work with! This article demonstrates how to model multiple outcomes. \n\nThe data that we'll use has three outcomes. From `?modeldata::meats`:\n\n> \"These data are recorded on a Tecator Infratec Food and Feed Analyzer working in the wavelength range 850 - 1050 nm by the Near Infrared Transmission (NIT) principle. Each sample contains finely chopped pure meat with different moisture, fat and protein contents.\n\n> \"For each meat sample the data consists of a 100 channel spectrum of absorbances and the contents of moisture (water), fat and protein. The absorbance is `-log10` of the transmittance measured by the spectrometer. The three contents, measured in percent, are determined by analytic chemistry.\"\n\nThe goal is to predict the proportion of the three substances using the chemistry test. There can often be a high degree of between-variable correlations in predictors, and that is certainly the case here. \n\nTo start, let's take the two data matrices (called `endpoints` and `absorp`) and bind them together in a data frame:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(modeldata)\ndata(meats)\n```\n:::\n\n\nThe three _outcomes_ have fairly high correlations also. \n\n## Preprocessing the data\n\nIf the outcomes can be predicted using a linear model, partial least squares (PLS) is an ideal method. PLS models the data as a function of a set of unobserved _latent_ variables that are derived in a manner similar to principal component analysis (PCA). \n\nPLS, unlike PCA, also incorporates the outcome data when creating the PLS components. Like PCA, it tries to maximize the variance of the predictors that are explained by the components but it also tries to simultaneously maximize the correlation between those components and the outcomes. In this way, PLS _chases_ variation of the predictors and outcomes. \n\nSince we are working with variances and covariances, we need to standardize the data. The recipe will center and scale all of the variables. \n\nMany base R functions that deal with multivariate outcomes using a formula require the use of `cbind()` on the left-hand side of the formula to work with the traditional formula methods. In tidymodels, recipes do not; the outcomes can be symbolically \"added\" together on the left-hand side:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nnorm_rec <- \n recipe(water + fat + protein ~ ., data = meats) |>\n step_normalize(everything()) \n```\n:::\n\n\nBefore we can finalize the PLS model, the number of PLS components to retain must be determined. This can be done using performance metrics such as the root mean squared error. However, we can also calculate the proportion of variance explained by the components for the _predictors and each of the outcomes_. This allows an informed choice to be made based on the level of evidence that the situation requires. \n\nSince the data set isn't large, let's use resampling to measure these proportions. With ten repeats of 10-fold cross-validation, we build the PLS model on 90% of the data and evaluate on the heldout 10%. For each of the 100 models, we extract and save the proportions. \n\nThe folds can be created using the [rsample](https://rsample.tidymodels.org/) package and the recipe can be estimated for each resample using the [`prepper()`](https://rsample.tidymodels.org/reference/prepper.html) function: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(57343)\nfolds <- vfold_cv(meats, repeats = 10)\n\nfolds <- \n folds |>\n mutate(recipes = map(splits, prepper, recipe = norm_rec))\n```\n:::\n\n\n## Partial least squares\n\nThe complicated parts for moving forward are:\n\n1. Formatting the predictors and outcomes into the format that the pls package requires, and\n2. Estimating the proportions. \n\nFor the first part, the standardized outcomes and predictors need to be formatted into two separate matrices. Since we used `retain = TRUE` when prepping the recipes, we can `bake()` with `new_data = NULl` to get the processed data back out. To save the data as a matrix, the option `composition = \"matrix\"` will avoid saving the data as tibbles and use the required format. \n\nThe pls package expects a simple formula to specify the model, but each side of the formula should _represent a matrix_. In other words, we need a data set with two columns where each column is a matrix. The secret to doing this is to \"protect\" the two matrices using `I()` when adding them to the data frame.\n\nThe calculation for the proportion of variance explained is straightforward for the predictors; the function `pls::explvar()` will compute that. For the outcomes, the process is more complicated. A ready-made function to compute these is not obvious but there is some code inside of the summary function to do the computation (see below). \n\nThe function `get_var_explained()` shown here will do all these computations and return a data frame with columns `components`, `source` (for the predictors, water, etc), and the `proportion` of variance that is explained by the components. \n\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(pls)\n\nget_var_explained <- function(recipe, ...) {\n \n # Extract the predictors and outcomes into their own matrices\n y_mat <- bake(recipe, new_data = NULL, composition = \"matrix\", all_outcomes())\n x_mat <- bake(recipe, new_data = NULL, composition = \"matrix\", all_predictors())\n \n # The pls package prefers the data in a data frame where the outcome\n # and predictors are in _matrices_. To make sure this is formatted\n # properly, use the `I()` function to inhibit `data.frame()` from making\n # all the individual columns. `pls_format` should have two columns.\n pls_format <- data.frame(\n endpoints = I(y_mat),\n measurements = I(x_mat)\n )\n # Fit the model\n mod <- plsr(endpoints ~ measurements, data = pls_format)\n \n # Get the proportion of the predictor variance that is explained\n # by the model for different number of components. \n xve <- explvar(mod)/100 \n\n # To do the same for the outcome, it is more complex. This code \n # was extracted from pls:::summary.mvr. \n explained <- \n drop(pls::R2(mod, estimate = \"train\", intercept = FALSE)$val) |> \n # transpose so that components are in rows\n t() |> \n as_tibble() |>\n # Add the predictor proportions\n mutate(predictors = cumsum(xve) |> as.vector(),\n components = seq_along(xve)) |>\n # Put into a tidy format that is tall\n pivot_longer(\n cols = c(-components),\n names_to = \"source\",\n values_to = \"proportion\"\n )\n}\n```\n:::\n\n\nWe compute this data frame for each resample and save the results in the different columns. \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nfolds <- \n folds |>\n mutate(var = map(recipes, get_var_explained),\n var = unname(var))\n```\n:::\n\n\nTo extract and aggregate these data, simple row binding can be used to stack the data vertically. Most of the action happens in the first 15 components so let's filter the data and compute the _average_ proportion.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nvariance_data <- \n bind_rows(folds[[\"var\"]]) |>\n filter(components <= 15) |>\n group_by(components, source) |>\n summarize(proportion = mean(proportion))\n#> `summarise()` has regrouped the output.\n#> ℹ Summaries were computed grouped by components and source.\n#> ℹ Output is grouped by components.\n#> ℹ Use `summarise(.groups = \"drop_last\")` to silence this message.\n#> ℹ Use `summarise(.by = c(components, source))` for per-operation\n#> grouping (`?dplyr::dplyr_by`) instead.\n```\n:::\n\n\nThe plot below shows that, if the protein measurement is important, you might require 10 or so components to achieve a good representation of that outcome. Note that the predictor variance is captured extremely well using a single component. This is due to the high degree of correlation in those data. \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nggplot(variance_data, aes(x = components, y = proportion, col = source)) + \n geom_line(alpha = 0.5, linewidth = 1.2) + \n geom_point() \n```\n\n::: {.cell-output-display}\n![](figs/plot-1.svg){fig-align='center' width=100%}\n:::\n:::\n\n\n\n## Session information {#session-info}\n\n\n::: {.cell layout-align=\"center\"}\n\n```\n#> ─ Session info ─────────────────────────────────────────────────────\n#> version R version 4.6.0 (2026-04-24)\n#> language (EN)\n#> pandoc 3.1.3\n#> quarto 1.9.37\n#> \n#> ─ Packages ─────────────────────────────────────────────────────────\n#> package version date (UTC)\n#> broom 1.0.13 2026-05-14\n#> dials 1.4.3 2026-04-11\n#> dplyr 1.2.1 2026-04-03\n#> ggplot2 4.0.3 2026-04-22\n#> infer 1.1.0 2025-12-18\n#> parsnip 1.6.0 2026-05-14\n#> pls 2.9-0 2026-02-21\n#> purrr 1.2.2 2026-04-10\n#> recipes 1.3.2 2026-04-02\n#> rlang 1.2.0 2026-04-06\n#> rsample 1.3.2 2026-01-30\n#> sessioninfo 1.2.3 2025-02-05\n#> tibble 3.3.1 2026-01-11\n#> tidymodels 1.5.0 2026-04-23\n#> tune 2.1.0 2026-04-17\n#> workflows 1.3.0 2025-08-27\n#> yardstick 1.4.0 2026-04-07\n#> \n#> ────────────────────────────────────────────────────────────────────\n```\n:::\n\n \n", + "markdown": "---\ntitle: \"Multivariate analysis using partial least squares\"\ncategories:\n - model fitting\n - pre-processing\n - multivariate analysis\n - correlation\ntype: learn-subsection\nweight: 6\ndescription: | \n Build and fit a predictive model with more than one outcome.\ntoc: true\ntoc-depth: 2\nr-packages:\n - tidymodels\n - pls\n - sessioninfo\ninclude-after-body: ../../../html/resources.html\n---\n\n\n\n\n\n\n## Introduction\n\nTo use code in this article, you will need to install the following packages: pls, sessioninfo, and tidymodels.\n\n\"Multivariate analysis\" usually refers to multiple _outcomes_ being modeled, analyzed, and/or predicted. There are multivariate versions of many common statistical tools. For example, suppose there was a data set with columns `y1` and `y2` representing two outcomes to be predicted. The `lm()` function would look something like:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlm(cbind(y1, y2) ~ ., data = dat)\n```\n:::\n\n\nThis `cbind()` call is pretty awkward and is a consequence of how the traditional formula infrastructure works. The recipes package is a lot easier to work with! This article demonstrates how to model multiple outcomes. \n\nThe data that we'll use has three outcomes. From `?modeldata::meats`:\n\n> \"These data are recorded on a Tecator Infratec Food and Feed Analyzer working in the wavelength range 850 - 1050 nm by the Near Infrared Transmission (NIT) principle. Each sample contains finely chopped pure meat with different moisture, fat and protein contents.\n\n> \"For each meat sample the data consists of a 100 channel spectrum of absorbances and the contents of moisture (water), fat and protein. The absorbance is `-log10` of the transmittance measured by the spectrometer. The three contents, measured in percent, are determined by analytic chemistry.\"\n\nThe goal is to predict the proportion of the three substances using the chemistry test. There can often be a high degree of between-variable correlations in predictors, and that is certainly the case here. \n\nTo start, let's take the two data matrices (called `endpoints` and `absorp`) and bind them together in a data frame:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(modeldata)\ndata(meats)\n```\n:::\n\n\nThe three _outcomes_ have fairly high correlations also. \n\n## Preprocessing the data\n\nIf the outcomes can be predicted using a linear model, partial least squares (PLS) is an ideal method. PLS models the data as a function of a set of unobserved _latent_ variables that are derived in a manner similar to principal component analysis (PCA). \n\nPLS, unlike PCA, also incorporates the outcome data when creating the PLS components. Like PCA, it tries to maximize the variance of the predictors that are explained by the components but it also tries to simultaneously maximize the correlation between those components and the outcomes. In this way, PLS _chases_ variation of the predictors and outcomes. \n\nSince we are working with variances and covariances, we need to standardize the data. The recipe will center and scale all of the variables. \n\nMany base R functions that deal with multivariate outcomes using a formula require the use of `cbind()` on the left-hand side of the formula to work with the traditional formula methods. In tidymodels, recipes do not; the outcomes can be symbolically \"added\" together on the left-hand side:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nnorm_rec <- \n recipe(water + fat + protein ~ ., data = meats) |>\n step_normalize(everything()) \n```\n:::\n\n\nBefore we can finalize the PLS model, the number of PLS components to retain must be determined. This can be done using performance metrics such as the root mean squared error. However, we can also calculate the proportion of variance explained by the components for the _predictors and each of the outcomes_. This allows an informed choice to be made based on the level of evidence that the situation requires. \n\nSince the data set isn't large, let's use resampling to measure these proportions. With ten repeats of 10-fold cross-validation, we build the PLS model on 90% of the data and evaluate on the heldout 10%. For each of the 100 models, we extract and save the proportions. \n\nThe folds can be created using the [rsample](https://rsample.tidymodels.org/) package and the recipe can be estimated for each resample using the [`prepper()`](https://rsample.tidymodels.org/reference/prepper.html) function: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(57343)\nfolds <- vfold_cv(meats, repeats = 10)\n\nfolds <- \n folds |>\n mutate(recipes = map(splits, prepper, recipe = norm_rec))\n```\n:::\n\n\n## Partial least squares\n\nThe complicated parts for moving forward are:\n\n1. Formatting the predictors and outcomes into the format that the pls package requires, and\n2. Estimating the proportions. \n\nFor the first part, the standardized outcomes and predictors need to be formatted into two separate matrices. Since we used `retain = TRUE` when prepping the recipes, we can `bake()` with `new_data = NULl` to get the processed data back out. To save the data as a matrix, the option `composition = \"matrix\"` will avoid saving the data as tibbles and use the required format. \n\nThe pls package expects a simple formula to specify the model, but each side of the formula should _represent a matrix_. In other words, we need a data set with two columns where each column is a matrix. The secret to doing this is to \"protect\" the two matrices using `I()` when adding them to the data frame.\n\nThe calculation for the proportion of variance explained is straightforward for the predictors; the function `pls::explvar()` will compute that. For the outcomes, the process is more complicated. A ready-made function to compute these is not obvious but there is some code inside of the summary function to do the computation (see below). \n\nThe function `get_var_explained()` shown here will do all these computations and return a data frame with columns `components`, `source` (for the predictors, water, etc), and the `proportion` of variance that is explained by the components. \n\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(pls)\n\nget_var_explained <- function(recipe, ...) {\n \n # Extract the predictors and outcomes into their own matrices\n y_mat <- bake(recipe, new_data = NULL, composition = \"matrix\", all_outcomes())\n x_mat <- bake(recipe, new_data = NULL, composition = \"matrix\", all_predictors())\n \n # The pls package prefers the data in a data frame where the outcome\n # and predictors are in _matrices_. To make sure this is formatted\n # properly, use the `I()` function to inhibit `data.frame()` from making\n # all the individual columns. `pls_format` should have two columns.\n pls_format <- data.frame(\n endpoints = I(y_mat),\n measurements = I(x_mat)\n )\n # Fit the model\n mod <- plsr(endpoints ~ measurements, data = pls_format)\n \n # Get the proportion of the predictor variance that is explained\n # by the model for different number of components. \n xve <- explvar(mod)/100 \n\n # To do the same for the outcome, it is more complex. This code \n # was extracted from pls:::summary.mvr. \n explained <- \n drop(pls::R2(mod, estimate = \"train\", intercept = FALSE)$val) |> \n # transpose so that components are in rows\n t() |> \n as_tibble() |>\n # Add the predictor proportions\n mutate(predictors = cumsum(xve) |> as.vector(),\n components = seq_along(xve)) |>\n # Put into a tidy format that is tall\n pivot_longer(\n cols = c(-components),\n names_to = \"source\",\n values_to = \"proportion\"\n )\n}\n```\n:::\n\n\nWe compute this data frame for each resample and save the results in the different columns. \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nfolds <- \n folds |>\n mutate(var = map(recipes, get_var_explained),\n var = unname(var))\n```\n:::\n\n\nTo extract and aggregate these data, simple row binding can be used to stack the data vertically. Most of the action happens in the first 15 components so let's filter the data and compute the _average_ proportion.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nvariance_data <- \n bind_rows(folds[[\"var\"]]) |>\n filter(components <= 15) |>\n group_by(components, source) |>\n summarize(proportion = mean(proportion))\n#> `summarise()` has regrouped the output.\n#> ℹ Summaries were computed grouped by components and source.\n#> ℹ Output is grouped by components.\n#> ℹ Use `summarise(.groups = \"drop_last\")` to silence this message.\n#> ℹ Use `summarise(.by = c(components, source))` for per-operation\n#> grouping (`?dplyr::dplyr_by`) instead.\n```\n:::\n\n\nThe plot below shows that, if the protein measurement is important, you might require 10 or so components to achieve a good representation of that outcome. Note that the predictor variance is captured extremely well using a single component. This is due to the high degree of correlation in those data. \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nggplot(variance_data, aes(x = components, y = proportion, col = source)) + \n geom_line(alpha = 0.5, linewidth = 1.2) + \n geom_point() \n```\n\n::: {.cell-output-display}\n![](figs/plot-1.svg){fig-align='center' width=100%}\n:::\n:::\n\n\n\n## Session information {#session-info}\n\n\n::: {.cell layout-align=\"center\"}\n\n```\n#> ─ Session info ─────────────────────────────────────────────────────\n#> version R version 4.6.0 (2026-04-24)\n#> language (EN)\n#> pandoc 3.1.3\n#> quarto 1.9.37\n#> \n#> ─ Packages ─────────────────────────────────────────────────────────\n#> package version date (UTC)\n#> broom 1.0.13 2026-05-14\n#> dials 1.4.3 2026-04-11\n#> dplyr 1.2.1 2026-04-03\n#> ggplot2 4.0.3 2026-04-22\n#> infer 1.1.0 2025-12-18\n#> parsnip 1.6.0 2026-05-14\n#> pls 2.9-0 2026-02-21\n#> purrr 1.2.2 2026-04-10\n#> recipes 1.3.2 2026-04-02\n#> rlang 1.2.0 2026-04-06\n#> rsample 1.3.2 2026-01-30\n#> sessioninfo 1.2.3 2025-02-05\n#> tibble 3.3.1 2026-01-11\n#> tidymodels 1.5.0 2026-04-23\n#> tune 2.1.0 2026-04-17\n#> workflows 1.3.0 2025-08-27\n#> yardstick 1.4.0 2026-04-07\n#> \n#> ────────────────────────────────────────────────────────────────────\n```\n:::\n\n \n", "supporting": [], "filters": [ "rmarkdown/pagebreak.lua" diff --git a/_freeze/learn/work/bayes-opt/index/execute-results/html.json b/_freeze/learn/work/bayes-opt/index/execute-results/html.json index 9e240790..fd4ad862 100644 --- a/_freeze/learn/work/bayes-opt/index/execute-results/html.json +++ b/_freeze/learn/work/bayes-opt/index/execute-results/html.json @@ -1,8 +1,8 @@ { - "hash": "711430e758ebffe1ec77c8f8f5a898ef", + "hash": "53c2e354d4dd032db9fc105cb3f1045b", "result": { "engine": "knitr", - "markdown": "---\ntitle: \"Iterative Bayesian optimization of a classification model\"\ncategories:\n - tuning\n - SVMs\ntype: learn-subsection\nweight: 3\ndescription: | \n Identify the best hyperparameters for a model using Bayesian optimization of iterative search.\ntoc: true\ntoc-depth: 2\nr-packages:\n - tidymodels\n - kernlab\n - themis\n - future\ninclude-after-body: ../../../html/resources.html\n---\n\n\n \n\n\n## Introduction\n\nTo use code in this article, you will need to install the following packages: future, kernlab, themis, and tidymodels.\n\nMany of the examples for model tuning focus on [grid search](/learn/work/tune-svm/). For that method, all the candidate tuning parameter combinations are defined prior to evaluation. Alternatively, _iterative search_ can be used to analyze the existing tuning parameter results and then _predict_ which tuning parameters to try next. \n\nThere are a variety of methods for iterative search and the focus in this article is on _Bayesian optimization_. For more information on this method, these resources might be helpful:\n\n* [_Practical bayesian optimization of machine learning algorithms_](https://scholar.google.com/scholar?hl=en&as_sdt=0%2C7&q=Practical+Bayesian+Optimization+of+Machine+Learning+Algorithms&btnG=) (2012). J Snoek, H Larochelle, and RP Adams. Advances in neural information. \n\n* [_A Tutorial on Bayesian Optimization for Machine Learning_](https://www.cs.toronto.edu/~rgrosse/courses/csc411_f18/tutorials/tut8_adams_slides.pdf) (2018). R Adams.\n\n * [_Gaussian Processes for Machine Learning_](http://www.gaussianprocess.org/gpml/) (2006). C E Rasmussen and C Williams.\n\n* [Other articles!](https://scholar.google.com/scholar?hl=en&as_sdt=0%2C7&q=\"Bayesian+Optimization\"&btnG=)\n\n\n## Cell segmenting revisited\n\nTo demonstrate this approach to tuning models, let's return to the cell segmentation data from the [Getting Started](/start/resampling/) article on resampling: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(tidymodels)\nlibrary(modeldata)\n\n# Load data\ndata(cells)\n\nset.seed(2369)\ntr_te_split <- initial_split(cells |> select(-case), prop = 3/4)\ncell_train <- training(tr_te_split)\ncell_test <- testing(tr_te_split)\n\nset.seed(1697)\nfolds <- vfold_cv(cell_train, v = 10)\n```\n:::\n\n\n## The tuning scheme\n\nSince the predictors are highly correlated, we can used a recipe to convert the original predictors to principal component scores. There is also slight class imbalance in these data; about 64% of the data are poorly segmented. To mitigate this, the data will be down-sampled at the end of the pre-processing so that the number of poorly and well segmented cells occur with equal frequency. We can use a recipe for all this pre-processing, but the number of principal components will need to be _tuned_ so that we have enough (but not too many) representations of the data. \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(themis)\n\ncell_pre_proc <-\n recipe(class ~ ., data = cell_train) |>\n step_YeoJohnson(all_predictors()) |>\n step_normalize(all_predictors()) |>\n step_pca(all_predictors(), num_comp = tune()) |>\n step_downsample(class)\n```\n:::\n\n\nIn this analysis, we will use a support vector machine to model the data. Let's use a radial basis function (RBF) kernel and tune its main parameter ($\\sigma$). Additionally, the main SVM parameter, the cost value, also needs optimization. \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nsvm_mod <-\n svm_rbf(mode = \"classification\", cost = tune(), rbf_sigma = tune()) |>\n set_engine(\"kernlab\")\n```\n:::\n\n\nThese two objects (the recipe and model) will be combined into a single object via the `workflow()` function from the [workflows](https://workflows.tidymodels.org/) package; this object will be used in the optimization process. \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nsvm_wflow <-\n workflow() |>\n add_model(svm_mod) |>\n add_recipe(cell_pre_proc)\n```\n:::\n\n\nFrom this object, we can derive information about what parameters are slated to be tuned. A parameter set is derived by: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nsvm_set <- extract_parameter_set_dials(svm_wflow)\nsvm_set\n#> Collection of 3 parameters for tuning\n#> \n#> identifier type object\n#> cost cost nparam[+]\n#> rbf_sigma rbf_sigma nparam[+]\n#> num_comp num_comp nparam[+]\n#> \n```\n:::\n\n\nThe default range for the number of PCA components is rather small for this data set. A member of the parameter set can be modified using the `update()` function. Let's constrain the search to one to twenty components by updating the `num_comp` parameter. Additionally, the lower bound of this parameter is set to zero which specifies that the original predictor set should also be evaluated (i.e., with no PCA step at all): \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nsvm_set <- \n svm_set |> \n update(num_comp = num_comp(c(0L, 20L)))\n```\n:::\n\n\n## Sequential tuning \n\nBayesian optimization is a sequential method that uses a model to predict new candidate parameters for assessment. When scoring potential parameter value, the mean and variance of performance are predicted. The strategy used to define how these two statistical quantities are used is defined by an _acquisition function_. \n\nFor example, one approach for scoring new candidates is to use a confidence bound. Suppose accuracy is being optimized. For a metric that we want to maximize, a lower confidence bound can be used. The multiplier on the standard error (denoted as $\\kappa$) is a value that can be used to make trade-offs between **exploration** and **exploitation**. \n\n * **Exploration** means that the search will consider candidates in untested space.\n\n * **Exploitation** focuses in areas where the previous best results occurred. \n\nThe variance predicted by the Bayesian model is mostly spatial variation; the value will be large for candidate values that are not close to values that have already been evaluated. If the standard error multiplier is high, the search process will be more likely to avoid areas without candidate values in the vicinity. \n\nWe'll use another acquisition function, _expected improvement_, that determines which candidates are likely to be helpful relative to the current best results. This is the default acquisition function. More information on these functions can be found in the [package vignette for acquisition functions](https://tune.tidymodels.org/articles/acquisition_functions.html). \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(12)\nsearch_res <-\n svm_wflow |> \n tune_bayes(\n resamples = folds,\n # To use non-default parameter ranges\n param_info = svm_set,\n # Generate five at semi-random to start\n initial = 5,\n iter = 50,\n # How to measure performance?\n metrics = metric_set(roc_auc),\n control = control_bayes(no_improve = 30, verbose = TRUE)\n )\n#> \n#> ❯ Generating a set of 5 initial parameter results\n#> maximum number of iterations reached 0.02644788 -4.678954e-05maximum number of iterations reached 4.883479e-05 -1.405376e-11maximum number of iterations reached 0.0006198503 -7.83456e-07maximum number of iterations reached 0.02616499 -4.815298e-05maximum number of iterations reached 4.783692e-05 -3.531675e-12maximum number of iterations reached 0.000532615 -6.369734e-07maximum number of iterations reached 0.02563573 -3.389226e-05maximum number of iterations reached 4.597584e-05 -1.510886e-11maximum number of iterations reached 0.0009844145 -1.428013e-06\n#> ✓ Initialization complete\n#> \n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> maximum number of iterations reached 0.01591622 -1.060394e-06maximum number of iterations reached 0.01572637 -9.970078e-07\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> maximum number of iterations reached 0.01522807 -7.939643e-07\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n```\n:::\n\n\nThe resulting tibble is a stacked set of rows of the rsample object with an additional column for the iteration number:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nsearch_res\n#> # Tuning results\n#> # 10-fold cross-validation \n#> # A tibble: 510 × 5\n#> splits id .metrics .notes .iter\n#> \n#> 1 Fold01 0\n#> 2 Fold02 0\n#> 3 Fold03 0\n#> 4 Fold04 0\n#> 5 Fold05 0\n#> 6 Fold06 0\n#> 7 Fold07 0\n#> 8 Fold08 0\n#> 9 Fold09 0\n#> 10 Fold10 0\n#> # ℹ 500 more rows\n```\n:::\n\n\nAs with grid search, we can summarize the results over resamples:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nestimates <- \n collect_metrics(search_res) |> \n arrange(.iter)\n\nestimates\n#> # A tibble: 55 × 10\n#> cost rbf_sigma num_comp .metric .estimator mean n std_err .config \n#> \n#> 1 0.177 1 e-10 0 roc_auc binary 0.357 10 0.112 pre1_mod…\n#> 2 0.000977 3.16e- 3 5 roc_auc binary 0.346 10 0.114 pre2_mod…\n#> 3 2.38 1 e+ 0 10 roc_auc binary 0.821 10 0.0141 pre3_mod…\n#> 4 32 3.16e- 8 15 roc_auc binary 0.349 10 0.114 pre4_mod…\n#> 5 0.0131 1 e- 5 20 roc_auc binary 0.347 10 0.116 pre5_mod…\n#> 6 31.2 1.61e- 5 0 roc_auc binary 0.877 10 0.0122 iter01 \n#> 7 0.00105 3.49e-10 10 roc_auc binary 0.238 10 0.0585 iter02 \n#> 8 24.2 4.41e- 2 1 roc_auc binary 0.770 10 0.0109 iter03 \n#> 9 3.31 1.06e- 6 6 roc_auc binary 0.347 10 0.115 iter04 \n#> 10 20.6 4.71e- 1 0 roc_auc binary 0.814 10 0.0114 iter05 \n#> # ℹ 45 more rows\n#> # ℹ 1 more variable: .iter \n```\n:::\n\n\n\nThe best performance of the initial set of candidate values was `AUC = 0.8214598 `. The best results were achieved at iteration 38 with a corresponding AUC value of 0.9006475. The five best results are:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nshow_best(search_res, metric = \"roc_auc\")\n#> # A tibble: 5 × 10\n#> cost rbf_sigma num_comp .metric .estimator mean n std_err .config .iter\n#> \n#> 1 4.70 0.0288 9 roc_auc binary 0.901 10 0.00926 iter38 38\n#> 2 4.93 0.0310 9 roc_auc binary 0.900 10 0.00933 iter48 48\n#> 3 5.09 0.0309 9 roc_auc binary 0.900 10 0.00934 iter30 30\n#> 4 5.28 0.0245 8 roc_auc binary 0.899 10 0.00982 iter45 45\n#> 5 4.80 0.0385 8 roc_auc binary 0.898 10 0.00995 iter43 43\n```\n:::\n\n\nA plot of the search iterations can be created via:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nautoplot(search_res, type = \"performance\")\n```\n\n::: {.cell-output-display}\n![](figs/bo-plot-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nThere are many parameter combinations have roughly equivalent results. \n\nHow did the parameters change over iterations? \n\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nautoplot(search_res, type = \"parameters\") + \n labs(x = \"Iterations\", y = NULL)\n```\n\n::: {.cell-output-display}\n![](figs/bo-param-plot-1.svg){fig-align='center' width=864}\n:::\n:::\n\n\n\n\n\n## Session information {#session-info}\n\n\n::: {.cell layout-align=\"center\"}\n\n```\n#> ─ Session info ─────────────────────────────────────────────────────\n#> version R version 4.6.0 (2026-04-24)\n#> language (EN)\n#> pandoc 3.1.3\n#> quarto 1.9.37\n#> \n#> ─ Packages ─────────────────────────────────────────────────────────\n#> package version date (UTC)\n#> broom 1.0.13 2026-05-14\n#> dials 1.4.3 2026-04-11\n#> dplyr 1.2.1 2026-04-03\n#> future 1.70.0 2026-03-14\n#> ggplot2 4.0.3 2026-04-22\n#> infer 1.1.0 2025-12-18\n#> kernlab 0.9-33 2024-08-13\n#> parsnip 1.6.0 2026-05-14\n#> purrr 1.2.2 2026-04-10\n#> recipes 1.3.2 2026-04-02\n#> rlang 1.2.0 2026-04-06\n#> rsample 1.3.2 2026-01-30\n#> themis 1.0.3 2025-01-23\n#> tibble 3.3.1 2026-01-11\n#> tidymodels 1.5.0 2026-04-23\n#> tune 2.1.0 2026-04-17\n#> workflows 1.3.0 2025-08-27\n#> yardstick 1.4.0 2026-04-07\n#> \n#> ────────────────────────────────────────────────────────────────────\n```\n:::\n\n \n", + "markdown": "---\ntitle: \"Iterative Bayesian optimization of a classification model\"\ncategories:\n - tuning and workflows\n - tuning\n - SVMs\ntype: learn-subsection\nweight: 3\ndescription: | \n Identify the best hyperparameters for a model using Bayesian optimization of iterative search.\ntoc: true\ntoc-depth: 2\nr-packages:\n - tidymodels\n - kernlab\n - themis\n - future\ninclude-after-body: ../../../html/resources.html\n---\n\n\n \n\n\n## Introduction\n\nTo use code in this article, you will need to install the following packages: future, kernlab, themis, and tidymodels.\n\nMany of the examples for model tuning focus on [grid search](/learn/work/tune-svm/). For that method, all the candidate tuning parameter combinations are defined prior to evaluation. Alternatively, _iterative search_ can be used to analyze the existing tuning parameter results and then _predict_ which tuning parameters to try next. \n\nThere are a variety of methods for iterative search and the focus in this article is on _Bayesian optimization_. For more information on this method, these resources might be helpful:\n\n* [_Practical bayesian optimization of machine learning algorithms_](https://scholar.google.com/scholar?hl=en&as_sdt=0%2C7&q=Practical+Bayesian+Optimization+of+Machine+Learning+Algorithms&btnG=) (2012). J Snoek, H Larochelle, and RP Adams. Advances in neural information. \n\n* [_A Tutorial on Bayesian Optimization for Machine Learning_](https://www.cs.toronto.edu/~rgrosse/courses/csc411_f18/tutorials/tut8_adams_slides.pdf) (2018). R Adams.\n\n * [_Gaussian Processes for Machine Learning_](http://www.gaussianprocess.org/gpml/) (2006). C E Rasmussen and C Williams.\n\n* [Other articles!](https://scholar.google.com/scholar?hl=en&as_sdt=0%2C7&q=\"Bayesian+Optimization\"&btnG=)\n\n\n## Cell segmenting revisited\n\nTo demonstrate this approach to tuning models, let's return to the cell segmentation data from the [Getting Started](/start/resampling/) article on resampling: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(tidymodels)\nlibrary(modeldata)\n\n# Load data\ndata(cells)\n\nset.seed(2369)\ntr_te_split <- initial_split(cells |> select(-case), prop = 3/4)\ncell_train <- training(tr_te_split)\ncell_test <- testing(tr_te_split)\n\nset.seed(1697)\nfolds <- vfold_cv(cell_train, v = 10)\n```\n:::\n\n\n## The tuning scheme\n\nSince the predictors are highly correlated, we can used a recipe to convert the original predictors to principal component scores. There is also slight class imbalance in these data; about 64% of the data are poorly segmented. To mitigate this, the data will be down-sampled at the end of the pre-processing so that the number of poorly and well segmented cells occur with equal frequency. We can use a recipe for all this pre-processing, but the number of principal components will need to be _tuned_ so that we have enough (but not too many) representations of the data. \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(themis)\n\ncell_pre_proc <-\n recipe(class ~ ., data = cell_train) |>\n step_YeoJohnson(all_predictors()) |>\n step_normalize(all_predictors()) |>\n step_pca(all_predictors(), num_comp = tune()) |>\n step_downsample(class)\n```\n:::\n\n\nIn this analysis, we will use a support vector machine to model the data. Let's use a radial basis function (RBF) kernel and tune its main parameter ($\\sigma$). Additionally, the main SVM parameter, the cost value, also needs optimization. \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nsvm_mod <-\n svm_rbf(mode = \"classification\", cost = tune(), rbf_sigma = tune()) |>\n set_engine(\"kernlab\")\n```\n:::\n\n\nThese two objects (the recipe and model) will be combined into a single object via the `workflow()` function from the [workflows](https://workflows.tidymodels.org/) package; this object will be used in the optimization process. \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nsvm_wflow <-\n workflow() |>\n add_model(svm_mod) |>\n add_recipe(cell_pre_proc)\n```\n:::\n\n\nFrom this object, we can derive information about what parameters are slated to be tuned. A parameter set is derived by: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nsvm_set <- extract_parameter_set_dials(svm_wflow)\nsvm_set\n#> Collection of 3 parameters for tuning\n#> \n#> identifier type object\n#> cost cost nparam[+]\n#> rbf_sigma rbf_sigma nparam[+]\n#> num_comp num_comp nparam[+]\n#> \n```\n:::\n\n\nThe default range for the number of PCA components is rather small for this data set. A member of the parameter set can be modified using the `update()` function. Let's constrain the search to one to twenty components by updating the `num_comp` parameter. Additionally, the lower bound of this parameter is set to zero which specifies that the original predictor set should also be evaluated (i.e., with no PCA step at all): \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nsvm_set <- \n svm_set |> \n update(num_comp = num_comp(c(0L, 20L)))\n```\n:::\n\n\n## Sequential tuning \n\nBayesian optimization is a sequential method that uses a model to predict new candidate parameters for assessment. When scoring potential parameter value, the mean and variance of performance are predicted. The strategy used to define how these two statistical quantities are used is defined by an _acquisition function_. \n\nFor example, one approach for scoring new candidates is to use a confidence bound. Suppose accuracy is being optimized. For a metric that we want to maximize, a lower confidence bound can be used. The multiplier on the standard error (denoted as $\\kappa$) is a value that can be used to make trade-offs between **exploration** and **exploitation**. \n\n * **Exploration** means that the search will consider candidates in untested space.\n\n * **Exploitation** focuses in areas where the previous best results occurred. \n\nThe variance predicted by the Bayesian model is mostly spatial variation; the value will be large for candidate values that are not close to values that have already been evaluated. If the standard error multiplier is high, the search process will be more likely to avoid areas without candidate values in the vicinity. \n\nWe'll use another acquisition function, _expected improvement_, that determines which candidates are likely to be helpful relative to the current best results. This is the default acquisition function. More information on these functions can be found in the [package vignette for acquisition functions](https://tune.tidymodels.org/articles/acquisition_functions.html). \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(12)\nsearch_res <-\n svm_wflow |> \n tune_bayes(\n resamples = folds,\n # To use non-default parameter ranges\n param_info = svm_set,\n # Generate five at semi-random to start\n initial = 5,\n iter = 50,\n # How to measure performance?\n metrics = metric_set(roc_auc),\n control = control_bayes(no_improve = 30, verbose = TRUE)\n )\n#> \n#> ❯ Generating a set of 5 initial parameter results\n#> maximum number of iterations reached 0.02644788 -4.678954e-05maximum number of iterations reached 4.883479e-05 -1.405376e-11maximum number of iterations reached 0.0006198503 -7.83456e-07maximum number of iterations reached 0.02616499 -4.815298e-05maximum number of iterations reached 4.783692e-05 -3.531675e-12maximum number of iterations reached 0.000532615 -6.369734e-07maximum number of iterations reached 0.02563573 -3.389226e-05maximum number of iterations reached 4.597584e-05 -1.510886e-11maximum number of iterations reached 0.0009844145 -1.428013e-06\n#> ✓ Initialization complete\n#> \n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> maximum number of iterations reached 0.01591622 -1.060394e-06maximum number of iterations reached 0.01572637 -9.970078e-07\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> maximum number of iterations reached 0.01522807 -7.939643e-07\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n#> i Estimating performance\n#> i Fold01: preprocessor 1/1\n#> i Fold01: preprocessor 1/1 (prediction data)\n#> i Fold01: preprocessor 1/1, model 1/1\n#> i Fold01: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold02: preprocessor 1/1\n#> i Fold02: preprocessor 1/1 (prediction data)\n#> i Fold02: preprocessor 1/1, model 1/1\n#> i Fold02: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold03: preprocessor 1/1\n#> i Fold03: preprocessor 1/1 (prediction data)\n#> i Fold03: preprocessor 1/1, model 1/1\n#> i Fold03: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold04: preprocessor 1/1\n#> i Fold04: preprocessor 1/1 (prediction data)\n#> i Fold04: preprocessor 1/1, model 1/1\n#> i Fold04: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold05: preprocessor 1/1\n#> i Fold05: preprocessor 1/1 (prediction data)\n#> i Fold05: preprocessor 1/1, model 1/1\n#> i Fold05: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold06: preprocessor 1/1\n#> i Fold06: preprocessor 1/1 (prediction data)\n#> i Fold06: preprocessor 1/1, model 1/1\n#> i Fold06: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold07: preprocessor 1/1\n#> i Fold07: preprocessor 1/1 (prediction data)\n#> i Fold07: preprocessor 1/1, model 1/1\n#> i Fold07: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold08: preprocessor 1/1\n#> i Fold08: preprocessor 1/1 (prediction data)\n#> i Fold08: preprocessor 1/1, model 1/1\n#> i Fold08: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold09: preprocessor 1/1\n#> i Fold09: preprocessor 1/1 (prediction data)\n#> i Fold09: preprocessor 1/1, model 1/1\n#> i Fold09: preprocessor 1/1, model 1/1 (predictions)\n#> i Fold10: preprocessor 1/1\n#> i Fold10: preprocessor 1/1 (prediction data)\n#> i Fold10: preprocessor 1/1, model 1/1\n#> i Fold10: preprocessor 1/1, model 1/1 (predictions)\n#> ✓ Estimating performance\n```\n:::\n\n\nThe resulting tibble is a stacked set of rows of the rsample object with an additional column for the iteration number:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nsearch_res\n#> # Tuning results\n#> # 10-fold cross-validation \n#> # A tibble: 510 × 5\n#> splits id .metrics .notes .iter\n#> \n#> 1 Fold01 0\n#> 2 Fold02 0\n#> 3 Fold03 0\n#> 4 Fold04 0\n#> 5 Fold05 0\n#> 6 Fold06 0\n#> 7 Fold07 0\n#> 8 Fold08 0\n#> 9 Fold09 0\n#> 10 Fold10 0\n#> # ℹ 500 more rows\n```\n:::\n\n\nAs with grid search, we can summarize the results over resamples:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nestimates <- \n collect_metrics(search_res) |> \n arrange(.iter)\n\nestimates\n#> # A tibble: 55 × 10\n#> cost rbf_sigma num_comp .metric .estimator mean n std_err .config \n#> \n#> 1 0.177 1 e-10 0 roc_auc binary 0.357 10 0.112 pre1_mod…\n#> 2 0.000977 3.16e- 3 5 roc_auc binary 0.346 10 0.114 pre2_mod…\n#> 3 2.38 1 e+ 0 10 roc_auc binary 0.821 10 0.0141 pre3_mod…\n#> 4 32 3.16e- 8 15 roc_auc binary 0.349 10 0.114 pre4_mod…\n#> 5 0.0131 1 e- 5 20 roc_auc binary 0.347 10 0.116 pre5_mod…\n#> 6 31.2 1.61e- 5 0 roc_auc binary 0.877 10 0.0122 iter01 \n#> 7 0.00105 3.49e-10 10 roc_auc binary 0.238 10 0.0585 iter02 \n#> 8 24.2 4.41e- 2 1 roc_auc binary 0.770 10 0.0109 iter03 \n#> 9 3.31 1.06e- 6 6 roc_auc binary 0.347 10 0.115 iter04 \n#> 10 20.6 4.71e- 1 0 roc_auc binary 0.814 10 0.0114 iter05 \n#> # ℹ 45 more rows\n#> # ℹ 1 more variable: .iter \n```\n:::\n\n\n\nThe best performance of the initial set of candidate values was `AUC = 0.8214598 `. The best results were achieved at iteration 38 with a corresponding AUC value of 0.9006475. The five best results are:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nshow_best(search_res, metric = \"roc_auc\")\n#> # A tibble: 5 × 10\n#> cost rbf_sigma num_comp .metric .estimator mean n std_err .config .iter\n#> \n#> 1 4.70 0.0288 9 roc_auc binary 0.901 10 0.00926 iter38 38\n#> 2 4.93 0.0310 9 roc_auc binary 0.900 10 0.00933 iter48 48\n#> 3 5.09 0.0309 9 roc_auc binary 0.900 10 0.00934 iter30 30\n#> 4 5.28 0.0245 8 roc_auc binary 0.899 10 0.00982 iter45 45\n#> 5 4.80 0.0385 8 roc_auc binary 0.898 10 0.00995 iter43 43\n```\n:::\n\n\nA plot of the search iterations can be created via:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nautoplot(search_res, type = \"performance\")\n```\n\n::: {.cell-output-display}\n![](figs/bo-plot-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nThere are many parameter combinations have roughly equivalent results. \n\nHow did the parameters change over iterations? \n\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nautoplot(search_res, type = \"parameters\") + \n labs(x = \"Iterations\", y = NULL)\n```\n\n::: {.cell-output-display}\n![](figs/bo-param-plot-1.svg){fig-align='center' width=864}\n:::\n:::\n\n\n\n\n\n## Session information {#session-info}\n\n\n::: {.cell layout-align=\"center\"}\n\n```\n#> ─ Session info ─────────────────────────────────────────────────────\n#> version R version 4.6.0 (2026-04-24)\n#> language (EN)\n#> pandoc 3.1.3\n#> quarto 1.9.37\n#> \n#> ─ Packages ─────────────────────────────────────────────────────────\n#> package version date (UTC)\n#> broom 1.0.13 2026-05-14\n#> dials 1.4.3 2026-04-11\n#> dplyr 1.2.1 2026-04-03\n#> future 1.70.0 2026-03-14\n#> ggplot2 4.0.3 2026-04-22\n#> infer 1.1.0 2025-12-18\n#> kernlab 0.9-33 2024-08-13\n#> parsnip 1.6.0 2026-05-14\n#> purrr 1.2.2 2026-04-10\n#> recipes 1.3.2 2026-04-02\n#> rlang 1.2.0 2026-04-06\n#> rsample 1.3.2 2026-01-30\n#> themis 1.0.3 2025-01-23\n#> tibble 3.3.1 2026-01-11\n#> tidymodels 1.5.0 2026-04-23\n#> tune 2.1.0 2026-04-17\n#> workflows 1.3.0 2025-08-27\n#> yardstick 1.4.0 2026-04-07\n#> \n#> ────────────────────────────────────────────────────────────────────\n```\n:::\n\n \n", "supporting": [], "filters": [ "rmarkdown/pagebreak.lua" diff --git a/_freeze/learn/work/case-weights/index/execute-results/html.json b/_freeze/learn/work/case-weights/index/execute-results/html.json index 6dbb4c1f..10711725 100644 --- a/_freeze/learn/work/case-weights/index/execute-results/html.json +++ b/_freeze/learn/work/case-weights/index/execute-results/html.json @@ -1,8 +1,8 @@ { - "hash": "428e8020e4d058cd51c4e3dae81f3fed", + "hash": "a3b0ac4ac8fc7a706b4391c5ccad4f73", "result": { "engine": "knitr", - "markdown": "---\ntitle: \"Creating case weights based on time\"\ncategories:\n - model fitting\n - time series\ntype: learn-subsection\nweight: 5\ndescription: | \n Create models that use case weights, extract them from fitted models, and visualize them.\ntoc: true\ntoc-depth: 2\nr-packages:\n - tidymodels\ninclude-after-body: ../../../html/resources.html\n---\n\n\n \n\n\n## Introduction\n\nTo use code in this article, you will need to install the following packages: tidymodels.\n\nThis article demonstrates how to create and use importance weights in a predictive model. Using importance weights is a way to have our model care more about some observations than others.\n\n## Example Data\n\nTo demonstrate we will use the Chicago data from the modeldata package.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(tidymodels)\ndata(Chicago)\n\nChicago <- Chicago |>\n select(ridership, date, one_of(stations))\n```\n:::\n\n\nFrom `?Chicago`\n\n> These data are from Kuhn and Johnson (2020) and contain an abbreviated training set for modeling the number of people (in thousands) who enter the Clark and Lake L station.\n\n> The date column corresponds to the current date. The columns with station names (Austin through California) are a sample of the columns used in the original analysis (for filesize reasons). These are 14 day lag variables (i.e. date - 14 days). There are columns related to weather and sports team schedules.\n\nFor simplicity, we have limited our view to the date and station variables.\n\n## Creating weights\n\nThis data set contains daily information from 2001-01-22 to 2016-08-28. We will pretend that it is January 1st, 2016 and we want to predict the ridership for the remainder of 2016 using the date and station variables as predictors. Without any weighting, all the previous observations would have the same influence on the model. This may not be ideal since some observations appear a long time ago and not be as representative of the future as more recent observations. \n\nWe could just use recent observations to fit the model, ensuring that the training data stays as close to the testing data as possible. While a tempting idea, it would throw out a lot of informative data. Instead let us assign a weight to each observation, related to how long ago the observation was taken. This way we are not completely throwing away any observation; we are only giving less weight to data farther in the past. \n\nWe need to decide on a way to calculate the case weights. The main thing constraint is that the weight cannot be negative, and it would be nice if today was weighted as 1. So we need a function that is 1 when `x = 0` and decreasing otherwise. There are many kinds of functions like that, and we will be using this exponential decay function\n\n$$ weight = base ^ x $$\n\nwhere `base` is some constant and `x` is the number of days. To make sure that we select a reasonable `base`, we need to do some manual testing, starting with looking at how old the oldest observation is.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ndifftime(\"2016-01-01\", min(Chicago$date))\n#> Time difference of 5457 days\n```\n:::\n\n\nUsing this information we can visualize the weight curve, to see if we like the value of `base`.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ntibble_days <- tibble(days = 0:5457)\n\ntibble_days |>\n ggplot(aes(days)) +\n geom_function(fun = ~ 0.99 ^ .x)\n```\n\n::: {.cell-output-display}\n![](figs/plot-single-decay-curve-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nsetting `base` to 0.99 appears to be down weighted too much. Any observation more than a year old would have no influence.\n\nLet us try a few more values to find \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nmap_dfr(\n c(0.99, 0.999, 0.9999),\n ~ tibble_days |> mutate(base = factor(.x), value = .x ^ days)\n) |>\n ggplot(aes(days, value, group = base, color = base)) +\n geom_line()\n```\n\n::: {.cell-output-display}\n![](figs/plot-compare-decay-bases-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nFrom this, we could pick something around 0.999 since it gives a better balance. Let's create a small function to help us encode this weight. \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nweights_from_dates <- function(x, ref) {\n if_else(\n condition = x >= ref,\n true = 1, # <- Notice that I'm setting any future weight to 1.\n false = 0.999 ^ as.numeric(difftime(ref, x, units = \"days\"))\n )\n}\n```\n:::\n\n\nWe then modify `Chicago` to add a weight column, explicitly making it an importance weight with `importance_weight()`.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nChicago <- Chicago |>\n mutate(weight = weights_from_dates(date, \"2016-01-01\"),\n weight = importance_weights(weight))\n```\n:::\n\n\nThis approach to creating importance weights from dates is not limited to cases where we have daily observations. You are free to create similar weights if you have gaps or repeated observations within the same day. Likewise, you don't need to use days as the unit. Seconds, weeks, or years could be used as well.\n\n## Modeling\n\nWe start by splitting up our data into a training and testing set based on the day `\"2016-01-01\"`. We added weights to the data set before splitting it so each set has weights.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nChicago_train <- Chicago |> filter(date < \"2016-01-01\")\nChicago_test <- Chicago |> filter(date >= \"2016-01-01\")\n```\n:::\n\n\nNext, we are going to create a recipe. The weights won't have any influence on the preprocessing since none of these operations are supervised and we are using importance weights.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nbase_recipe <-\n recipe(ridership ~ ., data = Chicago_train) |>\n # Create date features\n step_date(date) |>\n step_holiday(date, keep_original_cols = FALSE) |>\n # Remove any columns with a single unique value\n step_zv(all_predictors()) |>\n # Normalize all the numerical features\n step_normalize(all_numeric_predictors()) |>\n # Perform PCA to reduce the correlation bet the stations\n step_pca(all_numeric_predictors(), threshold = 0.95)\n```\n:::\n\n\nNext we need to build the rest of the workflow. We use a linear regression specification\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlm_spec <-\n linear_reg() |>\n set_engine(\"lm\")\n```\n:::\n\n\nand we add these together in the workflow. To activate the case weights, we use the `add_case_weights()` function to specify the name of the case weights being used.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlm_wflow <-\n workflow() |> \n add_case_weights(weight) |>\n add_recipe(base_recipe) |>\n add_model(lm_spec)\n\nlm_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Recipe\n#> Model: linear_reg()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> 5 Recipe Steps\n#> \n#> • step_date()\n#> • step_holiday()\n#> • step_zv()\n#> • step_normalize()\n#> • step_pca()\n#> \n#> ── Case Weights ──────────────────────────────────────────────────────\n#> weight\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Linear Regression Model Specification (regression)\n#> \n#> Computational engine: lm\n```\n:::\n\n\nWith all that done we can fit the workflow with the usual syntax: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlm_fit <- fit(lm_wflow, data = Chicago_train)\nlm_fit\n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Recipe\n#> Model: linear_reg()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> 5 Recipe Steps\n#> \n#> • step_date()\n#> • step_holiday()\n#> • step_zv()\n#> • step_normalize()\n#> • step_pca()\n#> \n#> ── Case Weights ──────────────────────────────────────────────────────\n#> weight\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> \n#> Call:\n#> stats::lm(formula = ..y ~ ., data = data, weights = weights)\n#> \n#> Coefficients:\n#> (Intercept) date_dowMon date_dowTue date_dowWed date_dowThu \n#> 1.762599 13.307654 14.689027 14.620178 14.382313 \n#> date_dowFri date_dowSat date_monthFeb date_monthMar date_monthApr \n#> 13.695433 1.228233 0.364342 1.348229 1.409897 \n#> date_monthMay date_monthJun date_monthJul date_monthAug date_monthSep \n#> 1.188189 2.598296 2.219721 2.406998 1.932061 \n#> date_monthOct date_monthNov date_monthDec PC1 PC2 \n#> 2.655552 0.909007 -0.004751 0.073014 -1.591021 \n#> PC3 PC4 PC5 \n#> -0.608386 -0.205305 0.696010\n```\n:::\n\n\n## Session information {#session-info}\n\n\n::: {.cell layout-align=\"center\"}\n\n```\n#> ─ Session info ─────────────────────────────────────────────────────\n#> version R version 4.6.0 (2026-04-24)\n#> language (EN)\n#> pandoc 3.1.3\n#> quarto 1.9.37\n#> \n#> ─ Packages ─────────────────────────────────────────────────────────\n#> package version date (UTC)\n#> broom 1.0.13 2026-05-14\n#> dials 1.4.3 2026-04-11\n#> dplyr 1.2.1 2026-04-03\n#> ggplot2 4.0.3 2026-04-22\n#> infer 1.1.0 2025-12-18\n#> parsnip 1.6.0 2026-05-14\n#> purrr 1.2.2 2026-04-10\n#> recipes 1.3.2 2026-04-02\n#> rlang 1.2.0 2026-04-06\n#> rsample 1.3.2 2026-01-30\n#> tibble 3.3.1 2026-01-11\n#> tidymodels 1.5.0 2026-04-23\n#> tune 2.1.0 2026-04-17\n#> workflows 1.3.0 2025-08-27\n#> yardstick 1.4.0 2026-04-07\n#> \n#> ────────────────────────────────────────────────────────────────────\n```\n:::\n\n", + "markdown": "---\ntitle: \"Creating case weights based on time\"\ncategories:\n - tuning and workflows\n - model fitting\n - time series\ntype: learn-subsection\nweight: 5\ndescription: | \n Create models that use case weights, extract them from fitted models, and visualize them.\ntoc: true\ntoc-depth: 2\nr-packages:\n - tidymodels\ninclude-after-body: ../../../html/resources.html\n---\n\n\n \n\n\n## Introduction\n\nTo use code in this article, you will need to install the following packages: tidymodels.\n\nThis article demonstrates how to create and use importance weights in a predictive model. Using importance weights is a way to have our model care more about some observations than others.\n\n## Example Data\n\nTo demonstrate we will use the Chicago data from the modeldata package.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(tidymodels)\ndata(Chicago)\n\nChicago <- Chicago |>\n select(ridership, date, one_of(stations))\n```\n:::\n\n\nFrom `?Chicago`\n\n> These data are from Kuhn and Johnson (2020) and contain an abbreviated training set for modeling the number of people (in thousands) who enter the Clark and Lake L station.\n\n> The date column corresponds to the current date. The columns with station names (Austin through California) are a sample of the columns used in the original analysis (for filesize reasons). These are 14 day lag variables (i.e. date - 14 days). There are columns related to weather and sports team schedules.\n\nFor simplicity, we have limited our view to the date and station variables.\n\n## Creating weights\n\nThis data set contains daily information from 2001-01-22 to 2016-08-28. We will pretend that it is January 1st, 2016 and we want to predict the ridership for the remainder of 2016 using the date and station variables as predictors. Without any weighting, all the previous observations would have the same influence on the model. This may not be ideal since some observations appear a long time ago and not be as representative of the future as more recent observations. \n\nWe could just use recent observations to fit the model, ensuring that the training data stays as close to the testing data as possible. While a tempting idea, it would throw out a lot of informative data. Instead let us assign a weight to each observation, related to how long ago the observation was taken. This way we are not completely throwing away any observation; we are only giving less weight to data farther in the past. \n\nWe need to decide on a way to calculate the case weights. The main thing constraint is that the weight cannot be negative, and it would be nice if today was weighted as 1. So we need a function that is 1 when `x = 0` and decreasing otherwise. There are many kinds of functions like that, and we will be using this exponential decay function\n\n$$ weight = base ^ x $$\n\nwhere `base` is some constant and `x` is the number of days. To make sure that we select a reasonable `base`, we need to do some manual testing, starting with looking at how old the oldest observation is.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ndifftime(\"2016-01-01\", min(Chicago$date))\n#> Time difference of 5457 days\n```\n:::\n\n\nUsing this information we can visualize the weight curve, to see if we like the value of `base`.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ntibble_days <- tibble(days = 0:5457)\n\ntibble_days |>\n ggplot(aes(days)) +\n geom_function(fun = ~ 0.99 ^ .x)\n```\n\n::: {.cell-output-display}\n![](figs/plot-single-decay-curve-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nsetting `base` to 0.99 appears to be down weighted too much. Any observation more than a year old would have no influence.\n\nLet us try a few more values to find \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nmap_dfr(\n c(0.99, 0.999, 0.9999),\n ~ tibble_days |> mutate(base = factor(.x), value = .x ^ days)\n) |>\n ggplot(aes(days, value, group = base, color = base)) +\n geom_line()\n```\n\n::: {.cell-output-display}\n![](figs/plot-compare-decay-bases-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nFrom this, we could pick something around 0.999 since it gives a better balance. Let's create a small function to help us encode this weight. \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nweights_from_dates <- function(x, ref) {\n if_else(\n condition = x >= ref,\n true = 1, # <- Notice that I'm setting any future weight to 1.\n false = 0.999 ^ as.numeric(difftime(ref, x, units = \"days\"))\n )\n}\n```\n:::\n\n\nWe then modify `Chicago` to add a weight column, explicitly making it an importance weight with `importance_weight()`.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nChicago <- Chicago |>\n mutate(weight = weights_from_dates(date, \"2016-01-01\"),\n weight = importance_weights(weight))\n```\n:::\n\n\nThis approach to creating importance weights from dates is not limited to cases where we have daily observations. You are free to create similar weights if you have gaps or repeated observations within the same day. Likewise, you don't need to use days as the unit. Seconds, weeks, or years could be used as well.\n\n## Modeling\n\nWe start by splitting up our data into a training and testing set based on the day `\"2016-01-01\"`. We added weights to the data set before splitting it so each set has weights.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nChicago_train <- Chicago |> filter(date < \"2016-01-01\")\nChicago_test <- Chicago |> filter(date >= \"2016-01-01\")\n```\n:::\n\n\nNext, we are going to create a recipe. The weights won't have any influence on the preprocessing since none of these operations are supervised and we are using importance weights.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nbase_recipe <-\n recipe(ridership ~ ., data = Chicago_train) |>\n # Create date features\n step_date(date) |>\n step_holiday(date, keep_original_cols = FALSE) |>\n # Remove any columns with a single unique value\n step_zv(all_predictors()) |>\n # Normalize all the numerical features\n step_normalize(all_numeric_predictors()) |>\n # Perform PCA to reduce the correlation bet the stations\n step_pca(all_numeric_predictors(), threshold = 0.95)\n```\n:::\n\n\nNext we need to build the rest of the workflow. We use a linear regression specification\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlm_spec <-\n linear_reg() |>\n set_engine(\"lm\")\n```\n:::\n\n\nand we add these together in the workflow. To activate the case weights, we use the `add_case_weights()` function to specify the name of the case weights being used.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlm_wflow <-\n workflow() |> \n add_case_weights(weight) |>\n add_recipe(base_recipe) |>\n add_model(lm_spec)\n\nlm_wflow\n#> ══ Workflow ══════════════════════════════════════════════════════════\n#> Preprocessor: Recipe\n#> Model: linear_reg()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> 5 Recipe Steps\n#> \n#> • step_date()\n#> • step_holiday()\n#> • step_zv()\n#> • step_normalize()\n#> • step_pca()\n#> \n#> ── Case Weights ──────────────────────────────────────────────────────\n#> weight\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> Linear Regression Model Specification (regression)\n#> \n#> Computational engine: lm\n```\n:::\n\n\nWith all that done we can fit the workflow with the usual syntax: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlm_fit <- fit(lm_wflow, data = Chicago_train)\nlm_fit\n#> ══ Workflow [trained] ════════════════════════════════════════════════\n#> Preprocessor: Recipe\n#> Model: linear_reg()\n#> \n#> ── Preprocessor ──────────────────────────────────────────────────────\n#> 5 Recipe Steps\n#> \n#> • step_date()\n#> • step_holiday()\n#> • step_zv()\n#> • step_normalize()\n#> • step_pca()\n#> \n#> ── Case Weights ──────────────────────────────────────────────────────\n#> weight\n#> \n#> ── Model ─────────────────────────────────────────────────────────────\n#> \n#> Call:\n#> stats::lm(formula = ..y ~ ., data = data, weights = weights)\n#> \n#> Coefficients:\n#> (Intercept) date_dowMon date_dowTue date_dowWed date_dowThu \n#> 1.762599 13.307654 14.689027 14.620178 14.382313 \n#> date_dowFri date_dowSat date_monthFeb date_monthMar date_monthApr \n#> 13.695433 1.228233 0.364342 1.348229 1.409897 \n#> date_monthMay date_monthJun date_monthJul date_monthAug date_monthSep \n#> 1.188189 2.598296 2.219721 2.406998 1.932061 \n#> date_monthOct date_monthNov date_monthDec PC1 PC2 \n#> 2.655552 0.909007 -0.004751 0.073014 -1.591021 \n#> PC3 PC4 PC5 \n#> -0.608386 -0.205305 0.696010\n```\n:::\n\n\n## Session information {#session-info}\n\n\n::: {.cell layout-align=\"center\"}\n\n```\n#> ─ Session info ─────────────────────────────────────────────────────\n#> version R version 4.6.0 (2026-04-24)\n#> language (EN)\n#> pandoc 3.1.3\n#> quarto 1.9.37\n#> \n#> ─ Packages ─────────────────────────────────────────────────────────\n#> package version date (UTC)\n#> broom 1.0.13 2026-05-14\n#> dials 1.4.3 2026-04-11\n#> dplyr 1.2.1 2026-04-03\n#> ggplot2 4.0.3 2026-04-22\n#> infer 1.1.0 2025-12-18\n#> parsnip 1.6.0 2026-05-14\n#> purrr 1.2.2 2026-04-10\n#> recipes 1.3.2 2026-04-02\n#> rlang 1.2.0 2026-04-06\n#> rsample 1.3.2 2026-01-30\n#> tibble 3.3.1 2026-01-11\n#> tidymodels 1.5.0 2026-04-23\n#> tune 2.1.0 2026-04-17\n#> workflows 1.3.0 2025-08-27\n#> yardstick 1.4.0 2026-04-07\n#> \n#> ────────────────────────────────────────────────────────────────────\n```\n:::\n\n", "supporting": [], "filters": [ "rmarkdown/pagebreak.lua" diff --git a/_freeze/learn/work/fairness-detectors/index/execute-results/html.json b/_freeze/learn/work/fairness-detectors/index/execute-results/html.json index bf1b7847..cc12be99 100644 --- a/_freeze/learn/work/fairness-detectors/index/execute-results/html.json +++ b/_freeze/learn/work/fairness-detectors/index/execute-results/html.json @@ -1,8 +1,8 @@ { - "hash": "c06e32a8ad17de5c34c383f903190259", + "hash": "70bfc4170bc3e2fccdede87395e2fc68", "result": { "engine": "knitr", - "markdown": "---\ntitle: \"Are GPT detectors fair? A machine learning fairness case study\"\ncategories:\n - yardstick\n - fairness\n - classification\ntype: learn-subsection\nweight: 6\ndescription: |\n Releases of language models are often soon followed up with detectors, claiming to detect whether text was written by a human or a given language model. A 2023 study argues that GPT detectors disproportionately classify real writing from non-native English writers as AI-generated.\ntoc: true\ntoc-depth: 2\nbibliography: refs.bib\nr-packages:\n - tidymodels\n - detectors\ninclude-after-body: ../../../html/resources.html\n---\n\n------------------------------------------------------------------------\n\n\n\n\n\n\nThe popularity and capability of large language models like [Generative Pre-trained Transformer 4](https://openai.com/research/gpt-4) (GPT-4), which generate human-like text based on simple prompts, has rapidly increased in recent years. The releases of more powerful next-generation models are often accompanied by the development of GPT *detectors*, which are machine learning models intended to estimate the probability that inputted text was generated by a GPT.\n\nIn 2023, a group of researchers collected a corpus of essays, some of which were written by real humans and others that were generated using various language models. Notably, some of the essays from real humans were written by people who did not write in English natively, while others were written by people who did.\n\n::: callout-note\nWhat does it mean to be a \"native writer\" of a language? Keep an eye out for this idea later on.\n:::\n\nThe researchers passed each essay to a number of marketed GPT detectors, recording the predicted probabilities that each essay was written by a human or GPT model. Juxtaposing detector predictions for papers written by native and non-native English writers, @liang2023 argued in their paper [*GPT detectors are biased against non-native English writers*](https://doi.org/10.1016/j.patter.2023.100779) that GPT detectors disproportionately classify real writing from non-native English writers as AI-generated. Some news media channels that cite the study use it as evidence that GPT detector models behave unfairly. The statements \n\n1. GPT detectors are biased against non-native English writers, \n2. GPT detectors disproportionately classify real writing from non-native English writers as AI-generated, and \n3. GPT detector models behave unfairly\n\nare only equivalent, though, given a shared set of social values surrounding what *fairness* means.\n\nPerhaps the core question that machine learning fairness as a research field has tried to address is exactly what a machine learning model acting fairly entails. As a recent primer notes, \"\\[t\\]he rapid growth of this new field has led to wildly inconsistent motivations, terminology, and notation, presenting a serious challenge for cataloging and comparing definitions\" [@mitchell2021]. Fairness is a social construct, and means different things to different people; the operationalization of fairness as a quantitative metric is not at all trivial. The aim of evaluating a machine learning system's fairness is thus to state clearly one's definition of fairness, critique what that definition means in the problem context, and assess fairness under the chosen definition rigorously. The tidymodels packages in R provide a suite of functionality to support this kind of analysis, and we'll use the data from Liang et al. as an example to demonstrate a fairness-oriented analysis of model performance.\n\nAfter loading needed packages, we'll conduct an exploratory analysis of the study's data. Once we have a sense for the distributions of relevant variables, we'll explore the different ways that we could try to measure fairness in this problem context. We will see that different ways of measuring fairness encode social values held by different stakeholders.\n\n# Getting set up\n\nLoading the tidymodels meta-package will load all of the functionality we need:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(tidymodels)\n```\n:::\n\n\nA tidied version of the study's data is available in the detectors R package. Loading the package and taking a look at the data:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(detectors)\n\nstr(detectors)\n#> tibble [6,185 × 9] (S3: tbl_df/tbl/data.frame)\n#> $ kind : Factor w/ 2 levels \"AI\",\"Human\": 2 2 2 1 1 2 1 1 2 2 ...\n#> $ .pred_AI : num [1:6185] 0.999994 0.828145 0.000214 0 0.001784 ...\n#> $ .pred_class: Factor w/ 2 levels \"AI\",\"Human\": 1 1 2 2 2 2 1 2 2 1 ...\n#> $ detector : chr [1:6185] \"Sapling\" \"Crossplag\" \"Crossplag\" \"ZeroGPT\" ...\n#> $ native : chr [1:6185] \"No\" \"No\" \"Yes\" NA ...\n#> $ name : chr [1:6185] \"Real TOEFL\" \"Real TOEFL\" \"Real College Essays\" \"Fake CS224N - GPT3\" ...\n#> $ model : chr [1:6185] \"Human\" \"Human\" \"Human\" \"GPT3\" ...\n#> $ document_id: num [1:6185] 497 278 294 671 717 855 533 484 781 460 ...\n#> $ prompt : chr [1:6185] NA NA NA \"Plain\" ...\n```\n:::\n\n\nEach row in this data corresponds to an essay (`document_id`) passed to a given GPT detector. For each row, we have the predicted probability `.pred_AI` from the `detector` giving whether the essay was written by AI. Notably, for the essays written by humans, we also have whether the author was a `native` English writer.\n\n# Exploratory Analysis\n\nThis analysis will focus on the `kind`, `.pred_AI`, `.pred_class`, and `native` variables.\n\nInitially, we can tabulate the observed and predicted classes:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ndetectors |>\n count(kind, .pred_class)\n#> # A tibble: 4 × 3\n#> kind .pred_class n\n#> \n#> 1 AI AI 1158\n#> 2 AI Human 2559\n#> 3 Human AI 449\n#> 4 Human Human 2019\n```\n:::\n\n\nA perfect detector would predict that all essays written by AI were written by AI, the converse for humans, and zeroes elsewhere. Perhaps these models would perform better with [calibration](https://probably.tidymodels.org/articles/where-to-use.html), but as-is, they have a long way to go. For example, of the 3717 essays written by AI in this data, 2559 were predicted to be written by humans. Of the 2468 essays written by humans in this data, 449 were predicted to be written by AI.\n\nAdding `native` into the tabulation:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ndetectors |> \n count(native, kind, .pred_class)\n#> # A tibble: 6 × 4\n#> native kind .pred_class n\n#> \n#> 1 No Human AI 390\n#> 2 No Human Human 247\n#> 3 Yes Human AI 59\n#> 4 Yes Human Human 1772\n#> 5 AI AI 1158\n#> 6 AI Human 2559\n```\n:::\n\n\nNote that, for essays written by AI (`kind == \"AI\"`), the `native` variable isn't well-defined. Those entries thus match the numbers from the above table.\n\nFor essays written by `Human`s, a plot perhaps better demonstrates the disparity:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ndetectors |> \n filter(!is.na(native)) |>\n ggplot(aes(x = native, fill = .pred_class)) +\n geom_bar() +\n labs(x = \"Native English Writer\", fill = \"Predicted Class\")\n```\n\n::: {.cell-output-display}\n![](figs/plot-human-preds-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nMost of the essays written by non-native English writers are incorrectly classified as written by AI, while nearly all of the essays written by native English writers are correctly classified as written by humans. The same effect can be seen in the underlying probability distributions:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ndetectors |>\n filter(!is.na(native)) |>\n mutate(native = case_when(native == \"Yes\" ~ \"Native English writer\",\n native == \"No\" ~ \"Non-native English writer\")) |>\n ggplot(aes(.pred_AI, fill = native)) +\n geom_histogram(bins = 30, show.legend = FALSE) +\n facet_wrap(vars(native), scales = \"free_y\", nrow = 2) +\n labs(x = \"Predicted Probability That Essay Was Written by AI\")\n```\n\n::: {.cell-output-display}\n![](figs/plot-human-preds-by-native-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nAgain, note that *all of the plotted essays were written by humans*. An effective detector would thus predict a probability near zero for all of these observations. In this plot, we see that the evidence in our initial table showing these detectors weren't performing well didn't tell the whole story. They perform *quite* well for native English writers, actually. For non-native English writers, though, they perform terribly.\n\nThe above plots aggregate observations across several `detectors`, though. Do some GPT detectors classify essays written by non-native English writers just as well as those from native English writers? We can recreate the above plot to examine this question by faceting across `detectors`.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ndetectors |>\n filter(!is.na(native)) |>\n mutate(native = case_when(native == \"Yes\" ~ \"Native English writer\",\n native == \"No\" ~ \"Non-native English writer\")) |>\n ggplot(aes(.pred_AI, fill = native)) +\n geom_histogram(bins = 30) +\n facet_grid(vars(native), vars(detector), scales = \"free_y\") +\n labs(x = \"Predicted Probability That Essay Was Written by AI\") +\n scale_x_continuous(breaks = c(0, 1)) +\n theme(legend.position = \"none\")\n```\n\n::: {.cell-output-display}\n![](figs/plot-human-preds-by-native-detector-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nEach column in this plot reflects roughly the same story as the plot that aggregates across detectors; the detectors work *very* well at correctly classifying real writing from native English writers, yet classify writing from non-native English writers incorrectly at least as often as they do so correctly.\n\n::: callout-note\n## Question for reflection\nExplore the source data further. Where are the essays written by native English writers collected from? How about non-native? What does our usage of \"native writer\" mean in this context, then?\n:::\n\n# Fairness assessment\n\nThe [1.3.0 release of yardstick](https://yardstick.tidymodels.org/news/index.html#yardstick-130) introduced a suite of functionality for fairness-oriented analysis of machine learning models. The release includes a tool to make fairness metrics, `new_groupwise_metric()`, as well as three outputs of that tool giving canonical fairness metrics: `equal_opportunity()`, `equalized_odds()`, and `demographic_parity()`. To better understand how to use metrics to quantify model fairness, we'll consider what a fairness assessment of this model would look like from the perspective of three different stakeholders.\n\n## Effective detection, group-blind\n\nImagine, first, the position that the most fair detection model is one that reliably differentiates between essays written by humans or generated by GPTs, regardless of the problem context. From this perspective, the most fair model is the model that detects GPT-generated essays most effectively; it is unfair to pass on an essay written by a GPT as one's own work. When analyzing this data, a stakeholder with this perspective would ignore the `native` variable in their analysis.\n\n- A **detector author** may take on such a perspective, since their model may be applied in a diverse set of unknown contexts.\n\n- A **student** who submits an essay of their own writing may feel it is unfair to have their work compared to work generated by GPTs.\n\n- An **instructor** of a course tasked with evaluating essays may feel it is unfair to such students to compare those students' work to GPT-generated essays. Additionally, this instructor may teach a course to only native English writers or only non-native English writers.\n\nIn this case, we calculate a chosen performance metric for each detector, across all of the data, and then consider the most performant detectors:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ndetectors |>\n group_by(detector) |>\n roc_auc(truth = kind, .pred_AI) |>\n arrange(desc(.estimate)) |>\n head(3)\n#> # A tibble: 3 × 4\n#> detector .metric .estimator .estimate\n#> \n#> 1 GPTZero roc_auc binary 0.750\n#> 2 OriginalityAI roc_auc binary 0.682\n#> 3 HFOpenAI roc_auc binary 0.614\n```\n:::\n\n\n::: callout-tip\nTo learn more about how the yardstick package handles groups in data, see the [\"Grouping behavior in yardstick\" vignette](https://yardstick.tidymodels.org/articles/grouping.html).\n:::\n\nFrom this perspective, the models with the highest `roc_auc()` estimates are the most fair.\n\nThe fairness assessment for this stakeholder doesn't need any of the functionality newly introduced in yardstick 1.3.0. An analysis that reconciles the role of the `native` variable in these models' predictions will, though.\n\n## Fair prediction on human-written essays\n\nNow, consider that our only priority was to correctly classify human-written text as human-written and incorrectly classify human-written text as generated by AI at the same rate for both native English writers and non-native English writers, ignoring predictions from essays generated by AI.\n\n- Another **student** whose work is evaluated by a detector model may take on such a perspective. This student could be a native English writer who does not want other students to be subjected to undue harm or a non-native English writer concerned with their own writing being incorrectly classified as GPT-generated.\n\n- Another **instructor** of a course may feel it is unfair to disproportionately classify writing from students who are non-native English writers as GPT-generated, regardless of how effectively the model detects GPT-generated essays.\n\nThe `equal_opportunity()` metric enables us to quantify the extent of this interpretation of unfairness. Equal opportunity is satisfied when a model's predictions have the same true positive and false negative rates across protected groups; a model predicts more fairly if it's equally likely to predict a positive outcome for each group.\n\nIn this example, a GPT detector satisfies equal opportunity when the detector correctly classifies human-written text as human-written and incorrectly classifies human-written text as generated by AI at the same rate for both native English writers and non-native English writers.\n\n::: callout-note\nThis definition does not consider the predictions based on essays generated by AI.\n:::\n\nTo calculate equal opportunity with yardstick, we use `equal_opportunity()` to create a metric function:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nequal_opportunity_by_native <- equal_opportunity(by = native)\n\nequal_opportunity_by_native\n#> A class metric | direction: minimize, group-wise on: native\n```\n:::\n\n\nThe function `equal_opportunity_by_native()` is a yardstick metric function like any other, except it knows to temporarily group by and summarize across a data-column called `native`. Applying it:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ndetectors |>\n filter(kind == \"Human\") |>\n group_by(detector) |>\n equal_opportunity_by_native(\n truth = kind, \n estimate = .pred_class, \n event_level = \"second\"\n ) |>\n arrange(.estimate) |>\n head(3)\n#> # A tibble: 3 × 5\n#> detector .metric .by .estimator .estimate\n#> \n#> 1 Crossplag equal_opportunity native binary 0.464\n#> 2 ZeroGPT equal_opportunity native binary 0.477\n#> 3 GPTZero equal_opportunity native binary 0.510\n```\n:::\n\n\nThe detectors with estimates closest to zero are most fair, by this definition of fairness.\n\n::: callout-important\nWhy do we need to specify `event_level = \"second\"`? See the \"Relevant level\" section in [yardstick's documentation](https://yardstick.tidymodels.org/reference/sens.html#relevant-level) to learn more.\n:::\n\nGiven this set of moral values, our analysis would offer a different set of recommendations for which detector to use.\n\n## Balancing two notions of fairness\n\nInstead of either of the options we have considered so far, imagine that we both feel that it is unfair for GPT-generated work to go undetected *and* believe that it's unfair for a GPT detector to disproportionately classify human-written work from non-native English writers as GPT-generated.\n\n- Another **instructor** of a course may feel it is unfair to disproportionately classify human-written work from non-native English writers as GPT-generated, but still values detection of GPT-generated content.\n\nIn this case, we could *first* ensure that a model detects GPT-generated work with some threshold of performance, and then choose the model among that set that predicts most fairly on human-written essays. This reflects the belief that it is more unfair to fail to detect GPT-generated work than it is to disproportionately classify human-written work from non-native English writers as GPT-generated, as it is possible that the model that most proportionately classifies human-written work from native and non-native English writers as GPT-generated is not among the recommended models.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nperformant_detectors <- \n detectors |>\n group_by(detector) |>\n roc_auc(truth = kind, .pred_AI) |>\n arrange(desc(.estimate)) |>\n head(3)\n\ndetectors |>\n filter(kind == \"Human\", detector %in% performant_detectors$detector) |>\n group_by(detector) |>\n equal_opportunity_by_native(\n truth = kind, \n estimate = .pred_class, \n event_level = \"second\"\n ) |>\n arrange(.estimate)\n#> # A tibble: 3 × 5\n#> detector .metric .by .estimator .estimate\n#> \n#> 1 GPTZero equal_opportunity native binary 0.510\n#> 2 HFOpenAI equal_opportunity native binary 0.549\n#> 3 OriginalityAI equal_opportunity native binary 0.709\n```\n:::\n\n\nWe could also reverse the process, reflecting the belief that it is more unfair to disproportionately classify human-written work from non-native English writers as GPT-generated than it is to pass on output from GPTs as one's own work. We first set a threshold based on `equal_opportunity()`, then choose the most performant model by `roc_auc()`:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nequal_opportunity_detectors <- \n detectors |>\n filter(kind == \"Human\") |>\n group_by(detector) |>\n equal_opportunity_by_native(\n truth = kind, \n estimate = .pred_class, \n event_level = \"second\"\n ) |>\n arrange(.estimate) |>\n head(3)\n\ndetectors |>\n filter(detector %in% equal_opportunity_detectors$detector) |>\n group_by(detector) |>\n roc_auc(truth = kind, .pred_AI) |>\n arrange(desc(.estimate))\n#> # A tibble: 3 × 4\n#> detector .metric .estimator .estimate\n#> \n#> 1 GPTZero roc_auc binary 0.750\n#> 2 Crossplag roc_auc binary 0.613\n#> 3 ZeroGPT roc_auc binary 0.603\n```\n:::\n\n\nIn this example, changing the prioritization of the criteria results in a different set of recommended models.\n\n# Choosing a detector\n\nIn the preceding section, we saw that the recommended detector we identify depends on our moral values. That is, the mathematical notion of fairness appropriate for a given analysis follows from the problem context. In this way, no statistical model is objectively more fair than another; our assessment of fairness depends on our personally held ideas of fairness.\n\n\n\nAs for the problem of GPT detection, while each stakeholder might find that some models are more fair than others, even the most fair models recommended in each approach are quite unfair. For instance, from the first stakeholder's perspective, even though it's the most performant model available, GPTZero's `roc_auc()` of 0.75 leaves much to be desired; a stakeholder ought to consider the potential harms resulting from the substantial number of errors made by this model when applied in context.\n\nThis analysis only considered one fairness metric, `equal_opportunity()`. We could have attempted to apply either of the other two fairness metrics included in yardstick 1.3.0, `equalized_odds()` and `demographic_parity()`, or a custom fairness metric. Are those two other metrics well-defined for this problem? Which stakeholders' interests are best represented by those metrics? Would they result in yet another set of discordant recommendations?\n\nWe also did not consider how the outputs of a chosen model be used. If a student's work is classified as written by a GPT model, what happens then? Would a misclassification be more harmful for one type of student than another? Could an instructor trust model output more readily for one type of student than another? Answers to these questions are a necessary component of a complete fairness analysis and, just like the choice of metric, depend heavily on the problem context.\n\nIn all, we've seen that applied fairness analysis is as much a social problem as it is a technical one. While we absolutely ought to strive to minimize harm in development and deployment of machine learning models, the fact that fairness is a moral concept, rather than a mathematical one, means that algorithmic unfairness cannot be automated away.\n\n# References\n\n::: {#refs}\n:::\n\n## Session information {#session-info}\n\n\n::: {.cell layout-align=\"center\"}\n\n```\n#> ─ Session info ─────────────────────────────────────────────────────\n#> version R version 4.6.0 (2026-04-24)\n#> language (EN)\n#> pandoc 3.1.3\n#> quarto 1.9.37\n#> \n#> ─ Packages ─────────────────────────────────────────────────────────\n#> package version date (UTC)\n#> broom 1.0.13 2026-05-14\n#> detectors 0.1.0 2023-10-26\n#> dials 1.4.3 2026-04-11\n#> dplyr 1.2.1 2026-04-03\n#> ggplot2 4.0.3 2026-04-22\n#> infer 1.1.0 2025-12-18\n#> parsnip 1.6.0 2026-05-14\n#> purrr 1.2.2 2026-04-10\n#> recipes 1.3.2 2026-04-02\n#> rlang 1.2.0 2026-04-06\n#> rsample 1.3.2 2026-01-30\n#> tibble 3.3.1 2026-01-11\n#> tidymodels 1.5.0 2026-04-23\n#> tune 2.1.0 2026-04-17\n#> workflows 1.3.0 2025-08-27\n#> yardstick 1.4.0 2026-04-07\n#> \n#> ────────────────────────────────────────────────────────────────────\n```\n:::\n\n \n", + "markdown": "---\ntitle: \"Are GPT detectors fair? A machine learning fairness case study\"\ncategories:\n - tuning and workflows\n - yardstick\n - fairness\n - classification\ntype: learn-subsection\nweight: 6\ndescription: |\n Releases of language models are often soon followed up with detectors, claiming to detect whether text was written by a human or a given language model. A 2023 study argues that GPT detectors disproportionately classify real writing from non-native English writers as AI-generated.\ntoc: true\ntoc-depth: 2\nbibliography: refs.bib\nr-packages:\n - tidymodels\n - detectors\ninclude-after-body: ../../../html/resources.html\n---\n\n------------------------------------------------------------------------\n\n\n\n\n\n\nThe popularity and capability of large language models like [Generative Pre-trained Transformer 4](https://openai.com/research/gpt-4) (GPT-4), which generate human-like text based on simple prompts, has rapidly increased in recent years. The releases of more powerful next-generation models are often accompanied by the development of GPT *detectors*, which are machine learning models intended to estimate the probability that inputted text was generated by a GPT.\n\nIn 2023, a group of researchers collected a corpus of essays, some of which were written by real humans and others that were generated using various language models. Notably, some of the essays from real humans were written by people who did not write in English natively, while others were written by people who did.\n\n::: callout-note\nWhat does it mean to be a \"native writer\" of a language? Keep an eye out for this idea later on.\n:::\n\nThe researchers passed each essay to a number of marketed GPT detectors, recording the predicted probabilities that each essay was written by a human or GPT model. Juxtaposing detector predictions for papers written by native and non-native English writers, @liang2023 argued in their paper [*GPT detectors are biased against non-native English writers*](https://doi.org/10.1016/j.patter.2023.100779) that GPT detectors disproportionately classify real writing from non-native English writers as AI-generated. Some news media channels that cite the study use it as evidence that GPT detector models behave unfairly. The statements \n\n1. GPT detectors are biased against non-native English writers, \n2. GPT detectors disproportionately classify real writing from non-native English writers as AI-generated, and \n3. GPT detector models behave unfairly\n\nare only equivalent, though, given a shared set of social values surrounding what *fairness* means.\n\nPerhaps the core question that machine learning fairness as a research field has tried to address is exactly what a machine learning model acting fairly entails. As a recent primer notes, \"\\[t\\]he rapid growth of this new field has led to wildly inconsistent motivations, terminology, and notation, presenting a serious challenge for cataloging and comparing definitions\" [@mitchell2021]. Fairness is a social construct, and means different things to different people; the operationalization of fairness as a quantitative metric is not at all trivial. The aim of evaluating a machine learning system's fairness is thus to state clearly one's definition of fairness, critique what that definition means in the problem context, and assess fairness under the chosen definition rigorously. The tidymodels packages in R provide a suite of functionality to support this kind of analysis, and we'll use the data from Liang et al. as an example to demonstrate a fairness-oriented analysis of model performance.\n\nAfter loading needed packages, we'll conduct an exploratory analysis of the study's data. Once we have a sense for the distributions of relevant variables, we'll explore the different ways that we could try to measure fairness in this problem context. We will see that different ways of measuring fairness encode social values held by different stakeholders.\n\n# Getting set up\n\nLoading the tidymodels meta-package will load all of the functionality we need:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(tidymodels)\n```\n:::\n\n\nA tidied version of the study's data is available in the detectors R package. Loading the package and taking a look at the data:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(detectors)\n\nstr(detectors)\n#> tibble [6,185 × 9] (S3: tbl_df/tbl/data.frame)\n#> $ kind : Factor w/ 2 levels \"AI\",\"Human\": 2 2 2 1 1 2 1 1 2 2 ...\n#> $ .pred_AI : num [1:6185] 0.999994 0.828145 0.000214 0 0.001784 ...\n#> $ .pred_class: Factor w/ 2 levels \"AI\",\"Human\": 1 1 2 2 2 2 1 2 2 1 ...\n#> $ detector : chr [1:6185] \"Sapling\" \"Crossplag\" \"Crossplag\" \"ZeroGPT\" ...\n#> $ native : chr [1:6185] \"No\" \"No\" \"Yes\" NA ...\n#> $ name : chr [1:6185] \"Real TOEFL\" \"Real TOEFL\" \"Real College Essays\" \"Fake CS224N - GPT3\" ...\n#> $ model : chr [1:6185] \"Human\" \"Human\" \"Human\" \"GPT3\" ...\n#> $ document_id: num [1:6185] 497 278 294 671 717 855 533 484 781 460 ...\n#> $ prompt : chr [1:6185] NA NA NA \"Plain\" ...\n```\n:::\n\n\nEach row in this data corresponds to an essay (`document_id`) passed to a given GPT detector. For each row, we have the predicted probability `.pred_AI` from the `detector` giving whether the essay was written by AI. Notably, for the essays written by humans, we also have whether the author was a `native` English writer.\n\n# Exploratory Analysis\n\nThis analysis will focus on the `kind`, `.pred_AI`, `.pred_class`, and `native` variables.\n\nInitially, we can tabulate the observed and predicted classes:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ndetectors |>\n count(kind, .pred_class)\n#> # A tibble: 4 × 3\n#> kind .pred_class n\n#> \n#> 1 AI AI 1158\n#> 2 AI Human 2559\n#> 3 Human AI 449\n#> 4 Human Human 2019\n```\n:::\n\n\nA perfect detector would predict that all essays written by AI were written by AI, the converse for humans, and zeroes elsewhere. Perhaps these models would perform better with [calibration](https://probably.tidymodels.org/articles/where-to-use.html), but as-is, they have a long way to go. For example, of the 3717 essays written by AI in this data, 2559 were predicted to be written by humans. Of the 2468 essays written by humans in this data, 449 were predicted to be written by AI.\n\nAdding `native` into the tabulation:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ndetectors |> \n count(native, kind, .pred_class)\n#> # A tibble: 6 × 4\n#> native kind .pred_class n\n#> \n#> 1 No Human AI 390\n#> 2 No Human Human 247\n#> 3 Yes Human AI 59\n#> 4 Yes Human Human 1772\n#> 5 AI AI 1158\n#> 6 AI Human 2559\n```\n:::\n\n\nNote that, for essays written by AI (`kind == \"AI\"`), the `native` variable isn't well-defined. Those entries thus match the numbers from the above table.\n\nFor essays written by `Human`s, a plot perhaps better demonstrates the disparity:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ndetectors |> \n filter(!is.na(native)) |>\n ggplot(aes(x = native, fill = .pred_class)) +\n geom_bar() +\n labs(x = \"Native English Writer\", fill = \"Predicted Class\")\n```\n\n::: {.cell-output-display}\n![](figs/plot-human-preds-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nMost of the essays written by non-native English writers are incorrectly classified as written by AI, while nearly all of the essays written by native English writers are correctly classified as written by humans. The same effect can be seen in the underlying probability distributions:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ndetectors |>\n filter(!is.na(native)) |>\n mutate(native = case_when(native == \"Yes\" ~ \"Native English writer\",\n native == \"No\" ~ \"Non-native English writer\")) |>\n ggplot(aes(.pred_AI, fill = native)) +\n geom_histogram(bins = 30, show.legend = FALSE) +\n facet_wrap(vars(native), scales = \"free_y\", nrow = 2) +\n labs(x = \"Predicted Probability That Essay Was Written by AI\")\n```\n\n::: {.cell-output-display}\n![](figs/plot-human-preds-by-native-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nAgain, note that *all of the plotted essays were written by humans*. An effective detector would thus predict a probability near zero for all of these observations. In this plot, we see that the evidence in our initial table showing these detectors weren't performing well didn't tell the whole story. They perform *quite* well for native English writers, actually. For non-native English writers, though, they perform terribly.\n\nThe above plots aggregate observations across several `detectors`, though. Do some GPT detectors classify essays written by non-native English writers just as well as those from native English writers? We can recreate the above plot to examine this question by faceting across `detectors`.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ndetectors |>\n filter(!is.na(native)) |>\n mutate(native = case_when(native == \"Yes\" ~ \"Native English writer\",\n native == \"No\" ~ \"Non-native English writer\")) |>\n ggplot(aes(.pred_AI, fill = native)) +\n geom_histogram(bins = 30) +\n facet_grid(vars(native), vars(detector), scales = \"free_y\") +\n labs(x = \"Predicted Probability That Essay Was Written by AI\") +\n scale_x_continuous(breaks = c(0, 1)) +\n theme(legend.position = \"none\")\n```\n\n::: {.cell-output-display}\n![](figs/plot-human-preds-by-native-detector-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nEach column in this plot reflects roughly the same story as the plot that aggregates across detectors; the detectors work *very* well at correctly classifying real writing from native English writers, yet classify writing from non-native English writers incorrectly at least as often as they do so correctly.\n\n::: callout-note\n## Question for reflection\nExplore the source data further. Where are the essays written by native English writers collected from? How about non-native? What does our usage of \"native writer\" mean in this context, then?\n:::\n\n# Fairness assessment\n\nThe [1.3.0 release of yardstick](https://yardstick.tidymodels.org/news/index.html#yardstick-130) introduced a suite of functionality for fairness-oriented analysis of machine learning models. The release includes a tool to make fairness metrics, `new_groupwise_metric()`, as well as three outputs of that tool giving canonical fairness metrics: `equal_opportunity()`, `equalized_odds()`, and `demographic_parity()`. To better understand how to use metrics to quantify model fairness, we'll consider what a fairness assessment of this model would look like from the perspective of three different stakeholders.\n\n## Effective detection, group-blind\n\nImagine, first, the position that the most fair detection model is one that reliably differentiates between essays written by humans or generated by GPTs, regardless of the problem context. From this perspective, the most fair model is the model that detects GPT-generated essays most effectively; it is unfair to pass on an essay written by a GPT as one's own work. When analyzing this data, a stakeholder with this perspective would ignore the `native` variable in their analysis.\n\n- A **detector author** may take on such a perspective, since their model may be applied in a diverse set of unknown contexts.\n\n- A **student** who submits an essay of their own writing may feel it is unfair to have their work compared to work generated by GPTs.\n\n- An **instructor** of a course tasked with evaluating essays may feel it is unfair to such students to compare those students' work to GPT-generated essays. Additionally, this instructor may teach a course to only native English writers or only non-native English writers.\n\nIn this case, we calculate a chosen performance metric for each detector, across all of the data, and then consider the most performant detectors:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ndetectors |>\n group_by(detector) |>\n roc_auc(truth = kind, .pred_AI) |>\n arrange(desc(.estimate)) |>\n head(3)\n#> # A tibble: 3 × 4\n#> detector .metric .estimator .estimate\n#> \n#> 1 GPTZero roc_auc binary 0.750\n#> 2 OriginalityAI roc_auc binary 0.682\n#> 3 HFOpenAI roc_auc binary 0.614\n```\n:::\n\n\n::: callout-tip\nTo learn more about how the yardstick package handles groups in data, see the [\"Grouping behavior in yardstick\" vignette](https://yardstick.tidymodels.org/articles/grouping.html).\n:::\n\nFrom this perspective, the models with the highest `roc_auc()` estimates are the most fair.\n\nThe fairness assessment for this stakeholder doesn't need any of the functionality newly introduced in yardstick 1.3.0. An analysis that reconciles the role of the `native` variable in these models' predictions will, though.\n\n## Fair prediction on human-written essays\n\nNow, consider that our only priority was to correctly classify human-written text as human-written and incorrectly classify human-written text as generated by AI at the same rate for both native English writers and non-native English writers, ignoring predictions from essays generated by AI.\n\n- Another **student** whose work is evaluated by a detector model may take on such a perspective. This student could be a native English writer who does not want other students to be subjected to undue harm or a non-native English writer concerned with their own writing being incorrectly classified as GPT-generated.\n\n- Another **instructor** of a course may feel it is unfair to disproportionately classify writing from students who are non-native English writers as GPT-generated, regardless of how effectively the model detects GPT-generated essays.\n\nThe `equal_opportunity()` metric enables us to quantify the extent of this interpretation of unfairness. Equal opportunity is satisfied when a model's predictions have the same true positive and false negative rates across protected groups; a model predicts more fairly if it's equally likely to predict a positive outcome for each group.\n\nIn this example, a GPT detector satisfies equal opportunity when the detector correctly classifies human-written text as human-written and incorrectly classifies human-written text as generated by AI at the same rate for both native English writers and non-native English writers.\n\n::: callout-note\nThis definition does not consider the predictions based on essays generated by AI.\n:::\n\nTo calculate equal opportunity with yardstick, we use `equal_opportunity()` to create a metric function:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nequal_opportunity_by_native <- equal_opportunity(by = native)\n\nequal_opportunity_by_native\n#> A class metric | direction: minimize, group-wise on: native\n```\n:::\n\n\nThe function `equal_opportunity_by_native()` is a yardstick metric function like any other, except it knows to temporarily group by and summarize across a data-column called `native`. Applying it:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ndetectors |>\n filter(kind == \"Human\") |>\n group_by(detector) |>\n equal_opportunity_by_native(\n truth = kind, \n estimate = .pred_class, \n event_level = \"second\"\n ) |>\n arrange(.estimate) |>\n head(3)\n#> # A tibble: 3 × 5\n#> detector .metric .by .estimator .estimate\n#> \n#> 1 Crossplag equal_opportunity native binary 0.464\n#> 2 ZeroGPT equal_opportunity native binary 0.477\n#> 3 GPTZero equal_opportunity native binary 0.510\n```\n:::\n\n\nThe detectors with estimates closest to zero are most fair, by this definition of fairness.\n\n::: callout-important\nWhy do we need to specify `event_level = \"second\"`? See the \"Relevant level\" section in [yardstick's documentation](https://yardstick.tidymodels.org/reference/sens.html#relevant-level) to learn more.\n:::\n\nGiven this set of moral values, our analysis would offer a different set of recommendations for which detector to use.\n\n## Balancing two notions of fairness\n\nInstead of either of the options we have considered so far, imagine that we both feel that it is unfair for GPT-generated work to go undetected *and* believe that it's unfair for a GPT detector to disproportionately classify human-written work from non-native English writers as GPT-generated.\n\n- Another **instructor** of a course may feel it is unfair to disproportionately classify human-written work from non-native English writers as GPT-generated, but still values detection of GPT-generated content.\n\nIn this case, we could *first* ensure that a model detects GPT-generated work with some threshold of performance, and then choose the model among that set that predicts most fairly on human-written essays. This reflects the belief that it is more unfair to fail to detect GPT-generated work than it is to disproportionately classify human-written work from non-native English writers as GPT-generated, as it is possible that the model that most proportionately classifies human-written work from native and non-native English writers as GPT-generated is not among the recommended models.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nperformant_detectors <- \n detectors |>\n group_by(detector) |>\n roc_auc(truth = kind, .pred_AI) |>\n arrange(desc(.estimate)) |>\n head(3)\n\ndetectors |>\n filter(kind == \"Human\", detector %in% performant_detectors$detector) |>\n group_by(detector) |>\n equal_opportunity_by_native(\n truth = kind, \n estimate = .pred_class, \n event_level = \"second\"\n ) |>\n arrange(.estimate)\n#> # A tibble: 3 × 5\n#> detector .metric .by .estimator .estimate\n#> \n#> 1 GPTZero equal_opportunity native binary 0.510\n#> 2 HFOpenAI equal_opportunity native binary 0.549\n#> 3 OriginalityAI equal_opportunity native binary 0.709\n```\n:::\n\n\nWe could also reverse the process, reflecting the belief that it is more unfair to disproportionately classify human-written work from non-native English writers as GPT-generated than it is to pass on output from GPTs as one's own work. We first set a threshold based on `equal_opportunity()`, then choose the most performant model by `roc_auc()`:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nequal_opportunity_detectors <- \n detectors |>\n filter(kind == \"Human\") |>\n group_by(detector) |>\n equal_opportunity_by_native(\n truth = kind, \n estimate = .pred_class, \n event_level = \"second\"\n ) |>\n arrange(.estimate) |>\n head(3)\n\ndetectors |>\n filter(detector %in% equal_opportunity_detectors$detector) |>\n group_by(detector) |>\n roc_auc(truth = kind, .pred_AI) |>\n arrange(desc(.estimate))\n#> # A tibble: 3 × 4\n#> detector .metric .estimator .estimate\n#> \n#> 1 GPTZero roc_auc binary 0.750\n#> 2 Crossplag roc_auc binary 0.613\n#> 3 ZeroGPT roc_auc binary 0.603\n```\n:::\n\n\nIn this example, changing the prioritization of the criteria results in a different set of recommended models.\n\n# Choosing a detector\n\nIn the preceding section, we saw that the recommended detector we identify depends on our moral values. That is, the mathematical notion of fairness appropriate for a given analysis follows from the problem context. In this way, no statistical model is objectively more fair than another; our assessment of fairness depends on our personally held ideas of fairness.\n\n\n\nAs for the problem of GPT detection, while each stakeholder might find that some models are more fair than others, even the most fair models recommended in each approach are quite unfair. For instance, from the first stakeholder's perspective, even though it's the most performant model available, GPTZero's `roc_auc()` of 0.75 leaves much to be desired; a stakeholder ought to consider the potential harms resulting from the substantial number of errors made by this model when applied in context.\n\nThis analysis only considered one fairness metric, `equal_opportunity()`. We could have attempted to apply either of the other two fairness metrics included in yardstick 1.3.0, `equalized_odds()` and `demographic_parity()`, or a custom fairness metric. Are those two other metrics well-defined for this problem? Which stakeholders' interests are best represented by those metrics? Would they result in yet another set of discordant recommendations?\n\nWe also did not consider how the outputs of a chosen model be used. If a student's work is classified as written by a GPT model, what happens then? Would a misclassification be more harmful for one type of student than another? Could an instructor trust model output more readily for one type of student than another? Answers to these questions are a necessary component of a complete fairness analysis and, just like the choice of metric, depend heavily on the problem context.\n\nIn all, we've seen that applied fairness analysis is as much a social problem as it is a technical one. While we absolutely ought to strive to minimize harm in development and deployment of machine learning models, the fact that fairness is a moral concept, rather than a mathematical one, means that algorithmic unfairness cannot be automated away.\n\n# References\n\n::: {#refs}\n:::\n\n## Session information {#session-info}\n\n\n::: {.cell layout-align=\"center\"}\n\n```\n#> ─ Session info ─────────────────────────────────────────────────────\n#> version R version 4.6.0 (2026-04-24)\n#> language (EN)\n#> pandoc 3.1.3\n#> quarto 1.9.37\n#> \n#> ─ Packages ─────────────────────────────────────────────────────────\n#> package version date (UTC)\n#> broom 1.0.13 2026-05-14\n#> detectors 0.1.0 2023-10-26\n#> dials 1.4.3 2026-04-11\n#> dplyr 1.2.1 2026-04-03\n#> ggplot2 4.0.3 2026-04-22\n#> infer 1.1.0 2025-12-18\n#> parsnip 1.6.0 2026-05-14\n#> purrr 1.2.2 2026-04-10\n#> recipes 1.3.2 2026-04-02\n#> rlang 1.2.0 2026-04-06\n#> rsample 1.3.2 2026-01-30\n#> tibble 3.3.1 2026-01-11\n#> tidymodels 1.5.0 2026-04-23\n#> tune 2.1.0 2026-04-17\n#> workflows 1.3.0 2025-08-27\n#> yardstick 1.4.0 2026-04-07\n#> \n#> ────────────────────────────────────────────────────────────────────\n```\n:::\n\n \n", "supporting": [ "index_files" ], diff --git a/_freeze/learn/work/fairness-readmission/index/execute-results/html.json b/_freeze/learn/work/fairness-readmission/index/execute-results/html.json index 39ebd043..279b4422 100644 --- a/_freeze/learn/work/fairness-readmission/index/execute-results/html.json +++ b/_freeze/learn/work/fairness-readmission/index/execute-results/html.json @@ -1,8 +1,8 @@ { - "hash": "066f966d279a2845ac14903ab33698c1", + "hash": "0dcb267421e11ff8bb3d74838508657f", "result": { "engine": "knitr", - "markdown": "---\ntitle: \"Fair prediction of hospital readmission: a machine learning fairness case study\"\ncategories:\n - yardstick\n - fairness\n - tuning\n - classification\ntype: learn-subsection\nweight: 6\ndescription: |\n With information on a diabetes patient's stay at a hospital like demographics, diagnostic results, payment, and medications, a hospital can train a machine learning model to predict reasonably well whether a patient will be readmitted within 30 days. What harms to patients could result from using such a model, though?\ntoc: true\ntoc-depth: 2\nbibliography: refs.bib\nr-packages:\n - tidymodels\n - readmission\n - baguette\n - GGally\n - desirability2\ninclude-after-body: ../../../html/resources.html\n---\n\n------------------------------------------------------------------------\n\n\n\n\n\nIn 2019, @obermeyer2019 published an analysis of predictions from a machine learning model that health care providers use to allocate resources. The model's outputs are used to recommend a patient for _high-risk care management_ programs:\n\n> These programs seek to improve the care of patients with complex health needs by providing additional resources, including greater attention from trained providers, to help ensure that care is well coordinated. Most health systems use these programs as the cornerstone of population health management efforts, and they are widely considered effective at improving outcomes and satisfaction while reducing costs. [...] Because the programs are themselves expensive---with costs going toward teams of dedicated nurses, extra primary care appointment slots, and other scarce resources---**health systems rely extensively on algorithms to identify patients who will benefit the most.**\n\nThey argue in their analysis that the model in question exhibits substantial racial bias, where \"Black patients assigned the same level of risk by the algorithm are sicker than White patients.\" In practice, this resulted in the reduction of \"the number of Black patients identified for extra care by more than half.\"\n\nThis article will demonstrate a fairness-oriented workflow for training a machine learning model to identify high-risk patients. Throughout the model development process, we'll consider the social impacts that different modeling decisions may have when such a model is deployed in context.\n\n## Setup\n\nThe data we'll use in this analysis is a publicly available database containing information on 71,515 hospital stays from diabetes patients. The data comes from a study by @strack2014, where the authors model the effectiveness of a particular lab test in predicting readmission. A version of that data is available in the [readmission R package](https://simonpcouch.github.io/readmission/):\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(readmission)\n\nreadmission\n#> # A tibble: 71,515 × 12\n#> readmitted race sex age admission_source blood_glucose insurer duration\n#> \n#> 1 Yes Afric… Male [60-… Referral 7\n#> 2 No Cauca… Fema… [50-… Emergency Normal Private 4\n#> 3 Yes Cauca… Fema… [70-… Referral Medica… 5\n#> 4 No Cauca… Fema… [80-… Referral Private 5\n#> 5 No Cauca… Fema… [70-… Referral 4\n#> 6 No Cauca… Male [50-… Emergency Very High 2\n#> 7 Yes Afric… Fema… [70-… Referral Private 3\n#> 8 No Cauca… Fema… [20-… Emergency 1\n#> 9 No Cauca… Male [60-… Other 12\n#> 10 No Cauca… Fema… [80-… Referral Medica… 1\n#> # ℹ 71,505 more rows\n#> # ℹ 4 more variables: n_previous_visits , n_diagnoses ,\n#> # n_procedures , n_medications \n```\n:::\n\n\nThe first variable in this data, `readmitted`, gives whether the patient was readmitted within 30 days of discharge. We'll use this variable as a proxy for \"unmet need for additional care,\" in that readmission within one month indicates that the patient may have benefited from additional attention during their hospital stay; if a machine learning model consistently identifies lesser need (via prediction of non-readmission) in one subgroup than another, the subgroup allocated lesser need will go without care they'd benefit from. We'd like to train a model that is both fair with regard to how it treats `race` groups and is as performant as possible. The tidymodels framework provides the tools needed to identify such disparities.\n\nLoading needed packages:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(tidymodels)\nlibrary(baguette)\nlibrary(desirability2)\nlibrary(GGally)\n```\n:::\n\n\n## Exploratory Analysis\n\nLet's start off our analysis with some explanatory summarization and plotting. First, taking a look at our outcome variable:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nreadmission |>\n count(readmitted)\n#> # A tibble: 2 × 2\n#> readmitted n\n#> \n#> 1 Yes 6293\n#> 2 No 65222\n```\n:::\n\n\n8.8% of patients were readmitted within 30 days after being discharged from the hospital. This is an example of a modeling problem with a _class imbalance_, where one value of the outcome variable is much more common than another. Now, taking a look at the counts of those in each protected class:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nreadmission |>\n count(race)\n#> # A tibble: 6 × 2\n#> race n\n#> \n#> 1 African American 12887\n#> 2 Asian 497\n#> 3 Caucasian 53491\n#> 4 Hispanic 1517\n#> 5 Other 1177\n#> 6 Unknown 1946\n```\n:::\n\n\n::: callout-note\nWe'll refer to the `race` groups in this data by their actual value (e.g. `\"Caucasian\"` rather than Caucasian) so as to not take for granted the choices that the dataset authors made in choosing these categorizations. Racial categorizations are not stable across time and place---as you read on, consider how a change in the categories used in data collection might affect this analysis [@omi1994].\n:::\n\nA vast majority of patients are labeled `\"Caucasian\"` (74.8%) or `\"African American\"` (18%). The counts for the remaining racial categorizations are quite a bit smaller, and when we split the data up into resamples, those counts will reduce even further. As a result, the variability associated with the estimates `\"Asian\"`, `\"Hispanic\"`, `\"Other\"`, and `\"Unknown\"` will be larger than those for `\"African American\"` and `\"Causasian\"`. As an example:\n\n\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nreadmission |>\n # randomly split into groups\n mutate(group = sample(1:10, n(), replace = TRUE)) |>\n group_by(race, group) |>\n # compute proportion readmitted by race + random `group`ing\n summarize(prop = mean(readmitted == \"Yes\"), n = n()) |>\n # compute variation in the proportion by race.\n # note that, by default, the output from above is grouped by race only.\n summarize(mean = mean(prop), sd = sd(prop), n = sum(n))\n#> # A tibble: 6 × 4\n#> race mean sd n\n#> \n#> 1 African American 0.0849 0.00713 12887\n#> 2 Asian 0.0825 0.0295 497\n#> 3 Caucasian 0.0900 0.00369 53491\n#> 4 Hispanic 0.0805 0.0285 1517\n#> 5 Other 0.0680 0.0265 1177\n#> 6 Unknown 0.0720 0.0178 1946\n```\n:::\n\n\nThe standard deviations `sd` are much larger for groups with a smaller number of observations `n`, even if the average proportions `mean` are similar. This variation will affect our analysis downstream in that fairness metrics measure variation across groups; the noise associated with proportions calculated for `race` groups with fewer values may overwhelm the signal associated with actual disparities in care for those groups.\n\nWe have several options when considering how to address this:\n\n* **Remove rows arising from infrequent classes.** In this example, this would mean removing all rows with `race` values other than `\"Caucasian\"` or `\"African American\"`. This is the approach taken by other public analyses of these data, including the original study that this data arose from. An analysis resulting from this approach would ignore any disparities in care for `race` groups other than `\"Caucasian\"` or `\"African American\"`.\n* **Construct custom fairness metrics that address the increased variability associated with smaller counts.** We could scale the variability of the estimates for each `race` group before calculating fairness metrics.\n* **Bin the values for infrequent classes.** This would entail collapsing values of `race` other than `\"Caucasian\"` and `\"African American\"` into one factor level. This approach is somewhat of a hybrid of the two approaches above; we lose some granularity in information regarding care for `race` groups other than `\"Caucasian\"` and `\"African American\"`, but reduce the variability associated with the estimates for those groups in the process.\n\nFor this analysis, we'll go with the last option.\n\n::: callout-note\nWhile we won't construct custom fairness metrics in this example, you can do so using the `new_groupwise_metric()` function in yardstick.\n:::\n\nRecoding that data column:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nreadmission_collapsed <-\n readmission |>\n mutate(\n race = case_when(\n !(race %in% c(\"Caucasian\", \"African American\")) ~ \"Other\",\n .default = race\n ),\n race = factor(race)\n )\n\nreadmission_collapsed |>\n count(race)\n#> # A tibble: 3 × 2\n#> race n\n#> \n#> 1 African American 12887\n#> 2 Caucasian 53491\n#> 3 Other 5137\n```\n:::\n\n\nPlotting distributions of remaining predictors:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nreadmission_collapsed |>\n ggplot(aes(x = age)) +\n geom_bar() +\n facet_grid(rows = vars(admission_source))\n```\n\n::: {.cell-output-display}\n![](figs/plot-age-by-source-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nMost patients in this data are in their 60s and 70s. Emergencies account for most admissions in this data, though many others are from referrals or other sources.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nreadmission_collapsed |>\n ggplot(aes(x = insurer)) +\n geom_bar()\n```\n\n::: {.cell-output-display}\n![](figs/plot-insurer-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nWhile payment information on most patients is missing, most patients in this data are covered under Medicare.\n\n::: callout-tip\nThe payment method is one way in which societal unfairness may be reflected in the source data besides the variables on protected groups themselves. Medicaid coverage is only available to people below a certain income, and many self-pay patients do not have medical insurance because they cannot afford it. Relatedly, poverty rates differ drastically among racial groups in the U.S.\n:::\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nreadmission_collapsed |>\n pivot_longer(starts_with(\"n_\")) |>\n ggplot(aes(x = value)) +\n geom_histogram() +\n facet_wrap(vars(name), scales = \"free_x\")\n```\n\n::: {.cell-output-display}\n![](figs/plot-n-variables-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nFor many patients, this was their first inpatient visit to this hospital system. During their stay, many patients receive 10-20 medications and experience several procedures.\n\nWith a sense for the distributions of variables in this dataset, we can move on to splitting data up for modeling.\n\n## Resampling Data\n\nFirst, splitting data into training and testing:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(1)\nreadmission_splits <- initial_split(readmission_collapsed, strata = readmitted)\nreadmission_train <- training(readmission_splits)\nreadmission_test <- testing(readmission_splits)\n```\n:::\n\n\n:::callout-note\nWe've set `strata = readmitted` here to stratify our sample by the outcome variable in order to address the class imbalance. To learn more about class imbalances and stratification, see the [\"Common Methods for Splitting Data\"](https://www.tmwr.org/splitting#splitting-methods) chapter of [_Tidy Modeling with R_](https://www.tmwr.org/) [@kuhn2022].\n:::\n\nWe'll set the 17,879-row `readmission_test` test set to the side for the remainder of the analysis until we compute a final estimate of our performance on the chosen model. Splitting the 53,636 rows of the training data into 10 resamples:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nreadmission_folds <- vfold_cv(readmission_train, strata = readmitted)\n\nreadmission_folds\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> \n#> 1 Fold01\n#> 2 Fold02\n#> 3 Fold03\n#> 4 Fold04\n#> 5 Fold05\n#> 6 Fold06\n#> 7 Fold07\n#> 8 Fold08\n#> 9 Fold09\n#> 10 Fold10\n```\n:::\n\n\nEach split contains an analysis and assessment set: one for model fitting and the other for evaluation. Averaging performance estimates across resamples will give us a sense for how well the model performs on data it hasn't yet seen.\n\n## Training and Evaluating Models\n\nWe'll define a diverse set of models and pre-processing strategies and then evaluate them against our resamples.\n\n### Model Workflows\n\nWe'll first define a basic recipe that first sets a factor level for missing values and then centers and scales numeric data:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nrecipe_basic <-\n recipe(readmitted ~ ., data = readmission) |>\n step_unknown(all_nominal_predictors()) |>\n step_YeoJohnson(all_numeric_predictors()) |>\n step_normalize(all_numeric_predictors()) |>\n step_dummy(all_nominal_predictors())\n```\n:::\n\n\nThe other preprocessor that we'll try encodes the age as a numeric variable rather than the bins as in the source data:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\n# e.g. \"[10-20]\" -> 15\nage_bin_to_midpoint <- function(age_bin) {\n # ensure factors are treated as their label\n age <- as.character(age_bin) \n # take the second character, e.g. \"[10-20]\" -> \"1\"\n age <- substr(age, 2, 2)\n # convert to numeric, e.g. \"1\" -> 1\n age <- as.numeric(age)\n # scale to bin's midpoint, e.g. 1 -> 10 + 5 -> 15\n age * 10 + 5\n}\n\nrecipe_age <-\n recipe(readmitted ~ ., data = readmission) |>\n step_mutate(age_num = age_bin_to_midpoint(age)) |>\n step_rm(age) |>\n step_unknown(all_nominal_predictors()) |>\n step_YeoJohnson(all_numeric_predictors()) |>\n step_normalize(all_numeric_predictors()) |>\n step_dummy(all_nominal_predictors())\n```\n:::\n\n\nBoth of these preprocessors will be combined with one of three models. Logistic regressions, XGBoost, and bagged neural networks make a diverse set of assumptions about the underlying data generating process. Defining model specifications for each:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nspec_lr <- \n logistic_reg(\"classification\")\n\nspec_bt <-\n boost_tree(\"classification\", mtry = tune(), learn_rate = tune(), trees = 500)\n\nspec_nn <-\n bag_mlp(\"classification\", hidden_units = tune(), penalty = tune())\n```\n:::\n\n\nWe now combine each unique combination of recipe and preprocessor in a workflow set:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nwflow_set <- \n workflow_set(\n preproc = list(basic = recipe_basic, age = recipe_age),\n models = list(lr = spec_lr, bt = spec_bt, nn = spec_nn)\n )\n\nwflow_set\n#> # A workflow set/tibble: 6 × 4\n#> wflow_id info option result \n#> \n#> 1 basic_lr \n#> 2 basic_bt \n#> 3 basic_nn \n#> 4 age_lr \n#> 5 age_bt \n#> 6 age_nn \n```\n:::\n\n\nEach workflow in the workflow set is now ready to be evaluated. We now need to decide how to best evaluate these modeling workflows, though.\n\n### Metrics\n\nThe metrics with which we choose to evaluate our models are the core of our fairness analysis. In addition to the default metrics for classification in tune, `accuracy()` and `roc_auc()`, we'll compute a set of fairness metrics: `equal_opportunity()`, `equalized_odds()`, and `demographic_parity()`.\n\n* `equal_opportunity()`: Equal opportunity is satisfied when a model's predictions have the same true positive and false negative rates across protected groups. In this example, a model satisfies equal opportunity if it correctly predicts readmission and incorrectly predicts non-readmission at the same rate across `race` groups. In this case, the metric represents the interests of the patient; a patient would like to be just as likely to receive additional care resources as another if they are of equal need, and no more likely to go without unneeded care than another. Since this metric does not consider false positives, it notably does not penalize disparately providing additional care resources to a patient who may not need them.\n\n* `equalized_odds()`: Equalized odds is satisfied when a model's predictions have the same false positive, true positive, false negative, and true negative rates across protected groups. This definition is a special case of the one above, where there's additionally a constraint placed on the false positive and true negative rates. In this example, a model satisfies equalized odds if it correctly predicts both readmission and non-readmission _and_ incorrectly predicts readmission and non-readmission at the same rate across `race` groups. Similar to equal opportunity, the stakeholders for the metric in this case can be generally understood to be those who are subject to the model's predictions, except that this metric also aims to prevent disparately 1) providing additional care resources to those who may not need them and 2) identifying patients who do _not_ need additional care resources correctly.\n\n* `demographic_parity()`: Demographic parity is satisfied when a model's predictions have the same predicted positive rate across groups. In this example, a model satisfies demographic parity if it predicts readmission at the same rate across `race` groups. Note that this metric does not depend on the true outcome value, `readmitted`. The interests of a stakeholder who would like to see additional care resources provisioned at the same rate across `race` groups, even if the actual need for those resources differs among groups, are represented by this metric. As demographic parity is broadly accepted as part of a legal definition of machine learning fairness, hospital systems might consider this metric to protect themselves legally [@ecfr].\n\nFor each of the above metrics, values closer to zero indicate that a model is more fair.\n\nThe above three metrics are defined specifically with fairness in mind. By another view of fairness, though, `accuracy()` and `roc_auc()` are _also_ fairness metrics. Some stakeholders may believe that the most performant model regardless of group membership---i.e. the model that predicts readmission most accurately across groups---is the most fair model. \n\nTo evaluate each of these metrics against the specified workflows, we create a metric set like so:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nm_set <-\n metric_set(\n accuracy,\n roc_auc,\n equal_opportunity(race),\n equalized_odds(race),\n demographic_parity(race)\n )\n\nm_set\n#> A metric set, consisting of:\n#> - `accuracy()`, a class metric | direction: maximize\n#> - `roc_auc()`, a probability metric | direction: maximize\n#> - `equal_opportunity(race)()`, a class metric | direction: minimize,\n#> group-wise on: race\n#> - `equalized_odds(race)()`, a class metric | direction: minimize,\n#> group-wise on: race\n#> - `demographic_parity(race)()`, a class metric | direction: minimize,\n#> group-wise on: race\n```\n:::\n\n\n::: callout-note\nThe first two inputs, `accuracy()` and `roc_auc()`, are standard [yardstick](https://yardstick.tidymodels.org) metrics. The latter three are also yardstick metrics like any other, though are created using the [_metric factories_](https://yardstick.tidymodels.org/reference/new_groupwise_metric.html) `equal_opportunity()`, `equalized_odds()`, and `demographic_parity()`. When passed a data-column, metric factories output yardstick metrics. \n:::\n\n### Evaluation\n\nWe can now evaluate the workflows we've defined against resamples using our metric set. The `workflow_map()` function will call `tune_grid()` on each workflow:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(1)\nwflow_set_fit <- \n workflow_map(\n wflow_set, \n verbose = TRUE, \n seed = 1, \n metrics = m_set,\n resamples = readmission_folds\n )\n```\n:::\n\n\n\n\nA fitted workflow set looks just like the unfitted workflow set we saw previously, except that information on the tuning process is now stored in the `option` and `result` variables for each modeling workflow:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nwflow_set_fit\n#> # A workflow set/tibble: 6 × 4\n#> wflow_id info option result \n#> \n#> 1 basic_lr \n#> 2 basic_bt \n#> 3 basic_nn \n#> 4 age_lr \n#> 5 age_bt \n#> 6 age_nn \n```\n:::\n\n\n## Model Selection\n\nNow that we've evaluated a number of models with a variety of metrics, we can explore the results to determine our optimal model. Beginning with a quick exploratory plot of the distributions of our metrics:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nwflow_set_fit |>\n collect_metrics() |>\n pivot_wider(\n id_cols = c(wflow_id, .config), \n names_from = .metric, \n values_from = mean\n ) |>\n select(-c(wflow_id, .config)) |>\n ggpairs() +\n theme(axis.text.x = element_text(angle = 45, vjust = .8, hjust = .9))\n```\n\n::: {.cell-output-display}\n![](figs/plot-wflow-set-fit-corr-1.svg){fig-align='center' width=80%}\n:::\n:::\n\n\nThe fairness metrics `demographic_parity()`, `equal_opportunity()`, and `equalized_odds()` all take values very close to zero for many models. Also, the metric values are highly correlated with each other, including correlations between fairness metrics and the more general-purpose performance metrics. That is, the most performant models also seem to be among the most fair.\n\nMore concretely, we can rank the model configurations to examine only the most performant models:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nrank_results(wflow_set_fit, rank_metric = \"roc_auc\") |>\n filter(.metric == \"roc_auc\")\n#> # A tibble: 42 × 9\n#> wflow_id .config .metric mean std_err n preprocessor model rank\n#> \n#> 1 age_bt Preprocessor1_… roc_auc 0.605 0.00424 10 recipe boos… 1\n#> 2 basic_bt Preprocessor1_… roc_auc 0.605 0.00423 10 recipe boos… 2\n#> 3 age_bt Preprocessor1_… roc_auc 0.604 0.00378 10 recipe boos… 3\n#> 4 age_bt Preprocessor1_… roc_auc 0.603 0.00421 10 recipe boos… 4\n#> 5 basic_bt Preprocessor1_… roc_auc 0.603 0.00410 10 recipe boos… 5\n#> 6 basic_bt Preprocessor1_… roc_auc 0.603 0.00436 10 recipe boos… 6\n#> 7 age_bt Preprocessor1_… roc_auc 0.602 0.00372 10 recipe boos… 7\n#> 8 basic_nn Preprocessor1_… roc_auc 0.602 0.00403 10 recipe bag_… 8\n#> 9 age_nn Preprocessor1_… roc_auc 0.600 0.00431 10 recipe bag_… 9\n#> 10 age_nn Preprocessor1_… roc_auc 0.600 0.00412 10 recipe bag_… 10\n#> # ℹ 32 more rows\n```\n:::\n\n\nAlmost all of the most performant model configurations arise from the boosted tree modeling workflow. Let's examine the results for specifically the modeling workflow that encodes age as a number more thoroughly:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nautoplot(wflow_set_fit, id = \"age_bt\")\n```\n\n::: {.cell-output-display}\n![](figs/autoplot-age-bt-1.svg){fig-align='center' width=70%}\n:::\n:::\n\n\nThe learning rate `learn_rate` seems to have a more pronounced effect on the results metrics than the number of randomly selected predictors `mtry`. As before, we see that the most performant models with respect to `roc_auc()` also tend to be the most fair according to our fairness metrics. Further, the values of each of the fairness metrics plotted above seem highly correlated. \n\nFrom the perspective of a practitioner hoping to satisfy various stakeholders, the fact that these metrics are highly correlated makes the model selection process much easier. We can choose one fairness metric that we'd like to optimize for, and likely end up with a near-optimal configuration for the other metrics as a byproduct.\n\n:::callout-note\nIn machine learning fairness, \"impossibility theorems\" show that fairness definitions \"are not mathematically or morally compatible in general\" [@mitchell2021]. More concretely, unless we live in a world with no inequality, there is no way to satisfy many definitions of fairness at once. However, recent research emphasizes that near-fairness among more limited sets of metrics, like the three we've used here, is both possible and relevant [@bell2023].\n:::\n\nTo choose a model that performs well both with respect to a typical performance metric like `roc_auc()` and the fairness metrics we've chosen, we will make use of [desirability functions](https://www.tidyverse.org/blog/2023/05/desirability2/), which allow us to optimize based on multiple metrics at once.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nbest_params <-\n # extract the tuning results for the boosted tree model\n extract_workflow_set_result(wflow_set_fit, \"age_bt\") |>\n # collect the metrics associated with it\n collect_metrics() |>\n # pivot the metrics so that each is in a column\n pivot_wider(\n id_cols = c(mtry, learn_rate), \n names_from = .metric, \n values_from = mean\n ) |>\n mutate(\n # higher roc values are better; detect max and min from the data\n d_roc = d_max(roc_auc, use_data = TRUE),\n # lower equalized odds are better; detect max and min from the data\n d_e_odds = d_min(equalized_odds, use_data = TRUE),\n # compute overall desirability based on d_roc and d_e_odds\n d_overall = d_overall(across(starts_with(\"d_\")))\n ) |>\n # pick the model with the highest desirability value\n slice_max(d_overall)\n```\n:::\n\n\nThe result is a tibble giving the parameter values that resulted in the best model:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nbest_params\n#> # A tibble: 1 × 10\n#> mtry learn_rate accuracy demographic_parity equal_opportunity equalized_odds\n#> \n#> 1 7 0.00456 0.912 0 0 0\n#> # ℹ 4 more variables: roc_auc , d_roc , d_e_odds ,\n#> # d_overall \n```\n:::\n\n\nWe can use that tibble to finalize a workflow that we'll use to generate our final model fit:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nfinal_model_config <-\n extract_workflow(wflow_set_fit, \"age_bt\") |>\n finalize_workflow(best_params)\n```\n:::\n\n\nFinally, generating our final model fit:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nfinal_model <- \n last_fit(final_model_config, readmission_splits, metrics = m_set)\n```\n:::\n\n\n\n\nWe can see the metrics associated with the final fit using `collect_metrics()`, just as with a tuning result:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ncollect_metrics(final_model)\n#> # A tibble: 5 × 5\n#> .metric .estimator .estimate .by .config \n#> \n#> 1 accuracy binary 0.912 Preprocessor1_Model1\n#> 2 equal_opportunity binary 0 race Preprocessor1_Model1\n#> 3 equalized_odds binary 0 race Preprocessor1_Model1\n#> 4 demographic_parity binary 0 race Preprocessor1_Model1\n#> 5 roc_auc binary 0.602 Preprocessor1_Model1\n```\n:::\n\n\n\n\nThe model we've selected has near-fairness with respect to the set of metrics we've chosen here. The accuracy of the model is 91.16%, quite similar to the accuracy that would result if the model just always predicted a patient would not readmit (91.2%). The `roc_auc()` value 0.602 indicates that the model indeed correctly predicts readmission in some cases, though still has a lot of room for improvement. A further analysis of these models might measure performance using a metric that specifically evaluates predictions on observations from the minority class---as in, patients that did actually readmit---like [`sens()`](https://yardstick.tidymodels.org/reference/sens.html?q=sens#details).\n\nExtracting the model fit from the `last_fit` object:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nfinal_model_fit <- extract_workflow(final_model)\n```\n:::\n\n\nThe `final_model_fit` object is now ready to predict on new data! Models generated with tidymodels are easily versioned, deployed, and monitored using the vetiver framework; learn more about the framework on the [vetiver website](https://vetiver.tidymodels.org/get-started/).\n\nIn this article, we've demonstrated a fairness-oriented modeling analysis. Modeling with fairness in mind is not simply a numerical optimization problem, but a holistic process of examining the moral meanings of fairness in our problem context and how they are represented---or not represented---by various mathematical notions of fairness. \n\nWhile this analysis allowed us to train models that are near-fair with respect to a limited set of fairness metrics, it leaves many questions to be answered about the impacts of using such a model. Among other things:\n\n* We've only evaluated the fairness of the model with `race` in mind. We also have information on the patient's `sex`. Could this model behave unfairly with respect to the patient's `sex`? How about with respect to the intersections of `race` and `sex`? A growing body of research shows us that black women experience pronounced discrimination as patients [@johnson2019; @okoro2020; @gopal2021].\n* The categorizations of `race` and `sex` that we have access to are coarse. Race/ethnicity and sex/gender are richly and diversely experienced, and the limited set of categories that patients are presented with (or unknowingly assigned to) in healthcare contexts are only a proxy for how a patient may experience them. \n* We've used _readmission_ as a proxy for _need for additional care_. What factors might influence whether a patient willingly readmits? Or is admitted to the hospital in the first place? Note, especially, the disproportionate financial burden of healthcare for poorer patients.\n* We don't know how the predictions resulting from this model will be used by practitioners or trusted by patients. The outputs of the model, of course, should be evaluated in conjunction with other evidence collected by a healthcare team throughout a patient's stay. Knowing this, though, will the model's outputs be differently interpreted for different groups in the patient population? For example, would a positive prediction of readmission be considered more seriously as evidence for needed additional care for a white man than for others? Further, given historical exploitation and unethical practice, would protected groups believe that algorithmic recommendation for additional care is likely discriminatory and mistrust the recommendations offered to them? [See @rajkomar2018.] \n\nMachine learning models can both have significant positive impacts on our lives and at the same time cause significant harms. Given the tremendous reach of these models in our society, efforts to include fairness as a criteria for evaluating machine learning models are as necessary as ever.\n\n## Session information {#session-info}\n\n\n::: {.cell layout-align=\"center\"}\n\n```\n#> ─ Session info ─────────────────────────────────────────────────────\n#> version R version 4.6.0 (2026-04-24)\n#> language (EN)\n#> pandoc 3.1.3\n#> quarto 1.9.37\n#> \n#> ─ Packages ─────────────────────────────────────────────────────────\n#> package version date (UTC)\n#> baguette 1.1.0 2025-01-28\n#> broom 1.0.13 2026-05-14\n#> desirability2 0.2.0 2025-08-22\n#> dials 1.4.3 2026-04-11\n#> dplyr 1.2.1 2026-04-03\n#> GGally 2.4.0 2025-08-23\n#> ggplot2 4.0.3 2026-04-22\n#> infer 1.1.0 2025-12-18\n#> parsnip 1.6.0 2026-05-14\n#> purrr 1.2.2 2026-04-10\n#> readmission 0.1.0 2023-12-07\n#> recipes 1.3.2 2026-04-02\n#> rlang 1.2.0 2026-04-06\n#> rsample 1.3.2 2026-01-30\n#> tibble 3.3.1 2026-01-11\n#> tidymodels 1.5.0 2026-04-23\n#> tune 2.1.0 2026-04-17\n#> workflows 1.3.0 2025-08-27\n#> yardstick 1.4.0 2026-04-07\n#> \n#> ────────────────────────────────────────────────────────────────────\n```\n:::\n\n", + "markdown": "---\ntitle: \"Fair prediction of hospital readmission: a machine learning fairness case study\"\ncategories:\n - tuning and workflows\n - yardstick\n - fairness\n - tuning\n - classification\ntype: learn-subsection\nweight: 6\ndescription: |\n With information on a diabetes patient's stay at a hospital like demographics, diagnostic results, payment, and medications, a hospital can train a machine learning model to predict reasonably well whether a patient will be readmitted within 30 days. What harms to patients could result from using such a model, though?\ntoc: true\ntoc-depth: 2\nbibliography: refs.bib\nr-packages:\n - tidymodels\n - readmission\n - baguette\n - GGally\n - desirability2\ninclude-after-body: ../../../html/resources.html\n---\n\n------------------------------------------------------------------------\n\n\n\n\n\nIn 2019, @obermeyer2019 published an analysis of predictions from a machine learning model that health care providers use to allocate resources. The model's outputs are used to recommend a patient for _high-risk care management_ programs:\n\n> These programs seek to improve the care of patients with complex health needs by providing additional resources, including greater attention from trained providers, to help ensure that care is well coordinated. Most health systems use these programs as the cornerstone of population health management efforts, and they are widely considered effective at improving outcomes and satisfaction while reducing costs. [...] Because the programs are themselves expensive---with costs going toward teams of dedicated nurses, extra primary care appointment slots, and other scarce resources---**health systems rely extensively on algorithms to identify patients who will benefit the most.**\n\nThey argue in their analysis that the model in question exhibits substantial racial bias, where \"Black patients assigned the same level of risk by the algorithm are sicker than White patients.\" In practice, this resulted in the reduction of \"the number of Black patients identified for extra care by more than half.\"\n\nThis article will demonstrate a fairness-oriented workflow for training a machine learning model to identify high-risk patients. Throughout the model development process, we'll consider the social impacts that different modeling decisions may have when such a model is deployed in context.\n\n## Setup\n\nThe data we'll use in this analysis is a publicly available database containing information on 71,515 hospital stays from diabetes patients. The data comes from a study by @strack2014, where the authors model the effectiveness of a particular lab test in predicting readmission. A version of that data is available in the [readmission R package](https://simonpcouch.github.io/readmission/):\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(readmission)\n\nreadmission\n#> # A tibble: 71,515 × 12\n#> readmitted race sex age admission_source blood_glucose insurer duration\n#> \n#> 1 Yes Afric… Male [60-… Referral 7\n#> 2 No Cauca… Fema… [50-… Emergency Normal Private 4\n#> 3 Yes Cauca… Fema… [70-… Referral Medica… 5\n#> 4 No Cauca… Fema… [80-… Referral Private 5\n#> 5 No Cauca… Fema… [70-… Referral 4\n#> 6 No Cauca… Male [50-… Emergency Very High 2\n#> 7 Yes Afric… Fema… [70-… Referral Private 3\n#> 8 No Cauca… Fema… [20-… Emergency 1\n#> 9 No Cauca… Male [60-… Other 12\n#> 10 No Cauca… Fema… [80-… Referral Medica… 1\n#> # ℹ 71,505 more rows\n#> # ℹ 4 more variables: n_previous_visits , n_diagnoses ,\n#> # n_procedures , n_medications \n```\n:::\n\n\nThe first variable in this data, `readmitted`, gives whether the patient was readmitted within 30 days of discharge. We'll use this variable as a proxy for \"unmet need for additional care,\" in that readmission within one month indicates that the patient may have benefited from additional attention during their hospital stay; if a machine learning model consistently identifies lesser need (via prediction of non-readmission) in one subgroup than another, the subgroup allocated lesser need will go without care they'd benefit from. We'd like to train a model that is both fair with regard to how it treats `race` groups and is as performant as possible. The tidymodels framework provides the tools needed to identify such disparities.\n\nLoading needed packages:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(tidymodels)\nlibrary(baguette)\nlibrary(desirability2)\nlibrary(GGally)\n```\n:::\n\n\n## Exploratory Analysis\n\nLet's start off our analysis with some explanatory summarization and plotting. First, taking a look at our outcome variable:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nreadmission |>\n count(readmitted)\n#> # A tibble: 2 × 2\n#> readmitted n\n#> \n#> 1 Yes 6293\n#> 2 No 65222\n```\n:::\n\n\n8.8% of patients were readmitted within 30 days after being discharged from the hospital. This is an example of a modeling problem with a _class imbalance_, where one value of the outcome variable is much more common than another. Now, taking a look at the counts of those in each protected class:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nreadmission |>\n count(race)\n#> # A tibble: 6 × 2\n#> race n\n#> \n#> 1 African American 12887\n#> 2 Asian 497\n#> 3 Caucasian 53491\n#> 4 Hispanic 1517\n#> 5 Other 1177\n#> 6 Unknown 1946\n```\n:::\n\n\n::: callout-note\nWe'll refer to the `race` groups in this data by their actual value (e.g. `\"Caucasian\"` rather than Caucasian) so as to not take for granted the choices that the dataset authors made in choosing these categorizations. Racial categorizations are not stable across time and place---as you read on, consider how a change in the categories used in data collection might affect this analysis [@omi1994].\n:::\n\nA vast majority of patients are labeled `\"Caucasian\"` (74.8%) or `\"African American\"` (18%). The counts for the remaining racial categorizations are quite a bit smaller, and when we split the data up into resamples, those counts will reduce even further. As a result, the variability associated with the estimates `\"Asian\"`, `\"Hispanic\"`, `\"Other\"`, and `\"Unknown\"` will be larger than those for `\"African American\"` and `\"Causasian\"`. As an example:\n\n\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nreadmission |>\n # randomly split into groups\n mutate(group = sample(1:10, n(), replace = TRUE)) |>\n group_by(race, group) |>\n # compute proportion readmitted by race + random `group`ing\n summarize(prop = mean(readmitted == \"Yes\"), n = n()) |>\n # compute variation in the proportion by race.\n # note that, by default, the output from above is grouped by race only.\n summarize(mean = mean(prop), sd = sd(prop), n = sum(n))\n#> # A tibble: 6 × 4\n#> race mean sd n\n#> \n#> 1 African American 0.0849 0.00713 12887\n#> 2 Asian 0.0825 0.0295 497\n#> 3 Caucasian 0.0900 0.00369 53491\n#> 4 Hispanic 0.0805 0.0285 1517\n#> 5 Other 0.0680 0.0265 1177\n#> 6 Unknown 0.0720 0.0178 1946\n```\n:::\n\n\nThe standard deviations `sd` are much larger for groups with a smaller number of observations `n`, even if the average proportions `mean` are similar. This variation will affect our analysis downstream in that fairness metrics measure variation across groups; the noise associated with proportions calculated for `race` groups with fewer values may overwhelm the signal associated with actual disparities in care for those groups.\n\nWe have several options when considering how to address this:\n\n* **Remove rows arising from infrequent classes.** In this example, this would mean removing all rows with `race` values other than `\"Caucasian\"` or `\"African American\"`. This is the approach taken by other public analyses of these data, including the original study that this data arose from. An analysis resulting from this approach would ignore any disparities in care for `race` groups other than `\"Caucasian\"` or `\"African American\"`.\n* **Construct custom fairness metrics that address the increased variability associated with smaller counts.** We could scale the variability of the estimates for each `race` group before calculating fairness metrics.\n* **Bin the values for infrequent classes.** This would entail collapsing values of `race` other than `\"Caucasian\"` and `\"African American\"` into one factor level. This approach is somewhat of a hybrid of the two approaches above; we lose some granularity in information regarding care for `race` groups other than `\"Caucasian\"` and `\"African American\"`, but reduce the variability associated with the estimates for those groups in the process.\n\nFor this analysis, we'll go with the last option.\n\n::: callout-note\nWhile we won't construct custom fairness metrics in this example, you can do so using the `new_groupwise_metric()` function in yardstick.\n:::\n\nRecoding that data column:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nreadmission_collapsed <-\n readmission |>\n mutate(\n race = case_when(\n !(race %in% c(\"Caucasian\", \"African American\")) ~ \"Other\",\n .default = race\n ),\n race = factor(race)\n )\n\nreadmission_collapsed |>\n count(race)\n#> # A tibble: 3 × 2\n#> race n\n#> \n#> 1 African American 12887\n#> 2 Caucasian 53491\n#> 3 Other 5137\n```\n:::\n\n\nPlotting distributions of remaining predictors:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nreadmission_collapsed |>\n ggplot(aes(x = age)) +\n geom_bar() +\n facet_grid(rows = vars(admission_source))\n```\n\n::: {.cell-output-display}\n![](figs/plot-age-by-source-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nMost patients in this data are in their 60s and 70s. Emergencies account for most admissions in this data, though many others are from referrals or other sources.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nreadmission_collapsed |>\n ggplot(aes(x = insurer)) +\n geom_bar()\n```\n\n::: {.cell-output-display}\n![](figs/plot-insurer-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nWhile payment information on most patients is missing, most patients in this data are covered under Medicare.\n\n::: callout-tip\nThe payment method is one way in which societal unfairness may be reflected in the source data besides the variables on protected groups themselves. Medicaid coverage is only available to people below a certain income, and many self-pay patients do not have medical insurance because they cannot afford it. Relatedly, poverty rates differ drastically among racial groups in the U.S.\n:::\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nreadmission_collapsed |>\n pivot_longer(starts_with(\"n_\")) |>\n ggplot(aes(x = value)) +\n geom_histogram() +\n facet_wrap(vars(name), scales = \"free_x\")\n```\n\n::: {.cell-output-display}\n![](figs/plot-n-variables-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nFor many patients, this was their first inpatient visit to this hospital system. During their stay, many patients receive 10-20 medications and experience several procedures.\n\nWith a sense for the distributions of variables in this dataset, we can move on to splitting data up for modeling.\n\n## Resampling Data\n\nFirst, splitting data into training and testing:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(1)\nreadmission_splits <- initial_split(readmission_collapsed, strata = readmitted)\nreadmission_train <- training(readmission_splits)\nreadmission_test <- testing(readmission_splits)\n```\n:::\n\n\n:::callout-note\nWe've set `strata = readmitted` here to stratify our sample by the outcome variable in order to address the class imbalance. To learn more about class imbalances and stratification, see the [\"Common Methods for Splitting Data\"](https://www.tmwr.org/splitting#splitting-methods) chapter of [_Tidy Modeling with R_](https://www.tmwr.org/) [@kuhn2022].\n:::\n\nWe'll set the 17,879-row `readmission_test` test set to the side for the remainder of the analysis until we compute a final estimate of our performance on the chosen model. Splitting the 53,636 rows of the training data into 10 resamples:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nreadmission_folds <- vfold_cv(readmission_train, strata = readmitted)\n\nreadmission_folds\n#> # 10-fold cross-validation using stratification \n#> # A tibble: 10 × 2\n#> splits id \n#> \n#> 1 Fold01\n#> 2 Fold02\n#> 3 Fold03\n#> 4 Fold04\n#> 5 Fold05\n#> 6 Fold06\n#> 7 Fold07\n#> 8 Fold08\n#> 9 Fold09\n#> 10 Fold10\n```\n:::\n\n\nEach split contains an analysis and assessment set: one for model fitting and the other for evaluation. Averaging performance estimates across resamples will give us a sense for how well the model performs on data it hasn't yet seen.\n\n## Training and Evaluating Models\n\nWe'll define a diverse set of models and pre-processing strategies and then evaluate them against our resamples.\n\n### Model Workflows\n\nWe'll first define a basic recipe that first sets a factor level for missing values and then centers and scales numeric data:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nrecipe_basic <-\n recipe(readmitted ~ ., data = readmission) |>\n step_unknown(all_nominal_predictors()) |>\n step_YeoJohnson(all_numeric_predictors()) |>\n step_normalize(all_numeric_predictors()) |>\n step_dummy(all_nominal_predictors())\n```\n:::\n\n\nThe other preprocessor that we'll try encodes the age as a numeric variable rather than the bins as in the source data:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\n# e.g. \"[10-20]\" -> 15\nage_bin_to_midpoint <- function(age_bin) {\n # ensure factors are treated as their label\n age <- as.character(age_bin) \n # take the second character, e.g. \"[10-20]\" -> \"1\"\n age <- substr(age, 2, 2)\n # convert to numeric, e.g. \"1\" -> 1\n age <- as.numeric(age)\n # scale to bin's midpoint, e.g. 1 -> 10 + 5 -> 15\n age * 10 + 5\n}\n\nrecipe_age <-\n recipe(readmitted ~ ., data = readmission) |>\n step_mutate(age_num = age_bin_to_midpoint(age)) |>\n step_rm(age) |>\n step_unknown(all_nominal_predictors()) |>\n step_YeoJohnson(all_numeric_predictors()) |>\n step_normalize(all_numeric_predictors()) |>\n step_dummy(all_nominal_predictors())\n```\n:::\n\n\nBoth of these preprocessors will be combined with one of three models. Logistic regressions, XGBoost, and bagged neural networks make a diverse set of assumptions about the underlying data generating process. Defining model specifications for each:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nspec_lr <- \n logistic_reg(\"classification\")\n\nspec_bt <-\n boost_tree(\"classification\", mtry = tune(), learn_rate = tune(), trees = 500)\n\nspec_nn <-\n bag_mlp(\"classification\", hidden_units = tune(), penalty = tune())\n```\n:::\n\n\nWe now combine each unique combination of recipe and preprocessor in a workflow set:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nwflow_set <- \n workflow_set(\n preproc = list(basic = recipe_basic, age = recipe_age),\n models = list(lr = spec_lr, bt = spec_bt, nn = spec_nn)\n )\n\nwflow_set\n#> # A workflow set/tibble: 6 × 4\n#> wflow_id info option result \n#> \n#> 1 basic_lr \n#> 2 basic_bt \n#> 3 basic_nn \n#> 4 age_lr \n#> 5 age_bt \n#> 6 age_nn \n```\n:::\n\n\nEach workflow in the workflow set is now ready to be evaluated. We now need to decide how to best evaluate these modeling workflows, though.\n\n### Metrics\n\nThe metrics with which we choose to evaluate our models are the core of our fairness analysis. In addition to the default metrics for classification in tune, `accuracy()` and `roc_auc()`, we'll compute a set of fairness metrics: `equal_opportunity()`, `equalized_odds()`, and `demographic_parity()`.\n\n* `equal_opportunity()`: Equal opportunity is satisfied when a model's predictions have the same true positive and false negative rates across protected groups. In this example, a model satisfies equal opportunity if it correctly predicts readmission and incorrectly predicts non-readmission at the same rate across `race` groups. In this case, the metric represents the interests of the patient; a patient would like to be just as likely to receive additional care resources as another if they are of equal need, and no more likely to go without unneeded care than another. Since this metric does not consider false positives, it notably does not penalize disparately providing additional care resources to a patient who may not need them.\n\n* `equalized_odds()`: Equalized odds is satisfied when a model's predictions have the same false positive, true positive, false negative, and true negative rates across protected groups. This definition is a special case of the one above, where there's additionally a constraint placed on the false positive and true negative rates. In this example, a model satisfies equalized odds if it correctly predicts both readmission and non-readmission _and_ incorrectly predicts readmission and non-readmission at the same rate across `race` groups. Similar to equal opportunity, the stakeholders for the metric in this case can be generally understood to be those who are subject to the model's predictions, except that this metric also aims to prevent disparately 1) providing additional care resources to those who may not need them and 2) identifying patients who do _not_ need additional care resources correctly.\n\n* `demographic_parity()`: Demographic parity is satisfied when a model's predictions have the same predicted positive rate across groups. In this example, a model satisfies demographic parity if it predicts readmission at the same rate across `race` groups. Note that this metric does not depend on the true outcome value, `readmitted`. The interests of a stakeholder who would like to see additional care resources provisioned at the same rate across `race` groups, even if the actual need for those resources differs among groups, are represented by this metric. As demographic parity is broadly accepted as part of a legal definition of machine learning fairness, hospital systems might consider this metric to protect themselves legally [@ecfr].\n\nFor each of the above metrics, values closer to zero indicate that a model is more fair.\n\nThe above three metrics are defined specifically with fairness in mind. By another view of fairness, though, `accuracy()` and `roc_auc()` are _also_ fairness metrics. Some stakeholders may believe that the most performant model regardless of group membership---i.e. the model that predicts readmission most accurately across groups---is the most fair model. \n\nTo evaluate each of these metrics against the specified workflows, we create a metric set like so:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nm_set <-\n metric_set(\n accuracy,\n roc_auc,\n equal_opportunity(race),\n equalized_odds(race),\n demographic_parity(race)\n )\n\nm_set\n#> A metric set, consisting of:\n#> - `accuracy()`, a class metric | direction: maximize\n#> - `roc_auc()`, a probability metric | direction: maximize\n#> - `equal_opportunity(race)()`, a class metric | direction: minimize,\n#> group-wise on: race\n#> - `equalized_odds(race)()`, a class metric | direction: minimize,\n#> group-wise on: race\n#> - `demographic_parity(race)()`, a class metric | direction: minimize,\n#> group-wise on: race\n```\n:::\n\n\n::: callout-note\nThe first two inputs, `accuracy()` and `roc_auc()`, are standard [yardstick](https://yardstick.tidymodels.org) metrics. The latter three are also yardstick metrics like any other, though are created using the [_metric factories_](https://yardstick.tidymodels.org/reference/new_groupwise_metric.html) `equal_opportunity()`, `equalized_odds()`, and `demographic_parity()`. When passed a data-column, metric factories output yardstick metrics. \n:::\n\n### Evaluation\n\nWe can now evaluate the workflows we've defined against resamples using our metric set. The `workflow_map()` function will call `tune_grid()` on each workflow:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(1)\nwflow_set_fit <- \n workflow_map(\n wflow_set, \n verbose = TRUE, \n seed = 1, \n metrics = m_set,\n resamples = readmission_folds\n )\n```\n:::\n\n\n\n\nA fitted workflow set looks just like the unfitted workflow set we saw previously, except that information on the tuning process is now stored in the `option` and `result` variables for each modeling workflow:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nwflow_set_fit\n#> # A workflow set/tibble: 6 × 4\n#> wflow_id info option result \n#> \n#> 1 basic_lr \n#> 2 basic_bt \n#> 3 basic_nn \n#> 4 age_lr \n#> 5 age_bt \n#> 6 age_nn \n```\n:::\n\n\n## Model Selection\n\nNow that we've evaluated a number of models with a variety of metrics, we can explore the results to determine our optimal model. Beginning with a quick exploratory plot of the distributions of our metrics:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nwflow_set_fit |>\n collect_metrics() |>\n pivot_wider(\n id_cols = c(wflow_id, .config), \n names_from = .metric, \n values_from = mean\n ) |>\n select(-c(wflow_id, .config)) |>\n ggpairs() +\n theme(axis.text.x = element_text(angle = 45, vjust = .8, hjust = .9))\n```\n\n::: {.cell-output-display}\n![](figs/plot-wflow-set-fit-corr-1.svg){fig-align='center' width=80%}\n:::\n:::\n\n\nThe fairness metrics `demographic_parity()`, `equal_opportunity()`, and `equalized_odds()` all take values very close to zero for many models. Also, the metric values are highly correlated with each other, including correlations between fairness metrics and the more general-purpose performance metrics. That is, the most performant models also seem to be among the most fair.\n\nMore concretely, we can rank the model configurations to examine only the most performant models:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nrank_results(wflow_set_fit, rank_metric = \"roc_auc\") |>\n filter(.metric == \"roc_auc\")\n#> # A tibble: 42 × 9\n#> wflow_id .config .metric mean std_err n preprocessor model rank\n#> \n#> 1 age_bt Preprocessor1_… roc_auc 0.605 0.00424 10 recipe boos… 1\n#> 2 basic_bt Preprocessor1_… roc_auc 0.605 0.00423 10 recipe boos… 2\n#> 3 age_bt Preprocessor1_… roc_auc 0.604 0.00378 10 recipe boos… 3\n#> 4 age_bt Preprocessor1_… roc_auc 0.603 0.00421 10 recipe boos… 4\n#> 5 basic_bt Preprocessor1_… roc_auc 0.603 0.00410 10 recipe boos… 5\n#> 6 basic_bt Preprocessor1_… roc_auc 0.603 0.00436 10 recipe boos… 6\n#> 7 age_bt Preprocessor1_… roc_auc 0.602 0.00372 10 recipe boos… 7\n#> 8 basic_nn Preprocessor1_… roc_auc 0.602 0.00403 10 recipe bag_… 8\n#> 9 age_nn Preprocessor1_… roc_auc 0.600 0.00431 10 recipe bag_… 9\n#> 10 age_nn Preprocessor1_… roc_auc 0.600 0.00412 10 recipe bag_… 10\n#> # ℹ 32 more rows\n```\n:::\n\n\nAlmost all of the most performant model configurations arise from the boosted tree modeling workflow. Let's examine the results for specifically the modeling workflow that encodes age as a number more thoroughly:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nautoplot(wflow_set_fit, id = \"age_bt\")\n```\n\n::: {.cell-output-display}\n![](figs/autoplot-age-bt-1.svg){fig-align='center' width=70%}\n:::\n:::\n\n\nThe learning rate `learn_rate` seems to have a more pronounced effect on the results metrics than the number of randomly selected predictors `mtry`. As before, we see that the most performant models with respect to `roc_auc()` also tend to be the most fair according to our fairness metrics. Further, the values of each of the fairness metrics plotted above seem highly correlated. \n\nFrom the perspective of a practitioner hoping to satisfy various stakeholders, the fact that these metrics are highly correlated makes the model selection process much easier. We can choose one fairness metric that we'd like to optimize for, and likely end up with a near-optimal configuration for the other metrics as a byproduct.\n\n:::callout-note\nIn machine learning fairness, \"impossibility theorems\" show that fairness definitions \"are not mathematically or morally compatible in general\" [@mitchell2021]. More concretely, unless we live in a world with no inequality, there is no way to satisfy many definitions of fairness at once. However, recent research emphasizes that near-fairness among more limited sets of metrics, like the three we've used here, is both possible and relevant [@bell2023].\n:::\n\nTo choose a model that performs well both with respect to a typical performance metric like `roc_auc()` and the fairness metrics we've chosen, we will make use of [desirability functions](https://www.tidyverse.org/blog/2023/05/desirability2/), which allow us to optimize based on multiple metrics at once.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nbest_params <-\n # extract the tuning results for the boosted tree model\n extract_workflow_set_result(wflow_set_fit, \"age_bt\") |>\n # collect the metrics associated with it\n collect_metrics() |>\n # pivot the metrics so that each is in a column\n pivot_wider(\n id_cols = c(mtry, learn_rate), \n names_from = .metric, \n values_from = mean\n ) |>\n mutate(\n # higher roc values are better; detect max and min from the data\n d_roc = d_max(roc_auc, use_data = TRUE),\n # lower equalized odds are better; detect max and min from the data\n d_e_odds = d_min(equalized_odds, use_data = TRUE),\n # compute overall desirability based on d_roc and d_e_odds\n d_overall = d_overall(across(starts_with(\"d_\")))\n ) |>\n # pick the model with the highest desirability value\n slice_max(d_overall)\n```\n:::\n\n\nThe result is a tibble giving the parameter values that resulted in the best model:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nbest_params\n#> # A tibble: 1 × 10\n#> mtry learn_rate accuracy demographic_parity equal_opportunity equalized_odds\n#> \n#> 1 7 0.00456 0.912 0 0 0\n#> # ℹ 4 more variables: roc_auc , d_roc , d_e_odds ,\n#> # d_overall \n```\n:::\n\n\nWe can use that tibble to finalize a workflow that we'll use to generate our final model fit:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nfinal_model_config <-\n extract_workflow(wflow_set_fit, \"age_bt\") |>\n finalize_workflow(best_params)\n```\n:::\n\n\nFinally, generating our final model fit:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nfinal_model <- \n last_fit(final_model_config, readmission_splits, metrics = m_set)\n```\n:::\n\n\n\n\nWe can see the metrics associated with the final fit using `collect_metrics()`, just as with a tuning result:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ncollect_metrics(final_model)\n#> # A tibble: 5 × 5\n#> .metric .estimator .estimate .by .config \n#> \n#> 1 accuracy binary 0.912 Preprocessor1_Model1\n#> 2 equal_opportunity binary 0 race Preprocessor1_Model1\n#> 3 equalized_odds binary 0 race Preprocessor1_Model1\n#> 4 demographic_parity binary 0 race Preprocessor1_Model1\n#> 5 roc_auc binary 0.602 Preprocessor1_Model1\n```\n:::\n\n\n\n\nThe model we've selected has near-fairness with respect to the set of metrics we've chosen here. The accuracy of the model is 91.16%, quite similar to the accuracy that would result if the model just always predicted a patient would not readmit (91.2%). The `roc_auc()` value 0.602 indicates that the model indeed correctly predicts readmission in some cases, though still has a lot of room for improvement. A further analysis of these models might measure performance using a metric that specifically evaluates predictions on observations from the minority class---as in, patients that did actually readmit---like [`sens()`](https://yardstick.tidymodels.org/reference/sens.html?q=sens#details).\n\nExtracting the model fit from the `last_fit` object:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nfinal_model_fit <- extract_workflow(final_model)\n```\n:::\n\n\nThe `final_model_fit` object is now ready to predict on new data! Models generated with tidymodels are easily versioned, deployed, and monitored using the vetiver framework; learn more about the framework on the [vetiver website](https://vetiver.tidymodels.org/get-started/).\n\nIn this article, we've demonstrated a fairness-oriented modeling analysis. Modeling with fairness in mind is not simply a numerical optimization problem, but a holistic process of examining the moral meanings of fairness in our problem context and how they are represented---or not represented---by various mathematical notions of fairness. \n\nWhile this analysis allowed us to train models that are near-fair with respect to a limited set of fairness metrics, it leaves many questions to be answered about the impacts of using such a model. Among other things:\n\n* We've only evaluated the fairness of the model with `race` in mind. We also have information on the patient's `sex`. Could this model behave unfairly with respect to the patient's `sex`? How about with respect to the intersections of `race` and `sex`? A growing body of research shows us that black women experience pronounced discrimination as patients [@johnson2019; @okoro2020; @gopal2021].\n* The categorizations of `race` and `sex` that we have access to are coarse. Race/ethnicity and sex/gender are richly and diversely experienced, and the limited set of categories that patients are presented with (or unknowingly assigned to) in healthcare contexts are only a proxy for how a patient may experience them. \n* We've used _readmission_ as a proxy for _need for additional care_. What factors might influence whether a patient willingly readmits? Or is admitted to the hospital in the first place? Note, especially, the disproportionate financial burden of healthcare for poorer patients.\n* We don't know how the predictions resulting from this model will be used by practitioners or trusted by patients. The outputs of the model, of course, should be evaluated in conjunction with other evidence collected by a healthcare team throughout a patient's stay. Knowing this, though, will the model's outputs be differently interpreted for different groups in the patient population? For example, would a positive prediction of readmission be considered more seriously as evidence for needed additional care for a white man than for others? Further, given historical exploitation and unethical practice, would protected groups believe that algorithmic recommendation for additional care is likely discriminatory and mistrust the recommendations offered to them? [See @rajkomar2018.] \n\nMachine learning models can both have significant positive impacts on our lives and at the same time cause significant harms. Given the tremendous reach of these models in our society, efforts to include fairness as a criteria for evaluating machine learning models are as necessary as ever.\n\n## Session information {#session-info}\n\n\n::: {.cell layout-align=\"center\"}\n\n```\n#> ─ Session info ─────────────────────────────────────────────────────\n#> version R version 4.6.0 (2026-04-24)\n#> language (EN)\n#> pandoc 3.1.3\n#> quarto 1.9.37\n#> \n#> ─ Packages ─────────────────────────────────────────────────────────\n#> package version date (UTC)\n#> baguette 1.1.0 2025-01-28\n#> broom 1.0.13 2026-05-14\n#> desirability2 0.2.0 2025-08-22\n#> dials 1.4.3 2026-04-11\n#> dplyr 1.2.1 2026-04-03\n#> GGally 2.4.0 2025-08-23\n#> ggplot2 4.0.3 2026-04-22\n#> infer 1.1.0 2025-12-18\n#> parsnip 1.6.0 2026-05-14\n#> purrr 1.2.2 2026-04-10\n#> readmission 0.1.0 2023-12-07\n#> recipes 1.3.2 2026-04-02\n#> rlang 1.2.0 2026-04-06\n#> rsample 1.3.2 2026-01-30\n#> tibble 3.3.1 2026-01-11\n#> tidymodels 1.5.0 2026-04-23\n#> tune 2.1.0 2026-04-17\n#> workflows 1.3.0 2025-08-27\n#> yardstick 1.4.0 2026-04-07\n#> \n#> ────────────────────────────────────────────────────────────────────\n```\n:::\n\n", "supporting": [], "filters": [ "rmarkdown/pagebreak.lua" diff --git a/_freeze/learn/work/nested-resampling/index/execute-results/html.json b/_freeze/learn/work/nested-resampling/index/execute-results/html.json index 727b8b8d..d2c27d63 100644 --- a/_freeze/learn/work/nested-resampling/index/execute-results/html.json +++ b/_freeze/learn/work/nested-resampling/index/execute-results/html.json @@ -1,8 +1,8 @@ { - "hash": "eaf839a184c1acec6a22c5e013bcc26e", + "hash": "7adeb415798511d93e4ef9acf565685c", "result": { "engine": "knitr", - "markdown": "---\ntitle: \"Nested resampling\"\ncategories:\n - SVMs\ntype: learn-subsection\nweight: 2\ndescription: | \n Estimate the best hyperparameters for a model using nested resampling.\ntoc: true\ntoc-depth: 2\nr-packages:\n - tidymodels\n - scales\n - mlbench\n - kernlab\n - furrr\ninclude-after-body: ../../../html/resources.html\n---\n\n\n\n\n\n## Introduction\n\nTo use code in this article, you will need to install the following packages: furrr, kernlab, mlbench, scales, and tidymodels.\n\nIn this article, we discuss an alternative method for evaluating and tuning models, called [nested resampling](https://scholar.google.com/scholar?hl=en&as_sdt=0%2C7&q=%22nested+resampling%22+inner+outer&btnG=). While it is more computationally taxing and challenging to implement than other resampling methods, it has the potential to produce better estimates of model performance.\n\n## Resampling models\n\nA typical scheme for splitting the data when developing a predictive model is to create an initial split of the data into a training and test set. If resampling is used, it is executed on the training set. A series of binary splits is created. In rsample, we use the term *analysis set* for the data that are used to fit the model and the term *assessment set* for the set used to compute performance:\n\n\n::: {.cell layout-align=\"center\"}\n::: {.cell-output-display}\n![](img/resampling.svg){fig-align='center' width=70%}\n:::\n:::\n\n\nA common method for tuning models is [grid search](/learn/work/tune-svm/) where a candidate set of tuning parameters is created. The full set of models for every combination of the tuning parameter grid and the resamples is fitted. Each time, the assessment data are used to measure performance and the average value is determined for each tuning parameter.\n\nThe potential problem is that once we pick the tuning parameter associated with the best performance, this performance value is usually quoted as the performance of the model. There is serious potential for *optimization bias* since we use the same data to tune the model and to assess performance. This would result in an optimistic estimate of performance.\n\nNested resampling uses an additional layer of resampling that separates the tuning activities from the process used to estimate the efficacy of the model. An *outer* resampling scheme is used and, for every split in the outer resample, another full set of resampling splits are created on the original analysis set. For example, if 10-fold cross-validation is used on the outside and 5-fold cross-validation on the inside, a total of 500 models will be fit. The parameter tuning will be conducted 10 times and the best parameters are determined from the average of the 5 assessment sets. This process occurs 10 times.\n\nOnce the tuning results are complete, a model is fit to each of the outer resampling splits using the best parameter associated with that resample. The average of the outer method's assessment sets are a unbiased estimate of the model.\n\nWe will simulate some regression data to illustrate the methods. The mlbench package has a function `mlbench::mlbench.friedman1()` that can simulate a complex regression data structure from the [original MARS publication](https://scholar.google.com/scholar?hl=en&q=%22Multivariate+adaptive+regression+splines%22&btnG=&as_sdt=1%2C7&as_sdtp=). A training set size of 100 data points are generated as well as a large set that will be used to characterize how well the resampling procedure performed.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(mlbench)\nsim_data <- function(n) {\n tmp <- mlbench.friedman1(n, sd = 1)\n tmp <- cbind(tmp$x, tmp$y)\n tmp <- as.data.frame(tmp)\n names(tmp)[ncol(tmp)] <- \"y\"\n tmp\n}\n\nset.seed(9815)\ntrain_dat <- sim_data(100)\nlarge_dat <- sim_data(10^5)\n```\n:::\n\n\n## Nested resampling\n\nTo get started, the types of resampling methods need to be specified. This isn't a large data set, so 5 repeats of 10-fold cross validation will be used as the *outer* resampling method for generating the estimate of overall performance. To tune the model, it would be good to have precise estimates for each of the values of the tuning parameter so let's use 25 iterations of the bootstrap. This means that there will eventually be `5 * 10 * 25 = 1250` models that are fit to the data *per tuning parameter*. These models will be discarded once the performance of the model has been quantified.\n\nTo create the tibble with the resampling specifications:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(tidymodels)\nresults <- nested_cv(train_dat, \n outside = vfold_cv(repeats = 5), \n inside = bootstraps(times = 25))\nresults\n#> # Nested resampling:\n#> # outer: 10-fold cross-validation repeated 5 times\n#> # inner: Bootstrap sampling\n#> # A tibble: 50 × 4\n#> splits id id2 inner_resamples\n#> \n#> 1 Repeat1 Fold01 \n#> 2 Repeat1 Fold02 \n#> 3 Repeat1 Fold03 \n#> 4 Repeat1 Fold04 \n#> 5 Repeat1 Fold05 \n#> 6 Repeat1 Fold06 \n#> 7 Repeat1 Fold07 \n#> 8 Repeat1 Fold08 \n#> 9 Repeat1 Fold09 \n#> 10 Repeat1 Fold10 \n#> # ℹ 40 more rows\n```\n:::\n\n\nThe splitting information for each resample is contained in the `split` objects. Focusing on the second fold of the first repeat:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nresults$splits[[2]]\n#> \n#> <90/10/100>\n```\n:::\n\n\n`<90/10/100>` indicates the number of observations in the analysis set, assessment set, and the original data.\n\nEach element of `inner_resamples` has its own tibble with the bootstrapping splits.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nresults$inner_resamples[[5]]\n#> # Bootstrap sampling \n#> # A tibble: 25 × 2\n#> splits id \n#> \n#> 1 Bootstrap01\n#> 2 Bootstrap02\n#> 3 Bootstrap03\n#> 4 Bootstrap04\n#> 5 Bootstrap05\n#> 6 Bootstrap06\n#> 7 Bootstrap07\n#> 8 Bootstrap08\n#> 9 Bootstrap09\n#> 10 Bootstrap10\n#> # ℹ 15 more rows\n```\n:::\n\n\nThese are self-contained, meaning that the bootstrap sample is aware that it is a sample of a specific 90% of the data:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nresults$inner_resamples[[5]]$splits[[1]]\n#> \n#> <90/31/90>\n```\n:::\n\n\nTo start, we need to define how the model will be created and measured. Let's use a radial basis support vector machine model via the function `kernlab::ksvm`. This model is generally considered to have *two* tuning parameters: the SVM cost value and the kernel parameter `sigma`. For illustration purposes here, only the cost value will be tuned and the function `kernlab::sigest` will be used to estimate `sigma` during each model fit. This is automatically done by `ksvm`.\n\nAfter the model is fit to the analysis set, the root-mean squared error (RMSE) is computed on the assessment set. **One important note:** for this model, it is critical to center and scale the predictors before computing dot products. We don't do this operation here because `mlbench.friedman1` simulates all of the predictors to be standardized uniform random variables.\n\nOur function to fit the model and compute the RMSE is:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(kernlab)\n\n# `object` will be an `rsplit` object from our `results` tibble\n# `cost` is the tuning parameter\nsvm_rmse <- function(object, cost = 1) {\n set.seed(1234)\n y_col <- ncol(object$data)\n mod <- \n svm_rbf(mode = \"regression\", cost = cost) |> \n set_engine(\"kernlab\") |> \n fit(y ~ ., data = analysis(object))\n \n holdout_pred <- \n predict(mod, assessment(object) |> dplyr::select(-y)) |> \n bind_cols(assessment(object) |> dplyr::select(y))\n rmse(holdout_pred, truth = y, estimate = .pred)$.estimate\n}\n\n# In some case, we want to parameterize the function over the tuning parameter:\nrmse_wrapper <- function(cost, object) svm_rmse(object, cost)\n```\n:::\n\n\nFor the nested resampling, a model needs to be fit for each tuning parameter and each bootstrap split. To do this, create a wrapper:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\n# `object` will be an `rsplit` object for the bootstrap samples\ntune_over_cost <- function(object) {\n tibble(cost = 2 ^ seq(-2, 8, by = 1)) |> \n mutate(RMSE = map_dbl(cost, rmse_wrapper, object = object))\n}\n```\n:::\n\n\nSince this will be called across the set of outer cross-validation splits, another wrapper is required:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\n# `object` is an `rsplit` object in `results$inner_resamples` \nsummarize_tune_results <- function(object) {\n # Return row-bound tibble that has the 25 bootstrap results\n map_df(object$splits, tune_over_cost) |>\n # For each value of the tuning parameter, compute the \n # average RMSE which is the inner bootstrap estimate. \n group_by(cost) |>\n summarize(mean_RMSE = mean(RMSE, na.rm = TRUE),\n n = length(RMSE),\n .groups = \"drop\")\n}\n```\n:::\n\n\nNow that those functions are defined, we can execute all the inner resampling loops:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ntuning_results <- map(results$inner_resamples, summarize_tune_results) \n```\n:::\n\n\nAlternatively, since these computations can be run in parallel, we can use the furrr package. Instead of using `map()`, the function `future_map()` parallelizes the iterations using the [future package](https://cran.r-project.org/web/packages/future/vignettes/future-1-overview.html). The `multisession` plan uses the local cores to process the inner resampling loop. The end results are the same as the sequential computations.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(furrr)\nplan(multisession)\n\ntuning_results <- future_map(results$inner_resamples, summarize_tune_results) \n```\n:::\n\n\nThe object `tuning_results` is a list of data frames for each of the 50 outer resamples.\n\nLet's make a plot of the averaged results to see what the relationship is between the RMSE and the tuning parameters for each of the inner bootstrapping operations:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(scales)\n\npooled_inner <- tuning_results |> bind_rows()\n\nbest_cost <- function(dat) dat[which.min(dat$mean_RMSE),]\n\np <- \n ggplot(pooled_inner, aes(x = cost, y = mean_RMSE)) + \n scale_x_continuous(trans = 'log2') +\n xlab(\"SVM Cost\") + ylab(\"Inner RMSE\")\n\nfor (i in 1:length(tuning_results))\n p <- p +\n geom_line(data = tuning_results[[i]], alpha = .2) +\n geom_point(data = best_cost(tuning_results[[i]]), pch = 16, alpha = 3/4)\n\np <- p + geom_smooth(data = pooled_inner, se = FALSE)\np\n```\n\n::: {.cell-output-display}\n![](figs/rmse-plot-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nEach gray line is a separate bootstrap resampling curve created from a different 90% of the data. The blue line is a LOESS smooth of all the results pooled together.\n\nTo determine the best parameter estimate for each of the outer resampling iterations:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ncost_vals <- \n tuning_results |> \n map_df(best_cost) |> \n select(cost)\n\nresults <- \n bind_cols(results, cost_vals) |> \n mutate(cost = factor(cost, levels = paste(2 ^ seq(-2, 8, by = 1))))\n\nggplot(results, aes(x = cost)) + \n geom_bar() + \n xlab(\"SVM Cost\") + \n scale_x_discrete(drop = FALSE)\n```\n\n::: {.cell-output-display}\n![](figs/choose-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nMost of the resamples produced an optimal cost value of 2.0, but the distribution is right-skewed due to the flat trend in the resampling profile once the cost value becomes 10 or larger.\n\nNow that we have these estimates, we can compute the outer resampling results for each of the 50 splits using the corresponding tuning parameter value:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nresults <- \n results |> \n mutate(RMSE = map2_dbl(splits, cost, svm_rmse))\n\nsummary(results$RMSE)\n#> Min. 1st Qu. Median Mean 3rd Qu. Max. \n#> 1.676 2.090 2.589 2.682 3.220 4.228\n```\n:::\n\n\nThe estimated RMSE for the model tuning process is 2.68.\n\nWhat is the RMSE estimate for the non-nested procedure when only the outer resampling method is used? For each cost value in the tuning grid, 50 SVM models are fit and their RMSE values are averaged. The table of cost values and mean RMSE estimates is used to determine the best cost value. The associated RMSE is the biased estimate.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nnot_nested <-\n map(results$splits, tune_over_cost) |>\n bind_rows()\n\nouter_summary <- not_nested |> \n group_by(cost) |> \n summarize(outer_RMSE = mean(RMSE), n = length(RMSE))\n\nouter_summary\n#> # A tibble: 11 × 3\n#> cost outer_RMSE n\n#> \n#> 1 0.25 3.54 50\n#> 2 0.5 3.11 50\n#> 3 1 2.78 50\n#> 4 2 2.63 50\n#> 5 4 2.66 50\n#> 6 8 2.78 50\n#> 7 16 2.84 50\n#> 8 32 2.84 50\n#> 9 64 2.84 50\n#> 10 128 2.84 50\n#> 11 256 2.84 50\n\nggplot(outer_summary, aes(x = cost, y = outer_RMSE)) + \n geom_point() + \n geom_line() + \n scale_x_continuous(trans = 'log2') +\n xlab(\"SVM Cost\") + ylab(\"RMSE\")\n```\n\n::: {.cell-output-display}\n![](figs/not-nested-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nThe non-nested procedure estimates the RMSE to be 2.63. Both estimates are fairly close.\n\nThe approximately true RMSE for an SVM model with a cost value of 2.0 can be approximated with the large sample that was simulated at the beginning.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nfinalModel <- ksvm(y ~ ., data = train_dat, C = 2)\nlarge_pred <- predict(finalModel, large_dat[, -ncol(large_dat)])\nsqrt(mean((large_dat$y - large_pred) ^ 2, na.rm = TRUE))\n#> [1] 2.695091\n```\n:::\n\n\nThe nested procedure produces a closer estimate to the approximate truth but the non-nested estimate is very similar.\n\n## Session information {#session-info}\n\n\n::: {.cell layout-align=\"center\"}\n\n```\n#> ─ Session info ─────────────────────────────────────────────────────\n#> version R version 4.6.0 (2026-04-24)\n#> language (EN)\n#> pandoc 3.1.3\n#> quarto 1.9.37\n#> \n#> ─ Packages ─────────────────────────────────────────────────────────\n#> package version date (UTC)\n#> broom 1.0.13 2026-05-14\n#> dials 1.4.3 2026-04-11\n#> dplyr 1.2.1 2026-04-03\n#> furrr 0.4.0 2026-03-31\n#> ggplot2 4.0.3 2026-04-22\n#> infer 1.1.0 2025-12-18\n#> kernlab 0.9-33 2024-08-13\n#> mlbench 2.1-8 2026-03-26\n#> parsnip 1.6.0 2026-05-14\n#> purrr 1.2.2 2026-04-10\n#> recipes 1.3.2 2026-04-02\n#> rlang 1.2.0 2026-04-06\n#> rsample 1.3.2 2026-01-30\n#> scales 1.4.0 2025-04-24\n#> tibble 3.3.1 2026-01-11\n#> tidymodels 1.5.0 2026-04-23\n#> tune 2.1.0 2026-04-17\n#> workflows 1.3.0 2025-08-27\n#> yardstick 1.4.0 2026-04-07\n#> \n#> ────────────────────────────────────────────────────────────────────\n```\n:::\n\n", + "markdown": "---\ntitle: \"Nested resampling\"\ncategories:\n - tuning and workflows\n - SVMs\ntype: learn-subsection\nweight: 2\ndescription: | \n Estimate the best hyperparameters for a model using nested resampling.\ntoc: true\ntoc-depth: 2\nr-packages:\n - tidymodels\n - scales\n - mlbench\n - kernlab\n - furrr\ninclude-after-body: ../../../html/resources.html\n---\n\n\n\n\n\n## Introduction\n\nTo use code in this article, you will need to install the following packages: furrr, kernlab, mlbench, scales, and tidymodels.\n\nIn this article, we discuss an alternative method for evaluating and tuning models, called [nested resampling](https://scholar.google.com/scholar?hl=en&as_sdt=0%2C7&q=%22nested+resampling%22+inner+outer&btnG=). While it is more computationally taxing and challenging to implement than other resampling methods, it has the potential to produce better estimates of model performance.\n\n## Resampling models\n\nA typical scheme for splitting the data when developing a predictive model is to create an initial split of the data into a training and test set. If resampling is used, it is executed on the training set. A series of binary splits is created. In rsample, we use the term *analysis set* for the data that are used to fit the model and the term *assessment set* for the set used to compute performance:\n\n\n::: {.cell layout-align=\"center\"}\n::: {.cell-output-display}\n![](img/resampling.svg){fig-align='center' width=70%}\n:::\n:::\n\n\nA common method for tuning models is [grid search](/learn/work/tune-svm/) where a candidate set of tuning parameters is created. The full set of models for every combination of the tuning parameter grid and the resamples is fitted. Each time, the assessment data are used to measure performance and the average value is determined for each tuning parameter.\n\nThe potential problem is that once we pick the tuning parameter associated with the best performance, this performance value is usually quoted as the performance of the model. There is serious potential for *optimization bias* since we use the same data to tune the model and to assess performance. This would result in an optimistic estimate of performance.\n\nNested resampling uses an additional layer of resampling that separates the tuning activities from the process used to estimate the efficacy of the model. An *outer* resampling scheme is used and, for every split in the outer resample, another full set of resampling splits are created on the original analysis set. For example, if 10-fold cross-validation is used on the outside and 5-fold cross-validation on the inside, a total of 500 models will be fit. The parameter tuning will be conducted 10 times and the best parameters are determined from the average of the 5 assessment sets. This process occurs 10 times.\n\nOnce the tuning results are complete, a model is fit to each of the outer resampling splits using the best parameter associated with that resample. The average of the outer method's assessment sets are a unbiased estimate of the model.\n\nWe will simulate some regression data to illustrate the methods. The mlbench package has a function `mlbench::mlbench.friedman1()` that can simulate a complex regression data structure from the [original MARS publication](https://scholar.google.com/scholar?hl=en&q=%22Multivariate+adaptive+regression+splines%22&btnG=&as_sdt=1%2C7&as_sdtp=). A training set size of 100 data points are generated as well as a large set that will be used to characterize how well the resampling procedure performed.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(mlbench)\nsim_data <- function(n) {\n tmp <- mlbench.friedman1(n, sd = 1)\n tmp <- cbind(tmp$x, tmp$y)\n tmp <- as.data.frame(tmp)\n names(tmp)[ncol(tmp)] <- \"y\"\n tmp\n}\n\nset.seed(9815)\ntrain_dat <- sim_data(100)\nlarge_dat <- sim_data(10^5)\n```\n:::\n\n\n## Nested resampling\n\nTo get started, the types of resampling methods need to be specified. This isn't a large data set, so 5 repeats of 10-fold cross validation will be used as the *outer* resampling method for generating the estimate of overall performance. To tune the model, it would be good to have precise estimates for each of the values of the tuning parameter so let's use 25 iterations of the bootstrap. This means that there will eventually be `5 * 10 * 25 = 1250` models that are fit to the data *per tuning parameter*. These models will be discarded once the performance of the model has been quantified.\n\nTo create the tibble with the resampling specifications:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(tidymodels)\nresults <- nested_cv(train_dat, \n outside = vfold_cv(repeats = 5), \n inside = bootstraps(times = 25))\nresults\n#> # Nested resampling:\n#> # outer: 10-fold cross-validation repeated 5 times\n#> # inner: Bootstrap sampling\n#> # A tibble: 50 × 4\n#> splits id id2 inner_resamples\n#> \n#> 1 Repeat1 Fold01 \n#> 2 Repeat1 Fold02 \n#> 3 Repeat1 Fold03 \n#> 4 Repeat1 Fold04 \n#> 5 Repeat1 Fold05 \n#> 6 Repeat1 Fold06 \n#> 7 Repeat1 Fold07 \n#> 8 Repeat1 Fold08 \n#> 9 Repeat1 Fold09 \n#> 10 Repeat1 Fold10 \n#> # ℹ 40 more rows\n```\n:::\n\n\nThe splitting information for each resample is contained in the `split` objects. Focusing on the second fold of the first repeat:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nresults$splits[[2]]\n#> \n#> <90/10/100>\n```\n:::\n\n\n`<90/10/100>` indicates the number of observations in the analysis set, assessment set, and the original data.\n\nEach element of `inner_resamples` has its own tibble with the bootstrapping splits.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nresults$inner_resamples[[5]]\n#> # Bootstrap sampling \n#> # A tibble: 25 × 2\n#> splits id \n#> \n#> 1 Bootstrap01\n#> 2 Bootstrap02\n#> 3 Bootstrap03\n#> 4 Bootstrap04\n#> 5 Bootstrap05\n#> 6 Bootstrap06\n#> 7 Bootstrap07\n#> 8 Bootstrap08\n#> 9 Bootstrap09\n#> 10 Bootstrap10\n#> # ℹ 15 more rows\n```\n:::\n\n\nThese are self-contained, meaning that the bootstrap sample is aware that it is a sample of a specific 90% of the data:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nresults$inner_resamples[[5]]$splits[[1]]\n#> \n#> <90/31/90>\n```\n:::\n\n\nTo start, we need to define how the model will be created and measured. Let's use a radial basis support vector machine model via the function `kernlab::ksvm`. This model is generally considered to have *two* tuning parameters: the SVM cost value and the kernel parameter `sigma`. For illustration purposes here, only the cost value will be tuned and the function `kernlab::sigest` will be used to estimate `sigma` during each model fit. This is automatically done by `ksvm`.\n\nAfter the model is fit to the analysis set, the root-mean squared error (RMSE) is computed on the assessment set. **One important note:** for this model, it is critical to center and scale the predictors before computing dot products. We don't do this operation here because `mlbench.friedman1` simulates all of the predictors to be standardized uniform random variables.\n\nOur function to fit the model and compute the RMSE is:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(kernlab)\n\n# `object` will be an `rsplit` object from our `results` tibble\n# `cost` is the tuning parameter\nsvm_rmse <- function(object, cost = 1) {\n set.seed(1234)\n y_col <- ncol(object$data)\n mod <- \n svm_rbf(mode = \"regression\", cost = cost) |> \n set_engine(\"kernlab\") |> \n fit(y ~ ., data = analysis(object))\n \n holdout_pred <- \n predict(mod, assessment(object) |> dplyr::select(-y)) |> \n bind_cols(assessment(object) |> dplyr::select(y))\n rmse(holdout_pred, truth = y, estimate = .pred)$.estimate\n}\n\n# In some case, we want to parameterize the function over the tuning parameter:\nrmse_wrapper <- function(cost, object) svm_rmse(object, cost)\n```\n:::\n\n\nFor the nested resampling, a model needs to be fit for each tuning parameter and each bootstrap split. To do this, create a wrapper:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\n# `object` will be an `rsplit` object for the bootstrap samples\ntune_over_cost <- function(object) {\n tibble(cost = 2 ^ seq(-2, 8, by = 1)) |> \n mutate(RMSE = map_dbl(cost, rmse_wrapper, object = object))\n}\n```\n:::\n\n\nSince this will be called across the set of outer cross-validation splits, another wrapper is required:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\n# `object` is an `rsplit` object in `results$inner_resamples` \nsummarize_tune_results <- function(object) {\n # Return row-bound tibble that has the 25 bootstrap results\n map_df(object$splits, tune_over_cost) |>\n # For each value of the tuning parameter, compute the \n # average RMSE which is the inner bootstrap estimate. \n group_by(cost) |>\n summarize(mean_RMSE = mean(RMSE, na.rm = TRUE),\n n = length(RMSE),\n .groups = \"drop\")\n}\n```\n:::\n\n\nNow that those functions are defined, we can execute all the inner resampling loops:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ntuning_results <- map(results$inner_resamples, summarize_tune_results) \n```\n:::\n\n\nAlternatively, since these computations can be run in parallel, we can use the furrr package. Instead of using `map()`, the function `future_map()` parallelizes the iterations using the [future package](https://cran.r-project.org/web/packages/future/vignettes/future-1-overview.html). The `multisession` plan uses the local cores to process the inner resampling loop. The end results are the same as the sequential computations.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(furrr)\nplan(multisession)\n\ntuning_results <- future_map(results$inner_resamples, summarize_tune_results) \n```\n:::\n\n\nThe object `tuning_results` is a list of data frames for each of the 50 outer resamples.\n\nLet's make a plot of the averaged results to see what the relationship is between the RMSE and the tuning parameters for each of the inner bootstrapping operations:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(scales)\n\npooled_inner <- tuning_results |> bind_rows()\n\nbest_cost <- function(dat) dat[which.min(dat$mean_RMSE),]\n\np <- \n ggplot(pooled_inner, aes(x = cost, y = mean_RMSE)) + \n scale_x_continuous(trans = 'log2') +\n xlab(\"SVM Cost\") + ylab(\"Inner RMSE\")\n\nfor (i in 1:length(tuning_results))\n p <- p +\n geom_line(data = tuning_results[[i]], alpha = .2) +\n geom_point(data = best_cost(tuning_results[[i]]), pch = 16, alpha = 3/4)\n\np <- p + geom_smooth(data = pooled_inner, se = FALSE)\np\n```\n\n::: {.cell-output-display}\n![](figs/rmse-plot-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nEach gray line is a separate bootstrap resampling curve created from a different 90% of the data. The blue line is a LOESS smooth of all the results pooled together.\n\nTo determine the best parameter estimate for each of the outer resampling iterations:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ncost_vals <- \n tuning_results |> \n map_df(best_cost) |> \n select(cost)\n\nresults <- \n bind_cols(results, cost_vals) |> \n mutate(cost = factor(cost, levels = paste(2 ^ seq(-2, 8, by = 1))))\n\nggplot(results, aes(x = cost)) + \n geom_bar() + \n xlab(\"SVM Cost\") + \n scale_x_discrete(drop = FALSE)\n```\n\n::: {.cell-output-display}\n![](figs/choose-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nMost of the resamples produced an optimal cost value of 2.0, but the distribution is right-skewed due to the flat trend in the resampling profile once the cost value becomes 10 or larger.\n\nNow that we have these estimates, we can compute the outer resampling results for each of the 50 splits using the corresponding tuning parameter value:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nresults <- \n results |> \n mutate(RMSE = map2_dbl(splits, cost, svm_rmse))\n\nsummary(results$RMSE)\n#> Min. 1st Qu. Median Mean 3rd Qu. Max. \n#> 1.676 2.090 2.589 2.682 3.220 4.228\n```\n:::\n\n\nThe estimated RMSE for the model tuning process is 2.68.\n\nWhat is the RMSE estimate for the non-nested procedure when only the outer resampling method is used? For each cost value in the tuning grid, 50 SVM models are fit and their RMSE values are averaged. The table of cost values and mean RMSE estimates is used to determine the best cost value. The associated RMSE is the biased estimate.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nnot_nested <-\n map(results$splits, tune_over_cost) |>\n bind_rows()\n\nouter_summary <- not_nested |> \n group_by(cost) |> \n summarize(outer_RMSE = mean(RMSE), n = length(RMSE))\n\nouter_summary\n#> # A tibble: 11 × 3\n#> cost outer_RMSE n\n#> \n#> 1 0.25 3.54 50\n#> 2 0.5 3.11 50\n#> 3 1 2.78 50\n#> 4 2 2.63 50\n#> 5 4 2.66 50\n#> 6 8 2.78 50\n#> 7 16 2.84 50\n#> 8 32 2.84 50\n#> 9 64 2.84 50\n#> 10 128 2.84 50\n#> 11 256 2.84 50\n\nggplot(outer_summary, aes(x = cost, y = outer_RMSE)) + \n geom_point() + \n geom_line() + \n scale_x_continuous(trans = 'log2') +\n xlab(\"SVM Cost\") + ylab(\"RMSE\")\n```\n\n::: {.cell-output-display}\n![](figs/not-nested-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nThe non-nested procedure estimates the RMSE to be 2.63. Both estimates are fairly close.\n\nThe approximately true RMSE for an SVM model with a cost value of 2.0 can be approximated with the large sample that was simulated at the beginning.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nfinalModel <- ksvm(y ~ ., data = train_dat, C = 2)\nlarge_pred <- predict(finalModel, large_dat[, -ncol(large_dat)])\nsqrt(mean((large_dat$y - large_pred) ^ 2, na.rm = TRUE))\n#> [1] 2.695091\n```\n:::\n\n\nThe nested procedure produces a closer estimate to the approximate truth but the non-nested estimate is very similar.\n\n## Session information {#session-info}\n\n\n::: {.cell layout-align=\"center\"}\n\n```\n#> ─ Session info ─────────────────────────────────────────────────────\n#> version R version 4.6.0 (2026-04-24)\n#> language (EN)\n#> pandoc 3.1.3\n#> quarto 1.9.37\n#> \n#> ─ Packages ─────────────────────────────────────────────────────────\n#> package version date (UTC)\n#> broom 1.0.13 2026-05-14\n#> dials 1.4.3 2026-04-11\n#> dplyr 1.2.1 2026-04-03\n#> furrr 0.4.0 2026-03-31\n#> ggplot2 4.0.3 2026-04-22\n#> infer 1.1.0 2025-12-18\n#> kernlab 0.9-33 2024-08-13\n#> mlbench 2.1-8 2026-03-26\n#> parsnip 1.6.0 2026-05-14\n#> purrr 1.2.2 2026-04-10\n#> recipes 1.3.2 2026-04-02\n#> rlang 1.2.0 2026-04-06\n#> rsample 1.3.2 2026-01-30\n#> scales 1.4.0 2025-04-24\n#> tibble 3.3.1 2026-01-11\n#> tidymodels 1.5.0 2026-04-23\n#> tune 2.1.0 2026-04-17\n#> workflows 1.3.0 2025-08-27\n#> yardstick 1.4.0 2026-04-07\n#> \n#> ────────────────────────────────────────────────────────────────────\n```\n:::\n\n", "supporting": [], "filters": [ "rmarkdown/pagebreak.lua" diff --git a/_freeze/learn/work/sparse-matrix/index/execute-results/html.json b/_freeze/learn/work/sparse-matrix/index/execute-results/html.json index e47477f3..bbde6b19 100644 --- a/_freeze/learn/work/sparse-matrix/index/execute-results/html.json +++ b/_freeze/learn/work/sparse-matrix/index/execute-results/html.json @@ -1,8 +1,8 @@ { - "hash": "9f9d18ced7826641d407c28ad0a259c7", + "hash": "87c83408f048a3a4a30d5ecb71fb5b42", "result": { "engine": "knitr", - "markdown": "---\ntitle: \"Model tuning using a sparse matrix\"\ncategories:\n - tuning\n - classification\n - sparse data\ntype: learn-subsection\nweight: 1\ndescription: | \n Fitting a model using tidymodels with a sparse matrix as the data.\ntoc: true\ntoc-depth: 2\nr-packages:\n - tidymodels\n - sparsevctrs\ninclude-after-body: ../../../html/resources.html\n---\n\n\n \n\n\n## Introduction\n\nTo use code in this article, you will need to install the following packages: sparsevctrs and tidymodels.\n\nThis article demonstrates how we can use a sparse matrix in tidymodels.\n\nWe use the term **sparse data** to denote a data set that contains a lot of 0s. Such data is commonly seen as a result of dealing with categorical variables, text tokenization, or graph data sets. The word sparse describes how the information is packed. Namely, it represents the presence of a lot of zeroes. For some tasks, we can easily get above 99% percent of 0s in the predictors. \n\nThe reason we use sparse data as a construct is that it is a lot more memory efficient to store the positions and values of the non-zero entries than to encode all the values. One could think of this as a compression, but one that is done such that data tasks are still fast. The following vector requires 25 values to store it normally (dense representation). This representation will be referred to as a **dense vector**.\n\n```r\nc(100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0)\n```\nThe sparse representation of this vector only requires 5 values. 1 value for the length (25), 2 values for the locations of the non-zero values (1, 22), and 2 values for the non-zero values (100, 1). This idea can also be extended to matrices as is done in the Matrix package. Where we instead store the dimensions of the matrix, row indexes, colomn indexes, and the values for the non-zero entries.\n\n## Example data\n\nThe data we will be using in this article is a larger sample of the [small_fine_foods](https://modeldata.tidymodels.org/reference/small_fine_foods.html) data set from the [modeldata](https://modeldata.tidymodels.org) package. The [raw data](https://snap.stanford.edu/data/web-FineFoods.html) was sliced down to 100,000 rows, tokenized, and saved as a sparse matrix. Data has been saved as [reviews.rds](reviews.rds) and the code to generate this data set is found at [generate-data.R](generate-data.R). This file takes up around 1MB compressed, and around 12MB once loaded into R. This data set is encoded as a sparse matrix from the Matrix package; if we were to turn it into a dense matrix, it would take up 3GB.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nreviews <- readr::read_rds(\"reviews.rds\")\nreviews |> head()\n#> 6 x 24818 sparse Matrix of class \"dgCMatrix\"\n#> [[ suppressing 34 column names 'SCORE', 'a', 'all' ... ]]\n#> \n#> 1 1 2 1 3 1 1 2 1 1 1 1 1 1 1 2 1 1 1 1 1 1 1 1 1 1 2 1 2 1 1 1 1 1 1 ......\n#> 2 . . . . . . . . . . . . . . . . . . . . . . . . . . . 2 . . . . . . ......\n#> 3 . 4 . 6 . . . . . . . . . . . 1 5 3 . . . . . . . 2 . . . . . . . . ......\n#> 4 . . . 1 . . . . . . . . 1 1 1 4 1 1 . . . . . . . . . . . . . . . . ......\n#> 5 1 4 . . . . . . . . . . . . . . 1 . . . . . . . . 1 . . . . . . . . ......\n#> 6 . 3 1 2 . . . . . . . . . . . 2 1 1 . . . . . . 4 1 . . . . . . . . ......\n#> \n#> .....suppressing 24784 columns in show(); maybe adjust options(max.print=, width=)\n#> ..............................\n```\n:::\n\n\n## Modeling\n\nWe start by loading tidymodels and the sparsevctrs package. The sparsevctrs package includes some helper functions that will allow us to more easily work with sparse matrices in tidymodels.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(tidymodels)\nlibrary(sparsevctrs)\n```\n:::\n\n\nWhile sparse matrices now work in parsnip, recipes, and workflows directly, we can use rsample's sampling functions as well if we turn it into a tibble. The usual `as_tibble()` would turn the object to a dense representation, greatly expanding the object size. However, sparsevctrs' `coerce_to_sparse_tibble()` will create a tibble with sparse columns, which we call a **sparse tibble**.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nreviews_tbl <- coerce_to_sparse_tibble(reviews)\nreviews_tbl\n#> # A tibble: 15,000 × 24,818\n#> SCORE a all and appreciates be better bought canned dog finicky\n#> \n#> 1 1 2 1 3 1 1 2 1 1 1 1\n#> 2 0 0 0 0 0 0 0 0 0 0 0\n#> 3 0 4 0 6 0 0 0 0 0 0 0\n#> 4 0 0 0 1 0 0 0 0 0 0 0\n#> 5 1 4 0 0 0 0 0 0 0 0 0\n#> 6 0 3 1 2 0 0 0 0 0 0 0\n#> 7 1 1 0 3 0 0 0 0 0 0 0\n#> 8 1 0 0 1 0 0 0 0 0 0 0\n#> 9 1 0 0 1 0 0 0 0 0 0 0\n#> 10 1 1 0 0 0 0 0 0 0 2 0\n#> # ℹ 14,990 more rows\n#> # ℹ 24,807 more variables: food , found , good , have ,\n#> # i , is , it , labrador , like , looks ,\n#> # meat , more , most , my , of , processed ,\n#> # product , products , quality , several , she ,\n#> # smells , stew , than , the , them , this ,\n#> # to , vitality , actually , an , arrived , …\n```\n:::\n\n\nDespite this tibble containing 15,000 rows and a little under 25,000 columns, it only takes up marginally more space than the sparse matrix.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlobstr::obj_size(reviews)\n#> 12.75 MB\nlobstr::obj_size(reviews_tbl)\n#> 18.27 MB\n```\n:::\n\n\nThe outcome `SCORE` is currently encoded as a double, but we want it to be a factor for it to work well with tidymodels, since tidymodels expects outcomes to be factors for classification.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nreviews_tbl <- reviews_tbl |>\n mutate(SCORE = factor(SCORE, levels = c(1, 0), labels = c(\"great\", \"other\")))\n```\n:::\n\n\nSince `reviews_tbl` is now a tibble, we can use `initial_split()` as we usually do.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(1234)\n\nreview_split <- initial_split(reviews_tbl)\nreview_train <- training(review_split)\nreview_test <- testing(review_split)\n\nreview_folds <- vfold_cv(review_train)\n```\n:::\n\n\nNext, we will specify our workflow. Since we are showcasing how sparse data works in tidymodels, we will stick to a simple lasso regression model. These models tend to work well with sparse predictors. `penalty` has been set to be tuned.\n\n::: callout-tip\nAll available models can be found at the [sparse models search](../../../find/sparse/index.qmd).\n:::\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nrec_spec <- recipe(SCORE ~ ., data = review_train)\n\nlm_spec <- logistic_reg(penalty = tune()) |>\n set_engine(\"glmnet\")\n\nwf_spec <- workflow(rec_spec, lm_spec)\n```\n:::\n\n\nWith everything in order, we can now evaluate several different values of `penalty` with `tune_grid()`.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ntune_res <- tune_grid(wf_spec, review_folds)\n```\n:::\n\n\nDespite the size of the data, this code runs quite quickly due to the sparse encoding of the data. Once the tuning process is done, then we can look at the performance for different values of regularization.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nautoplot(tune_res)\n```\n\n::: {.cell-output-display}\n![](figs/autoplot-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nWe can now finalize the workflow and fit the final model on the training data set.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nwf_final <- finalize_workflow(\n wf_spec, \n select_best(tune_res, metric = \"roc_auc\")\n)\n\nwf_fit <- fit(wf_final, review_train)\n```\n:::\n\n\nWith this fitted model, we can now predict with a sparse tibble.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\npredict(wf_fit, review_test)\n#> # A tibble: 3,750 × 1\n#> .pred_class\n#> \n#> 1 other \n#> 2 great \n#> 3 great \n#> 4 great \n#> 5 great \n#> 6 great \n#> 7 great \n#> 8 great \n#> 9 great \n#> 10 great \n#> # ℹ 3,740 more rows\n```\n:::\n\n\n`fit()` and `predict()` both accept sparse matrices as input. However if you want to tune a model with the tune package or perform data splitting with rsample then you will need a tibble, which can be done with `coerce_to_sparse_tibble()`.\n\nThis means that we could technically do predictions on our model directly on the sparse matrix using `predict()`.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\npredict(wf_fit, reviews)\n#> # A tibble: 15,000 × 1\n#> .pred_class\n#> \n#> 1 great \n#> 2 other \n#> 3 great \n#> 4 great \n#> 5 great \n#> 6 other \n#> 7 great \n#> 8 great \n#> 9 great \n#> 10 great \n#> # ℹ 14,990 more rows\n```\n:::\n\n\n## Session information {#session-info}\n\n\n::: {.cell layout-align=\"center\"}\n\n```\n#> ─ Session info ─────────────────────────────────────────────────────\n#> version R version 4.6.0 (2026-04-24)\n#> language (EN)\n#> pandoc 3.1.3\n#> quarto 1.9.37\n#> \n#> ─ Packages ─────────────────────────────────────────────────────────\n#> package version date (UTC)\n#> broom 1.0.13 2026-05-14\n#> dials 1.4.3 2026-04-11\n#> dplyr 1.2.1 2026-04-03\n#> ggplot2 4.0.3 2026-04-22\n#> infer 1.1.0 2025-12-18\n#> parsnip 1.6.0 2026-05-14\n#> purrr 1.2.2 2026-04-10\n#> recipes 1.3.2 2026-04-02\n#> rlang 1.2.0 2026-04-06\n#> rsample 1.3.2 2026-01-30\n#> sparsevctrs 0.3.6 2026-01-27\n#> tibble 3.3.1 2026-01-11\n#> tidymodels 1.5.0 2026-04-23\n#> tune 2.1.0 2026-04-17\n#> workflows 1.3.0 2025-08-27\n#> yardstick 1.4.0 2026-04-07\n#> \n#> ────────────────────────────────────────────────────────────────────\n```\n:::\n\n", + "markdown": "---\ntitle: \"Model tuning using a sparse matrix\"\ncategories:\n - tuning and workflows\n - tuning\n - classification\n - sparse data\ntype: learn-subsection\nweight: 1\ndescription: | \n Fitting a model using tidymodels with a sparse matrix as the data.\ntoc: true\ntoc-depth: 2\nr-packages:\n - tidymodels\n - sparsevctrs\ninclude-after-body: ../../../html/resources.html\n---\n\n\n \n\n\n## Introduction\n\nTo use code in this article, you will need to install the following packages: sparsevctrs and tidymodels.\n\nThis article demonstrates how we can use a sparse matrix in tidymodels.\n\nWe use the term **sparse data** to denote a data set that contains a lot of 0s. Such data is commonly seen as a result of dealing with categorical variables, text tokenization, or graph data sets. The word sparse describes how the information is packed. Namely, it represents the presence of a lot of zeroes. For some tasks, we can easily get above 99% percent of 0s in the predictors. \n\nThe reason we use sparse data as a construct is that it is a lot more memory efficient to store the positions and values of the non-zero entries than to encode all the values. One could think of this as a compression, but one that is done such that data tasks are still fast. The following vector requires 25 values to store it normally (dense representation). This representation will be referred to as a **dense vector**.\n\n```r\nc(100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0)\n```\nThe sparse representation of this vector only requires 5 values. 1 value for the length (25), 2 values for the locations of the non-zero values (1, 22), and 2 values for the non-zero values (100, 1). This idea can also be extended to matrices as is done in the Matrix package. Where we instead store the dimensions of the matrix, row indexes, colomn indexes, and the values for the non-zero entries.\n\n## Example data\n\nThe data we will be using in this article is a larger sample of the [small_fine_foods](https://modeldata.tidymodels.org/reference/small_fine_foods.html) data set from the [modeldata](https://modeldata.tidymodels.org) package. The [raw data](https://snap.stanford.edu/data/web-FineFoods.html) was sliced down to 100,000 rows, tokenized, and saved as a sparse matrix. Data has been saved as [reviews.rds](reviews.rds) and the code to generate this data set is found at [generate-data.R](generate-data.R). This file takes up around 1MB compressed, and around 12MB once loaded into R. This data set is encoded as a sparse matrix from the Matrix package; if we were to turn it into a dense matrix, it would take up 3GB.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nreviews <- readr::read_rds(\"reviews.rds\")\nreviews |> head()\n#> 6 x 24818 sparse Matrix of class \"dgCMatrix\"\n#> [[ suppressing 34 column names 'SCORE', 'a', 'all' ... ]]\n#> \n#> 1 1 2 1 3 1 1 2 1 1 1 1 1 1 1 2 1 1 1 1 1 1 1 1 1 1 2 1 2 1 1 1 1 1 1 ......\n#> 2 . . . . . . . . . . . . . . . . . . . . . . . . . . . 2 . . . . . . ......\n#> 3 . 4 . 6 . . . . . . . . . . . 1 5 3 . . . . . . . 2 . . . . . . . . ......\n#> 4 . . . 1 . . . . . . . . 1 1 1 4 1 1 . . . . . . . . . . . . . . . . ......\n#> 5 1 4 . . . . . . . . . . . . . . 1 . . . . . . . . 1 . . . . . . . . ......\n#> 6 . 3 1 2 . . . . . . . . . . . 2 1 1 . . . . . . 4 1 . . . . . . . . ......\n#> \n#> .....suppressing 24784 columns in show(); maybe adjust options(max.print=, width=)\n#> ..............................\n```\n:::\n\n\n## Modeling\n\nWe start by loading tidymodels and the sparsevctrs package. The sparsevctrs package includes some helper functions that will allow us to more easily work with sparse matrices in tidymodels.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(tidymodels)\nlibrary(sparsevctrs)\n```\n:::\n\n\nWhile sparse matrices now work in parsnip, recipes, and workflows directly, we can use rsample's sampling functions as well if we turn it into a tibble. The usual `as_tibble()` would turn the object to a dense representation, greatly expanding the object size. However, sparsevctrs' `coerce_to_sparse_tibble()` will create a tibble with sparse columns, which we call a **sparse tibble**.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nreviews_tbl <- coerce_to_sparse_tibble(reviews)\nreviews_tbl\n#> # A tibble: 15,000 × 24,818\n#> SCORE a all and appreciates be better bought canned dog finicky\n#> \n#> 1 1 2 1 3 1 1 2 1 1 1 1\n#> 2 0 0 0 0 0 0 0 0 0 0 0\n#> 3 0 4 0 6 0 0 0 0 0 0 0\n#> 4 0 0 0 1 0 0 0 0 0 0 0\n#> 5 1 4 0 0 0 0 0 0 0 0 0\n#> 6 0 3 1 2 0 0 0 0 0 0 0\n#> 7 1 1 0 3 0 0 0 0 0 0 0\n#> 8 1 0 0 1 0 0 0 0 0 0 0\n#> 9 1 0 0 1 0 0 0 0 0 0 0\n#> 10 1 1 0 0 0 0 0 0 0 2 0\n#> # ℹ 14,990 more rows\n#> # ℹ 24,807 more variables: food , found , good , have ,\n#> # i , is , it , labrador , like , looks ,\n#> # meat , more , most , my , of , processed ,\n#> # product , products , quality , several , she ,\n#> # smells , stew , than , the , them , this ,\n#> # to , vitality , actually , an , arrived , …\n```\n:::\n\n\nDespite this tibble containing 15,000 rows and a little under 25,000 columns, it only takes up marginally more space than the sparse matrix.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlobstr::obj_size(reviews)\n#> 12.75 MB\nlobstr::obj_size(reviews_tbl)\n#> 18.27 MB\n```\n:::\n\n\nThe outcome `SCORE` is currently encoded as a double, but we want it to be a factor for it to work well with tidymodels, since tidymodels expects outcomes to be factors for classification.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nreviews_tbl <- reviews_tbl |>\n mutate(SCORE = factor(SCORE, levels = c(1, 0), labels = c(\"great\", \"other\")))\n```\n:::\n\n\nSince `reviews_tbl` is now a tibble, we can use `initial_split()` as we usually do.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(1234)\n\nreview_split <- initial_split(reviews_tbl)\nreview_train <- training(review_split)\nreview_test <- testing(review_split)\n\nreview_folds <- vfold_cv(review_train)\n```\n:::\n\n\nNext, we will specify our workflow. Since we are showcasing how sparse data works in tidymodels, we will stick to a simple lasso regression model. These models tend to work well with sparse predictors. `penalty` has been set to be tuned.\n\n::: callout-tip\nAll available models can be found at the [sparse models search](../../../find/sparse/index.qmd).\n:::\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nrec_spec <- recipe(SCORE ~ ., data = review_train)\n\nlm_spec <- logistic_reg(penalty = tune()) |>\n set_engine(\"glmnet\")\n\nwf_spec <- workflow(rec_spec, lm_spec)\n```\n:::\n\n\nWith everything in order, we can now evaluate several different values of `penalty` with `tune_grid()`.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ntune_res <- tune_grid(wf_spec, review_folds)\n```\n:::\n\n\nDespite the size of the data, this code runs quite quickly due to the sparse encoding of the data. Once the tuning process is done, then we can look at the performance for different values of regularization.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nautoplot(tune_res)\n```\n\n::: {.cell-output-display}\n![](figs/autoplot-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nWe can now finalize the workflow and fit the final model on the training data set.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nwf_final <- finalize_workflow(\n wf_spec, \n select_best(tune_res, metric = \"roc_auc\")\n)\n\nwf_fit <- fit(wf_final, review_train)\n```\n:::\n\n\nWith this fitted model, we can now predict with a sparse tibble.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\npredict(wf_fit, review_test)\n#> # A tibble: 3,750 × 1\n#> .pred_class\n#> \n#> 1 other \n#> 2 great \n#> 3 great \n#> 4 great \n#> 5 great \n#> 6 great \n#> 7 great \n#> 8 great \n#> 9 great \n#> 10 great \n#> # ℹ 3,740 more rows\n```\n:::\n\n\n`fit()` and `predict()` both accept sparse matrices as input. However if you want to tune a model with the tune package or perform data splitting with rsample then you will need a tibble, which can be done with `coerce_to_sparse_tibble()`.\n\nThis means that we could technically do predictions on our model directly on the sparse matrix using `predict()`.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\npredict(wf_fit, reviews)\n#> # A tibble: 15,000 × 1\n#> .pred_class\n#> \n#> 1 great \n#> 2 other \n#> 3 great \n#> 4 great \n#> 5 great \n#> 6 other \n#> 7 great \n#> 8 great \n#> 9 great \n#> 10 great \n#> # ℹ 14,990 more rows\n```\n:::\n\n\n## Session information {#session-info}\n\n\n::: {.cell layout-align=\"center\"}\n\n```\n#> ─ Session info ─────────────────────────────────────────────────────\n#> version R version 4.6.0 (2026-04-24)\n#> language (EN)\n#> pandoc 3.1.3\n#> quarto 1.9.37\n#> \n#> ─ Packages ─────────────────────────────────────────────────────────\n#> package version date (UTC)\n#> broom 1.0.13 2026-05-14\n#> dials 1.4.3 2026-04-11\n#> dplyr 1.2.1 2026-04-03\n#> ggplot2 4.0.3 2026-04-22\n#> infer 1.1.0 2025-12-18\n#> parsnip 1.6.0 2026-05-14\n#> purrr 1.2.2 2026-04-10\n#> recipes 1.3.2 2026-04-02\n#> rlang 1.2.0 2026-04-06\n#> rsample 1.3.2 2026-01-30\n#> sparsevctrs 0.3.6 2026-01-27\n#> tibble 3.3.1 2026-01-11\n#> tidymodels 1.5.0 2026-04-23\n#> tune 2.1.0 2026-04-17\n#> workflows 1.3.0 2025-08-27\n#> yardstick 1.4.0 2026-04-07\n#> \n#> ────────────────────────────────────────────────────────────────────\n```\n:::\n\n", "supporting": [], "filters": [ "rmarkdown/pagebreak.lua" diff --git a/_freeze/learn/work/sparse-recipe/index/execute-results/html.json b/_freeze/learn/work/sparse-recipe/index/execute-results/html.json index de1a6d59..6bfd5690 100644 --- a/_freeze/learn/work/sparse-recipe/index/execute-results/html.json +++ b/_freeze/learn/work/sparse-recipe/index/execute-results/html.json @@ -1,8 +1,8 @@ { - "hash": "3c5e5301133f50908481bd51b213c0ec", + "hash": "39ac0a1802ee2718503ae8d756505bce", "result": { "engine": "knitr", - "markdown": "---\ntitle: \"Using recipes to create sparse data\"\ncategories:\n - tuning\n - classification\n - sparse data\ntype: learn-subsection\nweight: 1\ndescription: | \n Fitting a model using tidymodels where sparse data is created using a recipe.\ntoc: true\ntoc-depth: 2\nr-packages:\n - tidymodels\n - nycflights13\ninclude-after-body: ../../../html/resources.html\n---\n\n\n\n \n\n\n## Introduction\n\nTo use code in this article, you will need to install the following packages: nycflights13 and tidymodels.\n\n\nThis article demonstrates how we can use a recipe to generate data sparsity in tidymodels.\n\nWe use the term **sparse data** to denote a data set that contains a lot of 0s. Such data is commonly seen as a result of dealing with categorical variables, text tokenization, or graph data sets. The word sparse describes how the information is packed. Namely, it represents the presence of a lot of zeroes. For some tasks, we can easily get above 99% percent of 0s in the predictors. \n\nThe reason we use sparse data as a construct is that it is a lot more memory efficient to store the positions and values of the non-zero entries than to encode all the values. One could think of this as a compression, but one that is done such that data tasks are still fast. The following vector requires 25 values to store it normally (dense representation). This representation will be referred to as a **dense vector**.\n\n```r\nc(100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0)\n```\nThe sparse representation of this vector only requires 5 values. 1 value for the length (25), 2 values for the locations of the non-zero values (1, 22), and 2 values for the non-zero values (100, 1). This idea can also be extended to matrices as is done in the Matrix package.\n\nNot all modeling tasks can handle sparsity, we have a [list of compatible](../../../find/sparse/index.qmd) steps you can use to guide the recipe creation.\n\n## The data\n\nWe will be using the [nycflights13](https://nycflights13.tidyverse.org/) data set for this demonstration. We are using this data specifically because it contains a number of categorical with a lot of levels, that when converted to binary indicator columns (a.k.a. \"dummy variables\") will create a lot of sparse columns.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(tidymodels)\nlibrary(nycflights13)\n\nglimpse(flights)\n#> Rows: 336,776\n#> Columns: 19\n#> $ year 2013, 2013, 2013, 2013, 2013, 2013, 2013, 2013, 2013, 2…\n#> $ month 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1…\n#> $ day 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1…\n#> $ dep_time 517, 533, 542, 544, 554, 554, 555, 557, 557, 558, 558, …\n#> $ sched_dep_time 515, 529, 540, 545, 600, 558, 600, 600, 600, 600, 600, …\n#> $ dep_delay 2, 4, 2, -1, -6, -4, -5, -3, -3, -2, -2, -2, -2, -2, -1…\n#> $ arr_time 830, 850, 923, 1004, 812, 740, 913, 709, 838, 753, 849,…\n#> $ sched_arr_time 819, 830, 850, 1022, 837, 728, 854, 723, 846, 745, 851,…\n#> $ arr_delay 11, 20, 33, -18, -25, 12, 19, -14, -8, 8, -2, -3, 7, -1…\n#> $ carrier \"UA\", \"UA\", \"AA\", \"B6\", \"DL\", \"UA\", \"B6\", \"EV\", \"B6\", \"…\n#> $ flight 1545, 1714, 1141, 725, 461, 1696, 507, 5708, 79, 301, 4…\n#> $ tailnum \"N14228\", \"N24211\", \"N619AA\", \"N804JB\", \"N668DN\", \"N394…\n#> $ origin \"EWR\", \"LGA\", \"JFK\", \"JFK\", \"LGA\", \"EWR\", \"EWR\", \"LGA\",…\n#> $ dest \"IAH\", \"IAH\", \"MIA\", \"BQN\", \"ATL\", \"ORD\", \"FLL\", \"IAD\",…\n#> $ air_time 227, 227, 160, 183, 116, 150, 158, 53, 140, 138, 149, 1…\n#> $ distance 1400, 1416, 1089, 1576, 762, 719, 1065, 229, 944, 733, …\n#> $ hour 5, 5, 5, 5, 6, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 6, 6, 6…\n#> $ minute 15, 29, 40, 45, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0…\n#> $ time_hour 2013-01-01 05:00:00, 2013-01-01 05:00:00, 2013-01-01 0…\n```\n:::\n\n\nOur modeling objective is to fit a model that predicts the arrival delay, using a regression model. We could just as well have done a classification model on \"Will plane land on time?,\" but using the regression model we can hopefully be able to quantify how early or late the plane will be.\n\nWe are furthermore assuming that this prediction will take place at departure time. This means we have to exclude some variables as they contain information that is not yet available.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nflights <- flights |>\n select(-arr_time, -air_time)\n```\n:::\n\n\nThis data set contains a number of redundant variables. We don't need to know the departure time `dep_time`, scheduled departure time `sched_dep_time`, and the departure delay `dep_delay` as they are a linear combination of each other `dep_delay = dep_time - sched_dep_time`. So we can remove one of them and choose to get rid of `sched_dep_time`.\n\nLikewise, the `time_hour` variable is a datetime that contains data also located in `year`, `month`, `day`, `hour`, and `minute`. We will thus also remove that one.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nflights <- flights |>\n select(-time_hour, -sched_dep_time)\n```\n:::\n\n\nYou may or may not have noticed that `dep_time` and `sched_arr_time` have a weird encoding. What is happening is that `517` is actually `5:17` e.i. 17 minutes past 5 AM. So we need to update that, which we will use a little helper function for.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nget_minutes <- function(x) {\n minutes <- x %% 100\n hours <- x %/% 100\n\n hours * 60 + minutes\n}\n\nflights <- flights |>\n mutate(across(c(dep_time, sched_arr_time), get_minutes))\n```\n:::\n\n\nWe will fit a model using the first month of the year, and then try to assess how well it will generalize over the remaining years. We will also exclude any observations where `arr_delay` is `NA`.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nflights <- drop_na(flights, arr_delay)\nflights_train <- filter(flights, month == 1)\nflights_test <- filter(flights, month != 1)\n```\n:::\n\n\n## Creating a recipe\n\nThe data is quite simple in terms of types. We have numeric variables and categorical variables. We will do some simple imputation of the numeric variables and create dummy variables on the categorical predictors.\n\nWe'll use a recipe to preprocess the data. If you have never seen a recipe, see Chapter 8 of [_Tidy Models with R_](https://www.tmwr.org/recipes). \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nrec_spec <- recipe(arr_delay ~ ., data = flights_train) |>\n step_impute_mean(all_numeric_predictors()) |>\n step_zv(all_predictors()) |>\n step_normalize(all_numeric_predictors()) |>\n step_novel(all_nominal_predictors()) |>\n step_unknown(all_nominal_predictors()) |>\n step_dummy(all_nominal_predictors())\n\nrec_spec\n#> \n#> ── Recipe ────────────────────────────────────────────────────────────\n#> \n#> ── Inputs\n#> Number of variables by role\n#> outcome: 1\n#> predictor: 14\n#> \n#> ── Operations\n#> • Mean imputation for: all_numeric_predictors()\n#> • Zero variance filter on: all_predictors()\n#> • Centering and scaling for: all_numeric_predictors()\n#> • Novel factor level assignment for: all_nominal_predictors()\n#> • Unknown factor level assignment for: all_nominal_predictors()\n#> • Dummy variables from: all_nominal_predictors()\n```\n:::\n\n\nYou will notice we aren't doing anything special here to denote the recipe that will acknowledge or produce sparsity. Next, we will go into some details to explain how you, the user, should approach recipes when you suspect that sparsity will be produced.\n\n## How is sparsity handled in recipes\n\nThere have been made two types of changes to recipes steps regarding sparsity. \n\nFirst, some steps can augment the data set with many columns that are naturally sparse. Creating binary indicators from a factor predictor, via `step_dummy()`, is a good example. Because of this, a number of steps have gained a `sparse` argument, which toggles the creation of sparse vectors. \n\nThe second change is that a number of steps are now able to take sparse vectors as input and _preserve sparsity_. You can see a full [list of these steps](../../../find/sparse/index.qmd) at the link.\n\nMost of the changes with regard to sparsity are done to minimize the changes the user needs to make to their code. This means that in many cases you don't need to change anything, the steps will know when to produce sparse data or not.\n\nWhen a recipe is used in a workflow and it is being `fit()`, an internal check is being done to figure out whether or not to produce sparse features. This check looks at the sparsity of the data itself, what model is being used, and the recipe. Since [only some models](../../../find/sparse/index.qmd) support sparsity this is the first check. \n\nA rough estimate of the sparsity of the data that will come out of the recipe is calculated. This is done using the input data set, and the steps present in the recipe. But since this check has to happen before the recipe is prepped, it will be quite simple. What this means in practice is that it is fairly good at estimating the sparsity that is produced by sparsity-generating steps, but it isn't able to detect if those variables are passed to a different step that doesn't preserve it.\n\nThe following recipe would give an accurate estimate of how sparse the resulting data will be, as the dummy variables produced by `step_dummy()` aren't passed to any other steps. We want a good estimate of the resulting sparsity as it is key in determining whether the recipe should produce sparse data or not.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nrecipe(outcome ~ ., data = data_train) |>\n step_normalize(all_numeric_predictors()) |>\n step_dummy(all_nominal_predictors())\n```\n:::\n\n\nBut the next recipe would have the same sparsity estimate despite not being able to produce any sparsity since `step_normalize()` can't preserve the sparsity as it subtracts a constant value.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nrecipe(outcome ~ ., data = data_train) |>\n step_dummy(all_nominal_predictors()) |>\n step_normalize(all_numeric_predictors())\n```\n:::\n\n\nIf you were able to modify the above recipe to use `step_scale()` instead of `step_normalize()` then the estimate is still valid as `step_scale()` is a sparsity-preserving step.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nrecipe(outcome ~ ., data = data_train) |>\n step_dummy(all_nominal_predictors()) |>\n step_scale(all_numeric_predictors())\n```\n:::\n\n\nIs it for this reason the steps that produce sparsity have the `sparse` argument. It defaults to `\"auto\"`, which means that the estimating process in workflows decides whether or not sparsity should be created. This argument can take two other values `\"yes\"` and `\"no\"`. If you know for certain that the recipe should or shouldn't produce sparsity you can overwrite with this argument.\n\nThis means the recipe below wouldn't try to initially produce sparse vectors since they will immediately be turned into dense vectors by the next step.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nrecipe(outcome ~ ., data = data_train) |>\n step_dummy(all_nominal_predictors(), sparse = \"no\") |>\n step_normalize(all_numeric_predictors())\n```\n:::\n\n\nA lot of time went into trying to make `sparse = \"auto\"` work as well as possible, but since nothing is perfect you have the ability to overwrite. Setting `sparse = \"yes\"` or `sparse = \"no\"` is done as a overwrite, meaning that the above mentioned check in workflows doesn't run. The check is only done if `sparse = \"auto\"` is present in any of the steps.\n\nThe bad thing that happens if the check process is incorrect or you set the wrong value for `sparse` is that you get the speed and performance from previous versions before sparsity was enabled. Not worse speed and performance.\n\n## Modeling\n\nWe will finish the workflow using a model/engine combination that supports sparse data.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nmod_spec <- boost_tree() |>\n set_mode(\"regression\") |>\n set_engine(\"xgboost\")\n\nmod_spec\n#> Boosted Tree Model Specification (regression)\n#> \n#> Computational engine: xgboost\n```\n:::\n\n\nThen combine it in a workflow.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nwf_spec <- workflow(rec_spec, mod_spec)\n```\n:::\n\n\nAnd fit it like we usually do.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nwf_fit <- fit(wf_spec, flights_train)\n```\n:::\n\n\n::: {.callout-note}\nThe above code chunk was run locally and timed. With sparsity enabled `sparse = \"yes\"` it took around 0.45 seconds, wit sparsity disabled `sparse = \"no\"` it took around 30 seconds.\n:::\n\nNow that the model has been fit we can calculate the RMSE to see how well the model has performed.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ntrain_preds <- augment(wf_fit, flights_train)\n\nrmse(train_preds, arr_delay, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> \n#> 1 rmse standard 14.2\n```\n:::\n\n\n::: {.callout-note}\nThe above code chunk was run locally and timed. With sparsity enabled `sparse = \"yes\"` it took around 0.1 seconds, wit sparsity disabled `sparse = \"no\"` it took around 5 seconds.\n:::\n\nWe can also take a visual look at the performance by plotting the predicted values against the real values.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ntrain_preds |>\n ggplot(aes(arr_delay, .pred)) +\n geom_point(alpha = 0.2)\n```\n\n::: {.cell-output-display}\n![](figs/plot-train-predictions-1.svg){fig-align='center' fig-alt='Scatter chart. Arrival delay along the x-axis and predictions along the \ny-axis. The majority of the points are along the diagonal, with a shift\ndown.' width=672}\n:::\n:::\n\n\nThe model appears to work fairly well on the training data set. We notice the shift down, which would suggest that the model has a bias towards underestimating the delay.\n\nNow we will see how well the model performs in the remaining months.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ntest_preds <- augment(wf_fit, flights_test)\n\nrmse(test_preds, arr_delay, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> \n#> 1 rmse standard 20.9\n```\n:::\n\n\nAnd they see that the performance is quite a bit worse. Let's see how the performance goes on a month-by-month basis.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ntest_preds |>\n group_by(month) |>\n rmse(arr_delay, .pred) |>\n ggplot(aes(month, .estimate)) +\n geom_point()\n```\n\n::: {.cell-output-display}\n![](figs/plot-rmse-by-month-1.svg){fig-align='center' fig-alt='Scatter chart. Month along the x-axis, estimate of RSME along the y-axis.\nStarting on the second month with a value around 17, it goes up for each\nmonth to 23 in July, afterward it does back down to 16 in September, \nwith November having the same value and December having a value of 20.' width=672}\n:::\n:::\n\n\nWe see the same result that the model doesn't generalize to the other months. This should not be that surprising as the model was only fit in January. Furthermore, it appears that there is a seasonal trend happening, further showing us that fitting this model in January alone was not the best idea.\n\n## Session information {#session-info}\n\n\n::: {.cell layout-align=\"center\"}\n\n```\n#> ─ Session info ─────────────────────────────────────────────────────\n#> version R version 4.6.0 (2026-04-24)\n#> language (EN)\n#> pandoc 3.1.3\n#> quarto 1.9.37\n#> \n#> ─ Packages ─────────────────────────────────────────────────────────\n#> package version date (UTC)\n#> broom 1.0.13 2026-05-14\n#> dials 1.4.3 2026-04-11\n#> dplyr 1.2.1 2026-04-03\n#> ggplot2 4.0.3 2026-04-22\n#> infer 1.1.0 2025-12-18\n#> nycflights13 1.0.2 2021-04-12\n#> parsnip 1.6.0 2026-05-14\n#> purrr 1.2.2 2026-04-10\n#> recipes 1.3.2 2026-04-02\n#> rlang 1.2.0 2026-04-06\n#> rsample 1.3.2 2026-01-30\n#> tibble 3.3.1 2026-01-11\n#> tidymodels 1.5.0 2026-04-23\n#> tune 2.1.0 2026-04-17\n#> workflows 1.3.0 2025-08-27\n#> yardstick 1.4.0 2026-04-07\n#> \n#> ────────────────────────────────────────────────────────────────────\n```\n:::\n\n", + "markdown": "---\ntitle: \"Using recipes to create sparse data\"\ncategories:\n - tuning and workflows\n - tuning\n - classification\n - sparse data\ntype: learn-subsection\nweight: 1\ndescription: | \n Fitting a model using tidymodels where sparse data is created using a recipe.\ntoc: true\ntoc-depth: 2\nr-packages:\n - tidymodels\n - nycflights13\ninclude-after-body: ../../../html/resources.html\n---\n\n\n\n \n\n\n## Introduction\n\nTo use code in this article, you will need to install the following packages: nycflights13 and tidymodels.\n\n\nThis article demonstrates how we can use a recipe to generate data sparsity in tidymodels.\n\nWe use the term **sparse data** to denote a data set that contains a lot of 0s. Such data is commonly seen as a result of dealing with categorical variables, text tokenization, or graph data sets. The word sparse describes how the information is packed. Namely, it represents the presence of a lot of zeroes. For some tasks, we can easily get above 99% percent of 0s in the predictors. \n\nThe reason we use sparse data as a construct is that it is a lot more memory efficient to store the positions and values of the non-zero entries than to encode all the values. One could think of this as a compression, but one that is done such that data tasks are still fast. The following vector requires 25 values to store it normally (dense representation). This representation will be referred to as a **dense vector**.\n\n```r\nc(100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0)\n```\nThe sparse representation of this vector only requires 5 values. 1 value for the length (25), 2 values for the locations of the non-zero values (1, 22), and 2 values for the non-zero values (100, 1). This idea can also be extended to matrices as is done in the Matrix package.\n\nNot all modeling tasks can handle sparsity, we have a [list of compatible](../../../find/sparse/index.qmd) steps you can use to guide the recipe creation.\n\n## The data\n\nWe will be using the [nycflights13](https://nycflights13.tidyverse.org/) data set for this demonstration. We are using this data specifically because it contains a number of categorical with a lot of levels, that when converted to binary indicator columns (a.k.a. \"dummy variables\") will create a lot of sparse columns.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(tidymodels)\nlibrary(nycflights13)\n\nglimpse(flights)\n#> Rows: 336,776\n#> Columns: 19\n#> $ year 2013, 2013, 2013, 2013, 2013, 2013, 2013, 2013, 2013, 2…\n#> $ month 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1…\n#> $ day 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1…\n#> $ dep_time 517, 533, 542, 544, 554, 554, 555, 557, 557, 558, 558, …\n#> $ sched_dep_time 515, 529, 540, 545, 600, 558, 600, 600, 600, 600, 600, …\n#> $ dep_delay 2, 4, 2, -1, -6, -4, -5, -3, -3, -2, -2, -2, -2, -2, -1…\n#> $ arr_time 830, 850, 923, 1004, 812, 740, 913, 709, 838, 753, 849,…\n#> $ sched_arr_time 819, 830, 850, 1022, 837, 728, 854, 723, 846, 745, 851,…\n#> $ arr_delay 11, 20, 33, -18, -25, 12, 19, -14, -8, 8, -2, -3, 7, -1…\n#> $ carrier \"UA\", \"UA\", \"AA\", \"B6\", \"DL\", \"UA\", \"B6\", \"EV\", \"B6\", \"…\n#> $ flight 1545, 1714, 1141, 725, 461, 1696, 507, 5708, 79, 301, 4…\n#> $ tailnum \"N14228\", \"N24211\", \"N619AA\", \"N804JB\", \"N668DN\", \"N394…\n#> $ origin \"EWR\", \"LGA\", \"JFK\", \"JFK\", \"LGA\", \"EWR\", \"EWR\", \"LGA\",…\n#> $ dest \"IAH\", \"IAH\", \"MIA\", \"BQN\", \"ATL\", \"ORD\", \"FLL\", \"IAD\",…\n#> $ air_time 227, 227, 160, 183, 116, 150, 158, 53, 140, 138, 149, 1…\n#> $ distance 1400, 1416, 1089, 1576, 762, 719, 1065, 229, 944, 733, …\n#> $ hour 5, 5, 5, 5, 6, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 5, 6, 6, 6…\n#> $ minute 15, 29, 40, 45, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0…\n#> $ time_hour 2013-01-01 05:00:00, 2013-01-01 05:00:00, 2013-01-01 0…\n```\n:::\n\n\nOur modeling objective is to fit a model that predicts the arrival delay, using a regression model. We could just as well have done a classification model on \"Will plane land on time?,\" but using the regression model we can hopefully be able to quantify how early or late the plane will be.\n\nWe are furthermore assuming that this prediction will take place at departure time. This means we have to exclude some variables as they contain information that is not yet available.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nflights <- flights |>\n select(-arr_time, -air_time)\n```\n:::\n\n\nThis data set contains a number of redundant variables. We don't need to know the departure time `dep_time`, scheduled departure time `sched_dep_time`, and the departure delay `dep_delay` as they are a linear combination of each other `dep_delay = dep_time - sched_dep_time`. So we can remove one of them and choose to get rid of `sched_dep_time`.\n\nLikewise, the `time_hour` variable is a datetime that contains data also located in `year`, `month`, `day`, `hour`, and `minute`. We will thus also remove that one.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nflights <- flights |>\n select(-time_hour, -sched_dep_time)\n```\n:::\n\n\nYou may or may not have noticed that `dep_time` and `sched_arr_time` have a weird encoding. What is happening is that `517` is actually `5:17` e.i. 17 minutes past 5 AM. So we need to update that, which we will use a little helper function for.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nget_minutes <- function(x) {\n minutes <- x %% 100\n hours <- x %/% 100\n\n hours * 60 + minutes\n}\n\nflights <- flights |>\n mutate(across(c(dep_time, sched_arr_time), get_minutes))\n```\n:::\n\n\nWe will fit a model using the first month of the year, and then try to assess how well it will generalize over the remaining years. We will also exclude any observations where `arr_delay` is `NA`.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nflights <- drop_na(flights, arr_delay)\nflights_train <- filter(flights, month == 1)\nflights_test <- filter(flights, month != 1)\n```\n:::\n\n\n## Creating a recipe\n\nThe data is quite simple in terms of types. We have numeric variables and categorical variables. We will do some simple imputation of the numeric variables and create dummy variables on the categorical predictors.\n\nWe'll use a recipe to preprocess the data. If you have never seen a recipe, see Chapter 8 of [_Tidy Models with R_](https://www.tmwr.org/recipes). \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nrec_spec <- recipe(arr_delay ~ ., data = flights_train) |>\n step_impute_mean(all_numeric_predictors()) |>\n step_zv(all_predictors()) |>\n step_normalize(all_numeric_predictors()) |>\n step_novel(all_nominal_predictors()) |>\n step_unknown(all_nominal_predictors()) |>\n step_dummy(all_nominal_predictors())\n\nrec_spec\n#> \n#> ── Recipe ────────────────────────────────────────────────────────────\n#> \n#> ── Inputs\n#> Number of variables by role\n#> outcome: 1\n#> predictor: 14\n#> \n#> ── Operations\n#> • Mean imputation for: all_numeric_predictors()\n#> • Zero variance filter on: all_predictors()\n#> • Centering and scaling for: all_numeric_predictors()\n#> • Novel factor level assignment for: all_nominal_predictors()\n#> • Unknown factor level assignment for: all_nominal_predictors()\n#> • Dummy variables from: all_nominal_predictors()\n```\n:::\n\n\nYou will notice we aren't doing anything special here to denote the recipe that will acknowledge or produce sparsity. Next, we will go into some details to explain how you, the user, should approach recipes when you suspect that sparsity will be produced.\n\n## How is sparsity handled in recipes\n\nThere have been made two types of changes to recipes steps regarding sparsity. \n\nFirst, some steps can augment the data set with many columns that are naturally sparse. Creating binary indicators from a factor predictor, via `step_dummy()`, is a good example. Because of this, a number of steps have gained a `sparse` argument, which toggles the creation of sparse vectors. \n\nThe second change is that a number of steps are now able to take sparse vectors as input and _preserve sparsity_. You can see a full [list of these steps](../../../find/sparse/index.qmd) at the link.\n\nMost of the changes with regard to sparsity are done to minimize the changes the user needs to make to their code. This means that in many cases you don't need to change anything, the steps will know when to produce sparse data or not.\n\nWhen a recipe is used in a workflow and it is being `fit()`, an internal check is being done to figure out whether or not to produce sparse features. This check looks at the sparsity of the data itself, what model is being used, and the recipe. Since [only some models](../../../find/sparse/index.qmd) support sparsity this is the first check. \n\nA rough estimate of the sparsity of the data that will come out of the recipe is calculated. This is done using the input data set, and the steps present in the recipe. But since this check has to happen before the recipe is prepped, it will be quite simple. What this means in practice is that it is fairly good at estimating the sparsity that is produced by sparsity-generating steps, but it isn't able to detect if those variables are passed to a different step that doesn't preserve it.\n\nThe following recipe would give an accurate estimate of how sparse the resulting data will be, as the dummy variables produced by `step_dummy()` aren't passed to any other steps. We want a good estimate of the resulting sparsity as it is key in determining whether the recipe should produce sparse data or not.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nrecipe(outcome ~ ., data = data_train) |>\n step_normalize(all_numeric_predictors()) |>\n step_dummy(all_nominal_predictors())\n```\n:::\n\n\nBut the next recipe would have the same sparsity estimate despite not being able to produce any sparsity since `step_normalize()` can't preserve the sparsity as it subtracts a constant value.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nrecipe(outcome ~ ., data = data_train) |>\n step_dummy(all_nominal_predictors()) |>\n step_normalize(all_numeric_predictors())\n```\n:::\n\n\nIf you were able to modify the above recipe to use `step_scale()` instead of `step_normalize()` then the estimate is still valid as `step_scale()` is a sparsity-preserving step.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nrecipe(outcome ~ ., data = data_train) |>\n step_dummy(all_nominal_predictors()) |>\n step_scale(all_numeric_predictors())\n```\n:::\n\n\nIs it for this reason the steps that produce sparsity have the `sparse` argument. It defaults to `\"auto\"`, which means that the estimating process in workflows decides whether or not sparsity should be created. This argument can take two other values `\"yes\"` and `\"no\"`. If you know for certain that the recipe should or shouldn't produce sparsity you can overwrite with this argument.\n\nThis means the recipe below wouldn't try to initially produce sparse vectors since they will immediately be turned into dense vectors by the next step.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nrecipe(outcome ~ ., data = data_train) |>\n step_dummy(all_nominal_predictors(), sparse = \"no\") |>\n step_normalize(all_numeric_predictors())\n```\n:::\n\n\nA lot of time went into trying to make `sparse = \"auto\"` work as well as possible, but since nothing is perfect you have the ability to overwrite. Setting `sparse = \"yes\"` or `sparse = \"no\"` is done as a overwrite, meaning that the above mentioned check in workflows doesn't run. The check is only done if `sparse = \"auto\"` is present in any of the steps.\n\nThe bad thing that happens if the check process is incorrect or you set the wrong value for `sparse` is that you get the speed and performance from previous versions before sparsity was enabled. Not worse speed and performance.\n\n## Modeling\n\nWe will finish the workflow using a model/engine combination that supports sparse data.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nmod_spec <- boost_tree() |>\n set_mode(\"regression\") |>\n set_engine(\"xgboost\")\n\nmod_spec\n#> Boosted Tree Model Specification (regression)\n#> \n#> Computational engine: xgboost\n```\n:::\n\n\nThen combine it in a workflow.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nwf_spec <- workflow(rec_spec, mod_spec)\n```\n:::\n\n\nAnd fit it like we usually do.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nwf_fit <- fit(wf_spec, flights_train)\n```\n:::\n\n\n::: {.callout-note}\nThe above code chunk was run locally and timed. With sparsity enabled `sparse = \"yes\"` it took around 0.45 seconds, wit sparsity disabled `sparse = \"no\"` it took around 30 seconds.\n:::\n\nNow that the model has been fit we can calculate the RMSE to see how well the model has performed.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ntrain_preds <- augment(wf_fit, flights_train)\n\nrmse(train_preds, arr_delay, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> \n#> 1 rmse standard 14.2\n```\n:::\n\n\n::: {.callout-note}\nThe above code chunk was run locally and timed. With sparsity enabled `sparse = \"yes\"` it took around 0.1 seconds, wit sparsity disabled `sparse = \"no\"` it took around 5 seconds.\n:::\n\nWe can also take a visual look at the performance by plotting the predicted values against the real values.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ntrain_preds |>\n ggplot(aes(arr_delay, .pred)) +\n geom_point(alpha = 0.2)\n```\n\n::: {.cell-output-display}\n![](figs/plot-train-predictions-1.svg){fig-align='center' fig-alt='Scatter chart. Arrival delay along the x-axis and predictions along the \ny-axis. The majority of the points are along the diagonal, with a shift\ndown.' width=672}\n:::\n:::\n\n\nThe model appears to work fairly well on the training data set. We notice the shift down, which would suggest that the model has a bias towards underestimating the delay.\n\nNow we will see how well the model performs in the remaining months.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ntest_preds <- augment(wf_fit, flights_test)\n\nrmse(test_preds, arr_delay, .pred)\n#> # A tibble: 1 × 3\n#> .metric .estimator .estimate\n#> \n#> 1 rmse standard 20.9\n```\n:::\n\n\nAnd they see that the performance is quite a bit worse. Let's see how the performance goes on a month-by-month basis.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ntest_preds |>\n group_by(month) |>\n rmse(arr_delay, .pred) |>\n ggplot(aes(month, .estimate)) +\n geom_point()\n```\n\n::: {.cell-output-display}\n![](figs/plot-rmse-by-month-1.svg){fig-align='center' fig-alt='Scatter chart. Month along the x-axis, estimate of RSME along the y-axis.\nStarting on the second month with a value around 17, it goes up for each\nmonth to 23 in July, afterward it does back down to 16 in September, \nwith November having the same value and December having a value of 20.' width=672}\n:::\n:::\n\n\nWe see the same result that the model doesn't generalize to the other months. This should not be that surprising as the model was only fit in January. Furthermore, it appears that there is a seasonal trend happening, further showing us that fitting this model in January alone was not the best idea.\n\n## Session information {#session-info}\n\n\n::: {.cell layout-align=\"center\"}\n\n```\n#> ─ Session info ─────────────────────────────────────────────────────\n#> version R version 4.6.0 (2026-04-24)\n#> language (EN)\n#> pandoc 3.1.3\n#> quarto 1.9.37\n#> \n#> ─ Packages ─────────────────────────────────────────────────────────\n#> package version date (UTC)\n#> broom 1.0.13 2026-05-14\n#> dials 1.4.3 2026-04-11\n#> dplyr 1.2.1 2026-04-03\n#> ggplot2 4.0.3 2026-04-22\n#> infer 1.1.0 2025-12-18\n#> nycflights13 1.0.2 2021-04-12\n#> parsnip 1.6.0 2026-05-14\n#> purrr 1.2.2 2026-04-10\n#> recipes 1.3.2 2026-04-02\n#> rlang 1.2.0 2026-04-06\n#> rsample 1.3.2 2026-01-30\n#> tibble 3.3.1 2026-01-11\n#> tidymodels 1.5.0 2026-04-23\n#> tune 2.1.0 2026-04-17\n#> workflows 1.3.0 2025-08-27\n#> yardstick 1.4.0 2026-04-07\n#> \n#> ────────────────────────────────────────────────────────────────────\n```\n:::\n\n", "supporting": [], "filters": [ "rmarkdown/pagebreak.lua" diff --git a/_freeze/learn/work/tune-svm/index/execute-results/html.json b/_freeze/learn/work/tune-svm/index/execute-results/html.json index d03f18bc..b8daf807 100644 --- a/_freeze/learn/work/tune-svm/index/execute-results/html.json +++ b/_freeze/learn/work/tune-svm/index/execute-results/html.json @@ -1,8 +1,8 @@ { - "hash": "74d4ba5a47be1061d2c74acf42fb44ab", + "hash": "2c59fb1a80d88099e5d045667a35717d", "result": { "engine": "knitr", - "markdown": "---\ntitle: \"Model tuning via grid search\"\ncategories:\n - tuning\n - SVMs\n - classification\ntype: learn-subsection\nweight: 1\ndescription: | \n Choose hyperparameters for a model by training on a grid of many possible parameter values.\ntoc: true\ntoc-depth: 2\nr-packages:\n - tidymodels\n - mlbench\n - kernlab\n - future\ninclude-after-body: ../../../html/resources.html\n---\n\n\n \n\n\n## Introduction\n\nTo use code in this article, you will need to install the following packages: future, kernlab, mlbench, and tidymodels.\n\nThis article demonstrates how to tune a model using grid search. Many models have **hyperparameters** that can't be learned directly from a single data set when training the model. Instead, we can train many models in a grid of possible hyperparameter values and see which ones turn out best. \n\n## Example data\n\nTo demonstrate model tuning, we'll use the Ionosphere data in the mlbench package:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(tidymodels)\nlibrary(mlbench)\ndata(Ionosphere)\n```\n:::\n\n\nFrom `?Ionosphere`:\n\n> This radar data was collected by a system in Goose Bay, Labrador. This system consists of a phased array of 16 high-frequency antennas with a total transmitted power on the order of 6.4 kilowatts. See the paper for more details. The targets were free electrons in the ionosphere. \"good\" radar returns are those showing evidence of some type of structure in the ionosphere. \"bad\" returns are those that do not; their signals pass through the ionosphere.\n\n> Received signals were processed using an autocorrelation function whose arguments are the time of a pulse and the pulse number. There were 17 pulse numbers for the Goose Bay system. Instances in this databse are described by 2 attributes per pulse number, corresponding to the complex values returned by the function resulting from the complex electromagnetic signal. See cited below for more details.\n\nThere are 43 predictors and a factor outcome. Two of the predictors are factors (`V1` and `V2`) and the rest are numeric variables that have been scaled to a range of -1 to 1. Note that the two factor predictors have sparse distributions:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ntable(Ionosphere$V1)\n#> \n#> 0 1 \n#> 38 313\ntable(Ionosphere$V2)\n#> \n#> 0 \n#> 351\n```\n:::\n\n\nThere's no point of putting `V2` into any model since is is a zero-variance predictor. `V1` is not but it _could_ be if the resampling process ends up sampling all of the same value. Is this an issue? It might be since the standard R formula infrastructure fails when there is only a single observed value:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nglm(Class ~ ., data = Ionosphere, family = binomial)\n\n# Surprisingly, this doesn't help: \n\nglm(Class ~ . - V2, data = Ionosphere, family = binomial)\n```\n:::\n\n\nLet's remove these two problematic variables:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nIonosphere <- Ionosphere |> select(-V1, -V2)\n```\n:::\n\n\n## Inputs for the search\n\nTo demonstrate, we'll fit a radial basis function support vector machine to these data and tune the SVM cost parameter and the $\\sigma$ parameter in the kernel function:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nsvm_mod <-\n svm_rbf(cost = tune(), rbf_sigma = tune()) |>\n set_mode(\"classification\") |>\n set_engine(\"kernlab\")\n```\n:::\n\n\nIn this article, tuning will be demonstrated in two ways, using:\n\n- a standard R formula, and \n- a recipe.\n\nLet's create a simple recipe here:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\niono_rec <-\n recipe(Class ~ ., data = Ionosphere) |>\n # remove any zero variance predictors\n step_zv(all_predictors()) |> \n # remove any linear combinations\n step_lincomb(all_numeric())\n```\n:::\n\n\nThe only other required item for tuning is a resampling strategy as defined by an rsample object. Let's demonstrate using basic bootstrapping:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(4943)\niono_rs <- bootstraps(Ionosphere, times = 30)\n```\n:::\n\n\n## Optional inputs\n\nAn _optional_ step for model tuning is to specify which metrics should be computed using the out-of-sample predictions. For classification, the default is to calculate the log-likelihood statistic and overall accuracy. Instead of the defaults, the area under the ROC curve will be used. To do this, a yardstick package function can be used to create a metric set:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nroc_vals <- metric_set(roc_auc)\n```\n:::\n\n\nIf no grid or parameters are provided, a set of 10 hyperparameters are created using a space-filling design (via a Latin hypercube). A grid can be given in a data frame where the parameters are in columns and parameter combinations are in rows. Here, the default will be used.\n\nAlso, a control object can be passed that specifies different aspects of the search. Here, the verbose option is turned off and the option to save the out-of-sample predictions is turned on. \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nctrl <- control_grid(verbose = FALSE, save_pred = TRUE)\n```\n:::\n\n\n## Executing with a formula\n\nFirst, we can use the formula interface:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(35)\nformula_res <-\n svm_mod |> \n tune_grid(\n Class ~ .,\n resamples = iono_rs,\n metrics = roc_vals,\n control = ctrl\n )\n#> maximum number of iterations reached 0.002729295 -0.002728731maximum number of iterations reached 1.48662e-05 -1.48662e-05maximum number of iterations reached 0.005087422 -0.005081074maximum number of iterations reached 0.003476184 -0.003475215maximum number of iterations reached 1.983896e-05 -1.983895e-05maximum number of iterations reached 0.006626915 -0.006616544maximum number of iterations reached 0.002515968 -0.002515484maximum number of iterations reached 1.34078e-05 -1.340779e-05maximum number of iterations reached 0.004555265 -0.0045506maximum number of iterations reached 0.002469115 -0.002468711maximum number of iterations reached 1.330233e-05 -1.330233e-05maximum number of iterations reached 0.00467122 -0.004667136maximum number of iterations reached 0.001989876 -0.001989645maximum number of iterations reached 1.315534e-05 -1.315534e-05maximum number of iterations reached 0.003656559 -0.003654453maximum number of iterations reached 0.002283848 -0.002283544maximum number of iterations reached 1.242395e-05 -1.242395e-05maximum number of iterations reached 0.004336801 -0.004333636maximum number of iterations reached 0.002963038 -0.002962428maximum number of iterations reached 1.496312e-05 -1.496312e-05maximum number of iterations reached 0.005434167 -0.005428241maximum number of iterations reached 0.002604825 -0.002604363maximum number of iterations reached 1.429271e-05 -1.429271e-05maximum number of iterations reached 0.004979161 -0.004973656maximum number of iterations reached 0.002666438 -0.002665922maximum number of iterations reached 1.370558e-05 -1.370558e-05maximum number of iterations reached 0.005090946 -0.005085138maximum number of iterations reached 0.001743169 -0.001743017maximum number of iterations reached 1.059289e-05 -1.059289e-05maximum number of iterations reached 0.00326213 -0.003260567maximum number of iterations reached 0.002568834 -0.002568403maximum number of iterations reached 1.354358e-05 -1.354357e-05maximum number of iterations reached 0.004861762 -0.004856653maximum number of iterations reached 0.002818794 -0.002818223maximum number of iterations reached 1.575347e-05 -1.575346e-05maximum number of iterations reached 0.005249384 -0.005243307maximum number of iterations reached 0.002860941 -0.002860314maximum number of iterations reached 1.503557e-05 -1.503557e-05maximum number of iterations reached 0.005400004 -0.005392986maximum number of iterations reached 0.003400418 -0.003399556maximum number of iterations reached 1.793884e-05 -1.793883e-05maximum number of iterations reached 0.006313314 -0.006304658maximum number of iterations reached 0.001877079 -0.001876912maximum number of iterations reached 1.127929e-05 -1.127928e-05maximum number of iterations reached 0.003670281 -0.003668525maximum number of iterations reached 0.002657858 -0.00265742maximum number of iterations reached 1.399915e-05 -1.399914e-05maximum number of iterations reached 0.005267624 -0.005262646maximum number of iterations reached 0.003063188 -0.003062394maximum number of iterations reached 1.569352e-05 -1.569351e-05maximum number of iterations reached 0.005752102 -0.005743002maximum number of iterations reached 0.00341126 -0.003410316maximum number of iterations reached 1.876099e-05 -1.876098e-05maximum number of iterations reached 0.00666891 -0.00665703maximum number of iterations reached 0.003056492 -0.003055746maximum number of iterations reached 1.558056e-05 -1.558056e-05maximum number of iterations reached 0.005753023 -0.005745269maximum number of iterations reached 0.002616596 -0.002616129maximum number of iterations reached 1.432853e-05 -1.432853e-05maximum number of iterations reached 0.004808037 -0.004803453maximum number of iterations reached 0.002710034 -0.002709487maximum number of iterations reached 1.425954e-05 -1.425954e-05maximum number of iterations reached 0.005062868 -0.005056853maximum number of iterations reached 0.002314831 -0.002314496maximum number of iterations reached 1.35017e-05 -1.35017e-05maximum number of iterations reached 0.004284309 -0.004281096maximum number of iterations reached 0.003593176 -0.003592102maximum number of iterations reached 1.881228e-05 -1.881227e-05maximum number of iterations reached 0.00656715 -0.006557095maximum number of iterations reached 0.002894065 -0.002893471maximum number of iterations reached 1.578179e-05 -1.578178e-05maximum number of iterations reached 0.005494275 -0.005487745maximum number of iterations reached 0.002421177 -0.002420827maximum number of iterations reached 1.39692e-05 -1.396919e-05maximum number of iterations reached 0.004434554 -0.004430894maximum number of iterations reached 0.002863577 -0.002863015maximum number of iterations reached 1.60938e-05 -1.60938e-05maximum number of iterations reached 0.005575677 -0.005569426maximum number of iterations reached 0.003103257 -0.003102474maximum number of iterations reached 1.578427e-05 -1.578427e-05maximum number of iterations reached 0.005594288 -0.005586607maximum number of iterations reached 0.003683707 -0.003682607maximum number of iterations reached 1.91626e-05 -1.916259e-05maximum number of iterations reached 0.006760405 -0.006751376maximum number of iterations reached 0.002493337 -0.002492961maximum number of iterations reached 1.428742e-05 -1.428742e-05maximum number of iterations reached 0.004858875 -0.004854543maximum number of iterations reached 0.002770911 -0.00277028maximum number of iterations reached 1.449988e-05 -1.449988e-05maximum number of iterations reached 0.005037426 -0.005031439\nformula_res\n#> # Tuning results\n#> # Bootstrap sampling \n#> # A tibble: 30 × 5\n#> splits id .metrics .notes .predictions\n#> \n#> 1 Bootstrap01 \n#> 2 Bootstrap02 \n#> 3 Bootstrap03 \n#> 4 Bootstrap04 \n#> 5 Bootstrap05 \n#> 6 Bootstrap06 \n#> 7 Bootstrap07 \n#> 8 Bootstrap08 \n#> 9 Bootstrap09 \n#> 10 Bootstrap10 \n#> # ℹ 20 more rows\n```\n:::\n\n\nThe `.metrics` column contains tibbles of the performance metrics for each tuning parameter combination:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nformula_res |> \n select(.metrics) |> \n slice(1) |> \n pull(1)\n#> [[1]]\n#> # A tibble: 10 × 6\n#> cost rbf_sigma .metric .estimator .estimate .config \n#> \n#> 1 0.000977 0.000000215 roc_auc binary 0.838 pre0_mod01_post0\n#> 2 0.00310 0.00599 roc_auc binary 0.942 pre0_mod02_post0\n#> 3 0.00984 0.0000000001 roc_auc binary 0.815 pre0_mod03_post0\n#> 4 0.0312 0.00000278 roc_auc binary 0.832 pre0_mod04_post0\n#> 5 0.0992 0.0774 roc_auc binary 0.968 pre0_mod05_post0\n#> 6 0.315 0.00000000129 roc_auc binary 0.830 pre0_mod06_post0\n#> 7 1 0.0000359 roc_auc binary 0.837 pre0_mod07_post0\n#> 8 3.17 1 roc_auc binary 0.974 pre0_mod08_post0\n#> 9 10.1 0.0000000167 roc_auc binary 0.832 pre0_mod09_post0\n#> 10 32 0.000464 roc_auc binary 0.861 pre0_mod10_post0\n```\n:::\n\n\nTo get the final resampling estimates, the `collect_metrics()` function can be used on the grid object:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nestimates <- collect_metrics(formula_res)\nestimates\n#> # A tibble: 10 × 8\n#> cost rbf_sigma .metric .estimator mean n std_err .config \n#> \n#> 1 0.000977 0.000000215 roc_auc binary 0.871 30 0.00516 pre0_mod01_po…\n#> 2 0.00310 0.00599 roc_auc binary 0.959 30 0.00290 pre0_mod02_po…\n#> 3 0.00984 0.0000000001 roc_auc binary 0.822 30 0.00718 pre0_mod03_po…\n#> 4 0.0312 0.00000278 roc_auc binary 0.871 30 0.00531 pre0_mod04_po…\n#> 5 0.0992 0.0774 roc_auc binary 0.970 30 0.00261 pre0_mod05_po…\n#> 6 0.315 0.00000000129 roc_auc binary 0.857 30 0.00624 pre0_mod06_po…\n#> 7 1 0.0000359 roc_auc binary 0.873 30 0.00533 pre0_mod07_po…\n#> 8 3.17 1 roc_auc binary 0.971 30 0.00248 pre0_mod08_po…\n#> 9 10.1 0.0000000167 roc_auc binary 0.871 30 0.00534 pre0_mod09_po…\n#> 10 32 0.000464 roc_auc binary 0.927 30 0.00484 pre0_mod10_po…\n```\n:::\n\n\nThe top combinations are:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nshow_best(formula_res, metric = \"roc_auc\")\n#> # A tibble: 5 × 8\n#> cost rbf_sigma .metric .estimator mean n std_err .config \n#> \n#> 1 3.17 1 roc_auc binary 0.971 30 0.00248 pre0_mod08_post0\n#> 2 0.0992 0.0774 roc_auc binary 0.970 30 0.00261 pre0_mod05_post0\n#> 3 0.00310 0.00599 roc_auc binary 0.959 30 0.00290 pre0_mod02_post0\n#> 4 32 0.000464 roc_auc binary 0.927 30 0.00484 pre0_mod10_post0\n#> 5 1 0.0000359 roc_auc binary 0.873 30 0.00533 pre0_mod07_post0\n```\n:::\n\n\n## Executing with a recipe\n\nNext, we can use the same syntax but pass a *recipe* in as the pre-processor argument:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(325)\nrecipe_res <-\n svm_mod |> \n tune_grid(\n iono_rec,\n resamples = iono_rs,\n metrics = roc_vals,\n control = ctrl\n )\n#> maximum number of iterations reached 0.002742459 -0.0027419maximum number of iterations reached 1.41767e-05 -1.417669e-05maximum number of iterations reached 0.005063382 -0.005057462maximum number of iterations reached 0.003559933 -0.003558871maximum number of iterations reached 1.839635e-05 -1.839635e-05maximum number of iterations reached 0.006503999 -0.00649393maximum number of iterations reached 0.002514222 -0.002513769maximum number of iterations reached 1.337914e-05 -1.337914e-05maximum number of iterations reached 0.004701971 -0.004696892maximum number of iterations reached 0.002512296 -0.00251186maximum number of iterations reached 1.35322e-05 -1.353219e-05maximum number of iterations reached 0.004688318 -0.004684078maximum number of iterations reached 0.00201446 -0.002014239maximum number of iterations reached 1.181384e-05 -1.181384e-05maximum number of iterations reached 0.003806843 -0.003804562maximum number of iterations reached 0.002238874 -0.002238586maximum number of iterations reached 1.316689e-05 -1.316688e-05maximum number of iterations reached 0.004295401 -0.004292429maximum number of iterations reached 0.002887453 -0.002886869maximum number of iterations reached 1.481601e-05 -1.481601e-05maximum number of iterations reached 0.005540786 -0.005534606maximum number of iterations reached 0.002509744 -0.002509279maximum number of iterations reached 1.548108e-05 -1.548107e-05maximum number of iterations reached 0.004869151 -0.004863923maximum number of iterations reached 0.00271789 -0.002717372maximum number of iterations reached 1.344855e-05 -1.344854e-05maximum number of iterations reached 0.005088211 -0.005082189maximum number of iterations reached 0.001788083 -0.001787917maximum number of iterations reached 1.038728e-05 -1.038728e-05maximum number of iterations reached 0.00327505 -0.003273591maximum number of iterations reached 0.002565143 -0.002564693maximum number of iterations reached 1.340505e-05 -1.340505e-05maximum number of iterations reached 0.004797415 -0.004792541maximum number of iterations reached 0.002819414 -0.002818838maximum number of iterations reached 1.606027e-05 -1.606026e-05maximum number of iterations reached 0.005439585 -0.005432979maximum number of iterations reached 0.0027882 -0.002787594maximum number of iterations reached 1.524922e-05 -1.524922e-05maximum number of iterations reached 0.00519029 -0.005183748maximum number of iterations reached 0.003381619 -0.003380755maximum number of iterations reached 1.760863e-05 -1.760862e-05maximum number of iterations reached 0.006384695 -0.006376075maximum number of iterations reached 0.001945395 -0.001945211maximum number of iterations reached 1.065017e-05 -1.065016e-05maximum number of iterations reached 0.003457146 -0.003455547maximum number of iterations reached 0.002792796 -0.002792333maximum number of iterations reached 1.502743e-05 -1.502743e-05maximum number of iterations reached 0.005208387 -0.005203715maximum number of iterations reached 0.003050062 -0.00304923maximum number of iterations reached 1.520112e-05 -1.520112e-05maximum number of iterations reached 0.00570829 -0.005699808maximum number of iterations reached 0.003525424 -0.003524307maximum number of iterations reached 1.898649e-05 -1.898649e-05maximum number of iterations reached 0.00648104 -0.006469245maximum number of iterations reached 0.003044483 -0.003043757maximum number of iterations reached 1.531583e-05 -1.531583e-05maximum number of iterations reached 0.005595095 -0.005587695maximum number of iterations reached 0.002576566 -0.002576127maximum number of iterations reached 1.477244e-05 -1.477244e-05maximum number of iterations reached 0.004990352 -0.004985023maximum number of iterations reached 0.002763485 -0.002762901maximum number of iterations reached 1.446549e-05 -1.446549e-05maximum number of iterations reached 0.004964604 -0.004958588maximum number of iterations reached 0.002368979 -0.00236863maximum number of iterations reached 1.289772e-05 -1.289772e-05maximum number of iterations reached 0.004231297 -0.004228114maximum number of iterations reached 0.003641421 -0.003640323maximum number of iterations reached 1.969006e-05 -1.969005e-05maximum number of iterations reached 0.00653699 -0.006527311maximum number of iterations reached 0.002840029 -0.002839469maximum number of iterations reached 1.690192e-05 -1.690192e-05maximum number of iterations reached 0.005462389 -0.005456maximum number of iterations reached 0.002396492 -0.002396135maximum number of iterations reached 1.380691e-05 -1.380691e-05maximum number of iterations reached 0.004663851 -0.004659728maximum number of iterations reached 0.002949473 -0.002948877maximum number of iterations reached 1.616891e-05 -1.61689e-05maximum number of iterations reached 0.005622439 -0.00561606maximum number of iterations reached 0.002911029 -0.002910376maximum number of iterations reached 1.621631e-05 -1.621631e-05maximum number of iterations reached 0.005602028 -0.005594715maximum number of iterations reached 0.003762223 -0.003761115maximum number of iterations reached 1.905902e-05 -1.905901e-05maximum number of iterations reached 0.006985212 -0.006975203maximum number of iterations reached 0.002524708 -0.002524315maximum number of iterations reached 1.500886e-05 -1.500886e-05maximum number of iterations reached 0.004750492 -0.004746452maximum number of iterations reached 0.002741921 -0.002741331maximum number of iterations reached 1.492422e-05 -1.492421e-05maximum number of iterations reached 0.005142 -0.005135487\nrecipe_res\n#> # Tuning results\n#> # Bootstrap sampling \n#> # A tibble: 30 × 5\n#> splits id .metrics .notes .predictions\n#> \n#> 1 Bootstrap01 \n#> 2 Bootstrap02 \n#> 3 Bootstrap03 \n#> 4 Bootstrap04 \n#> 5 Bootstrap05 \n#> 6 Bootstrap06 \n#> 7 Bootstrap07 \n#> 8 Bootstrap08 \n#> 9 Bootstrap09 \n#> 10 Bootstrap10 \n#> # ℹ 20 more rows\n```\n:::\n\n\nThe best setting here is:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nshow_best(recipe_res, metric = \"roc_auc\")\n#> # A tibble: 5 × 8\n#> cost rbf_sigma .metric .estimator mean n std_err .config \n#> \n#> 1 3.17 1 roc_auc binary 0.971 30 0.00248 pre0_mod08_post0\n#> 2 0.0992 0.0774 roc_auc binary 0.970 30 0.00261 pre0_mod05_post0\n#> 3 0.00310 0.00599 roc_auc binary 0.959 30 0.00290 pre0_mod02_post0\n#> 4 32 0.000464 roc_auc binary 0.927 30 0.00484 pre0_mod10_post0\n#> 5 1 0.0000359 roc_auc binary 0.873 30 0.00533 pre0_mod07_post0\n```\n:::\n\n\n## Out-of-sample predictions\n\nIf we used `save_pred = TRUE` to keep the out-of-sample predictions for each resample during tuning, we can obtain those predictions, along with the tuning parameters and resample identifier, using `collect_predictions()`:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ncollect_predictions(recipe_res)\n#> # A tibble: 38,740 × 8\n#> .pred_bad .pred_good id Class .row cost rbf_sigma .config \n#> \n#> 1 0.333 0.667 Bootstrap01 good 1 0.000977 0.000000215 pre0_mod01…\n#> 2 0.333 0.667 Bootstrap01 good 9 0.000977 0.000000215 pre0_mod01…\n#> 3 0.333 0.667 Bootstrap01 bad 10 0.000977 0.000000215 pre0_mod01…\n#> 4 0.333 0.667 Bootstrap01 bad 12 0.000977 0.000000215 pre0_mod01…\n#> 5 0.333 0.667 Bootstrap01 bad 14 0.000977 0.000000215 pre0_mod01…\n#> 6 0.333 0.667 Bootstrap01 good 15 0.000977 0.000000215 pre0_mod01…\n#> 7 0.333 0.667 Bootstrap01 bad 16 0.000977 0.000000215 pre0_mod01…\n#> 8 0.333 0.667 Bootstrap01 bad 22 0.000977 0.000000215 pre0_mod01…\n#> 9 0.333 0.667 Bootstrap01 good 23 0.000977 0.000000215 pre0_mod01…\n#> 10 0.333 0.667 Bootstrap01 bad 24 0.000977 0.000000215 pre0_mod01…\n#> # ℹ 38,730 more rows\n```\n:::\n\n\nWe can obtain the hold-out sets for all the resamples augmented with the predictions using `augment()`, which provides opportunities for flexible visualization of model results:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\naugment(recipe_res) |>\n ggplot(aes(V3, .pred_good, color = Class)) +\n geom_point(show.legend = FALSE) +\n facet_wrap(~Class)\n```\n\n::: {.cell-output-display}\n![](figs/augment-preds-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\n## Session information {#session-info}\n\n\n::: {.cell layout-align=\"center\"}\n\n```\n#> ─ Session info ─────────────────────────────────────────────────────\n#> version R version 4.6.0 (2026-04-24)\n#> language (EN)\n#> pandoc 3.1.3\n#> quarto 1.9.37\n#> \n#> ─ Packages ─────────────────────────────────────────────────────────\n#> package version date (UTC)\n#> broom 1.0.13 2026-05-14\n#> dials 1.4.3 2026-04-11\n#> dplyr 1.2.1 2026-04-03\n#> future 1.70.0 2026-03-14\n#> ggplot2 4.0.3 2026-04-22\n#> infer 1.1.0 2025-12-18\n#> kernlab 0.9-33 2024-08-13\n#> mlbench 2.1-8 2026-03-26\n#> parsnip 1.6.0 2026-05-14\n#> purrr 1.2.2 2026-04-10\n#> recipes 1.3.2 2026-04-02\n#> rlang 1.2.0 2026-04-06\n#> rsample 1.3.2 2026-01-30\n#> tibble 3.3.1 2026-01-11\n#> tidymodels 1.5.0 2026-04-23\n#> tune 2.1.0 2026-04-17\n#> workflows 1.3.0 2025-08-27\n#> yardstick 1.4.0 2026-04-07\n#> \n#> ────────────────────────────────────────────────────────────────────\n```\n:::\n\n", + "markdown": "---\ntitle: \"Model tuning via grid search\"\ncategories:\n - tuning and workflows\n - tuning\n - SVMs\n - classification\ntype: learn-subsection\nweight: 1\ndescription: | \n Choose hyperparameters for a model by training on a grid of many possible parameter values.\ntoc: true\ntoc-depth: 2\nr-packages:\n - tidymodels\n - mlbench\n - kernlab\n - future\ninclude-after-body: ../../../html/resources.html\n---\n\n\n \n\n\n## Introduction\n\nTo use code in this article, you will need to install the following packages: future, kernlab, mlbench, and tidymodels.\n\nThis article demonstrates how to tune a model using grid search. Many models have **hyperparameters** that can't be learned directly from a single data set when training the model. Instead, we can train many models in a grid of possible hyperparameter values and see which ones turn out best. \n\n## Example data\n\nTo demonstrate model tuning, we'll use the Ionosphere data in the mlbench package:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(tidymodels)\nlibrary(mlbench)\ndata(Ionosphere)\n```\n:::\n\n\nFrom `?Ionosphere`:\n\n> This radar data was collected by a system in Goose Bay, Labrador. This system consists of a phased array of 16 high-frequency antennas with a total transmitted power on the order of 6.4 kilowatts. See the paper for more details. The targets were free electrons in the ionosphere. \"good\" radar returns are those showing evidence of some type of structure in the ionosphere. \"bad\" returns are those that do not; their signals pass through the ionosphere.\n\n> Received signals were processed using an autocorrelation function whose arguments are the time of a pulse and the pulse number. There were 17 pulse numbers for the Goose Bay system. Instances in this databse are described by 2 attributes per pulse number, corresponding to the complex values returned by the function resulting from the complex electromagnetic signal. See cited below for more details.\n\nThere are 43 predictors and a factor outcome. Two of the predictors are factors (`V1` and `V2`) and the rest are numeric variables that have been scaled to a range of -1 to 1. Note that the two factor predictors have sparse distributions:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ntable(Ionosphere$V1)\n#> \n#> 0 1 \n#> 38 313\ntable(Ionosphere$V2)\n#> \n#> 0 \n#> 351\n```\n:::\n\n\nThere's no point of putting `V2` into any model since is is a zero-variance predictor. `V1` is not but it _could_ be if the resampling process ends up sampling all of the same value. Is this an issue? It might be since the standard R formula infrastructure fails when there is only a single observed value:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nglm(Class ~ ., data = Ionosphere, family = binomial)\n\n# Surprisingly, this doesn't help: \n\nglm(Class ~ . - V2, data = Ionosphere, family = binomial)\n```\n:::\n\n\nLet's remove these two problematic variables:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nIonosphere <- Ionosphere |> select(-V1, -V2)\n```\n:::\n\n\n## Inputs for the search\n\nTo demonstrate, we'll fit a radial basis function support vector machine to these data and tune the SVM cost parameter and the $\\sigma$ parameter in the kernel function:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nsvm_mod <-\n svm_rbf(cost = tune(), rbf_sigma = tune()) |>\n set_mode(\"classification\") |>\n set_engine(\"kernlab\")\n```\n:::\n\n\nIn this article, tuning will be demonstrated in two ways, using:\n\n- a standard R formula, and \n- a recipe.\n\nLet's create a simple recipe here:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\niono_rec <-\n recipe(Class ~ ., data = Ionosphere) |>\n # remove any zero variance predictors\n step_zv(all_predictors()) |> \n # remove any linear combinations\n step_lincomb(all_numeric())\n```\n:::\n\n\nThe only other required item for tuning is a resampling strategy as defined by an rsample object. Let's demonstrate using basic bootstrapping:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(4943)\niono_rs <- bootstraps(Ionosphere, times = 30)\n```\n:::\n\n\n## Optional inputs\n\nAn _optional_ step for model tuning is to specify which metrics should be computed using the out-of-sample predictions. For classification, the default is to calculate the log-likelihood statistic and overall accuracy. Instead of the defaults, the area under the ROC curve will be used. To do this, a yardstick package function can be used to create a metric set:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nroc_vals <- metric_set(roc_auc)\n```\n:::\n\n\nIf no grid or parameters are provided, a set of 10 hyperparameters are created using a space-filling design (via a Latin hypercube). A grid can be given in a data frame where the parameters are in columns and parameter combinations are in rows. Here, the default will be used.\n\nAlso, a control object can be passed that specifies different aspects of the search. Here, the verbose option is turned off and the option to save the out-of-sample predictions is turned on. \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nctrl <- control_grid(verbose = FALSE, save_pred = TRUE)\n```\n:::\n\n\n## Executing with a formula\n\nFirst, we can use the formula interface:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(35)\nformula_res <-\n svm_mod |> \n tune_grid(\n Class ~ .,\n resamples = iono_rs,\n metrics = roc_vals,\n control = ctrl\n )\n#> maximum number of iterations reached 0.002729295 -0.002728731maximum number of iterations reached 1.48662e-05 -1.48662e-05maximum number of iterations reached 0.005087422 -0.005081074maximum number of iterations reached 0.003476184 -0.003475215maximum number of iterations reached 1.983896e-05 -1.983895e-05maximum number of iterations reached 0.006626915 -0.006616544maximum number of iterations reached 0.002515968 -0.002515484maximum number of iterations reached 1.34078e-05 -1.340779e-05maximum number of iterations reached 0.004555265 -0.0045506maximum number of iterations reached 0.002469115 -0.002468711maximum number of iterations reached 1.330233e-05 -1.330233e-05maximum number of iterations reached 0.00467122 -0.004667136maximum number of iterations reached 0.001989876 -0.001989645maximum number of iterations reached 1.315534e-05 -1.315534e-05maximum number of iterations reached 0.003656559 -0.003654453maximum number of iterations reached 0.002283848 -0.002283544maximum number of iterations reached 1.242395e-05 -1.242395e-05maximum number of iterations reached 0.004336801 -0.004333636maximum number of iterations reached 0.002963038 -0.002962428maximum number of iterations reached 1.496312e-05 -1.496312e-05maximum number of iterations reached 0.005434167 -0.005428241maximum number of iterations reached 0.002604825 -0.002604363maximum number of iterations reached 1.429271e-05 -1.429271e-05maximum number of iterations reached 0.004979161 -0.004973656maximum number of iterations reached 0.002666438 -0.002665922maximum number of iterations reached 1.370558e-05 -1.370558e-05maximum number of iterations reached 0.005090946 -0.005085138maximum number of iterations reached 0.001743169 -0.001743017maximum number of iterations reached 1.059289e-05 -1.059289e-05maximum number of iterations reached 0.00326213 -0.003260567maximum number of iterations reached 0.002568834 -0.002568403maximum number of iterations reached 1.354358e-05 -1.354357e-05maximum number of iterations reached 0.004861762 -0.004856653maximum number of iterations reached 0.002818794 -0.002818223maximum number of iterations reached 1.575347e-05 -1.575346e-05maximum number of iterations reached 0.005249384 -0.005243307maximum number of iterations reached 0.002860941 -0.002860314maximum number of iterations reached 1.503557e-05 -1.503557e-05maximum number of iterations reached 0.005400004 -0.005392986maximum number of iterations reached 0.003400418 -0.003399556maximum number of iterations reached 1.793884e-05 -1.793883e-05maximum number of iterations reached 0.006313314 -0.006304658maximum number of iterations reached 0.001877079 -0.001876912maximum number of iterations reached 1.127929e-05 -1.127928e-05maximum number of iterations reached 0.003670281 -0.003668525maximum number of iterations reached 0.002657858 -0.00265742maximum number of iterations reached 1.399915e-05 -1.399914e-05maximum number of iterations reached 0.005267624 -0.005262646maximum number of iterations reached 0.003063188 -0.003062394maximum number of iterations reached 1.569352e-05 -1.569351e-05maximum number of iterations reached 0.005752102 -0.005743002maximum number of iterations reached 0.00341126 -0.003410316maximum number of iterations reached 1.876099e-05 -1.876098e-05maximum number of iterations reached 0.00666891 -0.00665703maximum number of iterations reached 0.003056492 -0.003055746maximum number of iterations reached 1.558056e-05 -1.558056e-05maximum number of iterations reached 0.005753023 -0.005745269maximum number of iterations reached 0.002616596 -0.002616129maximum number of iterations reached 1.432853e-05 -1.432853e-05maximum number of iterations reached 0.004808037 -0.004803453maximum number of iterations reached 0.002710034 -0.002709487maximum number of iterations reached 1.425954e-05 -1.425954e-05maximum number of iterations reached 0.005062868 -0.005056853maximum number of iterations reached 0.002314831 -0.002314496maximum number of iterations reached 1.35017e-05 -1.35017e-05maximum number of iterations reached 0.004284309 -0.004281096maximum number of iterations reached 0.003593176 -0.003592102maximum number of iterations reached 1.881228e-05 -1.881227e-05maximum number of iterations reached 0.00656715 -0.006557095maximum number of iterations reached 0.002894065 -0.002893471maximum number of iterations reached 1.578179e-05 -1.578178e-05maximum number of iterations reached 0.005494275 -0.005487745maximum number of iterations reached 0.002421177 -0.002420827maximum number of iterations reached 1.39692e-05 -1.396919e-05maximum number of iterations reached 0.004434554 -0.004430894maximum number of iterations reached 0.002863577 -0.002863015maximum number of iterations reached 1.60938e-05 -1.60938e-05maximum number of iterations reached 0.005575677 -0.005569426maximum number of iterations reached 0.003103257 -0.003102474maximum number of iterations reached 1.578427e-05 -1.578427e-05maximum number of iterations reached 0.005594288 -0.005586607maximum number of iterations reached 0.003683707 -0.003682607maximum number of iterations reached 1.91626e-05 -1.916259e-05maximum number of iterations reached 0.006760405 -0.006751376maximum number of iterations reached 0.002493337 -0.002492961maximum number of iterations reached 1.428742e-05 -1.428742e-05maximum number of iterations reached 0.004858875 -0.004854543maximum number of iterations reached 0.002770911 -0.00277028maximum number of iterations reached 1.449988e-05 -1.449988e-05maximum number of iterations reached 0.005037426 -0.005031439\nformula_res\n#> # Tuning results\n#> # Bootstrap sampling \n#> # A tibble: 30 × 5\n#> splits id .metrics .notes .predictions\n#> \n#> 1 Bootstrap01 \n#> 2 Bootstrap02 \n#> 3 Bootstrap03 \n#> 4 Bootstrap04 \n#> 5 Bootstrap05 \n#> 6 Bootstrap06 \n#> 7 Bootstrap07 \n#> 8 Bootstrap08 \n#> 9 Bootstrap09 \n#> 10 Bootstrap10 \n#> # ℹ 20 more rows\n```\n:::\n\n\nThe `.metrics` column contains tibbles of the performance metrics for each tuning parameter combination:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nformula_res |> \n select(.metrics) |> \n slice(1) |> \n pull(1)\n#> [[1]]\n#> # A tibble: 10 × 6\n#> cost rbf_sigma .metric .estimator .estimate .config \n#> \n#> 1 0.000977 0.000000215 roc_auc binary 0.838 pre0_mod01_post0\n#> 2 0.00310 0.00599 roc_auc binary 0.942 pre0_mod02_post0\n#> 3 0.00984 0.0000000001 roc_auc binary 0.815 pre0_mod03_post0\n#> 4 0.0312 0.00000278 roc_auc binary 0.832 pre0_mod04_post0\n#> 5 0.0992 0.0774 roc_auc binary 0.968 pre0_mod05_post0\n#> 6 0.315 0.00000000129 roc_auc binary 0.830 pre0_mod06_post0\n#> 7 1 0.0000359 roc_auc binary 0.837 pre0_mod07_post0\n#> 8 3.17 1 roc_auc binary 0.974 pre0_mod08_post0\n#> 9 10.1 0.0000000167 roc_auc binary 0.832 pre0_mod09_post0\n#> 10 32 0.000464 roc_auc binary 0.861 pre0_mod10_post0\n```\n:::\n\n\nTo get the final resampling estimates, the `collect_metrics()` function can be used on the grid object:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nestimates <- collect_metrics(formula_res)\nestimates\n#> # A tibble: 10 × 8\n#> cost rbf_sigma .metric .estimator mean n std_err .config \n#> \n#> 1 0.000977 0.000000215 roc_auc binary 0.871 30 0.00516 pre0_mod01_po…\n#> 2 0.00310 0.00599 roc_auc binary 0.959 30 0.00290 pre0_mod02_po…\n#> 3 0.00984 0.0000000001 roc_auc binary 0.822 30 0.00718 pre0_mod03_po…\n#> 4 0.0312 0.00000278 roc_auc binary 0.871 30 0.00531 pre0_mod04_po…\n#> 5 0.0992 0.0774 roc_auc binary 0.970 30 0.00261 pre0_mod05_po…\n#> 6 0.315 0.00000000129 roc_auc binary 0.857 30 0.00624 pre0_mod06_po…\n#> 7 1 0.0000359 roc_auc binary 0.873 30 0.00533 pre0_mod07_po…\n#> 8 3.17 1 roc_auc binary 0.971 30 0.00248 pre0_mod08_po…\n#> 9 10.1 0.0000000167 roc_auc binary 0.871 30 0.00534 pre0_mod09_po…\n#> 10 32 0.000464 roc_auc binary 0.927 30 0.00484 pre0_mod10_po…\n```\n:::\n\n\nThe top combinations are:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nshow_best(formula_res, metric = \"roc_auc\")\n#> # A tibble: 5 × 8\n#> cost rbf_sigma .metric .estimator mean n std_err .config \n#> \n#> 1 3.17 1 roc_auc binary 0.971 30 0.00248 pre0_mod08_post0\n#> 2 0.0992 0.0774 roc_auc binary 0.970 30 0.00261 pre0_mod05_post0\n#> 3 0.00310 0.00599 roc_auc binary 0.959 30 0.00290 pre0_mod02_post0\n#> 4 32 0.000464 roc_auc binary 0.927 30 0.00484 pre0_mod10_post0\n#> 5 1 0.0000359 roc_auc binary 0.873 30 0.00533 pre0_mod07_post0\n```\n:::\n\n\n## Executing with a recipe\n\nNext, we can use the same syntax but pass a *recipe* in as the pre-processor argument:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(325)\nrecipe_res <-\n svm_mod |> \n tune_grid(\n iono_rec,\n resamples = iono_rs,\n metrics = roc_vals,\n control = ctrl\n )\n#> maximum number of iterations reached 0.002742459 -0.0027419maximum number of iterations reached 1.41767e-05 -1.417669e-05maximum number of iterations reached 0.005063382 -0.005057462maximum number of iterations reached 0.003559933 -0.003558871maximum number of iterations reached 1.839635e-05 -1.839635e-05maximum number of iterations reached 0.006503999 -0.00649393maximum number of iterations reached 0.002514222 -0.002513769maximum number of iterations reached 1.337914e-05 -1.337914e-05maximum number of iterations reached 0.004701971 -0.004696892maximum number of iterations reached 0.002512296 -0.00251186maximum number of iterations reached 1.35322e-05 -1.353219e-05maximum number of iterations reached 0.004688318 -0.004684078maximum number of iterations reached 0.00201446 -0.002014239maximum number of iterations reached 1.181384e-05 -1.181384e-05maximum number of iterations reached 0.003806843 -0.003804562maximum number of iterations reached 0.002238874 -0.002238586maximum number of iterations reached 1.316689e-05 -1.316688e-05maximum number of iterations reached 0.004295401 -0.004292429maximum number of iterations reached 0.002887453 -0.002886869maximum number of iterations reached 1.481601e-05 -1.481601e-05maximum number of iterations reached 0.005540786 -0.005534606maximum number of iterations reached 0.002509744 -0.002509279maximum number of iterations reached 1.548108e-05 -1.548107e-05maximum number of iterations reached 0.004869151 -0.004863923maximum number of iterations reached 0.00271789 -0.002717372maximum number of iterations reached 1.344855e-05 -1.344854e-05maximum number of iterations reached 0.005088211 -0.005082189maximum number of iterations reached 0.001788083 -0.001787917maximum number of iterations reached 1.038728e-05 -1.038728e-05maximum number of iterations reached 0.00327505 -0.003273591maximum number of iterations reached 0.002565143 -0.002564693maximum number of iterations reached 1.340505e-05 -1.340505e-05maximum number of iterations reached 0.004797415 -0.004792541maximum number of iterations reached 0.002819414 -0.002818838maximum number of iterations reached 1.606027e-05 -1.606026e-05maximum number of iterations reached 0.005439585 -0.005432979maximum number of iterations reached 0.0027882 -0.002787594maximum number of iterations reached 1.524922e-05 -1.524922e-05maximum number of iterations reached 0.00519029 -0.005183748maximum number of iterations reached 0.003381619 -0.003380755maximum number of iterations reached 1.760863e-05 -1.760862e-05maximum number of iterations reached 0.006384695 -0.006376075maximum number of iterations reached 0.001945395 -0.001945211maximum number of iterations reached 1.065017e-05 -1.065016e-05maximum number of iterations reached 0.003457146 -0.003455547maximum number of iterations reached 0.002792796 -0.002792333maximum number of iterations reached 1.502743e-05 -1.502743e-05maximum number of iterations reached 0.005208387 -0.005203715maximum number of iterations reached 0.003050062 -0.00304923maximum number of iterations reached 1.520112e-05 -1.520112e-05maximum number of iterations reached 0.00570829 -0.005699808maximum number of iterations reached 0.003525424 -0.003524307maximum number of iterations reached 1.898649e-05 -1.898649e-05maximum number of iterations reached 0.00648104 -0.006469245maximum number of iterations reached 0.003044483 -0.003043757maximum number of iterations reached 1.531583e-05 -1.531583e-05maximum number of iterations reached 0.005595095 -0.005587695maximum number of iterations reached 0.002576566 -0.002576127maximum number of iterations reached 1.477244e-05 -1.477244e-05maximum number of iterations reached 0.004990352 -0.004985023maximum number of iterations reached 0.002763485 -0.002762901maximum number of iterations reached 1.446549e-05 -1.446549e-05maximum number of iterations reached 0.004964604 -0.004958588maximum number of iterations reached 0.002368979 -0.00236863maximum number of iterations reached 1.289772e-05 -1.289772e-05maximum number of iterations reached 0.004231297 -0.004228114maximum number of iterations reached 0.003641421 -0.003640323maximum number of iterations reached 1.969006e-05 -1.969005e-05maximum number of iterations reached 0.00653699 -0.006527311maximum number of iterations reached 0.002840029 -0.002839469maximum number of iterations reached 1.690192e-05 -1.690192e-05maximum number of iterations reached 0.005462389 -0.005456maximum number of iterations reached 0.002396492 -0.002396135maximum number of iterations reached 1.380691e-05 -1.380691e-05maximum number of iterations reached 0.004663851 -0.004659728maximum number of iterations reached 0.002949473 -0.002948877maximum number of iterations reached 1.616891e-05 -1.61689e-05maximum number of iterations reached 0.005622439 -0.00561606maximum number of iterations reached 0.002911029 -0.002910376maximum number of iterations reached 1.621631e-05 -1.621631e-05maximum number of iterations reached 0.005602028 -0.005594715maximum number of iterations reached 0.003762223 -0.003761115maximum number of iterations reached 1.905902e-05 -1.905901e-05maximum number of iterations reached 0.006985212 -0.006975203maximum number of iterations reached 0.002524708 -0.002524315maximum number of iterations reached 1.500886e-05 -1.500886e-05maximum number of iterations reached 0.004750492 -0.004746452maximum number of iterations reached 0.002741921 -0.002741331maximum number of iterations reached 1.492422e-05 -1.492421e-05maximum number of iterations reached 0.005142 -0.005135487\nrecipe_res\n#> # Tuning results\n#> # Bootstrap sampling \n#> # A tibble: 30 × 5\n#> splits id .metrics .notes .predictions\n#> \n#> 1 Bootstrap01 \n#> 2 Bootstrap02 \n#> 3 Bootstrap03 \n#> 4 Bootstrap04 \n#> 5 Bootstrap05 \n#> 6 Bootstrap06 \n#> 7 Bootstrap07 \n#> 8 Bootstrap08 \n#> 9 Bootstrap09 \n#> 10 Bootstrap10 \n#> # ℹ 20 more rows\n```\n:::\n\n\nThe best setting here is:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nshow_best(recipe_res, metric = \"roc_auc\")\n#> # A tibble: 5 × 8\n#> cost rbf_sigma .metric .estimator mean n std_err .config \n#> \n#> 1 3.17 1 roc_auc binary 0.971 30 0.00248 pre0_mod08_post0\n#> 2 0.0992 0.0774 roc_auc binary 0.970 30 0.00261 pre0_mod05_post0\n#> 3 0.00310 0.00599 roc_auc binary 0.959 30 0.00290 pre0_mod02_post0\n#> 4 32 0.000464 roc_auc binary 0.927 30 0.00484 pre0_mod10_post0\n#> 5 1 0.0000359 roc_auc binary 0.873 30 0.00533 pre0_mod07_post0\n```\n:::\n\n\n## Out-of-sample predictions\n\nIf we used `save_pred = TRUE` to keep the out-of-sample predictions for each resample during tuning, we can obtain those predictions, along with the tuning parameters and resample identifier, using `collect_predictions()`:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ncollect_predictions(recipe_res)\n#> # A tibble: 38,740 × 8\n#> .pred_bad .pred_good id Class .row cost rbf_sigma .config \n#> \n#> 1 0.333 0.667 Bootstrap01 good 1 0.000977 0.000000215 pre0_mod01…\n#> 2 0.333 0.667 Bootstrap01 good 9 0.000977 0.000000215 pre0_mod01…\n#> 3 0.333 0.667 Bootstrap01 bad 10 0.000977 0.000000215 pre0_mod01…\n#> 4 0.333 0.667 Bootstrap01 bad 12 0.000977 0.000000215 pre0_mod01…\n#> 5 0.333 0.667 Bootstrap01 bad 14 0.000977 0.000000215 pre0_mod01…\n#> 6 0.333 0.667 Bootstrap01 good 15 0.000977 0.000000215 pre0_mod01…\n#> 7 0.333 0.667 Bootstrap01 bad 16 0.000977 0.000000215 pre0_mod01…\n#> 8 0.333 0.667 Bootstrap01 bad 22 0.000977 0.000000215 pre0_mod01…\n#> 9 0.333 0.667 Bootstrap01 good 23 0.000977 0.000000215 pre0_mod01…\n#> 10 0.333 0.667 Bootstrap01 bad 24 0.000977 0.000000215 pre0_mod01…\n#> # ℹ 38,730 more rows\n```\n:::\n\n\nWe can obtain the hold-out sets for all the resamples augmented with the predictions using `augment()`, which provides opportunities for flexible visualization of model results:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\naugment(recipe_res) |>\n ggplot(aes(V3, .pred_good, color = Class)) +\n geom_point(show.legend = FALSE) +\n facet_wrap(~Class)\n```\n\n::: {.cell-output-display}\n![](figs/augment-preds-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\n## Session information {#session-info}\n\n\n::: {.cell layout-align=\"center\"}\n\n```\n#> ─ Session info ─────────────────────────────────────────────────────\n#> version R version 4.6.0 (2026-04-24)\n#> language (EN)\n#> pandoc 3.1.3\n#> quarto 1.9.37\n#> \n#> ─ Packages ─────────────────────────────────────────────────────────\n#> package version date (UTC)\n#> broom 1.0.13 2026-05-14\n#> dials 1.4.3 2026-04-11\n#> dplyr 1.2.1 2026-04-03\n#> future 1.70.0 2026-03-14\n#> ggplot2 4.0.3 2026-04-22\n#> infer 1.1.0 2025-12-18\n#> kernlab 0.9-33 2024-08-13\n#> mlbench 2.1-8 2026-03-26\n#> parsnip 1.6.0 2026-05-14\n#> purrr 1.2.2 2026-04-10\n#> recipes 1.3.2 2026-04-02\n#> rlang 1.2.0 2026-04-06\n#> rsample 1.3.2 2026-01-30\n#> tibble 3.3.1 2026-01-11\n#> tidymodels 1.5.0 2026-04-23\n#> tune 2.1.0 2026-04-17\n#> workflows 1.3.0 2025-08-27\n#> yardstick 1.4.0 2026-04-07\n#> \n#> ────────────────────────────────────────────────────────────────────\n```\n:::\n\n", "supporting": [], "filters": [ "rmarkdown/pagebreak.lua" diff --git a/_freeze/learn/work/tune-text/index/execute-results/html.json b/_freeze/learn/work/tune-text/index/execute-results/html.json index 4ba67431..3ef2bf2b 100644 --- a/_freeze/learn/work/tune-text/index/execute-results/html.json +++ b/_freeze/learn/work/tune-text/index/execute-results/html.json @@ -1,8 +1,8 @@ { - "hash": "7e253705d445a2c647f36d1cf5b86968", + "hash": "1b1fdc6c9a4256928171b8be18350ddd", "result": { "engine": "knitr", - "markdown": "---\ntitle: \"Tuning text models\"\ncategories:\n - tuning\n - regression\n - sparse data\n\ntype: learn-subsection\nweight: 4\ndescription: | \n Prepare text data for predictive modeling and tune with both grid and iterative search.\ntoc: true\ntoc-depth: 2\nr-packages:\n - tidymodels\n - textrecipes\n - stopwords\n - text2vec\n - future\ninclude-after-body: ../../../html/resources.html\n---\n\n\n\n\n\n## Introduction\n\nTo use code in this article, you will need to install the following packages: future, stopwords, text2vec, textrecipes, and tidymodels.\n\nThis article demonstrates an advanced example for training and tuning models for text data. Text data must be processed and transformed to a numeric representation to be ready for computation in modeling; in tidymodels, we use a recipe for this preprocessing. This article also shows how to extract information from each model fit during tuning to use later on.\n\n\n## Text as data\n\nThe text data we'll use in this article are from Amazon: \n\n> This dataset consists of reviews of fine foods from amazon. The data span a period of more than 10 years, including all ~500,000 reviews up to October 2012. Reviews include product and user information, ratings, and a plaintext review.\n\nThis article uses a small subset of the total reviews [available at the original source](https://snap.stanford.edu/data/web-FineFoods.html). We sampled a single review from 5,000 random products and allocated 80% of these data to the training set, with the remaining 1,000 reviews held out for the test set. \n\nThere is a column for the product, a column for the text of the review, and a factor column for the outcome variable. The outcome is whether the reviewer gave the product a five-star rating or not.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(tidymodels)\n\ndata(\"small_fine_foods\")\ntraining_data\n#> # A tibble: 4,000 × 3\n#> product review score\n#> \n#> 1 B000J0LSBG \"this stuff is not stuffing its not good at all save yo… other\n#> 2 B000EYLDYE \"I absolutely LOVE this dried fruit. LOVE IT. Whenever I … great\n#> 3 B0026LIO9A \"GREAT DEAL, CONVENIENT TOO. Much cheaper than WalMart and… great\n#> 4 B00473P8SK \"Great flavor, we go through a ton of this sauce! I discove… great\n#> 5 B001SAWTNM \"This is excellent salsa/hot sauce, but you can get it for … great\n#> 6 B000FAG90U \"Again, this is the best dogfood out there. One suggestion… great\n#> 7 B006BXTCEK \"The box I received was filled with teas, hot chocolates, a… other\n#> 8 B002GWH5OY \"This is delicious coffee which compares favorably with muc… great\n#> 9 B003R0MFYY \"Don't let these little tiny cans fool you. They pack a lo… great\n#> 10 B001EO5ZXI \"One of the nicest, smoothest cup of chai I've made. Nice m… great\n#> # ℹ 3,990 more rows\n```\n:::\n\n\nOur modeling goal is to create modeling features from the text of the reviews to predict whether the review was five-star or not.\n\n## Inputs for the search\n\nText, perhaps more so than tabular data we often deal with, must be heavily processed to be used as predictor data for modeling. There are multiple ways to process and prepare text for modeling; let's add several steps together to create different kinds of features:\n\n* Create an initial set of count-based features, such as the number of words, spaces, lower- or uppercase characters, URLs, and so on; we can use the [step_textfeatures()](https://textrecipes.tidymodels.org/reference/step_textfeature.html) fun for this.\n\n* [Tokenize](https://smltar.com/tokenization.html) the text (i.e. break the text into smaller components such as words).\n\n* Remove stop words such as \"the\", \"an\", \"of\", etc.\n\n* [Stem](https://smltar.com/stemming.html) tokens to a common root where possible.\n\n* Convert tokens to dummy variables via a [unsigned, binary hash function](https://bookdown.org/max/FES/encoding-predictors-with-many-categories.html).\n\n* Optionally transform non-token features (the count-based features like number of lowercase characters) to a more symmetric state using a [Yeo-Johnson transformation](https://bookdown.org/max/FES/numeric-one-to-one.html).\n\n* Remove predictors with a single distinct value.\n\n* Center and scale all predictors. \n\n\n::: {.callout-note}\n We will end up with two kinds of features:\n\n- dummy/indicator variables for the count-based features like number of digits or punctuation characters \n- hash features for the tokens like \"salsa\" or \"delicious\". \n:::\n\nSome of these preprocessing steps (such as stemming) may or may not be good ideas but a full discussion of their effects is beyond the scope of this article. In this preprocessing approach, the main tuning parameter is the number of hashing features to use. \n\nBefore we start building our preprocessing recipe, we need some helper objects. For example, for the Yeo-Johnson transformation, we need to know the set of count-based text features: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nbasics <- names(textrecipes::count_functions)\nhead(basics)\n#> [1] \"n_words\" \"n_uq_words\" \"n_charS\" \"n_uq_charS\" \"n_digits\" \n#> [6] \"n_hashtags\"\n```\n:::\n\n\n::: {.callout-note}\nSince we will be using a glmnet model which is [sparse compatible](../../../find/sparse/index.qmd) then we will set `sparse = \"yes\"` in `step_texthash()` as we know it will produce plenty of sparse data. In this example, it will lead to a 4 times speed increase.\n:::\n\nNow, let's put this all together in one recipe:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(textrecipes)\n\npre_proc <-\n recipe(score ~ product + review, data = training_data) |>\n # Do not use the product ID as a predictor\n update_role(product, new_role = \"id\") |>\n # Make a copy of the raw text\n step_mutate(review_raw = review) |>\n # Compute the initial features. This removes the `review_raw` column\n step_textfeature(review_raw) |>\n # Make the feature names shorter\n step_rename_at(\n starts_with(\"textfeature_\"),\n fn = ~ gsub(\"textfeature_review_raw_\", \"\", .)\n ) |>\n step_tokenize(review) |>\n step_stopwords(review) |>\n step_stem(review) |>\n # Here is where the tuning parameter is declared\n step_texthash(review, signed = FALSE, num_terms = tune(), sparse = \"yes\") |>\n # Simplify these names\n step_rename_at(starts_with(\"review_hash\"), fn = ~ gsub(\"review_\", \"\", .)) |>\n # Transform the initial feature set\n step_YeoJohnson(one_of(!!basics)) |>\n step_zv(all_predictors()) |>\n step_scale(all_predictors())\n```\n:::\n\n\n::: {.callout-warning}\n Note that, when objects from the global environment are used, they are injected into the step objects via `!!`. For some parallel processing technologies, these objects may not be found by the worker processes. \n:::\n\nThe preprocessing recipe is long and complex (often typical for working with text data) but the model we'll use is more straightforward. Let's stick with a regularized logistic regression model: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlr_mod <-\n logistic_reg(penalty = tune(), mixture = tune()) |>\n set_engine(\"glmnet\")\n```\n:::\n\n\nThere are three tuning parameters for this data analysis:\n\n- `num_terms`, the number of feature hash variables to create\n- `penalty`, the amount of regularization for the model\n- `mixture`, the proportion of L1 regularization\n\n## Resampling\n\nThere is enough data here so that 5-fold resampling would hold out 800 reviews at a time to estimate performance. Performance estimates using this many observations have sufficiently low noise to measure and tune models. \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(8935)\nfolds <- vfold_cv(training_data, v = 5)\nfolds\n#> # 5-fold cross-validation \n#> # A tibble: 5 × 2\n#> splits id \n#> \n#> 1 Fold1\n#> 2 Fold2\n#> 3 Fold3\n#> 4 Fold4\n#> 5 Fold5\n```\n:::\n\n\n## Grid search\n\nLet's begin our tuning with [grid search](https://www.tidymodels.org/learn/work/tune-svm/) and a regular grid. For glmnet models, evaluating penalty values is fairly cheap because of the use of the [\"submodel-trick\"](https://tune.tidymodels.org/articles/extras/optimizations.html#sub-model-speed-ups-1). The grid will use 20 penalty values, 5 mixture values, and 3 values for the number of hash features. \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nfive_star_grid <- \n crossing(\n penalty = 10^seq(-3, 0, length = 20),\n mixture = c(0.01, 0.25, 0.50, 0.75, 1),\n num_terms = 2^c(8, 10, 12)\n )\nfive_star_grid\n#> # A tibble: 300 × 3\n#> penalty mixture num_terms\n#> \n#> 1 0.001 0.01 256\n#> 2 0.001 0.01 1024\n#> 3 0.001 0.01 4096\n#> 4 0.001 0.25 256\n#> 5 0.001 0.25 1024\n#> 6 0.001 0.25 4096\n#> 7 0.001 0.5 256\n#> 8 0.001 0.5 1024\n#> 9 0.001 0.5 4096\n#> 10 0.001 0.75 256\n#> # ℹ 290 more rows\n```\n:::\n\n\nNote that, for each resample, the (computationally expensive) text preprocessing recipe is only prepped 6 times. This increases the efficiency of the analysis by avoiding redundant work. \n\nLet's save information on the number of predictors by penalty value for each glmnet model. This can help us understand how many features were used across the penalty values. Use an extraction function to do this:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nglmnet_vars <- function(x) {\n # `x` will be a workflow object\n mod <- extract_fit_engine(x)\n # `df` is the number of model terms for each penalty value\n tibble(penalty = mod$lambda, num_vars = mod$df)\n}\n\nctrl <- control_grid(extract = glmnet_vars, verbose = TRUE)\n```\n:::\n\n\nFinally, let's run the grid search:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nroc_scores <- metric_set(roc_auc)\n\nset.seed(1559)\nfive_star_glmnet <-\n tune_grid(\n lr_mod,\n pre_proc,\n resamples = folds,\n grid = five_star_grid,\n metrics = roc_scores,\n control = ctrl\n )\n\nfive_star_glmnet\n#> # Tuning results\n#> # 5-fold cross-validation \n#> # A tibble: 5 × 5\n#> splits id .metrics .notes .extracts\n#> \n#> 1 Fold1 \n#> 2 Fold2 \n#> 3 Fold3 \n#> 4 Fold4 \n#> 5 Fold5 \n```\n:::\n\n\nWhat do the results look like? Let's get the resampling estimates of the area under the ROC curve for each tuning parameter:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ngrid_roc <- \n collect_metrics(five_star_glmnet) |> \n arrange(desc(mean))\ngrid_roc\n#> # A tibble: 300 × 9\n#> penalty mixture num_terms .metric .estimator mean n std_err .config \n#> \n#> 1 0.483 0.01 4096 roc_auc binary 0.799 5 0.00589 pre3_mod086…\n#> 2 0.336 0.01 4096 roc_auc binary 0.798 5 0.00509 pre3_mod081…\n#> 3 0.695 0.01 4096 roc_auc binary 0.798 5 0.00638 pre3_mod091…\n#> 4 0.0264 0.25 4096 roc_auc binary 0.795 5 0.00250 pre3_mod047…\n#> 5 0.234 0.01 4096 roc_auc binary 0.795 5 0.00465 pre3_mod076…\n#> 6 0.0379 0.25 4096 roc_auc binary 0.793 5 0.00354 pre3_mod052…\n#> 7 0.0127 0.5 4096 roc_auc binary 0.793 5 0.00245 pre3_mod038…\n#> 8 0.0546 0.25 4096 roc_auc binary 0.792 5 0.00589 pre3_mod057…\n#> 9 0.0183 0.5 4096 roc_auc binary 0.792 5 0.00329 pre3_mod043…\n#> 10 0.0183 0.25 4096 roc_auc binary 0.792 5 0.00230 pre3_mod042…\n#> # ℹ 290 more rows\n```\n:::\n\n\nThe best results have a fairly high penalty value and focus on the ridge penalty (i.e. no feature selection via the lasso's L1 penalty). The best solutions also use the largest number of hashing features. \n\nWhat is the relationship between performance and the tuning parameters? \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nautoplot(five_star_glmnet, metric = \"roc_auc\")\n```\n\n::: {.cell-output-display}\n![](figs/grid-plot-1.svg){fig-align='center' width=960}\n:::\n:::\n\n\n- We can definitely see that performance improves with the number of features included. In this article, we've used a small sample of the overall data set available. When more data are used, an even larger feature set is optimal. \n\n- The profiles with larger mixture values (greater than 0.01) have steep drop-offs in performance. What's that about? Those are cases where the lasso penalty is removing too many (and perhaps all) features from the model. \n- The panel with at least 4096 features shows that there are several parameter combinations that have about the same performance; there isn't much difference between the best performance for the different mixture values. A case could be made that we should choose a _larger_ mixture value and a _smaller_ penalty to select a simpler model that contains fewer predictors. \n\n- If more experimentation were conducted, a larger set of features (more than 4096) should also be considered. \n\nWe'll come back to the extracted glmnet components at the end of this article. \n\n## Directed search\n\nWhat if we had started with Bayesian optimization? Would a good set of conditions have been found more efficiently? \n\nLet's pretend that we haven't seen the grid search results. We'll initialize the Gaussian process model with five tuning parameter combinations chosen with a space-filling design. \n\nIt might be good to use a custom `dials` object for the number of hash terms. The default object, `num_terms()`, uses a linear range and tries to set the upper bound of the parameter using the data. Instead, let's create a parameter set, change the scale to be `log2`, and define the same range as was used in grid search. \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nhash_range <- num_terms(c(8, 12), trans = log2_trans())\nhash_range\n#> # Model Terms (quantitative)\n#> Transformer: log-2 [1e-100, Inf]\n#> Range (transformed scale): [8, 12]\n```\n:::\n\n\nTo use this, we have to merge the recipe and `parsnip` model object into a workflow:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nfive_star_wflow <-\n workflow() |>\n add_recipe(pre_proc) |>\n add_model(lr_mod)\n```\n:::\n\n\nThen we can extract and manipulate the corresponding parameter set:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nfive_star_set <-\n five_star_wflow |>\n extract_parameter_set_dials() |>\n update(\n num_terms = hash_range, \n penalty = penalty(c(-3, 0)),\n mixture = mixture(c(0.05, 1.00))\n )\n```\n:::\n\n\nThis is passed to the search function via the `param_info` argument. \n\nThe initial rounds of search can be biased more towards exploration of the parameter space (as opposed to staying near the current best results). If expected improvement is used as the acquisition function, the trade-off value can be slowly moved from exploration to exploitation over iterations (see the tune vignette on [acquisition functions](https://tune.tidymodels.org/articles/acquisition_functions.html) for more details). The tune package has a built-in function called `expo_decay()` that can help accomplish this:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ntrade_off_decay <- function(iter) {\n expo_decay(iter, start_val = .01, limit_val = 0, slope = 1/4)\n}\n```\n:::\n\n\nUsing these values, let's run the search:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(12)\nfive_star_search <-\n tune_bayes(\n five_star_wflow, \n resamples = folds,\n param_info = five_star_set,\n initial = 5,\n iter = 10,\n metrics = roc_scores,\n objective = exp_improve(trade_off_decay),\n control = control_bayes(verbose_iter = TRUE)\n )\n#> Optimizing roc_auc using the expected improvement with variable trade-off\n#> values.\n#> \n#> ── Iteration 1 ───────────────────────────────────────────────────────\n#> \n#> i Current best:\t\troc_auc=0.7809 (@iter 0)\n#> ✔ Gaussian process model (LOO R²: 75.5%)\n#> ℹ Generating 5000 candidates.\n#> i Trade-off value: 0.01\n#> i penalty=0.00474, mixture=0.0526, num_terms=1028\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results:\troc_auc=0.7302 (+/-0.00754)\n#> \n#> ── Iteration 2 ───────────────────────────────────────────────────────\n#> \n#> i Current best:\t\troc_auc=0.7809 (@iter 0)\n#> ⓧ GP has a LOO R² of 1.2% and is unreliable.\n#> ✔ Gaussian process model failed\n#> ℹ Generating a candidate as far away from existing points as possible.\n#> i Trade-off value: 0.007788\n#> ℹ Uncertainty sample\n#> i penalty=0.898, mixture=0.582, num_terms=291\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results:\troc_auc=0.5\n#> \n#> ── Iteration 3 ───────────────────────────────────────────────────────\n#> \n#> i Current best:\t\troc_auc=0.7809 (@iter 0)\n#> ✔ Gaussian process model (LOO R²: 75.3%)\n#> ℹ Generating 5000 candidates.\n#> i Trade-off value: 0.006065\n#> i penalty=0.0092, mixture=0.981, num_terms=3813\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ♥ Newest results:\troc_auc=0.7985 (+/-0.00368)\n#> \n#> ── Iteration 4 ───────────────────────────────────────────────────────\n#> \n#> i Current best:\t\troc_auc=0.7985 (@iter 3)\n#> ✔ Gaussian process model (LOO R²: 83.9%)\n#> ℹ Generating 5000 candidates.\n#> i Trade-off value: 0.004724\n#> i penalty=0.00496, mixture=0.982, num_terms=3680\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results:\troc_auc=0.7921 (+/-0.00591)\n#> \n#> ── Iteration 5 ───────────────────────────────────────────────────────\n#> \n#> i Current best:\t\troc_auc=0.7985 (@iter 3)\n#> ✔ Gaussian process model (LOO R²: 87.1%)\n#> ℹ Generating 5000 candidates.\n#> i Trade-off value: 0.003679\n#> i penalty=0.00713, mixture=0.982, num_terms=344\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results:\troc_auc=0.7525 (+/-0.0038)\n#> \n#> ── Iteration 6 ───────────────────────────────────────────────────────\n#> \n#> i Current best:\t\troc_auc=0.7985 (@iter 3)\n#> ✔ Gaussian process model (LOO R²: 86%)\n#> ℹ Generating 5000 candidates.\n#> i Trade-off value: 0.002865\n#> i penalty=0.0176, mixture=0.933, num_terms=3730\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results:\troc_auc=0.7854 (+/-0.00944)\n#> \n#> ── Iteration 7 ───────────────────────────────────────────────────────\n#> \n#> i Current best:\t\troc_auc=0.7985 (@iter 3)\n#> ✔ Gaussian process model (LOO R²: 89.2%)\n#> ℹ Generating 5000 candidates.\n#> i Trade-off value: 0.002231\n#> i penalty=0.0124, mixture=0.309, num_terms=4046\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ♥ Newest results:\troc_auc=0.8019 (+/-0.00607)\n#> \n#> ── Iteration 8 ───────────────────────────────────────────────────────\n#> \n#> i Current best:\t\troc_auc=0.8019 (@iter 7)\n#> ✔ Gaussian process model (LOO R²: 92%)\n#> ℹ Generating 5000 candidates.\n#> i Trade-off value: 0.001738\n#> i penalty=0.0163, mixture=0.0649, num_terms=3265\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results:\troc_auc=0.7683 (+/-0.0113)\n#> \n#> ── Iteration 9 ───────────────────────────────────────────────────────\n#> \n#> i Current best:\t\troc_auc=0.8019 (@iter 7)\n#> ✔ Gaussian process model (LOO R²: 88.6%)\n#> ℹ Generating 5000 candidates.\n#> i Trade-off value: 0.001353\n#> i penalty=0.0111, mixture=0.693, num_terms=3559\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ♥ Newest results:\troc_auc=0.8038 (+/-0.00636)\n#> \n#> ── Iteration 10 ──────────────────────────────────────────────────────\n#> \n#> i Current best:\t\troc_auc=0.8038 (@iter 9)\n#> ✔ Gaussian process model (LOO R²: 89.3%)\n#> ℹ Generating 5000 candidates.\n#> i Trade-off value: 0.001054\n#> i penalty=0.0121, mixture=0.729, num_terms=4070\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results:\troc_auc=0.8008 (+/-0.00718)\n\nfive_star_search\n#> # Tuning results\n#> # 5-fold cross-validation \n#> # A tibble: 55 × 5\n#> splits id .metrics .notes .iter\n#> \n#> 1 Fold1 0\n#> 2 Fold2 0\n#> 3 Fold3 0\n#> 4 Fold4 0\n#> 5 Fold5 0\n#> 6 Fold1 1\n#> 7 Fold2 1\n#> 8 Fold3 1\n#> 9 Fold4 1\n#> 10 Fold5 1\n#> # ℹ 45 more rows\n```\n:::\n\n\nThese results show some improvement over the initial set. One issue is that so many settings are sub-optimal (as shown in the plot above for grid search) so there are poor results periodically. There are regions where the penalty parameter becomes too large and all of the predictors are removed from the model. These regions are also dependent on the number of terms. There is a fairly narrow ridge (sorry, pun intended!) where good performance can be achieved. Using more iterations would probably result in the search finding better results. \nLet's look at a plot of model performance versus the search iterations:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nautoplot(five_star_search, type = \"performance\")\n```\n\n::: {.cell-output-display}\n![](figs/iter-plot-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\n::: {.callout-note}\nWhat would we do if we knew about the grid search results and wanted to try directed, iterative search? We would restrict the range for the number of hash features to be larger (especially with more data). We might also restrict the penalty and mixture parameters to have a lower upper bound. \n:::\n\n## Extracted results\n\nLet's return to the grid search results and examine the results of our `extract` function. For each _fitted model_, a tibble was saved that contains the relationship between the number of predictors and the penalty value. Let's look at these results for the best model:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nparams <- select_best(five_star_glmnet, metric = \"roc_auc\")\nparams\n#> # A tibble: 1 × 4\n#> penalty mixture num_terms .config \n#> \n#> 1 0.483 0.01 4096 pre3_mod086_post0\n```\n:::\n\n\nRecall that we saved the glmnet results in a tibble. The column `five_star_glmnet$.extracts` is a list of tibbles. As an example, the first element of the list is:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nfive_star_glmnet$.extracts[[1]]\n#> # A tibble: 300 × 5\n#> penalty mixture num_terms .extracts .config \n#> \n#> 1 0.001 0.01 256 pre1_mod001_post0\n#> 2 0.001 0.25 256 pre1_mod002_post0\n#> 3 0.001 0.5 256 pre1_mod003_post0\n#> 4 0.001 0.75 256 pre1_mod004_post0\n#> 5 0.001 1 256 pre1_mod005_post0\n#> 6 0.00144 0.01 256 pre1_mod006_post0\n#> 7 0.00144 0.25 256 pre1_mod007_post0\n#> 8 0.00144 0.5 256 pre1_mod008_post0\n#> 9 0.00144 0.75 256 pre1_mod009_post0\n#> 10 0.00144 1 256 pre1_mod010_post0\n#> # ℹ 290 more rows\n```\n:::\n\n\nMore nested tibbles! Let's `unnest()` the `five_star_glmnet$.extracts` column:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(tidyr)\nextracted <- \n five_star_glmnet |> \n dplyr::select(id, .extracts) |> \n unnest(cols = .extracts)\nextracted\n#> # A tibble: 1,500 × 6\n#> id penalty mixture num_terms .extracts .config \n#> \n#> 1 Fold1 0.001 0.01 256 pre1_mod001_post0\n#> 2 Fold1 0.001 0.25 256 pre1_mod002_post0\n#> 3 Fold1 0.001 0.5 256 pre1_mod003_post0\n#> 4 Fold1 0.001 0.75 256 pre1_mod004_post0\n#> 5 Fold1 0.001 1 256 pre1_mod005_post0\n#> 6 Fold1 0.00144 0.01 256 pre1_mod006_post0\n#> 7 Fold1 0.00144 0.25 256 pre1_mod007_post0\n#> 8 Fold1 0.00144 0.5 256 pre1_mod008_post0\n#> 9 Fold1 0.00144 0.75 256 pre1_mod009_post0\n#> 10 Fold1 0.00144 1 256 pre1_mod010_post0\n#> # ℹ 1,490 more rows\n```\n:::\n\n\nOne thing to realize here is that `tune_grid()` [may not fit all of the models](https://tune.tidymodels.org/articles/extras/optimizations.html) that are evaluated. In this case, for each value of `mixture` and `num_terms`, the model is fit over _all_ penalty values (this is a feature of this particular model and is not generally true for other engines). To select the best parameter set, we can exclude the `penalty` column in `extracted`:\n\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nextracted <- \n extracted |> \n dplyr::select(-penalty) |> \n inner_join(params, by = c(\"num_terms\", \"mixture\")) |> \n # Now remove it from the final results\n dplyr::select(-penalty)\nextracted\n#> # A tibble: 100 × 6\n#> id mixture num_terms .extracts .config.x .config.y \n#> \n#> 1 Fold1 0.01 4096 pre3_mod001_post0 pre3_mod086_pos…\n#> 2 Fold1 0.01 4096 pre3_mod006_post0 pre3_mod086_pos…\n#> 3 Fold1 0.01 4096 pre3_mod011_post0 pre3_mod086_pos…\n#> 4 Fold1 0.01 4096 pre3_mod016_post0 pre3_mod086_pos…\n#> 5 Fold1 0.01 4096 pre3_mod021_post0 pre3_mod086_pos…\n#> 6 Fold1 0.01 4096 pre3_mod026_post0 pre3_mod086_pos…\n#> 7 Fold1 0.01 4096 pre3_mod031_post0 pre3_mod086_pos…\n#> 8 Fold1 0.01 4096 pre3_mod036_post0 pre3_mod086_pos…\n#> 9 Fold1 0.01 4096 pre3_mod041_post0 pre3_mod086_pos…\n#> 10 Fold1 0.01 4096 pre3_mod046_post0 pre3_mod086_pos…\n#> # ℹ 90 more rows\n```\n:::\n\n\nNow we can get at the results that we want using another `unnest()`:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nextracted <- \n extracted |> \n unnest(col = .extracts) # <- these contain a `penalty` column\nextracted\n#> # A tibble: 10,000 × 7\n#> id mixture num_terms penalty num_vars .config.x .config.y \n#> \n#> 1 Fold1 0.01 4096 8.81 0 pre3_mod001_post0 pre3_mod086_post0\n#> 2 Fold1 0.01 4096 8.41 2 pre3_mod001_post0 pre3_mod086_post0\n#> 3 Fold1 0.01 4096 8.03 2 pre3_mod001_post0 pre3_mod086_post0\n#> 4 Fold1 0.01 4096 7.67 2 pre3_mod001_post0 pre3_mod086_post0\n#> 5 Fold1 0.01 4096 7.32 2 pre3_mod001_post0 pre3_mod086_post0\n#> 6 Fold1 0.01 4096 6.98 3 pre3_mod001_post0 pre3_mod086_post0\n#> 7 Fold1 0.01 4096 6.67 3 pre3_mod001_post0 pre3_mod086_post0\n#> 8 Fold1 0.01 4096 6.36 6 pre3_mod001_post0 pre3_mod086_post0\n#> 9 Fold1 0.01 4096 6.08 6 pre3_mod001_post0 pre3_mod086_post0\n#> 10 Fold1 0.01 4096 5.80 7 pre3_mod001_post0 pre3_mod086_post0\n#> # ℹ 9,990 more rows\n```\n:::\n\n\nLet's look at a plot of these results (per resample):\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nggplot(extracted, aes(x = penalty, y = num_vars)) + \n geom_line(aes(group = id, col = id), alpha = .5) + \n ylab(\"Number of retained predictors\") + \n scale_x_log10() + \n ggtitle(paste(\"mixture = \", params$mixture, \"and\", params$num_terms, \"features\")) + \n theme(legend.position = \"none\")\n```\n\n::: {.cell-output-display}\n![](figs/var-plot-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nThese results might help guide the choice of the `penalty` range if more optimization was conducted. \n\n## Session information {#session-info}\n\n\n::: {.cell layout-align=\"center\"}\n\n```\n#> ─ Session info ─────────────────────────────────────────────────────\n#> version R version 4.6.0 (2026-04-24)\n#> language (EN)\n#> pandoc 3.1.3\n#> quarto 1.9.37\n#> \n#> ─ Packages ─────────────────────────────────────────────────────────\n#> package version date (UTC)\n#> broom 1.0.13 2026-05-14\n#> dials 1.4.3 2026-04-11\n#> dplyr 1.2.1 2026-04-03\n#> future 1.70.0 2026-03-14\n#> ggplot2 4.0.3 2026-04-22\n#> infer 1.1.0 2025-12-18\n#> parsnip 1.6.0 2026-05-14\n#> purrr 1.2.2 2026-04-10\n#> recipes 1.3.2 2026-04-02\n#> rlang 1.2.0 2026-04-06\n#> rsample 1.3.2 2026-01-30\n#> stopwords 2.3 2021-10-28\n#> text2vec 0.6.6 2025-12-01\n#> textrecipes 1.1.0 2025-03-18\n#> tibble 3.3.1 2026-01-11\n#> tidymodels 1.5.0 2026-04-23\n#> tune 2.1.0 2026-04-17\n#> workflows 1.3.0 2025-08-27\n#> yardstick 1.4.0 2026-04-07\n#> \n#> ────────────────────────────────────────────────────────────────────\n```\n:::\n\n", + "markdown": "---\ntitle: \"Tuning text models\"\ncategories:\n - tuning and workflows\n - tuning\n - regression\n - sparse data\n\ntype: learn-subsection\nweight: 4\ndescription: | \n Prepare text data for predictive modeling and tune with both grid and iterative search.\ntoc: true\ntoc-depth: 2\nr-packages:\n - tidymodels\n - textrecipes\n - stopwords\n - text2vec\n - future\ninclude-after-body: ../../../html/resources.html\n---\n\n\n\n\n\n## Introduction\n\nTo use code in this article, you will need to install the following packages: future, stopwords, text2vec, textrecipes, and tidymodels.\n\nThis article demonstrates an advanced example for training and tuning models for text data. Text data must be processed and transformed to a numeric representation to be ready for computation in modeling; in tidymodels, we use a recipe for this preprocessing. This article also shows how to extract information from each model fit during tuning to use later on.\n\n\n## Text as data\n\nThe text data we'll use in this article are from Amazon: \n\n> This dataset consists of reviews of fine foods from amazon. The data span a period of more than 10 years, including all ~500,000 reviews up to October 2012. Reviews include product and user information, ratings, and a plaintext review.\n\nThis article uses a small subset of the total reviews [available at the original source](https://snap.stanford.edu/data/web-FineFoods.html). We sampled a single review from 5,000 random products and allocated 80% of these data to the training set, with the remaining 1,000 reviews held out for the test set. \n\nThere is a column for the product, a column for the text of the review, and a factor column for the outcome variable. The outcome is whether the reviewer gave the product a five-star rating or not.\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(tidymodels)\n\ndata(\"small_fine_foods\")\ntraining_data\n#> # A tibble: 4,000 × 3\n#> product review score\n#> \n#> 1 B000J0LSBG \"this stuff is not stuffing its not good at all save yo… other\n#> 2 B000EYLDYE \"I absolutely LOVE this dried fruit. LOVE IT. Whenever I … great\n#> 3 B0026LIO9A \"GREAT DEAL, CONVENIENT TOO. Much cheaper than WalMart and… great\n#> 4 B00473P8SK \"Great flavor, we go through a ton of this sauce! I discove… great\n#> 5 B001SAWTNM \"This is excellent salsa/hot sauce, but you can get it for … great\n#> 6 B000FAG90U \"Again, this is the best dogfood out there. One suggestion… great\n#> 7 B006BXTCEK \"The box I received was filled with teas, hot chocolates, a… other\n#> 8 B002GWH5OY \"This is delicious coffee which compares favorably with muc… great\n#> 9 B003R0MFYY \"Don't let these little tiny cans fool you. They pack a lo… great\n#> 10 B001EO5ZXI \"One of the nicest, smoothest cup of chai I've made. Nice m… great\n#> # ℹ 3,990 more rows\n```\n:::\n\n\nOur modeling goal is to create modeling features from the text of the reviews to predict whether the review was five-star or not.\n\n## Inputs for the search\n\nText, perhaps more so than tabular data we often deal with, must be heavily processed to be used as predictor data for modeling. There are multiple ways to process and prepare text for modeling; let's add several steps together to create different kinds of features:\n\n* Create an initial set of count-based features, such as the number of words, spaces, lower- or uppercase characters, URLs, and so on; we can use the [step_textfeatures()](https://textrecipes.tidymodels.org/reference/step_textfeature.html) fun for this.\n\n* [Tokenize](https://smltar.com/tokenization.html) the text (i.e. break the text into smaller components such as words).\n\n* Remove stop words such as \"the\", \"an\", \"of\", etc.\n\n* [Stem](https://smltar.com/stemming.html) tokens to a common root where possible.\n\n* Convert tokens to dummy variables via a [unsigned, binary hash function](https://bookdown.org/max/FES/encoding-predictors-with-many-categories.html).\n\n* Optionally transform non-token features (the count-based features like number of lowercase characters) to a more symmetric state using a [Yeo-Johnson transformation](https://bookdown.org/max/FES/numeric-one-to-one.html).\n\n* Remove predictors with a single distinct value.\n\n* Center and scale all predictors. \n\n\n::: {.callout-note}\n We will end up with two kinds of features:\n\n- dummy/indicator variables for the count-based features like number of digits or punctuation characters \n- hash features for the tokens like \"salsa\" or \"delicious\". \n:::\n\nSome of these preprocessing steps (such as stemming) may or may not be good ideas but a full discussion of their effects is beyond the scope of this article. In this preprocessing approach, the main tuning parameter is the number of hashing features to use. \n\nBefore we start building our preprocessing recipe, we need some helper objects. For example, for the Yeo-Johnson transformation, we need to know the set of count-based text features: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nbasics <- names(textrecipes::count_functions)\nhead(basics)\n#> [1] \"n_words\" \"n_uq_words\" \"n_charS\" \"n_uq_charS\" \"n_digits\" \n#> [6] \"n_hashtags\"\n```\n:::\n\n\n::: {.callout-note}\nSince we will be using a glmnet model which is [sparse compatible](../../../find/sparse/index.qmd) then we will set `sparse = \"yes\"` in `step_texthash()` as we know it will produce plenty of sparse data. In this example, it will lead to a 4 times speed increase.\n:::\n\nNow, let's put this all together in one recipe:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(textrecipes)\n\npre_proc <-\n recipe(score ~ product + review, data = training_data) |>\n # Do not use the product ID as a predictor\n update_role(product, new_role = \"id\") |>\n # Make a copy of the raw text\n step_mutate(review_raw = review) |>\n # Compute the initial features. This removes the `review_raw` column\n step_textfeature(review_raw) |>\n # Make the feature names shorter\n step_rename_at(\n starts_with(\"textfeature_\"),\n fn = ~ gsub(\"textfeature_review_raw_\", \"\", .)\n ) |>\n step_tokenize(review) |>\n step_stopwords(review) |>\n step_stem(review) |>\n # Here is where the tuning parameter is declared\n step_texthash(review, signed = FALSE, num_terms = tune(), sparse = \"yes\") |>\n # Simplify these names\n step_rename_at(starts_with(\"review_hash\"), fn = ~ gsub(\"review_\", \"\", .)) |>\n # Transform the initial feature set\n step_YeoJohnson(one_of(!!basics)) |>\n step_zv(all_predictors()) |>\n step_scale(all_predictors())\n```\n:::\n\n\n::: {.callout-warning}\n Note that, when objects from the global environment are used, they are injected into the step objects via `!!`. For some parallel processing technologies, these objects may not be found by the worker processes. \n:::\n\nThe preprocessing recipe is long and complex (often typical for working with text data) but the model we'll use is more straightforward. Let's stick with a regularized logistic regression model: \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlr_mod <-\n logistic_reg(penalty = tune(), mixture = tune()) |>\n set_engine(\"glmnet\")\n```\n:::\n\n\nThere are three tuning parameters for this data analysis:\n\n- `num_terms`, the number of feature hash variables to create\n- `penalty`, the amount of regularization for the model\n- `mixture`, the proportion of L1 regularization\n\n## Resampling\n\nThere is enough data here so that 5-fold resampling would hold out 800 reviews at a time to estimate performance. Performance estimates using this many observations have sufficiently low noise to measure and tune models. \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(8935)\nfolds <- vfold_cv(training_data, v = 5)\nfolds\n#> # 5-fold cross-validation \n#> # A tibble: 5 × 2\n#> splits id \n#> \n#> 1 Fold1\n#> 2 Fold2\n#> 3 Fold3\n#> 4 Fold4\n#> 5 Fold5\n```\n:::\n\n\n## Grid search\n\nLet's begin our tuning with [grid search](https://www.tidymodels.org/learn/work/tune-svm/) and a regular grid. For glmnet models, evaluating penalty values is fairly cheap because of the use of the [\"submodel-trick\"](https://tune.tidymodels.org/articles/extras/optimizations.html#sub-model-speed-ups-1). The grid will use 20 penalty values, 5 mixture values, and 3 values for the number of hash features. \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nfive_star_grid <- \n crossing(\n penalty = 10^seq(-3, 0, length = 20),\n mixture = c(0.01, 0.25, 0.50, 0.75, 1),\n num_terms = 2^c(8, 10, 12)\n )\nfive_star_grid\n#> # A tibble: 300 × 3\n#> penalty mixture num_terms\n#> \n#> 1 0.001 0.01 256\n#> 2 0.001 0.01 1024\n#> 3 0.001 0.01 4096\n#> 4 0.001 0.25 256\n#> 5 0.001 0.25 1024\n#> 6 0.001 0.25 4096\n#> 7 0.001 0.5 256\n#> 8 0.001 0.5 1024\n#> 9 0.001 0.5 4096\n#> 10 0.001 0.75 256\n#> # ℹ 290 more rows\n```\n:::\n\n\nNote that, for each resample, the (computationally expensive) text preprocessing recipe is only prepped 6 times. This increases the efficiency of the analysis by avoiding redundant work. \n\nLet's save information on the number of predictors by penalty value for each glmnet model. This can help us understand how many features were used across the penalty values. Use an extraction function to do this:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nglmnet_vars <- function(x) {\n # `x` will be a workflow object\n mod <- extract_fit_engine(x)\n # `df` is the number of model terms for each penalty value\n tibble(penalty = mod$lambda, num_vars = mod$df)\n}\n\nctrl <- control_grid(extract = glmnet_vars, verbose = TRUE)\n```\n:::\n\n\nFinally, let's run the grid search:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nroc_scores <- metric_set(roc_auc)\n\nset.seed(1559)\nfive_star_glmnet <-\n tune_grid(\n lr_mod,\n pre_proc,\n resamples = folds,\n grid = five_star_grid,\n metrics = roc_scores,\n control = ctrl\n )\n\nfive_star_glmnet\n#> # Tuning results\n#> # 5-fold cross-validation \n#> # A tibble: 5 × 5\n#> splits id .metrics .notes .extracts\n#> \n#> 1 Fold1 \n#> 2 Fold2 \n#> 3 Fold3 \n#> 4 Fold4 \n#> 5 Fold5 \n```\n:::\n\n\nWhat do the results look like? Let's get the resampling estimates of the area under the ROC curve for each tuning parameter:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ngrid_roc <- \n collect_metrics(five_star_glmnet) |> \n arrange(desc(mean))\ngrid_roc\n#> # A tibble: 300 × 9\n#> penalty mixture num_terms .metric .estimator mean n std_err .config \n#> \n#> 1 0.483 0.01 4096 roc_auc binary 0.799 5 0.00589 pre3_mod086…\n#> 2 0.336 0.01 4096 roc_auc binary 0.798 5 0.00509 pre3_mod081…\n#> 3 0.695 0.01 4096 roc_auc binary 0.798 5 0.00638 pre3_mod091…\n#> 4 0.0264 0.25 4096 roc_auc binary 0.795 5 0.00250 pre3_mod047…\n#> 5 0.234 0.01 4096 roc_auc binary 0.795 5 0.00465 pre3_mod076…\n#> 6 0.0379 0.25 4096 roc_auc binary 0.793 5 0.00354 pre3_mod052…\n#> 7 0.0127 0.5 4096 roc_auc binary 0.793 5 0.00245 pre3_mod038…\n#> 8 0.0546 0.25 4096 roc_auc binary 0.792 5 0.00589 pre3_mod057…\n#> 9 0.0183 0.5 4096 roc_auc binary 0.792 5 0.00329 pre3_mod043…\n#> 10 0.0183 0.25 4096 roc_auc binary 0.792 5 0.00230 pre3_mod042…\n#> # ℹ 290 more rows\n```\n:::\n\n\nThe best results have a fairly high penalty value and focus on the ridge penalty (i.e. no feature selection via the lasso's L1 penalty). The best solutions also use the largest number of hashing features. \n\nWhat is the relationship between performance and the tuning parameters? \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nautoplot(five_star_glmnet, metric = \"roc_auc\")\n```\n\n::: {.cell-output-display}\n![](figs/grid-plot-1.svg){fig-align='center' width=960}\n:::\n:::\n\n\n- We can definitely see that performance improves with the number of features included. In this article, we've used a small sample of the overall data set available. When more data are used, an even larger feature set is optimal. \n\n- The profiles with larger mixture values (greater than 0.01) have steep drop-offs in performance. What's that about? Those are cases where the lasso penalty is removing too many (and perhaps all) features from the model. \n- The panel with at least 4096 features shows that there are several parameter combinations that have about the same performance; there isn't much difference between the best performance for the different mixture values. A case could be made that we should choose a _larger_ mixture value and a _smaller_ penalty to select a simpler model that contains fewer predictors. \n\n- If more experimentation were conducted, a larger set of features (more than 4096) should also be considered. \n\nWe'll come back to the extracted glmnet components at the end of this article. \n\n## Directed search\n\nWhat if we had started with Bayesian optimization? Would a good set of conditions have been found more efficiently? \n\nLet's pretend that we haven't seen the grid search results. We'll initialize the Gaussian process model with five tuning parameter combinations chosen with a space-filling design. \n\nIt might be good to use a custom `dials` object for the number of hash terms. The default object, `num_terms()`, uses a linear range and tries to set the upper bound of the parameter using the data. Instead, let's create a parameter set, change the scale to be `log2`, and define the same range as was used in grid search. \n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nhash_range <- num_terms(c(8, 12), trans = log2_trans())\nhash_range\n#> # Model Terms (quantitative)\n#> Transformer: log-2 [1e-100, Inf]\n#> Range (transformed scale): [8, 12]\n```\n:::\n\n\nTo use this, we have to merge the recipe and `parsnip` model object into a workflow:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nfive_star_wflow <-\n workflow() |>\n add_recipe(pre_proc) |>\n add_model(lr_mod)\n```\n:::\n\n\nThen we can extract and manipulate the corresponding parameter set:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nfive_star_set <-\n five_star_wflow |>\n extract_parameter_set_dials() |>\n update(\n num_terms = hash_range, \n penalty = penalty(c(-3, 0)),\n mixture = mixture(c(0.05, 1.00))\n )\n```\n:::\n\n\nThis is passed to the search function via the `param_info` argument. \n\nThe initial rounds of search can be biased more towards exploration of the parameter space (as opposed to staying near the current best results). If expected improvement is used as the acquisition function, the trade-off value can be slowly moved from exploration to exploitation over iterations (see the tune vignette on [acquisition functions](https://tune.tidymodels.org/articles/acquisition_functions.html) for more details). The tune package has a built-in function called `expo_decay()` that can help accomplish this:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\ntrade_off_decay <- function(iter) {\n expo_decay(iter, start_val = .01, limit_val = 0, slope = 1/4)\n}\n```\n:::\n\n\nUsing these values, let's run the search:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nset.seed(12)\nfive_star_search <-\n tune_bayes(\n five_star_wflow, \n resamples = folds,\n param_info = five_star_set,\n initial = 5,\n iter = 10,\n metrics = roc_scores,\n objective = exp_improve(trade_off_decay),\n control = control_bayes(verbose_iter = TRUE)\n )\n#> Optimizing roc_auc using the expected improvement with variable trade-off\n#> values.\n#> \n#> ── Iteration 1 ───────────────────────────────────────────────────────\n#> \n#> i Current best:\t\troc_auc=0.7809 (@iter 0)\n#> ✔ Gaussian process model (LOO R²: 75.5%)\n#> ℹ Generating 5000 candidates.\n#> i Trade-off value: 0.01\n#> i penalty=0.00474, mixture=0.0526, num_terms=1028\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results:\troc_auc=0.7302 (+/-0.00754)\n#> \n#> ── Iteration 2 ───────────────────────────────────────────────────────\n#> \n#> i Current best:\t\troc_auc=0.7809 (@iter 0)\n#> ⓧ GP has a LOO R² of 1.2% and is unreliable.\n#> ✔ Gaussian process model failed\n#> ℹ Generating a candidate as far away from existing points as possible.\n#> i Trade-off value: 0.007788\n#> ℹ Uncertainty sample\n#> i penalty=0.898, mixture=0.582, num_terms=291\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results:\troc_auc=0.5\n#> \n#> ── Iteration 3 ───────────────────────────────────────────────────────\n#> \n#> i Current best:\t\troc_auc=0.7809 (@iter 0)\n#> ✔ Gaussian process model (LOO R²: 75.3%)\n#> ℹ Generating 5000 candidates.\n#> i Trade-off value: 0.006065\n#> i penalty=0.0092, mixture=0.981, num_terms=3813\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ♥ Newest results:\troc_auc=0.7985 (+/-0.00368)\n#> \n#> ── Iteration 4 ───────────────────────────────────────────────────────\n#> \n#> i Current best:\t\troc_auc=0.7985 (@iter 3)\n#> ✔ Gaussian process model (LOO R²: 83.9%)\n#> ℹ Generating 5000 candidates.\n#> i Trade-off value: 0.004724\n#> i penalty=0.00496, mixture=0.982, num_terms=3680\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results:\troc_auc=0.7921 (+/-0.00591)\n#> \n#> ── Iteration 5 ───────────────────────────────────────────────────────\n#> \n#> i Current best:\t\troc_auc=0.7985 (@iter 3)\n#> ✔ Gaussian process model (LOO R²: 87.1%)\n#> ℹ Generating 5000 candidates.\n#> i Trade-off value: 0.003679\n#> i penalty=0.00713, mixture=0.982, num_terms=344\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results:\troc_auc=0.7525 (+/-0.0038)\n#> \n#> ── Iteration 6 ───────────────────────────────────────────────────────\n#> \n#> i Current best:\t\troc_auc=0.7985 (@iter 3)\n#> ✔ Gaussian process model (LOO R²: 86%)\n#> ℹ Generating 5000 candidates.\n#> i Trade-off value: 0.002865\n#> i penalty=0.0176, mixture=0.933, num_terms=3730\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results:\troc_auc=0.7854 (+/-0.00944)\n#> \n#> ── Iteration 7 ───────────────────────────────────────────────────────\n#> \n#> i Current best:\t\troc_auc=0.7985 (@iter 3)\n#> ✔ Gaussian process model (LOO R²: 89.2%)\n#> ℹ Generating 5000 candidates.\n#> i Trade-off value: 0.002231\n#> i penalty=0.0124, mixture=0.309, num_terms=4046\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ♥ Newest results:\troc_auc=0.8019 (+/-0.00607)\n#> \n#> ── Iteration 8 ───────────────────────────────────────────────────────\n#> \n#> i Current best:\t\troc_auc=0.8019 (@iter 7)\n#> ✔ Gaussian process model (LOO R²: 92%)\n#> ℹ Generating 5000 candidates.\n#> i Trade-off value: 0.001738\n#> i penalty=0.0163, mixture=0.0649, num_terms=3265\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results:\troc_auc=0.7683 (+/-0.0113)\n#> \n#> ── Iteration 9 ───────────────────────────────────────────────────────\n#> \n#> i Current best:\t\troc_auc=0.8019 (@iter 7)\n#> ✔ Gaussian process model (LOO R²: 88.6%)\n#> ℹ Generating 5000 candidates.\n#> i Trade-off value: 0.001353\n#> i penalty=0.0111, mixture=0.693, num_terms=3559\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ♥ Newest results:\troc_auc=0.8038 (+/-0.00636)\n#> \n#> ── Iteration 10 ──────────────────────────────────────────────────────\n#> \n#> i Current best:\t\troc_auc=0.8038 (@iter 9)\n#> ✔ Gaussian process model (LOO R²: 89.3%)\n#> ℹ Generating 5000 candidates.\n#> i Trade-off value: 0.001054\n#> i penalty=0.0121, mixture=0.729, num_terms=4070\n#> i Estimating performance\n#> ✓ Estimating performance\n#> ⓧ Newest results:\troc_auc=0.8008 (+/-0.00718)\n\nfive_star_search\n#> # Tuning results\n#> # 5-fold cross-validation \n#> # A tibble: 55 × 5\n#> splits id .metrics .notes .iter\n#> \n#> 1 Fold1 0\n#> 2 Fold2 0\n#> 3 Fold3 0\n#> 4 Fold4 0\n#> 5 Fold5 0\n#> 6 Fold1 1\n#> 7 Fold2 1\n#> 8 Fold3 1\n#> 9 Fold4 1\n#> 10 Fold5 1\n#> # ℹ 45 more rows\n```\n:::\n\n\nThese results show some improvement over the initial set. One issue is that so many settings are sub-optimal (as shown in the plot above for grid search) so there are poor results periodically. There are regions where the penalty parameter becomes too large and all of the predictors are removed from the model. These regions are also dependent on the number of terms. There is a fairly narrow ridge (sorry, pun intended!) where good performance can be achieved. Using more iterations would probably result in the search finding better results. \nLet's look at a plot of model performance versus the search iterations:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nautoplot(five_star_search, type = \"performance\")\n```\n\n::: {.cell-output-display}\n![](figs/iter-plot-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\n::: {.callout-note}\nWhat would we do if we knew about the grid search results and wanted to try directed, iterative search? We would restrict the range for the number of hash features to be larger (especially with more data). We might also restrict the penalty and mixture parameters to have a lower upper bound. \n:::\n\n## Extracted results\n\nLet's return to the grid search results and examine the results of our `extract` function. For each _fitted model_, a tibble was saved that contains the relationship between the number of predictors and the penalty value. Let's look at these results for the best model:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nparams <- select_best(five_star_glmnet, metric = \"roc_auc\")\nparams\n#> # A tibble: 1 × 4\n#> penalty mixture num_terms .config \n#> \n#> 1 0.483 0.01 4096 pre3_mod086_post0\n```\n:::\n\n\nRecall that we saved the glmnet results in a tibble. The column `five_star_glmnet$.extracts` is a list of tibbles. As an example, the first element of the list is:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nfive_star_glmnet$.extracts[[1]]\n#> # A tibble: 300 × 5\n#> penalty mixture num_terms .extracts .config \n#> \n#> 1 0.001 0.01 256 pre1_mod001_post0\n#> 2 0.001 0.25 256 pre1_mod002_post0\n#> 3 0.001 0.5 256 pre1_mod003_post0\n#> 4 0.001 0.75 256 pre1_mod004_post0\n#> 5 0.001 1 256 pre1_mod005_post0\n#> 6 0.00144 0.01 256 pre1_mod006_post0\n#> 7 0.00144 0.25 256 pre1_mod007_post0\n#> 8 0.00144 0.5 256 pre1_mod008_post0\n#> 9 0.00144 0.75 256 pre1_mod009_post0\n#> 10 0.00144 1 256 pre1_mod010_post0\n#> # ℹ 290 more rows\n```\n:::\n\n\nMore nested tibbles! Let's `unnest()` the `five_star_glmnet$.extracts` column:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nlibrary(tidyr)\nextracted <- \n five_star_glmnet |> \n dplyr::select(id, .extracts) |> \n unnest(cols = .extracts)\nextracted\n#> # A tibble: 1,500 × 6\n#> id penalty mixture num_terms .extracts .config \n#> \n#> 1 Fold1 0.001 0.01 256 pre1_mod001_post0\n#> 2 Fold1 0.001 0.25 256 pre1_mod002_post0\n#> 3 Fold1 0.001 0.5 256 pre1_mod003_post0\n#> 4 Fold1 0.001 0.75 256 pre1_mod004_post0\n#> 5 Fold1 0.001 1 256 pre1_mod005_post0\n#> 6 Fold1 0.00144 0.01 256 pre1_mod006_post0\n#> 7 Fold1 0.00144 0.25 256 pre1_mod007_post0\n#> 8 Fold1 0.00144 0.5 256 pre1_mod008_post0\n#> 9 Fold1 0.00144 0.75 256 pre1_mod009_post0\n#> 10 Fold1 0.00144 1 256 pre1_mod010_post0\n#> # ℹ 1,490 more rows\n```\n:::\n\n\nOne thing to realize here is that `tune_grid()` [may not fit all of the models](https://tune.tidymodels.org/articles/extras/optimizations.html) that are evaluated. In this case, for each value of `mixture` and `num_terms`, the model is fit over _all_ penalty values (this is a feature of this particular model and is not generally true for other engines). To select the best parameter set, we can exclude the `penalty` column in `extracted`:\n\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nextracted <- \n extracted |> \n dplyr::select(-penalty) |> \n inner_join(params, by = c(\"num_terms\", \"mixture\")) |> \n # Now remove it from the final results\n dplyr::select(-penalty)\nextracted\n#> # A tibble: 100 × 6\n#> id mixture num_terms .extracts .config.x .config.y \n#> \n#> 1 Fold1 0.01 4096 pre3_mod001_post0 pre3_mod086_pos…\n#> 2 Fold1 0.01 4096 pre3_mod006_post0 pre3_mod086_pos…\n#> 3 Fold1 0.01 4096 pre3_mod011_post0 pre3_mod086_pos…\n#> 4 Fold1 0.01 4096 pre3_mod016_post0 pre3_mod086_pos…\n#> 5 Fold1 0.01 4096 pre3_mod021_post0 pre3_mod086_pos…\n#> 6 Fold1 0.01 4096 pre3_mod026_post0 pre3_mod086_pos…\n#> 7 Fold1 0.01 4096 pre3_mod031_post0 pre3_mod086_pos…\n#> 8 Fold1 0.01 4096 pre3_mod036_post0 pre3_mod086_pos…\n#> 9 Fold1 0.01 4096 pre3_mod041_post0 pre3_mod086_pos…\n#> 10 Fold1 0.01 4096 pre3_mod046_post0 pre3_mod086_pos…\n#> # ℹ 90 more rows\n```\n:::\n\n\nNow we can get at the results that we want using another `unnest()`:\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nextracted <- \n extracted |> \n unnest(col = .extracts) # <- these contain a `penalty` column\nextracted\n#> # A tibble: 10,000 × 7\n#> id mixture num_terms penalty num_vars .config.x .config.y \n#> \n#> 1 Fold1 0.01 4096 8.81 0 pre3_mod001_post0 pre3_mod086_post0\n#> 2 Fold1 0.01 4096 8.41 2 pre3_mod001_post0 pre3_mod086_post0\n#> 3 Fold1 0.01 4096 8.03 2 pre3_mod001_post0 pre3_mod086_post0\n#> 4 Fold1 0.01 4096 7.67 2 pre3_mod001_post0 pre3_mod086_post0\n#> 5 Fold1 0.01 4096 7.32 2 pre3_mod001_post0 pre3_mod086_post0\n#> 6 Fold1 0.01 4096 6.98 3 pre3_mod001_post0 pre3_mod086_post0\n#> 7 Fold1 0.01 4096 6.67 3 pre3_mod001_post0 pre3_mod086_post0\n#> 8 Fold1 0.01 4096 6.36 6 pre3_mod001_post0 pre3_mod086_post0\n#> 9 Fold1 0.01 4096 6.08 6 pre3_mod001_post0 pre3_mod086_post0\n#> 10 Fold1 0.01 4096 5.80 7 pre3_mod001_post0 pre3_mod086_post0\n#> # ℹ 9,990 more rows\n```\n:::\n\n\nLet's look at a plot of these results (per resample):\n\n\n::: {.cell layout-align=\"center\"}\n\n```{.r .cell-code}\nggplot(extracted, aes(x = penalty, y = num_vars)) + \n geom_line(aes(group = id, col = id), alpha = .5) + \n ylab(\"Number of retained predictors\") + \n scale_x_log10() + \n ggtitle(paste(\"mixture = \", params$mixture, \"and\", params$num_terms, \"features\")) + \n theme(legend.position = \"none\")\n```\n\n::: {.cell-output-display}\n![](figs/var-plot-1.svg){fig-align='center' width=672}\n:::\n:::\n\n\nThese results might help guide the choice of the `penalty` range if more optimization was conducted. \n\n## Session information {#session-info}\n\n\n::: {.cell layout-align=\"center\"}\n\n```\n#> ─ Session info ─────────────────────────────────────────────────────\n#> version R version 4.6.0 (2026-04-24)\n#> language (EN)\n#> pandoc 3.1.3\n#> quarto 1.9.37\n#> \n#> ─ Packages ─────────────────────────────────────────────────────────\n#> package version date (UTC)\n#> broom 1.0.13 2026-05-14\n#> dials 1.4.3 2026-04-11\n#> dplyr 1.2.1 2026-04-03\n#> future 1.70.0 2026-03-14\n#> ggplot2 4.0.3 2026-04-22\n#> infer 1.1.0 2025-12-18\n#> parsnip 1.6.0 2026-05-14\n#> purrr 1.2.2 2026-04-10\n#> recipes 1.3.2 2026-04-02\n#> rlang 1.2.0 2026-04-06\n#> rsample 1.3.2 2026-01-30\n#> stopwords 2.3 2021-10-28\n#> text2vec 0.6.6 2025-12-01\n#> textrecipes 1.1.0 2025-03-18\n#> tibble 3.3.1 2026-01-11\n#> tidymodels 1.5.0 2026-04-23\n#> tune 2.1.0 2026-04-17\n#> workflows 1.3.0 2025-08-27\n#> yardstick 1.4.0 2026-04-07\n#> \n#> ────────────────────────────────────────────────────────────────────\n```\n:::\n\n", "supporting": [], "filters": [ "rmarkdown/pagebreak.lua" diff --git a/learn/models/conformal-regression/index.html.md b/learn/models/conformal-regression/index.html.md index f2908c52..86ff564c 100644 --- a/learn/models/conformal-regression/index.html.md +++ b/learn/models/conformal-regression/index.html.md @@ -1,6 +1,7 @@ --- title: "Conformal inference for regression models" categories: + - model fitting - probably - parsnip - tuning diff --git a/learn/models/pls/index.html.md b/learn/models/pls/index.html.md index dd52e719..15ae6d9d 100644 --- a/learn/models/pls/index.html.md +++ b/learn/models/pls/index.html.md @@ -1,6 +1,7 @@ --- title: "Multivariate analysis using partial least squares" categories: + - model fitting - pre-processing - multivariate analysis - correlation diff --git a/learn/work/bayes-opt/index.html.md b/learn/work/bayes-opt/index.html.md index 69a3347c..d57f9ff3 100644 --- a/learn/work/bayes-opt/index.html.md +++ b/learn/work/bayes-opt/index.html.md @@ -1,6 +1,7 @@ --- title: "Iterative Bayesian optimization of a classification model" categories: + - tuning and workflows - tuning - SVMs type: learn-subsection diff --git a/learn/work/case-weights/index.html.md b/learn/work/case-weights/index.html.md index 91e9d533..e5565c1e 100644 --- a/learn/work/case-weights/index.html.md +++ b/learn/work/case-weights/index.html.md @@ -1,6 +1,7 @@ --- title: "Creating case weights based on time" categories: + - tuning and workflows - model fitting - time series type: learn-subsection diff --git a/learn/work/fairness-detectors/index.html.md b/learn/work/fairness-detectors/index.html.md index da09d406..e08d6b9b 100644 --- a/learn/work/fairness-detectors/index.html.md +++ b/learn/work/fairness-detectors/index.html.md @@ -1,6 +1,7 @@ --- title: "Are GPT detectors fair? A machine learning fairness case study" categories: + - tuning and workflows - yardstick - fairness - classification diff --git a/learn/work/fairness-readmission/index.html.md b/learn/work/fairness-readmission/index.html.md index 8d0198a3..25aaece7 100644 --- a/learn/work/fairness-readmission/index.html.md +++ b/learn/work/fairness-readmission/index.html.md @@ -1,6 +1,7 @@ --- title: "Fair prediction of hospital readmission: a machine learning fairness case study" categories: + - tuning and workflows - yardstick - fairness - tuning diff --git a/learn/work/nested-resampling/index.html.md b/learn/work/nested-resampling/index.html.md index c0bbf8c9..c9b04139 100644 --- a/learn/work/nested-resampling/index.html.md +++ b/learn/work/nested-resampling/index.html.md @@ -1,6 +1,7 @@ --- title: "Nested resampling" categories: + - tuning and workflows - SVMs type: learn-subsection weight: 2 diff --git a/learn/work/sparse-matrix/index.html.md b/learn/work/sparse-matrix/index.html.md index 98c3e501..d9345bf9 100644 --- a/learn/work/sparse-matrix/index.html.md +++ b/learn/work/sparse-matrix/index.html.md @@ -1,6 +1,7 @@ --- title: "Model tuning using a sparse matrix" categories: + - tuning and workflows - tuning - classification - sparse data diff --git a/learn/work/sparse-recipe/index.html.md b/learn/work/sparse-recipe/index.html.md index b6e4e7f0..27084599 100644 --- a/learn/work/sparse-recipe/index.html.md +++ b/learn/work/sparse-recipe/index.html.md @@ -1,6 +1,7 @@ --- title: "Using recipes to create sparse data" categories: + - tuning and workflows - tuning - classification - sparse data diff --git a/learn/work/tune-svm/index.html.md b/learn/work/tune-svm/index.html.md index d9e7e1f0..eeacfa06 100644 --- a/learn/work/tune-svm/index.html.md +++ b/learn/work/tune-svm/index.html.md @@ -1,6 +1,7 @@ --- title: "Model tuning via grid search" categories: + - tuning and workflows - tuning - SVMs - classification diff --git a/learn/work/tune-text/index.html.md b/learn/work/tune-text/index.html.md index 4ddec32d..ced3c1bb 100644 --- a/learn/work/tune-text/index.html.md +++ b/learn/work/tune-text/index.html.md @@ -1,6 +1,7 @@ --- title: "Tuning text models" categories: + - tuning and workflows - tuning - regression - sparse data