Add doc tests. (#12216)

Author: trivialfis

.github/workflows/misc.yml

@@ -133,9 +133,35 @@ jobs:
             --prefix ${BRANCH_NAME}/${{ github.event.pull_request.head.sha || github.sha }} --make-public \
             r-docs-${BRANCH_NAME}.tar.bz2
 
+  test-docs:
+    name: Test Sphinx documentation snippets
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+      - uses: actions/checkout@v6.0.2
+        with:
+          submodules: "true"
+      - name: Install system dependencies
+        run: |
+          sudo apt update
+          sudo apt install -y libuv1-dev
+      - uses: r-lib/actions/setup-r@v2
+        with:
+          r-version: release
+      - uses: dmlc/xgboost-devops/actions/miniforge-setup@main
+        with:
+          environment-name: xgboost_docs
+          environment-file: doc/xgboost_doc.yml
+      - uses: dmlc/xgboost-devops/actions/sccache@main
+      - name: Test Sphinx documentation snippets
+        run: bash ops/pipeline/test-docs.sh
+      - run: sccache --show-stats
+
   trigger-rtd-build:
     name: Trigger Read The Docs build
-    needs: [build-jvm-docs, build-r-docs]
+    needs: [build-jvm-docs, build-r-docs, test-docs]
     runs-on:
       - runs-on=${{ github.run_id }}
       - runner=linux-amd64-cpu

R-package/DESCRIPTION

@@ -71,6 +71,6 @@ Imports:
     data.table (>= 1.9.6),
     jsonlite (>= 1.0)
 Roxygen: list(markdown = TRUE)
-RoxygenNote: 7.3.3
 Encoding: UTF-8
 SystemRequirements: GNU make, C++17
+Config/roxygen2/version: 8.0.0

R-package/man/xgb.plot.deepness.Rd

@@ -1,14 +1,15 @@
 ## Install dependencies of R package for testing. The list might not be
 ## up-to-date, check DESCRIPTION for the latest list and update this one if
 ## inconsistent is found.
-pkgs <- c(
-  ## CI
+ci_pkgs <- c(
   "pkgbuild",
   "roxygen2",
   "XML",
   "cplm",
-  "e1071",
-  ## suggests
+  "e1071"
+)
+
+suggests_pkgs <- c(
   "knitr",
   "rmarkdown",
   "ggplot2",
@@ -23,31 +24,62 @@ pkgs <- c(
   "igraph",
   "float",
   "titanic",
-  "RhpcBLASctl",
-  ## imports
+  "RhpcBLASctl"
+)
+
+imports_pkgs <- c(
   "Matrix",
   "data.table",
   "jsonlite"
 )
 
+dependency_scopes <- list(
+  ci = ci_pkgs,
+  suggests = suggests_pkgs,
+  imports = imports_pkgs,
+  doc_test = c(imports_pkgs, "DirichletReg", "testthat")
+)
+
+scopes <- commandArgs(trailingOnly = TRUE)
+if (!length(scopes)) {
+  scopes <- c("ci", "suggests", "imports")
+}
+scopes <- gsub("-", "_", scopes, fixed = TRUE)
+if ("all" %in% scopes) {
+  scopes <- names(dependency_scopes)
+}
+
+unknown_scopes <- setdiff(scopes, names(dependency_scopes))
+if (length(unknown_scopes)) {
+  stop(
+    "Unknown dependency scope(s): ",
+    paste(unknown_scopes, collapse = ", "),
+    ". Valid scopes are: ",
+    paste(c(names(dependency_scopes), "all"), collapse = ", ")
+  )
+}
+
+pkgs <- unique(unlist(dependency_scopes[scopes], use.names = FALSE))
+
 ncpus <- parallel::detectCores()
 print(paste0("Using ", ncpus, " cores to install dependencies."))
+print(paste0("Installing dependency scopes: ", paste(scopes, collapse = ", ")))
 
 if (.Platform$OS.type == "unix") {
   print("Installing source packages on unix.")
   install.packages(
     pkgs,
-    repo = "https://cloud.r-project.org",
+    repos = "https://cloud.r-project.org",
     dependencies = c("Depends", "Imports", "LinkingTo"),
-    Ncpus = parallel::detectCores()
+    Ncpus = ncpus
   )
 } else {
   print("Installing binary packages on Windows.")
   install.packages(
     pkgs,
-    repo = "https://cloud.r-project.org",
+    repos = "https://cloud.r-project.org",
     dependencies = c("Depends", "Imports", "LinkingTo"),
-    Ncpus = parallel::detectCores(),
+    Ncpus = ncpus,
     type = "binary"
   )
 }

R-package/man/xgb.plot.shap.Rd

@@ -252,15 +252,31 @@ def is_readthedocs_build():
     "sphinxcontrib.jquery",
     "sphinx.ext.autodoc",
     "sphinx.ext.napoleon",
+    "sphinx.ext.doctest",
     "sphinx.ext.mathjax",
     "sphinx.ext.intersphinx",
     "sphinx_gallery.gen_gallery",
     "sphinx_issues",
     "sphinx_tabs.tabs",
     "breathe",
     "myst_parser",
+    "xgboost_doc_doctest",
 ]
 
+sphinx_tabs_valid_builders = ["html", "doctest"]
+
+# We need the real XGBoost library for running tests.
+doctest_global_setup = """
+import os
+import sys
+
+os.environ.pop("XGBOOST_BUILD_DOC", None)
+for name in list(sys.modules):
+    if name == "xgboost" or name.startswith("xgboost."):
+        del sys.modules[name]
+"""
+doctest_test_doctest_blocks = ""
+
 sphinx_gallery_conf = {
     # path to your example scripts
     "examples_dirs": [

R-package/tests/helper_scripts/install_deps.R

@@ -126,3 +126,22 @@ Examples
 * We are super excited to hear about your story. If you have blog posts,
   tutorials, or code solutions using XGBoost, please tell us, and we will add
   a link in the example pages.
+
+*********
+Doc Tests
+*********
+
+We use Sphinx doctest to test selected snippets in the documentation. At the moment, this
+only covers Python and R snippets written with ``.. code-tab:: python`` or ``.. code-tab::
+r``. Regular code blocks are rendered as examples but are not executed by the doctest job.
+
+The doctest job runs snippets from each ``.rst`` file as an independent group. Snippets in
+the same document share state, while snippets from different documents do not. To skip a
+tabbed snippet, add the ``no-doctest`` class:
+
+.. code-block:: rst
+
+  .. code-tab:: python
+     :class: no-doctest
+
+     # This snippet is rendered but not tested.

doc/conf.py

@@ -316,12 +316,12 @@ point, which means it will be a minimum rather than a maximum or saddle point).
             k = 3
             rng = np.random.default_rng(seed=123)
             x0 = rng.standard_normal(size=k)
-            y_sample = rng.dirichlet(np.exp(x0), size=5_000_000)
+            y_sample = rng.dirichlet(np.exp(x0), size=200_000)
             x_broadcast = np.broadcast_to(x0, (y_sample.shape[0], k))
             g_sample = dirichlet_grad(x_broadcast, y_sample)
             ref = (g_sample.T @ g_sample) / y_sample.shape[0]
             Ehess = dirichlet_expected_hess(x0.reshape((1,-1)))[0]
-            np.testing.assert_almost_equal(Ehess, ref, decimal=2)
+            np.testing.assert_allclose(Ehess, ref, rtol=5e-2, atol=5e-2)
         test_dirichlet_expected_hess()
 
     .. code-tab:: r R
@@ -344,14 +344,14 @@ point, which means it will be a minimum rather than a maximum or saddle point).
             set.seed(123)
             x0 <- rnorm(k)
             alpha <- exp(x0)
-            n.samples <- 5e6
+            n.samples <- 2e5
             y.samples <- rdirichlet(n.samples, alpha)
 
             x.broadcast <- rep(x0, n.samples) |> matrix(ncol=k, byrow=T)
             grad.samples <- dirichlet.grad(x.broadcast, y.samples)
             ref <- crossprod(grad.samples) / n.samples
             Ehess <- dirichlet.expected.hess(matrix(x0, nrow=1))
-            expect_equal(Ehess[1,,], ref, tolerance=1e-2)
+            expect_equal(Ehess[1,,], ref, tolerance=5e-2)
         })
 
 But note that this is still not usable for XGBoost, since the expected
@@ -528,6 +528,7 @@ Fitting an XGBoost model and making predictions:
             evals=[(dtrain, "Train")],
             evals_result=results,
             custom_metric=dirichlet_eval_metric,
+            verbose_eval=False,
         )
         yhat = softmax(booster.inplace_predict(X), axis=1)
 
@@ -655,6 +656,7 @@ Now fitting a model again, this time with the intercept:
             evals=[(dtrain, "Train")],
             evals_result=results,
             custom_metric=dirichlet_eval_metric,
+            verbose_eval=False,
         )
         yhat = softmax(
             booster.predict(

doc/contrib/docs.rst

@@ -12,11 +12,15 @@ dependencies:
   - numpy
   - scipy
   - scikit-learn
+  - dask
   - myst-parser
   - pyspark
   - pip:
     - breathe
     - sphinx_rtd_theme
+    - sphinx-issues
+    - sphinx-tabs
+    - sphinxcontrib-jquery
     - pydot-ng
     - graphviz
     - ray[train]