[wip][ci] Revise document build pipeline.

- Provide a script to build all document locally.
- Integrate with the CI.

Author: trivialfis

doc/conf.py

@@ -112,6 +112,10 @@ def get_sha(branch: str) -> str | None:
 
 def download_jvm_docs() -> None:
     """Fetch docs for the JVM packages"""
+    if os.environ.get("XGBOOST_SKIP_JVM_DOCS", None) == "1":
+        print("Skipping JVM docs (XGBOOST_SKIP_JVM_DOCS=1)")
+        return
+
     print("Download JVM documents from S3.")
     branch = get_branch()
     commit = get_sha(branch)
@@ -161,6 +165,10 @@ def try_fetch_jvm_doc(branch: str) -> bool:
 
 def download_r_docs() -> None:
     """Fetch R document from s3."""
+    if os.environ.get("XGBOOST_SKIP_R_DOCS", None) == "1":
+        print("Skipping R docs (XGBOOST_SKIP_R_DOCS=1)")
+        return
+
     branch = get_branch()
     commit = get_sha(branch)
     print("Download R documents from S3.")
@@ -328,7 +336,10 @@ def is_readthedocs_build():
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
-exclude_patterns = ["_build"]
+# Note: tmp/ contains pre-built HTML docs (R pkgdown, JVM javadocs/scaladocs, C++ doxygen)
+# that are copied via html_extra_path. We exclude them from Sphinx processing to avoid
+# "document isn't included in any toctree" warnings.
+exclude_patterns = ["_build", "tmp"]
 html_extra_path = []
 if is_readthedocs_build():
     html_extra_path = [TMP_DIR]

doc/requirements.txt

@@ -1,6 +1,7 @@
 sphinx
 mock
 sphinx_rtd_theme>=1.0.0
+sphinxcontrib-jquery
 breathe
 scikit-learn
 sh

doc/sphinx_util.py

@@ -1,45 +1,32 @@
 #!/bin/bash
 ## Build docs for the JVM packages and package it in a tarball.
-## Note: this script assumes that the user has already built libxgboost4j.so
-## and placed it in the lib/ directory.
+##
+## Prerequisites:
+##   - libxgboost4j.so must be built and placed in lib/
+##   - BRANCH_NAME environment variable must be set
+##
+## This script uses ops/script/build_local_docs.py for the actual build.
 
 set -euo pipefail
 
 if [[ -z ${BRANCH_NAME:-} ]]; then
-  echo "Make sure to define environment variable BRANCH_NAME."
+  echo "Error: BRANCH_NAME environment variable not set"
   exit 1
 fi
 
 if [[ ! -f lib/libxgboost4j.so ]]; then
-  echo "Must place libxgboost4j.so in lib/ first"
+  echo "Error: lib/libxgboost4j.so not found. Build it first with:"
+  echo "  python3 ops/script/build_local_docs.py jvm-lib"
   exit 2
 fi
 
-echo "--- Build JVM packages doc"
+echo "--- Building JVM documentation ---"
 set -x
 
-# Copy in libxgboost4j.so
-mkdir -p jvm-packages/xgboost4j/src/main/resources/lib/linux/x86_64/
-cp -v lib/libxgboost4j.so jvm-packages/xgboost4j/src/main/resources/lib/linux/x86_64/
+python3 ops/script/build_local_docs.py jvm --branch-name "${BRANCH_NAME}"
 
-cd jvm-packages/
-# Install JVM packages in local Maven repository
-mvn --no-transfer-progress install -Pdocs
-# Build Scaladocs
-mvn --no-transfer-progress scala:doc -Pdocs
-# Build Javadocs
-mvn --no-transfer-progress javadoc:javadoc -Pdocs
-
-# Package JVM docs in a tarball
-mkdir -p tmp/scaladocs
-cp -rv xgboost4j/target/reports/apidocs/ ./tmp/javadocs/
-cp -rv xgboost4j/target/site/scaladocs/ ./tmp/scaladocs/xgboost4j/
-cp -rv xgboost4j-spark/target/site/scaladocs/ ./tmp/scaladocs/xgboost4j-spark/
-cp -rv xgboost4j-spark-gpu/target/site/scaladocs/ ./tmp/scaladocs/xgboost4j-spark-gpu/
-cp -rv xgboost4j-flink/target/site/scaladocs/ ./tmp/scaladocs/xgboost4j-flink/
-
-cd tmp
-tar cvjf ${BRANCH_NAME}.tar.bz2 javadocs/ scaladocs/
-mv ${BRANCH_NAME}.tar.bz2 ..
-cd ..
-rm -rfv tmp/
+# Verify output
+[[ -f "jvm-packages/${BRANCH_NAME}.tar.bz2" ]] || {
+  echo "Error: jvm-packages/${BRANCH_NAME}.tar.bz2 not found"
+  exit 3
+}

ops/pipeline/build-jvm-doc.sh

@@ -1,44 +1,39 @@
 #!/bin/bash
 ## Build docs for the R package and package it in a tarball.
+##
+## Prerequisites:
+##   - BRANCH_NAME environment variable must be set
+##   - R and required packages must be available
+##
+## This script uses ops/script/build_local_docs.py for the actual build.
 
 set -euo pipefail
 
 if [[ -z ${BRANCH_NAME:-} ]]; then
-  echo "Make sure to define environment variable BRANCH_NAME."
+  echo "Error: BRANCH_NAME environment variable not set"
   exit 1
 fi
 
-if [[ -z "${R_LIBS_USER:-}" ]]; then
-  export R_LIBS_USER=/tmp/rtmpdir
-fi
-
-echo "--- Build R package doc"
+# Setup R_LIBS_USER
+export R_LIBS_USER="${R_LIBS_USER:-/tmp/rtmpdir}"
 echo "R_LIBS_USER: ${R_LIBS_USER}"
-set -x
 
-if [[ ! -d ${R_LIBS_USER} ]]; then
-  echo "Make ${R_LIBS_USER} for installing temporary R packages."
-  mkdir ${R_LIBS_USER}
-fi
+mkdir -p "${R_LIBS_USER}"
 
-# Used only in container environment
-if command -v gosu 2>&1 >/dev/null; then
-  gosu root chown -R $UID:$GROUPS ${R_LIBS_USER}
+# Container environment: fix permissions
+if command -v gosu &>/dev/null; then
+  gosu root chown -R "$UID:$GROUPS" "${R_LIBS_USER}"
 fi
 
-cd R-package
-
-MAKEFLAGS=-j$(nproc) Rscript ./tests/helper_scripts/install_deps.R
-# Some examples are failing
-MAKEFLAGS=-j$(nproc) Rscript -e "pkgdown::build_site(examples=FALSE)"
-# Install the package for vignettes
-MAKEFLAGS=-j$(nproc) R CMD INSTALL .
-
-cd -
-
-cd doc/R-package
-make -j$(nproc) all
+echo "--- Building R documentation ---"
+set -x
 
-cd ../../  # back to project root
+python3 ops/script/build_local_docs.py r \
+  --branch-name "${BRANCH_NAME}" \
+  --r-libs-user "${R_LIBS_USER}"
 
-tar cvjf r-docs-${BRANCH_NAME}.tar.bz2 R-package/docs doc/R-package/xgboost_introduction.md doc/R-package/xgboostfromJSON.md
+# Verify output
+[[ -f "r-docs-${BRANCH_NAME}.tar.bz2" ]] || {
+  echo "Error: r-docs-${BRANCH_NAME}.tar.bz2 not found"
+  exit 3
+}