diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index df36b003865eb..dd73c84133948 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1000,8 +1000,8 @@ jobs: apt-get install -y libcurl4-openssl-dev libgit2-dev libssl-dev libxml2-dev \ libfontconfig1-dev libharfbuzz-dev libfribidi-dev libfreetype6-dev libpng-dev \ libtiff5-dev libjpeg-dev - Rscript -e "install.packages(c('devtools'), repos='https://cloud.r-project.org/')" - Rscript -e "devtools::install_version('lintr', version='2.0.1', repos='https://cloud.r-project.org')" + Rscript -e "install.packages(c('remotes'), repos='https://cloud.r-project.org/')" + Rscript -e "remotes::install_version('lintr', version='2.0.1', repos='https://cloud.r-project.org')" - name: Install R linter dependencies and SparkR run: ./R/install-dev.sh - name: R linter @@ -1116,9 +1116,9 @@ jobs: apt-get update -y apt-get install -y libcurl4-openssl-dev pandoc apt-get install -y ruby ruby-dev - Rscript -e "install.packages(c('devtools', 'testthat', 'knitr', 'rmarkdown', 'markdown', 'e1071', 'roxygen2', 'ggplot2', 'mvtnorm', 'statmod'), repos='https://cloud.r-project.org/')" - Rscript -e "devtools::install_version('pkgdown', version='2.0.1', repos='https://cloud.r-project.org')" - Rscript -e "devtools::install_version('preferably', version='0.4', repos='https://cloud.r-project.org')" + Rscript -e "install.packages(c('remotes', 'testthat', 'knitr', 'rmarkdown', 'markdown', 'e1071', 'roxygen2', 'ggplot2', 'mvtnorm', 'statmod'), repos='https://cloud.r-project.org/')" + Rscript -e "remotes::install_version('pkgdown', version='2.0.1', repos='https://cloud.r-project.org')" + Rscript -e "remotes::install_version('preferably', version='0.4', repos='https://cloud.r-project.org')" # Should unpin 'sphinxcontrib-*' after upgrading sphinx>5 python3.9 -m pip install 'sphinx==4.5.0' mkdocs 'pydata_sphinx_theme>=0.13' sphinx-copybutton nbsphinx numpydoc jinja2 markupsafe 'pyzmq<24.0.0' 'sphinxcontrib-applehelp==1.0.4' 'sphinxcontrib-devhelp==1.0.2' 'sphinxcontrib-htmlhelp==2.0.1' 'sphinxcontrib-qthelp==1.0.3' 'sphinxcontrib-serializinghtml==1.1.5' python3.9 -m pip install ipython_genutils # See SPARK-38517 diff --git a/R/DOCUMENTATION.md b/R/DOCUMENTATION.md index d4ff3f5f4875b..1fb96f0955d86 100644 --- a/R/DOCUMENTATION.md +++ b/R/DOCUMENTATION.md @@ -22,8 +22,7 @@ SparkR documentation is generated by using in-source comments and annotated by u [`roxygen2`](https://cran.r-project.org/package=roxygen2). After making changes to the documentation and generating man pages, you can run the following from an R console in the SparkR home directory ```R -library(devtools) -devtools::document(pkg="./pkg", roclets=c("rd")) +roxygen2::roxygenize(package.dir="./pkg", roclets=c("rd")) ``` You can verify if your changes are good by running diff --git a/R/create-docs.sh b/R/create-docs.sh index 3deaefd0659dc..70a62ed020df8 100755 --- a/R/create-docs.sh +++ b/R/create-docs.sh @@ -18,7 +18,7 @@ # # Script to create API docs and vignettes for SparkR -# This requires `devtools`, `knitr` and `rmarkdown` to be installed on the machine. +# This requires `roxygen2`, `knitr`, `rmarkdown`, and `pkgdown` to be installed on the machine. # After running this script the html docs can be found in # $SPARK_HOME/R/pkg/html diff --git a/R/create-rd.sh b/R/create-rd.sh index 1f0527458f2f0..5851b622354bb 100755 --- a/R/create-rd.sh +++ b/R/create-rd.sh @@ -33,5 +33,5 @@ FWDIR="$(cd "`dirname "${BASH_SOURCE[0]}"`"; pwd)" pushd "$FWDIR" > /dev/null . "$FWDIR/find-r.sh" -# Generate Rd files if devtools is installed -"$R_SCRIPT_PATH/Rscript" -e ' if(requireNamespace("devtools", quietly=TRUE)) { setwd("'$FWDIR'"); devtools::document(pkg="./pkg", roclets="rd") }' +# Generate Rd files if roxygen2 is installed +"$R_SCRIPT_PATH/Rscript" -e ' if(requireNamespace("roxygen2", quietly=TRUE)) { setwd("'$FWDIR'"); roxygen2::roxygenize(package.dir="./pkg", roclets=c("rd")) }' diff --git a/R/pkg/README.md b/R/pkg/README.md index c05a75812245c..881f3081ca747 100644 --- a/R/pkg/README.md +++ b/R/pkg/README.md @@ -53,7 +53,7 @@ Once you have made your changes, please include unit tests for them and run exis #### Generating documentation -The SparkR documentation (Rd files and HTML files) are not a part of the source repository. To generate them you can run the script `R/create-docs.sh`. This script uses `devtools` and `knitr` to generate the docs and these packages need to be installed on the machine before using the script. Also, you may need to install these [prerequisites](https://github.com/apache/spark/tree/master/docs#prerequisites). See also, `R/DOCUMENTATION.md` +The SparkR documentation (Rd files and HTML files) are not a part of the source repository. To generate them you can run the script `R/create-docs.sh`. This script uses `roxygen2`, `knitr`, and `rmarkdown` to generate the docs and these packages need to be installed on the machine before using the script. Also, you may need to install these [prerequisites](https://github.com/apache/spark/tree/master/docs#prerequisites). See also, `R/DOCUMENTATION.md` ### Examples, Unit tests diff --git a/dev/create-release/spark-rm/Dockerfile.base b/dev/create-release/spark-rm/Dockerfile.base index 046942a9177e6..713a50355599c 100644 --- a/dev/create-release/spark-rm/Dockerfile.base +++ b/dev/create-release/spark-rm/Dockerfile.base @@ -91,13 +91,13 @@ RUN echo 'deb https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/' >> /et # Install R packages (same versions across all branches) # See more in SPARK-39959, roxygen2 < 7.2.1 -RUN Rscript -e "install.packages(c('devtools', 'remotes', 'knitr', 'markdown', \ +RUN Rscript -e "install.packages(c('remotes', 'knitr', 'markdown', \ 'rmarkdown', 'testthat', 'e1071', 'survival', 'arrow', \ 'ggplot2', 'mvtnorm', 'statmod', 'xml2'), repos='https://cloud.r-project.org/')" && \ - Rscript -e "devtools::install_version('roxygen2', version='7.2.0', repos='https://cloud.r-project.org')" && \ - Rscript -e "devtools::install_version('lintr', version='2.0.1', repos='https://cloud.r-project.org')" && \ - Rscript -e "devtools::install_version('preferably', version='0.4', repos='https://cloud.r-project.org')" && \ - Rscript -e "devtools::install_version('pkgdown', version='2.0.1', repos='https://cloud.r-project.org')" + Rscript -e "remotes::install_version('roxygen2', version='7.2.0', repos='https://cloud.r-project.org')" && \ + Rscript -e "remotes::install_version('lintr', version='2.0.1', repos='https://cloud.r-project.org')" && \ + Rscript -e "remotes::install_version('preferably', version='0.4', repos='https://cloud.r-project.org')" && \ + Rscript -e "remotes::install_version('pkgdown', version='2.0.1', repos='https://cloud.r-project.org')" # See more in SPARK-39735 ENV R_LIBS_SITE="/usr/local/lib/R/site-library:${R_LIBS_SITE}:/usr/lib/R/library" diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile index 1a5f558d08f7d..0db0214651506 100644 --- a/dev/infra/Dockerfile +++ b/dev/infra/Dockerfile @@ -74,13 +74,13 @@ RUN gpg -a --export E084DAB9 | apt-key add - RUN add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/' # See more in SPARK-39959, roxygen2 < 7.2.1 -RUN Rscript -e "install.packages(c('devtools', 'knitr', 'markdown', \ - 'rmarkdown', 'testthat', 'devtools', 'e1071', 'survival', 'arrow', \ - 'ggplot2', 'mvtnorm', 'statmod', 'xml2', 'remotes'), repos='https://cloud.r-project.org/')" && \ - Rscript -e "devtools::install_version('roxygen2', version='7.2.0', repos='https://cloud.r-project.org')" && \ - Rscript -e "devtools::install_version('lintr', version='2.0.1', repos='https://cloud.r-project.org')" && \ - Rscript -e "devtools::install_version('pkgdown', version='2.0.1', repos='https://cloud.r-project.org')" && \ - Rscript -e "devtools::install_version('preferably', version='0.4', repos='https://cloud.r-project.org')" +RUN Rscript -e "install.packages(c('remotes', 'knitr', 'markdown', \ + 'rmarkdown', 'testthat', 'e1071', 'survival', 'arrow', \ + 'ggplot2', 'mvtnorm', 'statmod', 'xml2'), repos='https://cloud.r-project.org/')" && \ + Rscript -e "remotes::install_version('roxygen2', version='7.2.0', repos='https://cloud.r-project.org')" && \ + Rscript -e "remotes::install_version('lintr', version='2.0.1', repos='https://cloud.r-project.org')" && \ + Rscript -e "remotes::install_version('pkgdown', version='2.0.1', repos='https://cloud.r-project.org')" && \ + Rscript -e "remotes::install_version('preferably', version='0.4', repos='https://cloud.r-project.org')" # See more in SPARK-39735 ENV R_LIBS_SITE="/usr/local/lib/R/site-library:${R_LIBS_SITE}:/usr/lib/R/library" diff --git a/dev/spark-test-image/docs/Dockerfile b/dev/spark-test-image/docs/Dockerfile index 1ddf1eb47237b..3b7306c417cf5 100644 --- a/dev/spark-test-image/docs/Dockerfile +++ b/dev/spark-test-image/docs/Dockerfile @@ -72,10 +72,10 @@ RUN apt-get update && apt-get install -y \ && rm -rf /var/lib/apt/lists/* # See more in SPARK-39959, roxygen2 < 7.2.1 -RUN Rscript -e "install.packages(c('devtools', 'knitr', 'markdown', 'rmarkdown', 'testthat', 'remotes'), repos='https://cloud.r-project.org/')" && \ - Rscript -e "devtools::install_version('roxygen2', version='7.2.0', repos='https://cloud.r-project.org')" && \ - Rscript -e "devtools::install_version('pkgdown', version='2.0.1', repos='https://cloud.r-project.org')" && \ - Rscript -e "devtools::install_version('preferably', version='0.4', repos='https://cloud.r-project.org')" +RUN Rscript -e "install.packages(c('remotes', 'knitr', 'markdown', 'rmarkdown', 'testthat'), repos='https://cloud.r-project.org/')" && \ + Rscript -e "remotes::install_version('roxygen2', version='7.2.0', repos='https://cloud.r-project.org')" && \ + Rscript -e "remotes::install_version('pkgdown', version='2.0.1', repos='https://cloud.r-project.org')" && \ + Rscript -e "remotes::install_version('preferably', version='0.4', repos='https://cloud.r-project.org')" # See more in SPARK-39735 ENV R_LIBS_SITE="/usr/local/lib/R/site-library:${R_LIBS_SITE}:/usr/lib/R/library" diff --git a/dev/spark-test-image/lint/Dockerfile b/dev/spark-test-image/lint/Dockerfile index ecc2fb31b53cd..799656614d828 100644 --- a/dev/spark-test-image/lint/Dockerfile +++ b/dev/spark-test-image/lint/Dockerfile @@ -64,10 +64,10 @@ RUN apt-get update && apt-get install -y \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -RUN Rscript -e "install.packages(c('devtools', 'knitr', 'markdown', 'rmarkdown', 'testthat', 'remotes'), repos='https://cloud.r-project.org/')" \ - && Rscript -e "devtools::install_version('pkgdown', version='2.0.1', repos='https://cloud.r-project.org')" \ - && Rscript -e "devtools::install_version('preferably', version='0.4', repos='https://cloud.r-project.org')" \ - && Rscript -e "devtools::install_version('lintr', version='2.0.1', repos='https://cloud.r-project.org')" +RUN Rscript -e "install.packages(c('remotes', 'knitr', 'markdown', 'rmarkdown', 'testthat'), repos='https://cloud.r-project.org/')" \ + && Rscript -e "remotes::install_version('pkgdown', version='2.0.1', repos='https://cloud.r-project.org')" \ + && Rscript -e "remotes::install_version('preferably', version='0.4', repos='https://cloud.r-project.org')" \ + && Rscript -e "remotes::install_version('lintr', version='2.0.1', repos='https://cloud.r-project.org')" # See more in SPARK-39735 ENV R_LIBS_SITE="/usr/local/lib/R/site-library:${R_LIBS_SITE}:/usr/lib/R/library" diff --git a/dev/spark-test-image/sparkr/Dockerfile b/dev/spark-test-image/sparkr/Dockerfile index e2dcf36d6081c..fcb2e090c00c6 100644 --- a/dev/spark-test-image/sparkr/Dockerfile +++ b/dev/spark-test-image/sparkr/Dockerfile @@ -67,13 +67,13 @@ RUN gpg -a --export E084DAB9 | apt-key add - RUN add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu jammy-cran40/' # See more in SPARK-39959, roxygen2 < 7.2.1 -RUN Rscript -e "install.packages(c('devtools', 'knitr', 'markdown', \ - 'rmarkdown', 'testthat', 'devtools', 'e1071', 'survival', 'arrow', \ - 'ggplot2', 'mvtnorm', 'statmod', 'xml2', 'remotes'), repos='https://cloud.r-project.org/')" && \ - Rscript -e "devtools::install_version('roxygen2', version='7.2.0', repos='https://cloud.r-project.org')" && \ - Rscript -e "devtools::install_version('lintr', version='2.0.1', repos='https://cloud.r-project.org')" && \ - Rscript -e "devtools::install_version('pkgdown', version='2.0.1', repos='https://cloud.r-project.org')" && \ - Rscript -e "devtools::install_version('preferably', version='0.4', repos='https://cloud.r-project.org')" +RUN Rscript -e "install.packages(c('remotes', 'knitr', 'markdown', \ + 'rmarkdown', 'testthat', 'e1071', 'survival', 'arrow', \ + 'ggplot2', 'mvtnorm', 'statmod', 'xml2'), repos='https://cloud.r-project.org/')" && \ + Rscript -e "remotes::install_version('roxygen2', version='7.2.0', repos='https://cloud.r-project.org')" && \ + Rscript -e "remotes::install_version('lintr', version='2.0.1', repos='https://cloud.r-project.org')" && \ + Rscript -e "remotes::install_version('pkgdown', version='2.0.1', repos='https://cloud.r-project.org')" && \ + Rscript -e "remotes::install_version('preferably', version='0.4', repos='https://cloud.r-project.org')" # See more in SPARK-39735 ENV R_LIBS_SITE="/usr/local/lib/R/site-library:${R_LIBS_SITE}:/usr/lib/R/library" diff --git a/docs/README.md b/docs/README.md index 1235efe91812b..d896cae0b09a8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -60,10 +60,10 @@ To generate the Python or R API docs, you'll also need to [install Pandoc](https If you'd like to generate R API documentation, install these libraries: ```sh -$ sudo Rscript -e 'install.packages(c("knitr", "devtools", "testthat", "rmarkdown"), repos="https://cloud.r-project.org/")' -$ sudo Rscript -e 'devtools::install_version("roxygen2", version = "7.1.2", repos="https://cloud.r-project.org/")' -$ sudo Rscript -e "devtools::install_version('pkgdown', version='2.0.1', repos='https://cloud.r-project.org')" -$ sudo Rscript -e "devtools::install_version('preferably', version='0.4', repos='https://cloud.r-project.org')" +$ sudo Rscript -e 'install.packages(c("remotes", "knitr", "testthat", "rmarkdown"), repos="https://cloud.r-project.org/")' +$ sudo Rscript -e 'remotes::install_version("roxygen2", version = "7.1.2", repos="https://cloud.r-project.org/")' +$ sudo Rscript -e "remotes::install_version('pkgdown', version='2.0.1', repos='https://cloud.r-project.org')" +$ sudo Rscript -e "remotes::install_version('preferably', version='0.4', repos='https://cloud.r-project.org')" ``` Note: Other versions of roxygen2 might work in SparkR documentation generation but `RoxygenNote` field in `$SPARK_HOME/R/pkg/DESCRIPTION` is 7.1.2, which is updated if the version is mismatched. diff --git a/docs/building-spark.md b/docs/building-spark.md index 1b93b337b4f7f..d8482de8c5269 100644 --- a/docs/building-spark.md +++ b/docs/building-spark.md @@ -250,7 +250,7 @@ The run-tests script also can be limited to a specific Python version or a speci To run the SparkR tests you will need to install the [knitr](https://cran.r-project.org/package=knitr), [rmarkdown](https://cran.r-project.org/package=rmarkdown), [testthat](https://cran.r-project.org/package=testthat), [e1071](https://cran.r-project.org/package=e1071) and [survival](https://cran.r-project.org/package=survival) packages first: - Rscript -e "install.packages(c('knitr', 'rmarkdown', 'devtools', 'testthat', 'e1071', 'survival'), repos='https://cloud.r-project.org/')" + Rscript -e "install.packages(c('knitr', 'rmarkdown', 'testthat', 'e1071', 'survival'), repos='https://cloud.r-project.org/')" You can run just the SparkR tests using the command: