Merge branch 'master' into feature/gp_function_reference_docs

Author: WardBrian

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,6 +1,7 @@
 #### Submission Checklist
 
 - [ ] Builds locally 
+- [ ] New functions marked with `` `r since("VERSION")` ``
 - [ ] Declare copyright holder and open-source license: see below
 
 #### Summary

Jenkinsfile

@@ -1,7 +1,7 @@
 #!/usr/bin/env groovy
 
 pipeline {
-    agent { label 'gelman-group-linux' }
+    agent { label 'osx' }
     options {
         skipDefaultCheckout()
         preserveStashes(buildCount: 5)
@@ -12,7 +12,7 @@ pipeline {
         string(defaultValue: '', name: 'last_docs_version_dir', description: "Last docs version found in /docs. Example: 2_21")
         booleanParam(defaultValue: true, description: 'Build docs and create a PR ?', name: 'buildDocs')
         booleanParam(defaultValue: true, description: 'Change docs version for stan-dev.github.io ?', name: 'docsStanDev')
-        booleanParam(defaultValue: false, description: 'Build cmdstan manual ?', name: 'cmdstanManual')
+        booleanParam(defaultValue: true, description: 'Link docs to latest?', name: 'linkDocs')
     }
     environment {
         GITHUB_TOKEN = credentials('6e7c1e8f-ca2c-4b11-a70e-d934d3f6b681')
@@ -44,17 +44,21 @@ pipeline {
                 }
 
                 /* Build docs */
-                sh "python build.py $major_version $minor_version"
+                sh "python3 build.py $major_version $minor_version"
 
                 /* Add redirects */
-                sh "python add_redirects.py $major_version $minor_version functions-reference"
-                sh "python add_redirects.py $major_version $minor_version reference-manual"
-                sh "python add_redirects.py $major_version $minor_version stan-users-guide"
-
-                /* Link docs to latest */
-                sh "ls -lhart docs"
-                sh "chmod +x add_links.sh"
-                sh "./add_links.sh $last_docs_version_dir"
+                sh "python3 add_redirects.py $major_version $minor_version functions-reference"
+                sh "python3 add_redirects.py $major_version $minor_version reference-manual"
+                sh "python3 add_redirects.py $major_version $minor_version stan-users-guide"
+                sh "python3 add_redirects.py $major_version $minor_version cmdstan-guide"
+                script {
+                    if (params.linkDocs) {
+                        /* Link docs to latest */
+                        sh "ls -lhart docs"
+                        sh "chmod +x add_links.sh"
+                        sh "./add_links.sh $last_docs_version_dir"
+                    }
+                }
 
                 /* Create Pull Request */
                 withCredentials([usernamePassword(credentialsId: 'a630aebc-6861-4e69-b497-fd7f496ec46b',
@@ -105,12 +109,20 @@ pipeline {
                     def new_version_underscore = major_version + "_" + minor_version
                     def new_version_dot = major_version + "." + minor_version
 
-                    sh """
-                        sed -i 's/$last_docs_version_dir/$new_version_underscore/g' users/documentation/index.md
-                        sed -i 's/$last_version_dot/$new_version_dot/g' users/documentation/index.md
+                    /* Linux Version */
+                    //sh """
+                        //sed -i 's/$last_docs_version_dir/$new_version_underscore/g' users/documentation/index.md
+                        //sed -i 's/$last_version_dot/$new_version_dot/g' users/documentation/index.md
+                        //sed -i 's/$last_docs_version_dir/$new_version_underscore/g' users/interfaces/cmdstan.md
+                        //sed -i 's/$last_version_dot/$new_version_dot/g' users/interfaces/cmdstan.md
+                    //"""
 
-                        sed -i 's/$last_docs_version_dir/$new_version_underscore/g' users/interfaces/cmdstan.md
-                        sed -i 's/$last_version_dot/$new_version_dot/g' users/interfaces/cmdstan.md
+                    /* Mac Version */
+                    sh """
+                        sed -i.bak 's/$last_docs_version_dir/$new_version_underscore/g' users/documentation/index.md
+                        sed -i.bak 's/$last_version_dot/$new_version_dot/g' users/documentation/index.md
+                        sed -i.bak 's/$last_docs_version_dir/$new_version_underscore/g' users/interfaces/cmdstan.md
+                        sed -i.bak 's/$last_version_dot/$new_version_dot/g' users/interfaces/cmdstan.md
                     """
 
                 }
@@ -133,29 +145,5 @@ pipeline {
 
             }
         }
-        stage('Checkout cmdstan and build manual') {
-            when {
-              expression {
-                params.cmdstanManual
-              }
-            }
-            steps {
-                /* Checkout source code */
-                deleteDir()
-                checkout([$class: 'GitSCM',
-                          branches: [[name: '*/master']],
-                          doGenerateSubmoduleConfigurations: false,
-                          extensions: [],
-                          submoduleCfg: [],
-                          userRemoteConfigs: [[url: "https://github.com/stan-dev/cmdstan.git", credentialsId: 'a630aebc-6861-4e69-b497-fd7f496ec46b']]]
-                )
-
-                /* make manual */
-                sh "make manual"
-
-                /* archive manual for it to be shown in job result artifacts */
-                archiveArtifacts 'doc/*.pdf'
-            }
-        }
     }
 }

README.md

@@ -1,42 +1,92 @@
 # Repository `docs`
 
-Documentation for Stan language and platform
+Repository for the sources and published documentation set, versioned for each Stan minor release.
 
-## GitHub Pages
+* The Stan User's Guide - example models and techniques for coding statistical models in Stan and using them to do inference and prediction.
+* The Stan Reference Manual - specification for Stan language and core inference algorithms.
+* The Stan Functions Reference - functions and distributions built into the Stan language.
+* The CmdStan Guide - guide to the reference command-line interface.
 
-This repository uses
-[GitHub Pages](https://help.github.com/categories/github-pages-basics)
-to serve the
-[project pages](https://help.github.com/articles/user-organization-and-project-pages/#project-pages-sites) site
-with URL `https://mc-stan.org/docs`.
-The publishing strategy is to serve the contents of the directory `docs` on branch `master`.
-The `docs` directory contains an empty file named `.nojekyll` so that GitHub will treat the contents
-as pre-generated HTML instead of trying to run [jekyll](https://jekyllrb.com).
 
-## Directory Structure
+## Repository directory structure
 
-* `src` : directory of source files for Stan manuals and book, each in its own named subdirectory.
+* `src` : directory of source files for Stan and CmdStan guides and reference manuals, each in its own named subdirectory:
+    + `src/cmdstan-guide` - CmdStan Guide
+    + `src/functions-reference` - Stan Functions Reference
+	+ `src/reference-manual` - Stan Reference Manual
+	+ `src/stan-users-guide` - Stan Users Guide
 
 * `docs`: the directory `docs` on branch `master` is the [publishing source](https://help.github.com/articles/configuring-a-publishing-source-for-github-pages/) for the project pages site.  Whenever a verified member of the Stan organization pushes to `docs` on branch `master`,
 GitHub (re)builds and (re)deploys the website.
 
-  
-* `build.py`: python script which compiles markdown files under `src` to html and pdf and populates the `docs` dir with the generated documentation.
+## Documentation toolset
+
+The documentation source files are written in [Rmarkdown](https://rmarkdown.rstudio.com)
+and the RStudio's [bookdown package](https://github.com/rstudio/bookdown) converts these to HTML and pdf.
+**Required bookdown version:** 2.23 or higher, cf [the bookdown changelog](https://github.com/rstudio/bookdown/blob/main/NEWS.md#changes-in-bookdown-version-023).
+Reported via [issue #380](https://github.com/stan-dev/docs/issues/380).
+
+
+The conversion engine is [Pandoc](https://pandoc.org).  It is bundled with RStudio.
+To use the build scripts to build the docset,
+you might need to [install Pandoc](https://pandoc.org/installing.html) separately.
+
+To build the pdf version of the docs, you will need to [install LaTeX](https://www.latex-project.org/get/) as well.
+The Stan documentation uses the [Lucida fonts](https://www.pctex.com/Lucida_Fonts.html),
+which must be [installed manually](https://tex.stackexchange.com/questions/88423/manual-font-installation).
+
+
+## Scripts to build and maintain the docset
+
+**`build.py`**
+
+The program `build.py` convert the markdown files under `src` to html and pdf and populates the `docs` dir with the generated documentation.
+Requires Python 3.7 or higher, due to call to `subprocess.run`, kwarg `capture_output`.
+  + 2 required argments:  <Major> <minor> Stan version, expecting 2 positive integer arguments, e.g. `2 28`
+  + 2 optional arguments:  <document> <format>.  The document name corresponds to the name of the `src` subdirectory or `all`.  The output format is either `html` or `pdf`.
 
-  + arg 1: MAJOR Stan version, required (expecting number, should be positive int)
-  + arg 2: MINOR Stan version, required (expecting number, should be positive int)
-  + arg 3: name of document (optional - will build entire docset)
-  + arg 4: output format, either "html" or "pdf" (optional - default builds both html and pdfs)
-  
-* `LICENSE`: licensing terms.
 
+**Build script examples**
 
-## Generating the static site
+* `python build.py 2 28` - creates directory `docs/2_28` as needed; populates it will all generated documentation.
+* `python build.py 2 28 functions-reference` - builds both HTML and pdf versions of the Stan functions reference, resulting documents are under `docs/2_28`
+* `python build.py 2 28 functions-reference pdf` - builds only the pdf version of the Stan functions reference,  resulting document is `docs/2_28/functions-reference_2_28.pdf`
+* `python build.py 2 28 all pdf` - builds all pdfs from the Stan documentation set, resulting pdfs are in `docs/2_28`.
+ 
 
-1. Install R packages:  bookdown, arm
+**Additional scripts**
 
-2. Install on OS and ensure on PATH: pandoc, pandoc-citeproc, pdflatex
+The release process generates a new documentation set and adds links and redirects across the docset.
 
-3. Run `python build.py <MAJOR> <MINOR> (<document> (<format>))`
+* `add_redirects.py` manages the redirects from unversioned links to the latest version.
+* `link_to_latest.py` adds the "latest version" link into a docset.
+
+The Stan Functions Reference contains HTML comments which describe the function signature for all functions.  The script `extract_function_sigs.py` is used to scrape these signatures into a plain text file.
+
+## Build a single docset in R:  `bookdown::render_book`
+
+To build a single documet, you must have R or RStudio installed.
+To build a document from the command line, first `cd` to the correct `src` dir,
+then use the `Rscript` utility.
+
+```
+# build html
+> Rscript -e "bookdown::render_book('index.Rmd', output_format='bookdown::gitbook')"
+
+# build pdf
+> Rscript -e "bookdown::render_book('index.Rmd', output_format='bookdown::pdf_book')"
+```
+
+The output will be written to subdirectory `_build`.
+
+## GitHub Pages
+
+This repository uses
+[GitHub Pages](https://help.github.com/categories/github-pages-basics)
+to serve the
+[project pages](https://help.github.com/articles/user-organization-and-project-pages/#project-pages-sites) site
+with URL https://mc-stan.org/docs.
+The publishing strategy is to serve the contents of the directory `docs` on branch `master`.
+The `docs` directory contains an empty file named `.nojekyll` so that GitHub will treat the contents
+as pre-generated HTML instead of trying to run [jekyll](https://jekyllrb.com).
 
-4. The generated html and/or pdfs will be in the versioned subdirectory of `docs`.

build.py

@@ -29,11 +29,18 @@ def pushd(new_dir):
     os.chdir(previous_dir)
 
 def shexec(command):
-    returncode = subprocess.call(command, shell=True)
+    returncode = subprocess.call(command, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
     if returncode != 0:
         print('Command {} failed'.format(command))
         raise Exception(returncode)
 
+def check_bookdown_2_23():
+    command = ["Rscript", "-e", "utils::packageVersion(\"bookdown\") > 0.22"]
+    check = subprocess.run(command, capture_output=True, text=True)
+    if not 'TRUE' in check.stdout:
+        print("R Package bookdown must be version 2.23, see README for build tools requirements")
+        sys.exit(1)
+
 def safe_rm(fname):
     if os.path.exists(fname):
         os.remove(fname)
@@ -43,13 +50,16 @@ def make_docs(docspath, version, document, formats):
     srcpath = os.path.join(path, "src", document)
     with pushd(srcpath):
         if ("pdf" in formats):
+            print('render {} as pdf'.format(document))
             command = "Rscript -e \"bookdown::render_book(\'index.Rmd\', output_format=\'bookdown::pdf_book\')\""
             shexec(command)
             srcpdf = os.path.join(srcpath, "_book", "_main.pdf")
             pdfname = ''.join([document,'-',version,".pdf"])
             pdfpath = os.path.join(docspath, pdfname)
             shutil.move(srcpdf, pdfpath)
+            print('output file: {}'.format(pdfpath))
         if ("html" in formats):
+            print('render {} as html'.format(document))
             command = "Rscript -e \"bookdown::render_book(\'index.Rmd\', output_format=\'bookdown::gitbook\')\""
             shexec(command)
             [safe_rm(f) for f in ("stan-manual.css", "_main.rds")]
@@ -58,11 +68,16 @@ def make_docs(docspath, version, document, formats):
             shutil.rmtree(htmlpath, ignore_errors=True)
             command = ' '.join(["mv _book", htmlpath])
             shexec(command)
+            print('output dir: {}'.format(htmlpath))
         else:
             [safe_rm(f) for f in ("stan-manual.css", "_main.rds")]
             shutil.rmtree("_book", ignore_errors=True)
 
 def main():
+    if sys.version_info < (3, 7):
+        print('requires Python 3.7 or higher, found {}'.format(sys.version))
+        sys.exit(1)
+    check_bookdown_2_23()
     global all_docs
     global all_formats
     if (len(sys.argv) > 2):

docs/2_24/cmdstan-guide/appendices.html

@@ -256,6 +256,10 @@ <h1>
           <div class="page-inner">
 
             <section class="normal" id="section-">
+
+<div>
+This is an old version, <a href="https://mc-stan.org/docs/cmdstan-guide/appendices.html">view current version</a>.
+</div>
 <div id="appendices" class="section level1">
 <h1><i style='font-size: 110%; color:#990017;'>Appendices</i></h1>
 <p>This section contains the following appendices:</p>

docs/2_24/cmdstan-guide/bibliography.html

@@ -256,6 +256,10 @@ <h1>
           <div class="page-inner">
 
             <section class="normal" id="section-">
+
+<div>
+This is an old version, <a href="https://mc-stan.org/docs/cmdstan-guide/bibliography.html">view current version</a>.
+</div>
 <div id="bibliography" class="section level1">
 <h1><i style='font-size: 110%; color:#990017;'>Bibliography</i></h1>
 

docs/2_24/cmdstan-guide/cmdstan-installation.html

@@ -256,6 +256,10 @@ <h1>
           <div class="page-inner">
 
             <section class="normal" id="section-">
+
+<div>
+This is an old version, <a href="https://mc-stan.org/docs/cmdstan-guide/cmdstan-installation.html">view current version</a>.
+</div>
 <div id="cmdstan-installation" class="section level1">
 <h1><span class="header-section-number">1</span> CmdStan Installation</h1>
 <p>To install CmdStan you need:</p>

docs/2_24/cmdstan-guide/cmdstan-tools.html

@@ -256,6 +256,10 @@ <h1>
           <div class="page-inner">
 
             <section class="normal" id="section-">
+
+<div>
+This is an old version, <a href="https://mc-stan.org/docs/cmdstan-guide/cmdstan-tools.html">view current version</a>.
+</div>
 <div id="cmdstan-tools" class="section level1">
 <h1><i style='font-size: 110%; color:#990017;'>CmdStan Tools</i></h1>
 <p>This section provides a reference for the CmdStan tools:</p>

docs/2_24/cmdstan-guide/command-line-interface-overview.html

@@ -256,6 +256,10 @@ <h1>
           <div class="page-inner">
 
             <section class="normal" id="section-">
+
+<div>
+This is an old version, <a href="https://mc-stan.org/docs/cmdstan-guide/command-line-interface-overview.html">view current version</a>.
+</div>
 <div id="command-line-interface-overview" class="section level1">
 <h1><span class="header-section-number">8</span> Command-Line Interface Overview</h1>
 <p>A CmdStan executable is built from the Stan model concept and the CmdStan command line parser.

docs/2_24/cmdstan-guide/compiling-a-stan-program.html

@@ -256,6 +256,10 @@ <h1>
           <div class="page-inner">
 
             <section class="normal" id="section-">
+
+<div>
+This is an old version, <a href="https://mc-stan.org/docs/cmdstan-guide/compiling-a-stan-program.html">view current version</a>.
+</div>
 <div id="compiling-a-stan-program" class="section level1">
 <h1><span class="header-section-number">3</span> Compiling a Stan Program</h1>
 <p>A Stan program must be in a file with extension <code>.stan</code>.