Split compile tests into smaller chunks to improve CI times (#1123)

Split compile tests into smaller chunks to improve CI times

Author: serban-nicusor-toptal

Jenkinsfile

@@ -1,7 +1,7 @@
 @Library('StanUtils')
 import org.stan.Utils
 
-def utils = new org.stan.Utils()
+utils = new org.stan.Utils()
 
 def skipExpressionTests = false
 def skipRemainingStages = false
@@ -24,6 +24,30 @@ def tagName() {
     }
 }
 
+def runPerformanceTests(String testsPath){
+    unstash 'ubuntu-exe'
+
+    sh """
+        git clone --recursive --depth 50 https://github.com/stan-dev/performance-tests-cmdstan
+    """
+
+    writeFile(file:"performance-tests-cmdstan/cmdstan/make/local", text:"CXX=${CXX}")
+
+    utils.checkout_pr("cmdstan", "performance-tests-cmdstan/cmdstan", params.cmdstan_pr)
+    utils.checkout_pr("stan", "performance-tests-cmdstan/cmdstan/stan", params.stan_pr)
+    utils.checkout_pr("math", "performance-tests-cmdstan/cmdstan/stan/lib/stan_math", params.math_pr)
+
+    sh """
+        cd performance-tests-cmdstan
+        mkdir cmdstan/bin
+        cp ../bin/stanc cmdstan/bin/linux-stanc
+        cd cmdstan; make clean-all;
+        echo 'O=0' >> make/local
+        make -j${env.PARALLEL} build; cd ..
+        ./runPerformanceTests.py -j${env.PARALLEL} --runs=0 ${testsPath}
+    """
+}
+
 pipeline {
     agent none
     parameters {
@@ -38,7 +62,7 @@ pipeline {
     options {parallelsAlwaysFailFast()}
     environment {
         CXX = 'clang++-6.0'
-        PARALLEL = 8
+        PARALLEL = 4
     }
     stages {
         stage('Kill previous builds') {
@@ -176,7 +200,8 @@ pipeline {
         }
         stage("CmdStan & Math tests") {
             parallel {
-                stage("Compile tests") {
+
+                stage("Compile tests - good") {
                     when {
                         beforeAgent true
                         expression {
@@ -191,26 +216,36 @@ pipeline {
                     }
                     steps {
                         script {
-                            unstash 'ubuntu-exe'
-                            sh """
-                                git clone --recursive --depth 50 https://github.com/stan-dev/performance-tests-cmdstan
-                            """
-
-                            writeFile(file:"performance-tests-cmdstan/cmdstan/make/local",
-                                    text:"O=0\nCXX=${CXX}")
+                            runPerformanceTests("../test/integration/good")
+                        }
 
-                            utils.checkout_pr("cmdstan", "performance-tests-cmdstan/cmdstan", params.cmdstan_pr)
-                            utils.checkout_pr("stan", "performance-tests-cmdstan/cmdstan/stan", params.stan_pr)
-                            utils.checkout_pr("math", "performance-tests-cmdstan/cmdstan/stan/lib/stan_math", params.math_pr)
+                        xunit([GoogleTest(
+                            deleteOutputFiles: false,
+                            failIfNotNew: true,
+                            pattern: 'performance-tests-cmdstan/performance.xml',
+                            skipNoTestFiles: false,
+                            stopProcessingIfError: false)
+                        ])
+                    }
+                    post { always { runShell("rm -rf ./*") }}
+                }
 
-                            sh """
-                                cd performance-tests-cmdstan
-                                mkdir cmdstan/bin
-                                cp ../bin/stanc cmdstan/bin/linux-stanc
-                                cd cmdstan; make clean-all; make -j${env.PARALLEL} build; cd ..
-                                ./runPerformanceTests.py -j${env.PARALLEL} --runs=0 ../test/integration/good
-                                ./runPerformanceTests.py -j${env.PARALLEL} --runs=0 example-models
-                                """
+                stage("Compile tests - example-models") {
+                    when {
+                        beforeAgent true
+                        expression {
+                            !skipCompileTests
+                        }
+                    }
+                    agent {
+                        docker {
+                            image 'stanorg/ci:gpu'
+                            label 'linux'
+                        }
+                    }
+                    steps {
+                        script {
+                            runPerformanceTests("example-models")
                         }
 
                         xunit([GoogleTest(
@@ -223,6 +258,7 @@ pipeline {
                     }
                     post { always { runShell("rm -rf ./*") }}
                 }
+
                 stage("Model end-to-end tests") {
                     when {
                         beforeAgent true

README.md

@@ -3,10 +3,9 @@ This repo contains a new compiler for Stan, stanc3, written in OCaml.
 Since version 2.26, this has been the default compiler for Stan. See [this wiki](https://github.com/stan-dev/stanc3/wiki/changes-from-stanc2) for a list of minor differences between this compiler and the previous Stan compiler.
 
 To read more about why we built this, see this [introductory blog post](https://statmodeling.stat.columbia.edu/2019/03/13/stanc3-rewriting-the-stan-compiler/). For some discussion as to how we chose OCaml, see [this accidental flamewar](https://discourse.mc-stan.org/t/choosing-the-new-stan-compilers-implementation-language/6203).
-We're testing [these models](https://jenkins.mc-stan.org/job/stanc3/job/master/) (listed under Test Results) on every pull request.
-
-[![Build Status](https://jenkins.mc-stan.org/buildStatus/icon?job=stanc3%2Fmaster&style=flat-square)](https://jenkins.mc-stan.org/job/stanc3/job/master/)
+We're testing [these models](https://jenkins.flatironinstitute.org/job/Stan/job/Stanc3/job/master/) (listed under Test Results) on every pull request.
 
+[![Build Status](https://jenkins.flatironinstitute.org/job/Stan/job/Stanc3/job/master/badge/icon?style=flat-square)](https://jenkins.flatironinstitute.org/job/Stan/job/Stanc3/job/master/)
 
 ## Documentation