Merge pull request #3307 from stan-dev/ci/distribution-tests-tweaks

Try to improve error handling of parallel distribution tests

Author: WardBrian

Jenkinsfile

@@ -487,6 +487,12 @@ pipeline {
             steps {
                 script {
                     retry(3) { checkout scm }
+
+                    if (params.withRowVector || isBranch('develop') || isBranch('master')) {
+                        sh "echo CXXFLAGS+=-DSTAN_TEST_ROW_VECTORS >> make/local"
+                        sh "echo CXXFLAGS+=-DSTAN_PROB_TEST_ALL >> make/local"
+                    }
+
                     if (params.runAllDistributions || isBranch('develop') || isBranch('master')) {
                         changedDistributionTests = sh(script:"python3 test/prob/getDependencies.py --pretend-all", returnStdout:true).trim().readLines()
                     } else {
@@ -511,12 +517,17 @@ pipeline {
             steps {
                 script {
                     def tests = [:]
-                    for (f in changedDistributionTests.collate(24)) {
+
+                    def executors = 10
+                    def tests_per_executor = Math.ceil((changedDistributionTests.size() / executors).doubleValue()).toInteger()
+                    def idx = 0
+                    for (f in changedDistributionTests.collate(tests_per_executor)) {
+                        idx = idx + 1
                         def names = f.join(" ")
-                        tests["Distribution Tests: ${names}"] = { node ("linux && docker && 8core") {
+                        tests["Distribution Tests: ${idx} / ${executors}"] = { node ("linux && docker && 8core") {
                             deleteDir()
                             docker.image('stanorg/ci:gpu-cpp17').inside {
-                                catchError {
+                                catchError(buildResult: "FAILURE", stageResult: "FAILURE") {
                                     unstash 'MathSetup'
                                     sh """
                                         echo CXX=${CLANG_CXX} > make/local

test/prob/getDependencies.py

@@ -9,7 +9,7 @@
 import sys
 import subprocess
 from pathlib import Path
-
+import glob
 
 def get_dependencies(file: Path) -> Set[str]:
     file_dot_d = file.with_suffix(".d")
@@ -36,6 +36,11 @@ def get_changed() -> Set[str]:
     return set(changed_files)
 
 
+def add_tests_from_hpp(tests_to_run, test):
+    pattern = str(test).replace("_test.hpp", "_0*_test.cpp")
+    tests = glob.glob(pattern)
+    tests_to_run.extend(tests)
+
 if __name__ == "__main__":
     if Path("makefile") not in Path(".").iterdir():
         raise ValueError("getDependencies must be ran from the top-level repository")
@@ -47,12 +52,16 @@ def get_changed() -> Set[str]:
 
     distribution_tests = Path("test", "prob")
 
+    subprocess.run(["make", "generate-tests"], stdout=subprocess.DEVNULL)
+
     for dist in distribution_tests.iterdir():
         if not dist.is_dir():
             continue
+        print(f"Considering {dist}.", file=sys.stderr)
         for test in dist.iterdir():
+            if test.suffix != ".hpp": continue
             if pretend:
-                tests_to_run.append(str(test).replace("_test.hpp", "_0*_test.cpp"))
+                add_tests_from_hpp(tests_to_run, test)
             else:
                 deps = get_dependencies(test)
                 intersection = changed & deps
@@ -61,7 +70,6 @@ def get_changed() -> Set[str]:
                         f"{test} has {len(intersection)} changed dependencies out of {len(deps)} files #included.",
                         file=sys.stderr,
                     )
+                    add_tests_from_hpp(tests_to_run, test)
 
-                    tests_to_run.append(str(test).replace("_test.hpp", "_0*_test.cpp"))
-
-    print("\n".join(tests_to_run))
+    print("\n".join(set(tests_to_run)))