Update distribution test detector for better balancing

Author: WardBrian

Jenkinsfile

@@ -511,9 +511,14 @@ pipeline {
             steps {
                 script {
                     def tests = [:]
-                    for (f in changedDistributionTests.collate(18)) {
+                    def tests_per_executor = 180
+                    def idx = 0
+                    // roughly 10 executors when all tests run
+                    def executors = (changedDistributionTests.size() + tests_per_executor - 1) / tests_per_executor
+                    for (f in changedDistributionTests.collate(tests_per_executor)) {
+                        idx = idx + 1
                         def names = f.join(" ")
-                        tests["Distribution Tests: ${names}"] = { node ("linux && docker && 8core") {
+                        tests["Distribution Tests: ${idx} / ${executors}"] = { node ("linux && docker && 8core") {
                             deleteDir()
                             docker.image('stanorg/ci:gpu-cpp17').inside {
                                 catchError(buildResult: "FAILURE", stageResult: "FAILURE") {
@@ -536,7 +541,8 @@ pipeline {
                             }
                         } }
                     }
-                    parallel(failfast: true) tests
+                    tests.failFast = true
+                    parallel tests
                 }
             }
             post {

test/prob/getDependencies.py

@@ -9,7 +9,7 @@
 import sys
 import subprocess
 from pathlib import Path
-
+import glob
 
 def get_dependencies(file: Path) -> Set[str]:
     file_dot_d = file.with_suffix(".d")
@@ -36,6 +36,11 @@ def get_changed() -> Set[str]:
     return set(changed_files)
 
 
+def add_tests_from_hpp(tests_to_run, test):
+    pattern = str(test).replace("_test.hpp", "_0*_test.cpp")
+    tests = glob.glob(pattern)
+    tests_to_run.extend(tests)
+
 if __name__ == "__main__":
     if Path("makefile") not in Path(".").iterdir():
         raise ValueError("getDependencies must be ran from the top-level repository")
@@ -47,12 +52,16 @@ def get_changed() -> Set[str]:
 
     distribution_tests = Path("test", "prob")
 
+    subprocess.run(["make", "generate-tests"], stdout=subprocess.DEVNULL)
+
     for dist in distribution_tests.iterdir():
         if not dist.is_dir():
             continue
+        print(f"Considering {dist}.", file=sys.stderr)
         for test in dist.iterdir():
+            if test.suffix != ".hpp": continue
             if pretend:
-                tests_to_run.append(str(test).replace("_test.hpp", "_0*_test.cpp"))
+                add_tests_from_hpp(tests_to_run, test)
             else:
                 deps = get_dependencies(test)
                 intersection = changed & deps
@@ -61,7 +70,6 @@ def get_changed() -> Set[str]:
                         f"{test} has {len(intersection)} changed dependencies out of {len(deps)} files #included.",
                         file=sys.stderr,
                     )
+                    add_tests_from_hpp(tests_to_run, test)
 
-                    tests_to_run.append(str(test).replace("_test.hpp", "_0*_test.cpp"))
-
-    print("\n".join(tests_to_run))
+    print("\n".join(set(tests_to_run)))