From 9d94d95c99c5d71fabc9064dac5a52a12a0ea2df Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Wed, 8 Oct 2025 15:32:42 -0400 Subject: [PATCH 1/5] ci: remove "max-workers" and test with our gradle defaults If our defaults need improvement, let's improve them to make tests run faster. We want to take advantage of the hardware. --- .github/workflows/run-checks-all.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run-checks-all.yml b/.github/workflows/run-checks-all.yml index 9efba2267423..6985d745ee86 100644 --- a/.github/workflows/run-checks-all.yml +++ b/.github/workflows/run-checks-all.yml @@ -58,7 +58,7 @@ jobs: echo "lucene.tool.ast-grep=ast-grep" >> build-options.local.properties - name: Run gradle check (without tests) - run: ./gradlew check -x test "-Ptask.times=true" --max-workers 2 + run: ./gradlew check -x test "-Ptask.times=true" # This runs all tests without any other validation checks. @@ -88,7 +88,7 @@ jobs: uses: ./.github/actions/prepare-for-build - name: Run gradle tests - run: ./gradlew test "-Ptask.times=true" --max-workers 2 + run: ./gradlew test "-Ptask.times=true" - name: List automatically-initialized gradle.properties run: cat gradle.properties From 69581cad0b7a2133437858ce5e3c6ddd0245f843 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Wed, 8 Oct 2025 18:04:26 -0400 Subject: [PATCH 2/5] build: fix cpu defaults Currently these don't use all the cpus: * Ubuntu and Windows runners have 4 cpus * MacOS X has 3 cpus (!) Dividing by two (assuming hyperthreads?) doesn't yield a speedup either. I set the value to 4 on 2-core system for this reason. --- .../org/apache/lucene/gradle/GradlePropertiesGenerator.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/GradlePropertiesGenerator.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/GradlePropertiesGenerator.java index 5436afe70f80..b80471371887 100644 --- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/GradlePropertiesGenerator.java +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/GradlePropertiesGenerator.java @@ -54,10 +54,10 @@ public void run(Path source, Path destination) throws IOException { } // Approximate a common-sense default for running gradle/tests with parallel - // workers: half the count of available cpus but not more than 12. + // workers: the count of available cpus but not more than 12. var cpus = Runtime.getRuntime().availableProcessors(); - var maxWorkers = (int) Math.max(1d, Math.min(cpus * 0.5d, 12)); - var testsJvms = (int) Math.max(1d, Math.min(cpus * 0.5d, 12)); + var maxWorkers = Math.min(cpus, 12); + var testsJvms = Math.min(cpus, 12); var replacements = Map.of("@MAX_WORKERS@", maxWorkers, "@TEST_JVMS@", testsJvms); From feca015d7b21cf4d37ad8ecec68760325e608082 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Wed, 8 Oct 2025 18:14:23 -0400 Subject: [PATCH 3/5] build: adjust cpu defaults here to match (for consistency) --- .../gradle/plugins/java/TestsAndRandomizationPlugin.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/TestsAndRandomizationPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/TestsAndRandomizationPlugin.java index a48ae6c61435..86d240067f4f 100644 --- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/TestsAndRandomizationPlugin.java +++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/TestsAndRandomizationPlugin.java @@ -186,9 +186,7 @@ public void apply(Project project) { .getProviders() .provider( () -> { - return ((int) - Math.max( - 1, Math.min(Runtime.getRuntime().availableProcessors() / 2.0, 4.0))); + return Math.min(12, Runtime.getRuntime().availableProcessors()); })); // GITHUB#13986: Allow easier configuration of the Panama Vectorization provider with newer Java From 94542647c09fc1f07697f93101f4cc56a6574be2 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Wed, 8 Oct 2025 18:23:57 -0400 Subject: [PATCH 4/5] docs: update the template inline doc around build parallelism --- gradle/template.gradle.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/template.gradle.properties b/gradle/template.gradle.properties index 59ae7422b0a0..1e808746e1b5 100644 --- a/gradle/template.gradle.properties +++ b/gradle/template.gradle.properties @@ -16,7 +16,7 @@ ############### # # Gradle build can run tasks in parallel but by default it consumes all CPU cores which -# is too optimistic a default for Lucene tests. You can disable the parallelism +# may be too optimistic a default for Lucene tests. You can disable the parallelism # entirely or assign it a 'low' priority with these properties: # # org.gradle.parallel=[true, false] @@ -24,7 +24,7 @@ # # The default level of parallelism is computed based on the number of cores on # your machine (on the first run of gradle build). By default these are fairly conservative -# settings (half the number of cores for workers, for example): +# settings (the number of cores for workers up to a limit of 12, for example): # # org.gradle.workers.max=[X] # tests.jvms=[N <= X] From 549c1827666b2533fbf6264f582fc65cd4b30485 Mon Sep 17 00:00:00 2001 From: Robert Muir Date: Thu, 9 Oct 2025 01:22:53 -0400 Subject: [PATCH 5/5] ci: fix performance issues with test runners --- .github/workflows/run-checks-all.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/run-checks-all.yml b/.github/workflows/run-checks-all.yml index 6985d745ee86..ecf516394549 100644 --- a/.github/workflows/run-checks-all.yml +++ b/.github/workflows/run-checks-all.yml @@ -87,8 +87,20 @@ jobs: - name: Configure tools uses: ./.github/actions/prepare-for-build + - name: Speedup MacOS runner + if: ${{ runner.os == 'macOS' }} + run: | + mkdir /tmp/tmpfs + sudo mount_tmpfs -o noowners -s 1g /tmp/tmpfs + sudo sysctl debug.lowpri_throttle_enabled=0 + echo "tests.workDir=/tmp/tmpfs/lucene" >> build-options.local.properties + - name: Run gradle tests run: ./gradlew test "-Ptask.times=true" + env: + # Set to the defaults to override the "CI"-based logic that would enable C2 + # we can't afford C2 on github runners. + TEST_JVM_ARGS: "-XX:TieredStopAtLevel=1 -XX:+UseParallelGC -XX:ActiveProcessorCount=1" - name: List automatically-initialized gradle.properties run: cat gradle.properties