Skip to content

Commit 73ce432

Browse files
triceotriceo
committed
ci: introduce Quickstarts performance workflow
In order to be able to run the workflow and test it on a branch, the workflow first needs to exist on the main branch. Therefore the first commit lands now.
1 parent 8f3dea2 commit 73ce432

2 files changed

Lines changed: 633 additions & 0 deletions

File tree

‎.github/workflows/performance_quickstarts.yml‎

Lines changed: 311 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,311 @@
1+
# Both baseline and SUT (Software Under Test) are built from source first,
2+
# with their binaries uploaded as artifacts.
3+
# This is done on GitHub infrastructure, to achieve maximum parallelization.
4+
#
5+
# The benchmark job downloads the binaries and runs them.
6+
# The baseline is established first, then the SUT is measured.
7+
# They both run in the same job,
8+
# to guarantee they run on the same machine with the same performance characteristics.
9+
# This is done on a self-hosted runner which we completely control.
10+
#
11+
# Each benchmark gives a 99.9 % confidence interval.
12+
# The confidence intervals are compared to determine if the branch under test is a regression or an improvement.
13+
# The error threshold is expected to be below +/- 3.0 %.
14+
#
15+
name: Quickstart Perf Regression Test
16+
permissions:
17+
contents: read
18+
19+
on:
20+
workflow_dispatch:
21+
inputs:
22+
jdk_baseline:
23+
description: 'JDK version'
24+
default: '25'
25+
required: true
26+
baseline:
27+
description: 'Baseline branch or tag'
28+
default: 'main'
29+
required: true
30+
jdk_branch:
31+
description: 'JDK version'
32+
default: '25'
33+
required: true
34+
branch:
35+
description: 'Branch to benchmark (needs to use 999-SNAPSHOT)'
36+
default: 'main'
37+
required: true
38+
branch_owner:
39+
description: 'User owning the branch'
40+
default: 'TimefoldAI'
41+
required: true
42+
runs:
43+
description: 'Solver runs per quickstart per version'
44+
default: '20'
45+
required: false
46+
time_limit:
47+
description: 'Solver time limit per run in seconds'
48+
default: '60'
49+
required: false
50+
51+
run-name: "TimefoldAI's ${{ github.event.inputs.baseline }} vs. ${{ github.event.inputs.branch_owner }}'s ${{ github.event.inputs.branch }} (Java ${{ github.event.inputs.jdk_baseline }} vs. ${{ github.event.inputs.jdk_branch }})"
52+
53+
jobs:
54+
build_baseline:
55+
runs-on: ubuntu-latest # Leverage massive parallelization of Github-hosted runners.
56+
strategy:
57+
fail-fast: true # If one compilation fails, abort everything.
58+
matrix:
59+
# When updating this list, use find-and-replace in the entire file to keep all such lists identical.
60+
example: [bed-allocation, conference-scheduling, employee-scheduling, facility-location, flight-crew-scheduling, food-packaging, maintenance-scheduling, meeting-scheduling, order-picking, project-job-scheduling, school-timetabling, sports-league-scheduling, task-assigning, tournament-scheduling, vehicle-routing]
61+
steps:
62+
- name: Checkout timefold-solver-quickstarts
63+
uses: actions/checkout@v4
64+
with:
65+
repository: TimefoldAI/timefold-solver-quickstarts
66+
path: ./timefold-solver-quickstarts
67+
ref: main
68+
69+
- name: Setup JDK and Maven
70+
uses: actions/setup-java@v5
71+
with:
72+
java-version: 25 # Always build with the least recent supported JDK.
73+
distribution: 'temurin'
74+
cache: 'maven'
75+
76+
- name: Checkout timefold-solver
77+
uses: actions/checkout@v4
78+
with:
79+
repository: TimefoldAI/timefold-solver
80+
ref: ${{ github.event.inputs.baseline }}
81+
path: ./timefold-solver
82+
83+
- name: Quickly build timefold-solver
84+
working-directory: ./timefold-solver
85+
shell: bash
86+
run: ./mvnw -B -Dquickly clean install
87+
88+
- name: Switch quickstarts to baseline branch if it exists
89+
working-directory: ./timefold-solver-quickstarts
90+
shell: bash
91+
env:
92+
TARGET_BRANCH: ${{ github.event.inputs.baseline }}
93+
run: |
94+
if git ls-remote --exit-code --heads origin "$TARGET_BRANCH" > /dev/null; then
95+
git fetch --depth=1 origin "$TARGET_BRANCH"
96+
git checkout -B "$TARGET_BRANCH" FETCH_HEAD
97+
fi
98+
git status
99+
100+
- name: Build the quickstart
101+
shell: bash
102+
run: |
103+
mvn -B -DskipTests package \
104+
-f timefold-solver-quickstarts/java/${{ matrix.example }}/pom.xml
105+
106+
- name: Upload the binaries
107+
uses: actions/upload-artifact@v4
108+
with:
109+
name: baseline-${{ matrix.example }}
110+
path: |
111+
./timefold-solver-quickstarts/java/${{ matrix.example }}/target/quarkus-app/
112+
if-no-files-found: error
113+
114+
build_sut:
115+
runs-on: ubuntu-latest # Leverage massive parallelization of Github-hosted runners.
116+
strategy:
117+
fail-fast: true # If one compilation fails, abort everything.
118+
matrix:
119+
# When updating this list, use find-and-replace in the entire file to keep all such lists identical.
120+
example: [bed-allocation, conference-scheduling, employee-scheduling, facility-location, flight-crew-scheduling, food-packaging, maintenance-scheduling, meeting-scheduling, order-picking, project-job-scheduling, school-timetabling, sports-league-scheduling, task-assigning, tournament-scheduling, vehicle-routing]
121+
steps:
122+
- name: Checkout timefold-solver-quickstarts
123+
uses: actions/checkout@v4
124+
with:
125+
repository: TimefoldAI/timefold-solver-quickstarts
126+
path: ./timefold-solver-quickstarts
127+
ref: main
128+
129+
- name: Setup JDK and Maven
130+
uses: actions/setup-java@v5
131+
with:
132+
java-version: 25 # Always build with the least recent supported JDK.
133+
distribution: 'temurin'
134+
cache: 'maven'
135+
136+
- name: Checkout timefold-solver
137+
uses: actions/checkout@v4
138+
with:
139+
repository: ${{ github.event.inputs.branch_owner }}/timefold-solver
140+
ref: ${{ github.event.inputs.branch }}
141+
path: ./timefold-solver
142+
143+
- name: Quickly build timefold-solver
144+
working-directory: ./timefold-solver
145+
shell: bash
146+
run: ./mvnw -B -Dquickly clean install
147+
148+
- name: Switch quickstarts to branch under test if it exists
149+
working-directory: ./timefold-solver-quickstarts
150+
shell: bash
151+
env:
152+
TARGET_BRANCH: ${{ github.event.inputs.branch }}
153+
run: |
154+
if git ls-remote --exit-code --heads origin "$TARGET_BRANCH" > /dev/null; then
155+
git fetch --depth=1 origin "$TARGET_BRANCH"
156+
git checkout -B "$TARGET_BRANCH" FETCH_HEAD
157+
fi
158+
git status
159+
160+
- name: Build the quickstart
161+
shell: bash
162+
run: |
163+
mvn -B -DskipTests package \
164+
-f timefold-solver-quickstarts/java/${{ matrix.example }}/pom.xml
165+
166+
- name: Upload the binaries
167+
uses: actions/upload-artifact@v4
168+
with:
169+
name: sut-${{ matrix.example }}
170+
path: |
171+
./timefold-solver-quickstarts/java/${{ matrix.example }}/target/quarkus-app/
172+
if-no-files-found: error
173+
174+
benchmark:
175+
needs: [ build_baseline, build_sut ]
176+
runs-on: self-hosted # We need a stable machine to actually run the benchmarks.
177+
strategy:
178+
fail-fast: false # Jobs fail if the benchmark error is over predefined thresholds; other benchmarks continue.
179+
matrix:
180+
# When updating this list, use find-and-replace in the entire file to keep all such lists identical.
181+
example: [bed-allocation, conference-scheduling, employee-scheduling, facility-location, flight-crew-scheduling, food-packaging, maintenance-scheduling, meeting-scheduling, order-picking, project-job-scheduling, school-timetabling, sports-league-scheduling, task-assigning, tournament-scheduling, vehicle-routing]
182+
steps:
183+
- name: Clean results of previous runs
184+
shell: bash
185+
env:
186+
BASELINE: ${{ github.event.inputs.baseline }}
187+
BRANCH: ${{ github.event.inputs.branch }}
188+
run: |
189+
# DIRs are different, so that we can run "main" against "main" and have separate results.
190+
# Strip CR/LF from inputs before writing to $GITHUB_ENV to prevent env-file injection.
191+
SANITIZED_BASELINE=$(echo "$BASELINE" | tr -d '\r\n' | sed 's/\//\-/g')
192+
SANITIZED_BRANCH=$(echo "$BRANCH" | tr -d '\r\n' | sed 's/\//\-/g')
193+
{
194+
echo "SANITIZED_BASELINE=$SANITIZED_BASELINE"
195+
echo "SANITIZED_BRANCH=$SANITIZED_BRANCH"
196+
} >> "$GITHUB_ENV"
197+
198+
- name: Setup Python
199+
uses: actions/setup-python@v5
200+
with:
201+
python-version: 'x'
202+
203+
- name: Checkout timefold-solver-benchmarks
204+
uses: actions/checkout@v4
205+
with:
206+
repository: TimefoldAI/timefold-solver-benchmarks
207+
path: ./timefold-solver-benchmarks
208+
209+
- name: Download the baseline binaries
210+
uses: actions/download-artifact@v4
211+
with:
212+
name: baseline-${{ matrix.example }}
213+
path: ./baseline-app
214+
215+
- name: Download the SUT binaries
216+
uses: actions/download-artifact@v4
217+
with:
218+
name: sut-${{ matrix.example }}
219+
path: ./sut-app
220+
221+
- name: (Baseline) Setup JDK
222+
uses: actions/setup-java@v5
223+
with:
224+
java-version: ${{ github.event.inputs.jdk_baseline }}
225+
distribution: 'temurin'
226+
check-latest: true
227+
228+
- name: (Baseline) Run the benchmark
229+
id: benchmark_baseline
230+
shell: bash
231+
run: |
232+
python3 timefold-solver-benchmarks/benchmark-quickstarts.py \
233+
${{ matrix.example }} ./baseline-app \
234+
--runs ${{ github.event.inputs.runs }} \
235+
--time-limit ${{ github.event.inputs.time_limit }} \
236+
--base-port $((8080 + ${{ strategy.job-index }})) \
237+
--output baseline.csv
238+
echo "RANGE_MID=$(tail -1 baseline.csv | cut -d',' -f3)" >> "$GITHUB_OUTPUT"
239+
echo "RANGE_START=$(tail -1 baseline.csv | cut -d',' -f7)" >> "$GITHUB_OUTPUT"
240+
echo "RANGE_END=$(tail -1 baseline.csv | cut -d',' -f8)" >> "$GITHUB_OUTPUT"
241+
242+
- name: (SUT) Setup JDK
243+
uses: actions/setup-java@v5
244+
with:
245+
java-version: ${{ github.event.inputs.jdk_branch }}
246+
distribution: 'temurin'
247+
check-latest: true
248+
249+
- name: (SUT) Run the benchmark
250+
id: benchmark_sut
251+
shell: bash
252+
run: |
253+
python3 timefold-solver-benchmarks/benchmark-quickstarts.py \
254+
${{ matrix.example }} ./sut-app \
255+
--runs ${{ github.event.inputs.runs }} \
256+
--time-limit ${{ github.event.inputs.time_limit }} \
257+
--base-port $((8080 + ${{ strategy.job-index }})) \
258+
--output sut.csv
259+
echo "RANGE_MID=$(tail -1 sut.csv | cut -d',' -f3)" >> "$GITHUB_OUTPUT"
260+
echo "RANGE_START=$(tail -1 sut.csv | cut -d',' -f7)" >> "$GITHUB_OUTPUT"
261+
echo "RANGE_END=$(tail -1 sut.csv | cut -d',' -f8)" >> "$GITHUB_OUTPUT"
262+
263+
- name: Report results
264+
env:
265+
BASELINE: ${{ github.event.inputs.baseline }}
266+
BRANCH: ${{ github.event.inputs.branch }}
267+
OWNER: ${{ github.event.inputs.branch_owner }}
268+
EXAMPLE: ${{ matrix.example }}
269+
BASELINE_RANGE_START: ${{ steps.benchmark_baseline.outputs.RANGE_START }}
270+
BASELINE_RANGE_MID: ${{ steps.benchmark_baseline.outputs.RANGE_MID }}
271+
BASELINE_RANGE_END: ${{ steps.benchmark_baseline.outputs.RANGE_END }}
272+
SUT_RANGE_START: ${{ steps.benchmark_sut.outputs.RANGE_START }}
273+
SUT_RANGE_MID: ${{ steps.benchmark_sut.outputs.RANGE_MID }}
274+
SUT_RANGE_END: ${{ steps.benchmark_sut.outputs.RANGE_END }}
275+
shell: bash
276+
run: |
277+
BASELINE_DEV=$(echo "scale=2; ($BASELINE_RANGE_MID / $BASELINE_RANGE_START) * 100 - 100" | bc)
278+
SUT_DEV=$(echo "scale=2; ($SUT_RANGE_MID / $SUT_RANGE_START) * 100 - 100" | bc)
279+
DIFF_MID=$(echo "scale=2; ($BASELINE_RANGE_MID / $SUT_RANGE_MID) * 100" | bc)
280+
FAIL=false
281+
282+
if (( $(echo "$DIFF_MID >= 97.00" | bc -l) && $(echo "$DIFF_MID <= 103.00" | bc -l) )); then
283+
echo "### ✅ Within tolerance" >> $GITHUB_STEP_SUMMARY
284+
elif [ "$SUT_RANGE_START" -gt "$BASELINE_RANGE_END" ]; then
285+
echo "### 🚀 Statistically significant improvement" >> $GITHUB_STEP_SUMMARY
286+
elif [ "$BASELINE_RANGE_START" -gt "$SUT_RANGE_END" ]; then
287+
echo "### ‼️ Statistically significant regression ‼️" >> $GITHUB_STEP_SUMMARY
288+
FAIL=true
289+
else
290+
echo "### ⁉️ Undetermined result ⁉️" >> $GITHUB_STEP_SUMMARY
291+
FAIL=true
292+
fi
293+
294+
BASELINE_URL="https://github.com/TimefoldAI/timefold-solver/tree/$BASELINE"
295+
SUT_URL="https://github.com/$OWNER/timefold-solver/tree/$BRANCH"
296+
297+
echo "| | **Ref** | **Mean** |" >> $GITHUB_STEP_SUMMARY
298+
echo "|:------:|:-----------:|:-----------------:|" >> $GITHUB_STEP_SUMMARY
299+
echo "| _Old_ | [TimefoldAI's $BASELINE]($BASELINE_URL) | $BASELINE_RANGE_MID ± $BASELINE_DEV % |" >> $GITHUB_STEP_SUMMARY
300+
echo "| _New_ | [$OWNER's $BRANCH]($SUT_URL) | $SUT_RANGE_MID ± $SUT_DEV % |" >> $GITHUB_STEP_SUMMARY
301+
echo "| _Diff_ | | $DIFF_MID % |" >> $GITHUB_STEP_SUMMARY
302+
303+
echo "" >> $GITHUB_STEP_SUMMARY
304+
echo "Quickstart: $EXAMPLE" >> $GITHUB_STEP_SUMMARY
305+
echo "Mean is in moves per second. Higher is better." >> $GITHUB_STEP_SUMMARY
306+
echo "Mean ± X % describes a 99.9 % confidence interval." >> $GITHUB_STEP_SUMMARY
307+
echo "Diff under 100 % represents an improvement, over 100 % a regression." >> $GITHUB_STEP_SUMMARY
308+
309+
if [ "$FAIL" = true ]; then
310+
exit 1
311+
fi

0 commit comments

Comments
 (0)