Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 53 additions & 41 deletions .github/workflows/performance_score_director.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,12 +50,14 @@ jobs:
outputs:
baseline_solver_version: ${{ steps.step1.outputs.version }}
needs_snapshot_built: ${{ steps.step1.outputs.needs_snapshot_built }}
env:
BASELINE: ${{ github.event.inputs.baseline }}
steps:
- name: Determine the baseline
id: step1
shell: bash
run: |
if [[ "${{ github.event.inputs.baseline }}" =~ ^v([0-9]+\.[0-9]+\.[0-9]+)$ ]]; then
if [[ "$BASELINE" =~ ^v([0-9]+\.[0-9]+\.[0-9]+)$ ]]; then
VERSION="${BASH_REMATCH[1]}"
NEEDS_SNAPSHOT_BUILT=false
echo "Baseline is a release tag ($VERSION)."
Expand All @@ -77,9 +79,6 @@ jobs:
matrix:
# When updating this list, use find-and-replace in the entire file to keep all such lists identical.
example: [cloud_balancing, conference_scheduling, curriculum_course, examination, flow_shop, machine_reassignment, meeting_scheduling, nurse_rostering, patient_admission_scheduling, task_assigning, traveling_tournament, vehicle_routing]
env:
MVN_USERNAME: '${{ secrets.JFROG_ENTERPRISE_READ_ONLY_ACCESS_USERNAME }}'
MVN_PASSWORD: '${{ secrets.JFROG_ENTERPRISE_READ_ONLY_ACCESS_TOKEN }}'
steps:
- name: Checkout timefold-solver-benchmarks
uses: actions/checkout@v6
Expand All @@ -94,9 +93,6 @@ jobs:
java-version: 25 # Always build with the least recent supported JDK.
distribution: 'temurin'
cache: 'maven'
server-id: 'timefold-solver-enterprise'
server-username: 'MVN_USERNAME'
server-password: 'MVN_PASSWORD'

# Only build the snapshots if determined by the decisions job.
- name: Checkout timefold-solver
Expand Down Expand Up @@ -129,8 +125,9 @@ jobs:
if: needs.decisions.outputs.needs_snapshot_built == 'true'
working-directory: ./timefold-solver-benchmarks
shell: bash
env:
TARGET_BRANCH: ${{ github.event.inputs.baseline }}
run: |
TARGET_BRANCH="${{ github.event.inputs.baseline }}"
if git ls-remote --exit-code --heads origin "$TARGET_BRANCH" > /dev/null; then
git fetch --depth=1 origin "$TARGET_BRANCH"
git checkout -B "$TARGET_BRANCH" FETCH_HEAD
Expand All @@ -139,8 +136,10 @@ jobs:
- name: Compile the benchmark
working-directory: ./timefold-solver-benchmarks
shell: bash
env:
VERSION: ${{ needs.decisions.outputs.baseline_solver_version }}
run: |
./mvnw clean install -B -Dquickly -Dversion.ai.timefold.solver=${{ needs.decisions.outputs.baseline_solver_version }}
./mvnw clean install -B -Dquickly -Dversion.ai.timefold.solver=$VERSION
mv target/benchmarks.jar ../benchmarks-baseline.jar

- name: Upload the binaries
Expand All @@ -158,9 +157,6 @@ jobs:
matrix:
# When updating this list, use find-and-replace in the entire file to keep all such lists identical.
example: [cloud_balancing, conference_scheduling, curriculum_course, examination, flow_shop, machine_reassignment, meeting_scheduling, nurse_rostering, patient_admission_scheduling, task_assigning, traveling_tournament, vehicle_routing]
env:
MVN_USERNAME: '${{ secrets.JFROG_ENTERPRISE_READ_ONLY_ACCESS_USERNAME }}'
MVN_PASSWORD: '${{ secrets.JFROG_ENTERPRISE_READ_ONLY_ACCESS_TOKEN }}'
steps:
- name: Checkout timefold-solver-benchmarks
uses: actions/checkout@v6
Expand All @@ -175,9 +171,6 @@ jobs:
java-version: 25 # Always build with the least recent supported JDK.
distribution: 'temurin'
cache: 'maven'
server-id: 'timefold-solver-enterprise'
server-username: 'MVN_USERNAME'
server-password: 'MVN_PASSWORD'

- name: Checkout timefold-solver
uses: actions/checkout@v6
Expand All @@ -201,8 +194,9 @@ jobs:
- name: Switch to correct Enterprise branch if it exists
working-directory: ./timefold-solver-enterprise
shell: bash
env:
TARGET_BRANCH: ${{ github.event.inputs.branch }}
run: |
TARGET_BRANCH="${{ github.event.inputs.branch }}"
if git ls-remote --exit-code --heads origin "$TARGET_BRANCH" > /dev/null; then
git fetch --depth=1 origin "$TARGET_BRANCH"
git checkout -B "$TARGET_BRANCH" FETCH_HEAD
Expand All @@ -218,8 +212,9 @@ jobs:
- name: Switch to correct Benchmarks branch if it exists
working-directory: ./timefold-solver-benchmarks
shell: bash
env:
TARGET_BRANCH: ${{ github.event.inputs.branch }}
run: |
TARGET_BRANCH="${{ github.event.inputs.branch }}"
if git ls-remote --exit-code --heads origin "$TARGET_BRANCH" > /dev/null; then
git fetch --depth=1 origin "$TARGET_BRANCH"
git checkout -B "$TARGET_BRANCH" FETCH_HEAD
Expand Down Expand Up @@ -251,14 +246,22 @@ jobs:
steps:
- name: Clean results of previous runs
shell: bash
env:
BASELINE: ${{ github.event.inputs.baseline }}
BRANCH: ${{ github.event.inputs.branch }}
run: |
rm -rf timefold-solver-benchmarks
# GitHub expressions can't do this, so we need to hack this in shell.
# DIRs are different, so that we can run "main" against "main" and have separate results.
echo "SANITIZED_BASELINE=$(echo "${{ github.event.inputs.baseline }}" | sed 's/\//\-/g')" >> $GITHUB_ENV
echo "BASELINE_DIR=baseline_$(echo "${{ github.event.inputs.baseline }}" | sed 's/\//\-/g')" >> $GITHUB_ENV
echo "SANITIZED_BRANCH=$(echo "${{ github.event.inputs.branch }}" | sed 's/\//\-/g')" >> $GITHUB_ENV
echo "SUT_DIR=sut_$(echo "${{ github.event.inputs.branch }}" | sed 's/\//\-/g')" >> $GITHUB_ENV
# Strip CR/LF from inputs before writing to $GITHUB_ENV to prevent env-file injection.
SANITIZED_BASELINE=$(echo "$BASELINE" | tr -d '\r\n' | sed 's/\//\-/g')
SANITIZED_BRANCH=$(echo "$BRANCH" | tr -d '\r\n' | sed 's/\//\-/g')
{
echo "SANITIZED_BASELINE=$SANITIZED_BASELINE"
echo "BASELINE_DIR=baseline_$SANITIZED_BASELINE"
echo "SANITIZED_BRANCH=$SANITIZED_BRANCH"
echo "SUT_DIR=sut_$SANITIZED_BRANCH"
} >> "$GITHUB_ENV"

- name: Checkout timefold-solver-benchmarks
uses: actions/checkout@v6
Expand All @@ -270,13 +273,15 @@ jobs:
- name: Configure the benchmark
working-directory: ./timefold-solver-benchmarks
shell: bash
env:
EXAMPLE: ${{ matrix.example }}
run: |
echo "forks=20" > scoredirector-benchmark.properties
echo "warmup_iterations=5" >> scoredirector-benchmark.properties
echo "measurement_iterations=5" >> scoredirector-benchmark.properties
echo "relative_score_error_threshold=0.02" >> scoredirector-benchmark.properties
echo "score_director_type=cs" >> scoredirector-benchmark.properties
echo "example=${{ matrix.example }}" >> scoredirector-benchmark.properties
echo "example=$EXAMPLE" >> scoredirector-benchmark.properties
cat scoredirector-benchmark.properties
chmod +x run-scoredirector.sh

Expand Down Expand Up @@ -309,9 +314,9 @@ jobs:
mkdir target
cp benchmarks-baseline.jar target/benchmarks.jar
./run-scoredirector.sh
echo "RANGE_START=$(jq '.[0].primaryMetric.scoreConfidence[0]|round' results/scoredirector/${{ env.RUN_ID }}/results.json)" >> "$GITHUB_OUTPUT"
echo "RANGE_END=$(jq '.[0].primaryMetric.scoreConfidence[1]|round' results/scoredirector/${{ env.RUN_ID }}/results.json)" >> "$GITHUB_OUTPUT"
echo "RANGE_MID=$(jq '.[0].primaryMetric.score|round' results/scoredirector/${{ env.RUN_ID }}/results.json)" >> "$GITHUB_OUTPUT"
echo "RANGE_START=$(jq '.[0].primaryMetric.scoreConfidence[0]|round' results/scoredirector/$RUN_ID/results.json)" >> "$GITHUB_OUTPUT"
echo "RANGE_END=$(jq '.[0].primaryMetric.scoreConfidence[1]|round' results/scoredirector/$RUN_ID/results.json)" >> "$GITHUB_OUTPUT"
echo "RANGE_MID=$(jq '.[0].primaryMetric.score|round' results/scoredirector/$RUN_ID/results.json)" >> "$GITHUB_OUTPUT"

- name: (SUT) Setup JDK and Maven
uses: actions/setup-java@v5
Expand All @@ -331,28 +336,32 @@ jobs:
rm target/benchmarks.jar
cp benchmarks-sut.jar target/benchmarks.jar
./run-scoredirector.sh
echo "RANGE_START=$(jq '.[0].primaryMetric.scoreConfidence[0]|round' results/scoredirector/${{ env.RUN_ID }}/results.json)" >> "$GITHUB_OUTPUT"
echo "RANGE_END=$(jq '.[0].primaryMetric.scoreConfidence[1]|round' results/scoredirector/${{ env.RUN_ID }}/results.json)" >> "$GITHUB_OUTPUT"
echo "RANGE_MID=$(jq '.[0].primaryMetric.score|round' results/scoredirector/${{ env.RUN_ID }}/results.json)" >> "$GITHUB_OUTPUT"
echo "RANGE_START=$(jq '.[0].primaryMetric.scoreConfidence[0]|round' results/scoredirector/$RUN_ID/results.json)" >> "$GITHUB_OUTPUT"
echo "RANGE_END=$(jq '.[0].primaryMetric.scoreConfidence[1]|round' results/scoredirector/$RUN_ID/results.json)" >> "$GITHUB_OUTPUT"
echo "RANGE_MID=$(jq '.[0].primaryMetric.score|round' results/scoredirector/$RUN_ID/results.json)" >> "$GITHUB_OUTPUT"

- name: Clean up the data
working-directory: ./timefold-solver-benchmarks
shell: bash
env:
BASELINE_DIR: ${{ env.BASELINE_DIR }}
SUT_DIR: ${{ env.SUT_DIR }}
EXAMPLE: ${{ matrix.example }}
run: |
cd results/scoredirector/${{ env.BASELINE_DIR }}
mv $(find . -maxdepth 1 -mindepth 1 -type d -print -quit) ../../../${{ env.BASELINE_DIR }}
cd results/scoredirector/$BASELINE_DIR
mv $(find . -maxdepth 1 -mindepth 1 -type d -print -quit) ../../../$BASELINE_DIR
cd ../../../
mv ${{ env.BASELINE_DIR }}/jfr-cpu.jfr ${{ env.BASELINE_DIR }}/${{ matrix.example }}-${{ env.BASELINE_DIR }}.jfr
mv $BASELINE_DIR/jfr-cpu.jfr $BASELINE_DIR/$EXAMPLE-$BASELINE_DIR.jfr

cd results/scoredirector/${{ env.SUT_DIR }}
mv $(find . -maxdepth 1 -mindepth 1 -type d -print -quit) ../../../${{ env.SUT_DIR }}
cd results/scoredirector/$SUT_DIR
mv $(find . -maxdepth 1 -mindepth 1 -type d -print -quit) ../../../$SUT_DIR
cd ../../../
mv ${{ env.SUT_DIR }}/jfr-cpu.jfr ${{ env.SUT_DIR }}/${{ matrix.example }}-${{ env.SUT_DIR }}.jfr
mv $SUT_DIR/jfr-cpu.jfr $SUT_DIR/$EXAMPLE-$SUT_DIR.jfr

- name: Archive benchmark data
uses: actions/upload-artifact@v7
with:
name: assets-${{ matrix.example }}-${{ env.SANITIZED_BASELINE }}_vs_${{ env.SANITIZED_BRANCH }}
name: assets-${{ matrix.example }}-${{ env.SANITIZED_BASELINE }}-vs-${{ env.SANITIZED_BRANCH }}
path: |
./timefold-solver-benchmarks/scoredirector-benchmark.properties
./timefold-solver-benchmarks/${{ env.BASELINE_DIR }}/*.jfr
Expand All @@ -365,6 +374,9 @@ jobs:
- name: Report results
working-directory: ./timefold-solver-benchmarks
env:
BASELINE: ${{ github.event.inputs.baseline }}
BRANCH: ${{ github.event.inputs.branch }}
OWNER: ${{ github.event.inputs.branch_owner }}
BASELINE_RANGE_START: ${{ steps.benchmark_baseline.outputs.RANGE_START }}
BASELINE_RANGE_MID: ${{ steps.benchmark_baseline.outputs.RANGE_MID }}
BASELINE_RANGE_END: ${{ steps.benchmark_baseline.outputs.RANGE_END }}
Expand All @@ -391,17 +403,17 @@ jobs:
FAIL=true
fi

if [[ "${{ github.event.inputs.baseline }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
BASELINE_URL="https://github.com/TimefoldAI/timefold-solver/releases/tag/${{ github.event.inputs.baseline }}"
if [[ "$BASELINE" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
BASELINE_URL="https://github.com/TimefoldAI/timefold-solver/releases/tag/$BASELINE"
else
BASELINE_URL="https://github.com/TimefoldAI/timefold-solver/tree/${{ github.event.inputs.baseline }}"
BASELINE_URL="https://github.com/TimefoldAI/timefold-solver/tree/$BASELINE"
fi
SUT_URL="https://github.com/${{ github.event.inputs.branch_owner }}/timefold-solver/tree/${{ github.event.inputs.branch }}"
SUT_URL="https://github.com/$OWNER/timefold-solver/tree/$BRANCH"

echo "| | **Ref** | **Mean** |" >> $GITHUB_STEP_SUMMARY
echo "|:------:|:-----------:|:-----------------:|" >> $GITHUB_STEP_SUMMARY
echo "| _Old_ | [TimefoldAI's ${{ github.event.inputs.baseline }}]($BASELINE_URL) | $BASELINE_RANGE_MID ± $BASELINE_DEV % |" >> $GITHUB_STEP_SUMMARY
echo "| _New_ | [${{ github.event.inputs.branch_owner }}'s ${{ github.event.inputs.branch }}]($SUT_URL) | $SUT_RANGE_MID ± $SUT_DEV % |" >> $GITHUB_STEP_SUMMARY
echo "| _Old_ | [TimefoldAI's $BASELINE]($BASELINE_URL) | $BASELINE_RANGE_MID ± $BASELINE_DEV % |" >> $GITHUB_STEP_SUMMARY
echo "| _New_ | [$OWNER's $BRANCH]($SUT_URL) | $SUT_RANGE_MID ± $SUT_DEV % |" >> $GITHUB_STEP_SUMMARY
echo "| _Diff_ | | $DIFF_MID % |" >> $GITHUB_STEP_SUMMARY

echo "" >> $GITHUB_STEP_SUMMARY
Expand Down
Loading