chore: Add a benchmark action to test various branches of quickstarts

.github/workflows/benchmark.yml

@@ -0,0 +1,41 @@
+name: Run bash.sh
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Version to benchmark (1.x, 2.x, or split)"
+        required: true
+        default: "1.x"
+        type: choice
+        options:
+          - 1.x
+          - 2.x
+          - split
+
+jobs:
+  all:
+    runs-on: self-hosted
+    env:
+      SKIP_CLONE: true
+    steps:
+      - uses: actions/checkout@v4
+      - name: Checkout timefold-solver
+        uses: actions/checkout@v4
+        with:
+          repository: Christopher-Chianelli/timefold-solver
+          ref: ${{ github.event.inputs.version == 'split' && 'perf/split-random' || (github.event.inputs.version == '2.x' && 'main' || '1.x') }}
+          path: timefold-solver
+      - name: Checkout timefold-solver-enterprise
+        uses: actions/checkout@v4
+        with:
+          repository: TimefoldAI/timefold-solver-enterprise
+          ref: ${{ github.event.inputs.version == 'split' && 'perf/split-random' || (github.event.inputs.version == '2.x' && 'main' || '1.x') }}
+          path: timefold-solver-enterprise
+      - name: Checkout timefold-quickstarts
+        uses: actions/checkout@v4
+        with:
+          repository: TimefoldAI/timefold-quickstarts
+          ref: ${{ github.event.inputs.version == 'split' && 'development' || (github.event.inputs.version == '2.x' && 'development' || 'development-1.x') }}
+          path: timefold-quickstarts
+      - run: bash ./benchmark.sh ${{ inputs.version }}

benchmark.sh

@@ -0,0 +1,179 @@
+#!/bin/bash
+BASE_DIR=$(pwd)
+SOLVE_TIME=2m
+#TIMEOUT_SECONDS=100
+
+declare -A DEMO_DATA
+declare -A SOLVE_ENDPOINT
+
+SOLVE_ENDPOINT[school-timetabling]="timetables"
+DEMO_DATA[school-timetabling]="/SMALL"
+
+SOLVE_ENDPOINT[bed-allocation]="schedules"
+DEMO_DATA[bed-allocation]=""
+
+SOLVE_ENDPOINT[conference-scheduling]="schedules"
+DEMO_DATA[conference-scheduling]=""
+
+SOLVE_ENDPOINT[employee-scheduling]="schedules"
+DEMO_DATA[employee-scheduling]="/LARGE"
+
+#SOLVE_ENDPOINT[facility-location]="flp/solve"
+#DEMO_DATA[facility-location]="MISSING"
+
+SOLVE_ENDPOINT[flight-crew-scheduling]="schedules"
+DEMO_DATA[flight-crew-scheduling]=""
+
+#SOLVE_ENDPOINT[food-packaging]="schedule"
+#DEMO_DATA[food-packaging]="MISSING"
+
+SOLVE_ENDPOINT[maintenance-scheduling]="schedules"
+DEMO_DATA[maintenance-scheduling]="/LARGE"
+
+SOLVE_ENDPOINT[meeting-scheduling]="schedules"
+DEMO_DATA[meeting-scheduling]=""
+
+#SOLVE_ENDPOINT[order-picking]="orderPicking"
+#DEMO_DATA[order-picking]="MISSING"
+
+SOLVE_ENDPOINT[project-job-scheduling]="schedules"
+DEMO_DATA[project-job-scheduling]=""
+
+SOLVE_ENDPOINT[sports-league-scheduling]="schedules"
+DEMO_DATA[sports-league-scheduling]=""
+
+SOLVE_ENDPOINT[task-assigning]="schedules"
+DEMO_DATA[task-assigning]=""
+
+SOLVE_ENDPOINT[tournament-scheduling]="schedules"
+DEMO_DATA[tournament-scheduling]=""
+
+SOLVE_ENDPOINT[vehicle-routing]="route-plans"
+DEMO_DATA[vehicle-routing]="/FIRENZE"
+
+RANDOM_SEEDS=(0 1 42)
+MOVE_THREAD_COUNTS=(NONE 2 4)
+
+quickstart_dir() {
+    echo "$BASE_DIR/timefold-quickstarts/java/$1"
+}
+
+quickstart_jar() {
+    echo "$(quickstart_dir $1)/target/quarkus-app/quarkus-run.jar"
+}
+
+run_quickstart() {
+  # Set $BASE_URL to override the default http://localhost:8080
+  cd $(quickstart_dir $1)
+  mvn -q clean package -Denterprise -DskipTests &> /dev/null
+  nohup java "-Dquarkus.timefold.solver.termination.spent-limit=${SOLVE_TIME}" "-Dquarkus.timefold.solver.random-seed=${RANDOM_SEED}" "-Dquarkus.timefold.solver.move-thread-count=${MOVE_THREAD_COUNT}" -jar $(quickstart_jar $1) &> /dev/null &
+  sleep 5
+  BASE_URL="${BASE_URL:-http://localhost:8080}"
+  # Step 1: GET /demo-data/SMALL
+  PROBLEM=$(curl -sf "$BASE_URL/demo-data/${DEMO_DATA[$1]}")
+
+  # Step 2: POST /$SOLVE_ENDPOINT to start solving
+  JOB_ID=$(echo "$PROBLEM" | curl -sf -X POST \
+    -H "Content-Type: application/json" \
+    --data @- \
+    "$BASE_URL/${SOLVE_ENDPOINT[$1]}")
+
+  # Step 3: Poll /$SOLVE_ENDPOINT/{jobId}/status until solverStatus == "NOT_SOLVING"
+  POLL_INTERVAL_MS=500
+  POLL_INTERVAL_SECONDS=0.5
+  START_TIME=$(date +%s)
+
+  while true; do
+    #CURRENT_TIME=$(date +%s)
+    #ELAPSED=$((CURRENT_TIME - START_TIME))
+    #if [ "$ELAPSED" -ge "$TIMEOUT_SECONDS" ]; then
+    #  kill $!
+    #  return 1
+    #fi
+    SOLVER_STATUS=$(curl -sf "$BASE_URL/${SOLVE_ENDPOINT[$1]}/$JOB_ID/status" | jq -r '.solverStatus')
+
+    if [ "$SOLVER_STATUS" = "NOT_SOLVING" ]; then
+      break
+    fi
+
+    sleep "$POLL_INTERVAL_SECONDS"
+  done
+
+  # Step 4: GET /$SOLVE_ENDPOINT/{jobId} to retrieve the solution
+  SOLUTION=$(curl -sf "$BASE_URL/${SOLVE_ENDPOINT[$1]}/$JOB_ID")
+
+  # Step 5: Assert solution is not null
+  if [ -z "$SOLUTION" ]; then
+    kill $!
+    return 1
+  fi
+
+  SCORE=$(echo "$SOLUTION" | jq -r '.score')
+  kill $!
+  echo "$SCORE"
+}
+
+case $1 in
+    1\.x)
+    COMMUNITY="1.x"
+    ENTERPRISE="1.x"
+    QUICKSTART="development-1.x"
+    ;;
+    2\.x)
+    COMMUNITY=main
+    ENTERPRISE=main
+    QUICKSTART=development
+    ;;
+    split)
+    COMMUNITY=perf/split-random
+    ENTERPRISE=perf/split-random
+    QUICKSTART=development
+    ;;
+    *)
+    echo "Usage: $0 1.x|2.x|split"
+    exit 1
+    ;;
+esac
+
+if [ -n "$USE_SSH" ]
+then
+    ENTERPRISE_URL=git@github.com:TimefoldAI/timefold-solver-enterprise.git
+else
+    ENTERPRISE_URL=https://github.com/TimefoldAI/timefold-solver-enterprise.git
+fi
+
+if [ -z "$SKIP_CLONE" ]
+then
+    rm -rf timefold-solver
+    rm -rf timefold-solver-enterprise
+    rm -rf timefold-quickstarts
+    git clone --depth 1 --branch $COMMUNITY https://github.com/Christopher-Chianelli/timefold-solver.git
+    git clone --depth 1 --branch $ENTERPRISE $ENTERPRISE_URL
+    git clone --depth 1 --branch $QUICKSTART https://github.com/TimefoldAI/timefold-quickstarts.git
+
+    echo "Building Solver and Solver Enterprise for branch $1"
+    cd timefold-solver
+    mvn -q clean install -Dquickly &> /dev/null
+    cd ..
+    cd timefold-solver-enterprise
+    mvn -q clean install -Dquickly &> /dev/null
+    cd ..
+fi
+
+echo "Quickstart,RandomSeed,MoveThreadCount,Score" > "$1".csv
+for quickstart in bed-allocation conference-scheduling employee-scheduling \
+                  flight-crew-scheduling maintenance-scheduling meeting-scheduling \
+                  project-job-scheduling school-timetabling sports-league-scheduling \
+                  task-assigning tournament-scheduling vehicle-routing
+do
+    for RANDOM_SEED in "${RANDOM_SEEDS[@]}"; do
+        for MOVE_THREAD_COUNT in "${MOVE_THREAD_COUNTS[@]}"; do
+            echo "Benchmarking Quickstart $quickstart (seed=$RANDOM_SEED, threads=$MOVE_THREAD_COUNT)"
+            echo "$quickstart,$RANDOM_SEED,$MOVE_THREAD_COUNT,$(run_quickstart $quickstart)" >> "$1".csv
+        done
+    done
+done
+
+echo "Finished Benchmarking for $1"
+echo "Results (stored in $1.csv)"
+cat "$1".csv