diff --git a/.github/workflows/nightly_performance_score_director.yml b/.github/workflows/nightly_performance.yml similarity index 80% rename from .github/workflows/nightly_performance_score_director.yml rename to .github/workflows/nightly_performance.yml index 08f59d7d..ef0a9c50 100644 --- a/.github/workflows/nightly_performance_score_director.yml +++ b/.github/workflows/nightly_performance.yml @@ -27,7 +27,10 @@ jobs: if git log --since="24 hours ago" --oneline | grep -q .; then cd ../timefold-solver-benchmarks echo '{}' | gh workflow run performance_score_director.yml --json - echo "Launched nightly perf tests." >> $GITHUB_STEP_SUMMARY + echo "Launched nightly score director perf tests." >> $GITHUB_STEP_SUMMARY + sleep 1m + echo '{}' | gh workflow run performance_quickstarts.yml --json + echo "Queued nightly quickstart perf tests." >> $GITHUB_STEP_SUMMARY else # Don't waste money. echo "No commits in the past 24 hours." >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/performance_quickstarts.yml b/.github/workflows/performance_quickstarts.yml index 8bf5a75a..1ed6b7ed 100644 --- a/.github/workflows/performance_quickstarts.yml +++ b/.github/workflows/performance_quickstarts.yml @@ -59,12 +59,12 @@ jobs: # When updating this list, use find-and-replace in the entire file to keep all such lists identical. example: [bed-allocation, conference-scheduling, employee-scheduling, facility-location, flight-crew-scheduling, food-packaging, maintenance-scheduling, meeting-scheduling, order-picking, project-job-scheduling, school-timetabling, sports-league-scheduling, task-assigning, tournament-scheduling, vehicle-routing] steps: - - name: Checkout timefold-solver-quickstarts + - name: Checkout timefold-quickstarts uses: actions/checkout@v4 with: - repository: TimefoldAI/timefold-solver-quickstarts - path: ./timefold-solver-quickstarts - ref: main + repository: TimefoldAI/timefold-quickstarts + path: ./timefold-quickstarts + ref: development - name: Setup JDK and Maven uses: actions/setup-java@v5 @@ -85,8 +85,21 @@ jobs: shell: bash run: ./mvnw -B -Dquickly clean install + - name: Checkout timefold-solver-enterprise + uses: actions/checkout@v4 + with: + repository: TimefoldAI/timefold-solver-enterprise + ref: ${{ github.event.inputs.baseline }} + token: ${{ secrets.BENCHMARK_PUBLISH_TOKEN }} + path: ./timefold-solver-enterprise + + - name: Quickly build timefold-solver-enterprise + working-directory: ./timefold-solver-enterprise + shell: bash + run: ./mvnw -B -Dquickly clean install + - name: Switch quickstarts to baseline branch if it exists - working-directory: ./timefold-solver-quickstarts + working-directory: ./timefold-quickstarts shell: bash env: TARGET_BRANCH: ${{ github.event.inputs.baseline }} @@ -100,15 +113,15 @@ jobs: - name: Build the quickstart shell: bash run: | - mvn -B -DskipTests package \ - -f timefold-solver-quickstarts/java/${{ matrix.example }}/pom.xml + mvn -B -DskipTests -Penterprise package \ + -f timefold-quickstarts/java/${{ matrix.example }}/pom.xml - name: Upload the binaries uses: actions/upload-artifact@v4 with: name: baseline-${{ matrix.example }} path: | - ./timefold-solver-quickstarts/java/${{ matrix.example }}/target/quarkus-app/ + ./timefold-quickstarts/java/${{ matrix.example }}/target/quarkus-app/ if-no-files-found: error build_sut: @@ -119,12 +132,12 @@ jobs: # When updating this list, use find-and-replace in the entire file to keep all such lists identical. example: [bed-allocation, conference-scheduling, employee-scheduling, facility-location, flight-crew-scheduling, food-packaging, maintenance-scheduling, meeting-scheduling, order-picking, project-job-scheduling, school-timetabling, sports-league-scheduling, task-assigning, tournament-scheduling, vehicle-routing] steps: - - name: Checkout timefold-solver-quickstarts + - name: Checkout timefold-quickstarts uses: actions/checkout@v4 with: - repository: TimefoldAI/timefold-solver-quickstarts - path: ./timefold-solver-quickstarts - ref: main + repository: TimefoldAI/timefold-quickstarts + path: ./timefold-quickstarts + ref: development - name: Setup JDK and Maven uses: actions/setup-java@v5 @@ -145,8 +158,33 @@ jobs: shell: bash run: ./mvnw -B -Dquickly clean install + - name: Checkout timefold-solver-enterprise + uses: actions/checkout@v4 + with: + repository: TimefoldAI/timefold-solver-enterprise + ref: main + token: ${{ secrets.BENCHMARK_PUBLISH_TOKEN }} + path: ./timefold-solver-enterprise + + - name: Switch timefold-solver-enterprise to branch under test if it exists + working-directory: ./timefold-solver-enterprise + shell: bash + env: + TARGET_BRANCH: ${{ github.event.inputs.branch }} + run: | + if git ls-remote --exit-code --heads origin "$TARGET_BRANCH" > /dev/null; then + git fetch --depth=1 origin "$TARGET_BRANCH" + git checkout -B "$TARGET_BRANCH" FETCH_HEAD + fi + git status + + - name: Quickly build timefold-solver-enterprise + working-directory: ./timefold-solver-enterprise + shell: bash + run: ./mvnw -B -Dquickly clean install + - name: Switch quickstarts to branch under test if it exists - working-directory: ./timefold-solver-quickstarts + working-directory: ./timefold-quickstarts shell: bash env: TARGET_BRANCH: ${{ github.event.inputs.branch }} @@ -160,15 +198,15 @@ jobs: - name: Build the quickstart shell: bash run: | - mvn -B -DskipTests package \ - -f timefold-solver-quickstarts/java/${{ matrix.example }}/pom.xml + mvn -B -DskipTests -Penterprise package \ + -f timefold-quickstarts/java/${{ matrix.example }}/pom.xml - name: Upload the binaries uses: actions/upload-artifact@v4 with: name: sut-${{ matrix.example }} path: | - ./timefold-solver-quickstarts/java/${{ matrix.example }}/target/quarkus-app/ + ./timefold-quickstarts/java/${{ matrix.example }}/target/quarkus-app/ if-no-files-found: error benchmark: @@ -228,6 +266,8 @@ jobs: - name: (Baseline) Run the benchmark id: benchmark_baseline shell: bash + env: + TIMEFOLD_ENTERPRISE_LICENSE: ${{ secrets.TIMEFOLD_SOLVER_CI_PROD_LICENSE }} run: | python3 timefold-solver-benchmarks/benchmark-quickstarts.py \ ${{ matrix.example }} ./baseline-app \ @@ -249,6 +289,8 @@ jobs: - name: (SUT) Run the benchmark id: benchmark_sut shell: bash + env: + TIMEFOLD_ENTERPRISE_LICENSE: ${{ secrets.TIMEFOLD_SOLVER_CI_PROD_LICENSE }} run: | python3 timefold-solver-benchmarks/benchmark-quickstarts.py \ ${{ matrix.example }} ./sut-app \ diff --git a/benchmark-quickstarts.py b/benchmark-quickstarts.py index d000475e..0deafef9 100644 --- a/benchmark-quickstarts.py +++ b/benchmark-quickstarts.py @@ -108,6 +108,7 @@ def _run_once( "java", f"-Dquarkus.http.port={port}", f"-Dquarkus.timefold.solver.termination.spent-limit={time_limit}s", + "-Dquarkus.timefold.solver.move-thread-count=NONE", "-Dquarkus.log.category.ai.timefold.solver.level=INFO", "-Dquarkus.console.color=false", "-jar", str(jar),