JSO Benchmarks

.github/workflows/jsobmarks.yml

@@ -0,0 +1,81 @@
+name: JSOBenchmarks
+on:
+  pull_request:
+    types: [labeled, opened, synchronize, reopened]
+  workflow_call:
+
+jobs:
+  bmark:
+    name: Julia ${{ matrix.version }} - macOS - ${{ matrix.arch }} - ${{ github.event_name }}
+    if: github.event_name == 'workflow_call' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run benchmarks'))
+    # FIXME: should run on hosted runner
+    runs-on: macOS-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - 1
+        arch:
+          - aarch64
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: ${{ matrix.version }}
+          arch: ${{ matrix.arch }}
+      - uses: julia-actions/julia-buildpkg@v1
+        # FIXME: register JSOBenchmarks
+      - name: Installing non-registered dependencies
+        run: |
+          using Pkg
+          pkg1 = PackageSpec(url = "https://github.com/JuliaSmoothOptimizers/JSOBenchmarks.jl.git", rev = "main")
+          pkg_list = [pkg1]
+          Pkg.add(pkg_list)
+        shell: julia --project=benchmark --color=yes {0}
+      - name: Install benchmark dependencies
+        run: julia --project=benchmark -e 'using Pkg; Pkg.instantiate()'
+      - name: Sanitize project name
+        id: sanitize
+        run: echo "REPONAME=${{ github.event.repository.name }}" | sed -e 's/\.jl$//' >> $GITHUB_OUTPUT
+      - name: Run benchmarks
+        run: julia --project=benchmark -e 'using JSOBenchmarks; run_benchmarks("${{ steps.sanitize.outputs.REPONAME }}", "benchmark", reference_branch = "main")'
+        env:
+          GITHUB_AUTH: ${{ secrets.GIST_TOKEN }}
+      # - name: Post benchmark results in PR
+      #   uses: thollander/actions-comment-pull-request@v2
+      #   with:
+      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      #     message: "Full benchmark results stored as artifacts. Summary: ${{ secrets.BMARK_GIST_URL }}"
+      - name: Build comment
+        id: build-comment
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ github.token }}
+          result-encoding: string
+          script: |
+            const fs = require('fs');
+            return fs.readFileSync("${{ github.workspace }}/bmark_${{ github.sha }}.md", "utf8").toString();
+      - name: Comment in PR
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          message: ${{ steps.build-comment.outputs.result }}
+      # - name: Upload benchmark plots
+      #   uses: edunad/actions-image@v2.0.0
+      #   with:
+      #     path: "./*.png"
+      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      #     title: "Benchmarks at a Glance"
+      #     annotationLevel: "notice"
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v7
+        with:
+          name: jso-benchmarks
+          path: |
+            profiles_this_commit_vs_reference_*.svg
+            *_vs_reference_*.jld2
+            bmark_*.md
+            reference.md
+            judgement_*.md

benchmark/Project.toml

@@ -0,0 +1,14 @@
+[deps]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+JSOBenchmarks = "9410e2bb-e8e7-4fa0-9768-685b196f59b5"
+PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"
+RegularizedOptimization = "20620ad1-4fe4-4467-ae46-fb087718fe7b"
+RegularizedProblems = "ea076b23-609f-44d2-bb12-a4ae45328278"
+
+[compat]
+julia = "1"
+BenchmarkTools = "1"
+PkgBenchmark = "0.2"
+# JSOBenchmarks = "0.1"
+RegularizedOptimization = "0.2"
+RegularizedProblems = "0.1"

benchmark/benchmarks.jl

@@ -0,0 +1,16 @@
+using BenchmarkTools
+using RegularizedProblems, RegularizedOptimization
+
+const SUITE = BenchmarkGroup()
+
+bpdn_l0, _ = setup_bpdn_l0()
+bpdn_l1, _ = setup_bpdn_l1()
+bpdn_B0, _ = setup_bpdn_B0()
+
+for solver ∈ (R2,)
+  solver_name = string(solver)
+  SUITE[solver_name] = BenchmarkGroup()
+  SUITE[solver_name]["bpdn_l0"] = @benchmarkable $solver($bpdn_l0)
+  SUITE[solver_name]["bpdn_l1"] = @benchmarkable $solver($bpdn_l1)
+  SUITE[solver_name]["bpdn_B0"] = @benchmarkable $solver($bpdn_B0)
+end