From 1430b4d48264db3d84a7b887c88743d9f228cc3c Mon Sep 17 00:00:00 2001 From: Dominique Date: Tue, 3 Mar 2026 15:58:35 -0500 Subject: [PATCH 01/12] begin integrating JSOBenchmarks --- .github/workflows/jsobmarks.yml | 80 +++++++++++++++++++++++++++++++++ benchmarks/Project.toml | 6 +++ benchmarks/benchmarks.jl | 15 +++++++ 3 files changed, 101 insertions(+) create mode 100644 .github/workflows/jsobmarks.yml create mode 100644 benchmarks/Project.toml create mode 100644 benchmarks/benchmarks.jl diff --git a/.github/workflows/jsobmarks.yml b/.github/workflows/jsobmarks.yml new file mode 100644 index 00000000..206a6927 --- /dev/null +++ b/.github/workflows/jsobmarks.yml @@ -0,0 +1,80 @@ +name: JSOBenchmarks +on: + pull_request_target: + types: + - labeled +jobs: + bmark: + name: Julia ${{ matrix.version }} - macOS - ${{ matrix.arch }} - ${{ github.event_name }} + if: contains(github.event.pull_request.labels.*.name, 'benchmarks') + # FIXME: should run on hosted runner + runs-on: macOS-latest + strategy: + fail-fast: false + matrix: + version: + - 1 + arch: + - aarch64 + steps: + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v2 + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + - uses: julia-actions/julia-buildpkg@v1 + # FIXME: register JSOBenchmarks + - name: Installing non-registered dependencies + run: | + using Pkg + pkg1 = PackageSpec(url = "https://github.com/JuliaSmoothOptimizers/JSOBenchmarks.jl.git", rev = "main") + pkg_list = [pkg1] + Pkg.add(pkg_list) + shell: julia --project=benchmark --color=yes {0} + - name: Install benchmark dependencies + run: julia --project=benchmark -e 'using Pkg; Pkg.instantiate()' + - name: Sanitize project name + id: sanitize + run: echo "REPONAME=${{ github.event.repository.name }}" | sed -e 's/\.jl$//' >> $GITHUB_OUTPUT + - name: Run benchmarks + # FIXME: reference branch should be main eventually + run: julia --project=benchmark -e 'using JSOBenchmarks; run_benchmarks("${{ steps.sanitize.outputs.REPONAME }}", "benchmark", reference_branch = "jsobmarks")' + env: + GITHUB_AUTH: ${{ secrets.GIST_TOKEN }} + # - name: Post benchmark results in PR + # uses: thollander/actions-comment-pull-request@v2 + # with: + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # message: "Full benchmark results stored as artifacts. Summary: ${{ secrets.BMARK_GIST_URL }}" + - name: Build comment + id: build-comment + uses: actions/github-script@v6 + with: + github-token: ${{ github.token }} + result-encoding: string + script: | + const fs = require('fs'); + return fs.readFileSync("${{ github.workspace }}/${{ github.sha }}.md", "utf8").toString(); + - name: Comment in PR + uses: thollander/actions-comment-pull-request@v2 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + message: ${{ steps.build-comment.outputs.result }} + - name: Upload benchmark plots + uses: edunad/actions-image@v2.0.0 + with: + path: "./*.png" + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + title: "Benchmarks at a Glance" + annotationLevel: "notice" + - name: Upload artifacts + uses: actions/upload-artifact@v7 + with: + name: jso-benchmarks + path: | + profiles_this_commit_vs_refererence_*.svg + *_vs_reference_*.jld2 + bmark_*.md + this_commit.md + reference.md + judgement.md diff --git a/benchmarks/Project.toml b/benchmarks/Project.toml new file mode 100644 index 00000000..fdaf07ee --- /dev/null +++ b/benchmarks/Project.toml @@ -0,0 +1,6 @@ +[deps] +BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +JSOBenchmarks = "9410e2bb-e8e7-4fa0-9768-685b196f59b5" +PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d" +RegularizedOptimization = "20620ad1-4fe4-4467-ae46-fb087718fe7b" +RegularizedProblems = "ea076b23-609f-44d2-bb12-a4ae45328278" diff --git a/benchmarks/benchmarks.jl b/benchmarks/benchmarks.jl new file mode 100644 index 00000000..7759862b --- /dev/null +++ b/benchmarks/benchmarks.jl @@ -0,0 +1,15 @@ +using BenchmarkTools +using RegularizedProblems, RegularizedOptimization + +const SUITE = BenchmarkGroup() + +for solver ∈ (R2,) + solver_name = string(solver) + SUITE[solver_name] = BenchmarkGroup() + bpdn_l0, _ = setup_bpdn_l0() + SUITE[solver_name]["bpdn_l0"] = @benchmarkable $solver($bpdn_l0) + bpdn_l1, _ = setup_bpdn_l1() + SUITE[solver_name]["bpdn_l1"] = @benchmarkable $solver($bpdn_l1) + bpdn_B0, _ = setup_bpdn_lB() + SUITE[solver_name]["bpdn_B0"] = @benchmarkable $solver($bpdn_B0) +end From a775bad1490ec12f144211c3fb48ff05dab82cfc Mon Sep 17 00:00:00 2001 From: Dominique Date: Tue, 3 Mar 2026 18:08:56 -0500 Subject: [PATCH 02/12] Change trigger event for JSOBenchmarks --- .github/workflows/jsobmarks.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/jsobmarks.yml b/.github/workflows/jsobmarks.yml index 206a6927..60ea89df 100644 --- a/.github/workflows/jsobmarks.yml +++ b/.github/workflows/jsobmarks.yml @@ -1,8 +1,9 @@ name: JSOBenchmarks on: - pull_request_target: - types: - - labeled + pull_request: + types: [labeled, opened, synchronize, reopened] + workflow_call: + jobs: bmark: name: Julia ${{ matrix.version }} - macOS - ${{ matrix.arch }} - ${{ github.event_name }} From c0d953c837b6be6dc460fc69dca5d66f381537b6 Mon Sep 17 00:00:00 2001 From: Dominique Date: Tue, 3 Mar 2026 18:23:35 -0500 Subject: [PATCH 03/12] Use dir name benchmark (no s) --- {benchmarks => benchmark}/Project.toml | 0 {benchmarks => benchmark}/benchmarks.jl | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {benchmarks => benchmark}/Project.toml (100%) rename {benchmarks => benchmark}/benchmarks.jl (100%) diff --git a/benchmarks/Project.toml b/benchmark/Project.toml similarity index 100% rename from benchmarks/Project.toml rename to benchmark/Project.toml diff --git a/benchmarks/benchmarks.jl b/benchmark/benchmarks.jl similarity index 100% rename from benchmarks/benchmarks.jl rename to benchmark/benchmarks.jl From 7f4e0cdd66dad936c317be2317755c160a799b5e Mon Sep 17 00:00:00 2001 From: Dominique Date: Tue, 3 Mar 2026 18:32:47 -0500 Subject: [PATCH 04/12] Fix typo in problem constructor --- benchmark/benchmarks.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 7759862b..59dd6dc2 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -10,6 +10,6 @@ for solver ∈ (R2,) SUITE[solver_name]["bpdn_l0"] = @benchmarkable $solver($bpdn_l0) bpdn_l1, _ = setup_bpdn_l1() SUITE[solver_name]["bpdn_l1"] = @benchmarkable $solver($bpdn_l1) - bpdn_B0, _ = setup_bpdn_lB() + bpdn_B0, _ = setup_bpdn_B0() SUITE[solver_name]["bpdn_B0"] = @benchmarkable $solver($bpdn_B0) end From 2b26744f2b531bdd2ef00c1f84088c88a25ef375 Mon Sep 17 00:00:00 2001 From: Dominique Date: Tue, 3 Mar 2026 18:45:03 -0500 Subject: [PATCH 05/12] Fetch all branches in benchmark workflow --- .github/workflows/jsobmarks.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/jsobmarks.yml b/.github/workflows/jsobmarks.yml index 60ea89df..dd951322 100644 --- a/.github/workflows/jsobmarks.yml +++ b/.github/workflows/jsobmarks.yml @@ -18,7 +18,9 @@ jobs: arch: - aarch64 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 + with: + fetch_depth: 0 - uses: julia-actions/setup-julia@v2 with: version: ${{ matrix.version }} From e9676309eaab7820fe4c2e02a5379a4035b1220f Mon Sep 17 00:00:00 2001 From: Dominique Date: Tue, 3 Mar 2026 19:31:03 -0500 Subject: [PATCH 06/12] Fix typo in workflow --- .github/workflows/jsobmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jsobmarks.yml b/.github/workflows/jsobmarks.yml index dd951322..84ea0087 100644 --- a/.github/workflows/jsobmarks.yml +++ b/.github/workflows/jsobmarks.yml @@ -20,7 +20,7 @@ jobs: steps: - uses: actions/checkout@v6 with: - fetch_depth: 0 + fetch-depth: 0 - uses: julia-actions/setup-julia@v2 with: version: ${{ matrix.version }} From 3598dfab64786e2e38987518e4e2ad8da95cc857 Mon Sep 17 00:00:00 2001 From: Dominique Date: Tue, 3 Mar 2026 19:44:37 -0500 Subject: [PATCH 07/12] Debug branch switching issue --- .github/workflows/jsobmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jsobmarks.yml b/.github/workflows/jsobmarks.yml index 84ea0087..fe34456c 100644 --- a/.github/workflows/jsobmarks.yml +++ b/.github/workflows/jsobmarks.yml @@ -41,7 +41,7 @@ jobs: run: echo "REPONAME=${{ github.event.repository.name }}" | sed -e 's/\.jl$//' >> $GITHUB_OUTPUT - name: Run benchmarks # FIXME: reference branch should be main eventually - run: julia --project=benchmark -e 'using JSOBenchmarks; run_benchmarks("${{ steps.sanitize.outputs.REPONAME }}", "benchmark", reference_branch = "jsobmarks")' + run: julia --project=benchmark -e 'using JSOBenchmarks; run_benchmarks("${{ steps.sanitize.outputs.REPONAME }}", "benchmark", reference_branch = "origin/jsobmarks")' env: GITHUB_AUTH: ${{ secrets.GIST_TOKEN }} # - name: Post benchmark results in PR From 7f6b36123430cab1c47c9d477120053765f6b8e2 Mon Sep 17 00:00:00 2001 From: Dominique Date: Wed, 4 Mar 2026 09:51:50 -0500 Subject: [PATCH 08/12] fixes in workflow --- .github/workflows/jsobmarks.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/jsobmarks.yml b/.github/workflows/jsobmarks.yml index fe34456c..66506882 100644 --- a/.github/workflows/jsobmarks.yml +++ b/.github/workflows/jsobmarks.yml @@ -7,7 +7,7 @@ on: jobs: bmark: name: Julia ${{ matrix.version }} - macOS - ${{ matrix.arch }} - ${{ github.event_name }} - if: contains(github.event.pull_request.labels.*.name, 'benchmarks') + if: github.event_name == 'workflow_call' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'benchmarks')) # FIXME: should run on hosted runner runs-on: macOS-latest strategy: @@ -40,7 +40,8 @@ jobs: id: sanitize run: echo "REPONAME=${{ github.event.repository.name }}" | sed -e 's/\.jl$//' >> $GITHUB_OUTPUT - name: Run benchmarks - # FIXME: reference branch should be main eventually + # FIXME: we use a reference branch named jsobmarks to debug + # FIXME: change reference branch to main when ready run: julia --project=benchmark -e 'using JSOBenchmarks; run_benchmarks("${{ steps.sanitize.outputs.REPONAME }}", "benchmark", reference_branch = "origin/jsobmarks")' env: GITHUB_AUTH: ${{ secrets.GIST_TOKEN }} @@ -57,7 +58,7 @@ jobs: result-encoding: string script: | const fs = require('fs'); - return fs.readFileSync("${{ github.workspace }}/${{ github.sha }}.md", "utf8").toString(); + return fs.readFileSync("${{ github.workspace }}/bmark_${{ github.sha }}.md", "utf8").toString(); - name: Comment in PR uses: thollander/actions-comment-pull-request@v2 with: @@ -75,9 +76,8 @@ jobs: with: name: jso-benchmarks path: | - profiles_this_commit_vs_refererence_*.svg + profiles_this_commit_vs_reference_*.svg *_vs_reference_*.jld2 bmark_*.md - this_commit.md reference.md - judgement.md + judgement_*.md From 4c759c039adb9e8dc2e3e56ede3f41e6cf0e9fd6 Mon Sep 17 00:00:00 2001 From: Dominique Date: Wed, 4 Mar 2026 10:45:34 -0500 Subject: [PATCH 09/12] comment out at-a-glance section --- .github/workflows/jsobmarks.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/jsobmarks.yml b/.github/workflows/jsobmarks.yml index 66506882..92912080 100644 --- a/.github/workflows/jsobmarks.yml +++ b/.github/workflows/jsobmarks.yml @@ -64,13 +64,13 @@ jobs: with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} message: ${{ steps.build-comment.outputs.result }} - - name: Upload benchmark plots - uses: edunad/actions-image@v2.0.0 - with: - path: "./*.png" - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - title: "Benchmarks at a Glance" - annotationLevel: "notice" + # - name: Upload benchmark plots + # uses: edunad/actions-image@v2.0.0 + # with: + # path: "./*.png" + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # title: "Benchmarks at a Glance" + # annotationLevel: "notice" - name: Upload artifacts uses: actions/upload-artifact@v7 with: From b2dc0cf440bf12a09ae884469a9940cd538ec0d8 Mon Sep 17 00:00:00 2001 From: Dominique Date: Wed, 4 Mar 2026 10:48:34 -0500 Subject: [PATCH 10/12] use main as reference branch --- .github/workflows/jsobmarks.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/jsobmarks.yml b/.github/workflows/jsobmarks.yml index 92912080..590d2f28 100644 --- a/.github/workflows/jsobmarks.yml +++ b/.github/workflows/jsobmarks.yml @@ -40,9 +40,7 @@ jobs: id: sanitize run: echo "REPONAME=${{ github.event.repository.name }}" | sed -e 's/\.jl$//' >> $GITHUB_OUTPUT - name: Run benchmarks - # FIXME: we use a reference branch named jsobmarks to debug - # FIXME: change reference branch to main when ready - run: julia --project=benchmark -e 'using JSOBenchmarks; run_benchmarks("${{ steps.sanitize.outputs.REPONAME }}", "benchmark", reference_branch = "origin/jsobmarks")' + run: julia --project=benchmark -e 'using JSOBenchmarks; run_benchmarks("${{ steps.sanitize.outputs.REPONAME }}", "benchmark", reference_branch = "main")' env: GITHUB_AUTH: ${{ secrets.GIST_TOKEN }} # - name: Post benchmark results in PR From 9e82ba3ed310194dc81b99e5c64cb096586ed1a5 Mon Sep 17 00:00:00 2001 From: Dominique Date: Wed, 4 Mar 2026 16:29:49 -0500 Subject: [PATCH 11/12] Add compat entries, define problems outside loop --- benchmark/Project.toml | 8 ++++++++ benchmark/benchmarks.jl | 7 ++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/benchmark/Project.toml b/benchmark/Project.toml index fdaf07ee..200ae487 100644 --- a/benchmark/Project.toml +++ b/benchmark/Project.toml @@ -4,3 +4,11 @@ JSOBenchmarks = "9410e2bb-e8e7-4fa0-9768-685b196f59b5" PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d" RegularizedOptimization = "20620ad1-4fe4-4467-ae46-fb087718fe7b" RegularizedProblems = "ea076b23-609f-44d2-bb12-a4ae45328278" + +[compat] +julia = "1" +BenchmarkTools = "1" +PkgBenchmark = "0.2" +# JSOBenchmarks = "0.1" +RegularizedOptimization = "0.2" +RegularizedProblems = "0.1" diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 59dd6dc2..d8d80d17 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -3,13 +3,14 @@ using RegularizedProblems, RegularizedOptimization const SUITE = BenchmarkGroup() +bpdn_l0, _ = setup_bpdn_l0() +bpdn_l1, _ = setup_bpdn_l1() +bpdn_B0, _ = setup_bpdn_B0() + for solver ∈ (R2,) solver_name = string(solver) SUITE[solver_name] = BenchmarkGroup() - bpdn_l0, _ = setup_bpdn_l0() SUITE[solver_name]["bpdn_l0"] = @benchmarkable $solver($bpdn_l0) - bpdn_l1, _ = setup_bpdn_l1() SUITE[solver_name]["bpdn_l1"] = @benchmarkable $solver($bpdn_l1) - bpdn_B0, _ = setup_bpdn_B0() SUITE[solver_name]["bpdn_B0"] = @benchmarkable $solver($bpdn_B0) end From 4dea55e18fb0a7f2414605339dffe67ae0faf7b4 Mon Sep 17 00:00:00 2001 From: Dominique Date: Wed, 4 Mar 2026 16:32:15 -0500 Subject: [PATCH 12/12] Update .github/workflows/jsobmarks.yml Co-authored-by: Maxence Gollier <134112149+MaxenceGollier@users.noreply.github.com> --- .github/workflows/jsobmarks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jsobmarks.yml b/.github/workflows/jsobmarks.yml index 590d2f28..d1d3a593 100644 --- a/.github/workflows/jsobmarks.yml +++ b/.github/workflows/jsobmarks.yml @@ -7,7 +7,7 @@ on: jobs: bmark: name: Julia ${{ matrix.version }} - macOS - ${{ matrix.arch }} - ${{ github.event_name }} - if: github.event_name == 'workflow_call' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'benchmarks')) + if: github.event_name == 'workflow_call' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run benchmarks')) # FIXME: should run on hosted runner runs-on: macOS-latest strategy: