Support SolverBenchmarks

Project.toml

@@ -11,6 +11,7 @@ Git = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2"
 GitHub = "bc5e4493-9b4d-5f90-b8aa-2b2bcaad7a26"
 JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"
 Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
@@ -24,6 +25,7 @@ Git = "1.3"
 GitHub = "5.9"
 JLD2 = "0.4,0.5"
 JSON = "0.21"
+LibGit2 = "1.11.0"
 Pkg = "1.9"
 PkgBenchmark = "0.2"
 Plots = "1.39"

src/JSOBenchmarks.jl

@@ -10,13 +10,16 @@ using Git
 using GitHub
 using JLD2
 using JSON
+using LibGit2
 using PkgBenchmark
 using Plots
 
 # JSO modules
 using SolverBenchmark
 
-export run_benchmarks
+include("solver_benchmarks.jl")
+
+export run_benchmarks, run_solver_benchmarks
 export profile_solvers_from_pkgbmark
 export create_gist_from_json_dict, create_gist_from_json_file
 export update_gist_from_json_dict, update_gist_from_json_file
@@ -310,12 +313,24 @@ function write_simple_md_report(
   open(fname, "w") do f
     println(f, "Gist: $(gist_url)\n")
     println(f, "Full results stored as artifacts\n")
-    write_md_svgs(f, "Overview", gist_url, svgs)
-    write_md(f, "Judgement", judgement)
-    println(f, "<br>")
-    write_md(f, "this_commit", this_commit)
-    println(f, "<br>")
-    write_md(f, "Reference", reference)
+
+    if !isempty(svgs)
+      write_md_svgs(f, "Overview", gist_url, svgs)
+    end
+
+    if judgement !== nothing
+      write_md(f, "Judgement", judgement)
+      println(f, "<br>")
+    end
+
+    if this_commit !== nothing
+      write_md(f, "this_commit", this_commit)
+      println(f, "<br>")
+    end
+
+    if reference !== nothing
+      write_md(f, "Reference", reference)
+    end
   end
 end
 

src/solver_benchmarks.jl

@@ -0,0 +1,152 @@
+function run_solver_benchmarks(
+  repo_name::AbstractString,
+  bmark_dir::AbstractString;
+  reference_branch::AbstractString = "main",
+  gist_url::Union{AbstractString, Nothing} = nothing,
+  script = "benchmarks.jl",
+  values = [(:elapsed_time, "CPU Time"), (:neval_obj, "# Objective Evals"), (:neval_grad, "# Gradient Evals")],
+)
+
+  update_gist = gist_url !== nothing
+  is_git = isdir(joinpath(bmark_dir, "..", ".git"))
+  @info "" is_git update_gist
+
+  local gist_id
+  if update_gist
+    gist_id = split(gist_url, "/")[end]
+    @info "" gist_id
+  end
+
+  # if we are running these benchmarks from the git repository
+  # we want to develop the package instead of using the release
+  if is_git
+    Pkg.develop(PackageSpec(path = joinpath(bmark_dir, "..")))
+  else
+    Pkg.activate(bmark_dir)
+  end
+  Pkg.instantiate()
+
+  # name the benchmark after the repo or the sha of HEAD
+  bmarkname = is_git ? readchomp(`$git rev-parse HEAD`) : lowercase(repo_name)
+  @info "" bmarkname
+
+  # Run the benchmark script on this commit
+  this_commit = Base.include(Main, joinpath(bmark_dir, script))
+  @assert this_commit isa Dict{Symbol, DataFrame} "Expected the benchmark script to return a Dict{Symbol, DataFrame}, but got $(typeof(this_commit)). Make sure your benchmark script returns a dict resulting from BenchmarkSolver.bmark_solver function"
+
+  # Run the benchmark script on the reference branch
+  local reference
+  if is_git
+    repo_dir = joinpath(bmark_dir, "..")
+    repo = LibGit2.GitRepo(repo_dir)
+    println("repo_dir : $repo_dir")
+    println("bmark_dir : $bmark_dir")
+    reference = _withcommit(joinpath(bmark_dir, script), repo, reference_branch)
+  end
+
+  #TODO: save in a jld2 file: see run_benchmarks
+
+  # Plotting
+  files_dict = Dict{String, Any}()
+  svgs = String[]
+  if is_git
+    for key in keys(this_commit)
+      if haskey(reference, key)
+        @info "Plotting $key"
+        stats_subset = Dict(:this_commit => this_commit[key], :reference => reference[key])
+        solved(df) = (df.status .== :first_order)
+        costs = [df -> .!solved(df) * Inf + getproperty(df, value[1]) for value in values]
+        costnames = [value[2] for value in values]
+
+        p = profile_solvers(stats_subset, costs, costnames;xlabel = "", ylabel = "")
+        fname = "this_commit_vs_reference_$(key)"
+        savefig("$(fname).svg")
+        push!(svgs, "$(fname).svg")
+        content = read("$(fname).svg", String)
+        files_dict["$(fname).svg"] = Dict("content" => content)
+      else
+        @warn "$(reference_branch) branch benchmarks do not run the solver $key. Please update the benchmark solver list in a separate PR and rebase."
+      end
+    end
+  end
+
+  @info "creating or updating gist"
+  # json description of gist
+  json_dict = Dict{String, Any}(
+    "description" => "$(repo_name) repository benchmark",
+    "public" => true,
+    "files" => files_dict,
+  )
+
+  if update_gist
+    json_dict["gist_id"] = gist_id
+  end
+
+  gist_json = "$(bmarkname).json"
+  open(gist_json, "w") do f
+    JSON.print(f, json_dict)
+  end
+
+  readme = "# $(repo_name) Solver Benchmarks\n\n"
+  readme *= "Comparison between current commit and $(reference_branch).\n\n"
+
+  files_dict["README.md"] = Dict("content" => readme)
+
+  local new_gist_url
+  if update_gist
+    update_gist_from_json_dict(gist_id, json_dict)
+  else
+    new_gist = create_gist_from_json_dict(json_dict)
+    new_gist_url = string(new_gist.html_url)
+  end
+
+  @info "preparing simple Markdown report"
+  is_git &&
+    write_simple_md_report(
+      "bmark_$(bmarkname).md",
+      nothing,
+      nothing,
+      nothing,
+      new_gist_url,
+      svgs,
+    )
+
+  @info "finished"
+  return nothing
+end
+
+
+# Runs a script at a commit on a repo and afterwards goes back
+# to the original commit / branch.
+# This code is based on https://github.com/JuliaCI/PkgBenchmark.jl/blob/master/src/util.jl
+function _withcommit(script, repo, commit)
+  println(repo)
+  original_commit = string(LibGit2.GitHash(LibGit2.GitObject(repo, "HEAD")))
+  local result
+  LibGit2.transact(repo) do r
+    branch = try LibGit2.branch(r) catch err; nothing end
+    try
+      LibGit2.checkout!(r, _shastring(r, commit))
+      result = Base.include(Main, script)
+      @assert result isa Dict{Symbol, DataFrame} "Expected the benchmark script to return a Dict{Symbol, DataFrame}, but got $(typeof(result)). Make sure your benchmark script returns a dict resulting from BenchmarkSolver.bmark_solver function"
+    catch err
+        rethrow(err)
+    finally
+      if branch !== nothing
+        LibGit2.branch!(r, branch)
+      else
+        LibGit2.checkout!(r, original_commit)
+      end
+    end
+  end
+  return result
+end
+
+function _shastring(r::LibGit2.GitRepo, targetname)
+  branch = LibGit2.lookup_branch(r, targetname)
+  branch = branch === nothing ? LibGit2.lookup_branch(r, targetname, true) : branch # Search remote as well if not found locally.
+  branch = branch === nothing ? LibGit2.lookup_branch(r, "origin/$(targetname)") : branch
+  branch = branch === nothing ? LibGit2.lookup_branch(r, "origin/$(targetname)", true) : branch # Search remote as well if not found locally.
+  @assert branch !== nothing "Branch $(targetname) not found in repository."
+  return string(LibGit2.GitHash(LibGit2.GitObject(r, LibGit2.name(branch))))
+end