|
| 1 | +using Pkg |
| 2 | +bmark_dir = @__DIR__ |
| 3 | +println(@__DIR__) |
| 4 | +Pkg.activate(bmark_dir) |
| 5 | +Pkg.instantiate() |
| 6 | +repo_name = string(split(ARGS[1], ".")[1]) |
| 7 | +bmarkname = lowercase(repo_name) |
| 8 | +using Git |
| 9 | + |
| 10 | +const git = Git.git() |
| 11 | + |
| 12 | +# if we are running these benchmarks from the git repository |
| 13 | +# we want to develop the package instead of using the release |
| 14 | +if isdir(joinpath(bmark_dir, "..", ".git")) |
| 15 | + Pkg.develop(PackageSpec(url = joinpath(bmark_dir, ".."))) |
| 16 | + bmarkname = readchomp(`$git rev-parse HEAD`) # sha of HEAD |
| 17 | +end |
| 18 | + |
| 19 | +using DataFrames |
| 20 | +using GitHub |
| 21 | +using JLD2 |
| 22 | +using JSON |
| 23 | +using PkgBenchmark |
| 24 | +using Plots |
| 25 | + |
| 26 | +using SolverBenchmark |
| 27 | + |
| 28 | +# NB: benchmarkpkg will run benchmarks/benchmarks.jl by default |
| 29 | +commit = benchmarkpkg(repo_name) # current state of repository |
| 30 | +main = benchmarkpkg(repo_name, "main") |
| 31 | +judgement = judge(commit, main) |
| 32 | + |
| 33 | +commit_stats = bmark_results_to_dataframes(commit) |
| 34 | +main_stats = bmark_results_to_dataframes(main) |
| 35 | +judgement_stats = judgement_results_to_dataframes(judgement) |
| 36 | + |
| 37 | +export_markdown("judgement_$(bmarkname).md", judgement) |
| 38 | +export_markdown("main.md", main) |
| 39 | +export_markdown("$(bmarkname).md", commit) |
| 40 | + |
| 41 | +function profile_solvers_from_pkgbmark(stats::Dict{Symbol, DataFrame}) |
| 42 | + # guard against zero gctimes |
| 43 | + costs = |
| 44 | + [df -> df[!, :time], df -> df[!, :memory], df -> df[!, :gctime] .+ 1, df -> df[!, :allocations]] |
| 45 | + profile_solvers(stats, costs, ["time", "memory", "gctime+1", "allocations"]) |
| 46 | +end |
| 47 | + |
| 48 | +# extract stats for each benchmark to plot profiles |
| 49 | +# files_dict will be part of json_dict below |
| 50 | +files_dict = Dict{String, Any}() |
| 51 | +file_num = 1 |
| 52 | +for k ∈ keys(judgement_stats) |
| 53 | + global file_num |
| 54 | + k_stats = Dict{Symbol, DataFrame}(:commit => commit_stats[k], :main => main_stats[k]) |
| 55 | + save_stats(k_stats, "ldl_$(bmarkname)_vs_main_$(k).jld2", force = true) |
| 56 | + |
| 57 | + k_profile = profile_solvers_from_pkgbmark(k_stats) |
| 58 | + savefig("profiles_commit_vs_main_$(k).svg") |
| 59 | + # read contents of svg file to add to gist |
| 60 | + k_svgfile = open("profiles_commit_vs_main_$(k).svg", "r") do fd |
| 61 | + readlines(fd) |
| 62 | + end |
| 63 | + # file_num makes sure svg files appear before md files (added below) |
| 64 | + files_dict["$(file_num)_$(k).svg"] = Dict{String, Any}("content" => join(k_svgfile)) |
| 65 | + file_num += 1 |
| 66 | +end |
| 67 | + |
| 68 | +for mdfile ∈ [:judgement, :main, :commit] |
| 69 | + global file_num |
| 70 | + files_dict["$(file_num)_$(mdfile).md"] = |
| 71 | + Dict{String, Any}("content" => "$(sprint(export_markdown, eval(mdfile)))") |
| 72 | + file_num += 1 |
| 73 | +end |
| 74 | + |
| 75 | +jldopen("ldl_$(bmarkname)_vs_main_judgement.jld2", "w") do file |
| 76 | + file["jstats"] = judgement_stats |
| 77 | +end |
| 78 | + |
| 79 | +# json description of gist |
| 80 | +json_dict = Dict{String, Any}( |
| 81 | + "description" => "$(repo_name) repository benchmark", |
| 82 | + "public" => true, |
| 83 | + "files" => files_dict, |
| 84 | +) |
| 85 | + |
| 86 | +open("$(bmarkname).json", "w") do f |
| 87 | + JSON.print(f, json_dict) |
| 88 | +end |
0 commit comments