qoco-gpu-benchmarks/make_sgm_table.py at main · qoco-org/qoco-gpu-benchmarks · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
import os
import numpy as np
import pandas as pd
import math

PROBLEMS = [
    "group_lasso",
    "huber",
    "portfolio",
    "multiperiod_portfolio",
    "tv_denoising",
]

SOLVED_STRINGS = ["QOCO_SOLVED", "SOLVED", "Solved", "optimal"]

solvers = {
    "QOCO-GPU": "qoco_cuda_results.csv",
    "QOCO": "qoco_results.csv",
    "CuClarabel": "cuclarabel_results.csv",
    "Mosek": "mosek_results.csv",
    "Gurobi": "gurobi_results.csv",
}


def compute_shifted_geometric_mean(tmax=3600):

    t = {s: [] for s in solvers}
    fail = {s: 0 for s in solvers}

    for problem in PROBLEMS:

        for solver, file in solvers.items():

            path = os.path.join(problem, file)
            df = pd.read_csv(path)

            runtime = df["setup_time"] + df["solve_time"]

            if "status" in df.columns:
                status = df["status"]
            else:
                status = ["optimal"] * len(df)

            for r, st in zip(runtime, status):

                if r > tmax or math.isnan(r) or st not in SOLVED_STRINGS:
                    fail[solver] += 1
                    t[solver].append(tmax)
                else:
                    t[solver].append(r)

    n_prob = len(next(iter(t.values())))

    # shifted geometric mean (same formula as your code)
    rs = {}
    for s in solvers:
        prod = 1.0
        for p in range(n_prob):
            prod *= 1 + t[s][p]
        rs[s] = prod ** (1 / n_prob) - 1

    # normalize by best solver
    mings = min(rs.values())

    for s in solvers:
        rs[s] /= mings
        fail[s] = 100 * fail[s] / n_prob
    write_latex_table(rs, fail)


def write_latex_table(rs, fail):

    solver_names = list(solvers.keys())

    best_sgm = min(rs.values())
    best_fail = min(fail.values())

    lines = []
    lines.append(r"\begin{tabular}{l" + "c" * len(solver_names) + "}")
    lines.append(r"\toprule")

    lines.append(" & " + " & ".join(solver_names) + r" \\")
    lines.append(r"\midrule")

    # Shifted GM row
    row = "Shifted GM"
    for s in solver_names:
        val = "%.2f" % rs[s]
        if abs(rs[s] - best_sgm) < 1e-12:
            val = r"\textbf{" + val + "}"
        row += " & " + val
    row += r" \\"
    lines.append(row)

    # Failure rate row
    row = "Failure Rate (\%)"
    for s in solver_names:
        val = "%.1f" % fail[s]
        if abs(fail[s] - best_fail) < 1e-12:
            val = r"\textbf{" + val + "}"
        row += " & " + val
    row += r" \\"
    lines.append(row)

    lines.append(r"\bottomrule")
    lines.append(r"\end{tabular}")

    with open("figures/sgm_table.tex", "w") as f:
        f.write("\n".join(lines))


if __name__ == "__main__":
    compute_shifted_geometric_mean()