diff --git a/development/graph/lifted_multicut/evaluate_solvers.py b/development/graph/lifted_multicut/evaluate_solvers.py index a3b2278..a2972fd 100644 --- a/development/graph/lifted_multicut/evaluate_solvers.py +++ b/development/graph/lifted_multicut/evaluate_solvers.py @@ -1,6 +1,7 @@ from __future__ import annotations import argparse +import sys from dataclasses import dataclass from statistics import mean from time import perf_counter @@ -159,6 +160,22 @@ def format_float(value: float) -> str: return f"{value:.6g}" +def print_progress(row: dict) -> None: + print( + ( + f"[done] {row['problem']} / {row['solver']}: " + f"bic_energy={format_float(row['bic_energy'])}, " + f"nifty_energy={format_float(row['nifty_energy'])}, " + f"delta={format_float(row['energy_diff'])}, " + f"bic_runtime={format_float(row['bic_runtime_s'])}s, " + f"nifty_runtime={format_float(row['nifty_runtime_s'])}s, " + f"ratio={format_float(row['runtime_ratio'])}" + ), + file=sys.stderr, + flush=True, + ) + + def print_markdown_table(rows: list[dict]) -> None: headers = [ "problem", @@ -224,7 +241,9 @@ def main() -> None: rows = [] for problem_name in args.problems: for solver_name in args.solvers: - rows.append(evaluate(problem_name, solver_name, configs[solver_name], args)) + row = evaluate(problem_name, solver_name, configs[solver_name], args) + rows.append(row) + print_progress(row) print_markdown_table(rows) diff --git a/development/graph/multicut/PERFORMANCE_NOTES.md b/development/graph/multicut/PERFORMANCE_NOTES.md new file mode 100644 index 0000000..54a990a --- /dev/null +++ b/development/graph/multicut/PERFORMANCE_NOTES.md @@ -0,0 +1,92 @@ +# Multicut Performance Notes + +State of the multicut solvers vs nifty on the standard benchmark problems and +notes on remaining optimization headroom. Read this before the next round of +perf work. + +## Current benchmark matrix + +Produced by `python evaluate_solvers.py` (2026-05-17). Small problems were run +with both implementations in one pass: + +```bash +python evaluate_solvers.py --problems A_small B_small C_small \ + --results-jsonl benchmark_results/small_both.jsonl +``` + +Medium problems were run in two passes so completed rows were preserved even if +a long nifty row had to be stopped: + +```bash +python evaluate_solvers.py --problems A_medium B_medium C_medium \ + --backend bic --results-jsonl benchmark_results/medium_bic.jsonl +python evaluate_solvers.py --problems A_medium B_medium C_medium \ + --backend nifty --results-jsonl benchmark_results/medium_nifty.jsonl +``` + +All runs are single-threaded with `n_repeats=1`. `KernighanLinMulticut` and the +KL stage inside `ChainedMulticutSolvers` use 5 outer iterations. Fusion-move +uses watershed proposals, `numberOfIterations=10`, `stopIfNoImprovement=4`, and +a greedy-additive fusion sub-solver on the nifty side. `runtime ratio` is +`nifty_runtime / bic_runtime` - values reported as "faster" mean bic is faster. + +| Problem | Solver | bic energy | nifty energy | Δenergy | bic runtime | nifty runtime | runtime ratio | +|---|---|---|---|---|---|---|---| +| A_small | greedy_additive | -76 914.52 | -76 914.52 | 0.00 | 0.28 s | 0.38 s | 1.32x faster | +| A_small | kernighan_lin | -76 916.06 | -76 916.06 | 0.00 | 2.04 s | 2.54 s | 1.24x faster | +| A_small | greedy_fixation | -76 914.13 | -76 914.13 | 0.00 | 0.28 s | 1.96 s | 7.09x faster | +| A_small | chained | -76 916.06 | -76 916.06 | 0.00 | 2.02 s | 2.56 s | 1.27x faster | +| A_small | decomposer | -76 914.52 | -76 914.52 | 0.00 | 0.26 s | 0.34 s | 1.30x faster | +| A_small | fusion_move | -76 915.29 | -76 915.29 | 0.00 | 1.60 s | 2.35 s | 1.46x faster | +| B_small | greedy_additive | -437 001.4 | -437 001.4 | 0.00 | 0.32 s | 0.44 s | 1.36x faster | +| B_small | kernighan_lin | -437 023.6 | -437 023.6 | 0.00 | 4.40 s | 5.32 s | 1.21x faster | +| B_small | greedy_fixation | -436 943.8 | -436 943.8 | 0.00 | 0.34 s | 1.86 s | 5.43x faster | +| B_small | chained | -437 023.6 | -437 023.6 | 0.00 | 4.36 s | 5.27 s | 1.21x faster | +| B_small | decomposer | -437 001.4 | -437 001.4 | 0.00 | 0.32 s | 0.49 s | 1.50x faster | +| B_small | fusion_move | -437 034.1 | -437 034.1 | 0.00 | 1.60 s | 2.62 s | 1.64x faster | +| C_small | greedy_additive | -24 189.93 | -24 189.93 | 0.00 | 0.74 s | 0.87 s | 1.17x faster | +| C_small | kernighan_lin | -24 191.91 | -24 191.95 | +0.03698 | 6.52 s | 7.41 s | 1.14x faster | +| C_small | greedy_fixation | -24 162.43 | -24 162.43 | 0.00 | 0.76 s | 2.79 s | 3.66x faster | +| C_small | chained | -24 191.91 | -24 191.95 | +0.03698 | 6.52 s | 7.33 s | 1.12x faster | +| C_small | decomposer | -24 189.93 | -24 189.93 | 0.00 | 0.74 s | 0.91 s | 1.23x faster | +| C_small | fusion_move | -24 191.45 | -24 191.45 | 0.00 | 4.90 s | 7.42 s | 1.52x faster | +| A_medium | greedy_additive | -535 228.2 | -535 228.2 | 0.00 | 6.21 s | 6.72 s | 1.08x faster | +| A_medium | kernighan_lin | -535 251.2 | -535 251.3 | +0.0821 | 124.75 s | 137.72 s | 1.10x faster | +| A_medium | greedy_fixation | -535 214.6 | -535 214.6 | 0.00 | 6.16 s | 25.26 s | 4.10x faster | +| A_medium | chained | -535 251.2 | -535 251.3 | +0.0821 | 122.41 s | 134.68 s | 1.10x faster | +| A_medium | decomposer | -535 228.2 | -535 228.2 | 0.00 | 6.15 s | 5.81 s | 1.06x slower | +| A_medium | fusion_move | -535 235.3 | -535 235.3 | 0.00 | 51.34 s | 65.61 s | 1.28x faster | +| B_medium | greedy_additive | -2 141 349 | -2 141 349 | 0.00 | 5.71 s | 6.78 s | 1.19x faster | +| B_medium | kernighan_lin | -2 141 705 | -2 141 700 | -5.246 | 263.46 s | 157.91 s | 1.67x slower | +| B_medium | greedy_fixation | -2 141 097 | -2 141 097 | 0.00 | 5.63 s | 24.22 s | 4.30x faster | +| B_medium | chained | -2 141 705 | -2 141 700 | -5.246 | 259.35 s | 156.60 s | 1.66x slower | +| B_medium | decomposer | -2 141 349 | -2 141 349 | 0.00 | 5.62 s | 7.00 s | 1.25x faster | +| B_medium | fusion_move | -2 141 572 | -2 141 572 | 0.00 | 43.82 s | 67.12 s | 1.53x faster | +| C_medium | greedy_additive | -90 732.51 | -90 732.51 | 0.00 | 11.05 s | 11.53 s | 1.04x faster | +| C_medium | kernighan_lin | -90 768.01 | -90 768.56 | +0.5491 | 219.36 s | 240.26 s | 1.10x faster | +| C_medium | greedy_fixation | -90 631.67 | -90 631.67 | 0.00 | 11.11 s | 30.28 s | 2.73x faster | +| C_medium | chained | -90 768.01 | -90 768.56 | +0.5491 | 218.77 s | 234.48 s | 1.07x faster | +| C_medium | decomposer | -90 732.51 | -90 732.51 | 0.00 | 10.84 s | 12.12 s | 1.12x faster | +| C_medium | fusion_move | -90 746.68 | -90 746.68 | 0.00 | 58.34 s | 86.42 s | 1.48x faster | + +Δenergy = bic - nifty; negative means bic found the lower-energy labeling, +positive means nifty did. The only nonzero differences are KL/chained rows, and +they are tiny relative to the objective scale. Fusion-move, greedy-additive, +greedy-fixation, and decomposer match nifty energies on every benchmark row. + +## Current read + +`KernighanLinMulticut` is the dominant runtime target. It is faster than nifty +on most rows, but `B_medium` is the clear exception: bic KL/chained takes about +260 s vs nifty's 157 s. Because `ChainedMulticutSolvers` is greedy-additive + +KL, it inherits the same behavior. + +Fusion-move is not the immediate bottleneck. It is faster than nifty on every +row in the matrix, including all medium problems, with exact energy matches. + +Greedy fixation is substantially faster than nifty, and greedy additive plus +the decomposer are already close enough that further work there is unlikely to +move end-to-end runtimes unless a downstream workflow uses only those solvers. + +Raw intermediate rows are in `benchmark_results/*.jsonl`; the `.md` files next +to them contain the direct stdout tables from each run. diff --git a/development/graph/multicut/evaluate_solvers.py b/development/graph/multicut/evaluate_solvers.py index 3f57426..0e50933 100644 --- a/development/graph/multicut/evaluate_solvers.py +++ b/development/graph/multicut/evaluate_solvers.py @@ -1,6 +1,8 @@ from __future__ import annotations import argparse +import json +import sys from dataclasses import dataclass from statistics import mean from time import perf_counter @@ -141,42 +143,80 @@ def evaluate(problem_name: str, solver_name: str, config: SolverConfig, args): nifty_runtimes = [] for _ in range(args.n_repeats): - bic_objective = bic.graph.MulticutObjective(bic_graph, costs) - start = perf_counter() - bic_labels = config.make_bic_solver(args.threads).optimize(bic_objective) - bic_runtimes.append(perf_counter() - start) - bic_energies.append(bic_energy(bic_graph, costs, bic_labels)) - - nifty_objective = nmc.multicutObjective(nifty_graph, costs) - start = perf_counter() - nifty_labels = ( - config.make_nifty_factory(nifty_objective, args.threads) - .create(nifty_objective) - .optimize() - ) - nifty_runtimes.append(perf_counter() - start) - nifty_energies.append(nifty_energy(nifty_graph, costs, nifty_labels)) - - bic_runtime = mean(bic_runtimes) - nifty_runtime = mean(nifty_runtimes) + if args.backend in ("both", "bic"): + bic_objective = bic.graph.MulticutObjective(bic_graph, costs) + start = perf_counter() + bic_labels = config.make_bic_solver(args.threads).optimize(bic_objective) + bic_runtimes.append(perf_counter() - start) + bic_energies.append(bic_energy(bic_graph, costs, bic_labels)) + + if args.backend in ("both", "nifty"): + nifty_objective = nmc.multicutObjective(nifty_graph, costs) + start = perf_counter() + nifty_labels = ( + config.make_nifty_factory(nifty_objective, args.threads) + .create(nifty_objective) + .optimize() + ) + nifty_runtimes.append(perf_counter() - start) + nifty_energies.append(nifty_energy(nifty_graph, costs, nifty_labels)) + + bic_runtime = mean(bic_runtimes) if bic_runtimes else None + nifty_runtime = mean(nifty_runtimes) if nifty_runtimes else None + bic_energy_value = mean(bic_energies) if bic_energies else None + nifty_energy_value = mean(nifty_energies) if nifty_energies else None + energy_diff = ( + bic_energy_value - nifty_energy_value + if bic_energy_value is not None and nifty_energy_value is not None + else None + ) return { "problem": problem_name, "solver": solver_name, "nodes": int(bic_graph.number_of_nodes), "edges": int(bic_graph.number_of_edges), - "bic_energy": mean(bic_energies), - "nifty_energy": mean(nifty_energies), - "energy_diff": mean(bic_energies) - mean(nifty_energies), + "bic_energy": bic_energy_value, + "nifty_energy": nifty_energy_value, + "energy_diff": energy_diff, "bic_runtime_s": bic_runtime, "nifty_runtime_s": nifty_runtime, - "runtime_ratio": nifty_runtime / bic_runtime if bic_runtime > 0 else float("inf"), + "runtime_ratio": ( + nifty_runtime / bic_runtime + if bic_runtime is not None and nifty_runtime is not None and bic_runtime > 0 + else None + ), } def format_float(value: float) -> str: + if value is None: + return "" return f"{value:.6g}" +def print_progress(row: dict) -> None: + print( + ( + f"[done] {row['problem']} / {row['solver']}: " + f"bic_energy={format_float(row['bic_energy'])}, " + f"nifty_energy={format_float(row['nifty_energy'])}, " + f"delta={format_float(row['energy_diff'])}, " + f"bic_runtime={format_float(row['bic_runtime_s'])}s, " + f"nifty_runtime={format_float(row['nifty_runtime_s'])}s, " + f"ratio={format_float(row['runtime_ratio'])}" + ), + file=sys.stderr, + flush=True, + ) + + +def append_jsonl(path: str, row: dict) -> None: + with open(path, "a", encoding="utf-8") as f: + json.dump(row, f, sort_keys=True) + f.write("\n") + f.flush() + + def print_markdown_table(rows: list[dict]) -> None: headers = [ "problem", @@ -233,6 +273,16 @@ def main() -> None: parser.add_argument("--n-repeats", type=int, default=1) parser.add_argument("--threads", type=int, default=1) parser.add_argument("--timeout", type=float, default=30.0) + parser.add_argument( + "--backend", + choices=("both", "bic", "nifty"), + default="both", + help="Which implementation to run. Defaults to both.", + ) + parser.add_argument( + "--results-jsonl", + help="Append each completed row to this JSONL file.", + ) args = parser.parse_args() if args.n_repeats < 1: @@ -243,7 +293,11 @@ def main() -> None: rows = [] for problem_name in args.problems: for solver_name in args.solvers: - rows.append(evaluate(problem_name, solver_name, configs[solver_name], args)) + row = evaluate(problem_name, solver_name, configs[solver_name], args) + rows.append(row) + print_progress(row) + if args.results_jsonl is not None: + append_jsonl(args.results_jsonl, row) print_markdown_table(rows)