diff --git a/bench_runner/__main__.py b/bench_runner/__main__.py index b72a022f..8465fbb5 100644 --- a/bench_runner/__main__.py +++ b/bench_runner/__main__.py @@ -19,6 +19,7 @@ "remove_benchmark": "Remove specific benchmarks from the data set", "run_benchmarks": "Run benchmarks (in timing, pyperf or perf modes)", "should_run": "Determine whether we need to rerun results for the current commit", + "synthesize_loops_file": "Create a loops file from multiple benchmark results", "notify": "Send a notification about the completion of the workflow", } diff --git a/bench_runner/scripts/synthesize_loops_file.py b/bench_runner/scripts/synthesize_loops_file.py new file mode 100644 index 00000000..c52c641e --- /dev/null +++ b/bench_runner/scripts/synthesize_loops_file.py @@ -0,0 +1,270 @@ +import argparse +import collections +import errno +import json +import pathlib +import sys +from typing import Iterable + +import rich_argparse + +# pyperf/pyperformance run benchmarks by benchmark name, but store results, +# including loops used, by data point name (as reported by the benchmark). +# There's no mapping from the one to the other that we can easily use (other +# than running benchmarks one by one and checking what data points they +# report), so here's a hand-written mapping instead. Benchmarks that use +# their own name for the data point are omitted. This will probably (slowly) +# get out of date, but so be it. +# +# (Sorted by value, then key.) +DATAPOINT_TO_BENCHMARK = { + "many_optionals": "argparse", + "subparsers": "argparse", + "async_tree_none": "async_tree", + "async_tree_none_tg": "async_tree_tg", + "bench_mp_pool": "concurrent_imap", + "bench_thread_pool": "concurrent_imap", + "deepcopy_memo": "deepcopy", + "deepcopy_reduce": "deepcopy", + "create_gc_cycles": "gc_collect", + "genshi_text": "genshi", + "genshi_xml": "genshi", + "logging_format": "logging", + "logging_silent": "logging", + "logging_simple": "logging", + "shortest_path": "networkx", + "connected_components": "networkx_connected_components", + "k_core": "networkx_k_core", + "pprint_pformat": "pprint", + "pprint_safe_repr": "pprint", + "scimark_fft": "scimark", + "scimark_lu": "scimark", + "scimark_monte_carlo": "scimark", + "scimark_sor": "scimark", + "scimark_sparse_mat_mult": "scimark", + "sqlglot_v2_normalize": "sqlglot_v2", + "sympy_expand": "sympy", + "sympy_integrate": "sympy", + "sympy_str": "sympy", + "sympy_sum": "sympy", + "xml_etree_generate": "xml_etree", + "xml_etree_iterparse": "xml_etree", + "xml_etree_parse": "xml_etree", + "xml_etree_process": "xml_etree", +} + +# The list of bm_* directories in pyperformance and pyston-benchmarks, plus +# the aliases defined in their MANIFEST files (entries with +# '') +KNOWN_BENCHMARKS = { + "2to3", + "aiohttp", + "argparse", + "argparse_subparsers", + "async_generators", + "async_tree", + "async_tree_cpu_io_mixed", + "async_tree_cpu_io_mixed_tg", + "async_tree_eager", + "async_tree_eager_cpu_io_mixed", + "async_tree_eager_cpu_io_mixed_tg", + "async_tree_eager_io", + "async_tree_eager_io_tg", + "async_tree_eager_memoization", + "async_tree_eager_memoization_tg", + "async_tree_eager_tg", + "async_tree_io", + "async_tree_io_tg", + "async_tree_memoization", + "async_tree_memoization_tg", + "async_tree_tg", + "asyncio_tcp", + "asyncio_tcp_ssl", + "asyncio_websockets", + "bpe_tokeniser", + "chameleon", + "chaos", + "comprehensions", + "concurrent_imap", + "coroutines", + "coverage", + "crypto_pyaes", + "dask", + "decimal_factorial", + "decimal_pi", + "deepcopy", + "deltablue", + "django_template", + "djangocms", + "docutils", + "dulwich_log", + "fannkuch", + "flaskblogging", + "float", + "gc_collect", + "gc_traversal", + "generators", + "genshi", + "gevent_hub", + "go", + "gunicorn", + "hexiom", + "hg_startup", + "html5lib", + "json", + "json_dumps", + "json_loads", + "kinto", + "logging", + "mako", + "mdp", + "meteor_contest", + "mypy2", + "nbody", + "networkx", + "networkx_connected_components", + "networkx_k_core", + "nqueens", + "pathlib", + "pickle", + "pickle_dict", + "pickle_list", + "pickle_pure_python", + "pidigits", + "pprint", + "pycparser", + "pyflate", + "pylint", + "python_startup", + "python_startup_no_site", + "pytorch_alexnet_inference", + "raytrace", + "regex_compile", + "regex_dna", + "regex_effbot", + "regex_v8", + "richards", + "richards_super", + "scimark", + "spectral_norm", + "sphinx", + "sqlalchemy_declarative", + "sqlalchemy_imperative", + "sqlglot_v2", + "sqlglot_v2_optimize", + "sqlglot_v2_parse", + "sqlglot_v2_transpile", + "sqlite_synth", + "sympy", + "telco", + "thrift", + "tomli_loads", + "tornado_http", + "typing_runtime_protocols", + "unpack_sequence", + "unpickle", + "unpickle_list", + "unpickle_pure_python", + "xml_etree", +} + + +def parse_result(results_file, benchmark_data): + with results_file.open() as f: + result = json.load(f) + bms = result["benchmarks"] + if len(bms) == 1 and "metadata" not in bms[0]: + # Sometimes a .json file contains just a single benchmark. + bms = [result] + for bm in bms: + if "metadata" not in bm: + raise RuntimeError(f"Invalid data {bm.keys()!r} in {results_file}") + return + name = bm["metadata"]["name"] + name = DATAPOINT_TO_BENCHMARK.get(name, name) + assert name is not None # to satisfy pyright. + if name not in KNOWN_BENCHMARKS: + print( + f"WARNING: unknown benchmark {name!r} in {results_file}", + file=sys.stderr, + ) + # Avoid repeated warnings. + KNOWN_BENCHMARKS.add(name) + benchmark_data[name].append(bm["metadata"]["loops"]) + + +def _main( + loops_file: pathlib.Path, + update: bool, + overwrite: bool, + merger: str, + results: Iterable[pathlib.Path], +): + if not update and not overwrite and loops_file.exists(): + raise OSError( + errno.EEXIST, + f"{loops_file} exists (use -f to overwrite, -u to merge data)", + ) + benchmark_data = collections.defaultdict(list) + if update: + parse_result(loops_file, benchmark_data) + for result_file in results: + parse_result(result_file, benchmark_data) + + merge_func = { + "max": max, + "min": min, + }[merger] + + # pyperformance expects a specific layout, and needs the top-level + # metadata even if it's empty. + loops_data = {"benchmarks": [], "metadata": {}} + for bm in sorted(benchmark_data): + loops = merge_func(benchmark_data[bm]) + bm_result = {"metadata": {"name": bm, "loops": loops}} + loops_data["benchmarks"].append(bm_result) + with loops_file.open("w") as f: + json.dump(loops_data, f, sort_keys=True, indent=4) + f.write("\n") + + +def main(): + parser = argparse.ArgumentParser( + description=""" + Synthesize a loops.json file for use with `pyperformance`'s + `--same-loops` (or `PYPERFORMANCE_LOOPS_FILE`) from one or more + benchmark results. + """, + formatter_class=rich_argparse.ArgumentDefaultsRichHelpFormatter, + ) + parser.add_argument( + "-o", "--loops_file", help="loops file to write to", required=True + ) + group = parser.add_mutually_exclusive_group(required=False) + group.add_argument( + "-u", "--update", action="store_true", help="add to existing loops file" + ) + group.add_argument( + "-f", "--overwrite", action="store_true", help="replace loops file" + ) + parser.add_argument( + "-s", + "--select", + choices=("max", "min"), + default="max", + help="how to merge multiple runs", + ) + parser.add_argument("results", nargs="+", help="benchmark results to parse") + args = parser.parse_args() + + _main( + pathlib.Path(args.loops_file), + args.update, + args.overwrite, + args.select, + [pathlib.Path(r) for r in args.results], + ) + + +if __name__ == "__main__": + main() diff --git a/tests/test_synthesize_loops_file.py b/tests/test_synthesize_loops_file.py new file mode 100644 index 00000000..90a681b8 --- /dev/null +++ b/tests/test_synthesize_loops_file.py @@ -0,0 +1,82 @@ +import json +import pathlib +import sys +import tempfile + +import pytest + +from bench_runner.scripts import synthesize_loops_file + +DATA_PATH = pathlib.Path(__file__).parent / "data" + + +def run_synthesize( + output: pathlib.Path, + datadir: pathlib.Path, + *, + update: bool = False, + overwrite: bool = False, + merger: str = "max", +): + files = datadir.glob("results/**/*.json") + synthesize_loops_file._main( + loops_file=output, + update=update, + overwrite=overwrite, + merger=merger, + results=files, + ) + + +def check_loops(output: pathlib.Path): + with output.open() as f: + data = json.load(f) + assert "benchmarks" in data + assert "metadata" in data + seen = set() + for bm in data["benchmarks"]: + assert "metadata" in bm + assert "loops" in bm["metadata"] + assert isinstance(bm["metadata"]["loops"], int) + assert "name" in bm["metadata"] + assert (name := bm["metadata"]["name"]) not in seen + assert isinstance(name, str) + seen.add(name) + data["benchmarks"].sort(key=lambda item: item["metadata"]["name"]) + return data + + +def set_loops(output, value): + with output.open() as f: + data = json.load(f) + for bm in data["benchmarks"]: + bm["metadata"]["loops"] = value + with output.open("w") as f: + json.dump(data, f, sort_keys=True, indent=4) + + +def test_synthesize(): + with tempfile.TemporaryDirectory() as tmpdir: + output = pathlib.Path(tmpdir) / "loops.json" + run_synthesize(output, DATA_PATH) + expected_data = check_loops(output) + + with pytest.raises(FileExistsError): + run_synthesize(output, DATA_PATH) + + run_synthesize(output, DATA_PATH, update=True) + assert expected_data == check_loops(output) + + set_loops(output, 0) + run_synthesize(output, DATA_PATH, update=True) + assert expected_data == check_loops(output) + + set_loops(output, sys.maxsize) + run_synthesize(output, DATA_PATH, overwrite=True) + assert expected_data == check_loops(output) + + run_synthesize(output, DATA_PATH, overwrite=True, merger="min") + expected_data = check_loops(output) + set_loops(output, sys.maxsize) + run_synthesize(output, DATA_PATH, update=True, merger="min") + assert expected_data == check_loops(output)