From 2124b6d13cf2c00fa141366cc169727591712036 Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Fri, 28 Mar 2025 14:15:05 -0400
Subject: [PATCH 01/30] Refactor so as much work as possible is done in Python

---
 bench_runner/__main__.py                     |   4 +-
 bench_runner/benchmark_definitions.py        |  41 +++
 bench_runner/git.py                          |  40 ++
 bench_runner/scripts/get_merge_base.py       |   4 +-
 bench_runner/scripts/install.py              |   5 +-
 bench_runner/scripts/run_benchmarks.py       |   3 +-
 bench_runner/scripts/should_run.py           | 109 ------
 bench_runner/scripts/workflow.py             | 369 +++++++++++++++++++
 bench_runner/templates/_benchmark.src.yml    | 206 +----------
 bench_runner/templates/_pystats.src.yml      |  61 +--
 bench_runner/templates/benchmark.src.yml     |   2 -
 bench_runner/templates/env.yml               |   2 -
 bench_runner/templates/workflow_bootstrap.py | 163 ++++++++
 bench_runner/util.py                         |  31 +-
 tests/test_run_benchmarks.py                 |  81 ++--
 15 files changed, 704 insertions(+), 417 deletions(-)
 create mode 100644 bench_runner/benchmark_definitions.py
 delete mode 100644 bench_runner/scripts/should_run.py
 create mode 100644 bench_runner/scripts/workflow.py
 delete mode 100644 bench_runner/templates/env.yml
 create mode 100644 bench_runner/templates/workflow_bootstrap.py

diff --git a/bench_runner/__main__.py b/bench_runner/__main__.py
index 8465fbb5..b843b9d3 100644
--- a/bench_runner/__main__.py
+++ b/bench_runner/__main__.py
@@ -14,13 +14,13 @@
         "Get the merge base of the selected commit, and determine if it should run"
     ),
     "install": "Install the workflow files into a results repository",
+    "notify": "Send a notification about the completion of the workflow",
     "profiling_plot": "Generate the profiling plots from raw data",
     "purge": "Purge old results from a results repository",
     "remove_benchmark": "Remove specific benchmarks from the data set",
     "run_benchmarks": "Run benchmarks (in timing, pyperf or perf modes)",
-    "should_run": "Determine whether we need to rerun results for the current commit",
     "synthesize_loops_file": "Create a loops file from multiple benchmark results",
-    "notify": "Send a notification about the completion of the workflow",
+    "workflow": "Run the full compile/benchmark workflow",
 }
 
 if __name__ == "__main__":
diff --git a/bench_runner/benchmark_definitions.py b/bench_runner/benchmark_definitions.py
new file mode 100644
index 00000000..a5202630
--- /dev/null
+++ b/bench_runner/benchmark_definitions.py
@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+
+import dataclasses
+import hashlib
+from pathlib import Path
+
+
+from . import git
+
+
+@dataclasses.dataclass
+class BenchmarkRepo:
+    hash: str
+    url: str
+    dirname: str
+
+
+BENCHMARK_REPOS = [
+    BenchmarkRepo(
+        "56d12a8fd7cc1432835965d374929bfa7f6f7a07",
+        "https://github.com/mdboom/pyperformance.git",
+        "pyperformance",
+    ),
+    BenchmarkRepo(
+        "265655e7f03ace13ec1e00e1ba299179e69f8a00",
+        "https://github.com/pyston/python-macrobenchmarks.git",
+        "pyston-benchmarks",
+    ),
+]
+
+
+def get_benchmark_hash() -> str:
+    hash = hashlib.sha256()
+    for repo in BENCHMARK_REPOS:
+        if Path(repo.dirname).is_dir():
+            current_hash = git.get_git_hash(Path(repo.dirname))
+        else:
+            current_hash = repo.hash
+        hash.update(current_hash.encode("ascii")[:7])
+    return hash.hexdigest()[:6]
diff --git a/bench_runner/git.py b/bench_runner/git.py
index e6209e59..7f8120ed 100644
--- a/bench_runner/git.py
+++ b/bench_runner/git.py
@@ -2,9 +2,12 @@
 from __future__ import annotations
 
 
+import contextlib
 import datetime
 from pathlib import Path
+import shutil
 import subprocess
+import re
 
 
 import rich
@@ -128,3 +131,40 @@ def get_commits_between(dirname: PathLike, ref1: str, ref2: str) -> list[str]:
 def bisect_commits(dirname: PathLike, ref1: str, ref2: str) -> str:
     commits = get_commits_between(dirname, ref1, ref2)
     return commits[len(commits) // 2]
+
+
+def clone(
+    dirname: PathLike,
+    url: str,
+    *,
+    branch: str | None = None,
+    depth: int = 1,
+) -> None:
+    is_hash = re.match(r"^[0-9a-f]{40}$", branch) if branch else False
+
+    dirname = Path(dirname)
+    if dirname.is_dir():
+        if is_hash and (dirname / ".git").is_dir() and get_git_hash(dirname) == branch:
+            # This is a git repo, and the hash matches
+            return
+        shutil.rmtree(dirname)
+
+    # Fetching a hash and fetching a branch require different approaches
+
+    if is_hash:
+        assert branch is not None
+        dirname.mkdir()
+        with contextlib.chdir(dirname):
+            subprocess.check_call(["git", "init"])
+            subprocess.check_call(["git", "remote", "add", "origin", url])
+            subprocess.check_call(
+                ["git", "fetch", "--depth", str(depth), "origin", branch]
+            )
+            subprocess.check_call(["git", "checkout", branch])
+    else:
+        args = ["git", "clone", url, str(dirname)]
+        if branch is not None:
+            args += ["--branch", branch]
+        if depth is not None:
+            args += ["--depth", str(depth)]
+        subprocess.check_call(args)
diff --git a/bench_runner/scripts/get_merge_base.py b/bench_runner/scripts/get_merge_base.py
index 2ea7cfd6..4209d4bd 100644
--- a/bench_runner/scripts/get_merge_base.py
+++ b/bench_runner/scripts/get_merge_base.py
@@ -6,10 +6,10 @@
 import rich_argparse
 
 
+from bench_runner import benchmark_definitions
 from bench_runner import flags as mflags
 from bench_runner import git
 from bench_runner.result import has_result
-from bench_runner import util
 from bench_runner.util import PathLike
 
 
@@ -55,7 +55,7 @@ def _main(
                     machine,
                     pystats,
                     flags,
-                    util.get_benchmark_hash(),
+                    benchmark_definitions.get_benchmark_hash(),
                     progress=False,
                 )
                 is None
diff --git a/bench_runner/scripts/install.py b/bench_runner/scripts/install.py
index ec80b305..9275338e 100644
--- a/bench_runner/scripts/install.py
+++ b/bench_runner/scripts/install.py
@@ -241,13 +241,11 @@ def generate_generic(dst: Any) -> Any:
 def _main(check: bool) -> None:
     WORKFLOW_PATH.mkdir(parents=True, exist_ok=True)
 
-    env = load_yaml(TEMPLATE_PATH / "env.yml")
-
     for path in TEMPLATE_PATH.glob("*"):
         if path.name.endswith(".src.yml") or path.name == "env.yml":
             continue
 
-        if not (ROOT_PATH / path.name).is_file():
+        if not (ROOT_PATH / path.name).is_file() or path.suffix == ".py":
             if check:
                 fail_check(ROOT_PATH / path.name)
             else:
@@ -258,7 +256,6 @@ def _main(check: bool) -> None:
         generator = GENERATORS.get(src_path.name, generate_generic)
         src = load_yaml(src_path)
         dst = generator(src)
-        dst = {"env": env, **dst}
         write_yaml(dst_path, dst, check)
 
 
diff --git a/bench_runner/scripts/run_benchmarks.py b/bench_runner/scripts/run_benchmarks.py
index 01a3ad66..fa1249dc 100644
--- a/bench_runner/scripts/run_benchmarks.py
+++ b/bench_runner/scripts/run_benchmarks.py
@@ -18,6 +18,7 @@
 import rich_argparse
 
 
+from bench_runner import benchmark_definitions
 from bench_runner import flags
 from bench_runner import git
 from bench_runner.result import Result
@@ -278,7 +279,7 @@ def update_metadata(
     merge_base = git.get_git_merge_base(cpython)
     if merge_base is not None:
         metadata["commit_merge_base"] = merge_base
-    metadata["benchmark_hash"] = util.get_benchmark_hash()
+    metadata["benchmark_hash"] = benchmark_definitions.get_benchmark_hash()
     if run_id is not None:
         metadata["github_action_url"] = f"{GITHUB_URL}/actions/runs/{run_id}"
     actor = os.environ.get("GITHUB_ACTOR")
diff --git a/bench_runner/scripts/should_run.py b/bench_runner/scripts/should_run.py
deleted file mode 100644
index 3d8f42f4..00000000
--- a/bench_runner/scripts/should_run.py
+++ /dev/null
@@ -1,109 +0,0 @@
-# Determines if this should run.
-# If force is `true`, we always run, otherwise, we only run if we don't have
-# results.
-
-import argparse
-from pathlib import Path
-import subprocess
-import sys
-
-
-import rich_argparse
-
-
-# NOTE: This file should import in Python 3.9 or later so it can at least print
-# the error message that the version of Python is too old.
-
-
-def _main(
-    force: bool,
-    fork: str,
-    ref: str,
-    machine: str,
-    pystats: bool,
-    flag_str: str,
-    cpython: Path = Path("cpython"),
-    results_dir: Path = Path("results"),
-) -> None:
-    if sys.version_info[:2] < (3, 10):
-        print(
-            "The benchmarking infrastructure requires Python 3.10 or later.",
-            file=sys.stderr,
-        )
-        sys.exit(1)
-
-    # Now that we've assert we are Python 3.11 or later, we can import
-    # parts of our library.
-    from bench_runner import flags as mflags
-    from bench_runner import git
-    from bench_runner.result import has_result
-    from bench_runner import util
-
-    flags = mflags.parse_flags(flag_str)
-
-    if "PYTHON_UOPS" in flags and "JIT" in flags:
-        print("Tier 2 interpreter and JIT may not be selected at the same time")
-        sys.exit(1)
-
-    try:
-        commit_hash = git.get_git_hash(cpython)
-    except subprocess.CalledProcessError:
-        # This will fail if the cpython checkout failed for some reason. Print
-        # a nice error message since the one the checkout itself gives is
-        # totally inscrutable.
-        print("The checkout of cpython failed.", file=sys.stderr)
-        print(f"You specified fork {fork!r} and ref {ref!r}.", file=sys.stderr)
-        print("Are you sure you entered the fork and ref correctly?", file=sys.stderr)
-        # Fail the rest of the workflow
-        sys.exit(1)
-
-    found_result = has_result(
-        results_dir,
-        commit_hash,
-        machine,
-        pystats,
-        flags,
-        util.get_benchmark_hash(),
-        progress=False,
-    )
-
-    if force:
-        if found_result is not None:
-            for filepath in found_result.filename.parent.iterdir():
-                if filepath.suffix != ".json":
-                    git.remove(results_dir.parent, filepath)
-        should_run = True
-    else:
-        should_run = (machine in ("__really_all", "all")) or found_result is None
-
-    print(f"should_run={str(should_run).lower()}")
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="Do we need to run this commit?",
-        formatter_class=rich_argparse.ArgumentDefaultsRichHelpFormatter,
-    )
-    parser.add_argument(
-        "force",
-        help="If true, force a re-run",
-    )
-    parser.add_argument("fork")
-    parser.add_argument("ref")
-    parser.add_argument("machine")
-    parser.add_argument("pystats")
-    parser.add_argument("flags")
-    args = parser.parse_args()
-
-    _main(
-        args.force != "false",
-        args.fork,
-        args.ref,
-        args.machine,
-        args.pystats != "false",
-        args.flags,
-    )
-
-
-if __name__ == "__main__":
-    main()
diff --git a/bench_runner/scripts/workflow.py b/bench_runner/scripts/workflow.py
new file mode 100644
index 00000000..1a6003a3
--- /dev/null
+++ b/bench_runner/scripts/workflow.py
@@ -0,0 +1,369 @@
+from __future__ import annotations
+
+
+import argparse
+import contextlib
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+
+
+import rich_argparse
+
+
+from bench_runner import benchmark_definitions
+from bench_runner import flags as mflags
+from bench_runner import git
+from bench_runner.result import has_result
+from bench_runner import util
+from bench_runner.util import PathLike
+
+
+from bench_runner.scripts import run_benchmarks as mrun_benchmarks
+
+
+def get_windows_build_dir(force_32bit: bool) -> Path:
+    if force_32bit:
+        return Path("PCbuild") / "win32"
+    return Path("PCbuild") / "amd64"
+
+
+def get_exe_path(cpython: Path, flags: list[str], force_32bit: bool) -> Path:
+    if sys.platform.startswith("linux"):
+        return cpython / "python"
+    elif sys.platform == "darwin":
+        return cpython / "python.exe"
+    elif sys.platform.startswith("win32"):
+        build_dir = get_windows_build_dir(force_32bit)
+        if "NOGIL" in flags:
+            exe = next(build_dir.glob("python3.*.exe"))
+        else:
+            exe = "python.exe"
+        return cpython / get_windows_build_dir(force_32bit) / exe
+    else:
+        raise RuntimeError(f"Unsupported platform: {sys.platform}")
+
+
+def run_in_venv(
+    venv: PathLike, module: str, cmd: list[str], sudo: bool = False
+) -> None:
+    venv = Path(venv)
+
+    if sys.platform == "win32":
+        exe = Path("Scripts") / "python.exe"
+    else:
+        exe = Path("bin") / "python"
+
+    args = [
+        str(venv / exe),
+        "-m",
+        module,
+        *cmd,
+    ]
+
+    if sudo:
+        ld_library_path = os.environ.get("LD_LIBRARY_PATH", "")
+        args = ["sudo", f"LD_LIBRARY_PATH={ld_library_path}"] + args
+
+    print("Running command:", " ".join(args))
+    subprocess.check_call(args)
+
+
+def should_run(
+    force: bool,
+    fork: str,
+    ref: str,
+    machine: str,
+    pystats: bool,
+    flags: list[str],
+    cpython: Path = Path("cpython"),
+    results_dir: Path = Path("results"),
+) -> bool:
+    try:
+        commit_hash = git.get_git_hash(cpython)
+    except subprocess.CalledProcessError:
+        # This will fail if the cpython checkout failed for some reason. Print
+        # a nice error message since the one the checkout itself gives is
+        # totally inscrutable.
+        print("The checkout of cpython failed.", file=sys.stderr)
+        print(f"You specified fork {fork!r} and ref {ref!r}.", file=sys.stderr)
+        print("Are you sure you entered the fork and ref correctly?", file=sys.stderr)
+        # Fail the rest of the workflow
+        sys.exit(1)
+
+    found_result = has_result(
+        results_dir,
+        commit_hash,
+        machine,
+        pystats,
+        flags,
+        benchmark_definitions.get_benchmark_hash(),
+        progress=False,
+    )
+
+    if force:
+        if found_result is not None:
+            for filepath in found_result.filename.parent.iterdir():
+                if filepath.suffix != ".json":
+                    git.remove(results_dir.parent, filepath)
+        should_run = True
+    else:
+        should_run = (machine in ("__really_all", "all")) or found_result is None
+
+    return should_run
+
+
+def checkout_cpython(fork: str, ref: str, cpython: PathLike = Path("cpython")):
+    git.clone(cpython, f"https://github.com/{fork}/cpython.git", branch=ref, depth=50)
+
+
+def checkout_benchmarks():
+    for repo in benchmark_definitions.BENCHMARK_REPOS:
+        git.clone(
+            Path(repo.dirname),
+            repo.url,
+            branch=repo.hash,
+            depth=1,
+        )
+
+
+def compile_unix(cpython: PathLike, flags: list[str], pgo: bool, pystats: bool) -> None:
+    cpython = Path(cpython)
+
+    env = os.environ.copy()
+    if "CLANG" in flags:
+        if sys.platform.startswith("linux"):
+            env["CC"] = util.safe_which("clang-19")
+            env["LLVM_AR"] = util.safe_which("llvm-ar-19")
+            env["LLVM_PROFDATA"] = util.safe_which("llvm-profdata-19")
+        elif sys.platform == "darwin":
+            llvm_prefix = util.get_brew_prefix("llvm")
+            env["CC"] = f"{llvm_prefix}/bin/clang"
+            env["LDFLAGS"] = f"-L{llvm_prefix}/lib"
+            env["CFLAGS"] = f"-I{llvm_prefix}/include"
+
+    if sys.platform == "darwin":
+        openssl_prefix = util.get_brew_prefix("openssl@1.1")
+        env["PKG_CONFIG_PATH"] = f"{openssl_prefix}/lib/pkgconfig"
+
+    args = []
+    if pystats:
+        args.append("--with-pystats")
+    if pgo:
+        args.extend(["--enable-optimizations", "--with-lto=full"])
+    if "PYTHON_UOPS" in flags:
+        assert "JIT" not in flags
+        args.append("--enable-experimental-jit=interpreter")
+    if "JIT" in flags:
+        assert "PYTHON_UOPS" not in flags
+        args.append("--enable-experimental-jit=yes")
+    if "NOGIL" in flags:
+        args.append("--disable-gil")
+    if "CLANG" in flags:
+        args.append("--with-tail-call-interp")
+
+    with contextlib.chdir(cpython):
+        subprocess.check_call(["./configure", *args], env=env)
+        subprocess.check_call(["make", "-j"], env=env)
+
+
+def compile_windows(
+    cpython: PathLike, flags: list[str], pgo: bool, force_32bit: bool
+) -> None:
+    cpython = Path(cpython)
+
+    args = []
+    if force_32bit:
+        args.extend(["-p", "win32"])
+    args.extend(["-c", "Release"])
+    if pgo:
+        args.append("--pgo")
+    if "JIT" in flags:
+        args.append("--experimental-jit")
+    if "PYTHON_UOPS" in flags:
+        args.append("--experimental-jit-interpreter")
+    if "NOGIL" in flags:
+        args.append("--disable-gil")
+    if "CLANG" in flags:
+        args.extend(
+            [
+                r"/p:PlatformToolset=clangcl",
+                r"/p:LLVMInstallDir=C:\Program Files\LLVM",
+                r"/p:LLVMToolsVersion=19.1.6",
+                "--tail-call-interp",
+            ]
+        )
+
+    with contextlib.chdir(cpython):
+        subprocess.check_call(
+            [
+                Path("PCbuild") / "build.bat",
+                *args,
+            ]
+        )
+        shutil.copytree(get_windows_build_dir(force_32bit), "libs", dirs_exist_ok=True)
+
+
+def install_pyperformance(venv: PathLike) -> None:
+    run_in_venv(venv, "pip", ["install", "./pyperformance"])
+
+
+def tune_system(venv: PathLike, perf: bool) -> None:
+    # System tuning is Linux only
+    if not sys.platform.startswith("linux"):
+        return
+
+    run_in_venv(venv, "pyperf", ["system", perf and "reset" or "tune"], sudo=True)
+
+    if perf:
+        subprocess.check_call(
+            [
+                "sudo",
+                "bash",
+                "-c",
+                "echo 0 > /proc/sys/kernel/perf_event_max_sample_rate",
+            ]
+        )
+
+
+def reset_system(venv: PathLike) -> None:
+    # System tuning is Linux only
+    if not sys.platform.startswith("linux"):
+        return
+
+    run_in_venv(
+        venv,
+        "pyperf",
+        ["system", "reset"],
+        sudo=True,
+    )
+
+
+def _main(
+    fork: str,
+    ref: str,
+    machine: str,
+    benchmarks: str,
+    flags: list[str],
+    force: bool,
+    pgo: bool,
+    perf: bool,
+    pystats: bool,
+    force_32bit: bool,
+    run_id: str | None = None,
+):
+    venv = Path("venv")
+    cpython = Path("cpython")
+
+    checkout_cpython(fork, ref, cpython)
+
+    if not should_run(force, fork, ref, machine, False, flags, cpython=cpython):
+        print("No need to run benchmarks.  Skipping...")
+        return
+
+    checkout_benchmarks()
+
+    if sys.platform.startswith("linux") or sys.platform == "darwin":
+        compile_unix(cpython, flags, pgo, pystats)
+    elif sys.platform == "win32":
+        compile_windows(cpython, flags, pgo, force_32bit)
+    else:
+        raise RuntimeError(f"Unsupported platform: {sys.platform}")
+
+    # Print out the version of Python we built just so we can confirm it's the
+    # right thing in the logs
+    subprocess.check_call([get_exe_path(cpython, flags, force_32bit), "-VV"])
+
+    install_pyperformance(venv)
+    tune_system(venv, perf)
+
+    try:
+        if Path(".debug").exists():
+            shutil.rmtree(".debug")
+
+        pystats_dir = Path("/tmp") / "pystats"
+        if pystats:
+            shutil.rmtree(pystats_dir, ignore_errors=True)
+            pystats_dir.mkdir(parents=True)
+
+        if perf:
+            mode = "perf"
+        elif pystats:
+            mode = "pystats"
+        else:
+            mode = "benchmark"
+
+        mrun_benchmarks._main(
+            mode,
+            get_exe_path(cpython, flags, force_32bit),
+            fork,
+            ref,
+            benchmarks,
+            flags=flags,
+            run_id=run_id,
+            test_mode=False,
+            individual=pystats,
+        )
+    finally:
+        reset_system(venv)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="""
+        Run the full compile/benchmark workflow.
+        """,
+        formatter_class=rich_argparse.ArgumentDefaultsRichHelpFormatter,
+    )
+    parser.add_argument("fork", help="The fork of CPython")
+    parser.add_argument("ref", help="The git ref in the fork")
+    parser.add_argument(
+        "machine",
+        help="The machine to run the benchmarks on.",
+    )
+    parser.add_argument("benchmarks", help="The benchmarks to run")
+    parser.add_argument("flags", help="Configuration flags")
+    parser.add_argument("--force", action="store_true", help="Force a re-run")
+    parser.add_argument(
+        "--pgo",
+        action="store_true",
+        help="Build with profiling guided optimization",
+    )
+    parser.add_argument(
+        "--perf",
+        action="store_true",
+        help="Collect Linux perf profiling data (Linux only)",
+    )
+    parser.add_argument(
+        "--pystats",
+        action="store_true",
+        help="Enable Pystats (Linux only)",
+    )
+    parser.add_argument(
+        "--32bit",
+        action="store_true",
+        dest="force_32bit",
+        help="Do a 32-bit build (Windows only)",
+    )
+    parser.add_argument("--run_id", default=None, type=str, help="The github run id")
+    args = parser.parse_args()
+
+    _main(
+        args.fork,
+        args.ref,
+        args.machine,
+        args.benchmarks,
+        mflags.parse_flags(args.flags),
+        args.force,
+        args.pgo,
+        args.perf,
+        args.pystats,
+        args.force_32bit,
+        args.run_id,
+    )
+
+
+if __name__ == "__main__":
+    main()
diff --git a/bench_runner/templates/_benchmark.src.yml b/bench_runner/templates/_benchmark.src.yml
index 5b80eba8..2939c9bc 100644
--- a/bench_runner/templates/_benchmark.src.yml
+++ b/bench_runner/templates/_benchmark.src.yml
@@ -71,65 +71,12 @@ jobs:
       - name: git gc
         run: |
           git gc
-      - name: Checkout CPython
-        uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-          repository: ${{ inputs.fork }}/cpython
-          path: cpython
-          ref: ${{ inputs.ref }}
-          fetch-depth: 50
-      - name: Install dependencies from PyPI
-        run: |
-          Remove-Item venv -Recurse -ErrorAction SilentlyContinue
-          py -m venv venv
-          venv\Scripts\python.exe -m pip install --upgrade pip
-          venv\Scripts\python.exe -m pip install -r requirements.txt
-      - name: Should we run?
-        if: ${{ always() }}
-        id: should_run
-        run: |
-          venv\Scripts\python.exe -m bench_runner should_run ${{ inputs.force }} ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.machine }} false "${{ env.flags }}" >> $GITHUB_OUTPUT
-      - name: Checkout python-macrobenchmarks
-        uses: actions/checkout@v4
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
-        with:
-          persist-credentials: false
-          repository: pyston/python-macrobenchmarks
-          path: pyston-benchmarks
-          ref: ${{ env.PYSTON_BENCHMARKS_HASH }}
-      - name: Checkout pyperformance
-        uses: actions/checkout@v4
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
-        with:
-          persist-credentials: false
-          repository: mdboom/pyperformance
-          path: pyperformance
-          ref: ${{ env.PYPERFORMANCE_HASH }}
-      - name: Build Python
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
-        # The build.bat script is much easier to use from cmd
+      - name: Building Python and running pyperformance
         shell: cmd
         run: |
-          cd cpython
-          PCbuild\build.bat %BUILD_FLAGS% ${{ (inputs.pgo == true) && '--pgo' || '' }} ${{ inputs.clang == true && '--tail-call-interp' || '' }} ${{ inputs.jit == true && '--experimental-jit' || '' }} ${{ inputs.tier2 == true && '--experimental-jit-interpreter' || '' }} ${{ inputs.nogil == true && '--disable-gil' || '' }} -c Release ${{ inputs.clang == true && '"/p:PlatformToolset=clangcl"' || '' }} ${{ inputs.clang == true && '"/p:LLVMInstallDir=C:\Program Files\LLVM"' || '' }} ${{ inputs.clang == true && '"/p:LLVMToolsVersion=19.1.6"' || '' }}
-      - name: Copy Python to different location
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
-        run: |
-          # Copy the build products to a place that libraries can find them.
-          cd cpython
-          Copy-Item -Path $env:BUILD_DEST -Destination "libs" -Recurse
-      - name: Install pyperformance
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
-        run: |
-          venv\Scripts\python.exe -m pip install .\pyperformance
-      - name: Running pyperformance
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
-        run: |
-          venv\Scripts\python.exe -m bench_runner run_benchmarks benchmark ${{ (inputs.nogil == true && '(get-item cpython/$env:BUILD_DEST/python3.*.exe).FullName' || 'cpython/$env:BUILD_DEST/python.exe') }} ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.benchmarks || 'all' }} "${{ env.flags }}" --run_id ${{ github.run_id }}
+          python workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.machine }} ${{ inputs.benchmarks || 'all' }} "${{ env.flags }}" ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }} %BUILD_FLAGS%
       # Pull again, since another job may have committed results in the meantime
       - name: Pull benchmarking
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
         run: |
           # Another benchmarking task may have created results for the same
           # commit while the above was running. This "magic" incantation means
@@ -137,12 +84,10 @@ jobs:
           # just pulled in in that case.
           git pull -s recursive -X ours --autostash --rebase
       - name: Add data to repo
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
         uses: EndBug/add-and-commit@v9
         with:
           add: results
       - name: Upload artifacts
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
         uses: actions/upload-artifact@v4
         with:
           name: benchmark
@@ -161,79 +106,17 @@ jobs:
         run: |
           git gc
       - uses: fregante/setup-git-user@v2
-      - name: Checkout CPython
-        uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-          repository: ${{ inputs.fork }}/cpython
-          path: cpython
-          ref: ${{ inputs.ref }}
-          fetch-depth: 50
-      - name: Install dependencies from PyPI
-        run: |
-          rm -rf venv
-          python -m venv venv
-          venv/bin/python -m pip install --upgrade pip
-          venv/bin/python -m pip install -r requirements.txt
-      - name: Should we run?
-        if: ${{ always() }}
-        id: should_run
-        run: |
-          venv/bin/python -m bench_runner should_run ${{ inputs.force }} ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.machine }} false ${{ env.flags }} >> $GITHUB_OUTPUT
-      - name: Checkout python-macrobenchmarks
-        uses: actions/checkout@v4
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
-        with:
-          persist-credentials: false
-          repository: pyston/python-macrobenchmarks
-          path: pyston-benchmarks
-          ref: ${{ env.PYSTON_BENCHMARKS_HASH }}
-      - name: Checkout pyperformance
-        uses: actions/checkout@v4
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
+      - name: Setup system Python
+        if: ${{ runner.arch == 'X64' }}
+        uses: actions/setup-python@v5
         with:
-          persist-credentials: false
-          repository: mdboom/pyperformance
-          path: pyperformance
-          ref: ${{ env.PYPERFORMANCE_HASH }}
-      - name: Build with clang
-        if: ${{ inputs.clang }}
+          python-version: "3.11"
+      - name: Building Python and running pyperformance
         run: |
-          echo "CC=`which clang-19`" >> $GITHUB_ENV
-          echo "LLVM_AR=`which llvm-ar-19`" >> $GITHUB_ENV
-          echo "LLVM_PROFDATA=`which llvm-profdata-19`" >> $GITHUB_ENV
-      - name: Build Python
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
-        run: |
-          cd cpython
-          ./configure --enable-option-checking=fatal ${{ inputs.pgo == true && '--enable-optimizations --with-lto=full' || '' }} ${{ inputs.tier2 == true && '--enable-experimental-jit=interpreter' || '' }} ${{ inputs.jit == true && '--enable-experimental-jit=yes' || '' }} ${{ inputs.nogil == true && '--disable-gil' || '' }} ${{ inputs.clang == true && '--with-tail-call-interp' || '' }} ${PYTHON_CONFIGURE_FLAGS:-}
-          make ${{ runner.arch == 'ARM64' && '-j' || '-j4' }}
-          ./python -VV
-      - name: Install pyperformance
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
-        run: |
-          venv/bin/python -m pip install ./pyperformance
-      - name: Tune system
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
-        run: |
-          sudo LD_LIBRARY_PATH=$LD_LIBRARY_PATH venv/bin/python -m pyperf system ${{ inputs.perf && 'reset' || 'tune ${CPU_AFFINITY:+--affinity="$CPU_AFFINITY"}' }}
-      - name: Tune for (Linux) perf
-        if: ${{ steps.should_run.outputs.should_run != 'false' && inputs.perf }}
-        run: |
-          # Must match the PERF_PERIOD value in profiling_plot.py
-          sudo bash -c "echo 100000 > /proc/sys/kernel/perf_event_max_sample_rate"
-      - name: Running pyperformance
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
-        run: |
-          rm -rf ~/.debug/*
-          venv/bin/python -m bench_runner run_benchmarks ${{ inputs.perf && 'perf' || 'benchmark' }} cpython/python ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} --run_id ${{ github.run_id }}
-      - name: Untune system
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
-        run: |
-          sudo LD_LIBRARY_PATH=$LD_LIBRARY_PATH venv/bin/python -m pyperf system reset
+          python workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.machine }} ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} ${{ inputs.perf && '--perf' || '' }} --run_id ${{ github.run_id }}
       # Pull again, since another job may have committed results in the meantime
       - name: Pull benchmarking
-        if: ${{ steps.should_run.outputs.should_run != 'false' && !inputs.perf }}
+        if: ${{ !inputs.perf }}
         run: |
           # Another benchmarking task may have created results for the same
           # commit while the above was running. This "magic" incantation means
@@ -241,12 +124,12 @@ jobs:
           # just pulled in in that case.
           git pull -s recursive -X ours --autostash --rebase
       - name: Adding data to repo
-        if: ${{ steps.should_run.outputs.should_run != 'false' && !inputs.perf }}
+        if: ${{ !inputs.perf }}
         uses: EndBug/add-and-commit@v9
         with:
           add: results
       - name: Upload benchmark artifacts
-        if: ${{ steps.should_run.outputs.should_run != 'false' && !inputs.perf }}
+        if: ${{ !inputs.perf }}
         uses: actions/upload-artifact@v4
         with:
           name: benchmark
@@ -254,7 +137,7 @@ jobs:
             benchmark.json
           overwrite: true
       - name: Upload perf artifacts
-        if: ${{ steps.should_run.outputs.should_run != 'false' && inputs.perf }}
+        if: ${{ inputs.perf }}
         uses: actions/upload-artifact@v4
         with:
           name: perf
@@ -270,70 +153,11 @@ jobs:
       - name: git gc
         run: |
           git gc
-      - name: Checkout CPython
-        uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-          repository: ${{ inputs.fork }}/cpython
-          path: cpython
-          ref: ${{ inputs.ref }}
-          fetch-depth: 50
-      - name: Install dependencies from PyPI
-        run: |
-          rm -rf venv
-          python3 -m venv venv
-          venv/bin/python -m pip install --upgrade pip
-          venv/bin/python -m pip install -r requirements.txt
-      - name: Should we run?
-        if: ${{ always() }}
-        id: should_run
-        run: |
-          venv/bin/python -m bench_runner should_run ${{ inputs.force }} ${{ inputs.force }} ${{ inputs.ref }} ${{ inputs.machine }} false ${{ env.flags }} >> $GITHUB_OUTPUT
-      - name: Checkout python-macrobenchmarks
-        uses: actions/checkout@v4
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
-        with:
-          persist-credentials: false
-          repository: pyston/python-macrobenchmarks
-          path: pyston-benchmarks
-          ref: ${{ env.PYSTON_BENCHMARKS_HASH }}
-      - name: Checkout pyperformance
-        uses: actions/checkout@v4
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
-        with:
-          persist-credentials: false
-          repository: mdboom/pyperformance
-          path: pyperformance
-          ref: ${{ env.PYPERFORMANCE_HASH }}
-      - name: Setup environment
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
-        run: |
-          echo "PKG_CONFIG_PATH=$(brew --prefix openssl@1.1)/lib/pkgconfig" >> $GITHUB_ENV
-      - name: Build with clang
-        if: ${{ inputs.clang }}
-        run: |
-          echo "PATH=$(brew --prefix llvm)/bin:$PATH" >> $GITHUB_ENV
-          echo "CC=$(brew --prefix llvm)/bin/clang" >> $GITHUB_ENV
-          echo "LDFLAGS=-L$(brew --prefix llvm)/lib" >> $GITHUB_ENV
-          echo "CFLAGS=-I$(brew --prefix llvm)/include" >> $GITHUB_ENV
-      - name: Build Python
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
-        run: |
-          cd cpython
-          ./configure --enable-option-checking=fatal ${{ inputs.pgo == true && '--enable-optimizations --with-lto=full' || '' }} ${{ inputs.tier2 == true && '--enable-experimental-jit=interpreter' || '' }} ${{ inputs.jit == true && '--enable-experimental-jit=yes' || '' }} ${{ inputs.nogil == true && '--disable-gil' || '' }} ${{ inputs.clang == true && '--with-tail-call-interp' || '' }} ${PYTHON_CONFIGURE_FLAGS:-}
-          make -j4
-          ./python.exe -VV
-      - name: Install pyperformance
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
-        run: |
-          venv/bin/python -m pip install ./pyperformance
-      - name: Running pyperformance
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
+      - name: Building Python and running pyperformance
         run: |
-          venv/bin/python -m bench_runner run_benchmarks benchmark cpython/python.exe ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} --run_id ${{ github.run_id }}
+          python3 workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.machine }} ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }}
       # Pull again, since another job may have committed results in the meantime
       - name: Pull benchmarking
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
         run: |
           # Another benchmarking task may have created results for the same
           # commit while the above was running. This "magic" incantation means
@@ -341,12 +165,10 @@ jobs:
           # just pulled in in that case.
           git pull -s recursive -X ours --autostash --rebase
       - name: Add data to repo
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
         uses: EndBug/add-and-commit@v9
         with:
           add: results
       - name: Upload artifacts
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
         uses: actions/upload-artifact@v4
         with:
           name: benchmark
diff --git a/bench_runner/templates/_pystats.src.yml b/bench_runner/templates/_pystats.src.yml
index b5272edb..6bae3f0f 100644
--- a/bench_runner/templates/_pystats.src.yml
+++ b/bench_runner/templates/_pystats.src.yml
@@ -18,9 +18,6 @@ name: _pystats
       force:
         description: "Rerun and replace results if commit already exists"
         type: boolean
-      individual:
-        description: "Collect pystats for each individual benchmark"
-        type: boolean
 
   workflow_call:
     inputs:
@@ -39,9 +36,6 @@ name: _pystats
       force:
         description: "Rerun and replace results if commit already exists"
         type: boolean
-      individual:
-        description: "Collect pystats for each individual benchmark"
-        type: boolean
 
 jobs:
   collect-stats:
@@ -56,61 +50,10 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: "3.11"
-      - name: Checkout CPython
-        uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-          repository: ${{ inputs.fork }}/cpython
-          ref: ${{ inputs.ref }}
-          path: cpython
-          fetch-depth: 50
-      - name: Install dependencies from PyPI
-        run: |
-          rm -rf venv
-          python -m venv venv
-          venv/bin/python -m pip install -r requirements.txt
-      - name: Should we run?
-        if: ${{ always() }}
-        id: should_run
-        run: |
-          venv/bin/python -m bench_runner should_run ${{ inputs.force }} ${{ inputs.fork }} ${{ inputs.ref }} all true ${{ env.flags }} >> $GITHUB_OUTPUT
-      - name: Checkout python-macrobenchmarks
-        uses: actions/checkout@v4
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
-        with:
-          persist-credentials: false
-          repository: pyston/python-macrobenchmarks
-          path: pyston-benchmarks
-          ref: ${{ env.PYSTON_BENCHMARKS_HASH }}
-      - name: Checkout pyperformance
-        uses: actions/checkout@v4
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
-        with:
-          persist-credentials: false
-          repository: mdboom/pyperformance
-          path: pyperformance
-          ref: ${{ env.PYPERFORMANCE_HASH }}
-      - name: Create pystats directory
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
-        run: |
-          # If we don't do this, stats are printed to the console
-          rm -rf /tmp/py_stats
-          mkdir /tmp/py_stats
-      - name: Build Python
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
-        run: |
-          cd cpython
-          ./configure --enable-option-checking=fatal --enable-pystats --prefix=$PWD/install ${{ inputs.tier2 == true && '--enable-experimental-jit=interpreter' || '' }} ${{ inputs.jit == true && '--enable-experimental-jit=yes' || '' }} ${{ inputs.nogil == true && '--disable-gil' || '' }}
-          make -j4
-          make install
-      - name: Install pyperformance into the system python
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
-        run: |
-          venv/bin/python -m pip install --no-binary :all: ./pyperformance
-      - name: Running pyperformance
+      - name: Build CPython and run pyperformance benchmarks
         if: ${{ steps.should_run.outputs.should_run != 'false' }}
         run: |
-          venv/bin/python -m bench_runner run_benchmarks pystats cpython/python ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} --run_id ${{ github.run_id }} ${{ inputs.individual == true && '--individual' || '' }}
+          python workflow_bootstrap.py --pystats ${{ inputs.fork }} ${{ inputs.ref }} all ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} --run_id ${{ github.run_id }}
       - name: Pull benchmarking
         if: ${{ steps.should_run.outputs.should_run != 'false' }}
         run: |
diff --git a/bench_runner/templates/benchmark.src.yml b/bench_runner/templates/benchmark.src.yml
index f5e90c62..d569da9f 100644
--- a/bench_runner/templates/benchmark.src.yml
+++ b/bench_runner/templates/benchmark.src.yml
@@ -104,7 +104,6 @@ jobs:
       fork: ${{ inputs.fork }}
       ref: ${{ inputs.ref }}
       benchmarks: ${{ inputs.benchmarks }}
-      individual: true
       force: true
     secrets: inherit
 
@@ -116,7 +115,6 @@ jobs:
       fork: python
       ref: ${{ needs.determine_base.outputs.ref }}
       benchmarks: ${{ inputs.benchmarks }}
-      individual: true
       force: false
     secrets: inherit
 
diff --git a/bench_runner/templates/env.yml b/bench_runner/templates/env.yml
deleted file mode 100644
index e7268417..00000000
--- a/bench_runner/templates/env.yml
+++ /dev/null
@@ -1,2 +0,0 @@
-PYPERFORMANCE_HASH: 56d12a8fd7cc1432835965d374929bfa7f6f7a07
-PYSTON_BENCHMARKS_HASH: 265655e7f03ace13ec1e00e1ba299179e69f8a00
diff --git a/bench_runner/templates/workflow_bootstrap.py b/bench_runner/templates/workflow_bootstrap.py
new file mode 100644
index 00000000..d6608359
--- /dev/null
+++ b/bench_runner/templates/workflow_bootstrap.py
@@ -0,0 +1,163 @@
+# This script may only use the standard library, since it bootstraps setting up
+# the virtual environment to run the full bench_runner.
+
+
+# NOTE: This file should import in Python 3.9 or later so it can at least print
+# the error message that the version of Python is too old.
+
+
+import argparse
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+
+
+def create_venv(venv: Path) -> None:
+    if venv.exists():
+        shutil.rmtree(venv)
+
+    subprocess.check_call(
+        [
+            sys.executable,
+            "-m",
+            "venv",
+            str(venv),
+        ]
+    )
+
+
+def run_in_venv(
+    venv: Path, module: str, cmd: list[str], prefix: list[str] = []
+) -> None:
+    venv = Path(venv)
+
+    if sys.platform == "win32":
+        exe = Path("Scripts") / "python.exe"
+    else:
+        exe = Path("bin") / "python"
+
+    args = [
+        *prefix,
+        str(venv / exe),
+        "-m",
+        module,
+        *cmd,
+    ]
+
+    print("Running command:", " ".join(args))
+    subprocess.check_call(args)
+
+
+def install_requirements(venv: Path) -> None:
+    run_in_venv(venv, "pip", ["install", "--upgrade", "pip"])
+    run_in_venv(venv, "pip", ["install", "-r", "requirements.txt"])
+
+
+def _main(
+    fork: str,
+    ref: str,
+    machine: str,
+    benchmarks: str,
+    flags: str,
+    force: bool,
+    pgo: bool,
+    perf: bool,
+    pystats: bool,
+    force_32bit: bool,
+    run_id: str | None = None,
+):
+    if force_32bit and sys.platform != "win32":
+        raise RuntimeError("32-bit builds are only supported on Windows")
+    if perf and not sys.platform.startswith("linux"):
+        raise RuntimeError("perf profiling is only supported on Linux")
+    if pystats and not sys.platform.startswith("linux"):
+        raise RuntimeError("Pystats is only supported on Linux")
+
+    venv = Path("venv")
+    create_venv(venv)
+    install_requirements(venv)
+
+    # Now that we've installed the full bench_runner library,
+    # continue on in a new process...
+
+    args = ["workflow", fork, ref, machine, benchmarks, flags]
+    if force:
+        args.append("--force")
+    if pgo:
+        args.append("--pgo")
+    if perf:
+        args.append("--perf")
+    if pystats:
+        args.append("--pystats")
+    if force_32bit:
+        args.append("--32bit")
+    if run_id:
+        args.extend(["--run_id", run_id])
+
+    run_in_venv(venv, "bench_runner", args)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="""
+        Run the full compile/benchmark workflow.
+        """,
+    )
+    parser.add_argument("fork", help="The fork of CPython")
+    parser.add_argument("ref", help="The git ref in the fork")
+    parser.add_argument(
+        "machine",
+        help="The machine to run the benchmarks on.",
+    )
+    parser.add_argument("benchmarks", help="The benchmarks to run")
+    parser.add_argument("flags", help="Configuration flags")
+    parser.add_argument("--force", action="store_true", help="Force a re-run")
+    parser.add_argument(
+        "--pgo",
+        action="store_true",
+        help="Build with profiling guided optimization",
+    )
+    parser.add_argument(
+        "--perf",
+        action="store_true",
+        help="Collect Linux perf profiling data (Linux only)",
+    )
+    parser.add_argument(
+        "--pystats",
+        action="store_true",
+        help="Enable Pystats (Linux only)",
+    )
+    parser.add_argument(
+        "--32bit",
+        action="store_true",
+        dest="force_32bit",
+        help="Do a 32-bit build (Windows only)",
+    )
+    parser.add_argument("--run_id", default=None, type=str, help="The github run id")
+    args = parser.parse_args()
+
+    _main(
+        args.fork,
+        args.ref,
+        args.machine,
+        args.benchmarks,
+        args.flags,
+        args.force,
+        args.pgo,
+        args.perf,
+        args.pystats,
+        args.force_32bit,
+        args.run_id,
+    )
+
+
+if __name__ == "__main__":
+    if sys.version_info[:2] < (3, 11):
+        print(
+            "The benchmarking infrastructure requires Python 3.11 or later.",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    main()
diff --git a/bench_runner/util.py b/bench_runner/util.py
index 59cbc867..d2142208 100644
--- a/bench_runner/util.py
+++ b/bench_runner/util.py
@@ -1,8 +1,9 @@
 import functools
-import hashlib
 import itertools
 import os
 from pathlib import Path
+import shutil
+import subprocess
 from typing import TypeAlias, Union
 
 
@@ -12,13 +13,6 @@
 PathLike: TypeAlias = Union[str, os.PathLike]
 
 
-def get_benchmark_hash() -> str:
-    hash = hashlib.sha256()
-    hash.update(os.environ["PYPERFORMANCE_HASH"].encode("ascii")[:7])
-    hash.update(os.environ["PYSTON_BENCHMARKS_HASH"].encode("ascii")[:7])
-    return hash.hexdigest()[:6]
-
-
 TYPE_TO_ICON = {
     "table": "📄",
     "time plot": "📈",
@@ -55,3 +49,24 @@ def has_any_element(iterable):
         return True  # If successful, the generator is not empty
     except StopIteration:
         return False  # If StopIteration is raised, the generator is empty
+
+
+def safe_which(cmd: str) -> str:
+    """
+    shutil, but raises a RuntimeError if the command is not found.
+    """
+    path = shutil.which(cmd)
+    if path is None:
+        raise RuntimeError(f"Command {cmd} not found in PATH")
+    return path
+
+
+def get_brew_prefix(command: str) -> str:
+    """
+    Get the prefix of the Homebrew installation.
+    """
+    try:
+        prefix = subprocess.check_output(["brew", "--prefix", command])
+    except subprocess.CalledProcessError:
+        raise RuntimeError(f"Unable to find brew installation prefix for {command}")
+    return prefix.decode("utf-8").strip()
diff --git a/tests/test_run_benchmarks.py b/tests/test_run_benchmarks.py
index 9d24ede3..99a14106 100644
--- a/tests/test_run_benchmarks.py
+++ b/tests/test_run_benchmarks.py
@@ -9,11 +9,11 @@
 import pytest
 
 
+from bench_runner import benchmark_definitions
 from bench_runner import git
 from bench_runner.scripts import generate_results
 from bench_runner.scripts import run_benchmarks
-from bench_runner.scripts import should_run
-from bench_runner import util
+from bench_runner.scripts import workflow
 
 
 DATA_PATH = Path(__file__).parent / "data"
@@ -32,8 +32,16 @@ def dummy(*args, **kwargs):
     monkeypatch.setattr(git, "get_git_merge_base", dummy)
 
 
+def hardcode_benchmark_hash(monkeypatch):
+    def dummy(*args, **kwargs):
+        return "215d35"
+
+    monkeypatch.setattr(benchmark_definitions, "get_benchmark_hash", dummy)
+
+
 def test_update_metadata(benchmarks_checkout, monkeypatch):
     dont_get_git_merge_base(monkeypatch)
+    hardcode_benchmark_hash(monkeypatch)
 
     shutil.copy(
         DATA_PATH
@@ -67,7 +75,9 @@ def test_update_metadata(benchmarks_checkout, monkeypatch):
     )
 
 
-def test_run_benchmarks(benchmarks_checkout):
+def test_run_benchmarks(benchmarks_checkout, monkeypatch):
+    hardcode_benchmark_hash(monkeypatch)
+
     shutil.copyfile(
         DATA_PATH / "bench_runner.toml", benchmarks_checkout / "bench_runner.toml"
     )
@@ -148,88 +158,88 @@ def test_run_benchmarks(benchmarks_checkout):
     assert returncode == 1
 
 
-def test_should_run_exists_noforce(benchmarks_checkout, capsys, monkeypatch):
+def test_should_run_exists_noforce(benchmarks_checkout, monkeypatch):
+    hardcode_benchmark_hash(monkeypatch)
     repo = _copy_repo(benchmarks_checkout)
     monkeypatch.chdir(repo)
 
-    should_run._main(
+    result = workflow.should_run(
         False,
         "python",
         "main",
         "linux-x86_64-linux",
         False,
-        ",,",
+        [],
         benchmarks_checkout / "cpython",
         repo / "results",
     )
 
-    captured = capsys.readouterr()
-    assert captured.out.strip() == "should_run=false"
+    assert result is False
     assert (repo / "results" / "bm-20220323-3.10.4-9d38120").is_dir()
 
 
-def test_should_run_diff_machine_noforce(benchmarks_checkout, capsys, monkeypatch):
+def test_should_run_diff_machine_noforce(benchmarks_checkout, monkeypatch):
     repo = _copy_repo(benchmarks_checkout)
     monkeypatch.chdir(repo)
 
-    should_run._main(
+    result = workflow.should_run(
         False,
         "python",
         "main",
         "darwin-x86_64-darwin",
         False,
-        ",,",
+        [],
         benchmarks_checkout / "cpython",
         repo / "results",
     )
 
-    captured = capsys.readouterr()
-    assert captured.out.strip() == "should_run=true"
+    assert result is True
     assert len(list((repo / "results" / "bm-20220323-3.10.4-9d38120").iterdir())) == 1
 
 
-def test_should_run_all_noforce(benchmarks_checkout, capsys, monkeypatch):
+def test_should_run_all_noforce(benchmarks_checkout, monkeypatch):
     repo = _copy_repo(benchmarks_checkout)
     monkeypatch.chdir(repo)
 
-    should_run._main(
+    result = workflow.should_run(
         False,
         "python",
         "main",
         "all",
         False,
-        ",,",
+        [],
         benchmarks_checkout / "cpython",
         repo / "results",
     )
 
-    captured = capsys.readouterr()
-    assert captured.out.strip() == "should_run=true"
+    assert result is True
     assert len(list((repo / "results" / "bm-20220323-3.10.4-9d38120").iterdir())) == 1
 
 
-def test_should_run_noexists_noforce(benchmarks_checkout, capsys, monkeypatch):
+def test_should_run_noexists_noforce(benchmarks_checkout, monkeypatch):
+    hardcode_benchmark_hash(monkeypatch)
     repo = _copy_repo(benchmarks_checkout)
     monkeypatch.chdir(repo)
     shutil.rmtree(repo / "results" / "bm-20220323-3.10.4-9d38120")
 
-    should_run._main(
+    result = workflow.should_run(
         False,
         "python",
         "main",
         "linux-x86_64-linux",
         False,
-        ",,",
+        [],
         benchmarks_checkout / "cpython",
         repo / "results",
     )
 
-    captured = capsys.readouterr()
-    assert captured.out.strip() == "should_run=true"
+    assert result is True
     assert not (repo / "results" / "bm-20220323-3.10.4-9d38120").is_dir()
 
 
-def test_should_run_exists_force(benchmarks_checkout, capsys, monkeypatch):
+def test_should_run_exists_force(benchmarks_checkout, monkeypatch):
+    hardcode_benchmark_hash(monkeypatch)
+
     repo = _copy_repo(benchmarks_checkout)
     monkeypatch.chdir(repo)
 
@@ -242,19 +252,18 @@ def remove(repo, path):
     monkeypatch.setattr(git, "remove", remove)
 
     generate_results._main(repo, force=False, bases=["3.11.0b3"])
-    should_run._main(
+    result = workflow.should_run(
         True,
         "python",
         "main",
         "linux-x86_64-linux",
         False,
-        ",,",
+        [],
         benchmarks_checkout / "cpython",
         repo / "results",
     )
 
-    captured = capsys.readouterr()
-    assert captured.out.splitlines()[-1].strip() == "should_run=true"
+    assert result is True
     assert (repo / "results" / "bm-20220323-3.10.4-9d38120").is_dir()
     assert set(x.name for x in removed_paths) == {
         "bm-20220323-linux-x86_64-python-main-3.10.4-9d38120-vs-3.11.0b3.svg",
@@ -263,24 +272,24 @@ def remove(repo, path):
     }
 
 
-def test_should_run_noexists_force(benchmarks_checkout, capsys, monkeypatch):
+def test_should_run_noexists_force(benchmarks_checkout, monkeypatch):
+    hardcode_benchmark_hash(monkeypatch)
     repo = _copy_repo(benchmarks_checkout)
     monkeypatch.chdir(repo)
     shutil.rmtree(repo / "results" / "bm-20220323-3.10.4-9d38120")
 
-    should_run._main(
+    result = workflow.should_run(
         True,
         "python",
         "main",
         "linux-x86_64-linux",
         False,
-        ",,",
+        [],
         benchmarks_checkout / "cpython",
         repo / "results",
     )
 
-    captured = capsys.readouterr()
-    assert captured.out.strip() == "should_run=true"
+    assert result is True
     assert not (repo / "results" / "bm-20220323-3.10.4-9d38120").is_dir()
 
 
@@ -292,13 +301,13 @@ def test_should_run_checkout_failed(tmp_path, capsys, monkeypatch):
     subprocess.check_call(["git", "init"], cwd=cpython_path)
 
     with pytest.raises(SystemExit):
-        should_run._main(
+        workflow.should_run(
             True,
             "python",
             "main",
             "linux-x86_64-linux",
             False,
-            ",,",
+            [],
             cpython_path,
             repo / "results",
         )
@@ -350,4 +359,4 @@ def test_run_benchmarks_flags(benchmarks_checkout):
 
 
 def test_get_benchmark_hash():
-    assert util.get_benchmark_hash() == "215d35"
+    assert benchmark_definitions.get_benchmark_hash() == "dcfded"

From aa60ce0b50915ba87cc1ada42a35ba660c55a3b3 Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Mon, 31 Mar 2025 09:23:43 -0400
Subject: [PATCH 02/30] Remove _should_run checks in _pystats.src.yml

---
 bench_runner/templates/_pystats.src.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/bench_runner/templates/_pystats.src.yml b/bench_runner/templates/_pystats.src.yml
index 6bae3f0f..8ef1b143 100644
--- a/bench_runner/templates/_pystats.src.yml
+++ b/bench_runner/templates/_pystats.src.yml
@@ -51,11 +51,9 @@ jobs:
         with:
           python-version: "3.11"
       - name: Build CPython and run pyperformance benchmarks
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
         run: |
           python workflow_bootstrap.py --pystats ${{ inputs.fork }} ${{ inputs.ref }} all ${{ inputs.benchmarks || 'all' }} ${{ env.flags }} --run_id ${{ github.run_id }}
       - name: Pull benchmarking
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
         run: |
           # Another benchmarking task may have created results for the same
           # commit while the above was running. This "magic" incantation means
@@ -63,7 +61,6 @@ jobs:
           # just pulled in in that case.
           git pull -s recursive -X ours --autostash --rebase
       - name: Add data to repo
-        if: ${{ steps.should_run.outputs.should_run != 'false' }}
         uses: EndBug/add-and-commit@v9
         with:
           add: results

From 30baeea5b8e2c8b788c550fd020ba946ea55ce00 Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Mon, 31 Mar 2025 11:55:55 -0400
Subject: [PATCH 03/30] Update minimum Python version

---
 pyproject.toml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index cbe9ff70..fe033f2c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -9,7 +9,7 @@ authors = [
 ]
 description = "Faster CPython's benchmarking runner utilities"
 readme = "README.md"
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 license = {text = "BSD-3-Clause"}
 classifiers = [
     "Programming Language :: Python :: 3",
@@ -21,7 +21,7 @@ dependencies = [
     "rich-argparse==1.7.0",
     "ruamel.yaml==0.18.10",
     "scour==0.38.2",
-    "tomli==2.0.1; python_version < '3.11'",
+    "tomli==2.0.1",
     "wheel",
 ]
 dynamic = ["version"]

From 7ff47c72b14f31b329662ecc851824ad1270aebc Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Mon, 31 Mar 2025 14:25:45 -0400
Subject: [PATCH 04/30] Simplify bootstrapping

---
 bench_runner/templates/workflow_bootstrap.py | 96 ++------------------
 1 file changed, 7 insertions(+), 89 deletions(-)

diff --git a/bench_runner/templates/workflow_bootstrap.py b/bench_runner/templates/workflow_bootstrap.py
index d6608359..cb43789d 100644
--- a/bench_runner/templates/workflow_bootstrap.py
+++ b/bench_runner/templates/workflow_bootstrap.py
@@ -6,7 +6,6 @@
 # the error message that the version of Python is too old.
 
 
-import argparse
 from pathlib import Path
 import shutil
 import subprocess
@@ -54,26 +53,7 @@ def install_requirements(venv: Path) -> None:
     run_in_venv(venv, "pip", ["install", "-r", "requirements.txt"])
 
 
-def _main(
-    fork: str,
-    ref: str,
-    machine: str,
-    benchmarks: str,
-    flags: str,
-    force: bool,
-    pgo: bool,
-    perf: bool,
-    pystats: bool,
-    force_32bit: bool,
-    run_id: str | None = None,
-):
-    if force_32bit and sys.platform != "win32":
-        raise RuntimeError("32-bit builds are only supported on Windows")
-    if perf and not sys.platform.startswith("linux"):
-        raise RuntimeError("perf profiling is only supported on Linux")
-    if pystats and not sys.platform.startswith("linux"):
-        raise RuntimeError("Pystats is only supported on Linux")
-
+def main():
     venv = Path("venv")
     create_venv(venv)
     install_requirements(venv)
@@ -81,75 +61,13 @@ def _main(
     # Now that we've installed the full bench_runner library,
     # continue on in a new process...
 
-    args = ["workflow", fork, ref, machine, benchmarks, flags]
-    if force:
-        args.append("--force")
-    if pgo:
-        args.append("--pgo")
-    if perf:
-        args.append("--perf")
-    if pystats:
-        args.append("--pystats")
-    if force_32bit:
-        args.append("--32bit")
-    if run_id:
-        args.extend(["--run_id", run_id])
-
-    run_in_venv(venv, "bench_runner", args)
-
+    last_arg = sys.argv.find("workflow_bootstrap.py")
+    if last_arg == -1:
+        raise ValueError(
+            "The script should be run from the command line with the workflow_bootstrap.py argument"
+        )
 
-def main():
-    parser = argparse.ArgumentParser(
-        description="""
-        Run the full compile/benchmark workflow.
-        """,
-    )
-    parser.add_argument("fork", help="The fork of CPython")
-    parser.add_argument("ref", help="The git ref in the fork")
-    parser.add_argument(
-        "machine",
-        help="The machine to run the benchmarks on.",
-    )
-    parser.add_argument("benchmarks", help="The benchmarks to run")
-    parser.add_argument("flags", help="Configuration flags")
-    parser.add_argument("--force", action="store_true", help="Force a re-run")
-    parser.add_argument(
-        "--pgo",
-        action="store_true",
-        help="Build with profiling guided optimization",
-    )
-    parser.add_argument(
-        "--perf",
-        action="store_true",
-        help="Collect Linux perf profiling data (Linux only)",
-    )
-    parser.add_argument(
-        "--pystats",
-        action="store_true",
-        help="Enable Pystats (Linux only)",
-    )
-    parser.add_argument(
-        "--32bit",
-        action="store_true",
-        dest="force_32bit",
-        help="Do a 32-bit build (Windows only)",
-    )
-    parser.add_argument("--run_id", default=None, type=str, help="The github run id")
-    args = parser.parse_args()
-
-    _main(
-        args.fork,
-        args.ref,
-        args.machine,
-        args.benchmarks,
-        args.flags,
-        args.force,
-        args.pgo,
-        args.perf,
-        args.pystats,
-        args.force_32bit,
-        args.run_id,
-    )
+    run_in_venv(venv, "bench_runner", ["workflow", *sys.argv[last_arg + 1 :]])
 
 
 if __name__ == "__main__":

From 1feab3b2d23e324f1d28f2192804eb66dceb5748 Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Mon, 31 Mar 2025 14:28:54 -0400
Subject: [PATCH 05/30] Bugfixes

---
 bench_runner/scripts/workflow.py             | 7 +++++++
 bench_runner/templates/workflow_bootstrap.py | 6 ++----
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/bench_runner/scripts/workflow.py b/bench_runner/scripts/workflow.py
index 1a6003a3..5d39d102 100644
--- a/bench_runner/scripts/workflow.py
+++ b/bench_runner/scripts/workflow.py
@@ -257,6 +257,13 @@ def _main(
     venv = Path("venv")
     cpython = Path("cpython")
 
+    if force_32bit and sys.platform != "win32":
+        raise RuntimeError("32-bit builds are only supported on Windows")
+    if perf and not sys.platform.startswith("linux"):
+        raise RuntimeError("perf profiling is only supported on Linux")
+    if pystats and not sys.platform.startswith("linux"):
+        raise RuntimeError("Pystats is only supported on Linux")
+
     checkout_cpython(fork, ref, cpython)
 
     if not should_run(force, fork, ref, machine, False, flags, cpython=cpython):
diff --git a/bench_runner/templates/workflow_bootstrap.py b/bench_runner/templates/workflow_bootstrap.py
index cb43789d..af628b4b 100644
--- a/bench_runner/templates/workflow_bootstrap.py
+++ b/bench_runner/templates/workflow_bootstrap.py
@@ -61,11 +61,9 @@ def main():
     # Now that we've installed the full bench_runner library,
     # continue on in a new process...
 
-    last_arg = sys.argv.find("workflow_bootstrap.py")
+    last_arg = sys.argv.index("workflow_bootstrap.py")
     if last_arg == -1:
-        raise ValueError(
-            "The script should be run from the command line with the workflow_bootstrap.py argument"
-        )
+        raise ValueError("Couldn't parse command line")
 
     run_in_venv(venv, "bench_runner", ["workflow", *sys.argv[last_arg + 1 :]])
 

From 3dc10578724a0e28c0e1553706b7ad1528d9e96e Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Mon, 7 Apr 2025 13:30:58 -0400
Subject: [PATCH 06/30] Update bench_runner/scripts/workflow.py

---
 bench_runner/scripts/workflow.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bench_runner/scripts/workflow.py b/bench_runner/scripts/workflow.py
index 5d39d102..6f84cc4e 100644
--- a/bench_runner/scripts/workflow.py
+++ b/bench_runner/scripts/workflow.py
@@ -223,7 +223,7 @@ def tune_system(venv: PathLike, perf: bool) -> None:
                 "sudo",
                 "bash",
                 "-c",
-                "echo 0 > /proc/sys/kernel/perf_event_max_sample_rate",
+                "echo 100000 > /proc/sys/kernel/perf_event_max_sample_rate",
             ]
         )
 

From 3dbc349b71ca1d6b6a5e5ced396832f8d23a9eba Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Tue, 8 Apr 2025 08:51:06 -0400
Subject: [PATCH 07/30] Address comments from the PR

---
 bench_runner/scripts/workflow.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bench_runner/scripts/workflow.py b/bench_runner/scripts/workflow.py
index 6f84cc4e..0f4b6133 100644
--- a/bench_runner/scripts/workflow.py
+++ b/bench_runner/scripts/workflow.py
@@ -150,7 +150,7 @@ def compile_unix(cpython: PathLike, flags: list[str], pgo: bool, pystats: bool)
 
     args = []
     if pystats:
-        args.append("--with-pystats")
+        args.append("--enable-pystats")
     if pgo:
         args.extend(["--enable-optimizations", "--with-lto=full"])
     if "PYTHON_UOPS" in flags:
@@ -290,7 +290,7 @@ def _main(
         if Path(".debug").exists():
             shutil.rmtree(".debug")
 
-        pystats_dir = Path("/tmp") / "pystats"
+        pystats_dir = Path("/tmp") / "py_stats"
         if pystats:
             shutil.rmtree(pystats_dir, ignore_errors=True)
             pystats_dir.mkdir(parents=True)

From 859b068a10186826da46003450c63e7f421b804f Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Tue, 8 Apr 2025 08:57:44 -0400
Subject: [PATCH 08/30] Port #388 to Python

---
 bench_runner/scripts/workflow.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bench_runner/scripts/workflow.py b/bench_runner/scripts/workflow.py
index 0f4b6133..6931b7f9 100644
--- a/bench_runner/scripts/workflow.py
+++ b/bench_runner/scripts/workflow.py
@@ -163,6 +163,7 @@ def compile_unix(cpython: PathLike, flags: list[str], pgo: bool, pystats: bool)
         args.append("--disable-gil")
     if "CLANG" in flags:
         args.append("--with-tail-call-interp")
+    args.append("--enable-option-checking=fatal")
 
     with contextlib.chdir(cpython):
         subprocess.check_call(["./configure", *args], env=env)

From eb43dc3f95051eef1d61c7cd5e5cae473e7b093e Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Tue, 8 Apr 2025 08:59:54 -0400
Subject: [PATCH 09/30] Port #389 to Python

---
 bench_runner/scripts/workflow.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bench_runner/scripts/workflow.py b/bench_runner/scripts/workflow.py
index 6931b7f9..2a8f90f0 100644
--- a/bench_runner/scripts/workflow.py
+++ b/bench_runner/scripts/workflow.py
@@ -5,6 +5,7 @@
 import contextlib
 import os
 from pathlib import Path
+import shlex
 import shutil
 import subprocess
 import sys
@@ -164,6 +165,8 @@ def compile_unix(cpython: PathLike, flags: list[str], pgo: bool, pystats: bool)
     if "CLANG" in flags:
         args.append("--with-tail-call-interp")
     args.append("--enable-option-checking=fatal")
+    if configure_flags := os.environ.get("PYTHON_CONFIGURE_FLAGS"):
+        args.extend(shlex.split(configure_flags))
 
     with contextlib.chdir(cpython):
         subprocess.check_call(["./configure", *args], env=env)

From 4f84f80898ac42f9681b151bce2881e8a1b981d9 Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Tue, 8 Apr 2025 09:02:32 -0400
Subject: [PATCH 10/30] Port #390 to Python

---
 bench_runner/scripts/workflow.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/bench_runner/scripts/workflow.py b/bench_runner/scripts/workflow.py
index 2a8f90f0..4313bb1c 100644
--- a/bench_runner/scripts/workflow.py
+++ b/bench_runner/scripts/workflow.py
@@ -219,7 +219,11 @@ def tune_system(venv: PathLike, perf: bool) -> None:
     if not sys.platform.startswith("linux"):
         return
 
-    run_in_venv(venv, "pyperf", ["system", perf and "reset" or "tune"], sudo=True)
+    args = ["system", perf and "reset" or "tune"]
+    if cpu_affinity := os.environ.get("CPU_AFFINITY"):
+        args.append(f'--affinity="{cpu_affinity}"')
+
+    run_in_venv(venv, "pyperf", args, sudo=True)
 
     if perf:
         subprocess.check_call(

From 7c9d6497c3f1e2a4069b4687b1b33c13af2ae2f4 Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Tue, 8 Apr 2025 11:48:53 -0400
Subject: [PATCH 11/30] Bugfix for machines that don't match up with ordering

---
 bench_runner/scripts/generate_results.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/bench_runner/scripts/generate_results.py b/bench_runner/scripts/generate_results.py
index f8f55361..cfb101b3 100644
--- a/bench_runner/scripts/generate_results.py
+++ b/bench_runner/scripts/generate_results.py
@@ -126,7 +126,11 @@ def sort_runner_names(runner_names: Iterable[str]) -> list[str]:
     def sorter(val):
         if val is None:
             return ()
-        return order.index(val.split()[0]), val
+        try:
+            idx = order.index(val.split()[0])
+        except ValueError:
+            idx = -1
+        return idx, val
 
     return sorted(runner_names, key=sorter)
 

From 89150359d444c75df1871507c528ed3d9e5f4aba Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Wed, 9 Apr 2025 08:11:30 -0400
Subject: [PATCH 12/30] Don't use my personal fork

---
 bench_runner/benchmark_definitions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bench_runner/benchmark_definitions.py b/bench_runner/benchmark_definitions.py
index a5202630..5ea214b4 100644
--- a/bench_runner/benchmark_definitions.py
+++ b/bench_runner/benchmark_definitions.py
@@ -19,7 +19,7 @@ class BenchmarkRepo:
 BENCHMARK_REPOS = [
     BenchmarkRepo(
         "56d12a8fd7cc1432835965d374929bfa7f6f7a07",
-        "https://github.com/mdboom/pyperformance.git",
+        "https://github.com/python/pyperformance.git",
         "pyperformance",
     ),
     BenchmarkRepo(

From 27dc96fda20172c53700e6a6245d03ade7b7cb4b Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Wed, 9 Apr 2025 17:53:10 -0400
Subject: [PATCH 13/30] Extend PATH on Darwin for clang

---
 bench_runner/scripts/workflow.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bench_runner/scripts/workflow.py b/bench_runner/scripts/workflow.py
index 4313bb1c..910b335a 100644
--- a/bench_runner/scripts/workflow.py
+++ b/bench_runner/scripts/workflow.py
@@ -141,6 +141,7 @@ def compile_unix(cpython: PathLike, flags: list[str], pgo: bool, pystats: bool)
             env["LLVM_PROFDATA"] = util.safe_which("llvm-profdata-19")
         elif sys.platform == "darwin":
             llvm_prefix = util.get_brew_prefix("llvm")
+            env["PATH"] = f"{llvm_prefix}/bin:{env['PATH']}"
             env["CC"] = f"{llvm_prefix}/bin/clang"
             env["LDFLAGS"] = f"-L{llvm_prefix}/lib"
             env["CFLAGS"] = f"-I{llvm_prefix}/include"

From a8b142f952f817231ba2db26115317004d425569 Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Thu, 10 Apr 2025 10:39:27 -0400
Subject: [PATCH 14/30] Limit cores on specific machines

---
 README.md                        |  6 ++++++
 bench_runner/config.py           | 10 ++++++++++
 bench_runner/result.py           |  3 +--
 bench_runner/runners.py          | 11 +++++++++--
 bench_runner/scripts/workflow.py | 10 +++++++++-
 5 files changed, 35 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 416fc158..816f3f27 100644
--- a/README.md
+++ b/README.md
@@ -93,6 +93,12 @@ If you don't want a machine to be included when the user selects "machine == 'al
 include_in_all = false
 ```
 
+You may limit the number of cores used to build Python with the `use_cores` option. This may be necessary, for example, on cloud VMs.
+
+```
+use_cores = 2
+```
+
 ### Try a benchmarking run
 
 There are instructions for running a benchmarking action already in the `README.md` of your repo. Look there and give it a try!
diff --git a/bench_runner/config.py b/bench_runner/config.py
index 1ebd6b2d..62d33bb1 100644
--- a/bench_runner/config.py
+++ b/bench_runner/config.py
@@ -4,6 +4,7 @@
 
 import functools
 from pathlib import Path
+from typing import Any
 
 try:
     import tomllib
@@ -11,9 +12,18 @@
     import tomli as tomllib  # type: ignore
 
 
+from . import runners
+
+
 @functools.cache
 def get_bench_runner_config(
     filepath: Path | str = Path("bench_runner.toml"),
 ):
     with Path(filepath).open("rb") as fd:
         return tomllib.load(fd)
+
+
+def get_config_for_current_runner() -> dict[str, Any]:
+    config = get_bench_runner_config()
+    runner = runners.get_runner_for_hostname()
+    return config.get("runners", {}).get(runner.nickname, {})
diff --git a/bench_runner/result.py b/bench_runner/result.py
index 2a47b759..9b3274a0 100644
--- a/bench_runner/result.py
+++ b/bench_runner/result.py
@@ -9,7 +9,6 @@
 from operator import itemgetter
 from pathlib import Path
 import re
-import socket
 import subprocess
 import sys
 from typing import Any, Callable, Iterable, Sequence
@@ -524,7 +523,7 @@ def from_scratch(
         flags: Iterable[str] | None = None,
     ) -> "Result":
         result = cls(
-            _clean(runners.get_nickname_for_hostname(socket.gethostname())),
+            _clean(runners.get_nickname_for_hostname()),
             _clean(_get_architecture(python)),
             _clean_for_url(fork),
             _clean(ref[:20]),
diff --git a/bench_runner/runners.py b/bench_runner/runners.py
index dc96b4b3..f8b4e5eb 100644
--- a/bench_runner/runners.py
+++ b/bench_runner/runners.py
@@ -3,6 +3,7 @@
 
 import functools
 import os
+import socket
 
 
 from . import config
@@ -80,13 +81,19 @@ def get_runners_by_nickname() -> dict[str, Runner]:
     return {x.nickname: x for x in get_runners()}
 
 
-def get_nickname_for_hostname(hostname: str) -> str:
+def get_nickname_for_hostname(hostname: str | None = None) -> str:
     # The envvar BENCHMARK_MACHINE_NICKNAME is used to override the machine that
     # results are reported for.
     if "BENCHMARK_MACHINE_NICKNAME" in os.environ:
         return os.environ["BENCHMARK_MACHINE_NICKNAME"]
-    return get_runners_by_hostname().get(hostname, unknown_runner).nickname
+    return get_runner_for_hostname(hostname).nickname
 
 
 def get_runner_by_nickname(nickname: str) -> Runner:
     return get_runners_by_nickname().get(nickname, unknown_runner)
+
+
+def get_runner_for_hostname(hostname: str | None = None) -> Runner:
+    if hostname is None:
+        hostname = socket.gethostname()
+    return get_runners_by_hostname().get(hostname, unknown_runner)
diff --git a/bench_runner/scripts/workflow.py b/bench_runner/scripts/workflow.py
index 910b335a..74d86a7b 100644
--- a/bench_runner/scripts/workflow.py
+++ b/bench_runner/scripts/workflow.py
@@ -15,6 +15,7 @@
 
 
 from bench_runner import benchmark_definitions
+from bench_runner import config
 from bench_runner import flags as mflags
 from bench_runner import git
 from bench_runner.result import has_result
@@ -132,6 +133,7 @@ def checkout_benchmarks():
 
 def compile_unix(cpython: PathLike, flags: list[str], pgo: bool, pystats: bool) -> None:
     cpython = Path(cpython)
+    cfg = config.get_config_for_current_runner()
 
     env = os.environ.copy()
     if "CLANG" in flags:
@@ -169,9 +171,15 @@ def compile_unix(cpython: PathLike, flags: list[str], pgo: bool, pystats: bool)
     if configure_flags := os.environ.get("PYTHON_CONFIGURE_FLAGS"):
         args.extend(shlex.split(configure_flags))
 
+    make_args = []
+    if cores := cfg.get("use_cores", None):
+        make_args.extend(["-j", str(cores)])
+    else:
+        make_args.extend(["-j"])
+
     with contextlib.chdir(cpython):
         subprocess.check_call(["./configure", *args], env=env)
-        subprocess.check_call(["make", "-j"], env=env)
+        subprocess.check_call(["make", *make_args], env=env)
 
 
 def compile_windows(

From b0ce77564983e87d5f5490d924f632a2068da7a9 Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Thu, 10 Apr 2025 11:09:42 -0400
Subject: [PATCH 15/30] Bugfix

---
 bench_runner/config.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/bench_runner/config.py b/bench_runner/config.py
index 62d33bb1..2fffc7ae 100644
--- a/bench_runner/config.py
+++ b/bench_runner/config.py
@@ -26,4 +26,7 @@ def get_bench_runner_config(
 def get_config_for_current_runner() -> dict[str, Any]:
     config = get_bench_runner_config()
     runner = runners.get_runner_for_hostname()
-    return config.get("runners", {}).get(runner.nickname, {})
+    all_runners = config.get("runners", [])
+    if len(all_runners) >= 1:
+        return all_runners[0].get(runner.nickname, {})
+    return {}

From 4eca84e9825fa3d86fe03f23015560874fe61dab Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Thu, 10 Apr 2025 12:02:18 -0400
Subject: [PATCH 16/30] Investigate Windows failure

---
 bench_runner/scripts/workflow.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bench_runner/scripts/workflow.py b/bench_runner/scripts/workflow.py
index 74d86a7b..8d62919f 100644
--- a/bench_runner/scripts/workflow.py
+++ b/bench_runner/scripts/workflow.py
@@ -40,6 +40,7 @@ def get_exe_path(cpython: Path, flags: list[str], force_32bit: bool) -> Path:
     elif sys.platform.startswith("win32"):
         build_dir = get_windows_build_dir(force_32bit)
         if "NOGIL" in flags:
+            print(list(build_dir.glob("*.exe")))
             exe = next(build_dir.glob("python3.*.exe"))
         else:
             exe = "python.exe"

From 37c68b60aadbb0f18dd1564d682beabbcfeeb8f0 Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Thu, 10 Apr 2025 13:17:58 -0400
Subject: [PATCH 17/30] Windows bugfixes

---
 bench_runner/scripts/workflow.py          | 2 +-
 bench_runner/templates/_benchmark.src.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/bench_runner/scripts/workflow.py b/bench_runner/scripts/workflow.py
index 8d62919f..b3e67a2f 100644
--- a/bench_runner/scripts/workflow.py
+++ b/bench_runner/scripts/workflow.py
@@ -38,7 +38,7 @@ def get_exe_path(cpython: Path, flags: list[str], force_32bit: bool) -> Path:
     elif sys.platform == "darwin":
         return cpython / "python.exe"
     elif sys.platform.startswith("win32"):
-        build_dir = get_windows_build_dir(force_32bit)
+        build_dir = cpython / get_windows_build_dir(force_32bit)
         if "NOGIL" in flags:
             print(list(build_dir.glob("*.exe")))
             exe = next(build_dir.glob("python3.*.exe"))
diff --git a/bench_runner/templates/_benchmark.src.yml b/bench_runner/templates/_benchmark.src.yml
index 2939c9bc..9ea751aa 100644
--- a/bench_runner/templates/_benchmark.src.yml
+++ b/bench_runner/templates/_benchmark.src.yml
@@ -74,7 +74,7 @@ jobs:
       - name: Building Python and running pyperformance
         shell: cmd
         run: |
-          python workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.machine }} ${{ inputs.benchmarks || 'all' }} "${{ env.flags }}" ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }} %BUILD_FLAGS%
+          python workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.machine }} ${{ inputs.benchmarks || 'all' }} "${{ env.flags }}" ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }}
       # Pull again, since another job may have committed results in the meantime
       - name: Pull benchmarking
         run: |

From 6600630deb3cb4d810c27002867230f33aac671b Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Thu, 10 Apr 2025 13:46:27 -0400
Subject: [PATCH 18/30] Fix Windows again

---
 bench_runner/scripts/workflow.py             | 75 ++++++++++----------
 bench_runner/templates/workflow_bootstrap.py |  2 +-
 bench_runner/util.py                         | 17 ++++-
 3 files changed, 54 insertions(+), 40 deletions(-)

diff --git a/bench_runner/scripts/workflow.py b/bench_runner/scripts/workflow.py
index b3e67a2f..8ed5dbdf 100644
--- a/bench_runner/scripts/workflow.py
+++ b/bench_runner/scripts/workflow.py
@@ -33,20 +33,18 @@ def get_windows_build_dir(force_32bit: bool) -> Path:
 
 
 def get_exe_path(cpython: Path, flags: list[str], force_32bit: bool) -> Path:
-    if sys.platform.startswith("linux"):
-        return cpython / "python"
-    elif sys.platform == "darwin":
-        return cpython / "python.exe"
-    elif sys.platform.startswith("win32"):
-        build_dir = cpython / get_windows_build_dir(force_32bit)
-        if "NOGIL" in flags:
-            print(list(build_dir.glob("*.exe")))
-            exe = next(build_dir.glob("python3.*.exe"))
-        else:
-            exe = "python.exe"
-        return cpython / get_windows_build_dir(force_32bit) / exe
-    else:
-        raise RuntimeError(f"Unsupported platform: {sys.platform}")
+    match util.get_simple_platform():
+        case "linux":
+            return cpython / "python"
+        case "macos":
+            return cpython / "python.exe"
+        case "windows":
+            build_dir = cpython / get_windows_build_dir(force_32bit)
+            if "NOGIL" in flags:
+                exe = next(build_dir.glob("python3.*.exe"))
+            else:
+                exe = build_dir / "python.exe"
+            return exe
 
 
 def run_in_venv(
@@ -54,7 +52,7 @@ def run_in_venv(
 ) -> None:
     venv = Path(venv)
 
-    if sys.platform == "win32":
+    if util.get_simple_platform() == "windows":
         exe = Path("Scripts") / "python.exe"
     else:
         exe = Path("bin") / "python"
@@ -138,18 +136,19 @@ def compile_unix(cpython: PathLike, flags: list[str], pgo: bool, pystats: bool)
 
     env = os.environ.copy()
     if "CLANG" in flags:
-        if sys.platform.startswith("linux"):
-            env["CC"] = util.safe_which("clang-19")
-            env["LLVM_AR"] = util.safe_which("llvm-ar-19")
-            env["LLVM_PROFDATA"] = util.safe_which("llvm-profdata-19")
-        elif sys.platform == "darwin":
-            llvm_prefix = util.get_brew_prefix("llvm")
-            env["PATH"] = f"{llvm_prefix}/bin:{env['PATH']}"
-            env["CC"] = f"{llvm_prefix}/bin/clang"
-            env["LDFLAGS"] = f"-L{llvm_prefix}/lib"
-            env["CFLAGS"] = f"-I{llvm_prefix}/include"
-
-    if sys.platform == "darwin":
+        match util.get_simple_platform():
+            case "linux":
+                env["CC"] = util.safe_which("clang-19")
+                env["LLVM_AR"] = util.safe_which("llvm-ar-19")
+                env["LLVM_PROFDATA"] = util.safe_which("llvm-profdata-19")
+            case "macos":
+                llvm_prefix = util.get_brew_prefix("llvm")
+                env["PATH"] = f"{llvm_prefix}/bin:{env['PATH']}"
+                env["CC"] = f"{llvm_prefix}/bin/clang"
+                env["LDFLAGS"] = f"-L{llvm_prefix}/lib"
+                env["CFLAGS"] = f"-I{llvm_prefix}/include"
+
+    if util.get_simple_platform() == "macos":
         openssl_prefix = util.get_brew_prefix("openssl@1.1")
         env["PKG_CONFIG_PATH"] = f"{openssl_prefix}/lib/pkgconfig"
 
@@ -226,7 +225,7 @@ def install_pyperformance(venv: PathLike) -> None:
 
 def tune_system(venv: PathLike, perf: bool) -> None:
     # System tuning is Linux only
-    if not sys.platform.startswith("linux"):
+    if util.get_simple_platform() != "linux":
         return
 
     args = ["system", perf and "reset" or "tune"]
@@ -248,7 +247,7 @@ def tune_system(venv: PathLike, perf: bool) -> None:
 
 def reset_system(venv: PathLike) -> None:
     # System tuning is Linux only
-    if not sys.platform.startswith("linux"):
+    if util.get_simple_platform() != "linux":
         return
 
     run_in_venv(
@@ -274,12 +273,13 @@ def _main(
 ):
     venv = Path("venv")
     cpython = Path("cpython")
+    platform = util.get_simple_platform()
 
-    if force_32bit and sys.platform != "win32":
+    if force_32bit and platform != "windows":
         raise RuntimeError("32-bit builds are only supported on Windows")
-    if perf and not sys.platform.startswith("linux"):
+    if perf and platform != "linux":
         raise RuntimeError("perf profiling is only supported on Linux")
-    if pystats and not sys.platform.startswith("linux"):
+    if pystats and platform != "linux":
         raise RuntimeError("Pystats is only supported on Linux")
 
     checkout_cpython(fork, ref, cpython)
@@ -290,12 +290,11 @@ def _main(
 
     checkout_benchmarks()
 
-    if sys.platform.startswith("linux") or sys.platform == "darwin":
-        compile_unix(cpython, flags, pgo, pystats)
-    elif sys.platform == "win32":
-        compile_windows(cpython, flags, pgo, force_32bit)
-    else:
-        raise RuntimeError(f"Unsupported platform: {sys.platform}")
+    match platform:
+        case "linux" | "macos":
+            compile_unix(cpython, flags, pgo, pystats)
+        case "windows":
+            compile_windows(cpython, flags, pgo, force_32bit)
 
     # Print out the version of Python we built just so we can confirm it's the
     # right thing in the logs
diff --git a/bench_runner/templates/workflow_bootstrap.py b/bench_runner/templates/workflow_bootstrap.py
index af628b4b..f919c233 100644
--- a/bench_runner/templates/workflow_bootstrap.py
+++ b/bench_runner/templates/workflow_bootstrap.py
@@ -31,7 +31,7 @@ def run_in_venv(
 ) -> None:
     venv = Path(venv)
 
-    if sys.platform == "win32":
+    if sys.platform.startswith("win"):
         exe = Path("Scripts") / "python.exe"
     else:
         exe = Path("bin") / "python"
diff --git a/bench_runner/util.py b/bench_runner/util.py
index d2142208..63845c10 100644
--- a/bench_runner/util.py
+++ b/bench_runner/util.py
@@ -4,7 +4,8 @@
 from pathlib import Path
 import shutil
 import subprocess
-from typing import TypeAlias, Union
+import sys
+from typing import Literal, TypeAlias, Union
 
 
 from . import config
@@ -70,3 +71,17 @@ def get_brew_prefix(command: str) -> str:
     except subprocess.CalledProcessError:
         raise RuntimeError(f"Unable to find brew installation prefix for {command}")
     return prefix.decode("utf-8").strip()
+
+
+@functools.cache
+def get_simple_platform() -> Literal["linux", "macos", "windows"]:
+    """
+    Return a basic platform name: linux, macos or windows.
+    """
+    if sys.platform.startswith("linux"):
+        return "linux"
+    elif sys.platform == "darwin":
+        return "macos"
+    elif sys.platform.startswith("win"):
+        return "windows"
+    raise RuntimeError(f"Unsupported platform {sys.platform}.")

From 56852977d0ee91d101a4e99423383919b6737be0 Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Thu, 10 Apr 2025 14:03:51 -0400
Subject: [PATCH 19/30] Minor simplification

---
 bench_runner/scripts/workflow.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bench_runner/scripts/workflow.py b/bench_runner/scripts/workflow.py
index 8ed5dbdf..b0339dae 100644
--- a/bench_runner/scripts/workflow.py
+++ b/bench_runner/scripts/workflow.py
@@ -53,12 +53,12 @@ def run_in_venv(
     venv = Path(venv)
 
     if util.get_simple_platform() == "windows":
-        exe = Path("Scripts") / "python.exe"
+        exe = venv / "Scripts" / "python.exe"
     else:
-        exe = Path("bin") / "python"
+        exe = venv / "bin" / "python"
 
     args = [
-        str(venv / exe),
+        str(exe),
         "-m",
         module,
         *cmd,

From cead7aa83a8f638e0a28bc679b4b235d53f51e11 Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Thu, 10 Apr 2025 14:18:43 -0400
Subject: [PATCH 20/30] Add CHANGELOG.md

---
 CHANGELOG.md | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 CHANGELOG.md

diff --git a/CHANGELOG.md b/CHANGELOG.md
new file mode 100644
index 00000000..1ec4fb0f
--- /dev/null
+++ b/CHANGELOG.md
@@ -0,0 +1,14 @@
+## Unreleased
+
+## v2.0.0
+
+Most of the work has moved from GitHub Actions `.yml` files to Python code in `workflow.py`.
+In the future, this will allow supporting more workflow engines beyond just GitHub Actions.
+
+**Migration note**: After running `python -m bench_runner install` to update your local files, but sure to add the new `workflow_bootstrap.py` file to your git repository.
+
+### New configuration
+
+Runners have a new configuration `use_cores` to control the number of CPU cores
+used to build CPython. By default, this will use all available cores, but some
+Cloud VMs require using fewer.

From 68c07ab780ae7c0c24d004fa1978dbaed9611f40 Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Thu, 10 Apr 2025 14:48:42 -0400
Subject: [PATCH 21/30] Fix Clang

---
 bench_runner/scripts/workflow.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bench_runner/scripts/workflow.py b/bench_runner/scripts/workflow.py
index b0339dae..29438689 100644
--- a/bench_runner/scripts/workflow.py
+++ b/bench_runner/scripts/workflow.py
@@ -202,9 +202,9 @@ def compile_windows(
     if "CLANG" in flags:
         args.extend(
             [
-                r"/p:PlatformToolset=clangcl",
-                r"/p:LLVMInstallDir=C:\Program Files\LLVM",
-                r"/p:LLVMToolsVersion=19.1.6",
+                r'"/p:PlatformToolset=clangcl"',
+                r'"/p:LLVMInstallDir=C:\Program Files\LLVM"',
+                r'"/p:LLVMToolsVersion=19.1.6"',
                 "--tail-call-interp",
             ]
         )

From 2fd3804d74192e0b51680a53197da8761840403c Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Fri, 11 Apr 2025 12:14:21 -0400
Subject: [PATCH 22/30] Try to get CLANG on Windows working

---
 bench_runner/scripts/workflow.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/bench_runner/scripts/workflow.py b/bench_runner/scripts/workflow.py
index 29438689..bfff0a17 100644
--- a/bench_runner/scripts/workflow.py
+++ b/bench_runner/scripts/workflow.py
@@ -202,9 +202,9 @@ def compile_windows(
     if "CLANG" in flags:
         args.extend(
             [
-                r'"/p:PlatformToolset=clangcl"',
-                r'"/p:LLVMInstallDir=C:\Program Files\LLVM"',
-                r'"/p:LLVMToolsVersion=19.1.6"',
+                r"/p:PlatformToolset=clangcl",
+                r"/p:LLVMInstallDir=C:\Program Files\LLVM",
+                r"/p:LLVMToolsVersion=19.1.6",
                 "--tail-call-interp",
             ]
         )
@@ -214,7 +214,8 @@ def compile_windows(
             [
                 Path("PCbuild") / "build.bat",
                 *args,
-            ]
+            ],
+            shell=True,
         )
         shutil.copytree(get_windows_build_dir(force_32bit), "libs", dirs_exist_ok=True)
 

From 5b1c259e6c80481360e1e831d70797a44d6acbbb Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Fri, 11 Apr 2025 12:18:48 -0400
Subject: [PATCH 23/30] Testing Windows syntax

---
 bench_runner/scripts/workflow.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bench_runner/scripts/workflow.py b/bench_runner/scripts/workflow.py
index bfff0a17..0c8b54ed 100644
--- a/bench_runner/scripts/workflow.py
+++ b/bench_runner/scripts/workflow.py
@@ -202,7 +202,7 @@ def compile_windows(
     if "CLANG" in flags:
         args.extend(
             [
-                r"/p:PlatformToolset=clangcl",
+                r"'\"/p:PlatformToolset=clangcl\"'",
                 r"/p:LLVMInstallDir=C:\Program Files\LLVM",
                 r"/p:LLVMToolsVersion=19.1.6",
                 "--tail-call-interp",

From 9e29f83ce2be3d8ea79c908e8b75b75519de53fb Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Fri, 11 Apr 2025 12:55:12 -0400
Subject: [PATCH 24/30] Try to fix Windows

---
 bench_runner/scripts/workflow.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bench_runner/scripts/workflow.py b/bench_runner/scripts/workflow.py
index 0c8b54ed..0dffa04e 100644
--- a/bench_runner/scripts/workflow.py
+++ b/bench_runner/scripts/workflow.py
@@ -202,7 +202,7 @@ def compile_windows(
     if "CLANG" in flags:
         args.extend(
             [
-                r"'\"/p:PlatformToolset=clangcl\"'",
+                '"/p:PlatformToolset=clangcl"',
                 r"/p:LLVMInstallDir=C:\Program Files\LLVM",
                 r"/p:LLVMToolsVersion=19.1.6",
                 "--tail-call-interp",

From 82b1a5602d03955dd3f9010828e67f19157acfde Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Fri, 11 Apr 2025 12:58:30 -0400
Subject: [PATCH 25/30] Try to fix Windows

---
 bench_runner/scripts/workflow.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bench_runner/scripts/workflow.py b/bench_runner/scripts/workflow.py
index 0dffa04e..6383cc43 100644
--- a/bench_runner/scripts/workflow.py
+++ b/bench_runner/scripts/workflow.py
@@ -202,7 +202,7 @@ def compile_windows(
     if "CLANG" in flags:
         args.extend(
             [
-                '"/p:PlatformToolset=clangcl"',
+                '"/p:PlatformToolset`=clangcl"',
                 r"/p:LLVMInstallDir=C:\Program Files\LLVM",
                 r"/p:LLVMToolsVersion=19.1.6",
                 "--tail-call-interp",

From c1a0bc13543d91a4e0527b806aa3bc883f6a8f2b Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Fri, 11 Apr 2025 13:12:36 -0400
Subject: [PATCH 26/30] Please HELP with Windows syntax

---
 bench_runner/scripts/workflow.py          | 7 +++----
 bench_runner/templates/_benchmark.src.yml | 1 -
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/bench_runner/scripts/workflow.py b/bench_runner/scripts/workflow.py
index 6383cc43..73bb633a 100644
--- a/bench_runner/scripts/workflow.py
+++ b/bench_runner/scripts/workflow.py
@@ -202,9 +202,9 @@ def compile_windows(
     if "CLANG" in flags:
         args.extend(
             [
-                '"/p:PlatformToolset`=clangcl"',
-                r"/p:LLVMInstallDir=C:\Program Files\LLVM",
-                r"/p:LLVMToolsVersion=19.1.6",
+                "/p:PlatformToolset`=clangcl",
+                "/p:LLVMInstallDir`=C:\\Program Files\\LLVM",
+                "/p:LLVMToolsVersion`=19.1.6",
                 "--tail-call-interp",
             ]
         )
@@ -215,7 +215,6 @@ def compile_windows(
                 Path("PCbuild") / "build.bat",
                 *args,
             ],
-            shell=True,
         )
         shutil.copytree(get_windows_build_dir(force_32bit), "libs", dirs_exist_ok=True)
 
diff --git a/bench_runner/templates/_benchmark.src.yml b/bench_runner/templates/_benchmark.src.yml
index 9ea751aa..f7178a27 100644
--- a/bench_runner/templates/_benchmark.src.yml
+++ b/bench_runner/templates/_benchmark.src.yml
@@ -72,7 +72,6 @@ jobs:
         run: |
           git gc
       - name: Building Python and running pyperformance
-        shell: cmd
         run: |
           python workflow_bootstrap.py ${{ inputs.fork }} ${{ inputs.ref }} ${{ inputs.machine }} ${{ inputs.benchmarks || 'all' }} "${{ env.flags }}" ${{ inputs.force && '--force' || '' }} ${{ inputs.pgo && '--pgo' || '' }} --run_id ${{ github.run_id }}
       # Pull again, since another job may have committed results in the meantime

From 8e0c29317428c31585458870e753a98324e3bcaf Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Fri, 11 Apr 2025 13:19:29 -0400
Subject: [PATCH 27/30] Escaping

---
 bench_runner/scripts/workflow.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bench_runner/scripts/workflow.py b/bench_runner/scripts/workflow.py
index 73bb633a..8fca1d31 100644
--- a/bench_runner/scripts/workflow.py
+++ b/bench_runner/scripts/workflow.py
@@ -202,7 +202,7 @@ def compile_windows(
     if "CLANG" in flags:
         args.extend(
             [
-                "/p:PlatformToolset`=clangcl",
+                '"/p:PlatformToolset=clangcl"',
                 "/p:LLVMInstallDir`=C:\\Program Files\\LLVM",
                 "/p:LLVMToolsVersion`=19.1.6",
                 "--tail-call-interp",

From f9f4ba8c00c7d9ad25065e49caca30163e7e85bf Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Fri, 11 Apr 2025 13:35:23 -0400
Subject: [PATCH 28/30] Maybe I'll finally get lucky...

---
 bench_runner/scripts/workflow.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bench_runner/scripts/workflow.py b/bench_runner/scripts/workflow.py
index 8fca1d31..ad5f695f 100644
--- a/bench_runner/scripts/workflow.py
+++ b/bench_runner/scripts/workflow.py
@@ -187,7 +187,7 @@ def compile_windows(
 ) -> None:
     cpython = Path(cpython)
 
-    args = []
+    args = ["--%"]  # This is the PowerShell "stop parsing" flag
     if force_32bit:
         args.extend(["-p", "win32"])
     args.extend(["-c", "Release"])
@@ -203,8 +203,8 @@ def compile_windows(
         args.extend(
             [
                 '"/p:PlatformToolset=clangcl"',
-                "/p:LLVMInstallDir`=C:\\Program Files\\LLVM",
-                "/p:LLVMToolsVersion`=19.1.6",
+                '"/p:LLVMInstallDir=C:\\Program Files\\LLVM"',
+                '"/p:LLVMToolsVersion=19.1.6"',
                 "--tail-call-interp",
             ]
         )

From cc90a9586913090ef77b43ede55dcfa5728e3cb7 Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Fri, 11 Apr 2025 13:44:17 -0400
Subject: [PATCH 29/30] Try again

---
 bench_runner/scripts/workflow.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bench_runner/scripts/workflow.py b/bench_runner/scripts/workflow.py
index ad5f695f..0f3b04d4 100644
--- a/bench_runner/scripts/workflow.py
+++ b/bench_runner/scripts/workflow.py
@@ -212,6 +212,7 @@ def compile_windows(
     with contextlib.chdir(cpython):
         subprocess.check_call(
             [
+                "powershell.exe",
                 Path("PCbuild") / "build.bat",
                 *args,
             ],

From 9684fa4df99b192468098ab3b8d552157f79c9a0 Mon Sep 17 00:00:00 2001
From: Michael Droettboom <mdboom@gmail.com>
Date: Fri, 11 Apr 2025 13:48:10 -0400
Subject: [PATCH 30/30] Reorder arguments

---
 bench_runner/scripts/workflow.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bench_runner/scripts/workflow.py b/bench_runner/scripts/workflow.py
index 0f3b04d4..1364696a 100644
--- a/bench_runner/scripts/workflow.py
+++ b/bench_runner/scripts/workflow.py
@@ -202,10 +202,10 @@ def compile_windows(
     if "CLANG" in flags:
         args.extend(
             [
+                "--tail-call-interp",
                 '"/p:PlatformToolset=clangcl"',
                 '"/p:LLVMInstallDir=C:\\Program Files\\LLVM"',
                 '"/p:LLVMToolsVersion=19.1.6"',
-                "--tail-call-interp",
             ]
         )