Automate bisecting benchmarks

Author: msimacek

.agents/skills/rota-bench-regression-analysis/SKILL.md

@@ -18,7 +18,14 @@ scripts/compare_bench_regressions.py --rota --json-out /tmp/compare_bench_regres
 3. Focus on `plausible` regressions. Ignore `flaky` and `inconclusive` items unless they help explain a plausible shift.
 4. Split the summary into:
 - `Attributed`
-- `Unattributed`
+- `To bisect`
+- `To watch`
+5. Show the current summary
+6. Execute the bisect script for each "to bisect" entry in parallel, then wait for all of them to finish.
+   The builds can take many hours without the script showing any output, make sure you wait for them with a long timeout.
+   If running in codex: round-robin poll the processes with `write_stdin` and 1 hour timeout (the configuration might cap this at a lower timeout in practice)
+7. Collect the bisect results and move any benchmarks that were attributed by the bisections.
+8. Show the final summary. Note any failed bisects.
 
 ## Useful JSON Queries
 ```bash
@@ -67,10 +74,16 @@ git diff --stat GOOD..BAD
 
 ## Output Contract
 - List findings first, not process notes.
-- Keep two top-level sections: `Attributed` and `Unattributed`.
+- Keep three top-level sections (if not empty): `Attributed` and `To bisect` and `To watch`
 - In the attributed section, use this header format: `abcd1234efgh | author@oracle.com | Full subject`
-- In the unattributed section, say whether the item looks real, flaky, or likely the same cause as another attributed item.
+- Unattributed changes that look plausible go to "to bisect", flaky ones go to "to watch"
+- In the "to bisect" section, add an invocation (don't execute yet) of `scripts/bisect_benchmark_regression.py` that can bisect it (use unabbreviated commits in this case)
+- In the "to watch" section, say whether the item looks flaky, or likely the same cause as another attributed item.
 - Do not abbreviate commit subjects.
 - Keep author emails.
 - Abbreviate commit IDs to 12 characters.
-- Do not list every benchmark; only the worst examples from each affected suite.
+- Do not list every benchmark if there are many; only the worst examples from each affected suite. If you didn't list all, say "and X others".
+
+## Guardrails
+- If the script or you can't find `bench-cli`, ask the user to provide it from the `bench-server` repo.
+- Don't submit more than 5 bisect jobs. If there are more in the "to bisect" list, pick 5 that look the most serious and leave the rest as "to bisect".

bisect-benchmark.ini

@@ -3,7 +3,7 @@
 # Usage:
 # - Create a temporary branch based on the main branch (or the bad commit)
 # - Fill in this configuration file, preferably using the automated script
-#   graal-enterprise/graalpython-enterprise/scripts/create-bisect-config
+#   scripts/create_bisect_config.py
 # - Commit and push the file
 # - The push command output should give you a link to create a PR. Open it, but
 #   don't create a PR. Instead, you should execute the job on your commit using

ci/python-bench.libsonnet

@@ -240,7 +240,9 @@
         downloads: downloads(self.os, self.arch),
         name: "bisect-benchmark",
         targets: ['bench'],
-        logs +: logs(self.os, self.arch),
+        logs +: logs(self.os, self.arch) + [
+            "bisect-benchmark-result.json",
+        ],
         deploysArtifacts: true,
         packages +: packages(self.os, self.arch) + {
             "apache/ant": ">=1.9.4",

mx.graalpython/mx_graalpython_bisect.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2020, 2026, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -71,6 +71,8 @@ def print_line(l):
     GRAAL_DIR: GRAAL_ENTERPRISE_DIR,
 }
 
+RESULTS_JSON_PATH = 'bisect-benchmark-result.json'
+
 
 def get_commit(repo_path, ref='HEAD'):
     if repo_path:
@@ -178,6 +180,22 @@ def summarize(self):
                     .format(self.repo_name, self.bad_commit, get_message(self.repo_path, self.bad_commit)))
         return ''
 
+    def to_dict(self):
+        return {
+            'repo_name': self.repo_name,
+            'repo_path': str(self.repo_path),
+            'commits': self.commits,
+            'results': [result.to_dict() if result is not None else None for result in self.results],
+            'good_index': self.good_index,
+            'bad_index': self.bad_index,
+            'good_commit': self.good_commit,
+            'bad_commit': self.bad_commit,
+            'dependency_results': {
+                str(index): dependency_result.to_dict()
+                for index, dependency_result in self.dependency_results.items()
+            },
+        }
+
 
 class BenchmarkResult(abc.ABC):
     def __init__(self, value, unit=None):
@@ -201,6 +219,14 @@ def bound_is_significant(self, bad_result, epsilon):
     def is_good(self, good_result, bad_result):
         pass
 
+    def to_dict(self):
+        return {
+            'kind': type(self).__name__,
+            'value': self.value,
+            'unit': self.unit,
+            'display': str(self),
+        }
+
 
 class LowerIsBetterResult(BenchmarkResult):
     def is_good(self, good_result, bad_result):
@@ -358,6 +384,16 @@ def benchmark_callback(repo_path: Path, commit, bench_command=args.benchmark_com
     print()
     print(summary)
 
+    with open(RESULTS_JSON_PATH, 'w', encoding='utf-8') as result_file:
+        json.dump({
+            'bisect_id': bisect_id,
+            'summary': summary,
+            'visualization': visualization,
+            'result': result.to_dict(),
+            'build_url': os.environ.get('BUILD_URL'),
+        }, result_file, indent=2, sort_keys=True)
+        result_file.write('\n')
+
     if args.rerun_with_commands:
         print('\n\nRerunning the good and bad commits with extra benchmark commands:')
         repo_path = DIR