wip

Author: wakonig

.github/scripts/compare_benchmarks.py

@@ -45,7 +45,10 @@ class Comparison:
         delta_percent (float): Percent change from baseline to current.
         unit (str): Display unit for both values.
         metric (str): Current result metric used for comparison.
-        regressed (bool): Whether the change exceeds the configured threshold.
+        regressed (bool): Whether the change exceeds the configured threshold in
+            the worse direction.
+        improved (bool): Whether the change exceeds the configured threshold in
+            the better direction.
     """
 
     name: str
@@ -55,6 +58,7 @@ class Comparison:
     unit: str
     metric: str
     regressed: bool
+    improved: bool
 
 
 def _read_json(path: Path) -> Any:
@@ -246,8 +250,10 @@ def compare_benchmarks(
 
         if higher_is_better:
             regressed = delta_percent <= -threshold_percent
+            improved = delta_percent >= threshold_percent
         else:
             regressed = delta_percent >= threshold_percent
+            improved = delta_percent <= -threshold_percent
 
         comparisons.append(
             Comparison(
@@ -258,6 +264,7 @@ def compare_benchmarks(
                 unit=current_benchmark.unit or baseline_benchmark.unit,
                 metric=current_benchmark.metric,
                 regressed=regressed,
+                improved=improved,
             )
         )
 
@@ -282,6 +289,16 @@ def _format_value(value: float, unit: str) -> str:
     return f"{value:.6g}{suffix}"
 
 
+def _format_status(comparison: Comparison) -> str:
+    """Format a comparison status for Markdown output."""
+
+    if comparison.regressed:
+        return ":red_circle: regressed"
+    if comparison.improved:
+        return ":green_circle: improved"
+    return "ok"
+
+
 def write_summary(
     path: Path,
     comparisons: list[Comparison],
@@ -305,6 +322,7 @@ def write_summary(
     """
 
     regressions = [comparison for comparison in comparisons if comparison.regressed]
+    improvements = [comparison for comparison in comparisons if comparison.improved]
     direction = "higher is better" if higher_is_better else "lower is better"
     sorted_comparisons = sorted(comparisons, key=lambda comparison: comparison.name)
 
@@ -313,6 +331,7 @@ def write_summary(
         "## Benchmark comparison",
         "",
         f"Threshold: {threshold_percent:g}% ({direction}).",
+        f"Result: {len(regressions)} regression(s), {len(improvements)} improvement(s) beyond threshold.",
     ]
     lines.append("")
 
@@ -336,6 +355,28 @@ def write_summary(
     else:
         lines.append("No benchmark regression exceeded the configured threshold.")
 
+    lines.append("")
+
+    if improvements:
+        lines.extend(
+            [
+                f"{len(improvements)} benchmark(s) improved beyond the configured threshold.",
+                "",
+                "| Benchmark | Baseline | Current | Change |",
+                "| --- | ---: | ---: | ---: |",
+            ]
+        )
+        for comparison in improvements:
+            lines.append(
+                "| "
+                f"{comparison.name} | "
+                f"{_format_value(comparison.baseline, comparison.unit)} | "
+                f"{_format_value(comparison.current, comparison.unit)} | "
+                f"{comparison.delta_percent:+.2f}% |"
+            )
+    else:
+        lines.append("No benchmark improvement exceeded the configured threshold.")
+
     if sorted_comparisons:
         lines.extend(
             [
@@ -348,14 +389,13 @@ def write_summary(
             ]
         )
         for comparison in sorted_comparisons:
-            status = "regressed" if comparison.regressed else "ok"
             lines.append(
                 "| "
                 f"{comparison.name} | "
                 f"{_format_value(comparison.baseline, comparison.unit)} | "
                 f"{_format_value(comparison.current, comparison.unit)} | "
                 f"{comparison.delta_percent:+.2f}% | "
-                f"{status} |"
+                f"{_format_status(comparison)} |"
             )
         lines.extend(["", "</details>"])