|
| 1 | +# SPDX-License-Identifier: PMPL-1.0-or-later |
| 2 | +# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) <j.d.a.jewell@open.ac.uk> |
| 3 | +# |
| 4 | +# bench/compare.exs — Phase D regression-gate comparator (standards#99 scaffold) |
| 5 | +# |
| 6 | +# Reads: |
| 7 | +# • bench/results.json (produced by bench/gateway_latency.exs in this run) |
| 8 | +# • bench/baseline.json (checked-in baseline) |
| 9 | +# |
| 10 | +# Emits: |
| 11 | +# • A markdown table on stdout (also written to the GitHub step summary by CI) |
| 12 | +# • Exit 0 if all scenarios are within tolerance |
| 13 | +# • Exit 1 if any scenario regressed past tolerance |
| 14 | +# |
| 15 | +# SCAFFOLD MODE |
| 16 | +# ───────────── |
| 17 | +# Until baseline.json contains real numbers (its "_status" is "scaffold- |
| 18 | +# placeholder"), this script is INTENTIONALLY non-blocking — it reports |
| 19 | +# the live numbers and exits 0 with a clear "scaffold mode" banner. |
| 20 | +# Phase D-4 will flip "_status" to "active" once a real baseline lands; |
| 21 | +# the gate becomes blocking from that point on with no code change here. |
| 22 | + |
| 23 | +# `mix run` puts deps (including :jason) on the code path; nothing to start. |
| 24 | + |
| 25 | +defmodule Bench.Compare do |
| 26 | + @results_path "bench/results.json" |
| 27 | + @baseline_path "bench/baseline.json" |
| 28 | + |
| 29 | + def run do |
| 30 | + with {:ok, results_raw} <- File.read(@results_path), |
| 31 | + {:ok, baseline_raw} <- File.read(@baseline_path), |
| 32 | + {:ok, results} <- Jason.decode(results_raw), |
| 33 | + {:ok, baseline} <- Jason.decode(baseline_raw) do |
| 34 | + compare(results, baseline) |
| 35 | + else |
| 36 | + {:error, :enoent} -> |
| 37 | + IO.puts(:stderr, "ERROR: bench/results.json or bench/baseline.json missing") |
| 38 | + IO.puts(:stderr, "Did `mix run bench/gateway_latency.exs` run first?") |
| 39 | + System.halt(2) |
| 40 | + |
| 41 | + {:error, reason} -> |
| 42 | + IO.puts(:stderr, "ERROR reading baseline/results: #{inspect(reason)}") |
| 43 | + System.halt(2) |
| 44 | + end |
| 45 | + end |
| 46 | + |
| 47 | + defp compare(results, baseline) do |
| 48 | + status = Map.get(baseline, "_status", "unknown") |
| 49 | + tolerance = Map.get(baseline, "tolerance", %{}) |
| 50 | + |
| 51 | + IO.puts("# Phase D — Performance Regression Report") |
| 52 | + IO.puts("") |
| 53 | + IO.puts("Baseline status: **#{status}**") |
| 54 | + IO.puts("") |
| 55 | + |
| 56 | + case status do |
| 57 | + "scaffold-placeholder" -> |
| 58 | + IO.puts( |
| 59 | + "> SCAFFOLD MODE — bench/baseline.json has not been populated yet " <> |
| 60 | + "(Phase D-4 collects the real baseline). This run is informational " <> |
| 61 | + "only; the gate is **non-blocking** until baseline.json `_status` " <> |
| 62 | + "is flipped to `active`." |
| 63 | + ) |
| 64 | + |
| 65 | + IO.puts("") |
| 66 | + emit_table(results, nil, tolerance) |
| 67 | + System.halt(0) |
| 68 | + |
| 69 | + "active" -> |
| 70 | + emit_table(results, baseline["scenarios"], tolerance) |
| 71 | + |> case do |
| 72 | + :ok -> System.halt(0) |
| 73 | + :regressed -> System.halt(1) |
| 74 | + end |
| 75 | + |
| 76 | + other -> |
| 77 | + IO.puts(:stderr, "ERROR: unknown baseline _status: #{inspect(other)}") |
| 78 | + System.halt(2) |
| 79 | + end |
| 80 | + end |
| 81 | + |
| 82 | + # ── Pretty-print + regression check ──────────────────────────────────────── |
| 83 | + |
| 84 | + defp emit_table(results, baseline_scenarios, tolerance) do |
| 85 | + # Benchee JSON shape: top-level "statistics" -> per-scenario map. |
| 86 | + stats = Map.get(results, "statistics", %{}) |
| 87 | + |
| 88 | + IO.puts("| Scenario | p50 (µs) | p95 (µs) | p99 (µs) | Status |") |
| 89 | + IO.puts("|----------|----------|----------|----------|--------|") |
| 90 | + |
| 91 | + Enum.reduce(stats, :ok, fn {name, scenario_stats}, acc -> |
| 92 | + p50 = percentile_us(scenario_stats, "50") |
| 93 | + p95 = percentile_us(scenario_stats, "95") |
| 94 | + p99 = percentile_us(scenario_stats, "99") |
| 95 | + |
| 96 | + status = |
| 97 | + case baseline_scenarios do |
| 98 | + nil -> |
| 99 | + "scaffold" |
| 100 | + |
| 101 | + map -> |
| 102 | + base = Map.get(map, name) |
| 103 | + check_regression(base, p50, p95, p99, tolerance) |
| 104 | + end |
| 105 | + |
| 106 | + IO.puts("| #{name} | #{fmt(p50)} | #{fmt(p95)} | #{fmt(p99)} | #{status} |") |
| 107 | + |
| 108 | + cond do |
| 109 | + acc == :regressed -> :regressed |
| 110 | + status == "REGRESSED" -> :regressed |
| 111 | + true -> acc |
| 112 | + end |
| 113 | + end) |
| 114 | + end |
| 115 | + |
| 116 | + defp percentile_us(scenario_stats, p) do |
| 117 | + # Benchee reports nanoseconds in JSON; convert to µs for human readability |
| 118 | + # and to match the µs units used in docs/perf-contract.md. |
| 119 | + case get_in(scenario_stats, ["percentiles", p]) do |
| 120 | + nil -> nil |
| 121 | + ns when is_number(ns) -> ns / 1_000.0 |
| 122 | + end |
| 123 | + end |
| 124 | + |
| 125 | + defp fmt(nil), do: "—" |
| 126 | + defp fmt(n) when is_float(n), do: :erlang.float_to_binary(n, decimals: 2) |
| 127 | + defp fmt(n), do: to_string(n) |
| 128 | + |
| 129 | + defp check_regression(nil, _, _, _, _), do: "no baseline" |
| 130 | + |
| 131 | + defp check_regression(base, p50, p95, p99, tolerance) do |
| 132 | + bp50 = num(base["p50_us"]) |
| 133 | + bp95 = num(base["p95_us"]) |
| 134 | + bp99 = num(base["p99_us"]) |
| 135 | + |
| 136 | + t50 = Map.get(tolerance, "p50_max_ratio", 1.20) |
| 137 | + t95 = Map.get(tolerance, "p95_max_ratio", 1.30) |
| 138 | + t99 = Map.get(tolerance, "p99_max_ratio", 1.50) |
| 139 | + |
| 140 | + breached = |
| 141 | + (bp50 && p50 && p50 > bp50 * t50) or |
| 142 | + (bp95 && p95 && p95 > bp95 * t95) or |
| 143 | + (bp99 && p99 && p99 > bp99 * t99) |
| 144 | + |
| 145 | + if breached, do: "REGRESSED", else: "ok" |
| 146 | + end |
| 147 | + |
| 148 | + defp num(nil), do: nil |
| 149 | + defp num(n) when is_number(n), do: n |
| 150 | + defp num(_), do: nil |
| 151 | +end |
| 152 | + |
| 153 | +Bench.Compare.run() |
0 commit comments