Merge pull request #88 from eglitobias/benchmark-tooling

Add benchmark comparison tooling

Author: michaelklishin

.gitignore

@@ -16,3 +16,4 @@ bin/*
 *.bundle
 Makefile
 profiling/dumps/*
+benchmarks/results/*

ChangeLog.md

@@ -1,6 +1,13 @@
 ## Changes between 2.7.0 and 2.8.0 (unreleased)
 
-No changes yet.
+### Performance Improvements in Frame Decoding and Encoding
+
+Replacing `x == nil` with `x.nil?` in the frame layer hot path yields a
+consistent **+15–18% throughput improvement** in `Frame.decode_header`
+(called on every received frame) and **+12–14%** in `HeartbeatFrame.encode`,
+across Ruby 3.3, 3.4, and 4.0.
+
+See benchmarks/BENCHMARKS.md for instructions on how to reproduce these numbers on your machine.
 
 
 ## Changes between 2.6.0 and 2.7.0 (Mar 31, 2026)

benchmarks/BENCHMARKS.md

@@ -0,0 +1,61 @@
+# Benchmarking
+
+## Running the suite
+
+```bash
+ruby benchmarks/run_all.rb
+```
+
+Results are saved to `benchmarks/results/` with a timestamp.
+
+## Comparing two commits
+
+`benchmark_compare.sh` runs the full suite on both commits and prints a colour-coded delta summary. The after-sha defaults to `HEAD`.
+
+```bash
+bash benchmarks/benchmark_compare.sh <before-sha> [after-sha] [ruby-binary ...]
+```
+
+With the current ruby only:
+```bash
+bash benchmarks/benchmark_compare.sh 6d857de
+```
+
+With multiple rubies (asdf example):
+```bash
+bash benchmarks/benchmark_compare.sh 6d857de HEAD \
+  $(asdf where ruby 3.3.11)/bin/ruby \
+  $(asdf where ruby 3.4.9)/bin/ruby \
+  $(asdf where ruby 4.0.2)/bin/ruby
+```
+
+With rbenv:
+```bash
+bash benchmarks/benchmark_compare.sh 6d857de HEAD \
+  $(rbenv prefix 3.3.11)/bin/ruby \
+  $(rbenv prefix 3.4.9)/bin/ruby
+```
+
+You can also diff any two saved result files directly:
+```bash
+ruby benchmarks/compare_results.rb benchmarks/results/before_3_4_9.txt benchmarks/results/after_3_4_9.txt
+```
+
+## Getting reliable results
+
+Benchmark results are sensitive to system load. For trustworthy numbers:
+
+- Run on an otherwise idle machine (close browsers, pause background processes)
+- Results within ±5% of each other between runs are noise — `compare_results.rb` filters these out automatically
+- If a benchmark shows a large error margin (e.g. `±10%` in the raw output), discard that result and re-run
+
+## Individual benchmarks
+
+| Script | What it covers |
+|---|---|
+| `benchmarks/frame_encoding.rb` | Frame encode/decode — the core hot path |
+| `benchmarks/table_encoding.rb` | AMQP table encode/decode |
+| `benchmarks/method_encoding.rb` | Method/properties encode/decode, Basic.Publish |
+| `benchmarks/pack_unpack.rb` | Low-level pack/unpack primitives |
+
+Run any of them directly with `ruby benchmarks/<name>.rb`.

benchmarks/benchmark_compare.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+# Benchmarks two commits and prints a before/after delta summary.
+#
+# Usage:
+#   bash benchmarks/benchmark_compare.sh <before-sha> [after-sha] [ruby-binary ...]
+#
+# Defaults: after-sha = HEAD, ruby-binary = ruby
+#
+# Examples:
+#   bash benchmarks/benchmark_compare.sh 6d857de
+#   bash benchmarks/benchmark_compare.sh 6d857de HEAD
+#   bash benchmarks/benchmark_compare.sh 6d857de HEAD ruby3.3 ruby3.4
+#
+# With asdf:
+#   bash benchmarks/benchmark_compare.sh 6d857de HEAD \
+#     $(asdf where ruby 3.3.11)/bin/ruby \
+#     $(asdf where ruby 3.4.9)/bin/ruby
+#
+# With rbenv:
+#   bash benchmarks/benchmark_compare.sh 6d857de HEAD \
+#     $(rbenv prefix 3.3.11)/bin/ruby \
+#     $(rbenv prefix 3.4.9)/bin/ruby
+set -euo pipefail
+
+if [[ $# -lt 1 ]]; then
+  echo "Usage: $0 <before-sha> [after-sha] [ruby-binary ...]"
+  exit 1
+fi
+
+DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO="$(cd "$DIR/.." && pwd)"
+
+# Resolve to full SHAs immediately, before any git checkout changes HEAD
+BEFORE_LABEL=${1}
+AFTER_LABEL=${2:-HEAD}
+BEFORE=$(git -C "$REPO" rev-parse "$BEFORE_LABEL")
+AFTER=$(git -C "$REPO" rev-parse "$AFTER_LABEL")
+shift 2 2>/dev/null || shift $#
+RUBIES=("${@:-ruby}")
+RESULTS="$DIR/results"
+mkdir -p "$RESULTS"
+
+CURRENT_BRANCH=$(git -C "$REPO" rev-parse --abbrev-ref HEAD)
+
+restore() {
+  git -C "$REPO" checkout --quiet "$CURRENT_BRANCH"
+}
+trap restore EXIT
+
+run_suite() {
+  local label=$1 sha=$2 ruby_bin=$3
+  local ruby_label
+  ruby_label=$("$ruby_bin" -e 'print "#{RUBY_VERSION}"')
+  local tag="${label}_${ruby_label//./_}"
+  local out="$RESULTS/${tag}.txt"
+
+  echo
+  echo "=== [$ruby_label] $label ($sha) ==="
+  git -C "$REPO" checkout --quiet "$sha"
+  "$ruby_bin" -e 'require "benchmark/ips"' 2>/dev/null || \
+    "$ruby_bin" -S gem install benchmark-ips --quiet --no-document
+  # Prepend ruby's own directory to PATH so that bare `ruby` calls in older
+  # versions of run_all.rb (before the RbConfig.ruby fix) also use the right binary
+  PATH="$(dirname "$ruby_bin"):$PATH" "$ruby_bin" "$DIR/run_all.rb" 2>&1 | tee "$out"
+  echo ">>> Saved: $out"
+}
+
+for ruby_bin in "${RUBIES[@]}"; do
+  run_suite "before" "$BEFORE" "$ruby_bin"
+  run_suite "after"  "$AFTER"  "$ruby_bin"
+done
+
+# Restore before printing the summary so compare_results.rb is available
+restore
+
+echo
+echo "=================================================================="
+echo "Summary: before ($BEFORE_LABEL) vs after ($AFTER_LABEL)"
+echo "=================================================================="
+
+for ruby_bin in "${RUBIES[@]}"; do
+  ruby_label=$("$ruby_bin" -e 'print "#{RUBY_VERSION}"')
+  before_file="$RESULTS/before_${ruby_label//./_}.txt"
+  after_file="$RESULTS/after_${ruby_label//./_}.txt"
+  [[ -f "$before_file" && -f "$after_file" ]] || continue
+
+  echo
+  echo "  Ruby $ruby_label ($BEFORE_LABEL vs $AFTER_LABEL)"
+  "$ruby_bin" "$DIR/compare_results.rb" "$before_file" "$after_file"
+done

benchmarks/compare_results.rb

@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+# frozen_string_literal: true
+#
+# Prints a before/after delta summary from two benchmark result files.
+# Changes within ±5% are considered noise and hidden.
+#
+# Usage: ruby benchmarks/compare_results.rb <before.txt> <after.txt>
+
+NOISE_THRESHOLD = 5.0
+
+def parse_ips(file)
+  results = {}
+  in_comparison = false
+
+  File.readlines(file).each do |line|
+    in_comparison = true  if line.strip == "Comparison:"
+    in_comparison = false if in_comparison && line.strip.empty?
+    next unless in_comparison
+
+    results[$1.strip] = $2.to_f if line =~ /^(.+?):\s+([\d.]+) i\/s/
+  end
+
+  results
+end
+
+before = parse_ips(ARGV[0])
+after  = parse_ips(ARGV[1])
+
+deltas = (before.keys & after.keys).filter_map do |name|
+  b, a = before[name], after[name]
+  pct  = ((a - b) / b * 100).round(1)
+  pct.abs >= NOISE_THRESHOLD ? [name, pct] : nil
+end.sort_by { |_, pct| -pct }
+
+if deltas.empty?
+  puts "  No changes beyond ±#{NOISE_THRESHOLD}% noise threshold"
+else
+  deltas.each do |name, pct|
+    color = pct >= 0 ? "\e[32m" : "\e[31m"
+    sign  = pct >= 0 ? "+" : ""
+    puts "  #{("%-45s" % name)} #{color}#{sign}#{pct}%\e[0m"
+  end
+end

benchmarks/run_all.rb

@@ -46,7 +46,7 @@
       puts "\n>>> Running #{benchmark}..."
       puts
 
-      output = `ruby #{benchmark_path} 2>&1`
+      output = `#{RbConfig.ruby} #{benchmark_path} 2>&1`
       puts output
 
       f.puts ">>> #{benchmark}"