diff --git a/.gitignore b/.gitignore index e1dec40..8f338c9 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ bin/* *.bundle Makefile profiling/dumps/* +benchmarks/results/* diff --git a/ChangeLog.md b/ChangeLog.md index 6b74d08..c0ebaf1 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,6 +1,13 @@ ## Changes between 2.7.0 and 2.8.0 (unreleased) -No changes yet. +### Performance Improvements in Frame Decoding and Encoding + +Replacing `x == nil` with `x.nil?` in the frame layer hot path yields a +consistent **+15–18% throughput improvement** in `Frame.decode_header` +(called on every received frame) and **+12–14%** in `HeartbeatFrame.encode`, +across Ruby 3.3, 3.4, and 4.0. + +See benchmarks/BENCHMARKS.md for instructions on how to reproduce these numbers on your machine. ## Changes between 2.6.0 and 2.7.0 (Mar 31, 2026) diff --git a/benchmarks/BENCHMARKS.md b/benchmarks/BENCHMARKS.md new file mode 100644 index 0000000..7b0f061 --- /dev/null +++ b/benchmarks/BENCHMARKS.md @@ -0,0 +1,61 @@ +# Benchmarking + +## Running the suite + +```bash +ruby benchmarks/run_all.rb +``` + +Results are saved to `benchmarks/results/` with a timestamp. + +## Comparing two commits + +`benchmark_compare.sh` runs the full suite on both commits and prints a colour-coded delta summary. The after-sha defaults to `HEAD`. + +```bash +bash benchmarks/benchmark_compare.sh [after-sha] [ruby-binary ...] +``` + +With the current ruby only: +```bash +bash benchmarks/benchmark_compare.sh 6d857de +``` + +With multiple rubies (asdf example): +```bash +bash benchmarks/benchmark_compare.sh 6d857de HEAD \ + $(asdf where ruby 3.3.11)/bin/ruby \ + $(asdf where ruby 3.4.9)/bin/ruby \ + $(asdf where ruby 4.0.2)/bin/ruby +``` + +With rbenv: +```bash +bash benchmarks/benchmark_compare.sh 6d857de HEAD \ + $(rbenv prefix 3.3.11)/bin/ruby \ + $(rbenv prefix 3.4.9)/bin/ruby +``` + +You can also diff any two saved result files directly: +```bash +ruby benchmarks/compare_results.rb benchmarks/results/before_3_4_9.txt benchmarks/results/after_3_4_9.txt +``` + +## Getting reliable results + +Benchmark results are sensitive to system load. For trustworthy numbers: + +- Run on an otherwise idle machine (close browsers, pause background processes) +- Results within ±5% of each other between runs are noise — `compare_results.rb` filters these out automatically +- If a benchmark shows a large error margin (e.g. `±10%` in the raw output), discard that result and re-run + +## Individual benchmarks + +| Script | What it covers | +|---|---| +| `benchmarks/frame_encoding.rb` | Frame encode/decode — the core hot path | +| `benchmarks/table_encoding.rb` | AMQP table encode/decode | +| `benchmarks/method_encoding.rb` | Method/properties encode/decode, Basic.Publish | +| `benchmarks/pack_unpack.rb` | Low-level pack/unpack primitives | + +Run any of them directly with `ruby benchmarks/.rb`. diff --git a/benchmarks/benchmark_compare.sh b/benchmarks/benchmark_compare.sh new file mode 100644 index 0000000..2c318f7 --- /dev/null +++ b/benchmarks/benchmark_compare.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Benchmarks two commits and prints a before/after delta summary. +# +# Usage: +# bash benchmarks/benchmark_compare.sh [after-sha] [ruby-binary ...] +# +# Defaults: after-sha = HEAD, ruby-binary = ruby +# +# Examples: +# bash benchmarks/benchmark_compare.sh 6d857de +# bash benchmarks/benchmark_compare.sh 6d857de HEAD +# bash benchmarks/benchmark_compare.sh 6d857de HEAD ruby3.3 ruby3.4 +# +# With asdf: +# bash benchmarks/benchmark_compare.sh 6d857de HEAD \ +# $(asdf where ruby 3.3.11)/bin/ruby \ +# $(asdf where ruby 3.4.9)/bin/ruby +# +# With rbenv: +# bash benchmarks/benchmark_compare.sh 6d857de HEAD \ +# $(rbenv prefix 3.3.11)/bin/ruby \ +# $(rbenv prefix 3.4.9)/bin/ruby +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "Usage: $0 [after-sha] [ruby-binary ...]" + exit 1 +fi + +DIR="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "$DIR/.." && pwd)" + +# Resolve to full SHAs immediately, before any git checkout changes HEAD +BEFORE_LABEL=${1} +AFTER_LABEL=${2:-HEAD} +BEFORE=$(git -C "$REPO" rev-parse "$BEFORE_LABEL") +AFTER=$(git -C "$REPO" rev-parse "$AFTER_LABEL") +shift 2 2>/dev/null || shift $# +RUBIES=("${@:-ruby}") +RESULTS="$DIR/results" +mkdir -p "$RESULTS" + +CURRENT_BRANCH=$(git -C "$REPO" rev-parse --abbrev-ref HEAD) + +restore() { + git -C "$REPO" checkout --quiet "$CURRENT_BRANCH" +} +trap restore EXIT + +run_suite() { + local label=$1 sha=$2 ruby_bin=$3 + local ruby_label + ruby_label=$("$ruby_bin" -e 'print "#{RUBY_VERSION}"') + local tag="${label}_${ruby_label//./_}" + local out="$RESULTS/${tag}.txt" + + echo + echo "=== [$ruby_label] $label ($sha) ===" + git -C "$REPO" checkout --quiet "$sha" + "$ruby_bin" -e 'require "benchmark/ips"' 2>/dev/null || \ + "$ruby_bin" -S gem install benchmark-ips --quiet --no-document + # Prepend ruby's own directory to PATH so that bare `ruby` calls in older + # versions of run_all.rb (before the RbConfig.ruby fix) also use the right binary + PATH="$(dirname "$ruby_bin"):$PATH" "$ruby_bin" "$DIR/run_all.rb" 2>&1 | tee "$out" + echo ">>> Saved: $out" +} + +for ruby_bin in "${RUBIES[@]}"; do + run_suite "before" "$BEFORE" "$ruby_bin" + run_suite "after" "$AFTER" "$ruby_bin" +done + +# Restore before printing the summary so compare_results.rb is available +restore + +echo +echo "==================================================================" +echo "Summary: before ($BEFORE_LABEL) vs after ($AFTER_LABEL)" +echo "==================================================================" + +for ruby_bin in "${RUBIES[@]}"; do + ruby_label=$("$ruby_bin" -e 'print "#{RUBY_VERSION}"') + before_file="$RESULTS/before_${ruby_label//./_}.txt" + after_file="$RESULTS/after_${ruby_label//./_}.txt" + [[ -f "$before_file" && -f "$after_file" ]] || continue + + echo + echo " Ruby $ruby_label ($BEFORE_LABEL vs $AFTER_LABEL)" + "$ruby_bin" "$DIR/compare_results.rb" "$before_file" "$after_file" +done diff --git a/benchmarks/compare_results.rb b/benchmarks/compare_results.rb new file mode 100644 index 0000000..5588f65 --- /dev/null +++ b/benchmarks/compare_results.rb @@ -0,0 +1,44 @@ +#!/usr/bin/env ruby +# encoding: utf-8 +# frozen_string_literal: true +# +# Prints a before/after delta summary from two benchmark result files. +# Changes within ±5% are considered noise and hidden. +# +# Usage: ruby benchmarks/compare_results.rb + +NOISE_THRESHOLD = 5.0 + +def parse_ips(file) + results = {} + in_comparison = false + + File.readlines(file).each do |line| + in_comparison = true if line.strip == "Comparison:" + in_comparison = false if in_comparison && line.strip.empty? + next unless in_comparison + + results[$1.strip] = $2.to_f if line =~ /^(.+?):\s+([\d.]+) i\/s/ + end + + results +end + +before = parse_ips(ARGV[0]) +after = parse_ips(ARGV[1]) + +deltas = (before.keys & after.keys).filter_map do |name| + b, a = before[name], after[name] + pct = ((a - b) / b * 100).round(1) + pct.abs >= NOISE_THRESHOLD ? [name, pct] : nil +end.sort_by { |_, pct| -pct } + +if deltas.empty? + puts " No changes beyond ±#{NOISE_THRESHOLD}% noise threshold" +else + deltas.each do |name, pct| + color = pct >= 0 ? "\e[32m" : "\e[31m" + sign = pct >= 0 ? "+" : "" + puts " #{("%-45s" % name)} #{color}#{sign}#{pct}%\e[0m" + end +end diff --git a/benchmarks/run_all.rb b/benchmarks/run_all.rb index a0391e4..5f69ccc 100644 --- a/benchmarks/run_all.rb +++ b/benchmarks/run_all.rb @@ -46,7 +46,7 @@ puts "\n>>> Running #{benchmark}..." puts - output = `ruby #{benchmark_path} 2>&1` + output = `#{RbConfig.ruby} #{benchmark_path} 2>&1` puts output f.puts ">>> #{benchmark}"