Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 8 additions & 20 deletions benchmarks/lib/regression_report.rb
Original file line number Diff line number Diff line change
Expand Up @@ -47,26 +47,14 @@ module RegressionReport
FIRST_RUN_SUMMARY = "first_run_summary"
CONFIRMATION_SUMMARY = "confirmation_summary"

# TEMPORARY — benchmarks whose regressions must NOT open an issue, by the exact
# benchmark name Bencher reports (the leading-slash name shown in the summary table,
# matched against REGRESSED_BENCHMARKS in each suite's payload).
#
# /posts_page: Pro spent weeks hard-500ing on every request (failed_pct = 100), so the
# route was effectively an instant error and Bencher built its rps/latency baseline
# from those bogus-fast failure responses. Now that the route is fixed, its real
# (slower) timings read as large regressions against that fast baseline and would file
# a spurious regression issue on every main push. We can't surgically delete just this
# benchmark's history in Bencher (the delete is blocked by a FOREIGN KEY constraint
# from its reports/alerts), so we suppress it here instead — and short-circuit it
# BEFORE a confirmation rerun (plan_confirmation.rb), so we don't burn a fresh runner
# confirming a benchmark we would suppress anyway.
#
# REMOVE this entry (restoring unconditional filing) once the failure-era samples have
# rolled out of Bencher's 64-run t-test window for /posts_page: Pro on main — i.e. once
# the dashboard baseline reflects real timings again. After that a regression for it is
# genuine and must be reported. Tracking issue + full revert steps:
# https://github.com/shakacode/react_on_rails/issues/3669
IGNORED_REGRESSION_BENCHMARKS = ["/posts_page: Pro"].freeze
# Benchmarks whose regressions must NOT open an issue, by the exact benchmark name
# Bencher reports (the leading-slash name shown in the summary table, matched against
# REGRESSED_BENCHMARKS in each suite's payload). Entries here are TEMPORARY
# suppressions: plan_confirmation.rb short-circuits a candidate whose only regressed
# benchmarks are listed here BEFORE a confirmation rerun (no fresh runner, no issue).
# Empty means every confirmed regression is reported. Any entry added here must have a
# tracking issue stating the revert criteria.
IGNORED_REGRESSION_BENCHMARKS = [].freeze

# Build a structured alert hash for the ALERTS payload key.
def alert(benchmark, measure)
Expand Down
6 changes: 5 additions & 1 deletion benchmarks/spec/plan_confirmation_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,11 @@ def matrix_row(suite, shard_label: "1/1")
}
end

let(:ignored) { RegressionReport::IGNORED_REGRESSION_BENCHMARKS.first }
# The production list is empty (no active suppressions); stub a sample entry so these
# examples keep pinning the ignore-list short-circuit machinery.
let(:ignored) { "/ignored: Pro" }

before { stub_const("RegressionReport::IGNORED_REGRESSION_BENCHMARKS", [ignored]) }
Comment thread
justin808 marked this conversation as resolved.

describe ".fully_ignored?" do
it "is true only when every named benchmark is ignored" do
Expand Down
6 changes: 5 additions & 1 deletion benchmarks/spec/regression_report_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,11 @@ def alert(benchmark, measure)
end

describe "the ignore-list helpers" do
let(:ignored) { described_class::IGNORED_REGRESSION_BENCHMARKS.first }
# The production list is empty (no active suppressions); stub a sample entry so
# these examples keep pinning the general filtering machinery.
let(:ignored) { "/ignored: Pro" }

before { stub_const("RegressionReport::IGNORED_REGRESSION_BENCHMARKS", [ignored]) }

it "drops ignored benchmarks from alert pairs" do
alerts = [alert(ignored, "rps"), alert("/real: Pro", "rps")]
Expand Down
5 changes: 4 additions & 1 deletion benchmarks/spec/track_benchmarks_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -329,7 +329,10 @@ def pair(benchmark, measure)
end

it "ignores a candidate alert on a temporarily-ignored benchmark" do
ignored = RegressionReport::IGNORED_REGRESSION_BENCHMARKS.first
# The production list is empty (no active suppressions); stub a sample entry so
# this example keeps pinning the ignore handling in the confirmation outcome.
ignored = "/ignored: Pro"
stub_const("RegressionReport::IGNORED_REGRESSION_BENCHMARKS", [ignored])
report = BencherReport.parse(
JSON.generate(
"results" => [[result(ignored, [rps_measure])]],
Expand Down
Loading