diff --git a/benchmarks/lib/regression_report.rb b/benchmarks/lib/regression_report.rb index e68da7f429..5226a904a4 100644 --- a/benchmarks/lib/regression_report.rb +++ b/benchmarks/lib/regression_report.rb @@ -47,26 +47,14 @@ module RegressionReport FIRST_RUN_SUMMARY = "first_run_summary" CONFIRMATION_SUMMARY = "confirmation_summary" - # TEMPORARY — benchmarks whose regressions must NOT open an issue, by the exact - # benchmark name Bencher reports (the leading-slash name shown in the summary table, - # matched against REGRESSED_BENCHMARKS in each suite's payload). - # - # /posts_page: Pro spent weeks hard-500ing on every request (failed_pct = 100), so the - # route was effectively an instant error and Bencher built its rps/latency baseline - # from those bogus-fast failure responses. Now that the route is fixed, its real - # (slower) timings read as large regressions against that fast baseline and would file - # a spurious regression issue on every main push. We can't surgically delete just this - # benchmark's history in Bencher (the delete is blocked by a FOREIGN KEY constraint - # from its reports/alerts), so we suppress it here instead — and short-circuit it - # BEFORE a confirmation rerun (plan_confirmation.rb), so we don't burn a fresh runner - # confirming a benchmark we would suppress anyway. - # - # REMOVE this entry (restoring unconditional filing) once the failure-era samples have - # rolled out of Bencher's 64-run t-test window for /posts_page: Pro on main — i.e. once - # the dashboard baseline reflects real timings again. After that a regression for it is - # genuine and must be reported. Tracking issue + full revert steps: - # https://github.com/shakacode/react_on_rails/issues/3669 - IGNORED_REGRESSION_BENCHMARKS = ["/posts_page: Pro"].freeze + # Benchmarks whose regressions must NOT open an issue, by the exact benchmark name + # Bencher reports (the leading-slash name shown in the summary table, matched against + # REGRESSED_BENCHMARKS in each suite's payload). Entries here are TEMPORARY + # suppressions: plan_confirmation.rb short-circuits a candidate whose only regressed + # benchmarks are listed here BEFORE a confirmation rerun (no fresh runner, no issue). + # Empty means every confirmed regression is reported. Any entry added here must have a + # tracking issue stating the revert criteria. + IGNORED_REGRESSION_BENCHMARKS = [].freeze # Build a structured alert hash for the ALERTS payload key. def alert(benchmark, measure) diff --git a/benchmarks/spec/plan_confirmation_spec.rb b/benchmarks/spec/plan_confirmation_spec.rb index 8d11bd9673..cb2a34387b 100644 --- a/benchmarks/spec/plan_confirmation_spec.rb +++ b/benchmarks/spec/plan_confirmation_spec.rb @@ -29,7 +29,11 @@ def matrix_row(suite, shard_label: "1/1") } end - let(:ignored) { RegressionReport::IGNORED_REGRESSION_BENCHMARKS.first } + # The production list is empty (no active suppressions); stub a sample entry so these + # examples keep pinning the ignore-list short-circuit machinery. + let(:ignored) { "/ignored: Pro" } + + before { stub_const("RegressionReport::IGNORED_REGRESSION_BENCHMARKS", [ignored]) } describe ".fully_ignored?" do it "is true only when every named benchmark is ignored" do diff --git a/benchmarks/spec/regression_report_spec.rb b/benchmarks/spec/regression_report_spec.rb index b1ff1e4cac..d4c47e7cc4 100644 --- a/benchmarks/spec/regression_report_spec.rb +++ b/benchmarks/spec/regression_report_spec.rb @@ -60,7 +60,11 @@ def alert(benchmark, measure) end describe "the ignore-list helpers" do - let(:ignored) { described_class::IGNORED_REGRESSION_BENCHMARKS.first } + # The production list is empty (no active suppressions); stub a sample entry so + # these examples keep pinning the general filtering machinery. + let(:ignored) { "/ignored: Pro" } + + before { stub_const("RegressionReport::IGNORED_REGRESSION_BENCHMARKS", [ignored]) } it "drops ignored benchmarks from alert pairs" do alerts = [alert(ignored, "rps"), alert("/real: Pro", "rps")] diff --git a/benchmarks/spec/track_benchmarks_spec.rb b/benchmarks/spec/track_benchmarks_spec.rb index f32d66e8a3..fd52b21316 100644 --- a/benchmarks/spec/track_benchmarks_spec.rb +++ b/benchmarks/spec/track_benchmarks_spec.rb @@ -329,7 +329,10 @@ def pair(benchmark, measure) end it "ignores a candidate alert on a temporarily-ignored benchmark" do - ignored = RegressionReport::IGNORED_REGRESSION_BENCHMARKS.first + # The production list is empty (no active suppressions); stub a sample entry so + # this example keeps pinning the ignore handling in the confirmation outcome. + ignored = "/ignored: Pro" + stub_const("RegressionReport::IGNORED_REGRESSION_BENCHMARKS", [ignored]) report = BencherReport.parse( JSON.generate( "results" => [[result(ignored, [rps_measure])]],