Merge pull request #2325 from Kobzol/stable-benchmarks

Add stable benchmarks to the compile-time benchmark set

Author: Kobzol

collector/src/benchmark_set/compile_benchmarks.rs

@@ -1,8 +1,20 @@
-//! This file contains an exhaustive list of all **non-stable** compile-time benchmarks
+//! This file contains an exhaustive list of all compile-time benchmarks
 //! located in the `collector/compile-benchmarks` directory that are benchmarked in production.
 //! If new benchmarks are added/removed, they have to also be added/removed here, and in
 //! the [super::expand_benchmark_set] function.
 
+// Stable benchmarks
+pub(super) const CARGO: &str = "cargo";
+pub(super) const ENCODING: &str = "encoding";
+pub(super) const FUTURES: &str = "futures";
+pub(super) const HTML5EVER: &str = "html5ever";
+pub(super) const INFLATE: &str = "inflate";
+pub(super) const PISTON_IMAGE: &str = "piston-image";
+pub(super) const REGEX: &str = "regex";
+pub(super) const SYN: &str = "syn";
+pub(super) const TOKIO_WEBPUSH_SIMPLE: &str = "tokio-webpush-simple";
+
+// Non-stable benchmarks
 pub(super) const AWAIT_CALL_TREE: &str = "await-call-tree";
 pub(super) const BITMAPS_3_2_1: &str = "bitmaps-3.2.1";
 pub(super) const BITMAPS_3_2_1_NEW_SOLVER: &str = "bitmaps-3.2.1-new-solver";

collector/src/benchmark_set/mod.rs

@@ -48,6 +48,7 @@ pub fn expand_benchmark_set(id: BenchmarkSetId) -> Vec<BenchmarkSetMember> {
                 compile(AWAIT_CALL_TREE),
                 compile(BITMAPS_3_2_1),
                 compile(BITMAPS_3_2_1_NEW_SOLVER),
+                compile(CARGO),
                 compile(CARGO_0_87_1),
                 compile(CLAP_DERIVE_4_5_32),
                 compile(COERCIONS),
@@ -57,15 +58,19 @@ pub fn expand_benchmark_set(id: BenchmarkSetId) -> Vec<BenchmarkSetMember> {
                 compile(DEEPLY_NESTED_MULTI),
                 compile(DERIVE),
                 compile(DIESEL_2_2_10),
+                compile(ENCODING),
                 compile(EXTERNS),
                 compile(EZA_0_21_2),
+                compile(FUTURES),
                 compile(HELLOWORLD),
                 compile(HELLOWORLD_TINY),
+                compile(HTML5EVER),
                 compile(HTML5EVER_0_31_0),
                 compile(HTML5EVER_0_31_0_NEW_SOLVER),
                 compile(HYPER_1_6_0),
                 compile(IMAGE_0_25_6),
                 compile(INCLUDE_BLOB),
+                compile(INFLATE),
                 compile(ISSUE_46449),
                 compile(ISSUE_58319),
                 compile(ISSUE_88862),
@@ -75,7 +80,9 @@ pub fn expand_benchmark_set(id: BenchmarkSetId) -> Vec<BenchmarkSetMember> {
                 compile(MATCH_STRESS),
                 compile(NALGEBRA_0_33_0),
                 compile(NALGEBRA_0_33_0_NEW_SOLVER),
+                compile(PISTON_IMAGE),
                 compile(PROJECTION_CACHING),
+                compile(REGEX),
                 compile(REGEX_AUTOMATA_0_4_8),
                 compile(REGRESSION_31157),
                 compile(RIPGREP_14_1_1),
@@ -85,9 +92,11 @@ pub fn expand_benchmark_set(id: BenchmarkSetId) -> Vec<BenchmarkSetMember> {
                 compile(SERDE_1_0_219_THREADS4),
                 compile(SERDE_DERIVE_1_0_219),
                 compile(STM32F4_0_15_1),
+                compile(SYN),
                 compile(SYN_2_0_101),
                 compile(SYN_2_0_101_NEW_SOLVER),
                 compile(TOKEN_STREAM_STRESS),
+                compile(TOKIO_WEBPUSH_SIMPLE),
                 compile(TT_MUNCHER),
                 compile(TUPLE_STRESS),
                 compile(TYPENUM_1_18_0),
@@ -168,7 +177,6 @@ mod tests {
             get_compile_benchmarks(Path::new(BENCHMARK_DIR), CompileBenchmarkFilter::All)
                 .unwrap()
                 .into_iter()
-                .filter(|b| !b.category().is_stable())
                 .map(|b| b.name)
                 .collect::<Vec<BenchmarkName>>();
         for benchmark in &all_compile_benchmarks {

collector/src/bin/collector.rs

@@ -1785,6 +1785,11 @@ async fn create_benchmark_configs(
     let mut bench_runtime = false;
     let mut bench_compile_benchmarks = HashSet::new();
 
+    let is_release = match artifact_id {
+        ArtifactId::Commit(_) => false,
+        ArtifactId::Tag(_) => true,
+    };
+
     match job.kind() {
         database::BenchmarkJobKind::Runtime => {
             bench_runtime = true;
@@ -1799,32 +1804,27 @@ async fn create_benchmark_configs(
             }
         }
         database::BenchmarkJobKind::Rustc => {
-            bench_rustc = true;
+            bench_rustc = !is_release;
         }
     }
 
-    let is_release = match artifact_id {
-        ArtifactId::Commit(_) => false,
-        ArtifactId::Tag(_) => true,
-    };
-    if is_release {
-        bench_rustc = false;
-        bench_compile_benchmarks.retain(|benchmark| {
-            all_compile_benchmarks
-                .iter()
-                .find(|b| b.name == *benchmark)
-                .map(|b| b.category().is_stable())
-                .unwrap_or(false)
-        });
-    }
+    let benchmarks: Vec<Benchmark> = all_compile_benchmarks
+        .iter()
+        .filter(|b| {
+            if !bench_compile_benchmarks.contains(&b.name) {
+                return false;
+            }
+            // Run stable benchmarks for releases and non-stable benchmarks for everything
+            // else.
+            let is_stable = b.category().is_stable();
+            is_release == is_stable
+        })
+        .cloned()
+        .collect();
 
-    let compile_config = if bench_rustc || !bench_compile_benchmarks.is_empty() {
+    let compile_config = if bench_rustc || !benchmarks.is_empty() {
         Some(CompileBenchmarkConfig {
-            benchmarks: all_compile_benchmarks
-                .iter()
-                .filter(|b| bench_compile_benchmarks.contains(&b.name))
-                .cloned()
-                .collect(),
+            benchmarks,
             profiles: vec![job.profile().into()],
             scenarios: Scenario::all(),
             backends: vec![job.backend().into()],