fix: divan walltime metric; support variable iter count

Author: not-matthias

crates/cargo-codspeed/src/walltime_results.rs

@@ -53,10 +53,17 @@ impl WalltimeBenchmark {
 
 impl From<RawWallTimeData> for WalltimeBenchmark {
     fn from(value: RawWallTimeData) -> Self {
-        let times_ns: Vec<f64> = value.times_ns.iter().map(|&t| t as f64).collect();
-        let mut data = Data::new(times_ns.clone());
+        let total_time = value.times_per_round_ns.iter().sum::<u128>() as f64 / 1_000_000_000.0;
+        let time_per_iteration_per_round_ns: Vec<_> = value
+            .times_per_round_ns
+            .into_iter()
+            .zip(&value.iters_per_round)
+            .map(|(time_per_round, iter_per_round)| time_per_round / iter_per_round)
+            .map(|t| t as f64)
+            .collect::<Vec<f64>>();
+
+        let mut data = Data::new(time_per_iteration_per_round_ns);
         let rounds = data.len() as u64;
-        let total_time = times_ns.iter().sum::<f64>() / 1_000_000_000.0;
 
         let mean_ns = data.mean().unwrap();
 
@@ -91,7 +98,9 @@ impl From<RawWallTimeData> for WalltimeBenchmark {
         let min_ns = data.min();
         let max_ns = data.max();
 
-        let iter_per_round = value.iter_per_round as u64;
+        // TODO(COD-1056): We currently only support single iteration count per round
+        let iter_per_round = (value.iters_per_round.iter().sum::<u128>()
+            / value.iters_per_round.len() as u128) as u64;
         let warmup_iters = 0; // FIXME: add warmup detection
 
         let stats = BenchmarkStats {
@@ -172,9 +181,9 @@ mod tests {
         };
         let raw_bench = RawWallTimeData {
             metadata,
-            iter_per_round: 1,
+            iters_per_round: vec![1],
             max_time_ns: None,
-            times_ns: vec![42],
+            times_per_round_ns: vec![42],
         };
 
         let benchmark: WalltimeBenchmark = raw_bench.into();
@@ -183,4 +192,31 @@ mod tests {
         assert_eq!(benchmark.stats.max_ns, 42.);
         assert_eq!(benchmark.stats.mean_ns, 42.);
     }
+
+    #[test]
+    fn test_parse_bench_with_variable_iterations() {
+        let metadata = BenchmarkMetadata {
+            name: "benchmark".to_string(),
+            uri: "test::benchmark".to_string(),
+        };
+
+        let raw_bench = RawWallTimeData {
+            metadata,
+            iters_per_round: vec![1, 2, 3, 4, 5, 6],
+            max_time_ns: None,
+            times_per_round_ns: vec![42, 42 * 2, 42 * 3, 42 * 4, 42 * 5, 42 * 6],
+        };
+
+        let total_rounds = raw_bench.iters_per_round.iter().sum::<u128>() as f64;
+
+        let benchmark: WalltimeBenchmark = raw_bench.into();
+        assert_eq!(benchmark.stats.stdev_ns, 0.);
+        assert_eq!(benchmark.stats.min_ns, 42.);
+        assert_eq!(benchmark.stats.max_ns, 42.);
+        assert_eq!(benchmark.stats.mean_ns, 42.);
+        assert_eq!(
+            benchmark.stats.total_time,
+            42. * total_rounds / 1_000_000_000.0
+        );
+    }
 }

crates/codspeed/src/walltime.rs

@@ -15,24 +15,24 @@ pub struct BenchmarkMetadata {
 pub struct RawWallTimeData {
     #[serde(flatten)]
     pub metadata: BenchmarkMetadata,
-    pub iter_per_round: u32,
+    pub iters_per_round: Vec<u128>,
+    pub times_per_round_ns: Vec<u128>,
     pub max_time_ns: Option<u128>,
-    pub times_ns: Vec<u128>,
 }
 
 impl RawWallTimeData {
     fn from_runtime_data(
         name: String,
         uri: String,
-        iter_per_round: u32,
+        iters_per_round: Vec<u128>,
+        times_per_round_ns: Vec<u128>,
         max_time_ns: Option<u128>,
-        times_ns: Vec<u128>,
     ) -> Self {
         RawWallTimeData {
             metadata: BenchmarkMetadata { name, uri },
-            iter_per_round,
+            iters_per_round,
             max_time_ns,
-            times_ns,
+            times_per_round_ns,
         }
     }
 
@@ -50,13 +50,37 @@ impl RawWallTimeData {
 /// Entry point called in patched integration to harvest raw walltime data
 ///
 /// `CODSPEED_CARGO_WORKSPACE_ROOT` is expected to be set for this to work
+///
+/// # Arguments
+///
+/// - `scope`: The used integration, e.g. "divan" or "criterion"
+/// - `name`: The name of the benchmark
+/// - `uri`: The URI of the benchmark
+/// - `iters_per_round`: The number of iterations for each round (=sample_size), e.g. `[1, 2, 3]` (variable) or `[2, 2, 2, 2]` (constant).
+/// - `times_per_round_ns`: The measured time for each round in nanoseconds, e.g. `[1000, 2000, 3000]`
+/// - `max_time_ns`: The time limit for the benchmark in nanoseconds (if defined)
+///
+/// # Pseudo-code
+///
+/// ```text
+/// let sample_count = /* The number of executions for the same benchmark. */
+/// let sample_size = iters_per_round = vec![/* The number of iterations within each sample. */];
+/// for round in 0..sample_count {
+///     let times_per_round_ns = 0;
+///     for iteration in 0..sample_size[round] {
+///         run_benchmark();
+///         times_per_round_ns += /* measured execution time */;
+///     }
+/// }
+/// ```
+///
 pub fn collect_raw_walltime_results(
     scope: &str,
     name: String,
     uri: String,
-    iter_per_round: u32,
+    iters_per_round: Vec<u128>,
+    times_per_round_ns: Vec<u128>,
     max_time_ns: Option<u128>,
-    times_ns: Vec<u128>,
 ) {
     if !crate::utils::running_with_codspeed_runner() {
         return;
@@ -66,7 +90,13 @@ pub fn collect_raw_walltime_results(
         eprintln!("codspeed failed to get workspace root. skipping");
         return;
     };
-    let data = RawWallTimeData::from_runtime_data(name, uri, iter_per_round, max_time_ns, times_ns);
+    let data = RawWallTimeData::from_runtime_data(
+        name,
+        uri,
+        iters_per_round,
+        times_per_round_ns,
+        max_time_ns,
+    );
     data.dump_to_results(&workspace_root, scope);
 }
 

crates/criterion_compat/criterion_fork/src/analysis/mod.rs

@@ -258,7 +258,7 @@ pub(crate) fn common<M: Measurement, T: ?Sized>(
     }
 
     if criterion.should_save_baseline() && ::codspeed::utils::running_with_codspeed_runner() {
-        codspeed::collect_walltime_results(id, criterion, &iters, avg_times);
+        codspeed::collect_walltime_results(id, criterion, &iters, &times);
     }
 }
 
@@ -293,7 +293,7 @@ mod codspeed {
         id: &BenchmarkId,
         c: &Criterion<M>,
         iters: &[f64],
-        avg_times: &[f64],
+        times: &[f64],
     ) {
         let (uri, bench_name) = create_uri_and_name(id, c);
 
@@ -306,17 +306,17 @@ mod codspeed {
             }
         }
 
-        let avg_iter_per_round = iters.iter().sum::<f64>() / iters.len() as f64;
+        let iters_per_round = iters.iter().map(|t| *t as u128).collect();
+        let times_per_round_ns = times.iter().map(|t| *t as u128).collect();
         let max_time_ns = Some(c.config.measurement_time.as_nanos());
-        let times_ns = avg_times.iter().map(|t| *t as u128).collect();
 
         ::codspeed::walltime::collect_raw_walltime_results(
             "criterion",
             bench_name,
             uri,
-            avg_iter_per_round as u32,
+            iters_per_round,
+            times_per_round_ns,
             max_time_ns,
-            times_ns,
         );
     }
 }

crates/divan_compat/divan_fork/src/divan.rs

@@ -424,7 +424,9 @@ mod codspeed {
         };
 
         let iter_per_round = bench_context.samples.sample_size;
-        let times_ns: Vec<_> =
+        let iters_per_round =
+            vec![iter_per_round as u128; bench_context.samples.time_samples.len()];
+        let times_per_round_ns: Vec<_> =
             bench_context.samples.time_samples.iter().map(|s| s.duration.picos / 1_000).collect();
         let max_time_ns = bench_context.options.max_time.map(|t| t.as_nanos());
 
@@ -441,9 +443,9 @@ mod codspeed {
             "divan",
             bench_name,
             uri,
-            iter_per_round,
+            iters_per_round,
+            times_per_round_ns,
             max_time_ns,
-            times_ns,
         );
     }
 }