--wip-- [skip ci]

GuillaumeLagrange · GuillaumeLagrange · commit 7d5671fdb967 · 2026-05-21T13:11:04.000+02:00
diff --git a/crates/criterion_compat/benches/criterion_integration/iter_manual.rs b/crates/criterion_compat/benches/criterion_integration/iter_manual.rs
@@ -1,27 +1,11 @@
 use codspeed_criterion_compat::{criterion_group, Criterion, IterManualOptions};
 
-fn iter_manual_basic(c: &mut Criterion) {
-    // rounds=5 is below criterion's sample_size >= 10 floor — proves
-    // iter_manual fully bypasses the adaptive outer sampler.
-    c.bench_function("iter_manual_basic", |b| {
-        b.iter_manual(IterManualOptions::new(5, 100), || {
-            let mut s = 0u64;
-            for i in 0..32 {
-                s = s.wrapping_add(i);
-            }
-            s
-        });
-    });
-}
-
 fn iter_manual_with_setup(c: &mut Criterion) {
+    // rounds=3 is below criterion's sample_size >= 10 floor — proves
+    // iter_manual fully bypasses the adaptive outer sampler.
     c.bench_function("iter_manual_setup", |b| {
         b.iter_manual_setup(
-            IterManualOptions {
-                rounds: 3,
-                iterations: 100,
-                warmup_rounds: 2,
-            },
+            IterManualOptions::new().rounds(3).iters(100).warmup(2),
             || (0u64..256).collect::<Vec<_>>(),
             |input| input.iter().copied().sum::<u64>(),
         );
@@ -31,30 +15,31 @@ fn iter_manual_with_setup(c: &mut Criterion) {
 fn iter_manual_with_teardown(c: &mut Criterion) {
     c.bench_function("iter_manual_setup_teardown", |b| {
         b.iter_manual_setup_teardown(
-            IterManualOptions {
-                rounds: 7,
-                iterations: 50,
-                warmup_rounds: 1,
-            },
+            IterManualOptions::new().rounds(7).iters(50).warmup(1),
             || (0u64..128).collect::<Vec<_>>(),
             |input| input.iter().copied().sum::<u64>(),
             drop,
         );
     });
 }
 
-#[cfg(feature = "async_futures")]
-fn iter_manual_async_basic(c: &mut Criterion) {
-    use codspeed_criterion_compat::async_executor::FuturesExecutor;
-    c.bench_function("iter_manual_async_basic", |b| {
-        b.to_async(FuturesExecutor)
-            .iter_manual(IterManualOptions::new(5, 100), || async {
-                let mut s = 0u64;
-                for i in 0..32 {
-                    s = s.wrapping_add(i);
+// Setup body deliberately does a chunk of work so it stands out in flamegraphs.
+// The measured region should NOT include this work.
+fn iter_manual_heavy_setup(c: &mut Criterion) {
+    c.bench_function("iter_manual_heavy_setup", |b| {
+        b.iter_manual_setup(
+            IterManualOptions::new().rounds(3).iters(50).warmup(1),
+            || {
+                // Heavy setup: build a large vector. Should NOT appear inside
+                // the measured region of the flamegraph.
+                let mut v = Vec::with_capacity(10_000);
+                for i in 0..10_000u64 {
+                    v.push(i.wrapping_mul(31));
                 }
-                s
-            });
+                v
+            },
+            |input| input.iter().copied().sum::<u64>(),
+        );
     });
 }
 
@@ -63,11 +48,7 @@ fn iter_manual_async_with_setup(c: &mut Criterion) {
     use codspeed_criterion_compat::async_executor::FuturesExecutor;
     c.bench_function("iter_manual_async_setup", |b| {
         b.to_async(FuturesExecutor).iter_manual_setup(
-            IterManualOptions {
-                rounds: 3,
-                iterations: 100,
-                warmup_rounds: 2,
-            },
+            IterManualOptions::new().rounds(3).iters(100).warmup(2),
             || (0u64..256).collect::<Vec<_>>(),
             |input| {
                 let sum = input.iter().copied().sum::<u64>();
@@ -82,11 +63,7 @@ fn iter_manual_async_with_teardown(c: &mut Criterion) {
     use codspeed_criterion_compat::async_executor::FuturesExecutor;
     c.bench_function("iter_manual_async_setup_teardown", |b| {
         b.to_async(FuturesExecutor).iter_manual_setup_teardown(
-            IterManualOptions {
-                rounds: 7,
-                iterations: 50,
-                warmup_rounds: 1,
-            },
+            IterManualOptions::new().rounds(7).iters(50).warmup(1),
             || (0u64..128).collect::<Vec<_>>(),
             |input| {
                 let sum = input.iter().copied().sum::<u64>();
@@ -97,19 +74,16 @@ fn iter_manual_async_with_teardown(c: &mut Criterion) {
     });
 }
 
-#[cfg(not(feature = "async_futures"))]
-fn iter_manual_async_basic(_c: &mut Criterion) {}
 #[cfg(not(feature = "async_futures"))]
 fn iter_manual_async_with_setup(_c: &mut Criterion) {}
 #[cfg(not(feature = "async_futures"))]
 fn iter_manual_async_with_teardown(_c: &mut Criterion) {}
 
 criterion_group!(
     benches,
-    iter_manual_basic,
     iter_manual_with_setup,
     iter_manual_with_teardown,
-    iter_manual_async_basic,
+    iter_manual_heavy_setup,
     iter_manual_async_with_setup,
     iter_manual_async_with_teardown,
 );
diff --git a/crates/criterion_compat/criterion_fork/src/codspeed_iter_manual.rs b/crates/criterion_compat/criterion_fork/src/codspeed_iter_manual.rs
@@ -1,11 +1,11 @@
 //! CodSpeed addition: manual control over benchmark iteration counts.
 //!
-//! `iter_manual*` lets the user pin down the exact number of measurement
+//! `iter_manual_setup*` lets the user pin down the exact number of measurement
 //! rounds, inner iterations per round, and warmup rounds. It bypasses
 //! Criterion's adaptive sampler entirely — see `routine.rs::sample` for the
 //! short-circuit.
 
-use std::time::Instant;
+use std::time::{Duration, Instant};
 
 use codspeed::instrument_hooks::InstrumentHooks;
 
@@ -20,28 +20,55 @@ use crate::Bencher;
 #[cfg(feature = "async")]
 use std::future::Future;
 
-/// Options for the [`iter_manual`](Bencher::iter_manual) family.
+/// Options for the [`iter_manual_setup`](Bencher::iter_manual_setup) family.
+///
+/// Built with a consuming-self builder to match the rest of criterion's
+/// configuration APIs (e.g. [`Criterion`](crate::Criterion)).
 #[derive(Debug, Clone, Copy)]
 pub struct IterManualOptions {
-    /// Number of measurement rounds (each produces one sample).
-    pub rounds: u64,
-    /// Number of routine invocations inside the measured region of each round.
-    pub iterations: u64,
-    /// Number of unmeasured warmup rounds run before measurement starts.
-    pub warmup_rounds: u64,
+    rounds: u64,
+    iterations: u64,
+    warmup_iterations: u64,
 }
 
-impl IterManualOptions {
-    /// Build options with the given rounds and iterations; `warmup_rounds` defaults to 0.
-    pub fn new(rounds: u64, iterations: u64) -> Self {
+impl Default for IterManualOptions {
+    fn default() -> Self {
         Self {
-            rounds,
-            iterations,
-            warmup_rounds: 0,
+            rounds: 1,
+            iterations: 1,
+            warmup_iterations: 0,
         }
     }
 }
 
+impl IterManualOptions {
+    /// Start with defaults: 1 round, 1 iteration per round, 0 warmup rounds.
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Number of measurement rounds (each produces one sample).
+    #[must_use]
+    pub fn rounds(mut self, rounds: u64) -> Self {
+        self.rounds = rounds;
+        self
+    }
+
+    /// Number of routine invocations inside a measurement round.
+    #[must_use]
+    pub fn iters(mut self, iterations: u64) -> Self {
+        self.iterations = iterations;
+        self
+    }
+
+    /// Number of unmeasured warmup iterations run before measurement starts.
+    #[must_use]
+    pub fn warmup(mut self, warmup_iterations: u64) -> Self {
+        self.warmup_iterations = warmup_iterations;
+        self
+    }
+}
+
 /// Captured output of a manual run. Stored on the `Bencher` and read by
 /// `routine.rs::sample` to short-circuit the adaptive sampler.
 pub(crate) struct ManualMeasurement {
@@ -54,32 +81,40 @@ pub(crate) struct ManualMeasurement {
 
 impl<'a, M: Measurement> Bencher<'a, M> {
     /// Run `routine` exactly `opts.iterations` times inside each of `opts.rounds`
-    /// measurement rounds, optionally preceded by `opts.warmup_rounds` unmeasured rounds.
+    /// measurement rounds, with a `setup` closure producing fresh input for
+    /// each round (outside the measured region). Optionally preceded by
+    /// `opts.warmup_rounds` unmeasured rounds.
     ///
     /// Criterion's adaptive sampler is bypassed for this benchmark.
     #[inline(never)]
-    pub fn iter_manual<O, R>(&mut self, opts: IterManualOptions, mut routine: R)
-    where
-        R: FnMut() -> O,
-    {
-        self.iter_manual_setup_teardown(opts, || (), |_| routine(), |_| ());
-    }
-
-    /// Like [`iter_manual`](Self::iter_manual), with a `setup` closure producing
-    /// fresh input for each round. The routine borrows the input mutably.
-    #[inline(never)]
     pub fn iter_manual_setup<I, O, S, R>(&mut self, opts: IterManualOptions, setup: S, routine: R)
     where
         S: FnMut() -> I,
         R: FnMut(&mut I) -> O,
     {
-        self.iter_manual_setup_teardown(opts, setup, routine, |_| ());
+        self.__codspeed_root_frame__iter_manual_setup_teardown(opts, setup, routine, |_| ());
     }
 
     /// Like [`iter_manual_setup`](Self::iter_manual_setup), with a `teardown`
     /// closure called after each round, outside the measured region.
     #[inline(never)]
     pub fn iter_manual_setup_teardown<I, O, S, R, T>(
+        &mut self,
+        opts: IterManualOptions,
+        setup: S,
+        routine: R,
+        teardown: T,
+    ) where
+        S: FnMut() -> I,
+        R: FnMut(&mut I) -> O,
+        T: FnMut(I),
+    {
+        self.__codspeed_root_frame__iter_manual_setup_teardown(opts, setup, routine, teardown);
+    }
+
+    #[inline(never)]
+    #[allow(missing_docs, non_snake_case)]
+    pub fn __codspeed_root_frame__iter_manual_setup_teardown<I, O, S, R, T>(
         &mut self,
         opts: IterManualOptions,
         mut setup: S,
@@ -92,10 +127,9 @@ impl<'a, M: Measurement> Bencher<'a, M> {
     {
         self.iterated = true;
 
-        let bench_start = InstrumentHooks::current_timestamp();
-        let time_start = Instant::now();
+        let mut elapsed_time = Duration::ZERO;
 
-        for _ in 0..opts.warmup_rounds {
+        for _ in 0..opts.warmup_iterations {
             let mut input = black_box(setup());
             for _ in 0..opts.iterations {
                 black_box(routine(&mut input));
@@ -106,18 +140,23 @@ impl<'a, M: Measurement> Bencher<'a, M> {
         let mut samples = Vec::with_capacity(opts.rounds as usize);
         for _ in 0..opts.rounds {
             let mut input = black_box(setup());
+
+            let bench_start = InstrumentHooks::current_timestamp();
+            let round_start = Instant::now();
             let start = self.measurement.start();
             for _ in 0..opts.iterations {
                 black_box(routine(&mut input));
             }
             let value = self.measurement.end(start);
+            elapsed_time += round_start.elapsed();
+            let bench_end = InstrumentHooks::current_timestamp();
+            InstrumentHooks::instance().add_benchmark_timestamps(bench_start, bench_end);
+
             teardown(input);
             samples.push(self.measurement.to_f64(&value));
         }
 
-        self.elapsed_time = time_start.elapsed();
-        let bench_end = InstrumentHooks::current_timestamp();
-        InstrumentHooks::instance().add_benchmark_timestamps(bench_start, bench_end);
+        self.elapsed_time = elapsed_time;
 
         self.codspeed_manual = Some(ManualMeasurement {
             samples,
@@ -128,16 +167,6 @@ impl<'a, M: Measurement> Bencher<'a, M> {
 
 #[cfg(feature = "async")]
 impl<'a, 'b, A: AsyncExecutor, M: Measurement> AsyncBencher<'a, 'b, A, M> {
-    /// Async/await variant of [`Bencher::iter_manual`].
-    #[inline(never)]
-    pub fn iter_manual<O, R, F>(&mut self, opts: IterManualOptions, mut routine: R)
-    where
-        R: FnMut() -> F,
-        F: Future<Output = O>,
-    {
-        self.iter_manual_setup_teardown(opts, || (), |_| routine(), |_| std::future::ready(()));
-    }
-
     /// Async/await variant of [`Bencher::iter_manual_setup`].
     #[inline(never)]
     pub fn iter_manual_setup<I, O, S, R, F>(
@@ -150,12 +179,32 @@ impl<'a, 'b, A: AsyncExecutor, M: Measurement> AsyncBencher<'a, 'b, A, M> {
         R: FnMut(&mut I) -> F,
         F: Future<Output = O>,
     {
-        self.iter_manual_setup_teardown(opts, setup, routine, |_| std::future::ready(()));
+        self.__codspeed_root_frame__iter_manual_setup_teardown(opts, setup, routine, |_| {
+            std::future::ready(())
+        });
     }
 
     /// Async/await variant of [`Bencher::iter_manual_setup_teardown`].
     #[inline(never)]
     pub fn iter_manual_setup_teardown<I, O, S, R, T, RF, TF>(
+        &mut self,
+        opts: IterManualOptions,
+        setup: S,
+        routine: R,
+        teardown: T,
+    ) where
+        S: FnMut() -> I,
+        R: FnMut(&mut I) -> RF,
+        T: FnMut(I) -> TF,
+        RF: Future<Output = O>,
+        TF: Future<Output = ()>,
+    {
+        self.__codspeed_root_frame__iter_manual_setup_teardown(opts, setup, routine, teardown);
+    }
+
+    #[inline(never)]
+    #[allow(missing_docs, non_snake_case)]
+    pub fn __codspeed_root_frame__iter_manual_setup_teardown<I, O, S, R, T, RF, TF>(
         &mut self,
         opts: IterManualOptions,
         mut setup: S,
@@ -172,8 +221,7 @@ impl<'a, 'b, A: AsyncExecutor, M: Measurement> AsyncBencher<'a, 'b, A, M> {
         runner.block_on(async {
             b.iterated = true;
 
-            let bench_start = InstrumentHooks::current_timestamp();
-            let time_start = Instant::now();
+            let mut elapsed_time = Duration::ZERO;
 
             for _ in 0..opts.warmup_rounds {
                 let mut input = black_box(setup());
@@ -186,18 +234,23 @@ impl<'a, 'b, A: AsyncExecutor, M: Measurement> AsyncBencher<'a, 'b, A, M> {
             let mut samples = Vec::with_capacity(opts.rounds as usize);
             for _ in 0..opts.rounds {
                 let mut input = black_box(setup());
+
+                let bench_start = InstrumentHooks::current_timestamp();
+                let round_start = Instant::now();
                 let start = b.measurement.start();
                 for _ in 0..opts.iterations {
                     black_box(routine(&mut input).await);
                 }
                 let value = b.measurement.end(start);
+                elapsed_time += round_start.elapsed();
+                let bench_end = InstrumentHooks::current_timestamp();
+                InstrumentHooks::instance().add_benchmark_timestamps(bench_start, bench_end);
+
                 teardown(input).await;
                 samples.push(b.measurement.to_f64(&value));
             }
 
-            b.elapsed_time = time_start.elapsed();
-            let bench_end = InstrumentHooks::current_timestamp();
-            InstrumentHooks::instance().add_benchmark_timestamps(bench_start, bench_end);
+            b.elapsed_time = elapsed_time;
 
             b.codspeed_manual = Some(ManualMeasurement {
                 samples,
diff --git a/crates/criterion_compat/src/compat/bencher.rs b/crates/criterion_compat/src/compat/bencher.rs
