Merge pull request #127 from TheDragonCode/4.x

Add warmup support to benchmark iterations and tests

Author: andrey-helldar

README.md

@@ -86,6 +86,27 @@ new Benchmark()
     ->toConsole();
 ```
 
+### Warm-up
+
+Use the `warmup` method to run each callback a few times before the measured iterations. Warm-up runs behave
+identically to regular ones (`beforeEach` / `afterEach` are invoked, the progress bar advances), but their
+time and memory are not included in the final statistics.
+
+This stabilizes results by absorbing one-time costs such as cold caches, JIT/opcache priming and lazy autoloading.
+
+Disabled by default.
+
+```php
+use DragonCode\Benchmark\Benchmark;
+
+new Benchmark()
+    ->warmup(3) // run 3 warm-up iterations per callback before measuring
+    ->compare(
+        foo: fn () => /* some code */,
+        bar: fn () => /* some code */,
+    );
+```
+
 ### Round Precision
 
 Use the `round` method to set the number of decimal places in console output:

src/Benchmark.php

@@ -27,6 +27,8 @@ class Benchmark
 
     protected int $deviations = 1;
 
+    protected int $warmup = 0;
+
     public function __construct(
         protected RunnerService $runner = new RunnerService,
         protected ViewService $view = new ViewService,
@@ -57,6 +59,19 @@ public function snapshots(string $directory): static
         return $this;
     }
 
+    /**
+     * Specifies the number of warm-up iterations for each callback.
+     *
+     * @param  int<1, max>  $count
+     * @return $this
+     */
+    public function warmup(int $count = 1): static
+    {
+        $this->warmup = max(1, abs($count));
+
+        return $this;
+    }
+
     /**
      * Sets a callback to be executed before all iterations for each comparison.
      *
@@ -292,7 +307,10 @@ protected function withProgress(Closure $callback, int $total): void
      */
     protected function steps(array $callbacks, int $multiplier = 1): int
     {
-        return count($callbacks) * $this->iterations * $multiplier;
+        $count  = count($callbacks);
+        $warmup = $count * $this->warmup;
+
+        return $count * $this->iterations * $multiplier + $warmup;
     }
 
     /**
@@ -321,14 +339,18 @@ protected function chunks(array $callbacks, ProgressBar $progressBar): void
      */
     protected function run(mixed $name, Closure $callback, ProgressBar $progressBar): void
     {
-        for ($i = 1; $i <= $this->iterations; $i++) {
+        $warmedUp = $this->warmup === 0;
+
+        for ($i = 1; $i <= $this->iterations + $this->warmup; $i++) {
             $result = $this->callbacks->performBeforeEach($name, $i);
 
             [$time, $memory] = $this->call($callback, [$i, $result]);
 
             $this->callbacks->performAfterEach($name, $i, $time, $memory);
 
-            $this->push($name, $time, $memory);
+            if ($warmedUp) {
+                $this->push($name, $time, $memory);
+            }
 
             $progressBar->advance();
         }

tests/.pest/snapshots/Unit/WarmUp/WarmUpTest/warmup_with_data_set___10__.snap

@@ -0,0 +1,16 @@
+
+
+ 0/26 [░░░░░░░░░░░░░░░░░░░░░░░░░░░░] 0% 1/26 [▓░░░░░░░░░░░░░░░░░░░░░░░░░░░] 3% 2/26 [▓▓░░░░░░░░░░░░░░░░░░░░░░░░░░] 7% 3/26 [▓▓▓░░░░░░░░░░░░░░░░░░░░░░░░░] 11% 4/26 [▓▓▓▓░░░░░░░░░░░░░░░░░░░░░░░░] 15% 5/26 [▓▓▓▓▓░░░░░░░░░░░░░░░░░░░░░░░] 19% 6/26 [▓▓▓▓▓▓░░░░░░░░░░░░░░░░░░░░░░] 23% 7/26 [▓▓▓▓▓▓▓░░░░░░░░░░░░░░░░░░░░░] 26% 8/26 [▓▓▓▓▓▓▓▓░░░░░░░░░░░░░░░░░░░░] 30% 9/26 [▓▓▓▓▓▓▓▓▓░░░░░░░░░░░░░░░░░░░] 34% 10/26 [▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░░░░░░░░] 38% 11/26 [▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░░░░░░░] 42% 12/26 [▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░░░░░░] 46% 13/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░░░░] 50% 14/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░░░] 53% 15/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░░] 57% 16/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░] 61% 17/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░] 65% 18/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░] 69% 19/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░] 73% 20/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░] 76% 21/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░] 80% 22/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░] 84% 23/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░] 88% 24/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░] 92% 25/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░] 96% 26/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓] 100% 26/26 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓] 100%
+
+
+
++-------+------------------------+------------------------+
+| #     | 0                      | 1                      |
++-------+------------------------+------------------------+
+| min   | 15.6789 ms - 202 bytes | 2.3457 ms - 102 bytes  |
+| max   | 112.789 ms - 209 bytes | 9.7568 ms - 109 bytes  |
+| avg   | 53.0252 ms - 205 bytes | 5.9429 ms - 105 bytes  |
+| total | 424.202 ms - 1.61 KB   | 47.5432 ms - 844 bytes |
++-------+------------------------+------------------------+
+| order | 2                      | 1                      |
++-------+------------------------+------------------------+

tests/.pest/snapshots/Unit/WarmUp/WarmUpTest/warmup_with_data_set___1__.snap

@@ -0,0 +1,16 @@
+
+
+ 0/8 [░░░░░░░░░░░░░░░░░░░░░░░░░░░░] 0% 1/8 [▓▓▓░░░░░░░░░░░░░░░░░░░░░░░░░] 12% 2/8 [▓▓▓▓▓▓▓░░░░░░░░░░░░░░░░░░░░░] 25% 3/8 [▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░░░░░░░░] 37% 4/8 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░░░░] 50% 5/8 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░] 62% 6/8 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░] 75% 7/8 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░] 87% 8/8 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓] 100% 8/8 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓] 100%
+
+
+
++-------+------------------------+------------------------+
+| #     | 0                      | 1                      |
++-------+------------------------+------------------------+
+| min   | 15.6789 ms - 202 bytes | 2.3457 ms - 102 bytes  |
+| max   | 112.789 ms - 209 bytes | 9.7568 ms - 109 bytes  |
+| avg   | 53.0252 ms - 205 bytes | 5.9429 ms - 105 bytes  |
+| total | 424.202 ms - 1.61 KB   | 47.5432 ms - 844 bytes |
++-------+------------------------+------------------------+
+| order | 2                      | 1                      |
++-------+------------------------+------------------------+

tests/.pest/snapshots/Unit/WarmUp/WarmUpTest/warmup_with_data_set___5__.snap

@@ -0,0 +1,16 @@
+
+
+ 0/16 [░░░░░░░░░░░░░░░░░░░░░░░░░░░░] 0% 1/16 [▓░░░░░░░░░░░░░░░░░░░░░░░░░░░] 6% 2/16 [▓▓▓░░░░░░░░░░░░░░░░░░░░░░░░░] 12% 3/16 [▓▓▓▓▓░░░░░░░░░░░░░░░░░░░░░░░] 18% 4/16 [▓▓▓▓▓▓▓░░░░░░░░░░░░░░░░░░░░░] 25% 5/16 [▓▓▓▓▓▓▓▓░░░░░░░░░░░░░░░░░░░░] 31% 6/16 [▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░░░░░░░░] 37% 7/16 [▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░░░░░░] 43% 8/16 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░░░░] 50% 9/16 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░░░] 56% 10/16 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░░░] 62% 11/16 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░░░] 68% 12/16 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░░] 75% 13/16 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░░░] 81% 14/16 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░░░] 87% 15/16 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓░░] 93% 16/16 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓] 100% 16/16 [▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓] 100%
+
+
+
++-------+------------------------+------------------------+
+| #     | 0                      | 1                      |
++-------+------------------------+------------------------+
+| min   | 15.6789 ms - 202 bytes | 2.3457 ms - 102 bytes  |
+| max   | 112.789 ms - 209 bytes | 9.7568 ms - 109 bytes  |
+| avg   | 53.0252 ms - 205 bytes | 5.9429 ms - 105 bytes  |
+| total | 424.202 ms - 1.61 KB   | 47.5432 ms - 844 bytes |
++-------+------------------------+------------------------+
+| order | 2                      | 1                      |
++-------+------------------------+------------------------+

tests/Unit/WarmUp/WarmUpTest.php

@@ -0,0 +1,28 @@
+<?php
+
+declare(strict_types=1);
+
+use DragonCode\Benchmark\Benchmark;
+use Tests\Fixtures\CollectorFixture;
+
+test('warmup', function (int $warmup) {
+    $count = 0;
+
+    (new Benchmark(collector: new CollectorFixture))
+        ->iterations(3)
+        ->round(4)
+        ->warmup($warmup)
+        ->compare(
+            foo: function () use (&$count) {
+                $count++;
+            },
+            bar: function () use (&$count) {
+                $count++;
+            },
+        )
+        ->toConsole();
+
+    expect($count)->toBe(6 + $warmup * 2);
+
+    expectOutputToMatchSnapshot();
+})->with([1, 5, 10]);