fix(testing): incomplete reading of benchmark time in b.N loops

Author: not-matthias

testing/fork.sh

@@ -124,6 +124,7 @@ restore_files "${CODSPEED_FILES[@]}"
 apply_patch "patches/benchmark_stopbenchmark_fail.patch" 10 ".."
 apply_patch "patches/benchmark_stoptimer_mitigation.patch" 10 ".."
 apply_patch "patches/benchmark_benchmarkers_bloop.patch" 10 ".."
+apply_patch "patches/benchmark_savemeasurement_bug.patch" 10 ".."
 
 
 # Run pre-commit and format the code

testing/patches/benchmark_savemeasurement_bug.patch

@@ -0,0 +1,37 @@
+diff --git a/testing/testing/benchmark.go b/testing/testing/benchmark.go
+index 0040973..883976e 100644
+--- a/testing/testing/benchmark.go
++++ b/testing/testing/benchmark.go
+@@ -200,6 +200,12 @@ func (b *B) SaveMeasurement() {
+ 	}
+ 	b.savedMeasurement = true
+
++	// WARN: This function must not be called if the timer is on, because we
++	// would read an incomplete b.duration value.
++	if b.timerOn {
++		panic("SaveMeasurement called with timer on")
++	}
++
+ 	// For b.N loops: This will be called in runN which sets b.N to the number of iterations.
+ 	// For b.Loop() loops: loopSlowPath sets b.N to 0 to prevent b.N loops within b.Loop. However, since
+ 	// we're starting/stopping the timer for each iteration in the b.Loop() loop, we can use 1 as
+@@ -311,8 +317,8 @@ func (b *B) __codspeed_root_frame__runN(n int) {
+ 	b.ResetTimer()
+ 	b.StartTimer()
+ 	b.benchFunc(b)
+-	b.SaveMeasurement()
+ 	b.StopTimer()
++	b.SaveMeasurement()
+ 	b.previousN = n
+ 	b.previousDuration = b.duration
+
+@@ -641,8 +647,8 @@ func (b *B) loopSlowPath() bool {
+ // whereas b.N-based benchmarks must run the benchmark function (and any
+ // associated setup and cleanup) several times.
+ func (b *B) Loop() bool {
+-	b.SaveMeasurement()
+ 	b.StopTimerWithoutMarker()
++	b.SaveMeasurement()
+ 	// This is written such that the fast path is as fast as possible and can be
+ 	// inlined.
+ 	//

testing/testing/benchmark.go

@@ -200,6 +200,12 @@ func (b *B) SaveMeasurement() {
 	}
 	b.savedMeasurement = true
 
+	// WARN: This function must not be called if the timer is on, because we
+	// would read an incomplete b.duration value.
+	if b.timerOn {
+		panic("SaveMeasurement called with timer on")
+	}
+
 	// For b.N loops: This will be called in runN which sets b.N to the number of iterations.
 	// For b.Loop() loops: loopSlowPath sets b.N to 0 to prevent b.N loops within b.Loop. However, since
 	// we're starting/stopping the timer for each iteration in the b.Loop() loop, we can use 1 as
@@ -311,8 +317,8 @@ func (b *B) __codspeed_root_frame__runN(n int) {
 	b.ResetTimer()
 	b.StartTimer()
 	b.benchFunc(b)
-	b.SaveMeasurement()
 	b.StopTimer()
+	b.SaveMeasurement()
 	b.previousN = n
 	b.previousDuration = b.duration
 
@@ -641,8 +647,8 @@ func (b *B) loopSlowPath() bool {
 // whereas b.N-based benchmarks must run the benchmark function (and any
 // associated setup and cleanup) several times.
 func (b *B) Loop() bool {
-	b.SaveMeasurement()
 	b.StopTimerWithoutMarker()
+	b.SaveMeasurement()
 	// This is written such that the fast path is as fast as possible and can be
 	// inlined.
 	//