wip: add analysis mode impl

Author: not-matthias

example/rle_test.go

@@ -0,0 +1,47 @@
+package example
+
+import (
+	"strconv"
+	"strings"
+	"testing"
+)
+
+func rleBad(s string) string {
+	if len(s) == 0 {
+		return ""
+	}
+	var out string
+	run := 1
+	for i := 1; i <= len(s); i++ {
+		if i < len(s) && s[i] == s[i-1] {
+			run++
+			continue
+		}
+		out += string(s[i-1])
+		out += strconv.Itoa(run)
+		out += "|"
+		run = 1
+	}
+	return out
+}
+
+func BenchmarkRLEBad(b *testing.B) {
+	in := strings.Repeat("AAAABBBCCDAA", 200)
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		_ = rleBad(in)
+	}
+}
+
+func leakyFunction() {
+	s := make([]string, 3)
+	for i := 0; i < 10000000; i++ {
+		s = append(s, "magical pandas")
+	}
+}
+
+func BenchmarkLeakyFunction(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		leakyFunction()
+	}
+}

go-runner/overlay/benchmark1.24.0.go

@@ -284,7 +284,7 @@ var labelsOnce sync.Once
 // subbenchmarks. b must not have subbenchmarks.
 func (b *B) run() {
 	labelsOnce.Do(func() {
-		fmt.Fprintf(b.w, "Running with CodSpeed (mode: walltime)\n")
+		fmt.Fprintf(b.w, "Running with CodSpeed (mode: %s)\n", getCodspeedRunnerMode())
 
 		fmt.Fprintf(b.w, "goos: %s\n", runtime.GOOS)
 		fmt.Fprintf(b.w, "goarch: %s\n", runtime.GOARCH)
@@ -347,8 +347,9 @@ func (b *B) launch() {
 	// b.Loop does its own ramp-up logic so we just need to run it once.
 	// If b.loop.n is non zero, it means b.Loop has already run.
 	if b.loop.n == 0 {
-		// Run the benchmark for at least the specified amount of time.
-		if b.benchTime.n > 0 {
+		if isAnalysisMode() {
+			runBenchmarkSingle(b)
+		} else if b.benchTime.n > 0 {
 			// We already ran a single iteration in run1.
 			// If -benchtime=1x was requested, use that result.
 			// See https://golang.org/issue/32051.
@@ -421,7 +422,9 @@ func (b *B) loopSlowPath() bool {
 
 	if b.loop.n == 0 {
 		// It's the first call to b.Loop() in the benchmark function.
-		if b.benchTime.n > 0 {
+		if isAnalysisMode() {
+			b.loop.n = 1
+		} else if b.benchTime.n > 0 {
 			// Fixed iteration count.
 			b.loop.n = uint64(b.benchTime.n)
 		} else {
@@ -442,7 +445,9 @@ func (b *B) loopSlowPath() bool {
 
 	// Should we keep iterating?
 	var more bool
-	if b.benchTime.n > 0 {
+	if isAnalysisMode() {
+		more = false
+	} else if b.benchTime.n > 0 {
 		// The iteration count is fixed, so we should have run this many and now
 		// be done.
 		if b.loop.i != uint64(b.benchTime.n) {

go-runner/overlay/benchmark1.25.0.go

@@ -284,7 +284,7 @@ var labelsOnce sync.Once
 // subbenchmarks. b must not have subbenchmarks.
 func (b *B) run() {
 	labelsOnce.Do(func() {
-		fmt.Fprintf(b.w, "Running with CodSpeed (mode: walltime)\n")
+		fmt.Fprintf(b.w, "Running with CodSpeed (mode: %s)\n", getCodspeedRunnerMode())
 
 		fmt.Fprintf(b.w, "goos: %s\n", runtime.GOOS)
 		fmt.Fprintf(b.w, "goarch: %s\n", runtime.GOARCH)
@@ -347,8 +347,9 @@ func (b *B) launch() {
 	// b.Loop does its own ramp-up logic so we just need to run it once.
 	// If b.loop.n is non zero, it means b.Loop has already run.
 	if b.loop.n == 0 {
-		// Run the benchmark for at least the specified amount of time.
-		if b.benchTime.n > 0 {
+		if isAnalysisMode() {
+			runBenchmarkSingle(b)
+		} else if b.benchTime.n > 0 {
 			// We already ran a single iteration in run1.
 			// If -benchtime=1x was requested, use that result.
 			// See https://golang.org/issue/32051.
@@ -421,7 +422,9 @@ func (b *B) loopSlowPath() bool {
 
 	if b.loop.n == 0 {
 		// It's the first call to b.Loop() in the benchmark function.
-		if b.benchTime.n > 0 {
+		if isAnalysisMode() {
+			b.loop.n = 1
+		} else if b.benchTime.n > 0 {
 			// Fixed iteration count.
 			b.loop.n = uint64(b.benchTime.n)
 		} else {
@@ -442,7 +445,9 @@ func (b *B) loopSlowPath() bool {
 
 	// Should we keep iterating?
 	var more bool
-	if b.benchTime.n > 0 {
+	if isAnalysisMode() {
+		more = false
+	} else if b.benchTime.n > 0 {
 		// The iteration count is fixed, so we should have run this many and now
 		// be done.
 		if b.loop.i != uint64(b.benchTime.n) {

go-runner/overlay/codspeed.go

@@ -13,6 +13,28 @@ import (
 	"time"
 )
 
+type codspeedRunnerMode string
+
+const (
+	modeWalltime   codspeedRunnerMode = "walltime"
+	modeSimulation codspeedRunnerMode = "simulation"
+	modeMemory     codspeedRunnerMode = "memory"
+)
+
+func getCodspeedRunnerMode() codspeedRunnerMode {
+	v := strings.ToLower(strings.TrimSpace(os.Getenv("CODSPEED_RUNNER_MODE")))
+	switch codspeedRunnerMode(v) {
+	case modeSimulation, modeMemory:
+		return codspeedRunnerMode(v)
+	default:
+		return modeWalltime
+	}
+}
+
+func isAnalysisMode() bool {
+	return getCodspeedRunnerMode() != modeWalltime
+}
+
 type codspeed struct {
 	instrument_hooks *InstrumentHooks
 
@@ -254,6 +276,9 @@ func (b *B) sendAccumulatedTimestamps() {
 }
 
 func (b *B) SaveMeasurement() {
+	if isAnalysisMode() {
+		return
+	}
 	if b.savedMeasurement {
 		return
 	}
@@ -314,6 +339,13 @@ func (b *B) StartTimerWithoutMarker() {
 	}
 }
 
+func runBenchmarkSingle(b *B) {
+	b.codspeed.instrument_hooks.StartBenchmark()
+	b.runN(1)
+	b.codspeed.instrument_hooks.StopBenchmark()
+	b.sendAccumulatedTimestamps()
+}
+
 func runBenchmarkWithWarmup(b *B) {
 	warmupD := b.benchTime.d / 10
 	warmupN := int64(1)

go-runner/src/lib.rs

@@ -25,8 +25,11 @@ pub fn run_benchmarks<P: AsRef<Path>>(
         bail!("Failed to run benchmarks: {error}");
     }
 
-    let profile_dir = profile_dir.as_ref().to_path_buf();
-    collect_walltime_results(&profile_dir).unwrap();
+    let mode = std::env::var("CODSPEED_RUNNER_MODE").unwrap_or_else(|_| "walltime".into());
+    if mode == "walltime" {
+        let profile_dir = profile_dir.as_ref().to_path_buf();
+        collect_walltime_results(&profile_dir).unwrap();
+    }
 
     Ok(())
 }

go-runner/src/runner/mod.rs

@@ -44,6 +44,10 @@ fn run_cmd<P: AsRef<Path>>(
     cmd.env("GOCACHE", _dir.path().join("gocache"));
     cmd.env("GOMODCACHE", _dir.path().join("gomodcache"));
 
+    if let Ok(mode) = std::env::var("CODSPEED_RUNNER_MODE") {
+        cmd.env("CODSPEED_RUNNER_MODE", mode);
+    }
+
     Ok((_dir, cmd))
 }