testing: add -test.shuffle to order randomize test and benchmark order

Author: dgryski

compileopts/config.go

@@ -550,6 +550,7 @@ type TestConfig struct {
 	BenchRegexp       string
 	BenchTime         string
 	BenchMem          bool
+	Shuffle           string
 }
 
 // filterTags removes predefined build tags for a target if a conflicting option

main.go

@@ -251,6 +251,9 @@ func Test(pkgName string, stdout, stderr io.Writer, options *compileopts.Options
 	if testConfig.Count != nil && *testConfig.Count != 1 {
 		flags = append(flags, "-test.count="+strconv.Itoa(*testConfig.Count))
 	}
+	if testConfig.Shuffle != "" {
+		flags = append(flags, "-test.shuffle="+testConfig.Shuffle)
+	}
 
 	logToStdout := testConfig.Verbose || testConfig.BenchRegexp != ""
 
@@ -1442,6 +1445,7 @@ func main() {
 		flag.StringVar(&testConfig.BenchRegexp, "bench", "", "bench: regexp of benchmarks to run")
 		flag.StringVar(&testConfig.BenchTime, "benchtime", "", "run each benchmark for duration `d`")
 		flag.BoolVar(&testConfig.BenchMem, "benchmem", false, "show memory stats for benchmarks")
+		flag.StringVar(&testConfig.Shuffle, "shuffle", "", "shuffle the order the tests and benchmarks run")
 	}
 
 	// Early command processing, before commands are interpreted by the Go flag

src/testing/testing.go

@@ -15,7 +15,9 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
+	"math/rand"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 	"unicode"
@@ -28,6 +30,7 @@ var (
 	flagShort      bool
 	flagRunRegexp  string
 	flagSkipRegexp string
+	flagShuffle    string
 	flagCount      int
 )
 
@@ -44,6 +47,7 @@ func Init() {
 	flag.BoolVar(&flagShort, "test.short", false, "short: run smaller test suite to save time")
 	flag.StringVar(&flagRunRegexp, "test.run", "", "run: regexp of tests to run")
 	flag.StringVar(&flagSkipRegexp, "test.skip", "", "skip: regexp of tests to run")
+	flag.StringVar(&flagShuffle, "test.shuffle", "off", "shuffle: off, on, <numeric-seed>")
 
 	flag.IntVar(&flagCount, "test.count", 1, "run each test or benchmark `count` times")
 
@@ -479,6 +483,27 @@ type testDeps interface {
 	MatchString(pat, str string) (bool, error)
 }
 
+func (m *M) shuffle() error {
+	var n int64
+
+	if flagShuffle == "on" {
+		n = time.Now().UnixNano()
+	} else {
+		var err error
+		n, err = strconv.ParseInt(flagShuffle, 10, 64)
+		if err != nil {
+			m.exitCode = 2
+			return fmt.Errorf(`testing: -shuffle should be "off", "on", or a valid integer: %v`, err)
+		}
+	}
+
+	fmt.Println("-test.shuffle", n)
+	rng := rand.New(rand.NewSource(n))
+	rng.Shuffle(len(m.Tests), func(i, j int) { m.Tests[i], m.Tests[j] = m.Tests[j], m.Tests[i] })
+	rng.Shuffle(len(m.Benchmarks), func(i, j int) { m.Benchmarks[i], m.Benchmarks[j] = m.Benchmarks[j], m.Benchmarks[i] })
+	return nil
+}
+
 // Run runs the tests. It returns an exit code to pass to os.Exit.
 func (m *M) Run() (code int) {
 	defer func() {
@@ -489,6 +514,13 @@ func (m *M) Run() (code int) {
 		flag.Parse()
 	}
 
+	if flagShuffle != "off" {
+		if err := m.shuffle(); err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			return
+		}
+	}
+
 	testRan, testOk := runTests(m.deps.MatchString, m.Tests)
 	if !testRan && *matchBenchmarks == "" {
 		fmt.Fprintln(os.Stderr, "testing: warning: no tests to run")