feat!: Detect retries and flaky tests

ctrf/ctrf.go

@@ -147,6 +147,7 @@ type Summary struct {
 	Pending int   `json:"pending"`
 	Skipped int   `json:"skipped"`
 	Other   int   `json:"other"`
+	Flaky   int   `json:"flaky,omitempty"`
 	Suites  int   `json:"suites,omitempty"`
 	Start   int64 `json:"start"`
 	Stop    int64 `json:"stop"`
@@ -179,16 +180,18 @@ func (summary *Summary) Validate() []error {
 	if summary.Stop < 0 {
 		errs = append(errs, errors.New("invalid property 'results.summary.stop'"))
 	}
+	if summary.Flaky < 0 {
+		errs = append(errs, errors.New("invalid property 'results.summary.flaky'"))
+	}
 	if summary.Suites < 0 {
 		errs = append(errs, errors.New("invalid property 'results.summary.suites'"))
 	}
 	if summary.Start > summary.Stop {
 		errs = append(errs, errors.New("invalid summary timestamps: start can't be greater than stop"))
 	}
-	testsSum := summary.Passed + summary.Failed + summary.Pending + summary.Skipped + summary.Other
+	testsSum := summary.Passed + summary.Failed + summary.Pending + summary.Skipped + summary.Other + summary.Flaky
 	if summary.Tests != testsSum {
 		errs = append(errs, fmt.Errorf("invalid summary counts: tests (%d) must be the sum of passed, failed, pending, skipped, and other (%d)", summary.Tests, testsSum))
-
 	}
 	return errs
 }
@@ -203,27 +206,43 @@ const (
 	TestOther   TestStatus = "other"
 )
 
+type RetryAttempt struct {
+	Attempt  int        `json:"attempt"`
+	Status   TestStatus `json:"status"`
+	Duration int64      `json:"duration,omitempty"`
+	Message  string     `json:"message,omitempty"`
+	Trace    string     `json:"trace,omitempty"`
+	Line     int        `json:"line,omitempty"`
+	Snippet  string     `json:"snippet,omitempty"`
+	Stdout   []string   `json:"stdout,omitempty"`
+	Stderr   []string   `json:"stderr,omitempty"`
+	Start    int64      `json:"start,omitempty"`
+	Stop     int64      `json:"stop,omitempty"`
+	Extra    any        `json:"extra,omitempty"`
+}
+
 type TestResult struct {
-	Name       string     `json:"name"`
-	Status     TestStatus `json:"status"`
-	Duration   int64      `json:"duration"`
-	Start      int64      `json:"start,omitempty"`
-	Stop       int64      `json:"stop,omitempty"`
-	Suite      string     `json:"suite,omitempty"`
-	Message    string     `json:"message,omitempty"`
-	Trace      string     `json:"trace,omitempty"`
-	RawStatus  string     `json:"rawStatus,omitempty"`
-	Tags       []string   `json:"tags,omitempty"`
-	Type       string     `json:"type,omitempty"`
-	Filepath   string     `json:"filePath,omitempty"`
-	Retry      int        `json:"retry,omitempty"`
-	Flake      bool       `json:"flake,omitempty"`
-	Browser    string     `json:"browser,omitempty"`
-	Device     string     `json:"device,omitempty"`
-	Screenshot string     `json:"screenshot,omitempty"`
-	Parameters any        `json:"parameters,omitempty"`
-	Steps      []any      `json:"steps,omitempty"`
-	Extra      any        `json:"extra,omitempty"`
+	Name          string         `json:"name"`
+	Status        TestStatus     `json:"status"`
+	Duration      int64          `json:"duration"`
+	Start         int64          `json:"start,omitempty"`
+	Stop          int64          `json:"stop,omitempty"`
+	Suite         string         `json:"suite,omitempty"`
+	Message       string         `json:"message,omitempty"`
+	Trace         string         `json:"trace,omitempty"`
+	RawStatus     string         `json:"rawStatus,omitempty"`
+	Tags          []string       `json:"tags,omitempty"`
+	Type          string         `json:"type,omitempty"`
+	Filepath      string         `json:"filePath,omitempty"`
+	Retries       int            `json:"retries,omitempty"`
+	Flaky         bool           `json:"flaky,omitempty"`
+	Browser       string         `json:"browser,omitempty"`
+	Device        string         `json:"device,omitempty"`
+	Screenshot    string         `json:"screenshot,omitempty"`
+	Parameters    any            `json:"parameters,omitempty"`
+	Steps         []any          `json:"steps,omitempty"`
+	RetryAttempts []RetryAttempt `json:"retryAttempts,omitempty"`
+	Extra         any            `json:"extra,omitempty"`
 }
 
 type Environment struct {

ctrf/ctrf_test.go

@@ -20,12 +20,13 @@ func TestRequiredProperties(t *testing.T) {
 				Name: "my tool",
 			},
 			Summary: &Summary{
-				Tests:   15,
-				Passed:  5,
-				Failed:  4,
-				Pending: 3,
-				Skipped: 2,
-				Other:   1,
+				Tests:   21,
+				Passed:  6,
+				Failed:  5,
+				Pending: 4,
+				Skipped: 3,
+				Other:   2,
+				Flaky:   1,
 				Start:   42,
 				Stop:    1337,
 			},
@@ -68,12 +69,13 @@ func TestRequiredProperties(t *testing.T) {
       "name": "my tool"
     },
     "summary": {
-      "tests": 15,
-      "passed": 5,
-      "failed": 4,
-      "pending": 3,
-      "skipped": 2,
-      "other": 1,
+      "tests": 21,
+      "passed": 6,
+      "failed": 5,
+      "pending": 4,
+      "skipped": 3,
+      "other": 2,
+      "flaky": 1,
       "start": 42,
       "stop": 1337
     },

examples/flaky/.gitignore

@@ -0,0 +1 @@
+test.json

examples/flaky/README.md

@@ -0,0 +1,11 @@
+# Flaky Test Example
+
+This code in here is an example for a flaky test. It can be run to generate some output suitable for testing flaky behavior and the resulting json output.
+
+At the time of writing, Go does not have built-in support for re-running flaky tests, (though there is an [accepted proposal](https://github.com/golang/go/issues/62244) to address this). However, [gotestsum](https://github.com/gotestyourself/gotestsum) does include support to automatically re-run flaky tests, with the `--rerun-fails=n` and `--rerun-fails-max-failures=n` flags.
+
+The output for the unit test can be generated with the following command from the repo root:
+
+```shell
+gotestsum --jsonfile examples/flaky/test.json --rerun-fails --packages ./examples/flaky -- -count 1 -tags examples
+```

examples/flaky/flaky_test.go

@@ -0,0 +1,63 @@
+//go:build examples
+
+package flaky
+
+import (
+	"errors"
+	"io/fs"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+)
+
+func Test_Flaky_Pass(t *testing.T) {
+	time.Sleep(50 * time.Millisecond)
+}
+
+func Test_Flaky_Fail(t *testing.T) {
+	time.Sleep(50 * time.Millisecond)
+	t.Fatal("This test is designed to fail.")
+}
+
+func Test_Flaky_Skipped(t *testing.T) {
+	t.Skip("This test is designed to be skipped.")
+	time.Sleep(50 * time.Millisecond)
+}
+
+func Test_Flaky_Flaky(t *testing.T) {
+	time.Sleep(50 * time.Millisecond)
+	file := path.Join(os.TempDir(), "flaky_test.tmp")
+	if _, err := os.Stat(file); errors.Is(err, fs.ErrNotExist) {
+		// First run: file does not exist, create it with count "1", and fail.
+		if err := os.WriteFile(file, []byte("1"), 0644); err != nil {
+			t.Fatalf("Failed to create flaky test file: %v", err)
+		}
+		t.Fatal("Flaky Failure (attempt 1)")
+	} else {
+		// File exists, read the current count
+		data, err := os.ReadFile(file)
+		if err != nil {
+			t.Fatalf("Failed to read flaky test file: %v", err)
+		}
+		count, err := strconv.Atoi(strings.TrimSpace(string(data)))
+		if err != nil {
+			t.Fatalf("Failed to parse failure count: %v", err)
+		}
+
+		if count == 1 {
+			// Second run: increment count to 2 and fail again
+			if err := os.WriteFile(file, []byte("2"), 0644); err != nil {
+				t.Fatalf("Failed to update flaky test file: %v", err)
+			}
+			t.Fatal("Flaky Failure (attempt 2)")
+		} else if count == 2 {
+			// Third run: remove file and pass
+			if err := os.Remove(file); err != nil {
+				t.Fatalf("Failed to remove flaky test file: %v", err)
+			}
+		}
+	}
+}