diff --git a/ctrf/ctrf.go b/ctrf/ctrf.go index 9e48c9b..bf95538 100644 --- a/ctrf/ctrf.go +++ b/ctrf/ctrf.go @@ -147,6 +147,7 @@ type Summary struct { Pending int `json:"pending"` Skipped int `json:"skipped"` Other int `json:"other"` + Flaky int `json:"flaky,omitempty"` Suites int `json:"suites,omitempty"` Start int64 `json:"start"` Stop int64 `json:"stop"` @@ -179,16 +180,18 @@ func (summary *Summary) Validate() []error { if summary.Stop < 0 { errs = append(errs, errors.New("invalid property 'results.summary.stop'")) } + if summary.Flaky < 0 { + errs = append(errs, errors.New("invalid property 'results.summary.flaky'")) + } if summary.Suites < 0 { errs = append(errs, errors.New("invalid property 'results.summary.suites'")) } if summary.Start > summary.Stop { errs = append(errs, errors.New("invalid summary timestamps: start can't be greater than stop")) } - testsSum := summary.Passed + summary.Failed + summary.Pending + summary.Skipped + summary.Other + testsSum := summary.Passed + summary.Failed + summary.Pending + summary.Skipped + summary.Other + summary.Flaky if summary.Tests != testsSum { errs = append(errs, fmt.Errorf("invalid summary counts: tests (%d) must be the sum of passed, failed, pending, skipped, and other (%d)", summary.Tests, testsSum)) - } return errs } @@ -203,27 +206,43 @@ const ( TestOther TestStatus = "other" ) +type RetryAttempt struct { + Attempt int `json:"attempt"` + Status TestStatus `json:"status"` + Duration int64 `json:"duration,omitempty"` + Message string `json:"message,omitempty"` + Trace string `json:"trace,omitempty"` + Line int `json:"line,omitempty"` + Snippet string `json:"snippet,omitempty"` + Stdout []string `json:"stdout,omitempty"` + Stderr []string `json:"stderr,omitempty"` + Start int64 `json:"start,omitempty"` + Stop int64 `json:"stop,omitempty"` + Extra any `json:"extra,omitempty"` +} + type TestResult struct { - Name string `json:"name"` - Status TestStatus `json:"status"` - Duration int64 `json:"duration"` - Start int64 `json:"start,omitempty"` - Stop int64 `json:"stop,omitempty"` - Suite string `json:"suite,omitempty"` - Message string `json:"message,omitempty"` - Trace string `json:"trace,omitempty"` - RawStatus string `json:"rawStatus,omitempty"` - Tags []string `json:"tags,omitempty"` - Type string `json:"type,omitempty"` - Filepath string `json:"filePath,omitempty"` - Retry int `json:"retry,omitempty"` - Flake bool `json:"flake,omitempty"` - Browser string `json:"browser,omitempty"` - Device string `json:"device,omitempty"` - Screenshot string `json:"screenshot,omitempty"` - Parameters any `json:"parameters,omitempty"` - Steps []any `json:"steps,omitempty"` - Extra any `json:"extra,omitempty"` + Name string `json:"name"` + Status TestStatus `json:"status"` + Duration int64 `json:"duration"` + Start int64 `json:"start,omitempty"` + Stop int64 `json:"stop,omitempty"` + Suite string `json:"suite,omitempty"` + Message string `json:"message,omitempty"` + Trace string `json:"trace,omitempty"` + RawStatus string `json:"rawStatus,omitempty"` + Tags []string `json:"tags,omitempty"` + Type string `json:"type,omitempty"` + Filepath string `json:"filePath,omitempty"` + Retries int `json:"retries,omitempty"` + Flaky bool `json:"flaky,omitempty"` + Browser string `json:"browser,omitempty"` + Device string `json:"device,omitempty"` + Screenshot string `json:"screenshot,omitempty"` + Parameters any `json:"parameters,omitempty"` + Steps []any `json:"steps,omitempty"` + RetryAttempts []RetryAttempt `json:"retryAttempts,omitempty"` + Extra any `json:"extra,omitempty"` } type Environment struct { diff --git a/ctrf/ctrf_test.go b/ctrf/ctrf_test.go index 4ead549..365fb24 100644 --- a/ctrf/ctrf_test.go +++ b/ctrf/ctrf_test.go @@ -20,12 +20,13 @@ func TestRequiredProperties(t *testing.T) { Name: "my tool", }, Summary: &Summary{ - Tests: 15, - Passed: 5, - Failed: 4, - Pending: 3, - Skipped: 2, - Other: 1, + Tests: 21, + Passed: 6, + Failed: 5, + Pending: 4, + Skipped: 3, + Other: 2, + Flaky: 1, Start: 42, Stop: 1337, }, @@ -68,12 +69,13 @@ func TestRequiredProperties(t *testing.T) { "name": "my tool" }, "summary": { - "tests": 15, - "passed": 5, - "failed": 4, - "pending": 3, - "skipped": 2, - "other": 1, + "tests": 21, + "passed": 6, + "failed": 5, + "pending": 4, + "skipped": 3, + "other": 2, + "flaky": 1, "start": 42, "stop": 1337 }, diff --git a/examples/flaky/.gitignore b/examples/flaky/.gitignore new file mode 100644 index 0000000..0f71215 --- /dev/null +++ b/examples/flaky/.gitignore @@ -0,0 +1 @@ +test.json diff --git a/examples/flaky/README.md b/examples/flaky/README.md new file mode 100644 index 0000000..d35959f --- /dev/null +++ b/examples/flaky/README.md @@ -0,0 +1,11 @@ +# Flaky Test Example + +This code in here is an example for a flaky test. It can be run to generate some output suitable for testing flaky behavior and the resulting json output. + +At the time of writing, Go does not have built-in support for re-running flaky tests, (though there is an [accepted proposal](https://github.com/golang/go/issues/62244) to address this). However, [gotestsum](https://github.com/gotestyourself/gotestsum) does include support to automatically re-run flaky tests, with the `--rerun-fails=n` and `--rerun-fails-max-failures=n` flags. + +The output for the unit test can be generated with the following command from the repo root: + +```shell +gotestsum --jsonfile examples/flaky/test.json --rerun-fails --packages ./examples/flaky -- -count 1 -tags examples +``` diff --git a/examples/flaky/flaky_test.go b/examples/flaky/flaky_test.go new file mode 100644 index 0000000..b72e9a9 --- /dev/null +++ b/examples/flaky/flaky_test.go @@ -0,0 +1,63 @@ +//go:build examples + +package flaky + +import ( + "errors" + "io/fs" + "os" + "path" + "strconv" + "strings" + "testing" + "time" +) + +func Test_Flaky_Pass(t *testing.T) { + time.Sleep(50 * time.Millisecond) +} + +func Test_Flaky_Fail(t *testing.T) { + time.Sleep(50 * time.Millisecond) + t.Fatal("This test is designed to fail.") +} + +func Test_Flaky_Skipped(t *testing.T) { + t.Skip("This test is designed to be skipped.") + time.Sleep(50 * time.Millisecond) +} + +func Test_Flaky_Flaky(t *testing.T) { + time.Sleep(50 * time.Millisecond) + file := path.Join(os.TempDir(), "flaky_test.tmp") + if _, err := os.Stat(file); errors.Is(err, fs.ErrNotExist) { + // First run: file does not exist, create it with count "1", and fail. + if err := os.WriteFile(file, []byte("1"), 0644); err != nil { + t.Fatalf("Failed to create flaky test file: %v", err) + } + t.Fatal("Flaky Failure (attempt 1)") + } else { + // File exists, read the current count + data, err := os.ReadFile(file) + if err != nil { + t.Fatalf("Failed to read flaky test file: %v", err) + } + count, err := strconv.Atoi(strings.TrimSpace(string(data))) + if err != nil { + t.Fatalf("Failed to parse failure count: %v", err) + } + + if count == 1 { + // Second run: increment count to 2 and fail again + if err := os.WriteFile(file, []byte("2"), 0644); err != nil { + t.Fatalf("Failed to update flaky test file: %v", err) + } + t.Fatal("Flaky Failure (attempt 2)") + } else if count == 2 { + // Third run: remove file and pass + if err := os.Remove(file); err != nil { + t.Fatalf("Failed to remove flaky test file: %v", err) + } + } + } +} diff --git a/reporter/reporter.go b/reporter/reporter.go index dc544d6..b89e658 100644 --- a/reporter/reporter.go +++ b/reporter/reporter.go @@ -23,6 +23,16 @@ type TestEvent struct { Output string } +const ( + ActionBuildOutput = "build-output" + ActionBuildFail = "build-fail" + ActionOutput = "output" + ActionRun = "run" + ActionPass = "pass" + ActionFail = "fail" + ActionSkip = "skip" +) + var buildOutput []string func ParseTestResults(r io.Reader, verbose bool, env *ctrf.Environment) (*ctrf.Report, error) { @@ -32,6 +42,13 @@ func ParseTestResults(r io.Reader, verbose bool, env *ctrf.Environment) (*ctrf.R report := ctrf.NewReport("gotest", env) report.Results.Summary.Start = time.Now().UnixNano() / int64(time.Millisecond) + testStartTimes := make(map[string]int64) + extraMap := make(map[string]any) + buildOutputEvents := make([]TestEvent, 0) + buildFailEvents := make([]TestEvent, 0) + + report.Results.Extra = extraMap + for { var event TestEvent if err := decoder.Decode(&event); err == io.EOF { @@ -42,99 +59,102 @@ func ParseTestResults(r io.Reader, verbose bool, env *ctrf.Environment) (*ctrf.R testEvents = append(testEvents, event) if verbose { - if event.Action == "build-output" || event.Action == "output" { + if event.Action == ActionBuildOutput || event.Action == ActionOutput { fmt.Print(event.Output) } } } for i, event := range testEvents { + // If we see any test failures, mark an overall failure in the Extra fields + if event.Action == ActionFail { + extraMap["FailedBuild"] = true + } - if event.Action == "build-output" || event.Action == "build-fail" || event.Action == "fail" { - if report.Results.Extra == nil { - report.Results.Extra = make(map[string]any) - } - extraMap, ok := report.Results.Extra.(map[string]any) - if !ok { - return nil, fmt.Errorf("expected a map, but got %T instead", report.Results.Extra) - } - - if event.Action == "fail" { - if _, ok := extraMap["FailedBuild"]; !ok { - extraMap["FailedBuild"] = true - } - } + if event.Action == ActionBuildOutput { + // Capture the full events to the extras + buildOutputEvents = append(buildOutputEvents, event) + extraMap["buildOutput"] = buildOutputEvents - if event.Action == "build-output" { - if _, ok := extraMap["buildOutput"]; !ok { - extraMap["buildOutput"] = []TestEvent{} - } - buildOutputEvents := extraMap["buildOutput"].([]TestEvent) - extraMap["buildOutput"] = append(buildOutputEvents, event) - buildOutput = append(buildOutput, event.Output) - continue - } + // Capture the actual build output as well + buildOutput = append(buildOutput, event.Output) + continue + } - if event.Action == "build-fail" { - if _, ok := extraMap["buildFail"]; !ok { - extraMap["buildFail"] = []TestEvent{} - } - buildFailEvents := extraMap["buildFail"].([]TestEvent) - extraMap["buildFail"] = append(buildFailEvents, event) - break - } + // Mark if we see a build failure in the extras field + if event.Action == ActionBuildFail { + buildFailEvents = append(buildFailEvents, event) + extraMap["buildFail"] = buildFailEvents + break } - if event.Action == "output" { + if event.Action == ActionOutput { buildOutput = append(buildOutput, event.Output) } + // From this point, we only care about events associated with an actual test if event.Test == "" { continue } - startTime, err := parseTimeString(event.Time) - duration := secondsToMillis(event.Elapsed) + + // Parse timestamp data from the event + eventTime, err := parseTimeString(event.Time) if err != nil { fmt.Fprintf(os.Stderr, "error parsing test event start time '%s' : %v\n", event.Time, err) } else { - if report.Results.Summary.Start > startTime { - report.Results.Summary.Start = startTime + if eventTime < report.Results.Summary.Start { + report.Results.Summary.Start = eventTime + } + if eventTime > report.Results.Summary.Stop { + report.Results.Summary.Stop = eventTime } - endTime := startTime + duration - if report.Results.Summary.Stop < endTime { - report.Results.Summary.Stop = endTime + + // If this is a "run" event, record the start time of the test. We'll look this up later when + // we process the "pass"/"fail"/"skip" event for the test to create the TestResult + if event.Action == ActionRun { + testStartTimes[testNameKey(event.Package, event.Test)] = eventTime } } - if event.Action == "pass" { - report.Results.Summary.Tests++ - report.Results.Summary.Passed++ - report.Results.Tests = append(report.Results.Tests, &ctrf.TestResult{ - Suite: event.Package, - Name: event.Test, - Status: ctrf.TestPassed, - Duration: duration, - }) - } else if event.Action == "fail" { - report.Results.Summary.Tests++ - report.Results.Summary.Failed++ - report.Results.Tests = append(report.Results.Tests, &ctrf.TestResult{ - Suite: event.Package, - Name: event.Test, - Status: ctrf.TestFailed, - Duration: duration, - Message: getMessagesForTest(testEvents, i, event.Package, event.Test), - }) - } else if event.Action == "skip" { - report.Results.Summary.Tests++ - report.Results.Summary.Skipped++ - report.Results.Tests = append(report.Results.Tests, &ctrf.TestResult{ + + // From this point on, we only deal with pass, fail, and skip events, which indicate that the + // test has completed, and we can create/update a TestResult for it. + if event.Action == ActionPass || event.Action == ActionFail || event.Action == ActionSkip { + // Look up the start time, and use this event's time as the endTime, to mark the start/stop times + // for the test result. Duration we get from the event.Elapsed field, which better takes into + // account parallel tests, setup/teardown time, etc... + startTime, ok := testStartTimes[testNameKey(event.Package, event.Test)] + if ok { + delete(testStartTimes, testNameKey(event.Package, event.Test)) + } + stopTime := eventTime + + // Determine the message for this test result. We only include messages on failures though, + // per the CTRF spec, so if this is not a failure, we pass an empty string for the message. + message := "" + if event.Action == ActionFail { + message = getMessagesForTest(testEvents, i, event.Package, event.Test, startTime) + } + + // Build the TestResult for this test event, and add it to the report. + newResult := &ctrf.TestResult{ Suite: event.Package, Name: event.Test, - Status: ctrf.TestSkipped, - Duration: duration, - }) - } + Status: actionToTestResult(event.Action), + Duration: secondsToMillis(event.Elapsed), + Message: message, + Start: startTime, + Stop: stopTime, + } + // Search through the existing results for a prior run. If this is a duplicate of an existing failure, + // then this is likely a retry of a potentially flaky test, so update the existing test result instead + // of creating a new one. + if existingResult := findTest(event.Package, event.Test, report.Results.Tests); existingResult == nil { + addResult(report, newResult) + } else { + updateResult(report, existingResult, newResult) + } + } } enrichReportWithFilenames(report) @@ -142,6 +162,107 @@ func ParseTestResults(r io.Reader, verbose bool, env *ctrf.Environment) (*ctrf.R return report, nil } +// addResult adds a new test result to the report, filling out all the relevant details. +func addResult(report *ctrf.Report, result *ctrf.TestResult) { + // Update the overall test count in the Summary + report.Results.Summary.Tests++ + + // Update the sub-count based on the test result status + switch result.Status { + case ctrf.TestPassed: + report.Results.Summary.Passed++ + case ctrf.TestFailed: + report.Results.Summary.Failed++ + case ctrf.TestSkipped: + report.Results.Summary.Skipped++ + case ctrf.TestPending: + report.Results.Summary.Pending++ + default: + report.Results.Summary.Other++ + } + + // Append the result to the report's results + report.Results.Tests = append(report.Results.Tests, result) +} + +func updateResult(report *ctrf.Report, oldResult, newResult *ctrf.TestResult) { + // If the existing result does not have a retries field, initialize it, and move the + // results to the first RetryAttempts object + if oldResult.RetryAttempts == nil { + oldResult.Retries = 1 + oldResult.RetryAttempts = append(oldResult.RetryAttempts, ctrf.RetryAttempt{ + Attempt: 1, + Status: oldResult.Status, + Message: oldResult.Message, + Duration: oldResult.Duration, + Start: oldResult.Start, + Stop: oldResult.Stop, + }) + } + + // If this is a pass after a failure, mark the test as flaky, not failed, + // and update the summary counts accordingly + if oldResult.Status == ctrf.TestFailed && newResult.Status == ctrf.TestPassed { + oldResult.Flaky = true + report.Results.Summary.Flaky++ + report.Results.Summary.Failed-- + } + + // Update the overall test status to match that of the new result + oldResult.Status = newResult.Status + + // Clear out the top-level message on the overall result, since the messages are in the retries + oldResult.Message = "" + + // Update the times of the overall test result + oldResult.Duration += newResult.Duration + if newResult.Stop > oldResult.Stop { + oldResult.Stop = newResult.Stop + } + if newResult.Start < oldResult.Start { + oldResult.Start = newResult.Start + } + + // Now add the new attempt to the retries + oldResult.Retries++ + oldResult.RetryAttempts = append(oldResult.RetryAttempts, ctrf.RetryAttempt{ + Attempt: oldResult.Retries, + Status: newResult.Status, + Message: newResult.Message, + Duration: newResult.Duration, + Start: newResult.Start, + Stop: newResult.Stop, + }) +} + +// testNameKey generates a unique key for a map lookup based on the test name and suite. +func testNameKey(suite, name string) string { + return fmt.Sprintf("%s.%s", suite, name) +} + +func actionToTestResult(action string) ctrf.TestStatus { + switch action { + case ActionPass: + return ctrf.TestPassed + case ActionFail: + return ctrf.TestFailed + case ActionSkip: + return ctrf.TestSkipped + default: + return ctrf.TestOther + } +} + +// findTest searches through the already-parsed test results to find a matching test. +func findTest(suite string, name string, tests []*ctrf.TestResult) *ctrf.TestResult { + for _, test := range tests { + if test.Suite == suite && test.Name == name { + return test + } + } + return nil +} + func generateTestMap() map[string][]string { tests := map[string][]string{} @@ -180,11 +301,24 @@ func enrichReportWithFilenames(report *ctrf.Report) { } } -func getMessagesForTest(testEvents []TestEvent, index int, packageName, testName string) string { +func getMessagesForTest(testEvents []TestEvent, index int, packageName, testName string, startTime int64) string { var messages []string for i := index; i >= 0; i-- { if testEvents[i].Package == packageName && testEvents[i].Test == testName { - if testEvents[i].Action == "output" { + // If we are only getting the messages for a single test retry, then we only want the messages + // that occurred on or after the start time of that retry attempt. If startTime is 0, then we + // want all messages for the test regardless of time. + if startTime != 0 { + eventTime, err := parseTimeString(testEvents[i].Time) + if err != nil { + fmt.Fprintf(os.Stderr, "error parsing test event time '%s' : %v\n", testEvents[i].Time, err) + } + if eventTime < startTime { + continue + } + } + + if testEvents[i].Action == ActionOutput { messages = append(messages, testEvents[i].Output) } } diff --git a/reporter/reporter_test.go b/reporter/reporter_test.go index 4a98930..114a002 100644 --- a/reporter/reporter_test.go +++ b/reporter/reporter_test.go @@ -17,8 +17,12 @@ func Test_Enrich_Reporter(t *testing.T) { Status: "passed", Suite: "github.com/ctrf-io/go-ctrf-json-reporter/reporter", Filepath: "reporter_test.go", + Start: 1740874081832, + Stop: 1740874081832, }, }}} + + //nolint:lll // The test inputs are raw strings taken from real test runs input := `{"Time":"2025-03-02T01:08:01.832222033+01:00","Action":"start","Package":"github.com/ctrf-io/go-ctrf-json-reporter/reporter"} {"Time":"2025-03-02T01:08:01.832309292+01:00","Action":"run","Package":"github.com/ctrf-io/go-ctrf-json-reporter/reporter","Test":"Test_Enrich_Reporter"} {"Time":"2025-03-02T01:08:01.832321979+01:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/reporter","Test":"Test_Enrich_Reporter","Output":"=== RUN Test_Enrich_Reporter\n"} @@ -41,24 +45,32 @@ func Test_Enrich_ReporterWithUnorderedMessages(t *testing.T) { Status: "passed", Suite: "github.com/ctrf-io/go-ctrf-json-reporter/reporter", Filepath: "reporter_test.go", + Start: 1760718477126, + Stop: 1760718477126, }, { Name: "Test_Enrich_Reporter/Test2", Status: "passed", Suite: "github.com/ctrf-io/go-ctrf-json-reporter/reporter", Filepath: "reporter_test.go", + Start: 1760718477126, + Stop: 1760718477126, }, { Name: "Test_Enrich_Reporter/Test3", Status: "passed", Suite: "github.com/ctrf-io/go-ctrf-json-reporter/reporter", Filepath: "reporter_test.go", + Start: 1760718477126, + Stop: 1760718477126, }, { Name: "Test_Enrich_Reporter/Test4", Status: "passed", Suite: "github.com/ctrf-io/go-ctrf-json-reporter/reporter", Filepath: "reporter_test.go", + Start: 1760718477126, + Stop: 1760718477126, }, { Name: "Test_Enrich_Reporter/Test5", @@ -66,18 +78,24 @@ func Test_Enrich_ReporterWithUnorderedMessages(t *testing.T) { Suite: "github.com/ctrf-io/go-ctrf-json-reporter/reporter", Filepath: "reporter_test.go", Message: "=== RUN Test_Enrich_Reporter/Test5\n reporter:59: Something.Skip() = false, want true\n --- FAIL: Test_Enrich_Reporter/Test5 (0.00s)\n", + Start: 1760718477126, + Stop: 1760718477126, }, { Name: "Test_Enrich_Reporter/Test6", Status: "passed", Suite: "github.com/ctrf-io/go-ctrf-json-reporter/reporter", Filepath: "reporter_test.go", + Start: 1760718477126, + Stop: 1760718477126, }, { Name: "Test_Enrich_Reporter/Test7", Status: "passed", Suite: "github.com/ctrf-io/go-ctrf-json-reporter/reporter", Filepath: "reporter_test.go", + Start: 1760718477126, + Stop: 1760718477126, }, { Name: "Test_Enrich_Reporter", @@ -85,8 +103,12 @@ func Test_Enrich_ReporterWithUnorderedMessages(t *testing.T) { Suite: "github.com/ctrf-io/go-ctrf-json-reporter/reporter", Filepath: "reporter_test.go", Message: "=== RUN Test_Enrich_Reporter\n--- FAIL: Test_Enrich_Reporter (0.00s)\n", + Start: 1760718477126, + Stop: 1760718477126, }, }}} + + //nolint:lll // The test inputs are raw strings taken from real test runs input := `{"Time":"2025-10-17T12:27:57.126761-04:00","Action":"run","Package":"github.com/ctrf-io/go-ctrf-json-reporter/reporter","Test":"Test_Enrich_Reporter"} {"Time":"2025-10-17T12:27:57.126764-04:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/reporter","Test":"Test_Enrich_Reporter","Output":"=== RUN Test_Enrich_Reporter\n"} {"Time":"2025-10-17T12:27:57.126769-04:00","Action":"run","Package":"github.com/ctrf-io/go-ctrf-json-reporter/reporter","Test":"Test_Enrich_Reporter/Test1"} @@ -126,3 +148,153 @@ func Test_Enrich_ReporterWithUnorderedMessages(t *testing.T) { require.NoError(t, err) assert.Equal(t, expected.Results.Tests, actual.Results.Tests) } + +func TestDetectFlakyTests(t *testing.T) { + expected := &ctrf.Report{Results: &ctrf.Results{ + Summary: &ctrf.Summary{ + Tests: 4, + Passed: 1, + Failed: 1, + Skipped: 1, + Flaky: 1, + Start: 1775245677812, // The event timestamp of the first processed event + Stop: 1775245679646, // The event timestamp of the last processed event + }, + Extra: map[string]any{ + "FailedBuild": true, + }, + Tests: []*ctrf.TestResult{ + { + Name: "Test_Flaky_Pass", + Status: ctrf.TestPassed, + Suite: "github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky", + Filepath: "reporter_test.go", // Filepath enrichment is based on grepping the current path, so this file is the one that gets found + Start: 1775245677812, + Stop: 1775245677863, + Duration: 50, + }, + { + Name: "Test_Flaky_Fail", + Status: ctrf.TestFailed, + Suite: "github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky", + Filepath: "reporter_test.go", + Retries: 3, + Start: 1775245677863, // The start time of the first event for the first retry + Stop: 1775245679247, // The stop time of the last event for the last retry + Duration: 150, + Message: "", + RetryAttempts: []ctrf.RetryAttempt{ + { + Attempt: 1, Status: ctrf.TestFailed, Start: 1775245677863, Stop: 1775245677914, Duration: 50, + Message: "=== RUN Test_Flaky_Fail\n flaky_test.go:21: This test is designed to fail.\n--- FAIL: Test_Flaky_Fail (0.05s)\n", + }, + { + Attempt: 2, Status: ctrf.TestFailed, Start: 1775245678350, Stop: 1775245678401, Duration: 50, + Message: "=== RUN Test_Flaky_Fail\n flaky_test.go:21: This test is designed to fail.\n--- FAIL: Test_Flaky_Fail (0.05s)\n", + }, + { + Attempt: 3, Status: ctrf.TestFailed, Start: 1775245679196, Stop: 1775245679247, Duration: 50, + Message: "=== RUN Test_Flaky_Fail\n flaky_test.go:21: This test is designed to fail.\n--- FAIL: Test_Flaky_Fail (0.05s)\n", + }, + }, + }, + { + Name: "Test_Flaky_Skipped", + Status: ctrf.TestSkipped, + Suite: "github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky", + Filepath: "reporter_test.go", + Start: 1775245677914, + Stop: 1775245677914, + }, + { + Name: "Test_Flaky_Flaky", + Status: ctrf.TestPassed, + Suite: "github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky", + Filepath: "reporter_test.go", + Retries: 3, + Flaky: true, + Duration: 150, + Start: 1775245677914, + Stop: 1775245679646, + RetryAttempts: []ctrf.RetryAttempt{ + { + Attempt: 1, Status: ctrf.TestFailed, Duration: 50, Start: 1775245677914, Stop: 1775245677967, + Message: "=== RUN Test_Flaky_Flaky\n flaky_test.go:37: Flaky Failure (attempt 1)\n--- FAIL: Test_Flaky_Flaky (0.05s)\n", + }, + { + Attempt: 2, Status: ctrf.TestFailed, Duration: 50, Start: 1775245678784, Stop: 1775245678837, + Message: "=== RUN Test_Flaky_Flaky\n flaky_test.go:54: Flaky Failure (attempt 2)\n--- FAIL: Test_Flaky_Flaky (0.05s)\n", + }, + {Attempt: 3, Status: ctrf.TestPassed, Duration: 50, Start: 1775245679595, Stop: 1775245679646}, + }, + }, + }, + }} + + //nolint:lll // The test inputs are raw strings taken from real test runs + input := `{"Time":"2026-04-03T13:47:57.561046-06:00","Action":"start","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky"} +{"Time":"2026-04-03T13:47:57.812415-06:00","Action":"run","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Pass"} +{"Time":"2026-04-03T13:47:57.812541-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Pass","Output":"=== RUN Test_Flaky_Pass\n"} +{"Time":"2026-04-03T13:47:57.863054-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Pass","Output":"--- PASS: Test_Flaky_Pass (0.05s)\n"} +{"Time":"2026-04-03T13:47:57.863087-06:00","Action":"pass","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Pass","Elapsed":0.05} +{"Time":"2026-04-03T13:47:57.863153-06:00","Action":"run","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Fail"} +{"Time":"2026-04-03T13:47:57.863169-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Fail","Output":"=== RUN Test_Flaky_Fail\n"} +{"Time":"2026-04-03T13:47:57.914515-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Fail","Output":" flaky_test.go:21: This test is designed to fail.\n"} +{"Time":"2026-04-03T13:47:57.914602-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Fail","Output":"--- FAIL: Test_Flaky_Fail (0.05s)\n"} +{"Time":"2026-04-03T13:47:57.914618-06:00","Action":"fail","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Fail","Elapsed":0.05} +{"Time":"2026-04-03T13:47:57.914632-06:00","Action":"run","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Skipped"} +{"Time":"2026-04-03T13:47:57.914639-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Skipped","Output":"=== RUN Test_Flaky_Skipped\n"} +{"Time":"2026-04-03T13:47:57.914663-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Skipped","Output":" flaky_test.go:25: This test is designed to be skipped.\n"} +{"Time":"2026-04-03T13:47:57.914686-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Skipped","Output":"--- SKIP: Test_Flaky_Skipped (0.00s)\n"} +{"Time":"2026-04-03T13:47:57.914707-06:00","Action":"skip","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Skipped","Elapsed":0} +{"Time":"2026-04-03T13:47:57.914718-06:00","Action":"run","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Flaky"} +{"Time":"2026-04-03T13:47:57.914725-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Flaky","Output":"=== RUN Test_Flaky_Flaky\n"} +{"Time":"2026-04-03T13:47:57.96746-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Flaky","Output":" flaky_test.go:37: Flaky Failure (attempt 1)\n"} +{"Time":"2026-04-03T13:47:57.967528-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Flaky","Output":"--- FAIL: Test_Flaky_Flaky (0.05s)\n"} +{"Time":"2026-04-03T13:47:57.967543-06:00","Action":"fail","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Flaky","Elapsed":0.05} +{"Time":"2026-04-03T13:47:57.967559-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Output":"FAIL\n"} +{"Time":"2026-04-03T13:47:57.969244-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Output":"FAIL\tgithub.com/ctrf-io/go-ctrf-json-reporter/examples/flaky\t0.408s\n"} +{"Time":"2026-04-03T13:47:57.969316-06:00","Action":"fail","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Elapsed":0.408} +{"Time":"2026-04-03T13:47:58.151436-06:00","Action":"start","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky"} +{"Time":"2026-04-03T13:47:58.350514-06:00","Action":"run","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Fail"} +{"Time":"2026-04-03T13:47:58.350555-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Fail","Output":"=== RUN Test_Flaky_Fail\n"} +{"Time":"2026-04-03T13:47:58.40176-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Fail","Output":" flaky_test.go:21: This test is designed to fail.\n"} +{"Time":"2026-04-03T13:47:58.401856-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Fail","Output":"--- FAIL: Test_Flaky_Fail (0.05s)\n"} +{"Time":"2026-04-03T13:47:58.401873-06:00","Action":"fail","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Fail","Elapsed":0.05} +{"Time":"2026-04-03T13:47:58.401904-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Output":"FAIL\n"} +{"Time":"2026-04-03T13:47:58.403834-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Output":"FAIL\tgithub.com/ctrf-io/go-ctrf-json-reporter/examples/flaky\t0.252s\n"} +{"Time":"2026-04-03T13:47:58.403863-06:00","Action":"fail","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Elapsed":0.252} +{"Time":"2026-04-03T13:47:58.566312-06:00","Action":"start","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky"} +{"Time":"2026-04-03T13:47:58.784809-06:00","Action":"run","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Flaky"} +{"Time":"2026-04-03T13:47:58.784889-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Flaky","Output":"=== RUN Test_Flaky_Flaky\n"} +{"Time":"2026-04-03T13:47:58.837476-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Flaky","Output":" flaky_test.go:54: Flaky Failure (attempt 2)\n"} +{"Time":"2026-04-03T13:47:58.837581-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Flaky","Output":"--- FAIL: Test_Flaky_Flaky (0.05s)\n"} +{"Time":"2026-04-03T13:47:58.837595-06:00","Action":"fail","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Flaky","Elapsed":0.05} +{"Time":"2026-04-03T13:47:58.837643-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Output":"FAIL\n"} +{"Time":"2026-04-03T13:47:58.839298-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Output":"FAIL\tgithub.com/ctrf-io/go-ctrf-json-reporter/examples/flaky\t0.273s\n"} +{"Time":"2026-04-03T13:47:58.839328-06:00","Action":"fail","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Elapsed":0.273} +{"Time":"2026-04-03T13:47:58.994008-06:00","Action":"start","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky"} +{"Time":"2026-04-03T13:47:59.196497-06:00","Action":"run","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Fail"} +{"Time":"2026-04-03T13:47:59.196544-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Fail","Output":"=== RUN Test_Flaky_Fail\n"} +{"Time":"2026-04-03T13:47:59.247768-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Fail","Output":" flaky_test.go:21: This test is designed to fail.\n"} +{"Time":"2026-04-03T13:47:59.247873-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Fail","Output":"--- FAIL: Test_Flaky_Fail (0.05s)\n"} +{"Time":"2026-04-03T13:47:59.247889-06:00","Action":"fail","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Fail","Elapsed":0.05} +{"Time":"2026-04-03T13:47:59.247917-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Output":"FAIL\n"} +{"Time":"2026-04-03T13:47:59.250129-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Output":"FAIL\tgithub.com/ctrf-io/go-ctrf-json-reporter/examples/flaky\t0.256s\n"} +{"Time":"2026-04-03T13:47:59.250164-06:00","Action":"fail","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Elapsed":0.256} +{"Time":"2026-04-03T13:47:59.413168-06:00","Action":"start","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky"} +{"Time":"2026-04-03T13:47:59.595591-06:00","Action":"run","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Flaky"} +{"Time":"2026-04-03T13:47:59.595652-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Flaky","Output":"=== RUN Test_Flaky_Flaky\n"} +{"Time":"2026-04-03T13:47:59.646973-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Flaky","Output":"--- PASS: Test_Flaky_Flaky (0.05s)\n"} +{"Time":"2026-04-03T13:47:59.646984-06:00","Action":"pass","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Test":"Test_Flaky_Flaky","Elapsed":0.05} +{"Time":"2026-04-03T13:47:59.646998-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Output":"PASS\n"} +{"Time":"2026-04-03T13:47:59.648315-06:00","Action":"output","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Output":"ok \tgithub.com/ctrf-io/go-ctrf-json-reporter/examples/flaky\t0.235s\n"} +{"Time":"2026-04-03T13:47:59.648334-06:00","Action":"pass","Package":"github.com/ctrf-io/go-ctrf-json-reporter/examples/flaky","Elapsed":0.235}` + + actual, err := reporter.ParseTestResults(bytes.NewBufferString(input), true, &ctrf.Environment{}) + + require.NoError(t, err) + assert.Equal(t, expected.Results.Summary, actual.Results.Summary) + assert.Equal(t, expected.Results.Tests, actual.Results.Tests) + assert.Equal(t, expected.Results.Extra, actual.Results.Extra) +}