Concurrent Test Execution

Takes advantage of changes made to isolate test runs to execute as many tests in parallel as possible. For tests that must be run serially, the @serial tag has been added to the beginning of relevant feature file(s).

Signed-off-by: Daniel Franz <dfranz@redhat.com>
Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com>

Author: dtfranz

Makefile

@@ -315,7 +315,7 @@ test-experimental-e2e: COVERAGE_NAME := experimental-e2e
 test-experimental-e2e: export MANIFEST := $(EXPERIMENTAL_RELEASE_MANIFEST)
 test-experimental-e2e: export INSTALL_DEFAULT_CATALOGS := false
 test-experimental-e2e: PROMETHEUS_VALUES := helm/prom_experimental.yaml
-test-experimental-e2e: E2E_TIMEOUT := 20m
+test-experimental-e2e: E2E_TIMEOUT := 10m
 test-experimental-e2e: run-internal prometheus e2e e2e-coverage kind-clean #HELP Run experimental e2e test suite on local kind cluster
 
 .PHONY: prometheus

test/e2e/features/tls.feature

@@ -1,3 +1,4 @@
+@Serial
 Feature: TLS profile enforcement on metrics endpoints
 
   Background:

test/e2e/features_test.go

@@ -1,9 +1,12 @@
 package e2e
 
 import (
+	"bytes"
 	"fmt"
+	"io"
 	"log"
 	"os"
+	"strings"
 	"testing"
 
 	"github.com/cucumber/godog"
@@ -14,52 +17,121 @@ import (
 	"github.com/operator-framework/operator-controller/test/e2e/steps"
 )
 
-var opts = godog.Options{
-	Format:      "pretty",
-	Paths:       []string{"features"},
-	Output:      colors.Colored(os.Stdout),
-	Concurrency: 1,
-	NoColors:    true,
+var defaultOpts = godog.Options{
+	Format:   "pretty",
+	Paths:    []string{"features"},
+	Output:   colors.Colored(os.Stdout),
+	NoColors: true,
 }
 
 func init() {
-	godog.BindCommandLineFlags("godog.", &opts)
+	godog.BindCommandLineFlags("godog.", &defaultOpts)
 }
 
 func TestMain(m *testing.M) {
 	// parse CLI arguments
 	pflag.Parse()
-	opts.Paths = pflag.Args()
+	defaultOpts.Paths = pflag.Args()
 
-	// run tests
-	sc := godog.TestSuite{
+	// Create buffers to capture output for final summary
+	var parallelBuf, serialBuf bytes.Buffer
+
+	parallelOpts := defaultOpts
+	parallelOpts.Concurrency = 100
+	parallelOpts.Tags = "~@Serial"
+	parallelOpts.Randomize = -1
+	// Write to both stdout (live) and buffer (for summary)
+	parallelOpts.Output = io.MultiWriter(os.Stdout, &parallelBuf)
+	// run tests concurrently
+	scParallel := godog.TestSuite{
+		TestSuiteInitializer: InitializeSuite,
+		ScenarioInitializer:  InitializeScenario,
+		Options:              &parallelOpts,
+	}.Run()
+
+	fmt.Println("End of parallel run - beginning serial tests")
+
+	serialOpts := defaultOpts
+	serialOpts.Concurrency = 1
+	serialOpts.Tags = "@Serial"
+	// Write to both stdout (live) and buffer (for summary)
+	serialOpts.Output = io.MultiWriter(os.Stdout, &serialBuf)
+	// run tests serially
+	scSerial := godog.TestSuite{
 		TestSuiteInitializer: InitializeSuite,
 		ScenarioInitializer:  InitializeScenario,
-		Options:              &opts,
+		Options:              &serialOpts,
 	}.Run()
 
-	switch sc {
-	//	0 - success
-	case 0:
-
-		path := os.Getenv("E2E_SUMMARY_OUTPUT")
-		if path == "" {
-			fmt.Println("Note: E2E_SUMMARY_OUTPUT is unset; skipping summary generation")
-		} else {
-			if err := testutil.PrintSummary(path); err != nil {
-				// Fail the run if alerts are found
-				fmt.Printf("%v", err)
-				os.Exit(1)
-			}
+	// TODO We re-print the output of any failed steps here for easier debugging. However, it would be
+	// better to combine this with the E2E_SUMMARY_OUTPUT and show pass/fail + performance in one
+	// markdown output then preserve the console output for local testing.
+
+	// Print aggregated summary
+	fmt.Println("\n" + strings.Repeat("=", 80))
+	fmt.Println("TEST EXECUTION SUMMARY")
+	fmt.Println(strings.Repeat("=", 80))
+
+	fmt.Printf("\nParallel Tests Exit Code: %d\n", scParallel)
+	if scParallel != 0 {
+		failedSteps := extractFailedSteps(parallelBuf.String())
+		if failedSteps != "" {
+			fmt.Println("\nParallel Test Failures:")
+			fmt.Println(strings.Repeat("-", 80))
+			fmt.Println(failedSteps)
+		}
+	}
+
+	fmt.Printf("\nSerial Tests Exit Code: %d\n", scSerial)
+	if scSerial != 0 {
+		failedSteps := extractFailedSteps(serialBuf.String())
+		if failedSteps != "" {
+			fmt.Println("\nSerial Test Failures:")
+			fmt.Println(strings.Repeat("-", 80))
+			fmt.Println(failedSteps)
+		}
+	}
+
+	fmt.Println(strings.Repeat("=", 80))
+
+	if scParallel != 0 || scSerial != 0 {
+		//	1 - failed
+		//	2 - command line usage error
+		// 128 - or higher, os signal related error exit codes
+		log.Fatalf("non-zero status returned; parallel: (%d), serial: (%d), failed to run feature tests", scParallel, scSerial)
+	}
+
+	path := os.Getenv("E2E_SUMMARY_OUTPUT")
+	if path == "" {
+		fmt.Println("Note: E2E_SUMMARY_OUTPUT is unset; skipping summary generation")
+	} else {
+		if err := testutil.PrintSummary(path); err != nil {
+			// Fail the run if alerts are found
+			fmt.Printf("%v", err)
+			os.Exit(1)
+		}
+	}
+}
+
+// extractFailedSteps extracts the "--- Failed steps:" section from godog output
+func extractFailedSteps(output string) string {
+	lines := strings.Split(output, "\n")
+	var failedSection []string
+	capturing := false
+
+	for _, line := range lines {
+		if strings.Contains(line, "--- Failed steps:") {
+			capturing = true
+		}
+		if capturing {
+			failedSection = append(failedSection, line)
 		}
-		return
+	}
 
-	//	1 - failed
-	//	2 - command line usage error
-	// 128 - or higher, os signal related error exit codes
-	default:
-		log.Fatalf("non-zero status returned (%d), failed to run feature tests", sc)
+	if len(failedSection) == 0 {
+		return ""
 	}
+	return strings.Join(failedSection, "\n")
 }
 
 func InitializeSuite(tc *godog.TestSuiteContext) {