Concurrent Test Execution

Takes advantage of changes made to isolate test runs to execute as many tests in parallel as possible. For tests that must be run serially, the @serial tag has been added to the beginning of relevant feature file(s).

Signed-off-by: Daniel Franz <dfranz@redhat.com>
Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com>

Author: dtfranz

test/e2e/README.md

@@ -207,6 +207,10 @@ Use these variables in YAML templates:
 
 ### 5. Feature Tags
 
+Tags can be used for different purposes in the test suite:
+
+#### Feature Gate Tags
+
 Use tags to conditionally run scenarios based on feature gates:
 
 ```gherkin
@@ -216,6 +220,28 @@ Scenario: Install operator having webhooks
 
 Scenarios are skipped if the feature gate is not enabled on the deployed controller.
 
+#### Serial Execution Tag
+
+By default, scenarios run concurrently (up to 100 parallel scenarios). However, some tests must run serially, typically because they:
+- Modify shared cluster resources (e.g., cluster-wide TLS configuration)
+- Have resource constraints that prevent parallel execution
+- Require exclusive access to a resource
+
+To mark a test for serial execution, add the `@Serial` tag:
+
+```gherkin
+@Serial
+Feature: TLS profile enforcement on metrics endpoints
+
+  Scenario: Test TLS configuration
+    Given the "catalogd" deployment is configured with custom TLS settings
+    ...
+```
+
+The test runner automatically separates scenarios:
+- Scenarios **without** `@Serial` run concurrently in the first test phase
+- Scenarios **with** `@Serial` run sequentially in a separate serial test phase
+
 ## Running Tests
 
 ### Run All Tests
@@ -243,6 +269,12 @@ go test test/e2e/features_test.go -- features/install.feature
 go test test/e2e/features_test.go --godog.tags="@WebhookProviderCertManager"
 ```
 
+Note that setting the tags in this way will disable the automatic test parallelization. If running in parallel with custom tags is desired, set `--godog.concurrency=100` for instance to re-enable. If this is done adding `&& ~@Serial` to the tags as well is highly recommended:
+
+```bash
+go test test/e2e/features_test.go --godog.tags="@WebhookProviderCertManager && ~@Serial" --godog.concurrency=100
+```
+
 ### Run with Debug Logging
 
 ```bash

test/e2e/features/tls.feature

@@ -1,3 +1,4 @@
+@Serial
 Feature: TLS profile enforcement on metrics endpoints
 
   Background:

test/e2e/features_test.go

@@ -1,9 +1,12 @@
 package e2e
 
 import (
+	"bytes"
 	"fmt"
+	"io"
 	"log"
 	"os"
+	"strings"
 	"testing"
 
 	"github.com/cucumber/godog"
@@ -14,52 +17,144 @@ import (
 	"github.com/operator-framework/operator-controller/test/e2e/steps"
 )
 
-var opts = godog.Options{
-	Format:      "pretty",
-	Paths:       []string{"features"},
-	Output:      colors.Colored(os.Stdout),
-	Concurrency: 1,
-	NoColors:    true,
+var cliOpts = godog.Options{
+	Format:   "pretty",
+	Paths:    []string{"features"},
+	Output:   colors.Colored(os.Stdout),
+	NoColors: true,
 }
 
 func init() {
-	godog.BindCommandLineFlags("godog.", &opts)
+	godog.BindCommandLineFlags("godog.", &cliOpts)
 }
 
 func TestMain(m *testing.M) {
 	// parse CLI arguments
 	pflag.Parse()
-	opts.Paths = pflag.Args()
+	cliOpts.Paths = pflag.Args()
 
-	// run tests
-	sc := godog.TestSuite{
+	if cliOpts.Tags != "" {
+		fmt.Println("Note: Custom feature tags provided - disabling automatic test parallelization")
+		// run tests explicitly as requested by CLI
+		sc := godog.TestSuite{
+			TestSuiteInitializer: InitializeSuite,
+			ScenarioInitializer:  InitializeScenario,
+			Options:              &cliOpts,
+		}.Run()
+
+		if sc != 0 {
+			//	1 - failed
+			//	2 - command line usage error
+			// 128 - or higher, os signal related error exit codes
+			log.Fatalf("non-zero status returned: (%d), failed to run feature tests", sc)
+		}
+	} else {
+		executeTestsParallel()
+	}
+
+	path := os.Getenv("E2E_SUMMARY_OUTPUT")
+	if path == "" {
+		fmt.Println("Note: E2E_SUMMARY_OUTPUT is unset; skipping summary generation")
+	} else {
+		if err := testutil.PrintSummary(path); err != nil {
+			// Fail the run if alerts are found
+			fmt.Printf("%v", err)
+			os.Exit(1)
+		}
+	}
+}
+
+func executeTestsParallel() {
+	// Create buffers to capture output for final summary
+	var parallelBuf, serialBuf bytes.Buffer
+
+	parallelOpts := cliOpts
+	if parallelOpts.Concurrency == 1 {
+		// Override default concurrency value with 100; otherwise use whatever was provided by CLI
+		parallelOpts.Concurrency = 100
+	}
+	parallelOpts.Tags = "~@Serial"
+	// Write to both specified output (live to stdout, by default) and buffer (for summary)
+	parallelOpts.Output = io.MultiWriter(parallelOpts.Output, &parallelBuf)
+	// run tests concurrently
+	scParallel := godog.TestSuite{
+		TestSuiteInitializer: InitializeSuite,
+		ScenarioInitializer:  InitializeScenario,
+		Options:              &parallelOpts,
+	}.Run()
+
+	fmt.Println("End of parallel run - beginning serial tests")
+
+	serialOpts := cliOpts
+	serialOpts.Concurrency = 1
+	serialOpts.Tags = "@Serial"
+	// Write to both specified output (live to stdout, by default) and buffer (for summary)
+	serialOpts.Output = io.MultiWriter(serialOpts.Output, &serialBuf)
+	// run tests serially
+	scSerial := godog.TestSuite{
 		TestSuiteInitializer: InitializeSuite,
 		ScenarioInitializer:  InitializeScenario,
-		Options:              &opts,
+		Options:              &serialOpts,
 	}.Run()
 
-	switch sc {
-	//	0 - success
-	case 0:
-
-		path := os.Getenv("E2E_SUMMARY_OUTPUT")
-		if path == "" {
-			fmt.Println("Note: E2E_SUMMARY_OUTPUT is unset; skipping summary generation")
-		} else {
-			if err := testutil.PrintSummary(path); err != nil {
-				// Fail the run if alerts are found
-				fmt.Printf("%v", err)
-				os.Exit(1)
-			}
+	// TODO We re-print the output of any failed steps here for easier debugging. However, it would be
+	// better to combine this with the E2E_SUMMARY_OUTPUT and show pass/fail + performance in one
+	// markdown output then preserve the console output for local testing.
+
+	// Print aggregated summary
+	fmt.Println("\n" + strings.Repeat("=", 80))
+	fmt.Println("TEST EXECUTION SUMMARY")
+	fmt.Println(strings.Repeat("=", 80))
+
+	fmt.Printf("\nParallel Tests Exit Code: %d\n", scParallel)
+	if scParallel != 0 {
+		failedSteps := extractFailedSteps(parallelBuf.String())
+		if failedSteps != "" {
+			fmt.Println("\nParallel Test Failures:")
+			fmt.Println(strings.Repeat("-", 80))
+			fmt.Println(failedSteps)
+		}
+	}
+
+	fmt.Printf("\nSerial Tests Exit Code: %d\n", scSerial)
+	if scSerial != 0 {
+		failedSteps := extractFailedSteps(serialBuf.String())
+		if failedSteps != "" {
+			fmt.Println("\nSerial Test Failures:")
+			fmt.Println(strings.Repeat("-", 80))
+			fmt.Println(failedSteps)
 		}
-		return
+	}
+
+	fmt.Println(strings.Repeat("=", 80))
+
+	if scParallel != 0 || scSerial != 0 {
+		//	1 - failed
+		//	2 - command line usage error
+		// 128 - or higher, os signal related error exit codes
+		log.Fatalf("non-zero status returned; parallel: (%d), serial: (%d), failed to run feature tests", scParallel, scSerial)
+	}
+}
+
+// extractFailedSteps extracts the "--- Failed steps:" section from godog output
+func extractFailedSteps(output string) string {
+	lines := strings.Split(output, "\n")
+	var failedSection []string
+	capturing := false
+
+	for _, line := range lines {
+		if strings.Contains(line, "--- Failed steps:") {
+			capturing = true
+		}
+		if capturing {
+			failedSection = append(failedSection, line)
+		}
+	}
 
-	//	1 - failed
-	//	2 - command line usage error
-	// 128 - or higher, os signal related error exit codes
-	default:
-		log.Fatalf("non-zero status returned (%d), failed to run feature tests", sc)
+	if len(failedSection) == 0 {
+		return ""
 	}
+	return strings.Join(failedSection, "\n")
 }
 
 func InitializeSuite(tc *godog.TestSuiteContext) {