E2E Summary Output Fix (#2751)

The serial e2e tests require pod restarts which causes the metrics to scrape two sets of results, which the summary generator did not allow. Now, the results from the new pod will be aggregated to the final set.

Signed-off-by: Daniel Franz <dfranz@redhat.com>

Author: dtfranz

internal/shared/util/test/utils.go

@@ -10,8 +10,8 @@ import (
 	"github.com/cucumber/godog/colors"
 	"github.com/spf13/pflag"
 
-	testutil "github.com/operator-framework/operator-controller/internal/shared/util/test"
 	"github.com/operator-framework/operator-controller/test/e2e/steps"
+	"github.com/operator-framework/operator-controller/test/internal/summary"
 )
 
 var opts = godog.Options{
@@ -47,10 +47,9 @@ func TestMain(m *testing.M) {
 		if path == "" {
 			fmt.Println("Note: E2E_SUMMARY_OUTPUT is unset; skipping summary generation")
 		} else {
-			if err := testutil.PrintSummary(path); err != nil {
-				// Fail the run if alerts are found
+			if err := summary.PrintSummary(path); err != nil {
+				// Alert but do not fail the run if alerts are found
 				fmt.Printf("%v", err)
-				os.Exit(1)
 			}
 		}
 		return

test/e2e/features_test.go

@@ -1,4 +1,4 @@
-package test
+package summary
 
 import (
 	"context"

internal/shared/util/test/artifacts.go test/internal/summary/artifacts.gointernal/shared/util/test/artifacts.go renamed to test/internal/summary/artifacts.go

@@ -1,4 +1,4 @@
-package test
+package summary
 
 import (
 	"context"
@@ -80,8 +80,8 @@ func (s *githubSummary) PerformanceQuery(title, pod, query, yLabel string, scale
 	matrix, ok := result.(model.Matrix)
 	if !ok {
 		return "", fmt.Errorf("typecast for metrics samples failed; aborting")
-	} else if len(matrix) != 1 {
-		return "", fmt.Errorf("expected 1 set of results; got: %d", len(matrix))
+	} else if len(matrix) == 0 {
+		return "", fmt.Errorf("expected 1 or more sets of results; got 0")
 	}
 	chart := xychart{
 		Title:  title,
@@ -92,10 +92,6 @@ func (s *githubSummary) PerformanceQuery(title, pod, query, yLabel string, scale
 	formattedData := make([]string, 0)
 	// matrix does not allow [] access, so we just do one iteration for the single result
 	for _, metric := range matrix {
-		if len(metric.Values) < 2 {
-			// A graph with one data point means something with the collection was wrong
-			return "", fmt.Errorf("expected at least two data points; got: %d", len(metric.Values))
-		}
 		for _, sample := range metric.Values {
 			floatSample := float64(sample.Value) * scaler
 			formattedData = append(formattedData, fmt.Sprintf("%f", floatSample))
@@ -160,7 +156,7 @@ func executeTemplate(templateFile string, obj any) (string, error) {
 	if err != nil {
 		return "", fmt.Errorf("failed to get working directory: %w", err)
 	}
-	tmpl, err := template.New(templateFile).ParseGlob(filepath.Join(wd, "../../internal/shared/util/test/templates", templateFile))
+	tmpl, err := template.New(templateFile).ParseGlob(filepath.Join(wd, "../internal/summary/templates", templateFile))
 	if err != nil {
 		return "", err
 	}