test(perfcheck): harden baseline capture artifacts (#491)

Signed-off-by: Roel de Cort <roel.decort@adfinis.com>

Author: dc-tec

.github/workflows/perf-baseline-capture.yml

@@ -23,6 +23,14 @@ on:
         description: Kind node image used for capture and verification.
         required: false
         default: kindest/node:v1.34.3
+      continue_on_sample_error:
+        description: Continue collecting remaining samples after a failed setup or scenario sample.
+        required: false
+        default: "true"
+      minimum_successful_samples:
+        description: Minimum passing measured samples required before writing a scenario baseline.
+        required: false
+        default: "3"
 
 permissions:
   contents: read
@@ -116,6 +124,9 @@ jobs:
         with:
           install-dir: bin/
 
+      - name: Remove checked-in scenario baseline
+        run: rm -rf "hack/perf/v2/baselines/${{ matrix.scenario }}"
+
       - name: Capture v2 baseline samples
         env:
           GOFLAGS: -mod=vendor
@@ -127,6 +138,8 @@ jobs:
           PERF_BASELINE_DIR: hack/perf/v2/baselines
           PERF_POLICY_FILE: hack/perf/v2/policies/weekly.yaml
           PERF_ARTIFACT_DIR: dist/perf
+          PERF_CONTINUE_ON_SAMPLE_ERROR: ${{ inputs.continue_on_sample_error }}
+          PERF_MIN_SUCCESSFUL_SAMPLES: ${{ inputs.minimum_successful_samples }}
         run: make perf-baseline
 
       - name: Render captured baseline report

hack/perfcheck/analysis.go

@@ -604,6 +604,7 @@ func readRunSummary(path string) (runSummaryDocument, error) {
 func writeScenarioBaseline(opts options, scenario scenarioSpec, samples []sampleDocument) error {
 	values := make(map[string][]float64)
 	var environment runEnvironment
+	successfulSamples := 0
 	allowedMeasurements := scenarioMeasurementSet(scenario)
 	for _, sample := range samples {
 		if environment == (runEnvironment{}) {
@@ -612,6 +613,7 @@ func writeScenarioBaseline(opts options, scenario scenarioSpec, samples []sample
 		if sample.Warmup || sample.Status != sampleStatusPass {
 			continue
 		}
+		successfulSamples++
 		for key, value := range sample.Measurements {
 			if math.IsNaN(value) || math.IsInf(value, 0) {
 				continue
@@ -622,6 +624,17 @@ func writeScenarioBaseline(opts options, scenario scenarioSpec, samples []sample
 			values[key] = append(values[key], value)
 		}
 	}
+	if opts.MinimumSuccessfulSamples > 0 && successfulSamples < opts.MinimumSuccessfulSamples {
+		return fmt.Errorf(
+			"scenario %q produced %d passing measured samples, need at least %d",
+			scenario.Name,
+			successfulSamples,
+			opts.MinimumSuccessfulSamples,
+		)
+	}
+	if len(values) == 0 {
+		return fmt.Errorf("scenario %q produced no baseline measurements", scenario.Name)
+	}
 
 	summary := make(map[string]measurementSummary, len(values))
 	for key, metricValues := range values {

hack/perfcheck/analysis_test.go

@@ -21,6 +21,58 @@ func TestSummarizeValues(t *testing.T) {
 	}
 }
 
+func TestWriteScenarioBaselineRequiresMinimumSuccessfulSamples(t *testing.T) {
+	t.Parallel()
+
+	opts := defaultOptions("capture")
+	opts.BaselineDir = t.TempDir()
+	opts.MinimumSuccessfulSamples = 2
+	scenario := scenarioSpec{
+		Name:    "lifecycle-convergence",
+		Primary: []string{metricClusterAvailableSeconds},
+	}
+	samples := []sampleDocument{
+		{
+			Version:     versionV2,
+			Scenario:    scenario.Name,
+			Sample:      1,
+			Status:      sampleStatusPass,
+			Environment: runEnvironment{Commit: "abc123"},
+			Measurements: map[string]float64{
+				metricClusterAvailableSeconds: 30,
+			},
+		},
+		{
+			Version:      versionV2,
+			Scenario:     scenario.Name,
+			Sample:       2,
+			Status:       sampleStatusScenarioError,
+			Error:        "setup failed",
+			Measurements: map[string]float64{},
+		},
+	}
+
+	err := writeScenarioBaseline(opts, scenario, samples)
+	if err == nil {
+		t.Fatalf("expected minimum successful sample error")
+	}
+	if !strings.Contains(err.Error(), "produced 1 passing measured samples, need at least 2") {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	opts.MinimumSuccessfulSamples = 1
+	if err := writeScenarioBaseline(opts, scenario, samples); err != nil {
+		t.Fatalf("write baseline with enough samples: %v", err)
+	}
+	doc, err := readScenarioBaseline(opts, scenario.Name)
+	if err != nil {
+		t.Fatalf("read baseline: %v", err)
+	}
+	if got := len(doc.Samples[metricClusterAvailableSeconds]); got != 1 {
+		t.Fatalf("baseline sample count = %d, want 1", got)
+	}
+}
+
 func TestCompareMeasurementsRequiresAbsoluteAndRelativeRegression(t *testing.T) {
 	current := map[string]measurementSummary{
 		metricSampleTotalSeconds: {Median: 125, UpperSample: 125, Min: 125, Max: 125, Count: 3},

hack/perfcheck/main.go

@@ -173,6 +173,12 @@ func bindExecutionFlags(fs *flag.FlagSet, opts *options) {
 		false,
 		"continue running remaining samples after a scenario or measurement error",
 	)
+	fs.IntVar(
+		&opts.MinimumSuccessfulSamples,
+		"minimum-successful-samples",
+		0,
+		"minimum passing measured samples required before writing a scenario baseline; 0 disables the guard",
+	)
 	fs.BoolVar(&opts.SkipImageBuild, "skip-image-build", false, "skip image build when supported by the executor")
 	fs.StringVar(&opts.OperatorImage, "operator-image", opts.OperatorImage, "operator image for native scenarios")
 	fs.StringVar(&opts.ConfigInitImage, "config-init-image", opts.ConfigInitImage, "config-init image")
@@ -263,6 +269,9 @@ func finalizeOptions(opts options) (options, error) {
 	if opts.TenantChurnCount < 1 {
 		return options{}, fmt.Errorf("tenant-churn-count must be >= 1")
 	}
+	if opts.MinimumSuccessfulSamples < 0 {
+		return options{}, fmt.Errorf("minimum-successful-samples must be >= 0")
+	}
 	return opts, nil
 }
 

hack/perfcheck/main_test.go

@@ -50,6 +50,17 @@ func TestFinalizeOptionsRejectsInvalidTenantChurnCount(t *testing.T) {
 	}
 }
 
+func TestFinalizeOptionsRejectsInvalidMinimumSuccessfulSamples(t *testing.T) {
+	t.Parallel()
+
+	opts := defaultOptions("capture")
+	opts.MinimumSuccessfulSamples = -1
+
+	if _, err := finalizeOptions(opts); err == nil {
+		t.Fatalf("expected minimum-successful-samples validation error")
+	}
+}
+
 func TestDefaultRollingUpgradeSourceUsesPatchUpgrade(t *testing.T) {
 	t.Setenv("PERF_UPGRADE_FROM_VERSION", "")
 	t.Setenv("PERF_UPGRADE_FROM_IMAGE", "")

hack/perfcheck/native_setup.go

@@ -194,15 +194,28 @@ func installCertManagerIfNeeded(ctx context.Context, opts options, cluster strin
 	}
 	if _, err := nativeKubectl(ctx, opts, cluster,
 		"wait",
+		"--for",
+		"condition=Established",
+		"crd/certificates.cert-manager.io",
+		"crd/issuers.cert-manager.io",
+		"--timeout",
+		"2m",
+	); err != nil {
+		return fmt.Errorf("wait for cert-manager CRDs: %w", err)
+	}
+	if _, err := nativeKubectl(ctx, opts, cluster,
+		"wait",
+		"deployment.apps/cert-manager",
+		"deployment.apps/cert-manager-cainjector",
 		"deployment.apps/cert-manager-webhook",
 		"--for",
 		"condition=Available",
 		"--namespace",
 		"cert-manager",
 		"--timeout",
-		"5m",
+		"10m",
 	); err != nil {
-		return fmt.Errorf("wait for cert-manager webhook: %w", err)
+		return fmt.Errorf("wait for cert-manager deployments: %w", err)
 	}
 	return nil
 }

hack/perfcheck/runner.go

@@ -47,10 +47,22 @@ func runCapture(opts options) error {
 		}
 	}
 
+	failedSamples := 0
 	for _, sample := range samples {
-		if sample.Status != sampleStatusPass && !sample.Warmup {
+		if sample.Status == sampleStatusPass || sample.Warmup {
+			continue
+		}
+		failedSamples++
+	}
+	if failedSamples > 0 {
+		if !opts.ContinueOnSampleError {
 			return fmt.Errorf("capture completed with scenario errors; inspect %s", opts.ArtifactDir)
 		}
+		fmt.Fprintf(
+			os.Stderr,
+			"warning: capture completed with %d failed measured samples; baselines use passing samples only\n",
+			failedSamples,
+		)
 	}
 	fmt.Printf("wrote v2 baselines under %s\n", opts.BaselineDir)
 	fmt.Printf("wrote v2 sample artifacts under %s\n", opts.ArtifactDir)
@@ -189,6 +201,9 @@ func executeScenarioSample(
 	if prepareErr != nil {
 		sample.Status = sampleStatusScenarioError
 		sample.Error = prepareErr.Error()
+		if err := collectKubernetesArtifacts(opts, scenarioDir, cluster, sample, ""); err != nil {
+			fmt.Fprintf(os.Stderr, "warning: collecting Kubernetes setup artifacts failed: %v\n", err)
+		}
 		return finishAndWriteSample(opts, sample)
 	}
 	defer func() {
@@ -1080,6 +1095,8 @@ func collectKubernetesArtifacts(
 		args []string
 	}{
 		{"pods.json", append([]string{"get", "pods"}, scopeArgs...)},
+		{"deployments.json", append([]string{"get", "deployments"}, scopeArgs...)},
+		{"replicasets.json", append([]string{"get", "replicasets"}, scopeArgs...)},
 		{"jobs.json", append([]string{"get", "jobs"}, scopeArgs...)},
 		{"events.json", append([]string{"get", "events"}, scopeArgs...)},
 		{"statefulsets.json", append([]string{"get", "statefulsets"}, scopeArgs...)},

hack/perfcheck/types.go

@@ -271,49 +271,50 @@ type analysisFinding struct {
 }
 
 type options struct {
-	Mode                   string
-	ScenarioNames          []string
-	ScenarioPath           string
-	PolicyPath             string
-	BaselineDir            string
-	ArtifactDir            string
-	PreviousSummaryPath    string
-	SummaryOut             string
-	ReportOut              string
-	FailOnFailures         bool
-	RunID                  string
-	EnvironmentID          string
-	NodeImage              string
-	KindBin                string
-	MakeBin                string
-	ScenarioTimeout        time.Duration
-	ClusterTimeout         time.Duration
-	CleanupTimeout         time.Duration
-	KeepOnFailure          bool
-	ContinueOnSampleError  bool
-	SamplesOverride        int
-	WarmupsOverride        int
-	ExistingClusterContext string
-	Namespace              string
-	NamespacePrefix        string
-	SkipImageBuild         bool
-	OperatorImage          string
-	ConfigInitImage        string
-	BackupExecutorImage    string
-	UpgradeExecutorImage   string
-	OpenBaoVersion         string
-	OpenBaoImage           string
-	UpgradeFromVersion     string
-	UpgradeFromImage       string
-	UpgradeToVersion       string
-	UpgradeToImage         string
-	APIServerCIDR          string
-	StorageClass           string
-	TenantChurnCount       int
-	OperatorNS             string
-	MetricsService         string
-	ServiceAccount         string
-	BindingName            string
+	Mode                     string
+	ScenarioNames            []string
+	ScenarioPath             string
+	PolicyPath               string
+	BaselineDir              string
+	ArtifactDir              string
+	PreviousSummaryPath      string
+	SummaryOut               string
+	ReportOut                string
+	FailOnFailures           bool
+	RunID                    string
+	EnvironmentID            string
+	NodeImage                string
+	KindBin                  string
+	MakeBin                  string
+	ScenarioTimeout          time.Duration
+	ClusterTimeout           time.Duration
+	CleanupTimeout           time.Duration
+	KeepOnFailure            bool
+	ContinueOnSampleError    bool
+	SamplesOverride          int
+	WarmupsOverride          int
+	ExistingClusterContext   string
+	Namespace                string
+	NamespacePrefix          string
+	SkipImageBuild           bool
+	OperatorImage            string
+	ConfigInitImage          string
+	BackupExecutorImage      string
+	UpgradeExecutorImage     string
+	OpenBaoVersion           string
+	OpenBaoImage             string
+	UpgradeFromVersion       string
+	UpgradeFromImage         string
+	UpgradeToVersion         string
+	UpgradeToImage           string
+	APIServerCIDR            string
+	StorageClass             string
+	TenantChurnCount         int
+	MinimumSuccessfulSamples int
+	OperatorNS               string
+	MetricsService           string
+	ServiceAccount           string
+	BindingName              string
 }
 
 type metricsSnapshot struct {

mk/development.mk

@@ -429,6 +429,7 @@ PERF_ENVIRONMENT ?= kind-v1.34.3
 PERF_PREVIOUS_SUMMARY ?=
 PERF_REPORT_FAIL_ON_FAILURES ?= false
 PERF_CONTINUE_ON_SAMPLE_ERROR ?= false
+PERF_MIN_SUCCESSFUL_SAMPLES ?= 0
 PERF_OPERATOR_IMAGE ?= example.com/openbao-operator:0.0.1
 PERF_CONFIG_INIT_IMAGE ?= openbao-init:dev
 PERF_BACKUP_EXECUTOR_IMAGE ?= openbao-backup:dev
@@ -692,6 +693,7 @@ perf-v2-capture: ## Capture v2 performance samples and update per-scenario distr
 		--environment="$(PERF_ENVIRONMENT)" \
 		--scenario-timeout="$(PERF_SCENARIO_TIMEOUT)" \
 		--continue-on-sample-error="$(PERF_CONTINUE_ON_SAMPLE_ERROR)" \
+		--minimum-successful-samples="$(PERF_MIN_SUCCESSFUL_SAMPLES)" \
 		--operator-image="$(PERF_OPERATOR_IMAGE)" \
 		--config-init-image="$(PERF_CONFIG_INIT_IMAGE)" \
 		--backup-executor-image="$(PERF_BACKUP_EXECUTOR_IMAGE)" \