Restrict test retries to an allowlist of CI jobs

Add a job-level allowlist for the "once" retry strategy. Only jobs
explicitly listed in retry_allowed_jobs.yaml are permitted to retry
failed tests. All other CI jobs must pass on the first attempt.

| Scenario                          | Retries? |
|-----------------------------------|----------|
| Job listed in the YAML allowlist  | Yes      |
| Job NOT in the allowlist          | No       |
| JOB_NAME not set (local/dev run)  | Yes      |
| Empty allowlist (jobs: [])        | No       |

The allowlist starts empty, meaning no CI job can retry today.
To allow retries for a job, add its name to
pkg/test/ginkgo/retry_allowed_jobs.yaml:

  jobs:
    - periodic-ci-openshift-release-master-ci-4.17-e2e-aws
    - periodic-ci-openshift-release-master-ci-4.17-e2e-gcp

The file is embedded at compile time via go:embed, so changes
require rebuilding openshift-tests.

Author: mkowalski

pkg/test/ginkgo/retries.go

@@ -2,6 +2,7 @@ package ginkgo
 
 import (
 	"context"
+	_ "embed"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -11,8 +12,32 @@ import (
 
 	"github.com/openshift/origin/pkg/dataloader"
 	"github.com/sirupsen/logrus"
+	"sigs.k8s.io/yaml"
 )
 
+//go:embed retry_allowed_jobs.yaml
+var retryAllowedJobsYAML []byte
+
+// retryAllowedJobsConfig represents the YAML structure for the retry allowlist.
+type retryAllowedJobsConfig struct {
+	Jobs []string `json:"jobs"`
+}
+
+// loadRetryAllowedJobs parses the embedded YAML and returns the set of job names
+// that are permitted to retry failed tests.
+func loadRetryAllowedJobs() map[string]bool {
+	var config retryAllowedJobsConfig
+	if err := yaml.Unmarshal(retryAllowedJobsYAML, &config); err != nil {
+		logrus.WithError(err).Error("Failed to parse retry_allowed_jobs.yaml, no jobs will be allowed to retry")
+		return map[string]bool{}
+	}
+	allowed := make(map[string]bool, len(config.Jobs))
+	for _, job := range config.Jobs {
+		allowed[job] = true
+	}
+	return allowed
+}
+
 const (
 	defaultRetryStrategy = "once"
 
@@ -61,11 +86,13 @@ type RetryStrategy interface {
 
 type RetryOnceStrategy struct {
 	PermittedRetryImageTags []string
+	AllowedRetryJobs        map[string]bool
 }
 
 func NewRetryOnceStrategy() *RetryOnceStrategy {
 	return &RetryOnceStrategy{
 		PermittedRetryImageTags: []string{"tests"}, // tests = openshift-tests image
+		AllowedRetryJobs:        loadRetryAllowedJobs(),
 	}
 }
 
@@ -74,7 +101,24 @@ func (s *RetryOnceStrategy) Name() string {
 }
 
 func (s *RetryOnceStrategy) ShouldAttemptRetries(failing []*testCase, suite *TestSuite) bool {
-	return len(failing) > 0 && len(failing) <= suite.MaximumAllowedFlakes
+	if len(failing) == 0 || len(failing) > suite.MaximumAllowedFlakes {
+		return false
+	}
+
+	jobName := os.Getenv("JOB_NAME")
+	if jobName == "" {
+		// When running outside of CI (no JOB_NAME set), allow retries.
+		logrus.Info("JOB_NAME not set, allowing retries")
+		return true
+	}
+
+	if !s.AllowedRetryJobs[jobName] {
+		logrus.Infof("Job %q is not in the retry allowlist, retries disabled", jobName)
+		return false
+	}
+
+	logrus.Infof("Job %q is in the retry allowlist, retries permitted", jobName)
+	return true
 }
 
 func (s *RetryOnceStrategy) GetMaxRetries(testCase *testCase) int {

pkg/test/ginkgo/retries_test.go

@@ -5,6 +5,108 @@ import (
 	"time"
 )
 
+func TestRetryOnceStrategy_ShouldAttemptRetries_Allowlist(t *testing.T) {
+	suite := &TestSuite{MaximumAllowedFlakes: 15}
+	failing := []*testCase{{name: "some-test", failed: true}}
+
+	tests := []struct {
+		name           string
+		allowedJobs    map[string]bool
+		jobNameEnv     string
+		expected       bool
+		description    string
+	}{
+		{
+			name:        "allowed_job_should_retry",
+			allowedJobs: map[string]bool{"periodic-ci-openshift-e2e-aws": true},
+			jobNameEnv:  "periodic-ci-openshift-e2e-aws",
+			expected:    true,
+			description: "Job in the allowlist should be permitted to retry",
+		},
+		{
+			name:        "disallowed_job_should_not_retry",
+			allowedJobs: map[string]bool{"periodic-ci-openshift-e2e-aws": true},
+			jobNameEnv:  "periodic-ci-openshift-e2e-gcp",
+			expected:    false,
+			description: "Job not in the allowlist should not be permitted to retry",
+		},
+		{
+			name:        "empty_allowlist_should_not_retry",
+			allowedJobs: map[string]bool{},
+			jobNameEnv:  "periodic-ci-openshift-e2e-aws",
+			expected:    false,
+			description: "Empty allowlist should block all jobs from retrying",
+		},
+		{
+			name:        "no_job_name_should_allow_retry",
+			allowedJobs: map[string]bool{},
+			jobNameEnv:  "",
+			expected:    true,
+			description: "When JOB_NAME is not set (local run), retries should be allowed",
+		},
+		{
+			name:        "no_failures_should_not_retry",
+			allowedJobs: map[string]bool{"periodic-ci-openshift-e2e-aws": true},
+			jobNameEnv:  "periodic-ci-openshift-e2e-aws",
+			expected:    false,
+			description: "No failing tests means no retries regardless of allowlist",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			strategy := &RetryOnceStrategy{
+				PermittedRetryImageTags: []string{"tests"},
+				AllowedRetryJobs:        tt.allowedJobs,
+			}
+
+			t.Setenv("JOB_NAME", tt.jobNameEnv)
+
+			testFailing := failing
+			if tt.name == "no_failures_should_not_retry" {
+				testFailing = nil
+			}
+
+			result := strategy.ShouldAttemptRetries(testFailing, suite)
+			if result != tt.expected {
+				t.Errorf("%s: expected %v, got %v", tt.description, tt.expected, result)
+			}
+		})
+	}
+}
+
+func TestRetryOnceStrategy_ShouldAttemptRetries_ExceedsMaxFlakes(t *testing.T) {
+	suite := &TestSuite{MaximumAllowedFlakes: 2}
+
+	// 3 failures exceeds MaximumAllowedFlakes of 2
+	failing := []*testCase{
+		{name: "test-1", failed: true},
+		{name: "test-2", failed: true},
+		{name: "test-3", failed: true},
+	}
+
+	strategy := &RetryOnceStrategy{
+		PermittedRetryImageTags: []string{"tests"},
+		AllowedRetryJobs:        map[string]bool{"my-job": true},
+	}
+
+	t.Setenv("JOB_NAME", "my-job")
+
+	result := strategy.ShouldAttemptRetries(failing, suite)
+	if result != false {
+		t.Errorf("Should not retry when failures exceed MaximumAllowedFlakes, got %v", result)
+	}
+}
+
+func TestLoadRetryAllowedJobs(t *testing.T) {
+	// This tests that the embedded YAML file can be parsed without error.
+	// The actual content depends on the YAML file, but parsing should not panic.
+	jobs := loadRetryAllowedJobs()
+	if jobs == nil {
+		t.Error("loadRetryAllowedJobs returned nil, expected an initialized map")
+	}
+}
+
 // createAttempts creates a slice of test attempts with the specified number of successes and failures
 func createAttempts(successes, failures int) []*testCase {
 	var attempts []*testCase

pkg/test/ginkgo/retry_allowed_jobs.yaml

@@ -0,0 +1,16 @@
+# Jobs that are allowed to retry failed tests once.
+# Any job NOT in this list must pass all tests on the first attempt.
+#
+# This list should be reviewed periodically and reduced over time.
+# To determine which jobs should remain here, query BigQuery:
+#
+#   SELECT JobName, COUNT(*) AS total_retried_tests,
+#     COUNTIF(FinalOutcome = 'flaky') AS passed_on_retry
+#   FROM retry_statistics
+#   WHERE _PARTITIONDATE >= DATE_SUB(CURRENT_DATE(), INTERVAL 30 DAY)
+#   GROUP BY JobName
+#   ORDER BY passed_on_retry DESC
+#
+# Jobs with passed_on_retry > 0 are benefiting from retries and should stay.
+# Jobs with passed_on_retry = 0 (or absent) should be removed from this list.
+jobs: []