feat: allow RateLimitRedisSettings.url to be sourced from a Secret (#9143)

* api: add urlRef to RateLimitRedisSettings (#9022)

Signed-off-by: Andrey Maltsev <maltsev.andrey@gmail.com>

Author: guanchzhou

api/v1alpha1/envoygateway_types.go

@@ -6,6 +6,7 @@
 package v1alpha1
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
@@ -682,17 +683,42 @@ type RedisTLSSettings struct {
 }
 
 // RateLimitRedisSettings defines the configuration for connecting to redis database.
+// +kubebuilder:validation:XValidation:rule="has(self.url) != has(self.urlRef)",message="exactly one of url or urlRef must be set"
 type RateLimitRedisSettings struct {
 	// URL of the Redis Database.
 	// This can reference a single Redis host or a comma delimited list for Sentinel and Cluster deployments of Redis.
-	URL string `json:"url"`
+	// Mutually exclusive with URLRef.
+	//
+	// +optional
+	URL string `json:"url,omitempty"`
+
+	// URLRef sources the Redis URL from a Kubernetes Secret key. Use this for GitOps
+	// flows where the Redis endpoint is provisioned by an external controller.
+	// The referenced Secret must exist in the namespace of the Envoy Gateway rate limit
+	// deployment. Mutually exclusive with URL.
+	//
+	// +optional
+	URLRef *RedisURLSource `json:"urlRef,omitempty"`
 
 	// TLS defines TLS configuration for connecting to redis database.
 	//
 	// +optional
 	TLS *RedisTLSSettings `json:"tls,omitempty"`
 }
 
+// RedisURLSource specifies where to source the Redis URL from.
+// +kubebuilder:validation:XValidation:rule="!has(self.secretKeyRef.optional) || !self.secretKeyRef.optional",message="urlRef.secretKeyRef.optional must not be true; the Secret is required"
+type RedisURLSource struct {
+	// SecretKeyRef references the Secret and key that hold the Redis URL.
+	// The Secret must be in the same namespace as the Envoy Gateway rate limit deployment.
+	// The reference is always required: optional must not be set to true, otherwise
+	// the rate limit pod could start with an unset REDIS_URL instead of waiting for
+	// the externally provisioned Secret.
+	//
+	// +kubebuilder:validation:Required
+	SecretKeyRef *corev1.SecretKeySelector `json:"secretKeyRef"`
+}
+
 // ExtensionManager defines the configuration for registering an extension manager to
 // the Envoy Gateway control plane.
 type ExtensionManager struct {

api/v1alpha1/validation/envoygateway_validate.go

@@ -189,11 +189,42 @@ func validateEnvoyGatewayRateLimit(rateLimit *egv1a1.RateLimit) error {
 	if rateLimit.Backend.Type != egv1a1.RedisBackendType {
 		return fmt.Errorf("unsupported ratelimit backend %v", rateLimit.Backend.Type)
 	}
-	if rateLimit.Backend.Redis == nil || rateLimit.Backend.Redis.URL == "" {
+	redis := rateLimit.Backend.Redis
+	if redis == nil {
 		return fmt.Errorf("empty ratelimit redis settings")
 	}
-	redisHosts := strings.Split(rateLimit.Backend.Redis.URL, ",")
-	for _, host := range redisHosts {
+
+	hasURL := redis.URL != ""
+	hasURLRef := redis.URLRef != nil
+	if hasURL == hasURLRef {
+		return fmt.Errorf("exactly one of ratelimit redis url or urlRef must be set")
+	}
+
+	if hasURLRef {
+		ref := redis.URLRef.SecretKeyRef
+		if ref == nil || ref.Name == "" || ref.Key == "" {
+			return fmt.Errorf("ratelimit redis urlRef.secretKeyRef must set both name and key")
+		}
+		// The Secret is required: the rate limit pod must wait for an
+		// externally provisioned Secret rather than start with an unset
+		// REDIS_URL. An optional reference would let the container boot
+		// nonfunctional, so reject optional=true.
+		if ref.Optional != nil && *ref.Optional {
+			return fmt.Errorf("ratelimit redis urlRef.secretKeyRef.optional must not be true; the Secret is required")
+		}
+		return nil
+	}
+
+	return ValidateRedisURL(redis.URL)
+}
+
+// ValidateRedisURL validates a ratelimit Redis URL string, which may be a single
+// host or a comma-delimited list of hosts for Sentinel and Cluster deployments.
+func ValidateRedisURL(redisURL string) error {
+	if redisURL == "" {
+		return fmt.Errorf("ratelimit redis url is empty")
+	}
+	for _, host := range strings.Split(redisURL, ",") {
 		if _, err := url.Parse(host); err != nil {
 			return fmt.Errorf("unknown ratelimit redis url format: %w", err)
 		}

api/v1alpha1/validation/envoygateway_validate_test.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
 
@@ -1462,3 +1463,123 @@ func TestLuaDisabled(t *testing.T) {
 		})
 	}
 }
+
+func TestValidateEnvoyGatewayRateLimitURLRef(t *testing.T) {
+	redisBackend := func(redis *egv1a1.RateLimitRedisSettings) *egv1a1.RateLimit {
+		return &egv1a1.RateLimit{
+			Backend: egv1a1.RateLimitDatabaseBackend{
+				Type:  egv1a1.RedisBackendType,
+				Redis: redis,
+			},
+		}
+	}
+	cases := []struct {
+		name      string
+		rateLimit *egv1a1.RateLimit
+		expectErr bool
+	}{
+		{
+			name:      "nil redis settings",
+			rateLimit: redisBackend(nil),
+			expectErr: true,
+		},
+		{
+			name:      "url only",
+			rateLimit: redisBackend(&egv1a1.RateLimitRedisSettings{URL: "redis.redis.svc:6379"}),
+			expectErr: false,
+		},
+		{
+			name: "urlRef only",
+			rateLimit: redisBackend(&egv1a1.RateLimitRedisSettings{
+				URLRef: &egv1a1.RedisURLSource{
+					SecretKeyRef: &corev1.SecretKeySelector{
+						LocalObjectReference: corev1.LocalObjectReference{Name: "redis-conn"},
+						Key:                  "REDIS_ENDPOINT",
+					},
+				},
+			}),
+			expectErr: false,
+		},
+		{
+			name: "both url and urlRef set",
+			rateLimit: redisBackend(&egv1a1.RateLimitRedisSettings{
+				URL: "redis.redis.svc:6379",
+				URLRef: &egv1a1.RedisURLSource{
+					SecretKeyRef: &corev1.SecretKeySelector{
+						LocalObjectReference: corev1.LocalObjectReference{Name: "redis-conn"},
+						Key:                  "REDIS_ENDPOINT",
+					},
+				},
+			}),
+			expectErr: true,
+		},
+		{
+			name:      "neither url nor urlRef",
+			rateLimit: redisBackend(&egv1a1.RateLimitRedisSettings{}),
+			expectErr: true,
+		},
+		{
+			name: "urlRef missing key",
+			rateLimit: redisBackend(&egv1a1.RateLimitRedisSettings{
+				URLRef: &egv1a1.RedisURLSource{
+					SecretKeyRef: &corev1.SecretKeySelector{
+						LocalObjectReference: corev1.LocalObjectReference{Name: "redis-conn"},
+					},
+				},
+			}),
+			expectErr: true,
+		},
+		{
+			name: "urlRef missing name",
+			rateLimit: redisBackend(&egv1a1.RateLimitRedisSettings{
+				URLRef: &egv1a1.RedisURLSource{
+					SecretKeyRef: &corev1.SecretKeySelector{Key: "REDIS_ENDPOINT"},
+				},
+			}),
+			expectErr: true,
+		},
+		{
+			name: "urlRef optional true",
+			rateLimit: redisBackend(&egv1a1.RateLimitRedisSettings{
+				URLRef: &egv1a1.RedisURLSource{
+					SecretKeyRef: &corev1.SecretKeySelector{
+						LocalObjectReference: corev1.LocalObjectReference{Name: "redis-conn"},
+						Key:                  "REDIS_ENDPOINT",
+						Optional:             new(true),
+					},
+				},
+			}),
+			expectErr: true,
+		},
+		{
+			name: "urlRef optional false",
+			rateLimit: redisBackend(&egv1a1.RateLimitRedisSettings{
+				URLRef: &egv1a1.RedisURLSource{
+					SecretKeyRef: &corev1.SecretKeySelector{
+						LocalObjectReference: corev1.LocalObjectReference{Name: "redis-conn"},
+						Key:                  "REDIS_ENDPOINT",
+						Optional:             new(false),
+					},
+				},
+			}),
+			expectErr: false,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := validateEnvoyGatewayRateLimit(tc.rateLimit)
+			if tc.expectErr {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+		})
+	}
+}
+
+func TestValidateRedisURL(t *testing.T) {
+	require.NoError(t, ValidateRedisURL("redis.redis.svc:6379"))
+	require.NoError(t, ValidateRedisURL("a.redis.svc:6379,b.redis.svc:6379"))
+	require.ErrorContains(t, ValidateRedisURL(""), "ratelimit redis url is empty")
+	require.ErrorContains(t, ValidateRedisURL(":foo"), "unknown ratelimit redis url format")
+}

api/v1alpha1/zz_generated.deepcopy.go

@@ -13,10 +13,13 @@ import (
 	"strings"
 
 	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
+	"github.com/envoyproxy/gateway/api/v1alpha1/validation"
 	"github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/resource"
 	"github.com/envoyproxy/gateway/internal/kubernetes"
 )
@@ -373,15 +376,20 @@ func expectedRateLimitContainerEnv(rateLimit *egv1a1.RateLimit, rateLimitDeploym
 	}
 
 	if rateLimit.Backend.Redis != nil {
+		redisURLEnv := corev1.EnvVar{Name: RedisURLEnvVar}
+		if rateLimit.Backend.Redis.URLRef != nil {
+			redisURLEnv.ValueFrom = &corev1.EnvVarSource{
+				SecretKeyRef: rateLimit.Backend.Redis.URLRef.SecretKeyRef,
+			}
+		} else {
+			redisURLEnv.Value = rateLimit.Backend.Redis.URL
+		}
 		env = append(env, []corev1.EnvVar{
 			{
 				Name:  RedisSocketTypeEnvVar,
 				Value: "tcp",
 			},
-			{
-				Name:  RedisURLEnvVar,
-				Value: rateLimit.Backend.Redis.URL,
-			},
+			redisURLEnv,
 		}...)
 	}
 
@@ -469,13 +477,37 @@ func expectedRateLimitContainerEnv(rateLimit *egv1a1.RateLimit, rateLimitDeploym
 	return resource.ExpectedContainerEnv(rateLimitDeployment.Container, env)
 }
 
-// Validate the ratelimit tls secret validating.
+// Validate validates the ratelimit redis url/tls secret references.
 func Validate(ctx context.Context, client client.Client, gateway *egv1a1.EnvoyGateway, namespace string) error {
-	if gateway.RateLimit.Backend.Redis != nil &&
-		gateway.RateLimit.Backend.Redis.TLS != nil &&
-		gateway.RateLimit.Backend.Redis.TLS.CertificateRef != nil {
-		certificateRef := gateway.RateLimit.Backend.Redis.TLS.CertificateRef
-		_, _, err := kubernetes.ValidateSecretObjectReference(ctx, client, certificateRef, namespace)
+	redis := gateway.RateLimit.Backend.Redis
+	if redis == nil {
+		return nil
+	}
+
+	if redis.URLRef != nil && redis.URLRef.SecretKeyRef != nil {
+		ref := redis.URLRef.SecretKeyRef
+		secret := &corev1.Secret{}
+		err := client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: ref.Name}, secret)
+		switch {
+		case apierrors.IsNotFound(err):
+			// The Secret may be provisioned after the EnvoyGateway config (e.g. by an
+			// external controller such as Crossplane). The ratelimit Deployment consumes
+			// the URL via valueFrom.secretKeyRef, so the kubelet starts the container once
+			// the Secret and key exist. Don't block infra creation on a not-yet-present Secret.
+		case err != nil:
+			return fmt.Errorf("failed to get Secret %s in namespace %s: %w", ref.Name, namespace, err)
+		default:
+			// Validate the value only when present; a missing key is resolved later by the kubelet.
+			if value, ok := secret.Data[ref.Key]; ok {
+				if verr := validation.ValidateRedisURL(string(value)); verr != nil {
+					return fmt.Errorf("invalid Redis URL in Secret %s/%s key %q: %w", namespace, ref.Name, ref.Key, verr)
+				}
+			}
+		}
+	}
+
+	if redis.TLS != nil && redis.TLS.CertificateRef != nil {
+		_, _, err := kubernetes.ValidateSecretObjectReference(ctx, client, redis.TLS.CertificateRef, namespace)
 		return err
 	}