feat: add automatic re-issuance of failed custom hostname certificates

internal/config/config.go

@@ -710,6 +710,35 @@ type GatewayConfig struct {
 	// When false: NSO emits ZERO EPPs and does NOT delete EPPs it did not create.
 	// When true (default): EPP emission proceeds as today.
 	EPPEmissionEnabled *bool `json:"eppEmissionEnabled,omitempty" yaml:"eppEmissionEnabled,omitempty"`
+
+	// CertificateReissuance controls how the gateway controller handles
+	// failed certificate issuance for custom hostnames. When a Certificate
+	// is stuck in a failed state, the controller deletes and recreates it
+	// to bypass cert-manager's exponential backoff. Kubernetes GC cascades
+	// the deletion through the entire chain (CertificateRequest, Order,
+	// Challenge, solver resources).
+	CertificateReissuance CertificateReissuanceConfig `json:"certificateReissuance,omitempty"`
+}
+
+// +k8s:deepcopy-gen=true
+
+// CertificateReissuanceConfig controls automatic recovery of failed certificate
+// issuance by deleting and recreating stuck Certificates.
+type CertificateReissuanceConfig struct {
+	// RetryInterval is the minimum time to wait after a Certificate failure
+	// before deleting it and recreating a fresh one. This prevents excessive
+	// requests to the ACME provider when the underlying issue persists (e.g.
+	// DNS not pointed to the Gateway).
+	//
+	// Defaults to 5m via GetRetryInterval().
+	RetryInterval metav1.Duration `json:"retryInterval,omitempty"`
+
+	// MaxRetries is the maximum number of times the gateway controller will
+	// fast-track re-issuance of a failed Certificate before falling back to
+	// cert-manager's built-in exponential backoff.
+	//
+	// +default=3
+	MaxRetries int `json:"maxRetries,omitempty"`
 }
 
 // HasDefaultListenerTLSSecret returns true when a shared TLS certificate
@@ -738,6 +767,24 @@ func (c *GatewayConfig) IsEPPEmissionEnabled() bool {
 	return *c.EPPEmissionEnabled
 }
 
+// GetRetryInterval returns the configured retry interval for certificate
+// re-issuance, defaulting to 5 minutes.
+func (c *CertificateReissuanceConfig) GetRetryInterval() time.Duration {
+	if c.RetryInterval.Duration > 0 {
+		return c.RetryInterval.Duration
+	}
+	return 5 * time.Minute
+}
+
+// GetMaxRetries returns the configured maximum number of fast-track
+// re-issuance attempts, defaulting to 3.
+func (c *CertificateReissuanceConfig) GetMaxRetries() int {
+	if c.MaxRetries > 0 {
+		return c.MaxRetries
+	}
+	return 3
+}
+
 func (c *GatewayConfig) GatewayDNSAddress(gateway *gatewayv1.Gateway) string {
 	seed := string(gateway.UID)
 	suffix := fmt.Sprintf(".%s", c.TargetDomain)

internal/controller/gateway_controller.go

@@ -7,6 +7,7 @@ import (
 	"crypto/tls"
 	"fmt"
 	"slices"
+	"strconv"
 	"strings"
 	"time"
 
@@ -59,6 +60,7 @@ const certificateIssuerTLSOption = "gateway.networking.datumapis.com/certificate
 // listenerTLSOptions. It means "use whichever real issuer the user's other
 // TLS listener on this gateway specified" — see resolveAutoIssuer.
 const autoIssuerSentinel = "auto"
+const annotationReissuanceCount = "networking.datumapis.com/reissuance-count"
 const KindGateway = "Gateway"
 const KindHTTPRoute = "HTTPRoute"
 const KindService = "Service"
@@ -806,6 +808,7 @@ func (r *GatewayReconciler) ensureListenerCertificates(
 	wildcardSuffix := "." + r.Config.Gateway.TargetDomain
 
 	desiredCerts := make(map[string]bool)
+	gatewayNeedsUpdate := false
 
 	// Wildcard-covered listeners are covered by the shared TLS secret only when
 	// one is configured (DefaultListenerTLSSecretName). Otherwise they need a
@@ -913,6 +916,13 @@ func (r *GatewayReconciler) ensureListenerCertificates(
 		var opResult string
 		var err error
 		if isNew {
+			reissuanceCount := getReissuanceCount(downstreamGateway, certName)
+			if reissuanceCount > 0 {
+				if cert.Annotations == nil {
+					cert.Annotations = make(map[string]string)
+				}
+				cert.Annotations[annotationReissuanceCount] = fmt.Sprintf("%d", reissuanceCount)
+			}
 			cert.Spec = desiredSpec
 			err = downstreamClient.Create(ctx, cert)
 			opResult = "created"
@@ -928,6 +938,27 @@ func (r *GatewayReconciler) ensureListenerCertificates(
 		if opResult != "" {
 			logger.Info("Certificate reconciled", "certificate", certName, "operation", opResult)
 		}
+
+		// Check if an existing Certificate is stuck in a failed state and
+		// should be deleted so we can recreate it fresh on the next reconcile.
+		if !isNew {
+			requeueAfter, gwChanged := r.reissueFailedCertificate(ctx, cert, certName, downstreamGateway, downstreamClient)
+			if gwChanged {
+				gatewayNeedsUpdate = true
+			}
+			if requeueAfter > 0 && (result.RequeueAfter == 0 || requeueAfter < result.RequeueAfter) {
+				result.RequeueAfter = requeueAfter
+			}
+		}
+	}
+
+	// Persist reissuance tracking annotations on the downstream Gateway if
+	// any Certificates were deleted for re-issuance.
+	if gatewayNeedsUpdate {
+		if err := downstreamClient.Update(ctx, downstreamGateway); err != nil {
+			result.Err = fmt.Errorf("failed to update downstream gateway reissuance annotations: %w", err)
+			return result
+		}
 	}
 
 	// Clean up Certificate resources for listeners that no longer need them.
@@ -968,6 +999,113 @@ func (r *GatewayReconciler) ensureListenerCertificates(
 	return result
 }
 
+// reissueFailedCertificate checks whether a Certificate is stuck in a failed
+// state and should be deleted so the gateway controller recreates it fresh on
+// the next reconcile (bypassing cert-manager's exponential backoff).
+//
+// The reissuance count is tracked as an annotation on the downstream Gateway
+// (keyed by certificate name) so it survives Certificate deletion.
+//
+// Returns:
+//   - requeueAfter: non-zero if the caller should requeue after this duration
+//   - gatewayChanged: true if the downstream gateway annotations were modified
+//     and the caller must persist the change
+func (r *GatewayReconciler) reissueFailedCertificate(
+	ctx context.Context,
+	cert *cmv1.Certificate,
+	certName string,
+	downstreamGateway *gatewayv1.Gateway,
+	downstreamClient client.Client,
+) (requeueAfter time.Duration, gatewayChanged bool) {
+	logger := log.FromContext(ctx)
+	reissuanceCfg := &r.Config.Gateway.CertificateReissuance
+
+	if cert.Status.LastFailureTime == nil {
+		if clearReissuanceCount(downstreamGateway, certName) {
+			logger.V(1).Info("cleared reissuance count for healthy Certificate", "certificate", certName)
+			gatewayChanged = true
+		}
+		return 0, gatewayChanged
+	}
+
+	retryCount := getReissuanceCount(downstreamGateway, certName)
+	maxRetries := reissuanceCfg.GetMaxRetries()
+	if retryCount >= maxRetries {
+		logger.V(1).Info("Certificate has exhausted fast-track re-issuance attempts, deferring to cert-manager backoff",
+			"certificate", certName,
+			"reissuanceCount", retryCount,
+			"maxRetries", maxRetries,
+		)
+		return 0, false
+	}
+
+	retryInterval := reissuanceCfg.GetRetryInterval()
+	sinceFailure := time.Since(cert.Status.LastFailureTime.Time)
+	if sinceFailure < retryInterval {
+		remaining := retryInterval - sinceFailure
+		logger.V(1).Info("Certificate failed but retry interval has not elapsed",
+			"certificate", certName,
+			"sinceFailure", sinceFailure,
+			"retryInterval", retryInterval,
+			"requeueAfter", remaining,
+		)
+		return remaining, false
+	}
+
+	setReissuanceCount(downstreamGateway, certName, retryCount+1)
+
+	logger.Info("deleting failed Certificate for re-issuance",
+		"certificate", certName,
+		"lastFailureTime", cert.Status.LastFailureTime.Time,
+		"reissuanceCount", retryCount+1,
+		"maxRetries", maxRetries,
+	)
+
+	if err := downstreamClient.Delete(ctx, cert); err != nil {
+		if !apierrors.IsNotFound(err) {
+			logger.Error(err, "failed to delete Certificate for re-issuance", "certificate", certName)
+		}
+		return 0, true
+	}
+
+	return 0, true
+}
+
+// reissuanceAnnotationKey returns the gateway annotation key used to track
+// reissuance count for a given certificate name.
+func reissuanceAnnotationKey(certName string) string {
+	return annotationReissuanceCount + "/" + certName
+}
+
+func getReissuanceCount(gw *gatewayv1.Gateway, certName string) int {
+	if gw.Annotations == nil {
+		return 0
+	}
+	count, err := strconv.Atoi(gw.Annotations[reissuanceAnnotationKey(certName)])
+	if err != nil {
+		return 0
+	}
+	return count
+}
+
+func setReissuanceCount(gw *gatewayv1.Gateway, certName string, count int) {
+	if gw.Annotations == nil {
+		gw.Annotations = make(map[string]string)
+	}
+	gw.Annotations[reissuanceAnnotationKey(certName)] = fmt.Sprintf("%d", count)
+}
+
+// clearReissuanceCount removes the reissuance tracking annotation for a
+// certificate that has recovered. Returns true if the annotation was present.
+func clearReissuanceCount(gw *gatewayv1.Gateway, certName string) bool {
+	key := reissuanceAnnotationKey(certName)
+	if _, ok := gw.Annotations[key]; ok {
+		delete(gw.Annotations, key)
+		return true
+	}
+	return false
+}
+
 func (r *GatewayReconciler) reconcileGatewayStatus(
 	ctx context.Context,
 	upstreamClient client.Client,