fix: make the TLSRoute watch optional too

TLSRoute graduated to the Gateway API standard channel in v1.5, so it is
subject to the same problem as ListenerSet and GRPCRoute: a cluster may
run a bundle that predates the graduation, or a managed offering may ship
only a curated subset of the standard channel, and the unconditional
watch then fails the manager's cache sync and crash-loops Envoy Gateway
at startup.

Guard the TLSRoute watch, its indexers, the reconcile-time processing and
the TLSRoute predicate List behind a CRD existence check, matching what
the previous commit does for ListenerSet and GRPCRoute. Also correct the
rationale comment: TCPRoute and UDPRoute are standard channel as of
Gateway API v1.6, they are not experimental-only.

Signed-off-by: Huabing (Robin) Zhao <zhaohuabing@gmail.com>

Author: zhaohuabing

internal/provider/kubernetes/controller.go

@@ -91,6 +91,7 @@ type gatewayAPIReconciler struct {
 	serviceImportCRDExists bool
 	spCRDExists            bool
 	tcpRouteCRDExists      bool
+	tlsRouteCRDExists      bool
 	udpRouteCRDExists      bool
 
 	clusterTrustBundleExits bool
@@ -1900,9 +1901,11 @@ func (r *gatewayAPIReconciler) processGateways(ctx context.Context, managedGC *g
 
 		// Route Processing
 
-		// Get TLSRoute objects and check if it exists.
-		if err := r.processTLSRoutes(ctx, gtwNamespacedName, resourceMap, resourceTree); err != nil {
-			return err
+		if r.tlsRouteCRDExists {
+			// Get TLSRoute objects and check if it exists.
+			if err := r.processTLSRoutes(ctx, gtwNamespacedName, resourceMap, resourceTree); err != nil {
+				return err
+			}
 		}
 
 		// Get HTTPRoute objects and check if it exists.
@@ -2316,14 +2319,17 @@ func (r *gatewayAPIReconciler) watchResources(ctx context.Context, mgr manager.M
 		return err
 	}
 
-	// Some managed Kubernetes offerings install a curated subset of the Gateway API standard channel
-	// CRDs and don't allow users to add the missing ones: for example, GKE's managed gateway-api-crds
-	// addon ships Gateway API v1.5 without ListenerSet and GRPCRoute, because the GKE Gateway
-	// controller doesn't implement them. Keep the ListenerSet and GRPCRoute watches optional so that
-	// Envoy Gateway starts on those clusters instead of crash-looping: the absent kind can't be used
-	// there anyway, so its absence should disable the feature, not take down the controller.
-	// TODO: remove these checks once ListenerSet and GRPCRoute are included in the Gateway API CRD
-	// bundles shipped by the major managed Kubernetes offerings.
+	// The watches for ListenerSet, GRPCRoute, TLSRoute, TCPRoute and UDPRoute are all optional. These
+	// kinds are in the Gateway API standard channel today (ListenerSet and TLSRoute graduated in
+	// v1.5, TCPRoute and UDPRoute in v1.6), but they are still not guaranteed to be installed:
+	//   - The cluster may run an older Gateway API bundle, or a standard channel bundle from before
+	//     the kind graduated, so the kind is simply not there.
+	//   - Some managed Kubernetes offerings install only a curated subset of the standard channel and
+	//     don't let users add the missing CRDs. For example, GKE's managed gateway-api-crds addon
+	//     ships Gateway API v1.5 without ListenerSet and GRPCRoute, because the GKE Gateway
+	//     controller doesn't implement them.
+	// Envoy Gateway must start on those clusters instead of crash-looping: an absent kind can't be
+	// used there anyway, so its absence should disable the feature, not take down the controller.
 	r.listenerSetCRDExists, err = checkCRD(resource.KindListenerSet, gwapiv1.GroupVersion.String())
 	if err != nil {
 		return err
@@ -2372,7 +2378,6 @@ func (r *gatewayAPIReconciler) watchResources(ctx context.Context, mgr manager.M
 		return err
 	}
 
-	// See the comment on the ListenerSet watch above for why this watch is optional.
 	r.grpcRouteCRDExists, err = checkCRD(resource.KindGRPCRoute, gwapiv1.GroupVersion.String())
 	if err != nil {
 		return err
@@ -2399,24 +2404,31 @@ func (r *gatewayAPIReconciler) watchResources(ctx context.Context, mgr manager.M
 			return err
 		}
 	}
-
-	// Watch TLSRoute CRUDs and process affected Gateways.
-	tlsrPredicates := commonPredicates[*gwapiv1.TLSRoute]()
-	if r.namespaceLabel != nil {
-		tlsrPredicates = append(tlsrPredicates, predicate.NewTypedPredicateFuncs(func(route *gwapiv1.TLSRoute) bool {
-			return r.hasMatchingNamespaceLabels(route)
-		}))
-	}
-	if err := c.Watch(
-		source.Kind(mgr.GetCache(), &gwapiv1.TLSRoute{},
-			handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, route *gwapiv1.TLSRoute) []reconcile.Request {
-				return r.enqueueClass(ctx, route)
-			}),
-			tlsrPredicates...)); err != nil {
+	r.tlsRouteCRDExists, err = checkCRD(resource.KindTLSRoute, gwapiv1.GroupVersion.String())
+	if err != nil {
 		return err
 	}
-	if err := addTLSRouteIndexers(ctx, mgr); err != nil {
-		return err
+	if !r.tlsRouteCRDExists {
+		r.log.Info("TLSRoute CRD not found, skipping TLSRoute watch")
+	} else {
+		// Watch TLSRoute CRUDs and process affected Gateways.
+		tlsrPredicates := commonPredicates[*gwapiv1.TLSRoute]()
+		if r.namespaceLabel != nil {
+			tlsrPredicates = append(tlsrPredicates, predicate.NewTypedPredicateFuncs(func(route *gwapiv1.TLSRoute) bool {
+				return r.hasMatchingNamespaceLabels(route)
+			}))
+		}
+		if err := c.Watch(
+			source.Kind(mgr.GetCache(), &gwapiv1.TLSRoute{},
+				handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, route *gwapiv1.TLSRoute) []reconcile.Request {
+					return r.enqueueClass(ctx, route)
+				}),
+				tlsrPredicates...)); err != nil {
+			return err
+		}
+		if err := addTLSRouteIndexers(ctx, mgr); err != nil {
+			return err
+		}
 	}
 
 	r.udpRouteCRDExists, err = checkCRD(resource.KindUDPRoute, gwapiv1.GroupVersion.String())

internal/provider/kubernetes/controller_offline.go

@@ -113,6 +113,7 @@ func NewOfflineGatewayAPIController(
 		serviceImportCRDExists: true,
 		spCRDExists:            true,
 		tcpRouteCRDExists:      true,
+		tlsRouteCRDExists:      true,
 		udpRouteCRDExists:      true,
 		backendCRDExists:       true,
 	}

internal/provider/kubernetes/predicates.go

@@ -575,15 +575,17 @@ func (r *gatewayAPIReconciler) isRouteReferencingBackend(nsName *types.Namespace
 		}
 	}
 
-	tlsRouteList := &gwapiv1.TLSRouteList{}
-	if err := r.client.List(ctx, tlsRouteList, &client.ListOptions{
-		FieldSelector: fields.OneTermEqualSelector(backendTLSRouteIndex, nsName.String()),
-	}); err != nil && !kerrors.IsNotFound(err) {
-		r.log.Error(err, "failed to find associated TLSRoutes")
-		return false
-	}
-	if len(tlsRouteList.Items) > 0 {
-		return true
+	if r.tlsRouteCRDExists {
+		tlsRouteList := &gwapiv1.TLSRouteList{}
+		if err := r.client.List(ctx, tlsRouteList, &client.ListOptions{
+			FieldSelector: fields.OneTermEqualSelector(backendTLSRouteIndex, nsName.String()),
+		}); err != nil && !kerrors.IsNotFound(err) {
+			r.log.Error(err, "failed to find associated TLSRoutes")
+			return false
+		}
+		if len(tlsRouteList.Items) > 0 {
+			return true
+		}
 	}
 
 	if r.tcpRouteCRDExists {

internal/provider/kubernetes/predicates_test.go

@@ -1155,9 +1155,11 @@ func TestValidateServiceForReconcile(t *testing.T) {
 
 	testCases := []struct {
 		name string
-		// grpcRouteCRDAbsent simulates a cluster whose Gateway API CRD bundle doesn't
-		// include GRPCRoute, e.g. GKE's managed gateway-api-crds addon.
+		// grpcRouteCRDAbsent and tlsRouteCRDAbsent simulate a cluster whose Gateway API
+		// CRD bundle doesn't include GRPCRoute or TLSRoute, e.g. GKE's managed
+		// gateway-api-crds addon.
 		grpcRouteCRDAbsent bool
+		tlsRouteCRDAbsent  bool
 		configs            []client.Object
 		service            client.Object
 		expect             bool
@@ -1267,6 +1269,18 @@ func TestValidateServiceForReconcile(t *testing.T) {
 			service: test.GetService(types.NamespacedName{Name: "service"}, nil, nil),
 			expect:  true,
 		},
+		{
+			name:              "tls route service routes exist but TLSRoute CRD is absent",
+			tlsRouteCRDAbsent: true,
+			configs: []client.Object{
+				test.GetGatewayClass("test-gc", egv1a1.GatewayControllerName, nil),
+				sampleGateway,
+				test.GetTLSRoute(types.NamespacedName{Name: "tlsroute-test"}, "scheduled-status-test",
+					types.NamespacedName{Name: "service"}, 443),
+			},
+			service: test.GetService(types.NamespacedName{Name: "service"}, nil, nil),
+			expect:  false,
+		},
 		{
 			name: "udp route service routes exist",
 			configs: []client.Object{
@@ -1511,6 +1525,7 @@ func TestValidateServiceForReconcile(t *testing.T) {
 
 	for _, tc := range testCases {
 		r.grpcRouteCRDExists = !tc.grpcRouteCRDAbsent
+		r.tlsRouteCRDExists = !tc.tlsRouteCRDAbsent
 		r.client = fakeclient.NewClientBuilder().
 			WithScheme(envoygateway.GetScheme()).
 			WithObjects(tc.configs...).

release-notes/current/bug_fixes/9583-optional-listenerset-grpcroute-crd-watches.md

@@ -0,0 +1 @@
+Fixed Envoy Gateway crash-looping at startup on clusters whose Gateway API CRD bundle omits ListenerSet, GRPCRoute or TLSRoute, such as GKE's managed gateway-api-crds addon, by making those watches conditional on the CRD being present.