Skip to content

Commit aec9ffd

Browse files
zhaohuabingzhaohuabing
authored
fix: make ListenerSet and GRPCRoute watches optional (#9583)
* fix: make ListenerSet and GRPCRoute watches optional (#8991) The Kubernetes provider watches, indexes and Lists ListenerSet and GRPCRoute unconditionally. When either CRD is absent, the manager's cache sync fails and Envoy Gateway crash-loops at startup with: error watching resources: no matches for kind "ListenerSet" in version "gateway.networking.k8s.io/v1" Both kinds are in the Gateway API standard channel, but some managed Kubernetes offerings install only a curated subset of it and don't let users add the missing CRDs. GKE's managed gateway-api-crds addon ships Gateway API v1.5 without listenersets and grpcroutes, because the GKE Gateway controller doesn't implement them; installing them out-of-band means a second owner for addon-managed CRDs. Guard both watches, their indexers, the reconcile-time processing and the GRPCRoute predicate Lists behind a CRD existence check, restoring the pre-#8365 behavior for these two kinds only. TLSRoute and BackendTLSPolicy stay unconditional. When the CRDs are present, behavior is unchanged; when one is absent, the controller starts and simply doesn't serve that kind instead of taking down the whole controller. The offline (file) provider keeps assuming all CRDs are present. Existence is probed once at startup, so a CRD installed later is not watched until the controller restarts, the same trade-off already accepted for EnvoyProxy, TCPRoute and UDPRoute. Signed-off-by: Huabing (Robin) Zhao <zhaohuabing@gmail.com> * fix: make the TLSRoute watch optional too TLSRoute graduated to the Gateway API standard channel in v1.5, so it is subject to the same problem as ListenerSet and GRPCRoute: a cluster may run a bundle that predates the graduation, or a managed offering may ship only a curated subset of the standard channel, and the unconditional watch then fails the manager's cache sync and crash-loops Envoy Gateway at startup. Guard the TLSRoute watch, its indexers, the reconcile-time processing and the TLSRoute predicate List behind a CRD existence check, matching what the previous commit does for ListenerSet and GRPCRoute. Also correct the rationale comment: TCPRoute and UDPRoute are standard channel as of Gateway API v1.6, they are not experimental-only. Signed-off-by: Huabing (Robin) Zhao <zhaohuabing@gmail.com> --------- Signed-off-by: Huabing (Robin) Zhao <zhaohuabing@gmail.com>
1 parent ddf2fc4 commit aec9ffd

5 files changed

Lines changed: 186 additions & 84 deletions

File tree

‎internal/provider/kubernetes/controller.go‎

Lines changed: 101 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -85,10 +85,13 @@ type gatewayAPIReconciler struct {
8585
eepCRDExists bool
8686
epCRDExists bool
8787
eppCRDExists bool
88+
grpcRouteCRDExists bool
8889
hrfCRDExists bool
90+
listenerSetCRDExists bool
8991
serviceImportCRDExists bool
9092
spCRDExists bool
9193
tcpRouteCRDExists bool
94+
tlsRouteCRDExists bool
9295
udpRouteCRDExists bool
9396

9497
clusterTrustBundleExits bool
@@ -1887,28 +1890,34 @@ func (r *gatewayAPIReconciler) processGateways(ctx context.Context, managedGC *g
18871890
gtwNamespacedName := utils.NamespacedName(gtw).String()
18881891

18891892
// ListenerSet Processing (must be done before route processing)
1890-
if err := r.processListenerSets(ctx, gtwNamespacedName, resourceMap, resourceTree); err != nil {
1891-
if isTransientError(err) {
1892-
return err
1893+
if r.listenerSetCRDExists {
1894+
if err := r.processListenerSets(ctx, gtwNamespacedName, resourceMap, resourceTree); err != nil {
1895+
if isTransientError(err) {
1896+
return err
1897+
}
1898+
r.log.Error(err, "failed to process ListenerSets for gateway", "namespace", gtw.Namespace, "name", gtw.Name)
18931899
}
1894-
r.log.Error(err, "failed to process ListenerSets for gateway", "namespace", gtw.Namespace, "name", gtw.Name)
18951900
}
18961901

18971902
// Route Processing
18981903

1899-
// Get TLSRoute objects and check if it exists.
1900-
if err := r.processTLSRoutes(ctx, gtwNamespacedName, resourceMap, resourceTree); err != nil {
1901-
return err
1904+
if r.tlsRouteCRDExists {
1905+
// Get TLSRoute objects and check if it exists.
1906+
if err := r.processTLSRoutes(ctx, gtwNamespacedName, resourceMap, resourceTree); err != nil {
1907+
return err
1908+
}
19021909
}
19031910

19041911
// Get HTTPRoute objects and check if it exists.
19051912
if err := r.processHTTPRoutes(ctx, gtwNamespacedName, resourceMap, resourceTree); err != nil {
19061913
return err
19071914
}
19081915

1909-
// Get GRPCRoute objects and check if it exists.
1910-
if err := r.processGRPCRoutes(ctx, gtwNamespacedName, resourceMap, resourceTree); err != nil {
1911-
return err
1916+
if r.grpcRouteCRDExists {
1917+
// Get GRPCRoute objects and check if it exists.
1918+
if err := r.processGRPCRoutes(ctx, gtwNamespacedName, resourceMap, resourceTree); err != nil {
1919+
return err
1920+
}
19121921
}
19131922

19141923
if r.tcpRouteCRDExists {
@@ -2310,25 +2319,44 @@ func (r *gatewayAPIReconciler) watchResources(ctx context.Context, mgr manager.M
23102319
return err
23112320
}
23122321

2313-
xlsPredicates := []predicate.TypedPredicate[*gwapiv1.ListenerSet]{
2314-
predicate.TypedGenerationChangedPredicate[*gwapiv1.ListenerSet]{},
2315-
}
2316-
if r.namespaceLabel != nil {
2317-
xlsPredicates = append(xlsPredicates, predicate.NewTypedPredicateFuncs(func(obj *gwapiv1.ListenerSet) bool {
2318-
return r.hasMatchingNamespaceLabels(obj)
2319-
}))
2320-
}
2321-
2322-
if err := c.Watch(
2323-
source.Kind(mgr.GetCache(), &gwapiv1.ListenerSet{},
2324-
handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, obj *gwapiv1.ListenerSet) []reconcile.Request {
2325-
return r.enqueueClass(ctx, obj)
2326-
}),
2327-
xlsPredicates...)); err != nil {
2322+
// The watches for ListenerSet, GRPCRoute, TLSRoute, TCPRoute and UDPRoute are all optional. These
2323+
// kinds are in the Gateway API standard channel today (ListenerSet and TLSRoute graduated in
2324+
// v1.5, TCPRoute and UDPRoute in v1.6), but they are still not guaranteed to be installed:
2325+
// - The cluster may run an older Gateway API bundle, or a standard channel bundle from before
2326+
// the kind graduated, so the kind is simply not there.
2327+
// - Some managed Kubernetes offerings install only a curated subset of the standard channel and
2328+
// don't let users add the missing CRDs. For example, GKE's managed gateway-api-crds addon
2329+
// ships Gateway API v1.5 without ListenerSet and GRPCRoute, because the GKE Gateway
2330+
// controller doesn't implement them.
2331+
// Envoy Gateway must start on those clusters instead of crash-looping: an absent kind can't be
2332+
// used there anyway, so its absence should disable the feature, not take down the controller.
2333+
r.listenerSetCRDExists, err = checkCRD(resource.KindListenerSet, gwapiv1.GroupVersion.String())
2334+
if err != nil {
23282335
return err
23292336
}
2330-
if err := addListenerSetIndexers(ctx, mgr); err != nil {
2331-
return err
2337+
if !r.listenerSetCRDExists {
2338+
r.log.Info("ListenerSet CRD not found, skipping ListenerSet watch")
2339+
} else {
2340+
xlsPredicates := []predicate.TypedPredicate[*gwapiv1.ListenerSet]{
2341+
predicate.TypedGenerationChangedPredicate[*gwapiv1.ListenerSet]{},
2342+
}
2343+
if r.namespaceLabel != nil {
2344+
xlsPredicates = append(xlsPredicates, predicate.NewTypedPredicateFuncs(func(obj *gwapiv1.ListenerSet) bool {
2345+
return r.hasMatchingNamespaceLabels(obj)
2346+
}))
2347+
}
2348+
2349+
if err := c.Watch(
2350+
source.Kind(mgr.GetCache(), &gwapiv1.ListenerSet{},
2351+
handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, obj *gwapiv1.ListenerSet) []reconcile.Request {
2352+
return r.enqueueClass(ctx, obj)
2353+
}),
2354+
xlsPredicates...)); err != nil {
2355+
return err
2356+
}
2357+
if err := addListenerSetIndexers(ctx, mgr); err != nil {
2358+
return err
2359+
}
23322360
}
23332361

23342362
// Watch HTTPRoute CRUDs and process affected Gateways.
@@ -2350,42 +2378,57 @@ func (r *gatewayAPIReconciler) watchResources(ctx context.Context, mgr manager.M
23502378
return err
23512379
}
23522380

2353-
// Watch GRPCRoute CRUDs and process affected Gateways.
2354-
grpcrPredicates := commonPredicates[*gwapiv1.GRPCRoute]()
2355-
if r.namespaceLabel != nil {
2356-
grpcrPredicates = append(grpcrPredicates, predicate.NewTypedPredicateFuncs(func(grpc *gwapiv1.GRPCRoute) bool {
2357-
return r.hasMatchingNamespaceLabels(grpc)
2358-
}))
2359-
}
2360-
if err := c.Watch(
2361-
source.Kind(mgr.GetCache(), &gwapiv1.GRPCRoute{},
2362-
handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, route *gwapiv1.GRPCRoute) []reconcile.Request {
2363-
return r.enqueueClass(ctx, route)
2364-
}),
2365-
grpcrPredicates...)); err != nil {
2366-
return err
2367-
}
2368-
if err := addGRPCRouteIndexers(ctx, mgr); err != nil {
2381+
r.grpcRouteCRDExists, err = checkCRD(resource.KindGRPCRoute, gwapiv1.GroupVersion.String())
2382+
if err != nil {
23692383
return err
23702384
}
2371-
2372-
// Watch TLSRoute CRUDs and process affected Gateways.
2373-
tlsrPredicates := commonPredicates[*gwapiv1.TLSRoute]()
2374-
if r.namespaceLabel != nil {
2375-
tlsrPredicates = append(tlsrPredicates, predicate.NewTypedPredicateFuncs(func(route *gwapiv1.TLSRoute) bool {
2376-
return r.hasMatchingNamespaceLabels(route)
2377-
}))
2385+
if !r.grpcRouteCRDExists {
2386+
r.log.Info("GRPCRoute CRD not found, skipping GRPCRoute watch")
2387+
} else {
2388+
// Watch GRPCRoute CRUDs and process affected Gateways.
2389+
grpcrPredicates := commonPredicates[*gwapiv1.GRPCRoute]()
2390+
if r.namespaceLabel != nil {
2391+
grpcrPredicates = append(grpcrPredicates, predicate.NewTypedPredicateFuncs(func(grpc *gwapiv1.GRPCRoute) bool {
2392+
return r.hasMatchingNamespaceLabels(grpc)
2393+
}))
2394+
}
2395+
if err := c.Watch(
2396+
source.Kind(mgr.GetCache(), &gwapiv1.GRPCRoute{},
2397+
handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, route *gwapiv1.GRPCRoute) []reconcile.Request {
2398+
return r.enqueueClass(ctx, route)
2399+
}),
2400+
grpcrPredicates...)); err != nil {
2401+
return err
2402+
}
2403+
if err := addGRPCRouteIndexers(ctx, mgr); err != nil {
2404+
return err
2405+
}
23782406
}
2379-
if err := c.Watch(
2380-
source.Kind(mgr.GetCache(), &gwapiv1.TLSRoute{},
2381-
handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, route *gwapiv1.TLSRoute) []reconcile.Request {
2382-
return r.enqueueClass(ctx, route)
2383-
}),
2384-
tlsrPredicates...)); err != nil {
2407+
r.tlsRouteCRDExists, err = checkCRD(resource.KindTLSRoute, gwapiv1.GroupVersion.String())
2408+
if err != nil {
23852409
return err
23862410
}
2387-
if err := addTLSRouteIndexers(ctx, mgr); err != nil {
2388-
return err
2411+
if !r.tlsRouteCRDExists {
2412+
r.log.Info("TLSRoute CRD not found, skipping TLSRoute watch")
2413+
} else {
2414+
// Watch TLSRoute CRUDs and process affected Gateways.
2415+
tlsrPredicates := commonPredicates[*gwapiv1.TLSRoute]()
2416+
if r.namespaceLabel != nil {
2417+
tlsrPredicates = append(tlsrPredicates, predicate.NewTypedPredicateFuncs(func(route *gwapiv1.TLSRoute) bool {
2418+
return r.hasMatchingNamespaceLabels(route)
2419+
}))
2420+
}
2421+
if err := c.Watch(
2422+
source.Kind(mgr.GetCache(), &gwapiv1.TLSRoute{},
2423+
handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, route *gwapiv1.TLSRoute) []reconcile.Request {
2424+
return r.enqueueClass(ctx, route)
2425+
}),
2426+
tlsrPredicates...)); err != nil {
2427+
return err
2428+
}
2429+
if err := addTLSRouteIndexers(ctx, mgr); err != nil {
2430+
return err
2431+
}
23892432
}
23902433

23912434
r.udpRouteCRDExists, err = checkCRD(resource.KindUDPRoute, gwapiv1.GroupVersion.String())

‎internal/provider/kubernetes/controller_offline.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,10 +107,13 @@ func NewOfflineGatewayAPIController(
107107
eepCRDExists: true,
108108
epCRDExists: true,
109109
eppCRDExists: true,
110+
grpcRouteCRDExists: true,
110111
hrfCRDExists: true,
112+
listenerSetCRDExists: true,
111113
serviceImportCRDExists: true,
112114
spCRDExists: true,
113115
tcpRouteCRDExists: true,
116+
tlsRouteCRDExists: true,
114117
udpRouteCRDExists: true,
115118
backendCRDExists: true,
116119
}

‎internal/provider/kubernetes/predicates.go‎

Lines changed: 26 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -562,26 +562,30 @@ func (r *gatewayAPIReconciler) isRouteReferencingBackend(nsName *types.Namespace
562562
return true
563563
}
564564

565-
grpcRouteList := &gwapiv1.GRPCRouteList{}
566-
if err := r.client.List(ctx, grpcRouteList, &client.ListOptions{
567-
FieldSelector: fields.OneTermEqualSelector(backendGRPCRouteIndex, nsName.String()),
568-
}); err != nil && !kerrors.IsNotFound(err) {
569-
r.log.Error(err, "failed to find associated GRPCRoutes")
570-
return false
571-
}
572-
if len(grpcRouteList.Items) > 0 {
573-
return true
565+
if r.grpcRouteCRDExists {
566+
grpcRouteList := &gwapiv1.GRPCRouteList{}
567+
if err := r.client.List(ctx, grpcRouteList, &client.ListOptions{
568+
FieldSelector: fields.OneTermEqualSelector(backendGRPCRouteIndex, nsName.String()),
569+
}); err != nil && !kerrors.IsNotFound(err) {
570+
r.log.Error(err, "failed to find associated GRPCRoutes")
571+
return false
572+
}
573+
if len(grpcRouteList.Items) > 0 {
574+
return true
575+
}
574576
}
575577

576-
tlsRouteList := &gwapiv1.TLSRouteList{}
577-
if err := r.client.List(ctx, tlsRouteList, &client.ListOptions{
578-
FieldSelector: fields.OneTermEqualSelector(backendTLSRouteIndex, nsName.String()),
579-
}); err != nil && !kerrors.IsNotFound(err) {
580-
r.log.Error(err, "failed to find associated TLSRoutes")
581-
return false
582-
}
583-
if len(tlsRouteList.Items) > 0 {
584-
return true
578+
if r.tlsRouteCRDExists {
579+
tlsRouteList := &gwapiv1.TLSRouteList{}
580+
if err := r.client.List(ctx, tlsRouteList, &client.ListOptions{
581+
FieldSelector: fields.OneTermEqualSelector(backendTLSRouteIndex, nsName.String()),
582+
}); err != nil && !kerrors.IsNotFound(err) {
583+
r.log.Error(err, "failed to find associated TLSRoutes")
584+
return false
585+
}
586+
if len(tlsRouteList.Items) > 0 {
587+
return true
588+
}
585589
}
586590

587591
if r.tcpRouteCRDExists {
@@ -1007,6 +1011,10 @@ func (r *gatewayAPIReconciler) isRouteReferencingHTTPRouteFilter(nsName *types.N
10071011
return true
10081012
}
10091013

1014+
if !r.grpcRouteCRDExists {
1015+
return false
1016+
}
1017+
10101018
grpcRouteList := &gwapiv1.GRPCRouteList{}
10111019
if err := r.client.List(ctx, grpcRouteList, &client.ListOptions{
10121020
FieldSelector: fields.OneTermEqualSelector(httpRouteFilterGRPCRouteIndex, nsName.String()),

‎internal/provider/kubernetes/predicates_test.go‎

Lines changed: 55 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1154,10 +1154,15 @@ func TestValidateServiceForReconcile(t *testing.T) {
11541154
sampleServiceBackendRef := test.GetServiceBackendRef(types.NamespacedName{Name: "service"}, 80)
11551155

11561156
testCases := []struct {
1157-
name string
1158-
configs []client.Object
1159-
service client.Object
1160-
expect bool
1157+
name string
1158+
// grpcRouteCRDAbsent and tlsRouteCRDAbsent simulate a cluster whose Gateway API
1159+
// CRD bundle doesn't include GRPCRoute or TLSRoute, e.g. GKE's managed
1160+
// gateway-api-crds addon.
1161+
grpcRouteCRDAbsent bool
1162+
tlsRouteCRDAbsent bool
1163+
configs []client.Object
1164+
service client.Object
1165+
expect bool
11611166
}{
11621167
{
11631168
name: "gateway service but deployment or daemonset does not exist",
@@ -1242,6 +1247,17 @@ func TestValidateServiceForReconcile(t *testing.T) {
12421247
service: test.GetService(types.NamespacedName{Name: "service"}, nil, nil),
12431248
expect: true,
12441249
},
1250+
{
1251+
name: "grpc route service routes exist but GRPCRoute CRD is absent",
1252+
grpcRouteCRDAbsent: true,
1253+
configs: []client.Object{
1254+
test.GetGatewayClass("test-gc", egv1a1.GatewayControllerName, nil),
1255+
sampleGateway,
1256+
test.GetGRPCRoute(types.NamespacedName{Name: "grpcroute-test"}, "scheduled-status-test", types.NamespacedName{Name: "service"}, 80),
1257+
},
1258+
service: test.GetService(types.NamespacedName{Name: "service"}, nil, nil),
1259+
expect: false,
1260+
},
12451261
{
12461262
name: "tls route service routes exist",
12471263
configs: []client.Object{
@@ -1253,6 +1269,18 @@ func TestValidateServiceForReconcile(t *testing.T) {
12531269
service: test.GetService(types.NamespacedName{Name: "service"}, nil, nil),
12541270
expect: true,
12551271
},
1272+
{
1273+
name: "tls route service routes exist but TLSRoute CRD is absent",
1274+
tlsRouteCRDAbsent: true,
1275+
configs: []client.Object{
1276+
test.GetGatewayClass("test-gc", egv1a1.GatewayControllerName, nil),
1277+
sampleGateway,
1278+
test.GetTLSRoute(types.NamespacedName{Name: "tlsroute-test"}, "scheduled-status-test",
1279+
types.NamespacedName{Name: "service"}, 443),
1280+
},
1281+
service: test.GetService(types.NamespacedName{Name: "service"}, nil, nil),
1282+
expect: false,
1283+
},
12561284
{
12571285
name: "udp route service routes exist",
12581286
configs: []client.Object{
@@ -1496,6 +1524,8 @@ func TestValidateServiceForReconcile(t *testing.T) {
14961524
}
14971525

14981526
for _, tc := range testCases {
1527+
r.grpcRouteCRDExists = !tc.grpcRouteCRDAbsent
1528+
r.tlsRouteCRDExists = !tc.tlsRouteCRDAbsent
14991529
r.client = fakeclient.NewClientBuilder().
15001530
WithScheme(envoygateway.GetScheme()).
15011531
WithObjects(tc.configs...).
@@ -1846,10 +1876,13 @@ func TestValidateHTTPRouteFilerForReconcile(t *testing.T) {
18461876
sampleHTTPRouteFilter := test.GetHTTPRouteFilter(types.NamespacedName{Name: "httproutefilter"})
18471877

18481878
testCases := []struct {
1849-
name string
1850-
configs []client.Object
1851-
httpRouteFilter client.Object
1852-
expect bool
1879+
name string
1880+
// grpcRouteCRDAbsent simulates a cluster whose Gateway API CRD bundle doesn't
1881+
// include GRPCRoute, e.g. GKE's managed gateway-api-crds addon.
1882+
grpcRouteCRDAbsent bool
1883+
configs []client.Object
1884+
httpRouteFilter client.Object
1885+
expect bool
18531886
}{
18541887
{
18551888
name: "httproutefilter but not referenced by route",
@@ -1886,6 +1919,19 @@ func TestValidateHTTPRouteFilerForReconcile(t *testing.T) {
18861919
httpRouteFilter: sampleHTTPRouteFilter,
18871920
expect: true,
18881921
},
1922+
{
1923+
name: "httproutefilter referenced by grpcroute but GRPCRoute CRD is absent",
1924+
grpcRouteCRDAbsent: true,
1925+
configs: []client.Object{
1926+
sampleGWC,
1927+
sampleGateway,
1928+
sampleService,
1929+
sampleHTTPRouteFilter,
1930+
test.GetGRPCRouteWithHTTPRouteFilter(types.NamespacedName{Name: "grpcroute-test"}, "scheduled-status-test", types.NamespacedName{Name: "service"}, 80, "httproutefilter"),
1931+
},
1932+
httpRouteFilter: sampleHTTPRouteFilter,
1933+
expect: false,
1934+
},
18891935
}
18901936

18911937
// Create the reconciler.
@@ -1897,6 +1943,7 @@ func TestValidateHTTPRouteFilerForReconcile(t *testing.T) {
18971943
}
18981944

18991945
for _, tc := range testCases {
1946+
r.grpcRouteCRDExists = !tc.grpcRouteCRDAbsent
19001947
r.client = fakeclient.NewClientBuilder().
19011948
WithScheme(envoygateway.GetScheme()).
19021949
WithObjects(tc.configs...).

‎release-notes/current/bug_fixes/9583-optional-standard-channel-crd-watches.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fixed Envoy Gateway crash-looping at startup on clusters whose Gateway API CRD bundle omits ListenerSet, GRPCRoute or TLSRoute, such as GKE's managed gateway-api-crds addon, by making those watches conditional on the CRD being present.

0 commit comments

Comments
 (0)