From dd4e592688dfa984ba3cc52063637b7880fc5721 Mon Sep 17 00:00:00 2001 From: Lin Moskovitch Date: Thu, 23 Jul 2026 00:03:48 +0300 Subject: [PATCH 1/7] feat(xds): add configurable max receive message size for xDS gRPC server Signed-off-by: Lin Moskovitch --- api/v1alpha1/envoygateway_types.go | 8 ++++++++ api/v1alpha1/validation/envoygateway_validate.go | 7 +++++++ .../validation/envoygateway_validate_test.go | 13 +++++++++++++ api/v1alpha1/zz_generated.deepcopy.go | 5 +++++ internal/xds/runner/runner.go | 6 ++++++ .../XXXXX-xds-grpc-max-recv-msg-size.md | 1 + site/content/en/latest/api/extension_types.md | 1 + 7 files changed, 41 insertions(+) create mode 100644 release-notes/current/new_features/XXXXX-xds-grpc-max-recv-msg-size.md diff --git a/api/v1alpha1/envoygateway_types.go b/api/v1alpha1/envoygateway_types.go index 280b60a6a1..f315a239ae 100644 --- a/api/v1alpha1/envoygateway_types.go +++ b/api/v1alpha1/envoygateway_types.go @@ -219,6 +219,14 @@ type XDSServer struct { // // +optional MaxConnectionAgeGrace *gwapiv1.Duration `json:"maxConnectionAgeGrace,omitempty"` + + // MaxRecvMsgSize defines the maximum message size in bytes the xDS gRPC server will accept from Envoy proxies. + // Increasing this is necessary at scale when the xDS snapshot exceeds the default 4MiB gRPC receive limit, + // which manifests as "received message larger than max" errors in the xDS stream. + // If unspecified, the default gRPC receive limit applies (4MiB). + // + // +optional + MaxRecvMsgSize *resource.Quantity `json:"maxRecvMsgSize,omitempty"` } // LeaderElection defines the desired leader election settings. diff --git a/api/v1alpha1/validation/envoygateway_validate.go b/api/v1alpha1/validation/envoygateway_validate.go index 513e6fd8b3..79ee28c066 100644 --- a/api/v1alpha1/validation/envoygateway_validate.go +++ b/api/v1alpha1/validation/envoygateway_validate.go @@ -333,6 +333,13 @@ func validateEnvoyGatewayXDSServer(xdsServer *egv1a1.XDSServer) error { } } + if xdsServer.MaxRecvMsgSize != nil { + v, ok := xdsServer.MaxRecvMsgSize.AsInt64() + if !ok || v <= 0 { + return fmt.Errorf("xdsServer.maxRecvMsgSize must be greater than zero") + } + } + return nil } diff --git a/api/v1alpha1/validation/envoygateway_validate_test.go b/api/v1alpha1/validation/envoygateway_validate_test.go index a5a9dd7b0b..c435bbea1c 100644 --- a/api/v1alpha1/validation/envoygateway_validate_test.go +++ b/api/v1alpha1/validation/envoygateway_validate_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/api/resource" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" @@ -1112,6 +1113,18 @@ func TestValidateEnvoyGatewayXDSServer(t *testing.T) { x := &egv1a1.XDSServer{MaxConnectionAgeGrace: &age} require.Error(t, validateEnvoyGatewayXDSServer(x)) }) + + t.Run("valid maxRecvMsgSize", func(t *testing.T) { + size := resource.MustParse("100Mi") + x := &egv1a1.XDSServer{MaxRecvMsgSize: &size} + require.NoError(t, validateEnvoyGatewayXDSServer(x)) + }) + + t.Run("invalid zero maxRecvMsgSize", func(t *testing.T) { + size := resource.MustParse("0") + x := &egv1a1.XDSServer{MaxRecvMsgSize: &size} + require.Error(t, validateEnvoyGatewayXDSServer(x)) + }) } func TestDefaultEnvoyGatewayLoggingLevel(t *testing.T) { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 7245a49145..e00b573343 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -8810,6 +8810,11 @@ func (in *XDSServer) DeepCopyInto(out *XDSServer) { *out = new(v1.Duration) **out = **in } + if in.MaxRecvMsgSize != nil { + in, out := &in.MaxRecvMsgSize, &out.MaxRecvMsgSize + x := (*in).DeepCopy() + *out = &x + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new XDSServer. diff --git a/internal/xds/runner/runner.go b/internal/xds/runner/runner.go index d497879cfa..c6b71b7ada 100644 --- a/internal/xds/runner/runner.go +++ b/internal/xds/runner/runner.go @@ -177,6 +177,12 @@ func (r *Runner) Start(ctx context.Context) error { grpc.KeepaliveParams(keepaliveParams), } + if r.EnvoyGateway.XDSServer != nil && r.EnvoyGateway.XDSServer.MaxRecvMsgSize != nil { + maxRecvMsgSize, _ := r.EnvoyGateway.XDSServer.MaxRecvMsgSize.AsInt64() + baseKeepaliveOptions = append(baseKeepaliveOptions, grpc.MaxRecvMsgSize(int(maxRecvMsgSize))) + r.Logger.Info("configured gRPC max receive message size", "maxRecvMsgSize", maxRecvMsgSize) + } + grpcOpts := append([]grpc.ServerOption{}, baseKeepaliveOptions...) grpcOpts = append(grpcOpts, grpc.Creds(credentials.NewTLS(tlsConfig))) diff --git a/release-notes/current/new_features/XXXXX-xds-grpc-max-recv-msg-size.md b/release-notes/current/new_features/XXXXX-xds-grpc-max-recv-msg-size.md new file mode 100644 index 0000000000..65f6d8af3b --- /dev/null +++ b/release-notes/current/new_features/XXXXX-xds-grpc-max-recv-msg-size.md @@ -0,0 +1 @@ +Added `xdsServer.maxRecvMsgSize` field to the EnvoyGateway API, allowing operators to increase the xDS gRPC server's maximum receive message size when large xDS snapshots at scale exceed the default 4MiB limit, which previously caused "received message larger than max" stream errors. diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index e5fcea8c43..078a87079b 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -6602,6 +6602,7 @@ _Appears in:_ | --- | --- | --- | --- | --- | | `maxConnectionAge` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ | false | | MaxConnectionAge is the maximum age of an active connection before Envoy Gateway will initiate a graceful close.
If unspecified, Envoy Gateway randomly selects a value between 10h and 12h to stagger reconnects across replicas. | | `maxConnectionAgeGrace` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ | false | | MaxConnectionAgeGrace is the grace period granted after reaching MaxConnectionAge before the connection is forcibly closed.
The default grace period is 2m. | +| `maxRecvMsgSize` | _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#quantity-resource-api)_ | false | | MaxRecvMsgSize defines the maximum message size in bytes the xDS gRPC server will accept from Envoy proxies.
Increasing this is necessary at scale when the xDS snapshot exceeds the default 4MiB gRPC receive limit,
which manifests as "received message larger than max" errors in the xDS stream.
If unspecified, the default gRPC receive limit applies (4MiB). | #### XDSTranslatorHook From ae41a939d35399a2e10d2d6293649aefc523e901 Mon Sep 17 00:00:00 2001 From: Lin Moskovitch Date: Thu, 23 Jul 2026 00:04:52 +0300 Subject: [PATCH 2/7] update release notes Signed-off-by: Lin Moskovitch --- ...pc-max-recv-msg-size.md => 9564-xds-grpc-max-recv-msg-size.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename release-notes/current/new_features/{XXXXX-xds-grpc-max-recv-msg-size.md => 9564-xds-grpc-max-recv-msg-size.md} (100%) diff --git a/release-notes/current/new_features/XXXXX-xds-grpc-max-recv-msg-size.md b/release-notes/current/new_features/9564-xds-grpc-max-recv-msg-size.md similarity index 100% rename from release-notes/current/new_features/XXXXX-xds-grpc-max-recv-msg-size.md rename to release-notes/current/new_features/9564-xds-grpc-max-recv-msg-size.md From 80f80c7b62931c6d168f2df3202676901fbacdcc Mon Sep 17 00:00:00 2001 From: Lin Moskovitch Date: Thu, 23 Jul 2026 13:52:48 +0300 Subject: [PATCH 3/7] test(xds): add coverage for MaxRecvMsgSize runner option Signed-off-by: Lin Moskovitch --- internal/xds/runner/runner_test.go | 31 ++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/internal/xds/runner/runner_test.go b/internal/xds/runner/runner_test.go index 77279c4b38..f3fb84236f 100644 --- a/internal/xds/runner/runner_test.go +++ b/internal/xds/runner/runner_test.go @@ -26,6 +26,7 @@ import ( "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" @@ -339,6 +340,36 @@ func TestRunner(t *testing.T) { }, time.Second*5, time.Millisecond*50) } +func TestRunner_withMaxRecvMsgSize(t *testing.T) { + caFile, certFile, keyFile, cleanup := setupTLSCerts(t) + defer cleanup() + + xdsIR := new(message.XdsIR) + pResource := new(message.ProviderResources) + cfg, err := config.New(os.Stdout, os.Stderr) + require.NoError(t, err) + + size := resource.MustParse("100Mi") + cfg.EnvoyGateway.XDSServer = &egv1a1.XDSServer{ + MaxRecvMsgSize: &size, + } + + r := New(&Config{ + Server: *cfg, + ProviderResources: pResource, + XdsIR: xdsIR, + TLSCertPath: certFile, + TLSKeyPath: keyFile, + TLSCaPath: caFile, + }) + + ctx, cancel := context.WithCancel(newTestTraceContext()) + defer cancel() + + err = r.Start(ctx) + require.NoError(t, err) +} + func TestRunner_withExtensionManager_FailOpen(t *testing.T) { // Setup TLS certificates caFile, certFile, keyFile, cleanup := setupTLSCerts(t) From 1a22b81f8133618dc0507c3d96c1664532598481 Mon Sep 17 00:00:00 2001 From: Lin Moskovitch Date: Thu, 23 Jul 2026 17:39:56 +0300 Subject: [PATCH 4/7] lint Signed-off-by: Lin Moskovitch --- api/v1alpha1/validation/envoygateway_validate_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/v1alpha1/validation/envoygateway_validate_test.go b/api/v1alpha1/validation/envoygateway_validate_test.go index c435bbea1c..606f2ebc44 100644 --- a/api/v1alpha1/validation/envoygateway_validate_test.go +++ b/api/v1alpha1/validation/envoygateway_validate_test.go @@ -11,8 +11,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" From 6fe1095190cbf2d0b327b4ea37c0495d8a1c2f0a Mon Sep 17 00:00:00 2001 From: Lin Moskovitch Date: Fri, 24 Jul 2026 11:19:56 +0300 Subject: [PATCH 5/7] refactor: rename MaxRecvMsgSize to MaxReceiveMessageSize for consistency Signed-off-by: Lin Moskovitch --- api/v1alpha1/envoygateway_types.go | 4 ++-- api/v1alpha1/validation/envoygateway_validate.go | 6 +++--- api/v1alpha1/validation/envoygateway_validate_test.go | 8 ++++---- api/v1alpha1/zz_generated.deepcopy.go | 4 ++-- internal/xds/runner/runner.go | 8 ++++---- internal/xds/runner/runner_test.go | 4 ++-- .../new_features/9564-xds-grpc-max-recv-msg-size.md | 2 +- site/content/en/latest/api/extension_types.md | 2 +- 8 files changed, 19 insertions(+), 19 deletions(-) diff --git a/api/v1alpha1/envoygateway_types.go b/api/v1alpha1/envoygateway_types.go index f315a239ae..7f131d1a3c 100644 --- a/api/v1alpha1/envoygateway_types.go +++ b/api/v1alpha1/envoygateway_types.go @@ -220,13 +220,13 @@ type XDSServer struct { // +optional MaxConnectionAgeGrace *gwapiv1.Duration `json:"maxConnectionAgeGrace,omitempty"` - // MaxRecvMsgSize defines the maximum message size in bytes the xDS gRPC server will accept from Envoy proxies. + // MaxReceiveMessageSize defines the maximum message size in bytes the xDS gRPC server will accept from Envoy proxies. // Increasing this is necessary at scale when the xDS snapshot exceeds the default 4MiB gRPC receive limit, // which manifests as "received message larger than max" errors in the xDS stream. // If unspecified, the default gRPC receive limit applies (4MiB). // // +optional - MaxRecvMsgSize *resource.Quantity `json:"maxRecvMsgSize,omitempty"` + MaxReceiveMessageSize *resource.Quantity `json:"maxReceiveMessageSize,omitempty"` } // LeaderElection defines the desired leader election settings. diff --git a/api/v1alpha1/validation/envoygateway_validate.go b/api/v1alpha1/validation/envoygateway_validate.go index 79ee28c066..6019db5e87 100644 --- a/api/v1alpha1/validation/envoygateway_validate.go +++ b/api/v1alpha1/validation/envoygateway_validate.go @@ -333,10 +333,10 @@ func validateEnvoyGatewayXDSServer(xdsServer *egv1a1.XDSServer) error { } } - if xdsServer.MaxRecvMsgSize != nil { - v, ok := xdsServer.MaxRecvMsgSize.AsInt64() + if xdsServer.MaxReceiveMessageSize != nil { + v, ok := xdsServer.MaxReceiveMessageSize.AsInt64() if !ok || v <= 0 { - return fmt.Errorf("xdsServer.maxRecvMsgSize must be greater than zero") + return fmt.Errorf("xdsServer.maxReceiveMessageSize must be greater than zero") } } diff --git a/api/v1alpha1/validation/envoygateway_validate_test.go b/api/v1alpha1/validation/envoygateway_validate_test.go index 606f2ebc44..4abd8cd48b 100644 --- a/api/v1alpha1/validation/envoygateway_validate_test.go +++ b/api/v1alpha1/validation/envoygateway_validate_test.go @@ -1114,15 +1114,15 @@ func TestValidateEnvoyGatewayXDSServer(t *testing.T) { require.Error(t, validateEnvoyGatewayXDSServer(x)) }) - t.Run("valid maxRecvMsgSize", func(t *testing.T) { + t.Run("valid maxReceiveMessageSize", func(t *testing.T) { size := resource.MustParse("100Mi") - x := &egv1a1.XDSServer{MaxRecvMsgSize: &size} + x := &egv1a1.XDSServer{MaxReceiveMessageSize: &size} require.NoError(t, validateEnvoyGatewayXDSServer(x)) }) - t.Run("invalid zero maxRecvMsgSize", func(t *testing.T) { + t.Run("invalid zero maxReceiveMessageSize", func(t *testing.T) { size := resource.MustParse("0") - x := &egv1a1.XDSServer{MaxRecvMsgSize: &size} + x := &egv1a1.XDSServer{MaxReceiveMessageSize: &size} require.Error(t, validateEnvoyGatewayXDSServer(x)) }) } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index e00b573343..1a91c26461 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -8810,8 +8810,8 @@ func (in *XDSServer) DeepCopyInto(out *XDSServer) { *out = new(v1.Duration) **out = **in } - if in.MaxRecvMsgSize != nil { - in, out := &in.MaxRecvMsgSize, &out.MaxRecvMsgSize + if in.MaxReceiveMessageSize != nil { + in, out := &in.MaxReceiveMessageSize, &out.MaxReceiveMessageSize x := (*in).DeepCopy() *out = &x } diff --git a/internal/xds/runner/runner.go b/internal/xds/runner/runner.go index c6b71b7ada..69fcd705f4 100644 --- a/internal/xds/runner/runner.go +++ b/internal/xds/runner/runner.go @@ -177,10 +177,10 @@ func (r *Runner) Start(ctx context.Context) error { grpc.KeepaliveParams(keepaliveParams), } - if r.EnvoyGateway.XDSServer != nil && r.EnvoyGateway.XDSServer.MaxRecvMsgSize != nil { - maxRecvMsgSize, _ := r.EnvoyGateway.XDSServer.MaxRecvMsgSize.AsInt64() - baseKeepaliveOptions = append(baseKeepaliveOptions, grpc.MaxRecvMsgSize(int(maxRecvMsgSize))) - r.Logger.Info("configured gRPC max receive message size", "maxRecvMsgSize", maxRecvMsgSize) + if r.EnvoyGateway.XDSServer != nil && r.EnvoyGateway.XDSServer.MaxReceiveMessageSize != nil { + maxReceiveMessageSize, _ := r.EnvoyGateway.XDSServer.MaxReceiveMessageSize.AsInt64() + baseKeepaliveOptions = append(baseKeepaliveOptions, grpc.MaxRecvMsgSize(int(maxReceiveMessageSize))) + r.Logger.Info("configured gRPC max receive message size", "maxReceiveMessageSize", maxReceiveMessageSize) } grpcOpts := append([]grpc.ServerOption{}, baseKeepaliveOptions...) diff --git a/internal/xds/runner/runner_test.go b/internal/xds/runner/runner_test.go index f3fb84236f..40b675d8b7 100644 --- a/internal/xds/runner/runner_test.go +++ b/internal/xds/runner/runner_test.go @@ -340,7 +340,7 @@ func TestRunner(t *testing.T) { }, time.Second*5, time.Millisecond*50) } -func TestRunner_withMaxRecvMsgSize(t *testing.T) { +func TestRunner_withMaxReceiveMessageSize(t *testing.T) { caFile, certFile, keyFile, cleanup := setupTLSCerts(t) defer cleanup() @@ -351,7 +351,7 @@ func TestRunner_withMaxRecvMsgSize(t *testing.T) { size := resource.MustParse("100Mi") cfg.EnvoyGateway.XDSServer = &egv1a1.XDSServer{ - MaxRecvMsgSize: &size, + MaxReceiveMessageSize: &size, } r := New(&Config{ diff --git a/release-notes/current/new_features/9564-xds-grpc-max-recv-msg-size.md b/release-notes/current/new_features/9564-xds-grpc-max-recv-msg-size.md index 65f6d8af3b..869c3abc99 100644 --- a/release-notes/current/new_features/9564-xds-grpc-max-recv-msg-size.md +++ b/release-notes/current/new_features/9564-xds-grpc-max-recv-msg-size.md @@ -1 +1 @@ -Added `xdsServer.maxRecvMsgSize` field to the EnvoyGateway API, allowing operators to increase the xDS gRPC server's maximum receive message size when large xDS snapshots at scale exceed the default 4MiB limit, which previously caused "received message larger than max" stream errors. +Added `xdsServer.maxReceiveMessageSize` field to the EnvoyGateway API, allowing operators to increase the xDS gRPC server's maximum receive message size when large xDS snapshots at scale exceed the default 4MiB limit, which previously caused "received message larger than max" stream errors. diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 078a87079b..8f0b8da43f 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -6602,7 +6602,7 @@ _Appears in:_ | --- | --- | --- | --- | --- | | `maxConnectionAge` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ | false | | MaxConnectionAge is the maximum age of an active connection before Envoy Gateway will initiate a graceful close.
If unspecified, Envoy Gateway randomly selects a value between 10h and 12h to stagger reconnects across replicas. | | `maxConnectionAgeGrace` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ | false | | MaxConnectionAgeGrace is the grace period granted after reaching MaxConnectionAge before the connection is forcibly closed.
The default grace period is 2m. | -| `maxRecvMsgSize` | _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#quantity-resource-api)_ | false | | MaxRecvMsgSize defines the maximum message size in bytes the xDS gRPC server will accept from Envoy proxies.
Increasing this is necessary at scale when the xDS snapshot exceeds the default 4MiB gRPC receive limit,
which manifests as "received message larger than max" errors in the xDS stream.
If unspecified, the default gRPC receive limit applies (4MiB). | +| `maxReceiveMessageSize` | _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#quantity-resource-api)_ | false | | MaxReceiveMessageSize defines the maximum message size in bytes the xDS gRPC server will accept from Envoy proxies.
Increasing this is necessary at scale when the xDS snapshot exceeds the default 4MiB gRPC receive limit,
which manifests as "received message larger than max" errors in the xDS stream.
If unspecified, the default gRPC receive limit applies (4MiB). | #### XDSTranslatorHook From ef7fcbe500ba6df08927950cc0f6465559b0d747 Mon Sep 17 00:00:00 2001 From: Lin Moskovitch Date: Sun, 26 Jul 2026 08:14:57 +0300 Subject: [PATCH 6/7] docs: improve MaxReceiveMessageSize comment and release note Signed-off-by: Lin Moskovitch --- api/v1alpha1/envoygateway_types.go | 17 +++++++++++++---- .../9564-xds-grpc-max-recv-msg-size.md | 2 +- site/content/en/latest/api/extension_types.md | 2 +- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/api/v1alpha1/envoygateway_types.go b/api/v1alpha1/envoygateway_types.go index 7f131d1a3c..2a3da50a38 100644 --- a/api/v1alpha1/envoygateway_types.go +++ b/api/v1alpha1/envoygateway_types.go @@ -220,10 +220,19 @@ type XDSServer struct { // +optional MaxConnectionAgeGrace *gwapiv1.Duration `json:"maxConnectionAgeGrace,omitempty"` - // MaxReceiveMessageSize defines the maximum message size in bytes the xDS gRPC server will accept from Envoy proxies. - // Increasing this is necessary at scale when the xDS snapshot exceeds the default 4MiB gRPC receive limit, - // which manifests as "received message larger than max" errors in the xDS stream. - // If unspecified, the default gRPC receive limit applies (4MiB). + // MaxReceiveMessageSize defines the maximum size of a single xDS message that the xDS gRPC + // server will accept from an Envoy proxy. + // + // Envoy's requests grow with the number of resources it holds: on every stream (re)connect, + // the first delta xDS request for each resource type echoes back the name and version of + // every resource the proxy currently has. At a large enough scale this exceeds the 4MiB + // default, and the stream fails immediately with "received message larger than max", leaving + // the proxy stuck on its last known-good configuration. + // + // Note this limit applies only to what Envoy Gateway receives; the configuration it sends to + // Envoy is not bounded by it. + // + // If unspecified, the gRPC default of 4MiB applies. // // +optional MaxReceiveMessageSize *resource.Quantity `json:"maxReceiveMessageSize,omitempty"` diff --git a/release-notes/current/new_features/9564-xds-grpc-max-recv-msg-size.md b/release-notes/current/new_features/9564-xds-grpc-max-recv-msg-size.md index 869c3abc99..46bf05b79f 100644 --- a/release-notes/current/new_features/9564-xds-grpc-max-recv-msg-size.md +++ b/release-notes/current/new_features/9564-xds-grpc-max-recv-msg-size.md @@ -1 +1 @@ -Added `xdsServer.maxReceiveMessageSize` field to the EnvoyGateway API, allowing operators to increase the xDS gRPC server's maximum receive message size when large xDS snapshots at scale exceed the default 4MiB limit, which previously caused "received message larger than max" stream errors. +Added `xdsServer.maxReceiveMessageSize` to the EnvoyGateway API, allowing operators to raise the xDS gRPC server's receive limit. At large resource counts, an Envoy proxy's delta xDS request on stream reconnect can exceed the 4MiB gRPC default, which previously broke the stream with "received message larger than max" errors and left the proxy on its last known-good configuration. diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 8f0b8da43f..73b1d63275 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -6602,7 +6602,7 @@ _Appears in:_ | --- | --- | --- | --- | --- | | `maxConnectionAge` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ | false | | MaxConnectionAge is the maximum age of an active connection before Envoy Gateway will initiate a graceful close.
If unspecified, Envoy Gateway randomly selects a value between 10h and 12h to stagger reconnects across replicas. | | `maxConnectionAgeGrace` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ | false | | MaxConnectionAgeGrace is the grace period granted after reaching MaxConnectionAge before the connection is forcibly closed.
The default grace period is 2m. | -| `maxReceiveMessageSize` | _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#quantity-resource-api)_ | false | | MaxReceiveMessageSize defines the maximum message size in bytes the xDS gRPC server will accept from Envoy proxies.
Increasing this is necessary at scale when the xDS snapshot exceeds the default 4MiB gRPC receive limit,
which manifests as "received message larger than max" errors in the xDS stream.
If unspecified, the default gRPC receive limit applies (4MiB). | +| `maxReceiveMessageSize` | _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#quantity-resource-api)_ | false | | MaxReceiveMessageSize defines the maximum size of a single xDS message that the xDS gRPC
server will accept from an Envoy proxy.
Envoy's requests grow with the number of resources it holds: on every stream (re)connect,
the first delta xDS request for each resource type echoes back the name and version of
every resource the proxy currently has. At a large enough scale this exceeds the 4MiB
default, and the stream fails immediately with "received message larger than max", leaving
the proxy stuck on its last known-good configuration.
Note this limit applies only to what Envoy Gateway receives; the configuration it sends to
Envoy is not bounded by it.
If unspecified, the gRPC default of 4MiB applies. | #### XDSTranslatorHook From 2d6fabf6adfd62296d890f01c3b59c34071534af Mon Sep 17 00:00:00 2001 From: Lin Moskovitch Date: Mon, 27 Jul 2026 13:50:29 +0300 Subject: [PATCH 7/7] feat(xds): set default max receive message size to 32MiB Signed-off-by: Lin Moskovitch --- api/v1alpha1/envoygateway_types.go | 2 +- internal/xds/runner/runner.go | 13 +++++++------ .../new_features/9564-xds-grpc-max-recv-msg-size.md | 3 ++- site/content/en/latest/api/extension_types.md | 2 +- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/api/v1alpha1/envoygateway_types.go b/api/v1alpha1/envoygateway_types.go index 2a3da50a38..a81e022663 100644 --- a/api/v1alpha1/envoygateway_types.go +++ b/api/v1alpha1/envoygateway_types.go @@ -232,7 +232,7 @@ type XDSServer struct { // Note this limit applies only to what Envoy Gateway receives; the configuration it sends to // Envoy is not bounded by it. // - // If unspecified, the gRPC default of 4MiB applies. + // If unspecified, defaults to 32MiB. // // +optional MaxReceiveMessageSize *resource.Quantity `json:"maxReceiveMessageSize,omitempty"` diff --git a/internal/xds/runner/runner.go b/internal/xds/runner/runner.go index 69fcd705f4..7864adfa54 100644 --- a/internal/xds/runner/runner.go +++ b/internal/xds/runner/runner.go @@ -172,16 +172,17 @@ func (r *Runner) Start(ctx context.Context) error { PermitWithoutStream: true, } + // Default to 32MiB + maxReceiveMessageSize := int64(32 * 1024 * 1024) + if r.EnvoyGateway.XDSServer != nil && r.EnvoyGateway.XDSServer.MaxReceiveMessageSize != nil { + maxReceiveMessageSize, _ = r.EnvoyGateway.XDSServer.MaxReceiveMessageSize.AsInt64() + } baseKeepaliveOptions := []grpc.ServerOption{ grpc.KeepaliveEnforcementPolicy(enforcementPolicy), grpc.KeepaliveParams(keepaliveParams), + grpc.MaxRecvMsgSize(int(maxReceiveMessageSize)), } - - if r.EnvoyGateway.XDSServer != nil && r.EnvoyGateway.XDSServer.MaxReceiveMessageSize != nil { - maxReceiveMessageSize, _ := r.EnvoyGateway.XDSServer.MaxReceiveMessageSize.AsInt64() - baseKeepaliveOptions = append(baseKeepaliveOptions, grpc.MaxRecvMsgSize(int(maxReceiveMessageSize))) - r.Logger.Info("configured gRPC max receive message size", "maxReceiveMessageSize", maxReceiveMessageSize) - } + r.Logger.Info("configured gRPC max receive message size", "maxReceiveMessageSize", maxReceiveMessageSize) grpcOpts := append([]grpc.ServerOption{}, baseKeepaliveOptions...) grpcOpts = append(grpcOpts, grpc.Creds(credentials.NewTLS(tlsConfig))) diff --git a/release-notes/current/new_features/9564-xds-grpc-max-recv-msg-size.md b/release-notes/current/new_features/9564-xds-grpc-max-recv-msg-size.md index 46bf05b79f..e66fc2f2fe 100644 --- a/release-notes/current/new_features/9564-xds-grpc-max-recv-msg-size.md +++ b/release-notes/current/new_features/9564-xds-grpc-max-recv-msg-size.md @@ -1 +1,2 @@ -Added `xdsServer.maxReceiveMessageSize` to the EnvoyGateway API, allowing operators to raise the xDS gRPC server's receive limit. At large resource counts, an Envoy proxy's delta xDS request on stream reconnect can exceed the 4MiB gRPC default, which previously broke the stream with "received message larger than max" errors and left the proxy on its last known-good configuration. +Added `xdsServer.maxReceiveMessageSize` to the EnvoyGateway API and raised the xDS gRPC server's default receive limit from 4MiB to 32MiB. At large resource counts, an Envoy proxy's delta xDS request on stream reconnect can exceed 4MiB, which previously broke the stream with "received message larger than max" errors and left the proxy on its last known-good configuration. + diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 73b1d63275..611743cc86 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -6602,7 +6602,7 @@ _Appears in:_ | --- | --- | --- | --- | --- | | `maxConnectionAge` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ | false | | MaxConnectionAge is the maximum age of an active connection before Envoy Gateway will initiate a graceful close.
If unspecified, Envoy Gateway randomly selects a value between 10h and 12h to stagger reconnects across replicas. | | `maxConnectionAgeGrace` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ | false | | MaxConnectionAgeGrace is the grace period granted after reaching MaxConnectionAge before the connection is forcibly closed.
The default grace period is 2m. | -| `maxReceiveMessageSize` | _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#quantity-resource-api)_ | false | | MaxReceiveMessageSize defines the maximum size of a single xDS message that the xDS gRPC
server will accept from an Envoy proxy.
Envoy's requests grow with the number of resources it holds: on every stream (re)connect,
the first delta xDS request for each resource type echoes back the name and version of
every resource the proxy currently has. At a large enough scale this exceeds the 4MiB
default, and the stream fails immediately with "received message larger than max", leaving
the proxy stuck on its last known-good configuration.
Note this limit applies only to what Envoy Gateway receives; the configuration it sends to
Envoy is not bounded by it.
If unspecified, the gRPC default of 4MiB applies. | +| `maxReceiveMessageSize` | _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#quantity-resource-api)_ | false | | MaxReceiveMessageSize defines the maximum size of a single xDS message that the xDS gRPC
server will accept from an Envoy proxy.
Envoy's requests grow with the number of resources it holds: on every stream (re)connect,
the first delta xDS request for each resource type echoes back the name and version of
every resource the proxy currently has. At a large enough scale this exceeds the 4MiB
default, and the stream fails immediately with "received message larger than max", leaving
the proxy stuck on its last known-good configuration.
Note this limit applies only to what Envoy Gateway receives; the configuration it sends to
Envoy is not bounded by it.
If unspecified, defaults to 32MiB. | #### XDSTranslatorHook