feat(xds): raise xDS gRPC server default receive limit and make it configurable (#9564)

linmosko · guydc · web-flow · commit 8e3217494067 · 2026-07-28T09:39:46.000-05:00
* feat(xds): add configurable max receive message size for xDS gRPC server

Signed-off-by: Lin Moskovitch &lt;lin.moskovitch@sap.com&gt;

* update release notes

Signed-off-by: Lin Moskovitch &lt;lin.moskovitch@sap.com&gt;

* test(xds): add coverage for MaxRecvMsgSize runner option

Signed-off-by: Lin Moskovitch &lt;lin.moskovitch@sap.com&gt;

* lint

Signed-off-by: Lin Moskovitch &lt;lin.moskovitch@sap.com&gt;

* refactor: rename MaxRecvMsgSize to MaxReceiveMessageSize for consistency

Signed-off-by: Lin Moskovitch &lt;lin.moskovitch@sap.com&gt;

* docs: improve MaxReceiveMessageSize comment and release note

Signed-off-by: Lin Moskovitch &lt;lin.moskovitch@sap.com&gt;

* feat(xds): set default max receive message size to 32MiB

Signed-off-by: Lin Moskovitch &lt;lin.moskovitch@sap.com&gt;

---------

Signed-off-by: Lin Moskovitch &lt;lin.moskovitch@sap.com&gt;
Co-authored-by: Guy Daich &lt;guy.daich@sap.com&gt;
diff --git a/api/v1alpha1/envoygateway_types.go b/api/v1alpha1/envoygateway_types.go
@@ -219,6 +219,23 @@ type XDSServer struct {
 	//
 	// +optional
 	MaxConnectionAgeGrace *gwapiv1.Duration `json:"maxConnectionAgeGrace,omitempty"`
+
+	// MaxReceiveMessageSize defines the maximum size of a single xDS message that the xDS gRPC
+	// server will accept from an Envoy proxy.
+	//
+	// Envoy's requests grow with the number of resources it holds: on every stream (re)connect,
+	// the first delta xDS request for each resource type echoes back the name and version of
+	// every resource the proxy currently has. At a large enough scale this exceeds the 4MiB
+	// default, and the stream fails immediately with "received message larger than max", leaving
+	// the proxy stuck on its last known-good configuration.
+	//
+	// Note this limit applies only to what Envoy Gateway receives; the configuration it sends to
+	// Envoy is not bounded by it.
+	//
+	// If unspecified, defaults to 32MiB.
+	//
+	// +optional
+	MaxReceiveMessageSize *resource.Quantity `json:"maxReceiveMessageSize,omitempty"`
 }
 
 // LeaderElection defines the desired leader election settings.
diff --git a/api/v1alpha1/validation/envoygateway_validate.go b/api/v1alpha1/validation/envoygateway_validate.go
@@ -334,6 +334,13 @@ func validateEnvoyGatewayXDSServer(xdsServer *egv1a1.XDSServer) error {
 		}
 	}
 
+	if xdsServer.MaxReceiveMessageSize != nil {
+		v, ok := xdsServer.MaxReceiveMessageSize.AsInt64()
+		if !ok || v <= 0 {
+			return fmt.Errorf("xdsServer.maxReceiveMessageSize must be greater than zero")
+		}
+	}
+
 	return nil
 }
 
diff --git a/api/v1alpha1/validation/envoygateway_validate_test.go b/api/v1alpha1/validation/envoygateway_validate_test.go
@@ -11,6 +11,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
 
@@ -1112,6 +1113,18 @@ func TestValidateEnvoyGatewayXDSServer(t *testing.T) {
 		x := &egv1a1.XDSServer{MaxConnectionAgeGrace: &age}
 		require.Error(t, validateEnvoyGatewayXDSServer(x))
 	})
+
+	t.Run("valid maxReceiveMessageSize", func(t *testing.T) {
+		size := resource.MustParse("100Mi")
+		x := &egv1a1.XDSServer{MaxReceiveMessageSize: &size}
+		require.NoError(t, validateEnvoyGatewayXDSServer(x))
+	})
+
+	t.Run("invalid zero maxReceiveMessageSize", func(t *testing.T) {
+		size := resource.MustParse("0")
+		x := &egv1a1.XDSServer{MaxReceiveMessageSize: &size}
+		require.Error(t, validateEnvoyGatewayXDSServer(x))
+	})
 }
 
 func TestDefaultEnvoyGatewayLoggingLevel(t *testing.T) {
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/internal/xds/runner/runner.go b/internal/xds/runner/runner.go
@@ -172,10 +172,17 @@ func (r *Runner) Start(ctx context.Context) error {
 		PermitWithoutStream: true,
 	}
 
+	// Default to 32MiB
+	maxReceiveMessageSize := int64(32 * 1024 * 1024)
+	if r.EnvoyGateway.XDSServer != nil && r.EnvoyGateway.XDSServer.MaxReceiveMessageSize != nil {
+		maxReceiveMessageSize, _ = r.EnvoyGateway.XDSServer.MaxReceiveMessageSize.AsInt64()
+	}
 	baseKeepaliveOptions := []grpc.ServerOption{
 		grpc.KeepaliveEnforcementPolicy(enforcementPolicy),
 		grpc.KeepaliveParams(keepaliveParams),
+		grpc.MaxRecvMsgSize(int(maxReceiveMessageSize)),
 	}
+	r.Logger.Info("configured gRPC max receive message size", "maxReceiveMessageSize", maxReceiveMessageSize)
 
 	grpcOpts := append([]grpc.ServerOption{}, baseKeepaliveOptions...)
 	grpcOpts = append(grpcOpts, grpc.Creds(credentials.NewTLS(tlsConfig)))
diff --git a/internal/xds/runner/runner_test.go b/internal/xds/runner/runner_test.go
@@ -26,6 +26,7 @@ import (
 	"go.opentelemetry.io/otel/trace"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
+	"k8s.io/apimachinery/pkg/api/resource"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 
 	egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
@@ -339,6 +340,36 @@ func TestRunner(t *testing.T) {
 	}, time.Second*5, time.Millisecond*50)
 }
 
+func TestRunner_withMaxReceiveMessageSize(t *testing.T) {
+	caFile, certFile, keyFile, cleanup := setupTLSCerts(t)
+	defer cleanup()
+
+	xdsIR := new(message.XdsIR)
+	pResource := new(message.ProviderResources)
+	cfg, err := config.New(os.Stdout, os.Stderr)
+	require.NoError(t, err)
+
+	size := resource.MustParse("100Mi")
+	cfg.EnvoyGateway.XDSServer = &egv1a1.XDSServer{
+		MaxReceiveMessageSize: &size,
+	}
+
+	r := New(&Config{
+		Server:            *cfg,
+		ProviderResources: pResource,
+		XdsIR:             xdsIR,
+		TLSCertPath:       certFile,
+		TLSKeyPath:        keyFile,
+		TLSCaPath:         caFile,
+	})
+
+	ctx, cancel := context.WithCancel(newTestTraceContext())
+	defer cancel()
+
+	err = r.Start(ctx)
+	require.NoError(t, err)
+}
+
 func TestRunner_withExtensionManager_FailOpen(t *testing.T) {
 	// Setup TLS certificates
 	caFile, certFile, keyFile, cleanup := setupTLSCerts(t)
diff --git a/release-notes/current/new_features/9564-xds-grpc-max-recv-msg-size.md b/release-notes/current/new_features/9564-xds-grpc-max-recv-msg-size.md
@@ -0,0 +1,2 @@
+Added `xdsServer.maxReceiveMessageSize` to the EnvoyGateway API and raised the xDS gRPC server's default receive limit from 4MiB to 32MiB. At large resource counts, an Envoy proxy's delta xDS request on stream reconnect can exceed 4MiB, which previously broke the stream with "received message larger than max" errors and left the proxy on its last known-good configuration.
+
diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md
@@ -6603,6 +6603,7 @@ _Appears in:_
 | ---   | ---  | ---      | ---     | ---         |
 | `maxConnectionAge` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ |  false  |  | MaxConnectionAge is the maximum age of an active connection before Envoy Gateway will initiate a graceful close.<br />If unspecified, Envoy Gateway randomly selects a value between 10h and 12h to stagger reconnects across replicas. |
 | `maxConnectionAgeGrace` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ |  false  |  | MaxConnectionAgeGrace is the grace period granted after reaching MaxConnectionAge before the connection is forcibly closed.<br />The default grace period is 2m. |
+| `maxReceiveMessageSize` | _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#quantity-resource-api)_ |  false  |  | MaxReceiveMessageSize defines the maximum size of a single xDS message that the xDS gRPC<br />server will accept from an Envoy proxy.<br />Envoy's requests grow with the number of resources it holds: on every stream (re)connect,<br />the first delta xDS request for each resource type echoes back the name and version of<br />every resource the proxy currently has. At a large enough scale this exceeds the 4MiB<br />default, and the stream fails immediately with "received message larger than max", leaving<br />the proxy stuck on its last known-good configuration.<br />Note this limit applies only to what Envoy Gateway receives; the configuration it sends to<br />Envoy is not bounded by it.<br />If unspecified, defaults to 32MiB. |
 
 
 #### XDSTranslatorHook

Original file line number	Diff line number	Diff line change
`@@ -334,6 +334,13 @@ func validateEnvoyGatewayXDSServer(xdsServer *egv1a1.XDSServer) error {`
`334`	`334`	`}`
`335`	`335`	`}`
`336`	`336`
	`337`	`+ if xdsServer.MaxReceiveMessageSize != nil {`
	`338`	`+ v, ok := xdsServer.MaxReceiveMessageSize.AsInt64()`
	`339`	`+ if !ok \|\| v <= 0 {`
	`340`	`+ return fmt.Errorf("xdsServer.maxReceiveMessageSize must be greater than zero")`
	`341`	`+ }`
	`342`	`+ }`
	`343`	`+`
`337`	`344`	`return nil`
`338`	`345`	`}`
`339`	`346`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	+Added `xdsServer.maxReceiveMessageSize` to the EnvoyGateway API and raised the xDS gRPC server's default receive limit from 4MiB to 32MiB. At large resource counts, an Envoy proxy's delta xDS request on stream reconnect can exceed 4MiB, which previously broke the stream with "received message larger than max" errors and left the proxy on its last known-good configuration.
	`2`	`+`