diff --git a/api/v1alpha1/envoygateway_types.go b/api/v1alpha1/envoygateway_types.go index 4853a35e0f..1c7e2f0df1 100644 --- a/api/v1alpha1/envoygateway_types.go +++ b/api/v1alpha1/envoygateway_types.go @@ -219,6 +219,23 @@ type XDSServer struct { // // +optional MaxConnectionAgeGrace *gwapiv1.Duration `json:"maxConnectionAgeGrace,omitempty"` + + // MaxReceiveMessageSize defines the maximum size of a single xDS message that the xDS gRPC + // server will accept from an Envoy proxy. + // + // Envoy's requests grow with the number of resources it holds: on every stream (re)connect, + // the first delta xDS request for each resource type echoes back the name and version of + // every resource the proxy currently has. At a large enough scale this exceeds the 4MiB + // default, and the stream fails immediately with "received message larger than max", leaving + // the proxy stuck on its last known-good configuration. + // + // Note this limit applies only to what Envoy Gateway receives; the configuration it sends to + // Envoy is not bounded by it. + // + // If unspecified, defaults to 32MiB. + // + // +optional + MaxReceiveMessageSize *resource.Quantity `json:"maxReceiveMessageSize,omitempty"` } // LeaderElection defines the desired leader election settings. diff --git a/api/v1alpha1/validation/envoygateway_validate.go b/api/v1alpha1/validation/envoygateway_validate.go index f822ed0231..f617e2d429 100644 --- a/api/v1alpha1/validation/envoygateway_validate.go +++ b/api/v1alpha1/validation/envoygateway_validate.go @@ -334,6 +334,13 @@ func validateEnvoyGatewayXDSServer(xdsServer *egv1a1.XDSServer) error { } } + if xdsServer.MaxReceiveMessageSize != nil { + v, ok := xdsServer.MaxReceiveMessageSize.AsInt64() + if !ok || v <= 0 { + return fmt.Errorf("xdsServer.maxReceiveMessageSize must be greater than zero") + } + } + return nil } diff --git a/api/v1alpha1/validation/envoygateway_validate_test.go b/api/v1alpha1/validation/envoygateway_validate_test.go index 7f62bee411..300b8d757e 100644 --- a/api/v1alpha1/validation/envoygateway_validate_test.go +++ b/api/v1alpha1/validation/envoygateway_validate_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" @@ -1112,6 +1113,18 @@ func TestValidateEnvoyGatewayXDSServer(t *testing.T) { x := &egv1a1.XDSServer{MaxConnectionAgeGrace: &age} require.Error(t, validateEnvoyGatewayXDSServer(x)) }) + + t.Run("valid maxReceiveMessageSize", func(t *testing.T) { + size := resource.MustParse("100Mi") + x := &egv1a1.XDSServer{MaxReceiveMessageSize: &size} + require.NoError(t, validateEnvoyGatewayXDSServer(x)) + }) + + t.Run("invalid zero maxReceiveMessageSize", func(t *testing.T) { + size := resource.MustParse("0") + x := &egv1a1.XDSServer{MaxReceiveMessageSize: &size} + require.Error(t, validateEnvoyGatewayXDSServer(x)) + }) } func TestDefaultEnvoyGatewayLoggingLevel(t *testing.T) { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index e079b476a8..9c32ace0fa 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -8815,6 +8815,11 @@ func (in *XDSServer) DeepCopyInto(out *XDSServer) { *out = new(v1.Duration) **out = **in } + if in.MaxReceiveMessageSize != nil { + in, out := &in.MaxReceiveMessageSize, &out.MaxReceiveMessageSize + x := (*in).DeepCopy() + *out = &x + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new XDSServer. diff --git a/internal/xds/runner/runner.go b/internal/xds/runner/runner.go index d497879cfa..7864adfa54 100644 --- a/internal/xds/runner/runner.go +++ b/internal/xds/runner/runner.go @@ -172,10 +172,17 @@ func (r *Runner) Start(ctx context.Context) error { PermitWithoutStream: true, } + // Default to 32MiB + maxReceiveMessageSize := int64(32 * 1024 * 1024) + if r.EnvoyGateway.XDSServer != nil && r.EnvoyGateway.XDSServer.MaxReceiveMessageSize != nil { + maxReceiveMessageSize, _ = r.EnvoyGateway.XDSServer.MaxReceiveMessageSize.AsInt64() + } baseKeepaliveOptions := []grpc.ServerOption{ grpc.KeepaliveEnforcementPolicy(enforcementPolicy), grpc.KeepaliveParams(keepaliveParams), + grpc.MaxRecvMsgSize(int(maxReceiveMessageSize)), } + r.Logger.Info("configured gRPC max receive message size", "maxReceiveMessageSize", maxReceiveMessageSize) grpcOpts := append([]grpc.ServerOption{}, baseKeepaliveOptions...) grpcOpts = append(grpcOpts, grpc.Creds(credentials.NewTLS(tlsConfig))) diff --git a/internal/xds/runner/runner_test.go b/internal/xds/runner/runner_test.go index 77279c4b38..40b675d8b7 100644 --- a/internal/xds/runner/runner_test.go +++ b/internal/xds/runner/runner_test.go @@ -26,6 +26,7 @@ import ( "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" @@ -339,6 +340,36 @@ func TestRunner(t *testing.T) { }, time.Second*5, time.Millisecond*50) } +func TestRunner_withMaxReceiveMessageSize(t *testing.T) { + caFile, certFile, keyFile, cleanup := setupTLSCerts(t) + defer cleanup() + + xdsIR := new(message.XdsIR) + pResource := new(message.ProviderResources) + cfg, err := config.New(os.Stdout, os.Stderr) + require.NoError(t, err) + + size := resource.MustParse("100Mi") + cfg.EnvoyGateway.XDSServer = &egv1a1.XDSServer{ + MaxReceiveMessageSize: &size, + } + + r := New(&Config{ + Server: *cfg, + ProviderResources: pResource, + XdsIR: xdsIR, + TLSCertPath: certFile, + TLSKeyPath: keyFile, + TLSCaPath: caFile, + }) + + ctx, cancel := context.WithCancel(newTestTraceContext()) + defer cancel() + + err = r.Start(ctx) + require.NoError(t, err) +} + func TestRunner_withExtensionManager_FailOpen(t *testing.T) { // Setup TLS certificates caFile, certFile, keyFile, cleanup := setupTLSCerts(t) diff --git a/release-notes/current/new_features/9564-xds-grpc-max-recv-msg-size.md b/release-notes/current/new_features/9564-xds-grpc-max-recv-msg-size.md new file mode 100644 index 0000000000..e66fc2f2fe --- /dev/null +++ b/release-notes/current/new_features/9564-xds-grpc-max-recv-msg-size.md @@ -0,0 +1,2 @@ +Added `xdsServer.maxReceiveMessageSize` to the EnvoyGateway API and raised the xDS gRPC server's default receive limit from 4MiB to 32MiB. At large resource counts, an Envoy proxy's delta xDS request on stream reconnect can exceed 4MiB, which previously broke the stream with "received message larger than max" errors and left the proxy on its last known-good configuration. + diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 24ce723004..ad331bbbdc 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -6603,6 +6603,7 @@ _Appears in:_ | --- | --- | --- | --- | --- | | `maxConnectionAge` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ | false | | MaxConnectionAge is the maximum age of an active connection before Envoy Gateway will initiate a graceful close.
If unspecified, Envoy Gateway randomly selects a value between 10h and 12h to stagger reconnects across replicas. | | `maxConnectionAgeGrace` | _[Duration](https://gateway-api.sigs.k8s.io/reference/api-spec/1.5/spec/#duration)_ | false | | MaxConnectionAgeGrace is the grace period granted after reaching MaxConnectionAge before the connection is forcibly closed.
The default grace period is 2m. | +| `maxReceiveMessageSize` | _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#quantity-resource-api)_ | false | | MaxReceiveMessageSize defines the maximum size of a single xDS message that the xDS gRPC
server will accept from an Envoy proxy.
Envoy's requests grow with the number of resources it holds: on every stream (re)connect,
the first delta xDS request for each resource type echoes back the name and version of
every resource the proxy currently has. At a large enough scale this exceeds the 4MiB
default, and the stream fails immediately with "received message larger than max", leaving
the proxy stuck on its last known-good configuration.
Note this limit applies only to what Envoy Gateway receives; the configuration it sends to
Envoy is not bounded by it.
If unspecified, defaults to 32MiB. | #### XDSTranslatorHook