diff --git a/internal/xds/translator/cluster.go b/internal/xds/translator/cluster.go index a80ac206d8..a4138d3ddd 100644 --- a/internal/xds/translator/cluster.go +++ b/internal/xds/translator/cluster.go @@ -15,6 +15,7 @@ import ( clusterv3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" endpointv3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + routev3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" commondnsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/common/dns/v3" dnsclusterv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/dns/v3" dfpv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/dynamic_forward_proxy/v3" @@ -1091,13 +1092,22 @@ func buildTypedExtensionProtocolOptions(args *xdsClusterArgs, requiresAutoHTTPCo requiresHTTPFilters := (len(args.settings) > 0 && args.settings[0].Filters != nil && args.settings[0].Filters.CredentialInjection != nil) || args.admissionControl != nil + var clusterHashPolicy []*routev3.RouteAction_HashPolicy + if !args.isRoute && args.loadBalancer != nil { + clusterHashPolicy = buildConsistentHashPolicy(args.loadBalancer.ConsistentHash) + } + requiredHTTPProtocolOptions := args.useClientProtocol || requiresAutoHTTPConfig || - requiresCommonHTTPOptions || requiresHTTP1Options || requiresHTTP2Options || requiresHTTPFilters || requiresAutoSNI || forceHTTP1UpstreamProtocol + requiresCommonHTTPOptions || requiresHTTP1Options || requiresHTTP2Options || requiresHTTPFilters || requiresAutoSNI || + forceHTTP1UpstreamProtocol || len(clusterHashPolicy) > 0 if !requiredHTTPProtocolOptions { return nil, nil, nil } protocolOptions := httpv3.HttpProtocolOptions{} + if len(clusterHashPolicy) > 0 { + protocolOptions.HashPolicy = clusterHashPolicy + } if requiresCommonHTTPOptions { protocolOptions.CommonHttpProtocolOptions = &corev3.HttpProtocolOptions{} if args.timeout != nil && args.timeout.HTTP != nil { diff --git a/internal/xds/translator/cluster_test.go b/internal/xds/translator/cluster_test.go index 4f582ec42c..9be366620d 100644 --- a/internal/xds/translator/cluster_test.go +++ b/internal/xds/translator/cluster_test.go @@ -17,6 +17,7 @@ import ( cswrrv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3" override_hostv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/override_host/v3" wrr_localityv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/wrr_locality/v3" + httpv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3" "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require" "google.golang.org/protobuf/encoding/protojson" @@ -295,6 +296,53 @@ func TestBuildClusterWithBackendUtilization(t *testing.T) { require.Equal(t, "type.googleapis.com/envoy.extensions.load_balancing_policies.client_side_weighted_round_robin.v3.ClientSideWeightedRoundRobin", policy.TypedExtensionConfig.TypedConfig.TypeUrl) } +func TestBuildXdsClusterWithClusterLevelHashPolicy(t *testing.T) { + sourceIP := true + args := &xdsClusterArgs{ + name: "test-cluster-consistent-hash", + endpointType: EndpointTypeStatic, + settings: []*ir.DestinationSetting{{ + Endpoints: []*ir.DestinationEndpoint{{Host: "127.0.0.1", Port: 8080}}, + }}, + loadBalancer: &ir.LoadBalancer{ + ConsistentHash: &ir.ConsistentHash{ + SourceIP: &sourceIP, + }, + }, + } + + result, err := buildXdsCluster(args) + require.NoError(t, err) + + options := &httpv3.HttpProtocolOptions{} + require.NotNil(t, result.cluster.TypedExtensionProtocolOptions) + require.NoError(t, result.cluster.TypedExtensionProtocolOptions[extensionOptionsKey].UnmarshalTo(options)) + require.Len(t, options.HashPolicy, 1) + require.True(t, options.HashPolicy[0].GetConnectionProperties().GetSourceIp()) +} + +func TestBuildXdsRouteClusterWithoutClusterLevelHashPolicy(t *testing.T) { + sourceIP := true + args := &xdsClusterArgs{ + name: "test-route-cluster-consistent-hash", + endpointType: EndpointTypeStatic, + settings: []*ir.DestinationSetting{{ + Endpoints: []*ir.DestinationEndpoint{{Host: "127.0.0.1", Port: 8080}}, + }}, + loadBalancer: &ir.LoadBalancer{ + ConsistentHash: &ir.ConsistentHash{ + SourceIP: &sourceIP, + }, + }, + isRoute: true, + } + + result, err := buildXdsCluster(args) + require.NoError(t, err) + + require.Nil(t, result.cluster.TypedExtensionProtocolOptions) +} + func TestBuildClusterWithBackendUtilizationSlowStart(t *testing.T) { window := 5 * time.Second args := &xdsClusterArgs{ diff --git a/internal/xds/translator/route.go b/internal/xds/translator/route.go index 21eb69951c..cf9b80e5e8 100644 --- a/internal/xds/translator/route.go +++ b/internal/xds/translator/route.go @@ -708,7 +708,13 @@ func buildHashPolicy(httpRoute *ir.HTTPRoute) []*routev3.RouteAction_HashPolicy return nil } - ch := httpRoute.Traffic.LoadBalancer.ConsistentHash + return buildConsistentHashPolicy(httpRoute.Traffic.LoadBalancer.ConsistentHash) +} + +func buildConsistentHashPolicy(ch *ir.ConsistentHash) []*routev3.RouteAction_HashPolicy { + if ch == nil { + return nil + } switch { case ch.Headers != nil: diff --git a/internal/xds/translator/testdata/in/xds-ir/ext-auth-retry.yaml b/internal/xds/translator/testdata/in/xds-ir/ext-auth-retry.yaml index 9b466d0157..4a44b236e1 100644 --- a/internal/xds/translator/testdata/in/xds-ir/ext-auth-retry.yaml +++ b/internal/xds/translator/testdata/in/xds-ir/ext-auth-retry.yaml @@ -29,6 +29,9 @@ http: extAuth: name: securitypolicy/default/policy-for-http-route-1 traffic: + loadBalancer: + consistentHash: + sourceIP: true retry: numRetries: 2 perRetry: @@ -86,6 +89,9 @@ http: extAuth: name: securitypolicy/default/policy-for-http-route-1 traffic: + loadBalancer: + consistentHash: + sourceIP: true retry: numRetries: 2 perRetry: diff --git a/internal/xds/translator/testdata/out/xds-ir/ext-auth-retry.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/ext-auth-retry.clusters.yaml index 5bf2b6b57d..648d9ce187 100644 --- a/internal/xds/translator/testdata/out/xds-ir/ext-auth-retry.clusters.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/ext-auth-retry.clusters.yaml @@ -82,11 +82,9 @@ loadBalancingPolicy: policies: - typedExtensionConfig: - name: envoy.load_balancing_policies.least_request + name: envoy.load_balancing_policies.maglev typedConfig: - '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest - localityLbConfig: - localityWeightedLbConfig: {} + '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.maglev.v3.Maglev name: securitypolicy/default/policy-for-http-route-1/default/grpc-backend perConnectionBufferLimitBytes: 32768 type: EDS @@ -97,6 +95,9 @@ http2ProtocolOptions: initialConnectionWindowSize: 1048576 initialStreamWindowSize: 65536 + hashPolicy: + - connectionProperties: + sourceIp: true - circuitBreakers: thresholds: - maxRetries: 1024 diff --git a/internal/xds/translator/testdata/out/xds-ir/jwt-with-backend-tls-retry.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/jwt-with-backend-tls-retry.clusters.yaml index e9d16815d0..7d92903143 100644 --- a/internal/xds/translator/testdata/out/xds-ir/jwt-with-backend-tls-retry.clusters.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/jwt-with-backend-tls-retry.clusters.yaml @@ -129,6 +129,9 @@ idleTimeout: 16s maxConnectionDuration: 17s maxRequestsPerConnection: 1 + hashPolicy: + - connectionProperties: + sourceIp: true upstreamConnectionOptions: tcpKeepalive: keepaliveInterval: 60 diff --git a/internal/xds/translator/testdata/out/xds-ir/oidc-backend-cluster-provider.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/oidc-backend-cluster-provider.clusters.yaml index 6188447b40..9297866e3b 100644 --- a/internal/xds/translator/testdata/out/xds-ir/oidc-backend-cluster-provider.clusters.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/oidc-backend-cluster-provider.clusters.yaml @@ -129,6 +129,9 @@ idleTimeout: 16s maxConnectionDuration: 17s maxRequestsPerConnection: 1 + hashPolicy: + - connectionProperties: + sourceIp: true upstreamConnectionOptions: tcpKeepalive: keepaliveInterval: 60 diff --git a/internal/xds/translator/testdata/out/xds-ir/oidc-provider-traffic-features.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/oidc-provider-traffic-features.clusters.yaml index 9cb89a04b1..27b556633c 100644 --- a/internal/xds/translator/testdata/out/xds-ir/oidc-provider-traffic-features.clusters.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/oidc-provider-traffic-features.clusters.yaml @@ -107,6 +107,9 @@ maxRequestsPerConnection: 1 explicitHttpConfig: httpProtocolOptions: {} + hashPolicy: + - connectionProperties: + sourceIp: true upstreamConnectionOptions: tcpKeepalive: keepaliveInterval: 60 diff --git a/release-notes/current/bug_fixes/9231-consistenthash-load-balancer-non-route-service-clusters.md b/release-notes/current/bug_fixes/9231-consistenthash-load-balancer-non-route-service-clusters.md new file mode 100644 index 0000000000..ca0be6d7c0 --- /dev/null +++ b/release-notes/current/bug_fixes/9231-consistenthash-load-balancer-non-route-service-clusters.md @@ -0,0 +1 @@ +Fixed `ConsistentHash` load balancer settings on non-route service clusters, such as SecurityPolicy extAuth backends, by emitting cluster-level HTTP hash policies. diff --git a/test/e2e/testdata/ext-auth-grpc-securitypolicy.yaml b/test/e2e/testdata/ext-auth-grpc-securitypolicy.yaml index f7dbc7d06b..9eb26d03c0 100644 --- a/test/e2e/testdata/ext-auth-grpc-securitypolicy.yaml +++ b/test/e2e/testdata/ext-auth-grpc-securitypolicy.yaml @@ -54,6 +54,11 @@ spec: - name: grpc-ext-auth namespace: gateway-conformance-infra port: 9002 + backendSettings: + loadBalancer: + type: ConsistentHash + consistentHash: + type: SourceIP --- apiVersion: gateway.networking.k8s.io/v1 kind: BackendTLSPolicy diff --git a/test/e2e/testdata/ext-auth-service.yaml b/test/e2e/testdata/ext-auth-service.yaml index 0ebce42f38..169801c1b5 100644 --- a/test/e2e/testdata/ext-auth-service.yaml +++ b/test/e2e/testdata/ext-auth-service.yaml @@ -56,7 +56,7 @@ metadata: name: envoy-ext-auth namespace: gateway-conformance-infra spec: - replicas: 1 + replicas: 2 selector: matchLabels: app: envoy-ext-auth