feat: support consistent hash for non-route backends (#9231)

* support consistent hash for non-route backends

Signed-off-by: Huabing (Robin) Zhao <zhaohuabing@gmail.com>

* update e2e tests

Signed-off-by: Huabing (Robin) Zhao <zhaohuabing@gmail.com>

---------

Signed-off-by: Huabing (Robin) Zhao <zhaohuabing@gmail.com>

Author: zhaohuabing

internal/xds/translator/cluster.go

@@ -15,6 +15,7 @@ import (
 	clusterv3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
 	corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	endpointv3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
+	routev3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
 	commondnsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/common/dns/v3"
 	dnsclusterv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/dns/v3"
 	dfpv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/dynamic_forward_proxy/v3"
@@ -1091,13 +1092,22 @@ func buildTypedExtensionProtocolOptions(args *xdsClusterArgs, requiresAutoHTTPCo
 	requiresHTTPFilters := (len(args.settings) > 0 && args.settings[0].Filters != nil && args.settings[0].Filters.CredentialInjection != nil) ||
 		args.admissionControl != nil
 
+	var clusterHashPolicy []*routev3.RouteAction_HashPolicy
+	if !args.isRoute && args.loadBalancer != nil {
+		clusterHashPolicy = buildConsistentHashPolicy(args.loadBalancer.ConsistentHash)
+	}
+
 	requiredHTTPProtocolOptions := args.useClientProtocol || requiresAutoHTTPConfig ||
-		requiresCommonHTTPOptions || requiresHTTP1Options || requiresHTTP2Options || requiresHTTPFilters || requiresAutoSNI || forceHTTP1UpstreamProtocol
+		requiresCommonHTTPOptions || requiresHTTP1Options || requiresHTTP2Options || requiresHTTPFilters || requiresAutoSNI ||
+		forceHTTP1UpstreamProtocol || len(clusterHashPolicy) > 0
 
 	if !requiredHTTPProtocolOptions {
 		return nil, nil, nil
 	}
 	protocolOptions := httpv3.HttpProtocolOptions{}
+	if len(clusterHashPolicy) > 0 {
+		protocolOptions.HashPolicy = clusterHashPolicy
+	}
 	if requiresCommonHTTPOptions {
 		protocolOptions.CommonHttpProtocolOptions = &corev3.HttpProtocolOptions{}
 		if args.timeout != nil && args.timeout.HTTP != nil {

internal/xds/translator/cluster_test.go

@@ -17,6 +17,7 @@ import (
 	cswrrv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3"
 	override_hostv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/override_host/v3"
 	wrr_localityv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/wrr_locality/v3"
+	httpv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3"
 	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/protobuf/encoding/protojson"
@@ -295,6 +296,53 @@ func TestBuildClusterWithBackendUtilization(t *testing.T) {
 	require.Equal(t, "type.googleapis.com/envoy.extensions.load_balancing_policies.client_side_weighted_round_robin.v3.ClientSideWeightedRoundRobin", policy.TypedExtensionConfig.TypedConfig.TypeUrl)
 }
 
+func TestBuildXdsClusterWithClusterLevelHashPolicy(t *testing.T) {
+	sourceIP := true
+	args := &xdsClusterArgs{
+		name:         "test-cluster-consistent-hash",
+		endpointType: EndpointTypeStatic,
+		settings: []*ir.DestinationSetting{{
+			Endpoints: []*ir.DestinationEndpoint{{Host: "127.0.0.1", Port: 8080}},
+		}},
+		loadBalancer: &ir.LoadBalancer{
+			ConsistentHash: &ir.ConsistentHash{
+				SourceIP: &sourceIP,
+			},
+		},
+	}
+
+	result, err := buildXdsCluster(args)
+	require.NoError(t, err)
+
+	options := &httpv3.HttpProtocolOptions{}
+	require.NotNil(t, result.cluster.TypedExtensionProtocolOptions)
+	require.NoError(t, result.cluster.TypedExtensionProtocolOptions[extensionOptionsKey].UnmarshalTo(options))
+	require.Len(t, options.HashPolicy, 1)
+	require.True(t, options.HashPolicy[0].GetConnectionProperties().GetSourceIp())
+}
+
+func TestBuildXdsRouteClusterWithoutClusterLevelHashPolicy(t *testing.T) {
+	sourceIP := true
+	args := &xdsClusterArgs{
+		name:         "test-route-cluster-consistent-hash",
+		endpointType: EndpointTypeStatic,
+		settings: []*ir.DestinationSetting{{
+			Endpoints: []*ir.DestinationEndpoint{{Host: "127.0.0.1", Port: 8080}},
+		}},
+		loadBalancer: &ir.LoadBalancer{
+			ConsistentHash: &ir.ConsistentHash{
+				SourceIP: &sourceIP,
+			},
+		},
+		isRoute: true,
+	}
+
+	result, err := buildXdsCluster(args)
+	require.NoError(t, err)
+
+	require.Nil(t, result.cluster.TypedExtensionProtocolOptions)
+}
+
 func TestBuildClusterWithBackendUtilizationSlowStart(t *testing.T) {
 	window := 5 * time.Second
 	args := &xdsClusterArgs{

internal/xds/translator/route.go

@@ -708,7 +708,13 @@ func buildHashPolicy(httpRoute *ir.HTTPRoute) []*routev3.RouteAction_HashPolicy
 		return nil
 	}
 
-	ch := httpRoute.Traffic.LoadBalancer.ConsistentHash
+	return buildConsistentHashPolicy(httpRoute.Traffic.LoadBalancer.ConsistentHash)
+}
+
+func buildConsistentHashPolicy(ch *ir.ConsistentHash) []*routev3.RouteAction_HashPolicy {
+	if ch == nil {
+		return nil
+	}
 
 	switch {
 	case ch.Headers != nil:

internal/xds/translator/testdata/in/xds-ir/ext-auth-retry.yaml

@@ -29,6 +29,9 @@ http:
           extAuth:
             name: securitypolicy/default/policy-for-http-route-1
             traffic:
+              loadBalancer:
+                consistentHash:
+                  sourceIP: true
               retry:
                 numRetries: 2
                 perRetry:
@@ -86,6 +89,9 @@ http:
           extAuth:
             name: securitypolicy/default/policy-for-http-route-1
             traffic:
+              loadBalancer:
+                consistentHash:
+                  sourceIP: true
               retry:
                 numRetries: 2
                 perRetry:

internal/xds/translator/testdata/out/xds-ir/ext-auth-retry.clusters.yaml

@@ -82,11 +82,9 @@
   loadBalancingPolicy:
     policies:
     - typedExtensionConfig:
-        name: envoy.load_balancing_policies.least_request
+        name: envoy.load_balancing_policies.maglev
         typedConfig:
-          '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest
-          localityLbConfig:
-            localityWeightedLbConfig: {}
+          '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.maglev.v3.Maglev
   name: securitypolicy/default/policy-for-http-route-1/default/grpc-backend
   perConnectionBufferLimitBytes: 32768
   type: EDS
@@ -97,6 +95,9 @@
         http2ProtocolOptions:
           initialConnectionWindowSize: 1048576
           initialStreamWindowSize: 65536
+      hashPolicy:
+      - connectionProperties:
+          sourceIp: true
 - circuitBreakers:
     thresholds:
     - maxRetries: 1024

internal/xds/translator/testdata/out/xds-ir/jwt-with-backend-tls-retry.clusters.yaml

@@ -129,6 +129,9 @@
         idleTimeout: 16s
         maxConnectionDuration: 17s
         maxRequestsPerConnection: 1
+      hashPolicy:
+      - connectionProperties:
+          sourceIp: true
   upstreamConnectionOptions:
     tcpKeepalive:
       keepaliveInterval: 60

internal/xds/translator/testdata/out/xds-ir/oidc-backend-cluster-provider.clusters.yaml

@@ -129,6 +129,9 @@
         idleTimeout: 16s
         maxConnectionDuration: 17s
         maxRequestsPerConnection: 1
+      hashPolicy:
+      - connectionProperties:
+          sourceIp: true
   upstreamConnectionOptions:
     tcpKeepalive:
       keepaliveInterval: 60

internal/xds/translator/testdata/out/xds-ir/oidc-provider-traffic-features.clusters.yaml

@@ -107,6 +107,9 @@
         maxRequestsPerConnection: 1
       explicitHttpConfig:
         httpProtocolOptions: {}
+      hashPolicy:
+      - connectionProperties:
+          sourceIp: true
   upstreamConnectionOptions:
     tcpKeepalive:
       keepaliveInterval: 60

release-notes/current/bug_fixes/9231-consistenthash-load-balancer-non-route-service-clusters.md

@@ -0,0 +1 @@
+Fixed `ConsistentHash` load balancer settings on non-route service clusters, such as SecurityPolicy extAuth backends, by emitting cluster-level HTTP hash policies.

test/e2e/testdata/ext-auth-grpc-securitypolicy.yaml

@@ -54,6 +54,11 @@ spec:
       - name: grpc-ext-auth
         namespace: gateway-conformance-infra
         port: 9002
+      backendSettings:
+        loadBalancer:
+          type: ConsistentHash
+          consistentHash:
+            type: SourceIP
 ---
 apiVersion: gateway.networking.k8s.io/v1
 kind: BackendTLSPolicy