Skip to content

Commit a176fb6

Browse files
andreademurtasandreademurtas
authored
fix(xds): use DnsCluster extension for DNS clusters (#9155)
* fix(xds): use DnsCluster extension for DNS clusters Replace the deprecated Cluster.dns_refresh_rate and respect_dns_ttl fields with Envoy's envoy.cluster.dns extension for DNS-based clusters. Behavior is preserved (30s default, respect_dns_ttl, dns_lookup_family carried into the extension via a name-based enum mapping); STRICT_DNS semantics kept via all_addresses_in_single_endpoint=false. Fixes #6192 Signed-off-by: Andrea De Murtas <56048157+andreademurtas@users.noreply.github.com>
1 parent 2646c11 commit a176fb6

65 files changed

Lines changed: 640 additions & 239 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.all.json‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -407,10 +407,18 @@
407407
}
408408
]
409409
},
410+
"clusterType": {
411+
"name": "envoy.cluster.dns",
412+
"typedConfig": {
413+
"@type": "type.googleapis.com/envoy.extensions.clusters.dns.v3.DnsCluster",
414+
"dnsLookupFamily": "V4_PREFERRED",
415+
"dnsRefreshRate": "30s",
416+
"respectDnsTtl": true
417+
}
418+
},
410419
"commonLbConfig": {},
411420
"connectTimeout": "10s",
412421
"dnsLookupFamily": "V4_PREFERRED",
413-
"dnsRefreshRate": "30s",
414422
"ignoreHealthOnHostRemoval": true,
415423
"loadAssignment": {
416424
"clusterName": "raw_githubusercontent_com_443",
@@ -453,7 +461,6 @@
453461
},
454462
"name": "raw_githubusercontent_com_443",
455463
"perConnectionBufferLimitBytes": 32768,
456-
"respectDnsTtl": true,
457464
"transportSocket": {
458465
"name": "envoy.transport_sockets.tls",
459466
"typedConfig": {
@@ -467,8 +474,7 @@
467474
},
468475
"sni": "raw.githubusercontent.com"
469476
}
470-
},
471-
"type": "STRICT_DNS"
477+
}
472478
}
473479
}
474480
]

‎internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.all.yaml‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -229,10 +229,16 @@ xds:
229229
circuitBreakers:
230230
thresholds:
231231
- maxRetries: 1024
232+
clusterType:
233+
name: envoy.cluster.dns
234+
typedConfig:
235+
'@type': type.googleapis.com/envoy.extensions.clusters.dns.v3.DnsCluster
236+
dnsLookupFamily: V4_PREFERRED
237+
dnsRefreshRate: 30s
238+
respectDnsTtl: true
232239
commonLbConfig: {}
233240
connectTimeout: 10s
234241
dnsLookupFamily: V4_PREFERRED
235-
dnsRefreshRate: 30s
236242
ignoreHealthOnHostRemoval: true
237243
loadAssignment:
238244
clusterName: raw_githubusercontent_com_443
@@ -257,7 +263,6 @@ xds:
257263
localityWeightedLbConfig: {}
258264
name: raw_githubusercontent_com_443
259265
perConnectionBufferLimitBytes: 32768
260-
respectDnsTtl: true
261266
transportSocket:
262267
name: envoy.transport_sockets.tls
263268
typedConfig:
@@ -267,7 +272,6 @@ xds:
267272
trustedCa:
268273
filename: /etc/ssl/certs/ca-certificates.crt
269274
sni: raw.githubusercontent.com
270-
type: STRICT_DNS
271275
- '@type': type.googleapis.com/envoy.admin.v3.ListenersConfigDump
272276
dynamicListeners:
273277
- activeState:

‎internal/cmd/egctl/testdata/translate/out/jwt-single-route-single-match-to-xds.cluster.yaml‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,10 +39,16 @@ xds:
3939
circuitBreakers:
4040
thresholds:
4141
- maxRetries: 1024
42+
clusterType:
43+
name: envoy.cluster.dns
44+
typedConfig:
45+
'@type': type.googleapis.com/envoy.extensions.clusters.dns.v3.DnsCluster
46+
dnsLookupFamily: V4_PREFERRED
47+
dnsRefreshRate: 30s
48+
respectDnsTtl: true
4249
commonLbConfig: {}
4350
connectTimeout: 10s
4451
dnsLookupFamily: V4_PREFERRED
45-
dnsRefreshRate: 30s
4652
ignoreHealthOnHostRemoval: true
4753
loadAssignment:
4854
clusterName: raw_githubusercontent_com_443
@@ -67,7 +73,6 @@ xds:
6773
localityWeightedLbConfig: {}
6874
name: raw_githubusercontent_com_443
6975
perConnectionBufferLimitBytes: 32768
70-
respectDnsTtl: true
7176
transportSocket:
7277
name: envoy.transport_sockets.tls
7378
typedConfig:
@@ -77,4 +82,3 @@ xds:
7782
trustedCa:
7883
filename: /etc/ssl/certs/ca-certificates.crt
7984
sni: raw.githubusercontent.com
80-
type: STRICT_DNS

‎internal/xds/translator/cluster.go‎

Lines changed: 47 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ import (
1515
clusterv3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
1616
corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
1717
endpointv3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
18+
commondnsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/common/dns/v3"
19+
dnsclusterv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/dns/v3"
1820
dfpv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/dynamic_forward_proxy/v3"
1921
commondfpv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/common/dynamic_forward_proxy/v3"
2022
codecv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/upstream_codec/v3"
@@ -58,6 +60,10 @@ const (
5860
tcpClusterPerConnectTimeout = 10 * time.Second
5961
dfpClusterTypeName = "envoy.clusters.dynamic_forward_proxy"
6062
dfpDNSCacheName = "envoy-gateway-dfp-cache"
63+
// dnsClusterTypeName is the registered name of Envoy's DNS cluster extension factory
64+
// (envoy.extensions.clusters.dns.v3.DnsCluster). Note the singular "cluster": this is the
65+
// factory name set in Envoy's ConfigurableClusterFactoryBase ctor, not the plural extension key.
66+
dnsClusterTypeName = "envoy.cluster.dns"
6167
)
6268

6369
type xdsClusterArgs struct {
@@ -169,6 +175,25 @@ func computeDNSLookupFamily(ipFamily *egv1a1.IPFamily, dns *ir.DNS) clusterv3.Cl
169175
return dnsLookupFamily
170176
}
171177

178+
// toCommonDNSLookupFamily maps the deprecated config.cluster.v3.Cluster_DnsLookupFamily enum to its
179+
// equivalent in extensions.clusters.common.dns.v3, used by the envoy.cluster.dns extension. The two
180+
// enums are NOT integer-compatible: the common enum has an extra UNSPECIFIED=0 value, so it is offset
181+
// by one. Mapping is done by name; any unknown value defaults to AUTO (the DnsCluster default).
182+
func toCommonDNSLookupFamily(f clusterv3.Cluster_DnsLookupFamily) commondnsv3.DnsLookupFamily {
183+
switch f {
184+
case clusterv3.Cluster_V4_ONLY:
185+
return commondnsv3.DnsLookupFamily_V4_ONLY
186+
case clusterv3.Cluster_V6_ONLY:
187+
return commondnsv3.DnsLookupFamily_V6_ONLY
188+
case clusterv3.Cluster_V4_PREFERRED:
189+
return commondnsv3.DnsLookupFamily_V4_PREFERRED
190+
case clusterv3.Cluster_ALL:
191+
return commondnsv3.DnsLookupFamily_ALL
192+
default:
193+
return commondnsv3.DnsLookupFamily_AUTO
194+
}
195+
}
196+
172197
func dfpCacheName(ipFamily *egv1a1.IPFamily, dns *ir.DNS) string {
173198
refresh := 30 * time.Second
174199
if dns != nil && dns.DNSRefreshRate != nil && dns.DNSRefreshRate.Duration > 0 {
@@ -515,17 +540,34 @@ func buildXdsCluster(args *xdsClusterArgs) (*buildClusterResult, error) {
515540
},
516541
}
517542
default:
518-
cluster.ClusterDiscoveryType = &clusterv3.Cluster_Type{Type: clusterv3.Cluster_STRICT_DNS}
519-
cluster.DnsRefreshRate = durationpb.New(30 * time.Second)
520-
cluster.RespectDnsTtl = true
543+
// DNS-based endpoints use Envoy's DNS cluster extension (envoy.cluster.dns) instead of the
544+
// deprecated top-level Cluster.dns_refresh_rate / respect_dns_ttl fields. Leaving
545+
// all_addresses_in_single_endpoint unset (false) is equivalent to the legacy STRICT_DNS type.
546+
dnsCluster := &dnsclusterv3.DnsCluster{
547+
DnsRefreshRate: durationpb.New(30 * time.Second),
548+
RespectDnsTtl: true,
549+
// When cluster_type is set, Envoy reads dns_lookup_family from the DnsCluster extension and
550+
// ignores the top-level Cluster.dns_lookup_family, so carry it over here to preserve behavior.
551+
DnsLookupFamily: toCommonDNSLookupFamily(dnsLookupFamily),
552+
}
521553
if args.dns != nil {
522554
if args.dns.DNSRefreshRate != nil {
523-
cluster.DnsRefreshRate = durationpb.New(args.dns.DNSRefreshRate.Duration)
555+
dnsCluster.DnsRefreshRate = durationpb.New(args.dns.DNSRefreshRate.Duration)
524556
}
525557
if args.dns.RespectDNSTTL != nil {
526-
cluster.RespectDnsTtl = ptr.Deref(args.dns.RespectDNSTTL, true)
558+
dnsCluster.RespectDnsTtl = ptr.Deref(args.dns.RespectDNSTTL, true)
527559
}
528560
}
561+
dnsClusterAny, err := proto.ToAnyWithValidation(dnsCluster)
562+
if err != nil {
563+
return nil, err
564+
}
565+
cluster.ClusterDiscoveryType = &clusterv3.Cluster_ClusterType{
566+
ClusterType: &clusterv3.Cluster_CustomClusterType{
567+
Name: dnsClusterTypeName,
568+
TypedConfig: dnsClusterAny,
569+
},
570+
}
529571
}
530572

531573
return &buildClusterResult{

‎internal/xds/translator/cluster_test.go‎

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ import (
1212
bootstrapv3 "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v3"
1313
clusterv3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3"
1414
endpointv3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
15+
commondnsv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/common/dns/v3"
16+
dnsclusterv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/dns/v3"
1517
cswrrv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3"
1618
override_hostv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/override_host/v3"
1719
wrr_localityv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/load_balancing_policies/wrr_locality/v3"
@@ -48,12 +50,54 @@ func TestBuildXdsCluster(t *testing.T) {
4850
require.NoError(t, err)
4951
dynamicXdsCluster := result.cluster
5052
require.Equal(t, bootstrapXdsCluster.Name, dynamicXdsCluster.Name)
51-
require.Equal(t, bootstrapXdsCluster.ClusterDiscoveryType, dynamicXdsCluster.ClusterDiscoveryType)
53+
// buildXdsCluster emits DNS-based clusters via the envoy.cluster.dns extension rather than the
54+
// (still-supported) STRICT_DNS type used by the static bootstrap xds_cluster, so assert the
55+
// extension cluster type and its DnsCluster config explicitly instead of comparing discovery types.
56+
clusterType := dynamicXdsCluster.GetClusterType()
57+
require.NotNil(t, clusterType)
58+
require.Equal(t, dnsClusterTypeName, clusterType.Name)
59+
dnsCluster := &dnsclusterv3.DnsCluster{}
60+
require.NoError(t, clusterType.TypedConfig.UnmarshalTo(dnsCluster))
61+
require.Equal(t, durationpb.New(30*time.Second), dnsCluster.DnsRefreshRate)
62+
require.True(t, dnsCluster.RespectDnsTtl)
63+
require.Equal(t, commondnsv3.DnsLookupFamily_V4_PREFERRED, dnsCluster.DnsLookupFamily)
5264
require.Equal(t, bootstrapXdsCluster.TransportSocket, dynamicXdsCluster.TransportSocket)
5365
requireCmpNoDiff(t, bootstrapXdsCluster.TransportSocket, dynamicXdsCluster.TransportSocket)
5466
requireCmpNoDiff(t, bootstrapXdsCluster.ConnectTimeout, dynamicXdsCluster.ConnectTimeout)
5567
}
5668

69+
func TestBuildXdsClusterDNSRefreshRateValidation(t *testing.T) {
70+
// A DNS refresh rate of 1ms or less is rejected by the envoy.cluster.dns extension
71+
// (DnsCluster.dns_refresh_rate must be greater than 1ms), so buildXdsCluster must surface the error.
72+
args := &xdsClusterArgs{
73+
name: "dns-cluster",
74+
endpointType: EndpointTypeDNS,
75+
dns: &ir.DNS{DNSRefreshRate: ir.MetaV1DurationPtr(500 * time.Microsecond)},
76+
}
77+
_, err := buildXdsCluster(args)
78+
require.ErrorContains(t, err, "DnsRefreshRate")
79+
}
80+
81+
func TestToCommonDNSLookupFamily(t *testing.T) {
82+
tests := []struct {
83+
name string
84+
in clusterv3.Cluster_DnsLookupFamily
85+
expected commondnsv3.DnsLookupFamily
86+
}{
87+
{"auto", clusterv3.Cluster_AUTO, commondnsv3.DnsLookupFamily_AUTO},
88+
{"v4 only", clusterv3.Cluster_V4_ONLY, commondnsv3.DnsLookupFamily_V4_ONLY},
89+
{"v6 only", clusterv3.Cluster_V6_ONLY, commondnsv3.DnsLookupFamily_V6_ONLY},
90+
{"v4 preferred", clusterv3.Cluster_V4_PREFERRED, commondnsv3.DnsLookupFamily_V4_PREFERRED},
91+
{"all", clusterv3.Cluster_ALL, commondnsv3.DnsLookupFamily_ALL},
92+
{"unknown defaults to auto", clusterv3.Cluster_DnsLookupFamily(99), commondnsv3.DnsLookupFamily_AUTO},
93+
}
94+
for _, tc := range tests {
95+
t.Run(tc.name, func(t *testing.T) {
96+
require.Equal(t, tc.expected, toCommonDNSLookupFamily(tc.in))
97+
})
98+
}
99+
}
100+
57101
func TestBuildXdsClusterLoadAssignment(t *testing.T) {
58102
bootstrapXdsCluster := getXdsClusterObjFromBootstrap(t)
59103
require.NotNil(t, bootstrapXdsCluster)

‎internal/xds/translator/testdata/out/xds-ir/accesslog-cel.clusters.yaml‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,16 @@
2424
- circuitBreakers:
2525
thresholds:
2626
- maxRetries: 1024
27+
clusterType:
28+
name: envoy.cluster.dns
29+
typedConfig:
30+
'@type': type.googleapis.com/envoy.extensions.clusters.dns.v3.DnsCluster
31+
dnsLookupFamily: V4_PREFERRED
32+
dnsRefreshRate: 30s
33+
respectDnsTtl: true
2734
commonLbConfig: {}
2835
connectTimeout: 10s
2936
dnsLookupFamily: V4_PREFERRED
30-
dnsRefreshRate: 30s
3137
ignoreHealthOnHostRemoval: true
3238
loadAssignment:
3339
clusterName: accesslog-0
@@ -52,8 +58,6 @@
5258
localityWeightedLbConfig: {}
5359
name: accesslog-0
5460
perConnectionBufferLimitBytes: 32768
55-
respectDnsTtl: true
56-
type: STRICT_DNS
5761
typedExtensionProtocolOptions:
5862
envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
5963
'@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions

‎internal/xds/translator/testdata/out/xds-ir/accesslog-endpoint-stats.clusters.yaml‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,16 @@
2626
- circuitBreakers:
2727
thresholds:
2828
- maxRetries: 1024
29+
clusterType:
30+
name: envoy.cluster.dns
31+
typedConfig:
32+
'@type': type.googleapis.com/envoy.extensions.clusters.dns.v3.DnsCluster
33+
dnsLookupFamily: V4_PREFERRED
34+
dnsRefreshRate: 30s
35+
respectDnsTtl: true
2936
commonLbConfig: {}
3037
connectTimeout: 10s
3138
dnsLookupFamily: V4_PREFERRED
32-
dnsRefreshRate: 30s
3339
ignoreHealthOnHostRemoval: true
3440
loadAssignment:
3541
clusterName: accesslog-0
@@ -54,10 +60,8 @@
5460
localityWeightedLbConfig: {}
5561
name: accesslog-0
5662
perConnectionBufferLimitBytes: 32768
57-
respectDnsTtl: true
5863
trackClusterStats:
5964
perEndpointStats: true
60-
type: STRICT_DNS
6165
typedExtensionProtocolOptions:
6266
envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
6367
'@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions

‎internal/xds/translator/testdata/out/xds-ir/accesslog-formatters.clusters.yaml‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,16 @@
2424
- circuitBreakers:
2525
thresholds:
2626
- maxRetries: 1024
27+
clusterType:
28+
name: envoy.cluster.dns
29+
typedConfig:
30+
'@type': type.googleapis.com/envoy.extensions.clusters.dns.v3.DnsCluster
31+
dnsLookupFamily: V4_PREFERRED
32+
dnsRefreshRate: 30s
33+
respectDnsTtl: true
2734
commonLbConfig: {}
2835
connectTimeout: 10s
2936
dnsLookupFamily: V4_PREFERRED
30-
dnsRefreshRate: 30s
3137
ignoreHealthOnHostRemoval: true
3238
loadAssignment:
3339
clusterName: accesslog-0
@@ -52,8 +58,6 @@
5258
localityWeightedLbConfig: {}
5359
name: accesslog-0
5460
perConnectionBufferLimitBytes: 32768
55-
respectDnsTtl: true
56-
type: STRICT_DNS
5761
typedExtensionProtocolOptions:
5862
envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
5963
'@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions

‎internal/xds/translator/testdata/out/xds-ir/accesslog-multi-cel.clusters.yaml‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,16 @@
2424
- circuitBreakers:
2525
thresholds:
2626
- maxRetries: 1024
27+
clusterType:
28+
name: envoy.cluster.dns
29+
typedConfig:
30+
'@type': type.googleapis.com/envoy.extensions.clusters.dns.v3.DnsCluster
31+
dnsLookupFamily: V4_PREFERRED
32+
dnsRefreshRate: 30s
33+
respectDnsTtl: true
2734
commonLbConfig: {}
2835
connectTimeout: 10s
2936
dnsLookupFamily: V4_PREFERRED
30-
dnsRefreshRate: 30s
3137
ignoreHealthOnHostRemoval: true
3238
loadAssignment:
3339
clusterName: accesslog-0
@@ -52,8 +58,6 @@
5258
localityWeightedLbConfig: {}
5359
name: accesslog-0
5460
perConnectionBufferLimitBytes: 32768
55-
respectDnsTtl: true
56-
type: STRICT_DNS
5761
typedExtensionProtocolOptions:
5862
envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
5963
'@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions

‎internal/xds/translator/testdata/out/xds-ir/accesslog-otel-headers.clusters.yaml‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,16 @@
2424
- circuitBreakers:
2525
thresholds:
2626
- maxRetries: 1024
27+
clusterType:
28+
name: envoy.cluster.dns
29+
typedConfig:
30+
'@type': type.googleapis.com/envoy.extensions.clusters.dns.v3.DnsCluster
31+
dnsLookupFamily: V4_PREFERRED
32+
dnsRefreshRate: 30s
33+
respectDnsTtl: true
2734
commonLbConfig: {}
2835
connectTimeout: 10s
2936
dnsLookupFamily: V4_PREFERRED
30-
dnsRefreshRate: 30s
3137
ignoreHealthOnHostRemoval: true
3238
loadAssignment:
3339
clusterName: accesslog-0
@@ -52,8 +58,6 @@
5258
localityWeightedLbConfig: {}
5359
name: accesslog-0
5460
perConnectionBufferLimitBytes: 32768
55-
respectDnsTtl: true
56-
type: STRICT_DNS
5761
typedExtensionProtocolOptions:
5862
envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
5963
'@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions

0 commit comments

Comments
 (0)