Skip to content

Commit 52f9e50

Browse files
authored
feat: add envoyproxy status (#8475)
* nit Signed-off-by: zirain <zirain2009@gmail.com> * add envoyproxy status Signed-off-by: zirain <zirain2009@gmail.com> * fix gen Signed-off-by: zirain <zirain2009@gmail.com> * lint Signed-off-by: zirain <zirain2009@gmail.com> * perf Signed-off-by: zirain <zirain2009@gmail.com> * fix Signed-off-by: zirain <zirain2009@gmail.com> * fix gen Signed-off-by: zirain <zirain2009@gmail.com> * comment and nit Signed-off-by: zirain <zirain2009@gmail.com> * fix status and more e2e Signed-off-by: zirain <zirain2009@gmail.com> * fix gen Signed-off-by: zirain <zirain2009@gmail.com> --------- Signed-off-by: zirain <zirain2009@gmail.com>
1 parent 1c60010 commit 52f9e50

62 files changed

Lines changed: 1468 additions & 34 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

charts/gateway-helm/templates/_rbac.tpl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ verbs:
100100
apiGroups:
101101
- gateway.envoyproxy.io
102102
resources:
103+
- envoyproxies/status
103104
- envoypatchpolicies/status
104105
- clienttrafficpolicies/status
105106
- backendtrafficpolicies/status

internal/cmd/egctl/testdata/translate/out/default-resources.all.yaml

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,18 @@ envoyProxyForGatewayClass:
174174
"@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig
175175
max_active_downstream_connections: 50000
176176
logging: {}
177-
status: {}
177+
status:
178+
ancestors:
179+
- ancestorRef:
180+
group: gateway.networking.k8s.io
181+
kind: GatewayClass
182+
name: eg
183+
conditions:
184+
- lastTransitionTime: null
185+
message: EnvoyProxy has been accepted.
186+
reason: Accepted
187+
status: "True"
188+
type: Accepted
178189
gatewayClass:
179190
apiVersion: gateway.networking.k8s.io/v1
180191
kind: GatewayClass

internal/cmd/egctl/testdata/translate/out/invalid-envoyproxy.all.yaml

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,18 @@ envoyProxyForGatewayClass:
2121
logging:
2222
level:
2323
default: warn
24-
status: {}
24+
status:
25+
ancestors:
26+
- ancestorRef:
27+
group: gateway.networking.k8s.io
28+
kind: GatewayClass
29+
name: eg
30+
conditions:
31+
- lastTransitionTime: null
32+
message: dynamic_resources cannot be modified
33+
reason: InvalidParameters
34+
status: "False"
35+
type: Accepted
2536
gatewayClass:
2637
apiVersion: gateway.networking.k8s.io/v1
2738
kind: GatewayClass

internal/cmd/egctl/testdata/translate/out/valid-envoyproxy.all.yaml

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,18 @@ envoyProxyForGatewayClass:
1616
externalTrafficPolicy: Local
1717
type: LoadBalancer
1818
type: Kubernetes
19-
status: {}
19+
status:
20+
ancestors:
21+
- ancestorRef:
22+
group: gateway.networking.k8s.io
23+
kind: GatewayClass
24+
name: eg
25+
conditions:
26+
- lastTransitionTime: null
27+
message: EnvoyProxy has been accepted.
28+
reason: Accepted
29+
status: "True"
30+
type: Accepted
2031
gatewayClass:
2132
apiVersion: gateway.networking.k8s.io/v1
2233
kind: GatewayClass

internal/gatewayapi/contexts.go

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,10 @@ import (
2929
type GatewayContext struct {
3030
*gwapiv1.Gateway
3131

32-
listeners []*ListenerContext
33-
envoyProxy *egv1a1.EnvoyProxy
32+
listeners []*ListenerContext
33+
envoyProxy *egv1a1.EnvoyProxy
34+
envoyProxyFromGateway bool
35+
3436
backendTLS *egv1a1.BackendTLSConfig
3537
}
3638

@@ -105,6 +107,7 @@ func (g *GatewayContext) attachEnvoyProxy(resources *resource.Resources, epMap m
105107
if string(ref.Group) == egv1a1.GroupVersion.Group && ref.Kind == egv1a1.KindEnvoyProxy {
106108
ep, exists := epMap[types.NamespacedName{Namespace: g.Namespace, Name: ref.Name}]
107109
if exists {
110+
g.envoyProxyFromGateway = true
108111
gatewayProxy = ep
109112
}
110113
}

internal/gatewayapi/runner/runner.go

Lines changed: 64 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,14 +112,33 @@ func mergePolicyStatus(aggregated aggregatedPolicyStatus, incoming *gwapiv1.Poli
112112
return aggregated
113113
}
114114

115-
aggregated.status.Ancestors = append(aggregated.status.Ancestors, incoming.Ancestors...)
115+
// Prevent self-merge when aggregated and incoming reference the same object
116+
if aggregated.status != incoming {
117+
aggregated.status.Ancestors = append(aggregated.status.Ancestors, incoming.Ancestors...)
118+
}
116119
if generation > aggregated.generation {
117120
aggregated.generation = generation
118121
}
119122

120123
return aggregated
121124
}
122125

126+
func mergeEnvoyProxyStatus(aggregated, incoming *egv1a1.EnvoyProxyStatus) *egv1a1.EnvoyProxyStatus {
127+
if incoming == nil {
128+
return aggregated
129+
}
130+
131+
if aggregated == nil {
132+
return incoming
133+
}
134+
135+
// Prevent self-merge when aggregated and incoming reference the same object
136+
if aggregated != incoming {
137+
aggregated.Ancestors = append(aggregated.Ancestors, incoming.Ancestors...)
138+
}
139+
return aggregated
140+
}
141+
123142
func New(cfg *Config) *Runner {
124143
return &Runner{
125144
Config: *cfg,
@@ -206,6 +225,13 @@ func (r *Runner) subscribeAndTranslate(sub <-chan watchable.Snapshot[string, *re
206225
// There is only 1 key which is the controller name
207226
// so when a delete is triggered, delete all keys
208227
if update.Delete || valWrapper == nil || valWrapper.Resources == nil {
228+
// Clear EnvoyProxy statuses before deleting to remove stale ancestor conditions
229+
for key := range r.keyCache.EnvoyProxyStatus {
230+
emptyStatus := &egv1a1.EnvoyProxyStatus{
231+
Ancestors: []egv1a1.EnvoyProxyAncestorStatus{},
232+
}
233+
r.ProviderResources.EnvoyProxyStatuses.Store(key, emptyStatus)
234+
}
209235
r.deleteAllKeys()
210236
return
211237
}
@@ -229,6 +255,7 @@ func (r *Runner) subscribeAndTranslate(sub <-chan watchable.Snapshot[string, *re
229255
var tlsRouteStatusCount, tcpRouteStatusCount, udpRouteStatusCount int
230256
var backendTLSPolicyStatusCount, clientTrafficPolicyStatusCount, backendTrafficPolicyStatusCount int
231257
var securityPolicyStatusCount, envoyExtensionPolicyStatusCount, backendStatusCount, extensionServerPolicyStatusCount int
258+
var envoyproxyStatusCount int
232259

233260
// `aggregatedStatuses` aggregates status result of resources from all
234261
// parents/ancestors, and then stores the status once for every resource.
@@ -244,6 +271,7 @@ func (r *Runner) subscribeAndTranslate(sub <-chan watchable.Snapshot[string, *re
244271
SecurityPolicies map[types.NamespacedName]aggregatedPolicyStatus
245272
EnvoyExtensionPolicies map[types.NamespacedName]aggregatedPolicyStatus
246273
ExtensionServerPolicies map[message.NamespacedNameAndGVK]aggregatedPolicyStatus
274+
EnvoyProxies map[types.NamespacedName]*egv1a1.EnvoyProxyStatus
247275
}{
248276
HTTPRoutes: make(map[types.NamespacedName]aggregatedRouteStatus),
249277
GRPCRoutes: make(map[types.NamespacedName]aggregatedRouteStatus),
@@ -256,6 +284,7 @@ func (r *Runner) subscribeAndTranslate(sub <-chan watchable.Snapshot[string, *re
256284
SecurityPolicies: make(map[types.NamespacedName]aggregatedPolicyStatus),
257285
EnvoyExtensionPolicies: make(map[types.NamespacedName]aggregatedPolicyStatus),
258286
ExtensionServerPolicies: make(map[message.NamespacedNameAndGVK]aggregatedPolicyStatus),
287+
EnvoyProxies: make(map[types.NamespacedName]*egv1a1.EnvoyProxyStatus),
259288
}
260289

261290
span.AddEvent("translate", trace.WithAttributes(attribute.Int("resources.count", len(*val))))
@@ -447,6 +476,15 @@ func (r *Runner) subscribeAndTranslate(sub <-chan watchable.Snapshot[string, *re
447476
aggregatedStatuses.ExtensionServerPolicies[key] = mergePolicyStatus(aggregatedStatuses.ExtensionServerPolicies[key], &policyStatus, extServerPolicy.GetGeneration())
448477
}
449478
}
479+
// EnvoyProxy status
480+
for _, ep := range result.EnvoyProxiesForGateways {
481+
r.Logger.Info("update envoyproxy status", "key", utils.NamespacedName(ep))
482+
aggregatedStatuses.EnvoyProxies[utils.NamespacedName(ep)] = mergeEnvoyProxyStatus(aggregatedStatuses.EnvoyProxies[utils.NamespacedName(ep)], &ep.Status)
483+
}
484+
if ep := result.EnvoyProxyForGatewayClass; ep != nil {
485+
r.Logger.Info("update envoyproxy status", "key", utils.NamespacedName(ep))
486+
aggregatedStatuses.EnvoyProxies[utils.NamespacedName(ep)] = mergeEnvoyProxyStatus(aggregatedStatuses.EnvoyProxies[utils.NamespacedName(ep)], &ep.Status)
487+
}
450488
statusUpdateSpan.End()
451489
}
452490

@@ -533,6 +571,15 @@ func (r *Runner) subscribeAndTranslate(sub <-chan watchable.Snapshot[string, *re
533571
delete(keysToDelete.ExtensionServerPolicyStatus, key)
534572
r.keyCache.ExtensionServerPolicyStatus[key] = true
535573
}
574+
for key, entry := range aggregatedStatuses.EnvoyProxies {
575+
s := egv1a1.EnvoyProxyStatus{
576+
Ancestors: entry.Ancestors,
577+
}
578+
r.ProviderResources.EnvoyProxyStatuses.Store(key, &s)
579+
envoyproxyStatusCount++
580+
delete(keysToDelete.EnvoyProxyStatus, key)
581+
r.keyCache.EnvoyProxyStatus[key] = true
582+
}
536583
// Publish aggregated metrics
537584
message.PublishMetric(message.Metadata{Runner: r.Name(), Message: message.InfraIRMessageName}, infraIRCount)
538585
message.PublishMetric(message.Metadata{Runner: r.Name(), Message: message.XDSIRMessageName}, xdsIRCount)
@@ -551,6 +598,7 @@ func (r *Runner) subscribeAndTranslate(sub <-chan watchable.Snapshot[string, *re
551598
message.PublishMetric(message.Metadata{Runner: r.Name(), Message: message.EnvoyExtensionPolicyStatusMessageName}, envoyExtensionPolicyStatusCount)
552599
message.PublishMetric(message.Metadata{Runner: r.Name(), Message: message.BackendStatusMessageName}, backendStatusCount)
553600
message.PublishMetric(message.Metadata{Runner: r.Name(), Message: message.ExtensionServerPoliciesStatusMessageName}, extensionServerPolicyStatusCount)
601+
message.PublishMetric(message.Metadata{Runner: r.Name(), Message: message.EnvoyProxyStatusMessageName}, envoyproxyStatusCount)
554602

555603
// Delete keys using mark and sweep
556604
r.deleteKeys(keysToDelete)
@@ -660,6 +708,9 @@ func (r *Runner) deleteAllKeys() {
660708
for key := range r.keyCache.BackendStatus {
661709
r.ProviderResources.BackendStatuses.Delete(key)
662710
}
711+
for key := range r.keyCache.EnvoyProxyStatus {
712+
r.ProviderResources.EnvoyProxyStatuses.Delete(key)
713+
}
663714

664715
// Clear all tracking
665716
r.keyCache = newKeyCache()
@@ -684,6 +735,7 @@ type KeyCache struct {
684735
SecurityPolicyStatus map[types.NamespacedName]bool
685736
EnvoyExtensionPolicyStatus map[types.NamespacedName]bool
686737
ExtensionServerPolicyStatus map[message.NamespacedNameAndGVK]bool
738+
EnvoyProxyStatus map[types.NamespacedName]bool
687739

688740
BackendStatus map[types.NamespacedName]bool
689741
}
@@ -737,6 +789,9 @@ func (kc *KeyCache) copy() *KeyCache {
737789
for key := range kc.ExtensionServerPolicyStatus {
738790
copied.ExtensionServerPolicyStatus[key] = true
739791
}
792+
for key := range kc.EnvoyProxyStatus {
793+
copied.EnvoyProxyStatus[key] = true
794+
}
740795
for key := range kc.BackendStatus {
741796
copied.BackendStatus[key] = true
742797
}
@@ -760,6 +815,7 @@ func newKeyCache() *KeyCache {
760815
SecurityPolicyStatus: make(map[types.NamespacedName]bool),
761816
EnvoyExtensionPolicyStatus: make(map[types.NamespacedName]bool),
762817
ExtensionServerPolicyStatus: make(map[message.NamespacedNameAndGVK]bool),
818+
EnvoyProxyStatus: make(map[types.NamespacedName]bool),
763819
BackendStatus: make(map[types.NamespacedName]bool),
764820
}
765821
}
@@ -812,6 +868,9 @@ func (r *Runner) populateKeyCache() {
812868
for key := range r.ProviderResources.ExtensionPolicyStatuses.LoadAll() {
813869
r.keyCache.ExtensionServerPolicyStatus[key] = true
814870
}
871+
for key := range r.ProviderResources.EnvoyProxyStatuses.LoadAll() {
872+
r.keyCache.EnvoyProxyStatus[key] = true
873+
}
815874
for key := range r.ProviderResources.BackendStatuses.LoadAll() {
816875
r.keyCache.BackendStatus[key] = true
817876
}
@@ -879,6 +938,10 @@ func (r *Runner) deleteKeys(kc *KeyCache) {
879938
r.ProviderResources.ExtensionPolicyStatuses.Delete(key)
880939
delete(r.keyCache.ExtensionServerPolicyStatus, key)
881940
}
941+
for key := range kc.EnvoyProxyStatus {
942+
r.ProviderResources.EnvoyProxyStatuses.Delete(key)
943+
delete(r.keyCache.EnvoyProxyStatus, key)
944+
}
882945
for key := range kc.BackendStatus {
883946
r.ProviderResources.BackendStatuses.Delete(key)
884947
delete(r.keyCache.BackendStatus, key)

internal/gatewayapi/runner/runner_test.go

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -304,6 +304,94 @@ func TestMergePolicyStatus(t *testing.T) {
304304
entry = mergePolicyStatus(entry, &gwapiv1.PolicyStatus{}, 4)
305305
require.Equal(t, int64(9), entry.generation)
306306
})
307+
308+
t.Run("self-merge prevention", func(t *testing.T) {
309+
status := &gwapiv1.PolicyStatus{
310+
Ancestors: []gwapiv1.PolicyAncestorStatus{
311+
{
312+
AncestorRef: gwapiv1.ParentReference{Name: gwapiv1.ObjectName("gw-a")},
313+
ControllerName: gwapiv1a2.GatewayController(controllerName),
314+
},
315+
},
316+
}
317+
318+
// First merge - status becomes both aggregated and incoming
319+
entry := mergePolicyStatus(aggregatedPolicyStatus{}, status, 1)
320+
// Second merge with same status - should not duplicate
321+
entry = mergePolicyStatus(entry, status, 2)
322+
323+
// Should still have only one ancestor, not duplicated
324+
require.Len(t, entry.status.Ancestors, 1)
325+
require.Equal(t, gwapiv1.ObjectName("gw-a"), entry.status.Ancestors[0].AncestorRef.Name)
326+
})
327+
}
328+
329+
func TestMergeEnvoyProxyStatus(t *testing.T) {
330+
t.Run("nil incoming keeps existing entry", func(t *testing.T) {
331+
existing := &egv1a1.EnvoyProxyStatus{
332+
Ancestors: []egv1a1.EnvoyProxyAncestorStatus{
333+
{
334+
AncestorRef: gwapiv1.ParentReference{Name: gwapiv1.ObjectName("gc-a")},
335+
},
336+
},
337+
}
338+
339+
got := mergeEnvoyProxyStatus(existing, nil)
340+
require.Same(t, existing, got)
341+
})
342+
343+
t.Run("nil existing takes incoming", func(t *testing.T) {
344+
incoming := &egv1a1.EnvoyProxyStatus{
345+
Ancestors: []egv1a1.EnvoyProxyAncestorStatus{
346+
{
347+
AncestorRef: gwapiv1.ParentReference{Name: gwapiv1.ObjectName("gc-a")},
348+
},
349+
},
350+
}
351+
352+
got := mergeEnvoyProxyStatus(nil, incoming)
353+
require.Same(t, incoming, got)
354+
})
355+
356+
t.Run("appends ancestors", func(t *testing.T) {
357+
first := &egv1a1.EnvoyProxyStatus{
358+
Ancestors: []egv1a1.EnvoyProxyAncestorStatus{
359+
{
360+
AncestorRef: gwapiv1.ParentReference{Name: gwapiv1.ObjectName("gc-a")},
361+
},
362+
},
363+
}
364+
second := &egv1a1.EnvoyProxyStatus{
365+
Ancestors: []egv1a1.EnvoyProxyAncestorStatus{
366+
{
367+
AncestorRef: gwapiv1.ParentReference{Name: gwapiv1.ObjectName("gw-b")},
368+
},
369+
},
370+
}
371+
372+
got := mergeEnvoyProxyStatus(first, second)
373+
374+
require.Len(t, got.Ancestors, 2)
375+
require.Equal(t, gwapiv1.ObjectName("gc-a"), got.Ancestors[0].AncestorRef.Name)
376+
require.Equal(t, gwapiv1.ObjectName("gw-b"), got.Ancestors[1].AncestorRef.Name)
377+
})
378+
379+
t.Run("self-merge prevention", func(t *testing.T) {
380+
status := &egv1a1.EnvoyProxyStatus{
381+
Ancestors: []egv1a1.EnvoyProxyAncestorStatus{
382+
{
383+
AncestorRef: gwapiv1.ParentReference{Name: gwapiv1.ObjectName("gc-a")},
384+
},
385+
},
386+
}
387+
388+
// Merge status with itself - should not duplicate
389+
got := mergeEnvoyProxyStatus(status, status)
390+
391+
// Should still have only one ancestor, not duplicated
392+
require.Len(t, got.Ancestors, 1)
393+
require.Equal(t, gwapiv1.ObjectName("gc-a"), got.Ancestors[0].AncestorRef.Name)
394+
})
307395
}
308396

309397
func TestMergeRouteStatus(t *testing.T) {
Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
// Copyright Envoy Gateway Authors
2+
// SPDX-License-Identifier: Apache-2.0
3+
// The full text of the Apache license is available in the LICENSE file at
4+
// the root of the repo.
5+
6+
package status
7+
8+
import (
9+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
10+
gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
11+
12+
egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
13+
)
14+
15+
func UpdateEnvoyProxyStatusAccepted(ep *egv1a1.EnvoyProxy, ancestor *gwapiv1.ParentReference,
16+
reason egv1a1.EnvoyProxyConditionReason, msg string,
17+
) {
18+
if ep == nil || ancestor == nil {
19+
return
20+
}
21+
status := metav1.ConditionTrue
22+
if reason != egv1a1.EnvoyProxyReasonAccepted {
23+
status = metav1.ConditionFalse
24+
}
25+
26+
cond := newCondition(string(egv1a1.EnvoyProxyConditionAccepted), status,
27+
string(reason), msg, ep.Generation)
28+
29+
for i := range ep.Status.Ancestors {
30+
item := ep.Status.Ancestors[i]
31+
if ancestorRefsEqual(&item.AncestorRef, ancestor) {
32+
ep.Status.Ancestors[i].Conditions = MergeConditions(item.Conditions, cond)
33+
return
34+
}
35+
}
36+
37+
// ancestor not found, append a new one
38+
ep.Status.Ancestors = append(ep.Status.Ancestors, egv1a1.EnvoyProxyAncestorStatus{
39+
AncestorRef: *ancestor,
40+
Conditions: []metav1.Condition{
41+
cond,
42+
},
43+
})
44+
}

0 commit comments

Comments
 (0)