diff --git a/charts/gateway-helm/templates/_rbac.tpl b/charts/gateway-helm/templates/_rbac.tpl index 4b45e2b635..1fa991d700 100644 --- a/charts/gateway-helm/templates/_rbac.tpl +++ b/charts/gateway-helm/templates/_rbac.tpl @@ -100,6 +100,7 @@ verbs: apiGroups: - gateway.envoyproxy.io resources: +- envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/internal/cmd/egctl/testdata/translate/out/default-resources.all.yaml b/internal/cmd/egctl/testdata/translate/out/default-resources.all.yaml index 41153fe438..89d9a4e741 100644 --- a/internal/cmd/egctl/testdata/translate/out/default-resources.all.yaml +++ b/internal/cmd/egctl/testdata/translate/out/default-resources.all.yaml @@ -174,7 +174,18 @@ envoyProxyForGatewayClass: "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig max_active_downstream_connections: 50000 logging: {} - status: {} + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: GatewayClass + name: eg + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted gatewayClass: apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass diff --git a/internal/cmd/egctl/testdata/translate/out/invalid-envoyproxy.all.yaml b/internal/cmd/egctl/testdata/translate/out/invalid-envoyproxy.all.yaml index 59d6d1bda9..846b37958b 100644 --- a/internal/cmd/egctl/testdata/translate/out/invalid-envoyproxy.all.yaml +++ b/internal/cmd/egctl/testdata/translate/out/invalid-envoyproxy.all.yaml @@ -21,7 +21,18 @@ envoyProxyForGatewayClass: logging: level: default: warn - status: {} + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: GatewayClass + name: eg + conditions: + - lastTransitionTime: null + message: dynamic_resources cannot be modified + reason: InvalidParameters + status: "False" + type: Accepted gatewayClass: apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass diff --git a/internal/cmd/egctl/testdata/translate/out/valid-envoyproxy.all.yaml b/internal/cmd/egctl/testdata/translate/out/valid-envoyproxy.all.yaml index 3341f5d606..f0db924237 100644 --- a/internal/cmd/egctl/testdata/translate/out/valid-envoyproxy.all.yaml +++ b/internal/cmd/egctl/testdata/translate/out/valid-envoyproxy.all.yaml @@ -16,7 +16,18 @@ envoyProxyForGatewayClass: externalTrafficPolicy: Local type: LoadBalancer type: Kubernetes - status: {} + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: GatewayClass + name: eg + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted gatewayClass: apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass diff --git a/internal/gatewayapi/contexts.go b/internal/gatewayapi/contexts.go index 96956810bb..c38cff0b02 100644 --- a/internal/gatewayapi/contexts.go +++ b/internal/gatewayapi/contexts.go @@ -29,8 +29,10 @@ import ( type GatewayContext struct { *gwapiv1.Gateway - listeners []*ListenerContext - envoyProxy *egv1a1.EnvoyProxy + listeners []*ListenerContext + envoyProxy *egv1a1.EnvoyProxy + envoyProxyFromGateway bool + backendTLS *egv1a1.BackendTLSConfig } @@ -105,6 +107,7 @@ func (g *GatewayContext) attachEnvoyProxy(resources *resource.Resources, epMap m if string(ref.Group) == egv1a1.GroupVersion.Group && ref.Kind == egv1a1.KindEnvoyProxy { ep, exists := epMap[types.NamespacedName{Namespace: g.Namespace, Name: ref.Name}] if exists { + g.envoyProxyFromGateway = true gatewayProxy = ep } } diff --git a/internal/gatewayapi/runner/runner.go b/internal/gatewayapi/runner/runner.go index e8ed6f5e56..c860589353 100644 --- a/internal/gatewayapi/runner/runner.go +++ b/internal/gatewayapi/runner/runner.go @@ -112,7 +112,10 @@ func mergePolicyStatus(aggregated aggregatedPolicyStatus, incoming *gwapiv1.Poli return aggregated } - aggregated.status.Ancestors = append(aggregated.status.Ancestors, incoming.Ancestors...) + // Prevent self-merge when aggregated and incoming reference the same object + if aggregated.status != incoming { + aggregated.status.Ancestors = append(aggregated.status.Ancestors, incoming.Ancestors...) + } if generation > aggregated.generation { aggregated.generation = generation } @@ -120,6 +123,22 @@ func mergePolicyStatus(aggregated aggregatedPolicyStatus, incoming *gwapiv1.Poli return aggregated } +func mergeEnvoyProxyStatus(aggregated, incoming *egv1a1.EnvoyProxyStatus) *egv1a1.EnvoyProxyStatus { + if incoming == nil { + return aggregated + } + + if aggregated == nil { + return incoming + } + + // Prevent self-merge when aggregated and incoming reference the same object + if aggregated != incoming { + aggregated.Ancestors = append(aggregated.Ancestors, incoming.Ancestors...) + } + return aggregated +} + func New(cfg *Config) *Runner { return &Runner{ Config: *cfg, @@ -206,6 +225,13 @@ func (r *Runner) subscribeAndTranslate(sub <-chan watchable.Snapshot[string, *re // There is only 1 key which is the controller name // so when a delete is triggered, delete all keys if update.Delete || valWrapper == nil || valWrapper.Resources == nil { + // Clear EnvoyProxy statuses before deleting to remove stale ancestor conditions + for key := range r.keyCache.EnvoyProxyStatus { + emptyStatus := &egv1a1.EnvoyProxyStatus{ + Ancestors: []egv1a1.EnvoyProxyAncestorStatus{}, + } + r.ProviderResources.EnvoyProxyStatuses.Store(key, emptyStatus) + } r.deleteAllKeys() return } @@ -229,6 +255,7 @@ func (r *Runner) subscribeAndTranslate(sub <-chan watchable.Snapshot[string, *re var tlsRouteStatusCount, tcpRouteStatusCount, udpRouteStatusCount int var backendTLSPolicyStatusCount, clientTrafficPolicyStatusCount, backendTrafficPolicyStatusCount int var securityPolicyStatusCount, envoyExtensionPolicyStatusCount, backendStatusCount, extensionServerPolicyStatusCount int + var envoyproxyStatusCount int // `aggregatedStatuses` aggregates status result of resources from all // parents/ancestors, and then stores the status once for every resource. @@ -244,6 +271,7 @@ func (r *Runner) subscribeAndTranslate(sub <-chan watchable.Snapshot[string, *re SecurityPolicies map[types.NamespacedName]aggregatedPolicyStatus EnvoyExtensionPolicies map[types.NamespacedName]aggregatedPolicyStatus ExtensionServerPolicies map[message.NamespacedNameAndGVK]aggregatedPolicyStatus + EnvoyProxies map[types.NamespacedName]*egv1a1.EnvoyProxyStatus }{ HTTPRoutes: make(map[types.NamespacedName]aggregatedRouteStatus), GRPCRoutes: make(map[types.NamespacedName]aggregatedRouteStatus), @@ -256,6 +284,7 @@ func (r *Runner) subscribeAndTranslate(sub <-chan watchable.Snapshot[string, *re SecurityPolicies: make(map[types.NamespacedName]aggregatedPolicyStatus), EnvoyExtensionPolicies: make(map[types.NamespacedName]aggregatedPolicyStatus), ExtensionServerPolicies: make(map[message.NamespacedNameAndGVK]aggregatedPolicyStatus), + EnvoyProxies: make(map[types.NamespacedName]*egv1a1.EnvoyProxyStatus), } span.AddEvent("translate", trace.WithAttributes(attribute.Int("resources.count", len(*val)))) @@ -447,6 +476,15 @@ func (r *Runner) subscribeAndTranslate(sub <-chan watchable.Snapshot[string, *re aggregatedStatuses.ExtensionServerPolicies[key] = mergePolicyStatus(aggregatedStatuses.ExtensionServerPolicies[key], &policyStatus, extServerPolicy.GetGeneration()) } } + // EnvoyProxy status + for _, ep := range result.EnvoyProxiesForGateways { + r.Logger.Info("update envoyproxy status", "key", utils.NamespacedName(ep)) + aggregatedStatuses.EnvoyProxies[utils.NamespacedName(ep)] = mergeEnvoyProxyStatus(aggregatedStatuses.EnvoyProxies[utils.NamespacedName(ep)], &ep.Status) + } + if ep := result.EnvoyProxyForGatewayClass; ep != nil { + r.Logger.Info("update envoyproxy status", "key", utils.NamespacedName(ep)) + aggregatedStatuses.EnvoyProxies[utils.NamespacedName(ep)] = mergeEnvoyProxyStatus(aggregatedStatuses.EnvoyProxies[utils.NamespacedName(ep)], &ep.Status) + } statusUpdateSpan.End() } @@ -533,6 +571,15 @@ func (r *Runner) subscribeAndTranslate(sub <-chan watchable.Snapshot[string, *re delete(keysToDelete.ExtensionServerPolicyStatus, key) r.keyCache.ExtensionServerPolicyStatus[key] = true } + for key, entry := range aggregatedStatuses.EnvoyProxies { + s := egv1a1.EnvoyProxyStatus{ + Ancestors: entry.Ancestors, + } + r.ProviderResources.EnvoyProxyStatuses.Store(key, &s) + envoyproxyStatusCount++ + delete(keysToDelete.EnvoyProxyStatus, key) + r.keyCache.EnvoyProxyStatus[key] = true + } // Publish aggregated metrics message.PublishMetric(message.Metadata{Runner: r.Name(), Message: message.InfraIRMessageName}, infraIRCount) message.PublishMetric(message.Metadata{Runner: r.Name(), Message: message.XDSIRMessageName}, xdsIRCount) @@ -551,6 +598,7 @@ func (r *Runner) subscribeAndTranslate(sub <-chan watchable.Snapshot[string, *re message.PublishMetric(message.Metadata{Runner: r.Name(), Message: message.EnvoyExtensionPolicyStatusMessageName}, envoyExtensionPolicyStatusCount) message.PublishMetric(message.Metadata{Runner: r.Name(), Message: message.BackendStatusMessageName}, backendStatusCount) message.PublishMetric(message.Metadata{Runner: r.Name(), Message: message.ExtensionServerPoliciesStatusMessageName}, extensionServerPolicyStatusCount) + message.PublishMetric(message.Metadata{Runner: r.Name(), Message: message.EnvoyProxyStatusMessageName}, envoyproxyStatusCount) // Delete keys using mark and sweep r.deleteKeys(keysToDelete) @@ -660,6 +708,9 @@ func (r *Runner) deleteAllKeys() { for key := range r.keyCache.BackendStatus { r.ProviderResources.BackendStatuses.Delete(key) } + for key := range r.keyCache.EnvoyProxyStatus { + r.ProviderResources.EnvoyProxyStatuses.Delete(key) + } // Clear all tracking r.keyCache = newKeyCache() @@ -684,6 +735,7 @@ type KeyCache struct { SecurityPolicyStatus map[types.NamespacedName]bool EnvoyExtensionPolicyStatus map[types.NamespacedName]bool ExtensionServerPolicyStatus map[message.NamespacedNameAndGVK]bool + EnvoyProxyStatus map[types.NamespacedName]bool BackendStatus map[types.NamespacedName]bool } @@ -737,6 +789,9 @@ func (kc *KeyCache) copy() *KeyCache { for key := range kc.ExtensionServerPolicyStatus { copied.ExtensionServerPolicyStatus[key] = true } + for key := range kc.EnvoyProxyStatus { + copied.EnvoyProxyStatus[key] = true + } for key := range kc.BackendStatus { copied.BackendStatus[key] = true } @@ -760,6 +815,7 @@ func newKeyCache() *KeyCache { SecurityPolicyStatus: make(map[types.NamespacedName]bool), EnvoyExtensionPolicyStatus: make(map[types.NamespacedName]bool), ExtensionServerPolicyStatus: make(map[message.NamespacedNameAndGVK]bool), + EnvoyProxyStatus: make(map[types.NamespacedName]bool), BackendStatus: make(map[types.NamespacedName]bool), } } @@ -812,6 +868,9 @@ func (r *Runner) populateKeyCache() { for key := range r.ProviderResources.ExtensionPolicyStatuses.LoadAll() { r.keyCache.ExtensionServerPolicyStatus[key] = true } + for key := range r.ProviderResources.EnvoyProxyStatuses.LoadAll() { + r.keyCache.EnvoyProxyStatus[key] = true + } for key := range r.ProviderResources.BackendStatuses.LoadAll() { r.keyCache.BackendStatus[key] = true } @@ -879,6 +938,10 @@ func (r *Runner) deleteKeys(kc *KeyCache) { r.ProviderResources.ExtensionPolicyStatuses.Delete(key) delete(r.keyCache.ExtensionServerPolicyStatus, key) } + for key := range kc.EnvoyProxyStatus { + r.ProviderResources.EnvoyProxyStatuses.Delete(key) + delete(r.keyCache.EnvoyProxyStatus, key) + } for key := range kc.BackendStatus { r.ProviderResources.BackendStatuses.Delete(key) delete(r.keyCache.BackendStatus, key) diff --git a/internal/gatewayapi/runner/runner_test.go b/internal/gatewayapi/runner/runner_test.go index 945d3b7499..b85eb102a3 100644 --- a/internal/gatewayapi/runner/runner_test.go +++ b/internal/gatewayapi/runner/runner_test.go @@ -304,6 +304,94 @@ func TestMergePolicyStatus(t *testing.T) { entry = mergePolicyStatus(entry, &gwapiv1.PolicyStatus{}, 4) require.Equal(t, int64(9), entry.generation) }) + + t.Run("self-merge prevention", func(t *testing.T) { + status := &gwapiv1.PolicyStatus{ + Ancestors: []gwapiv1.PolicyAncestorStatus{ + { + AncestorRef: gwapiv1.ParentReference{Name: gwapiv1.ObjectName("gw-a")}, + ControllerName: gwapiv1a2.GatewayController(controllerName), + }, + }, + } + + // First merge - status becomes both aggregated and incoming + entry := mergePolicyStatus(aggregatedPolicyStatus{}, status, 1) + // Second merge with same status - should not duplicate + entry = mergePolicyStatus(entry, status, 2) + + // Should still have only one ancestor, not duplicated + require.Len(t, entry.status.Ancestors, 1) + require.Equal(t, gwapiv1.ObjectName("gw-a"), entry.status.Ancestors[0].AncestorRef.Name) + }) +} + +func TestMergeEnvoyProxyStatus(t *testing.T) { + t.Run("nil incoming keeps existing entry", func(t *testing.T) { + existing := &egv1a1.EnvoyProxyStatus{ + Ancestors: []egv1a1.EnvoyProxyAncestorStatus{ + { + AncestorRef: gwapiv1.ParentReference{Name: gwapiv1.ObjectName("gc-a")}, + }, + }, + } + + got := mergeEnvoyProxyStatus(existing, nil) + require.Same(t, existing, got) + }) + + t.Run("nil existing takes incoming", func(t *testing.T) { + incoming := &egv1a1.EnvoyProxyStatus{ + Ancestors: []egv1a1.EnvoyProxyAncestorStatus{ + { + AncestorRef: gwapiv1.ParentReference{Name: gwapiv1.ObjectName("gc-a")}, + }, + }, + } + + got := mergeEnvoyProxyStatus(nil, incoming) + require.Same(t, incoming, got) + }) + + t.Run("appends ancestors", func(t *testing.T) { + first := &egv1a1.EnvoyProxyStatus{ + Ancestors: []egv1a1.EnvoyProxyAncestorStatus{ + { + AncestorRef: gwapiv1.ParentReference{Name: gwapiv1.ObjectName("gc-a")}, + }, + }, + } + second := &egv1a1.EnvoyProxyStatus{ + Ancestors: []egv1a1.EnvoyProxyAncestorStatus{ + { + AncestorRef: gwapiv1.ParentReference{Name: gwapiv1.ObjectName("gw-b")}, + }, + }, + } + + got := mergeEnvoyProxyStatus(first, second) + + require.Len(t, got.Ancestors, 2) + require.Equal(t, gwapiv1.ObjectName("gc-a"), got.Ancestors[0].AncestorRef.Name) + require.Equal(t, gwapiv1.ObjectName("gw-b"), got.Ancestors[1].AncestorRef.Name) + }) + + t.Run("self-merge prevention", func(t *testing.T) { + status := &egv1a1.EnvoyProxyStatus{ + Ancestors: []egv1a1.EnvoyProxyAncestorStatus{ + { + AncestorRef: gwapiv1.ParentReference{Name: gwapiv1.ObjectName("gc-a")}, + }, + }, + } + + // Merge status with itself - should not duplicate + got := mergeEnvoyProxyStatus(status, status) + + // Should still have only one ancestor, not duplicated + require.Len(t, got.Ancestors, 1) + require.Equal(t, gwapiv1.ObjectName("gc-a"), got.Ancestors[0].AncestorRef.Name) + }) } func TestMergeRouteStatus(t *testing.T) { diff --git a/internal/gatewayapi/status/envoyproxy.go b/internal/gatewayapi/status/envoyproxy.go new file mode 100644 index 0000000000..9f942d156f --- /dev/null +++ b/internal/gatewayapi/status/envoyproxy.go @@ -0,0 +1,44 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package status + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" +) + +func UpdateEnvoyProxyStatusAccepted(ep *egv1a1.EnvoyProxy, ancestor *gwapiv1.ParentReference, + reason egv1a1.EnvoyProxyConditionReason, msg string, +) { + if ep == nil || ancestor == nil { + return + } + status := metav1.ConditionTrue + if reason != egv1a1.EnvoyProxyReasonAccepted { + status = metav1.ConditionFalse + } + + cond := newCondition(string(egv1a1.EnvoyProxyConditionAccepted), status, + string(reason), msg, ep.Generation) + + for i := range ep.Status.Ancestors { + item := ep.Status.Ancestors[i] + if ancestorRefsEqual(&item.AncestorRef, ancestor) { + ep.Status.Ancestors[i].Conditions = MergeConditions(item.Conditions, cond) + return + } + } + + // ancestor not found, append a new one + ep.Status.Ancestors = append(ep.Status.Ancestors, egv1a1.EnvoyProxyAncestorStatus{ + AncestorRef: *ancestor, + Conditions: []metav1.Condition{ + cond, + }, + }) +} diff --git a/internal/gatewayapi/testdata/envoyextensionpolicy-with-dynamicmodule.out.yaml b/internal/gatewayapi/testdata/envoyextensionpolicy-with-dynamicmodule.out.yaml index 9120fc82c4..1eede11cad 100644 --- a/internal/gatewayapi/testdata/envoyextensionpolicy-with-dynamicmodule.out.yaml +++ b/internal/gatewayapi/testdata/envoyextensionpolicy-with-dynamicmodule.out.yaml @@ -80,6 +80,50 @@ envoyExtensionPolicies: status: "True" type: Overridden controllerName: gateway.envoyproxy.io/gatewayclass-controller +envoyProxiesForGateways: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + name: proxy-config + namespace: envoy-gateway + spec: + dynamicModules: + - name: my-auth-module + source: + local: + path: /usr/lib/envoy/modules/my_auth.so + type: Local + - name: remote-auth-module + source: + remote: + sha256: abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789 + url: https://modules.example.com/libremote_auth.so + type: Remote + - doNotClose: true + loadGlobally: true + name: ai-gateway-filter + source: + local: + path: /usr/lib/envoy/modules/ai_gateway.so + - name: path-loaded-module + source: + local: + path: /opt/modules/my_module.so + type: Local + logging: {} + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted gateways: - apiVersion: gateway.networking.k8s.io/v1 kind: Gateway @@ -234,7 +278,19 @@ infraIR: path: /opt/modules/my_module.so type: Local logging: {} - status: {} + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted listeners: - name: envoy-gateway/gateway-1/http ports: diff --git a/internal/gatewayapi/testdata/envoyextensionpolicy-with-invalid-dynamicmodule.out.yaml b/internal/gatewayapi/testdata/envoyextensionpolicy-with-invalid-dynamicmodule.out.yaml index 6d37e2e10f..7f24d96cf0 100644 --- a/internal/gatewayapi/testdata/envoyextensionpolicy-with-invalid-dynamicmodule.out.yaml +++ b/internal/gatewayapi/testdata/envoyextensionpolicy-with-invalid-dynamicmodule.out.yaml @@ -34,6 +34,32 @@ envoyExtensionPolicies: status: "True" type: Warning controllerName: gateway.envoyproxy.io/gatewayclass-controller +envoyProxiesForGateways: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + name: proxy-config + namespace: envoy-gateway + spec: + dynamicModules: + - name: allowed-module + source: + local: + path: /opt/modules/allowed.so + logging: {} + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted gateways: - apiVersion: gateway.networking.k8s.io/v1 kind: Gateway @@ -133,7 +159,19 @@ infraIR: local: path: /opt/modules/allowed.so logging: {} - status: {} + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted listeners: - name: envoy-gateway/gateway-1/http ports: diff --git a/internal/gatewayapi/testdata/envoyproxy-accesslog-file-json-no-format.out.yaml b/internal/gatewayapi/testdata/envoyproxy-accesslog-file-json-no-format.out.yaml index 378c3fe00e..4a6906ad2d 100644 --- a/internal/gatewayapi/testdata/envoyproxy-accesslog-file-json-no-format.out.yaml +++ b/internal/gatewayapi/testdata/envoyproxy-accesslog-file-json-no-format.out.yaml @@ -1,3 +1,80 @@ +envoyProxyForGatewayClass: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + name: test + namespace: envoy-gateway-system + spec: + logging: {} + provider: + kubernetes: + envoyDeployment: + container: + env: + - name: env_a + value: env_a_value + - name: env_b + value: env_b_name + image: envoyproxy/envoy:distroless-dev + resources: + requests: + cpu: 100m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + runAsUser: 2000 + pod: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cloud.google.com/gke-nodepool + operator: In + values: + - router-node + annotations: + key1: val1 + key2: val2 + securityContext: + fsGroup: 2000 + fsGroupChangePolicy: OnRootMismatch + runAsGroup: 3000 + runAsUser: 1000 + tolerations: + - effect: NoSchedule + key: node-type + operator: Exists + value: router + volumes: + - name: certs + secret: + secretName: envoy-cert + replicas: 2 + envoyService: + type: LoadBalancer + type: Kubernetes + telemetry: + accessLog: + settings: + - format: + type: JSON + sinks: + - file: + path: /dev/stdout + type: File + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: GatewayClass + name: envoy-gateway-class + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted gatewayClass: apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass @@ -125,7 +202,18 @@ infraIR: - file: path: /dev/stdout type: File - status: {} + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: GatewayClass + name: envoy-gateway-class + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted listeners: - name: envoy-gateway/gateway-1/http ports: diff --git a/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.out.yaml b/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.out.yaml index 525659b83f..d62455510b 100644 --- a/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.out.yaml +++ b/internal/gatewayapi/testdata/envoyproxy-accesslog-with-bad-sinks.out.yaml @@ -1,3 +1,84 @@ +envoyProxyForGatewayClass: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + name: test + namespace: envoy-gateway-system + spec: + logging: {} + provider: + kubernetes: + envoyDeployment: + container: + env: + - name: env_a + value: env_a_value + - name: env_b + value: env_b_name + image: envoyproxy/envoy:distroless-dev + resources: + requests: + cpu: 100m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + runAsUser: 2000 + pod: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cloud.google.com/gke-nodepool + operator: In + values: + - router-node + annotations: + key1: val1 + key2: val2 + securityContext: + fsGroup: 2000 + fsGroupChangePolicy: OnRootMismatch + runAsGroup: 3000 + runAsUser: 1000 + tolerations: + - effect: NoSchedule + key: node-type + operator: Exists + value: router + volumes: + - name: certs + secret: + secretName: envoy-cert + replicas: 2 + envoyService: + type: LoadBalancer + type: Kubernetes + telemetry: + accessLog: + settings: + - format: + text: | + [%START_TIME%] "%REQ(:METHOD)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"\n + type: Text + sinks: + - type: File + - type: ALS + - type: OpenTelemetry + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: GatewayClass + name: envoy-gateway-class + conditions: + - lastTransitionTime: null + message: '[unable to configure access log when using File sink type but "file" + field being empty, unable to configure access log when using OpenTelemetry + sink type but "openTelemetry" field being empty]' + reason: InvalidParameters + status: "False" + type: Accepted gatewayClass: apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass diff --git a/internal/gatewayapi/testdata/envoyproxy-endpoint-routing-for-gateway.out.yaml b/internal/gatewayapi/testdata/envoyproxy-endpoint-routing-for-gateway.out.yaml index 918f7822ef..43d38f6260 100644 --- a/internal/gatewayapi/testdata/envoyproxy-endpoint-routing-for-gateway.out.yaml +++ b/internal/gatewayapi/testdata/envoyproxy-endpoint-routing-for-gateway.out.yaml @@ -1,3 +1,25 @@ +envoyProxiesForGateways: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + name: test + namespace: envoy-gateway + spec: + logging: {} + routingType: Endpoint + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted gateways: - apiVersion: gateway.networking.k8s.io/v1 kind: Gateway @@ -89,7 +111,19 @@ infraIR: spec: logging: {} routingType: Endpoint - status: {} + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted listeners: - name: envoy-gateway/gateway-1/http ports: diff --git a/internal/gatewayapi/testdata/envoyproxy-for-gatewayclass.out.yaml b/internal/gatewayapi/testdata/envoyproxy-for-gatewayclass.out.yaml index e77ec4393b..cdb365ef4c 100644 --- a/internal/gatewayapi/testdata/envoyproxy-for-gatewayclass.out.yaml +++ b/internal/gatewayapi/testdata/envoyproxy-for-gatewayclass.out.yaml @@ -1,3 +1,68 @@ +envoyProxiesForGateways: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + name: valid + namespace: envoy-gateway + spec: + logging: {} + telemetry: + accessLog: + settings: + - format: + text: | + [%START_TIME%] "%REQ(:METHOD)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"\n + type: Text + sinks: + - file: + path: /dev/stdout + type: File + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: Gateway + name: ep-from-gtw + namespace: envoy-gateway + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted +envoyProxyForGatewayClass: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + name: invalid + namespace: envoy-gateway + spec: + logging: {} + telemetry: + accessLog: + settings: + - format: + text: | + [%START_TIME%] "%REQ(:METHOD)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"\n + type: Text + sinks: + - type: File + - type: ALS + - type: OpenTelemetry + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: GatewayClass + name: envoy-gateway-class + conditions: + - lastTransitionTime: null + message: '[unable to configure access log when using File sink type but "file" + field being empty, unable to configure access log when using OpenTelemetry + sink type but "openTelemetry" field being empty]' + reason: InvalidParameters + status: "False" + type: Accepted gatewayClass: apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass @@ -120,7 +185,19 @@ infraIR: - file: path: /dev/stdout type: File - status: {} + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: Gateway + name: ep-from-gtw + namespace: envoy-gateway + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted listeners: - name: envoy-gateway/ep-from-gtw/http ports: diff --git a/internal/gatewayapi/testdata/envoyproxy-gateway-accesslog-with-bad-sinks.out.yaml b/internal/gatewayapi/testdata/envoyproxy-gateway-accesslog-with-bad-sinks.out.yaml index 7b78a40a65..14d587a705 100644 --- a/internal/gatewayapi/testdata/envoyproxy-gateway-accesslog-with-bad-sinks.out.yaml +++ b/internal/gatewayapi/testdata/envoyproxy-gateway-accesslog-with-bad-sinks.out.yaml @@ -1,3 +1,85 @@ +envoyProxiesForGateways: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + name: test + namespace: envoy-gateway + spec: + logging: {} + provider: + kubernetes: + envoyDeployment: + container: + env: + - name: env_a + value: env_a_value + - name: env_b + value: env_b_name + image: envoyproxy/envoy:distroless-dev + resources: + requests: + cpu: 100m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + runAsUser: 2000 + pod: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: cloud.google.com/gke-nodepool + operator: In + values: + - router-node + annotations: + key1: val1 + key2: val2 + securityContext: + fsGroup: 2000 + fsGroupChangePolicy: OnRootMismatch + runAsGroup: 3000 + runAsUser: 1000 + tolerations: + - effect: NoSchedule + key: node-type + operator: Exists + value: router + volumes: + - name: certs + secret: + secretName: envoy-cert + replicas: 2 + envoyService: + type: LoadBalancer + type: Kubernetes + telemetry: + accessLog: + settings: + - format: + text: | + [%START_TIME%] "%REQ(:METHOD)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"\n + type: Text + sinks: + - type: File + - type: ALS + - type: OpenTelemetry + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + conditions: + - lastTransitionTime: null + message: '[unable to configure access log when using File sink type but "file" + field being empty, unable to configure access log when using OpenTelemetry + sink type but "openTelemetry" field being empty]' + reason: InvalidParameters + status: "False" + type: Accepted gateways: - apiVersion: gateway.networking.k8s.io/v1 kind: Gateway diff --git a/internal/gatewayapi/testdata/envoyproxy-metric-grpc.out.yaml b/internal/gatewayapi/testdata/envoyproxy-metric-grpc.out.yaml index 4939b6ee67..ad426eadd2 100644 --- a/internal/gatewayapi/testdata/envoyproxy-metric-grpc.out.yaml +++ b/internal/gatewayapi/testdata/envoyproxy-metric-grpc.out.yaml @@ -1,3 +1,50 @@ +envoyProxiesForGateways: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + name: override-disable + namespace: envoy-gateway + spec: + logging: {} + telemetry: + metrics: + enableGRPCStats: false + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + name: enable + namespace: envoy-gateway + spec: + logging: {} + telemetry: + metrics: + enableGRPCStats: true + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-2 + namespace: envoy-gateway + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted gateways: - apiVersion: gateway.networking.k8s.io/v1 kind: Gateway @@ -134,7 +181,19 @@ infraIR: telemetry: metrics: enableGRPCStats: false - status: {} + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted listeners: - name: envoy-gateway/gateway-1/http ports: @@ -164,7 +223,19 @@ infraIR: telemetry: metrics: enableGRPCStats: true - status: {} + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-2 + namespace: envoy-gateway + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted listeners: - name: envoy-gateway/gateway-2/http ports: diff --git a/internal/gatewayapi/testdata/envoyproxy-service-routing-for-gateway.out.yaml b/internal/gatewayapi/testdata/envoyproxy-service-routing-for-gateway.out.yaml index 9df65abd1b..2bdf941de3 100644 --- a/internal/gatewayapi/testdata/envoyproxy-service-routing-for-gateway.out.yaml +++ b/internal/gatewayapi/testdata/envoyproxy-service-routing-for-gateway.out.yaml @@ -1,3 +1,25 @@ +envoyProxiesForGateways: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + name: test + namespace: envoy-gateway + spec: + logging: {} + routingType: Service + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted gateways: - apiVersion: gateway.networking.k8s.io/v1 kind: Gateway @@ -89,7 +111,19 @@ infraIR: spec: logging: {} routingType: Service - status: {} + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted listeners: - name: envoy-gateway/gateway-1/http ports: diff --git a/internal/gatewayapi/testdata/gateway-with-infrastructure-parametersref.out.yaml b/internal/gatewayapi/testdata/gateway-with-infrastructure-parametersref.out.yaml index da9e436489..964c4a6418 100644 --- a/internal/gatewayapi/testdata/gateway-with-infrastructure-parametersref.out.yaml +++ b/internal/gatewayapi/testdata/gateway-with-infrastructure-parametersref.out.yaml @@ -1,3 +1,31 @@ +envoyProxiesForGateways: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + name: test + namespace: envoy-gateway + spec: + logging: {} + telemetry: + tracing: + provider: + host: otel-collector.monitoring.svc.cluster.local + port: 4317 + type: OpenTelemetry + samplingRate: 100 + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted gateways: - apiVersion: gateway.networking.k8s.io/v1beta1 kind: Gateway @@ -100,7 +128,19 @@ infraIR: port: 4317 type: OpenTelemetry samplingRate: 100 - status: {} + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted listeners: - name: envoy-gateway/gateway-1/http ports: diff --git a/internal/gatewayapi/testdata/securitypolicy-with-authorization-geoip-invalid-no-client-ip-detection.out.yaml b/internal/gatewayapi/testdata/securitypolicy-with-authorization-geoip-invalid-no-client-ip-detection.out.yaml index ef34db506d..68de904e34 100644 --- a/internal/gatewayapi/testdata/securitypolicy-with-authorization-geoip-invalid-no-client-ip-detection.out.yaml +++ b/internal/gatewayapi/testdata/securitypolicy-with-authorization-geoip-invalid-no-client-ip-detection.out.yaml @@ -1,3 +1,33 @@ +envoyProxyForGatewayClass: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + name: geoip-proxy + namespace: envoy-gateway + spec: + geoIP: + provider: + maxMind: + anonymousIpDbSource: + local: + path: /db/GeoIP2-Anonymous-IP.mmdb + cityDbSource: + local: + path: /db/GeoLite2-City.mmdb + type: MaxMind + logging: {} + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: GatewayClass + name: envoy-gateway-class + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted gatewayClass: apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass @@ -118,7 +148,18 @@ infraIR: path: /db/GeoLite2-City.mmdb type: MaxMind logging: {} - status: {} + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: GatewayClass + name: envoy-gateway-class + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted listeners: - name: envoy-gateway/gateway-1/http ports: diff --git a/internal/gatewayapi/testdata/securitypolicy-with-authorization-geoip-invalid-no-provider.out.yaml b/internal/gatewayapi/testdata/securitypolicy-with-authorization-geoip-invalid-no-provider.out.yaml index 7662d718ed..8a9d89a457 100644 --- a/internal/gatewayapi/testdata/securitypolicy-with-authorization-geoip-invalid-no-provider.out.yaml +++ b/internal/gatewayapi/testdata/securitypolicy-with-authorization-geoip-invalid-no-provider.out.yaml @@ -27,6 +27,26 @@ clientTrafficPolicies: status: "True" type: Accepted controllerName: gateway.envoyproxy.io/gatewayclass-controller +envoyProxyForGatewayClass: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + name: geoip-proxy + namespace: default + spec: + logging: {} + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: GatewayClass + name: envoy-gateway-class + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted gatewayClass: apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass @@ -135,7 +155,18 @@ infraIR: namespace: default spec: logging: {} - status: {} + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: GatewayClass + name: envoy-gateway-class + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted listeners: - name: default/gateway-1/http ports: diff --git a/internal/gatewayapi/testdata/securitypolicy-with-authorization-geoip-invalid-trusted-cidrs.out.yaml b/internal/gatewayapi/testdata/securitypolicy-with-authorization-geoip-invalid-trusted-cidrs.out.yaml index 7a91cda31b..6dbf3a13b2 100644 --- a/internal/gatewayapi/testdata/securitypolicy-with-authorization-geoip-invalid-trusted-cidrs.out.yaml +++ b/internal/gatewayapi/testdata/securitypolicy-with-authorization-geoip-invalid-trusted-cidrs.out.yaml @@ -27,6 +27,36 @@ clientTrafficPolicies: status: "True" type: Accepted controllerName: gateway.envoyproxy.io/gatewayclass-controller +envoyProxyForGatewayClass: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + name: geoip-proxy + namespace: default + spec: + geoIP: + provider: + maxMind: + anonymousIpDbSource: + local: + path: /db/GeoIP2-Anonymous-IP.mmdb + cityDbSource: + local: + path: /db/GeoLite2-City.mmdb + type: MaxMind + logging: {} + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: GatewayClass + name: envoy-gateway-class + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted gatewayClass: apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass @@ -145,7 +175,18 @@ infraIR: path: /db/GeoLite2-City.mmdb type: MaxMind logging: {} - status: {} + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: GatewayClass + name: envoy-gateway-class + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted listeners: - name: default/gateway-1/http ports: diff --git a/internal/gatewayapi/testdata/securitypolicy-with-authorization-geoip-mixed-valid-invalid-route-target.out.yaml b/internal/gatewayapi/testdata/securitypolicy-with-authorization-geoip-mixed-valid-invalid-route-target.out.yaml index 580a4fc98a..5e344ec166 100644 --- a/internal/gatewayapi/testdata/securitypolicy-with-authorization-geoip-mixed-valid-invalid-route-target.out.yaml +++ b/internal/gatewayapi/testdata/securitypolicy-with-authorization-geoip-mixed-valid-invalid-route-target.out.yaml @@ -29,6 +29,36 @@ clientTrafficPolicies: status: "True" type: Accepted controllerName: gateway.envoyproxy.io/gatewayclass-controller +envoyProxyForGatewayClass: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + name: geoip-proxy + namespace: default + spec: + geoIP: + provider: + maxMind: + anonymousIpDbSource: + local: + path: /db/GeoIP2-Anonymous-IP.mmdb + cityDbSource: + local: + path: /db/GeoLite2-City.mmdb + type: MaxMind + logging: {} + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: GatewayClass + name: envoy-gateway-class + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted gatewayClass: apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass @@ -213,7 +243,18 @@ infraIR: path: /db/GeoLite2-City.mmdb type: MaxMind logging: {} - status: {} + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: GatewayClass + name: envoy-gateway-class + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted listeners: - name: default/gateway-1/http-valid ports: diff --git a/internal/gatewayapi/testdata/securitypolicy-with-authorization-geoip-mixed-valid-invalid.out.yaml b/internal/gatewayapi/testdata/securitypolicy-with-authorization-geoip-mixed-valid-invalid.out.yaml index 9fc115c1f4..a5651b6833 100644 --- a/internal/gatewayapi/testdata/securitypolicy-with-authorization-geoip-mixed-valid-invalid.out.yaml +++ b/internal/gatewayapi/testdata/securitypolicy-with-authorization-geoip-mixed-valid-invalid.out.yaml @@ -29,6 +29,36 @@ clientTrafficPolicies: status: "True" type: Accepted controllerName: gateway.envoyproxy.io/gatewayclass-controller +envoyProxyForGatewayClass: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + name: geoip-proxy + namespace: default + spec: + geoIP: + provider: + maxMind: + anonymousIpDbSource: + local: + path: /db/GeoIP2-Anonymous-IP.mmdb + cityDbSource: + local: + path: /db/GeoLite2-City.mmdb + type: MaxMind + logging: {} + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: GatewayClass + name: envoy-gateway-class + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted gatewayClass: apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass @@ -213,7 +243,18 @@ infraIR: path: /db/GeoLite2-City.mmdb type: MaxMind logging: {} - status: {} + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: GatewayClass + name: envoy-gateway-class + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted listeners: - name: default/gateway-1/http-valid ports: diff --git a/internal/gatewayapi/testdata/securitypolicy-with-authorization-geoip.out.yaml b/internal/gatewayapi/testdata/securitypolicy-with-authorization-geoip.out.yaml index 36713c98a3..e653d5fc76 100644 --- a/internal/gatewayapi/testdata/securitypolicy-with-authorization-geoip.out.yaml +++ b/internal/gatewayapi/testdata/securitypolicy-with-authorization-geoip.out.yaml @@ -27,6 +27,36 @@ clientTrafficPolicies: status: "True" type: Accepted controllerName: gateway.envoyproxy.io/gatewayclass-controller +envoyProxyForGatewayClass: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + name: geoip-proxy + namespace: default + spec: + geoIP: + provider: + maxMind: + anonymousIpDbSource: + local: + path: /db/GeoIP2-Anonymous-IP.mmdb + cityDbSource: + local: + path: /db/GeoLite2-City.mmdb + type: MaxMind + logging: {} + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: GatewayClass + name: envoy-gateway-class + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted gatewayClass: apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass @@ -182,7 +212,18 @@ infraIR: path: /db/GeoLite2-City.mmdb type: MaxMind logging: {} - status: {} + status: + ancestors: + - ancestorRef: + group: gateway.networking.k8s.io + kind: GatewayClass + name: envoy-gateway-class + conditions: + - lastTransitionTime: null + message: EnvoyProxy has been accepted. + reason: Accepted + status: "True" + type: Accepted listeners: - name: default/gateway-1/http ports: diff --git a/internal/gatewayapi/translator.go b/internal/gatewayapi/translator.go index d87172bdcf..eae1551cfe 100644 --- a/internal/gatewayapi/translator.go +++ b/internal/gatewayapi/translator.go @@ -14,6 +14,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" @@ -139,6 +140,7 @@ type TranslateResult struct { func newTranslateResult( gc *gwapiv1.GatewayClass, + envoyProxyForGC *egv1a1.EnvoyProxy, gateways []*GatewayContext, httpRoutes []*HTTPRouteContext, grpcRoutes []*GRPCRouteContext, @@ -162,11 +164,24 @@ func newTranslateResult( } translateResult.GatewayClass = gc - + if gc != nil { + // this won't happen in real world, just a safety check to make sure the testdata won't be a mess. + translateResult.EnvoyProxyForGatewayClass = envoyProxyForGC + } + epProxiesSet := sets.New[types.NamespacedName]() if n := len(gateways); n > 0 { translateResult.Gateways = make([]*gwapiv1.Gateway, n) for i, gateway := range gateways { translateResult.Gateways[i] = gateway.Gateway + // If the EnvoyProxy is from the Gateway itself, add it to the result so that the status can be updated in the provider layer. + // If it's inherited from GatewayClass, it should be handled in EnvoyProxyForGatewayClass(L164). + if gateway.envoyProxyFromGateway && gateway.envoyProxy != nil { + epKey := utils.NamespacedName(gateway.envoyProxy) + if !epProxiesSet.Has(epKey) { + translateResult.EnvoyProxiesForGateways = append(translateResult.EnvoyProxiesForGateways, gateway.envoyProxy) + epProxiesSet.Insert(epKey) + } + } } } @@ -376,6 +391,7 @@ func (t *Translator) Translate(resources *resource.Resources) (*TranslateResult, allGateways = append(allGateways, failedGateways...) return newTranslateResult(resources.GatewayClass, + resources.EnvoyProxyForGatewayClass, allGateways, httpRoutes, grpcRoutes, tlsRoutes, tcpRoutes, udpRoutes, clientTrafficPolicies, backendTrafficPolicies, securityPolicies, resources.BackendTLSPolicies, envoyExtensionPolicies, @@ -402,6 +418,16 @@ func (t *Translator) GetRelevantGateways(resources *resource.Resources) ( // if EnvoyProxy found but invalid, set GC status to not accepted, // otherwise set GC status to accepted. if ep := resources.EnvoyProxyForGatewayClass; ep != nil { + var ancestor *gwapiv1.ParentReference + // TODO: remove this nil check after we update all the testdata. + if resources.GatewayClass != nil { + ancestor = &gwapiv1.ParentReference{ + Group: GroupPtr(gwapiv1.GroupName), + Kind: KindPtr(resource.KindGatewayClass), + Name: gwapiv1.ObjectName(resources.GatewayClass.Name), + } + } + err := validateEnvoyProxy(ep) if err != nil { envoyproxyValidationErrorMap[utils.NamespacedName(ep)] = err @@ -411,6 +437,9 @@ func (t *Translator) GetRelevantGateways(resources *resource.Resources) ( status.SetGatewayClassAccepted(resources.GatewayClass, false, string(gwapiv1.GatewayClassReasonInvalidParameters), fmt.Sprintf("%s: %v", status.MsgGatewayClassInvalidParams, err)) + + status.UpdateEnvoyProxyStatusAccepted(ep, ancestor, + egv1a1.EnvoyProxyReasonInvalidParameters, err.Error()) } else { // TODO: remove this nil check after we update all the testdata. if resources.GatewayClass != nil { @@ -424,6 +453,9 @@ func (t *Translator) GetRelevantGateways(resources *resource.Resources) ( key := utils.NamespacedName(ep) envoyproxyMap[key] = ep // we didn't append to envoyproxyValidatioErrorMap because it's valid. + + status.UpdateEnvoyProxyStatusAccepted(ep, ancestor, + egv1a1.EnvoyProxyReasonAccepted, "EnvoyProxy has been accepted.") } } @@ -462,6 +494,18 @@ func (t *Translator) GetRelevantGateways(resources *resource.Resources) ( continue } + var ancestor *gwapiv1.ParentReference + if gCtx.envoyProxyFromGateway { + // didn't need to update EnvoyProxy status if it's inherited + // from GatewayClass/EnvoyGateway. + ancestor = &gwapiv1.ParentReference{ + Group: GroupPtr(gwapiv1.GroupName), + Kind: KindPtr(resource.KindGateway), + Name: gwapiv1.ObjectName(gateway.Name), + Namespace: NamespacePtr(gateway.Namespace), + } + } + if ep := gCtx.envoyProxy; ep != nil { key := utils.NamespacedName(ep) if err, exits := envoyproxyValidationErrorMap[key]; exits { @@ -469,8 +513,18 @@ func (t *Translator) GetRelevantGateways(resources *resource.Resources) ( t.Logger.Info("EnvoyProxy for Gateway invalid", logKeysAndValues...) status.UpdateGatewayStatusNotAccepted(gCtx.Gateway, gwapiv1.GatewayReasonInvalidParameters, fmt.Sprintf("%s: %v", "Invalid parametersRef:", err.Error())) + + if gCtx.envoyProxyFromGateway { + status.UpdateEnvoyProxyStatusAccepted(ep, ancestor, + egv1a1.EnvoyProxyReasonInvalidParameters, err.Error()) + } continue } + + if gCtx.envoyProxyFromGateway { + status.UpdateEnvoyProxyStatusAccepted(ep, ancestor, + egv1a1.EnvoyProxyReasonAccepted, "EnvoyProxy has been accepted.") + } } if addrType := unsupportedAddressType(gateway); addrType != nil { diff --git a/internal/message/types.go b/internal/message/types.go index 836aa30b3b..10d2317ec6 100644 --- a/internal/message/types.go +++ b/internal/message/types.go @@ -115,6 +115,8 @@ type PolicyStatuses struct { BackendTLSPolicyStatuses watchable.Map[types.NamespacedName, *gwapiv1.PolicyStatus] EnvoyExtensionPolicyStatuses watchable.Map[types.NamespacedName, *gwapiv1.PolicyStatus] ExtensionPolicyStatuses watchable.Map[NamespacedNameAndGVK, *gwapiv1.PolicyStatus] + + EnvoyProxyStatuses watchable.Map[types.NamespacedName, *egv1a1.EnvoyProxyStatus] } // ExtensionStatuses contains statuses related to gw-api extension resources @@ -130,6 +132,7 @@ func (p *PolicyStatuses) Close() { p.BackendTLSPolicyStatuses.Close() p.EnvoyExtensionPolicyStatuses.Close() p.ExtensionPolicyStatuses.Close() + p.EnvoyProxyStatuses.Close() } func (e *ExtensionStatuses) Close() { @@ -225,4 +228,6 @@ const ( ListenerSetStatusMessageName MessageName = "listenerset-status" // GatewayClassStatusMessageName is a message containing updates to GatewayClass status GatewayClassStatusMessageName MessageName = "gatewayclass-status" + // EnvoyProxyStatusMessageName is a message containing updates to EnvoyProxy status + EnvoyProxyStatusMessageName MessageName = "envoyproxy-status" ) diff --git a/internal/provider/kubernetes/controller.go b/internal/provider/kubernetes/controller.go index 625079f8c2..8d73889b21 100644 --- a/internal/provider/kubernetes/controller.go +++ b/internal/provider/kubernetes/controller.go @@ -132,6 +132,7 @@ type subscriptions struct { securityPolicyStatuses <-chan watchable.Snapshot[types.NamespacedName, *gwapiv1.PolicyStatus] backendStatuses <-chan watchable.Snapshot[types.NamespacedName, *egv1a1.BackendStatus] extensionPolicyStatuses <-chan watchable.Snapshot[message.NamespacedNameAndGVK, *gwapiv1.PolicyStatus] + envoyProxyStatuses <-chan watchable.Snapshot[types.NamespacedName, *egv1a1.EnvoyProxyStatus] } // newGatewayAPIController @@ -254,6 +255,7 @@ func (r *gatewayAPIReconciler) subscribeToResources(ctx context.Context) { r.subscriptions.securityPolicyStatuses = r.resources.SecurityPolicyStatuses.Subscribe(ctx) r.subscriptions.backendStatuses = r.resources.BackendStatuses.Subscribe(ctx) r.subscriptions.extensionPolicyStatuses = r.resources.ExtensionPolicyStatuses.Subscribe(ctx) + r.subscriptions.envoyProxyStatuses = r.resources.EnvoyProxyStatuses.Subscribe(ctx) } func (r *gatewayAPIReconciler) backendAPIDisabled() bool { @@ -2923,9 +2925,10 @@ func (r *gatewayAPIReconciler) processGatewayParamsRef(ctx context.Context, gtw return fmt.Errorf("failed to find envoyproxy %s/%s for Gateway %s: %w", gtw.Namespace, ref.Name, gtw.Name, err) } - if err := r.processEnvoyProxy(ep, resourceMap); err != nil { - return err - } + // Discard Status to reduce memory consumption in watchable + // It will be recomputed by the gateway-api layer + ep.Status = egv1a1.EnvoyProxyStatus{} + r.processEnvoyProxy(ep, resourceMap) // Missing secret shouldn't stop the Gateway infrastructure from coming up if ep.Spec.BackendTLS != nil && ep.Spec.BackendTLS.ClientCertificateRef != nil { @@ -2961,19 +2964,20 @@ func (r *gatewayAPIReconciler) processGatewayClassParamsRef(ctx context.Context, return errors.New("using Merged Gateways with Gateway Namespace Mode is not supported") } - if err := r.processEnvoyProxy(ep, resourceMap); err != nil { - return err - } + // Discard Status to reduce memory consumption in watchable + // It will be recomputed by the gateway-api layer + ep.Status = egv1a1.EnvoyProxyStatus{} + r.processEnvoyProxy(ep, resourceMap) resourceTree.EnvoyProxyForGatewayClass = ep return nil } -// processEnvoyProxy processes the parametersRef of the provided GatewayClass. -func (r *gatewayAPIReconciler) processEnvoyProxy(ep *egv1a1.EnvoyProxy, resourceMap *resourceMappings) error { +// processEnvoyProxy processes the parametersRef of the provided GatewayClass/Gateway. +func (r *gatewayAPIReconciler) processEnvoyProxy(ep *egv1a1.EnvoyProxy, resourceMap *resourceMappings) { key := utils.NamespacedName(ep).String() if resourceMap.allAssociatedEnvoyProxies.Has(key) { r.log.Info("current EnvoyProxy has been processed already", "namespace", ep.Namespace, "name", ep.Name) - return nil + return } r.log.Info("processing EnvoyProxy", "namespace", ep.Namespace, "name", ep.Name) @@ -3021,7 +3025,6 @@ func (r *gatewayAPIReconciler) processEnvoyProxy(ep *egv1a1.EnvoyProxy, resource } resourceMap.allAssociatedEnvoyProxies.Insert(key) - return nil } // crdExists checks for the existence of the CRD in k8s APIServer before watching it. diff --git a/internal/provider/kubernetes/status.go b/internal/provider/kubernetes/status.go index 2e44131874..9b5a6b2703 100644 --- a/internal/provider/kubernetes/status.go +++ b/internal/provider/kubernetes/status.go @@ -645,6 +645,46 @@ func (r *gatewayAPIReconciler) updateStatusFromSubscriptions(ctx context.Context r.log.Info("extensionServerPolicies status subscriber shutting down") }() } + + // EnvoyProxy status updater + go func() { + message.HandleSubscription(r.log, + message.Metadata{Runner: string(egv1a1.LogComponentProviderRunner), Message: message.EnvoyProxyStatusMessageName}, + r.subscriptions.envoyProxyStatuses, + func(update message.Update[types.NamespacedName, *egv1a1.EnvoyProxyStatus], errChan chan error) { + // skip delete updates. + if update.Delete { + return + } + key := update.Key + val := update.Value + r.statusUpdater.Send(Update{ + NamespacedName: key, + Resource: new(egv1a1.EnvoyProxy), + Mutator: MutatorFunc(func(obj client.Object) client.Object { + t, ok := obj.(*egv1a1.EnvoyProxy) + if !ok { + err := fmt.Errorf("unsupported object type %T", obj) + errChan <- err + panic(err) + } + valCopy := val.DeepCopy() + setLastTransitionTimeInConditionsForEnvoyProxyStatus(valCopy, metav1.Now()) + tCopy := &egv1a1.EnvoyProxy{ + TypeMeta: t.TypeMeta, + ObjectMeta: t.ObjectMeta, + Spec: t.Spec, + Status: egv1a1.EnvoyProxyStatus{ + Ancestors: valCopy.Ancestors, + }, + } + return tCopy + }), + }) + }, + ) + r.log.Info("envoyProxy status subscriber shutting down") + }() } // mergeRouteParentStatus merges the old and new RouteParentStatus. @@ -764,6 +804,12 @@ func setLastTransitionTimeInConditionsForPolicyStatus(status *gwapiv1.PolicyStat } } +func setLastTransitionTimeInConditionsForEnvoyProxyStatus(status *egv1a1.EnvoyProxyStatus, now metav1.Time) { + for i := range status.Ancestors { + setLastTransitionTimeInConditions(status.Ancestors[i].Conditions, now) + } +} + // setLastTransitionTimeInConditionsForRouteStatus sets LastTransitionTime for Route status conditions func setLastTransitionTimeInConditionsForRouteStatus(status *gwapiv1.RouteStatus, now metav1.Time) { for i := range status.Parents { diff --git a/test/e2e/testdata/envoyproxy-status.yaml b/test/e2e/testdata/envoyproxy-status.yaml new file mode 100644 index 0000000000..53d00cbced --- /dev/null +++ b/test/e2e/testdata/envoyproxy-status.yaml @@ -0,0 +1,59 @@ +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: test-envoyproxy + namespace: gateway-conformance-infra +spec: + shutdown: + drainTimeout: 5s + minDrainDuration: 1s +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: test-gc1 +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: test-envoyproxy + namespace: gateway-conformance-infra +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: test-gtw1 + namespace: gateway-conformance-infra +spec: + gatewayClassName: test-gc1 + listeners: + - name: http + protocol: HTTP + port: 80 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: test-gc2 +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: test-envoyproxy + namespace: gateway-conformance-infra +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: test-gtw2 + namespace: gateway-conformance-infra +spec: + gatewayClassName: test-gc2 + listeners: + - name: http + protocol: HTTP + port: 80 +--- + diff --git a/test/e2e/tests/envoyproxy.go b/test/e2e/tests/envoyproxy.go index 3af172b211..3e7466bf4c 100644 --- a/test/e2e/tests/envoyproxy.go +++ b/test/e2e/tests/envoyproxy.go @@ -26,13 +26,16 @@ import ( "sigs.k8s.io/gateway-api/conformance/utils/kubernetes" "sigs.k8s.io/gateway-api/conformance/utils/suite" "sigs.k8s.io/gateway-api/conformance/utils/tlog" + + "github.com/envoyproxy/gateway/internal/gatewayapi" + "github.com/envoyproxy/gateway/internal/gatewayapi/resource" ) func init() { - ConformanceTests = append(ConformanceTests, EnvoyProxyCustomNameTest) + ConformanceTests = append(ConformanceTests, EnvoyProxyCustomName) } -var EnvoyProxyCustomNameTest = suite.ConformanceTest{ +var EnvoyProxyCustomName = suite.ConformanceTest{ ShortName: "EnvoyProxyCustomName", Description: "Test running Envoy with custom name", Manifests: []string{"testdata/envoyproxy-custom-name.yaml"}, @@ -70,6 +73,16 @@ var EnvoyProxyCustomNameTest = suite.ConformanceTest{ }, }) + EnvoyProxyMustBeAccepted(t, suite.Client, types.NamespacedName{ + Name: "deploy-custom-name", + Namespace: ns, + }, gwapiv1.ParentReference{ + Group: gatewayapi.GroupPtr(gwapiv1.GroupName), + Kind: gatewayapi.KindPtr(resource.KindGateway), + Namespace: gatewayapi.NamespacePtr(gwNN.Namespace), + Name: gwapiv1.ObjectName(gwNN.Name), + }) + err = checkDeployment(t, suite, gwNN, gatewayNS, "custom-", 1, 1) if err != nil { t.Fatalf("Failed to delete Gateway: %v", err) diff --git a/test/e2e/tests/envoyproxy_invalid.go b/test/e2e/tests/envoyproxy_invalid.go index b2e958adbc..f735dc5b87 100644 --- a/test/e2e/tests/envoyproxy_invalid.go +++ b/test/e2e/tests/envoyproxy_invalid.go @@ -21,6 +21,9 @@ import ( "sigs.k8s.io/gateway-api/conformance/utils/http" "sigs.k8s.io/gateway-api/conformance/utils/kubernetes" "sigs.k8s.io/gateway-api/conformance/utils/suite" + + "github.com/envoyproxy/gateway/internal/gatewayapi" + "github.com/envoyproxy/gateway/internal/gatewayapi/resource" ) func init() { @@ -64,6 +67,16 @@ var EnvoyProxyInvalidUpdateTest = suite.ConformanceTest{ Reason: string(gwapiv1.GatewayReasonInvalidParameters), }) + EnvoyProxyMustNotAccepted(t, suite.Client, types.NamespacedName{ + Name: "eg-invalid", + Namespace: ns, + }, gwapiv1.ParentReference{ + Group: gatewayapi.GroupPtr(gwapiv1.GroupName), + Kind: gatewayapi.KindPtr(resource.KindGateway), + Name: gwapiv1.ObjectName("eg-invalid"), + Namespace: gatewayapi.NamespacePtr(ns), + }) + podShouldBeSame, err := getPodNameForGateway(t, suite, expectedNs, gwNN) require.NoError(t, err) require.Equal(t, podName, podShouldBeSame) diff --git a/test/e2e/tests/envoyproxy_status.go b/test/e2e/tests/envoyproxy_status.go new file mode 100644 index 0000000000..e39d3e7602 --- /dev/null +++ b/test/e2e/tests/envoyproxy_status.go @@ -0,0 +1,91 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build e2e + +package tests + +import ( + "testing" + "time" + + "k8s.io/apimachinery/pkg/types" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/gateway-api/conformance/utils/http" + "sigs.k8s.io/gateway-api/conformance/utils/suite" + "sigs.k8s.io/gateway-api/conformance/utils/tlog" + + "github.com/envoyproxy/gateway/internal/gatewayapi" + "github.com/envoyproxy/gateway/internal/gatewayapi/resource" +) + +func init() { + ConformanceTests = append(ConformanceTests, EnvoyProxyStatus) +} + +var EnvoyProxyStatus = suite.ConformanceTest{ + ShortName: "EnvoyProxyStatus", + Description: "Make sure that the status of EnvoyProxy works as expected.", + Manifests: []string{"testdata/envoyproxy-status.yaml"}, + Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { + ns := "gateway-conformance-infra" + epName := "test-envoyproxy" + EnvoyProxyMustBeAccepted(t, suite.Client, types.NamespacedName{ + Name: epName, + Namespace: ns, + }, gwapiv1.ParentReference{ + Group: gatewayapi.GroupPtr(gwapiv1.GroupName), + Kind: gatewayapi.KindPtr(resource.KindGatewayClass), + Name: gwapiv1.ObjectName("test-gc1"), + }) + EnvoyProxyMustBeAccepted(t, suite.Client, types.NamespacedName{ + Name: epName, + Namespace: ns, + }, gwapiv1.ParentReference{ + Group: gatewayapi.GroupPtr(gwapiv1.GroupName), + Kind: gatewayapi.KindPtr(resource.KindGatewayClass), + Name: gwapiv1.ObjectName("test-gc2"), + }) + + // Remove parametersRef from gc1, and make sure the status is removed from EnvoyProxy. + http.AwaitConvergence(t, 1, suite.TimeoutConfig.MaxTimeToConsistency, func(_ time.Duration) bool { + // Get gc1 and remove parametersRef. + gc1 := &gwapiv1.GatewayClass{} + if err := suite.Client.Get(t.Context(), types.NamespacedName{ + Name: "test-gc1", + }, gc1); err != nil { + tlog.Logf(t, "failed to get gc1: %v", err) + return false + } + + gc1.Spec.ParametersRef = nil + if err := suite.Client.Update(t.Context(), gc1); err != nil { + tlog.Logf(t, "failed to update gc1: %v", err) + return false + } + + return true + }) + + // ParametersRef is removed from gc1, so the status of EnvoyProxy should be removed. + EnvoyProxyMustNotAccepted(t, suite.Client, types.NamespacedName{ + Name: epName, + Namespace: ns, + }, gwapiv1.ParentReference{ + Group: gatewayapi.GroupPtr(gwapiv1.GroupName), + Kind: gatewayapi.KindPtr(resource.KindGatewayClass), + Name: gwapiv1.ObjectName("test-gc1"), + }) + // The status of EnvoyProxy should still be accepted by gc2. + EnvoyProxyMustBeAccepted(t, suite.Client, types.NamespacedName{ + Name: epName, + Namespace: ns, + }, gwapiv1.ParentReference{ + Group: gatewayapi.GroupPtr(gwapiv1.GroupName), + Kind: gatewayapi.KindPtr(resource.KindGatewayClass), + Name: gwapiv1.ObjectName("test-gc2"), + }) + }, +} diff --git a/test/e2e/tests/utils.go b/test/e2e/tests/utils.go index 834a160ccf..75a89a39fa 100644 --- a/test/e2e/tests/utils.go +++ b/test/e2e/tests/utils.go @@ -295,6 +295,57 @@ func ClientTrafficPolicyMustBeAccepted(t *testing.T, client client.Client, polic require.NoErrorf(t, waitErr, "error waiting for ClientTrafficPolicy to be accepted") } +func EnvoyProxyMustBeAccepted(t *testing.T, client client.Client, epName types.NamespacedName, ancestorRef gwapiv1.ParentReference) { + t.Helper() + + waitErr := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) { + ep := &egv1a1.EnvoyProxy{} + err := client.Get(ctx, epName, ep) + if err != nil { + return false, fmt.Errorf("error fetching EnvoyProxy: %w", err) + } + + if envoyProxyAcceptedByAncestor(&ep.Status, ancestorRef) { + return true, nil + } + + tlog.Logf(t, "EnvoyProxy not yet accepted: %v", ep) + return false, nil + }) + + require.NoErrorf(t, waitErr, "error waiting for EnvoyProxy to be accepted") +} + +func EnvoyProxyMustNotAccepted(t *testing.T, client client.Client, epName types.NamespacedName, ancestorRef gwapiv1.ParentReference) { + t.Helper() + waitErr := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, 60*time.Second, true, func(ctx context.Context) (bool, error) { + ep := &egv1a1.EnvoyProxy{} + err := client.Get(ctx, epName, ep) + if err != nil { + return false, fmt.Errorf("error fetching EnvoyProxy: %w", err) + } + if !envoyProxyAcceptedByAncestor(&ep.Status, ancestorRef) { + return true, nil + } + tlog.Logf(t, "EnvoyProxy is accepted: %v", ep) + return false, nil + }) + require.NoErrorf(t, waitErr, "error waiting for EnvoyProxy to be not accepted") +} + +func envoyProxyAcceptedByAncestor(status *egv1a1.EnvoyProxyStatus, ancestorRef gwapiv1.ParentReference) bool { + for _, ancestor := range status.Ancestors { + if cmp.Equal(ancestor.AncestorRef, ancestorRef) { + for _, condition := range ancestor.Conditions { + if condition.Type == string(egv1a1.EnvoyProxyConditionAccepted) && condition.Status == metav1.ConditionTrue { + return true + } + } + } + } + return false +} + // AlmostEquals We use a solution similar to istio: // Given an offset, calculate whether the actual value is within the offset of the expected value func AlmostEquals(actual, expect, offset int) bool { diff --git a/test/helm/gateway-helm/certgen-annotations.out.yaml b/test/helm/gateway-helm/certgen-annotations.out.yaml index aa076f8c71..a152955fce 100644 --- a/test/helm/gateway-helm/certgen-annotations.out.yaml +++ b/test/helm/gateway-helm/certgen-annotations.out.yaml @@ -143,6 +143,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/certgen-args.out.yaml b/test/helm/gateway-helm/certgen-args.out.yaml index 9fc1812b0d..d5ca2b0f90 100644 --- a/test/helm/gateway-helm/certgen-args.out.yaml +++ b/test/helm/gateway-helm/certgen-args.out.yaml @@ -143,6 +143,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/certgen-labels.out.yaml b/test/helm/gateway-helm/certgen-labels.out.yaml index 43ba7a5bc7..06bf7a6580 100644 --- a/test/helm/gateway-helm/certgen-labels.out.yaml +++ b/test/helm/gateway-helm/certgen-labels.out.yaml @@ -143,6 +143,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/certjen-custom-scheduling.out.yaml b/test/helm/gateway-helm/certjen-custom-scheduling.out.yaml index 76bc243a92..9cbae51d53 100644 --- a/test/helm/gateway-helm/certjen-custom-scheduling.out.yaml +++ b/test/helm/gateway-helm/certjen-custom-scheduling.out.yaml @@ -143,6 +143,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/common-labels.out.yaml b/test/helm/gateway-helm/common-labels.out.yaml index d9f3e87119..47b83af26b 100644 --- a/test/helm/gateway-helm/common-labels.out.yaml +++ b/test/helm/gateway-helm/common-labels.out.yaml @@ -155,6 +155,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/control-plane-with-pdb.out.yaml b/test/helm/gateway-helm/control-plane-with-pdb.out.yaml index 8e60f9556a..6959a405ff 100644 --- a/test/helm/gateway-helm/control-plane-with-pdb.out.yaml +++ b/test/helm/gateway-helm/control-plane-with-pdb.out.yaml @@ -158,6 +158,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/default-config.out.yaml b/test/helm/gateway-helm/default-config.out.yaml index 6d9e279bb7..42e5f0a634 100644 --- a/test/helm/gateway-helm/default-config.out.yaml +++ b/test/helm/gateway-helm/default-config.out.yaml @@ -143,6 +143,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/deployment-annotations.out.yaml b/test/helm/gateway-helm/deployment-annotations.out.yaml index becccb04f9..83f7f8df1b 100644 --- a/test/helm/gateway-helm/deployment-annotations.out.yaml +++ b/test/helm/gateway-helm/deployment-annotations.out.yaml @@ -143,6 +143,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/deployment-custom-topology.out.yaml b/test/helm/gateway-helm/deployment-custom-topology.out.yaml index 9ac85ba274..366d48d84d 100644 --- a/test/helm/gateway-helm/deployment-custom-topology.out.yaml +++ b/test/helm/gateway-helm/deployment-custom-topology.out.yaml @@ -143,6 +143,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/deployment-extraenv.out.yaml b/test/helm/gateway-helm/deployment-extraenv.out.yaml index f290137ee2..8a410a0e69 100644 --- a/test/helm/gateway-helm/deployment-extraenv.out.yaml +++ b/test/helm/gateway-helm/deployment-extraenv.out.yaml @@ -143,6 +143,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/deployment-images-config.out.yaml b/test/helm/gateway-helm/deployment-images-config.out.yaml index 34cac7a706..3d903e8cc8 100644 --- a/test/helm/gateway-helm/deployment-images-config.out.yaml +++ b/test/helm/gateway-helm/deployment-images-config.out.yaml @@ -143,6 +143,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/deployment-priorityclass.out.yaml b/test/helm/gateway-helm/deployment-priorityclass.out.yaml index b8355aff16..d399aeb059 100644 --- a/test/helm/gateway-helm/deployment-priorityclass.out.yaml +++ b/test/helm/gateway-helm/deployment-priorityclass.out.yaml @@ -143,6 +143,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/deployment-repo-no-registry.out.yaml b/test/helm/gateway-helm/deployment-repo-no-registry.out.yaml index ec1a63b7cc..2528318815 100644 --- a/test/helm/gateway-helm/deployment-repo-no-registry.out.yaml +++ b/test/helm/gateway-helm/deployment-repo-no-registry.out.yaml @@ -143,6 +143,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/deployment-securitycontext.out.yaml b/test/helm/gateway-helm/deployment-securitycontext.out.yaml index dac27878d4..448ca64e0e 100644 --- a/test/helm/gateway-helm/deployment-securitycontext.out.yaml +++ b/test/helm/gateway-helm/deployment-securitycontext.out.yaml @@ -143,6 +143,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/deployment-volume-mounts.out.yaml b/test/helm/gateway-helm/deployment-volume-mounts.out.yaml index 2dcfbc0967..b6f9d3a94a 100644 --- a/test/helm/gateway-helm/deployment-volume-mounts.out.yaml +++ b/test/helm/gateway-helm/deployment-volume-mounts.out.yaml @@ -143,6 +143,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/envoy-gateway-config.out.yaml b/test/helm/gateway-helm/envoy-gateway-config.out.yaml index c15cbd412c..49786a774e 100644 --- a/test/helm/gateway-helm/envoy-gateway-config.out.yaml +++ b/test/helm/gateway-helm/envoy-gateway-config.out.yaml @@ -145,6 +145,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/envoy-gateway-gateway-namespace-config-watch.out.yaml b/test/helm/gateway-helm/envoy-gateway-gateway-namespace-config-watch.out.yaml index 0f957af34a..4f28cee2fe 100644 --- a/test/helm/gateway-helm/envoy-gateway-gateway-namespace-config-watch.out.yaml +++ b/test/helm/gateway-helm/envoy-gateway-gateway-namespace-config-watch.out.yaml @@ -223,6 +223,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status @@ -331,6 +332,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/envoy-gateway-gateway-namespace-config.out.yaml b/test/helm/gateway-helm/envoy-gateway-gateway-namespace-config.out.yaml index 3caecfa94d..ac37b5cb74 100644 --- a/test/helm/gateway-helm/envoy-gateway-gateway-namespace-config.out.yaml +++ b/test/helm/gateway-helm/envoy-gateway-gateway-namespace-config.out.yaml @@ -145,6 +145,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/envoy-gateway-gateway-namespace-namespaceselector.out.yaml b/test/helm/gateway-helm/envoy-gateway-gateway-namespace-namespaceselector.out.yaml index 57cde3adda..0d414a6e88 100644 --- a/test/helm/gateway-helm/envoy-gateway-gateway-namespace-namespaceselector.out.yaml +++ b/test/helm/gateway-helm/envoy-gateway-gateway-namespace-namespaceselector.out.yaml @@ -150,6 +150,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/global-images-config.out.yaml b/test/helm/gateway-helm/global-images-config.out.yaml index 54634076ca..b83f96e83b 100644 --- a/test/helm/gateway-helm/global-images-config.out.yaml +++ b/test/helm/gateway-helm/global-images-config.out.yaml @@ -159,6 +159,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/global-pullsecrets-override-deployment.out.yaml b/test/helm/gateway-helm/global-pullsecrets-override-deployment.out.yaml index 7dd187846e..a2de40ec7a 100644 --- a/test/helm/gateway-helm/global-pullsecrets-override-deployment.out.yaml +++ b/test/helm/gateway-helm/global-pullsecrets-override-deployment.out.yaml @@ -147,6 +147,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/global-pullsecrets-override-global.out.yaml b/test/helm/gateway-helm/global-pullsecrets-override-global.out.yaml index 2ebad096da..61d29b2f3f 100644 --- a/test/helm/gateway-helm/global-pullsecrets-override-global.out.yaml +++ b/test/helm/gateway-helm/global-pullsecrets-override-global.out.yaml @@ -147,6 +147,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/global-registry-override-deployment.out.yaml b/test/helm/gateway-helm/global-registry-override-deployment.out.yaml index afdae517a6..a6a8ceb170 100644 --- a/test/helm/gateway-helm/global-registry-override-deployment.out.yaml +++ b/test/helm/gateway-helm/global-registry-override-deployment.out.yaml @@ -143,6 +143,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/global-registry-override-global.out.yaml b/test/helm/gateway-helm/global-registry-override-global.out.yaml index f8494e3c77..f87ce6e269 100644 --- a/test/helm/gateway-helm/global-registry-override-global.out.yaml +++ b/test/helm/gateway-helm/global-registry-override-global.out.yaml @@ -143,6 +143,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/horizontal-pod-autoscaler-disabled.out.yaml b/test/helm/gateway-helm/horizontal-pod-autoscaler-disabled.out.yaml index e269aaa40e..671009c43c 100644 --- a/test/helm/gateway-helm/horizontal-pod-autoscaler-disabled.out.yaml +++ b/test/helm/gateway-helm/horizontal-pod-autoscaler-disabled.out.yaml @@ -143,6 +143,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/horizontal-pod-autoscaler.out.yaml b/test/helm/gateway-helm/horizontal-pod-autoscaler.out.yaml index 6cb862723c..51ffb2322b 100644 --- a/test/helm/gateway-helm/horizontal-pod-autoscaler.out.yaml +++ b/test/helm/gateway-helm/horizontal-pod-autoscaler.out.yaml @@ -143,6 +143,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/namespace-override.out.yaml b/test/helm/gateway-helm/namespace-override.out.yaml index 3f75d65463..d608f83b6a 100644 --- a/test/helm/gateway-helm/namespace-override.out.yaml +++ b/test/helm/gateway-helm/namespace-override.out.yaml @@ -143,6 +143,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/service-customization.out.yaml b/test/helm/gateway-helm/service-customization.out.yaml index f8ca7ad059..02140a3670 100644 --- a/test/helm/gateway-helm/service-customization.out.yaml +++ b/test/helm/gateway-helm/service-customization.out.yaml @@ -143,6 +143,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status diff --git a/test/helm/gateway-helm/webhook-disabled.out.yaml b/test/helm/gateway-helm/webhook-disabled.out.yaml index 9235d24e2e..547f014331 100644 --- a/test/helm/gateway-helm/webhook-disabled.out.yaml +++ b/test/helm/gateway-helm/webhook-disabled.out.yaml @@ -145,6 +145,7 @@ rules: - apiGroups: - gateway.envoyproxy.io resources: + - envoyproxies/status - envoypatchpolicies/status - clienttrafficpolicies/status - backendtrafficpolicies/status