Skip to content

Commit 34706ec

Browse files
JoseSzychoJoseSzycho
committed
feat: enable E2E testing for the envoy-gateway extension server and implement selective WAF policy injection based on configuration.
1 parent c2fe4e8 commit 34706ec

13 files changed

Lines changed: 302 additions & 15 deletions

File tree

‎Makefile‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,8 @@ test-e2e: chainsaw
8181
}
8282
$(KIND) get kubeconfig --name nso-standard > $(TMPDIR)/.kind-nso-standard.yaml
8383
$(KIND) get kubeconfig --name nso-infra > $(TMPDIR)/.kind-nso-infra.yaml
84-
$(CHAINSAW) test ./test/e2e \
84+
$(CHAINSAW) test $(or $(TEST_DIR),./test/e2e) \
85+
--parallel 1 \
8586
--cluster nso-standard=$(TMPDIR)/.kind-nso-standard.yaml \
8687
--cluster nso-infra=$(TMPDIR)/.kind-nso-infra.yaml
8788

@@ -153,7 +154,7 @@ set-image-controller: manifests kustomize
153154
cd config/manager && $(KUSTOMIZE) edit set image ghcr.io/datum-cloud/network-services-operator=${IMG}
154155

155156
.PHONY: prepare-infra-cluster
156-
prepare-infra-cluster: cert-manager envoy-gateway external-dns downstream-crds
157+
prepare-infra-cluster: cert-manager envoy-gateway external-dns downstream-crds billing-usage-collector load-image-nso-infra extension-server configure-eg-extension-manager
157158

158159
.PHONY: downstream-crds
159160
downstream-crds: ## Install NSO CRDs on the downstream (infra) cluster that the replicator mirrors into it.
@@ -168,12 +169,16 @@ prepare-e2e: chainsaw set-image-controller cert-manager load-image-all deploy-e2
168169
prepare-dev: chainsaw set-image-controller cert-manager install
169170

170171
.PHONY: load-image-all
171-
load-image-all: load-image-operator
172+
load-image-all: load-image-operator load-image-nso-infra
172173

173174
.PHONY: load-image-operator
174175
load-image-operator: docker-build kind
175176
$(KIND) load docker-image $(IMG) -n nso-standard
176177

178+
.PHONY: load-image-nso-infra
179+
load-image-nso-infra: docker-build kind ## Load operator image into nso-infra kind cluster (needed by the extension server).
180+
$(KIND) load docker-image $(IMG) -n nso-infra
181+
177182
.PHONY: cert-manager
178183
cert-manager: cmctl
179184
$(KUSTOMIZE) build --enable-helm config/tools/cert-manager | kubectl apply --server-side=true --force-conflicts -f -
@@ -187,6 +192,24 @@ envoy-gateway:
187192
external-dns:
188193
$(KUSTOMIZE) build --enable-helm config/tools/external-dns | kubectl apply --server-side=true --force-conflicts -f -
189194

195+
.PHONY: billing-usage-collector
196+
billing-usage-collector:
197+
$(KUSTOMIZE) build --enable-helm config/tools/billing-usage-collector | kubectl apply --server-side=true --force-conflicts -f -
198+
199+
.PHONY: extension-server
200+
extension-server: ## Deploy the NSO extension server to the infra cluster (e2e overlay with cert-manager-issued TLS).
201+
$(KUSTOMIZE) build --enable-helm config/extension-server-e2e | kubectl apply --server-side=true --force-conflicts -f -
202+
kubectl rollout restart deployment/network-services-operator-envoy-gateway-extension-server \
203+
-n network-services-operator-system
204+
kubectl rollout status deployment/network-services-operator-envoy-gateway-extension-server \
205+
-n network-services-operator-system --timeout=5m
206+
207+
.PHONY: configure-eg-extension-manager
208+
configure-eg-extension-manager: ## Patch the EG ConfigMap to enable extensionManager and restart the EG controller.
209+
$(KUSTOMIZE) build --enable-helm config/tools/envoy-gateway/overlays/e2e | kubectl apply --server-side=true --force-conflicts -f -
210+
kubectl rollout restart deployment/envoy-gateway -n envoy-gateway-system
211+
kubectl rollout status deployment/envoy-gateway -n envoy-gateway-system --timeout=3m
212+
190213
.PHONY: kind-standard-cluster
191214
kind-standard-cluster: kind
192215
$(KIND) create cluster --config=config/tools/kind/standard-cluster.yaml

‎config/extension-server-e2e/client-cert-patch.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: cert-manager.io/v1
2+
kind: Certificate
3+
metadata:
4+
name: envoy-gateway-extension-server-eg-client-tls
5+
spec:
6+
issuerRef:
7+
# Switch from the production placeholder ClusterIssuer to the e2e CA Issuer.
8+
# kustomize propagates the namePrefix so this resolves to
9+
# network-services-operator-nso-es-ca-issuer at apply time.
10+
name: nso-es-ca-issuer
11+
kind: Issuer
12+
group: cert-manager.io

‎config/extension-server-e2e/deployment-patch.yaml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: envoy-gateway-extension-server
5+
spec:
6+
# Single-node kind clusters can't satisfy the base DoNotSchedule topology constraint.
7+
replicas: 1
8+
template:
9+
spec:
10+
# Remove the hostname spread constraint — only 1 node in kind.
11+
topologySpreadConstraints: null
12+
volumes:
13+
# Replace cert-manager CSI driver mount with a regular Secret volume.
14+
# cert-manager Certificate nso-es-tls populates this Secret.
15+
- name: tls
16+
csi: null
17+
secret:
18+
secretName: nso-extension-server-tls
19+
# Replace ConfigMap CA bundle with the same Secret; the ca.crt key holds
20+
# the issuing CA cert that the extension server uses to verify EG's client cert.
21+
- name: tls-ca
22+
configMap: null
23+
secret:
24+
secretName: nso-extension-server-tls
25+
items:
26+
- key: ca.crt
27+
path: ca.crt
28+
# Extension server operator config — disables Coraza WAF injection
29+
# so standard (non-contrib) Envoy images work in e2e.
30+
- name: server-config
31+
configMap:
32+
name: extension-server-config
33+
containers:
34+
- name: envoy-gateway-extension-server
35+
env:
36+
- name: SERVER_CONFIG
37+
value: /server-config/config.yaml
38+
volumeMounts:
39+
- name: server-config
40+
mountPath: /server-config
41+
readOnly: true

‎config/extension-server-e2e/kustomization.yaml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
apiVersion: kustomize.config.k8s.io/v1beta1
2+
kind: Kustomization
3+
4+
namespace: network-services-operator-system
5+
namePrefix: network-services-operator-
6+
7+
resources:
8+
- namespace.yaml
9+
- ../extension-server
10+
- tls.yaml
11+
- server-config.yaml
12+
13+
patches:
14+
# Replace CSI + ConfigMap volumes with Secret-based mounts; reduce replicas to 1 for single-node kind.
15+
- path: deployment-patch.yaml
16+
target:
17+
kind: Deployment
18+
name: envoy-gateway-extension-server
19+
# Remove the PDB minAvailable constraint — kind clusters have only 1 node so minAvailable:1 prevents eviction.
20+
- path: pdb-patch.yaml
21+
target:
22+
kind: PodDisruptionBudget
23+
name: envoy-gateway-extension-server-pdb
24+
# Switch the EG client cert issuer from the placeholder to the e2e CA.
25+
- path: client-cert-patch.yaml
26+
target:
27+
kind: Certificate
28+
name: envoy-gateway-extension-server-eg-client-tls

‎config/extension-server-e2e/namespace.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
apiVersion: v1
2+
kind: Namespace
3+
metadata:
4+
name: network-services-operator-system

‎config/extension-server-e2e/pdb-patch.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
apiVersion: policy/v1
2+
kind: PodDisruptionBudget
3+
metadata:
4+
name: envoy-gateway-extension-server-pdb
5+
spec:
6+
# replicas=1 in e2e; minAvailable:1 would block any eviction. Set to 0 so kind
7+
# cluster teardown / upgrades don't stall.
8+
minAvailable: 0

‎config/extension-server-e2e/server-config.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
---
2+
# ConfigMap holding the extension server operator config for e2e.
3+
# Disables Coraza WAF injection — e2e uses the standard Envoy image which
4+
# does not have the golang filter compiled in, so injecting coraza-waf causes
5+
# Envoy to reject the listener configuration.
6+
apiVersion: v1
7+
kind: ConfigMap
8+
metadata:
9+
name: extension-server-config
10+
namespace: network-services-operator-system
11+
data:
12+
config.yaml: |
13+
apiVersion: apiserver.config.datumapis.com/v1alpha1
14+
kind: NetworkServicesOperator
15+
gateway:
16+
coraza:
17+
disabled: true

‎config/extension-server-e2e/tls.yaml‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
---
2+
# Self-signed bootstrap issuer; creates the e2e CA cert below.
3+
apiVersion: cert-manager.io/v1
4+
kind: Issuer
5+
metadata:
6+
name: nso-es-selfsigned
7+
spec:
8+
selfSigned: {}
9+
---
10+
# CA cert. cert-manager places the CA cert+key in Secret nso-extension-server-ca
11+
# (in the same namespace). The CA Issuer below references that Secret.
12+
apiVersion: cert-manager.io/v1
13+
kind: Certificate
14+
metadata:
15+
name: nso-es-ca
16+
spec:
17+
isCA: true
18+
commonName: nso-extension-server-ca
19+
secretName: nso-extension-server-ca
20+
privateKey:
21+
algorithm: ECDSA
22+
size: 256
23+
issuerRef:
24+
name: nso-es-selfsigned
25+
kind: Issuer
26+
group: cert-manager.io
27+
---
28+
# CA-backed issuer used by both the server cert and the EG client cert below.
29+
apiVersion: cert-manager.io/v1
30+
kind: Issuer
31+
metadata:
32+
name: nso-es-ca-issuer
33+
spec:
34+
ca:
35+
secretName: nso-extension-server-ca
36+
---
37+
# Extension-server TLS cert. cert-manager writes it to Secret nso-extension-server-tls,
38+
# which includes ca.crt = the CA cert — EG reads this field from certificateRef to verify
39+
# the extension server's presented cert.
40+
apiVersion: cert-manager.io/v1
41+
kind: Certificate
42+
metadata:
43+
name: nso-es-tls
44+
spec:
45+
secretName: nso-extension-server-tls
46+
dnsNames:
47+
- network-services-operator-envoy-gateway-extension-server.network-services-operator-system.svc
48+
- network-services-operator-envoy-gateway-extension-server.network-services-operator-system.svc.cluster.local
49+
privateKey:
50+
algorithm: ECDSA
51+
size: 256
52+
issuerRef:
53+
name: nso-es-ca-issuer
54+
kind: Issuer
55+
group: cert-manager.io
56+
usages:
57+
- server auth
58+
- digital signature
59+
- key encipherment

‎config/tools/envoy-gateway/overlays/e2e/envoy-gateway-extension-manager-config.yaml‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
---
2+
# Patches the Helm-managed envoy-gateway-config ConfigMap to enable the
3+
# extensionManager. Applied after the EG helm chart with --server-side
4+
# --force-conflicts so the extra field is owned by this apply, not helm.
5+
#
6+
# All other fields (extensionApis, runtimeFlags, provider image pins, etc.) are
7+
# reproduced verbatim from the v1.8.1 helm chart so the full ConfigMap value is
8+
# consistent — the data field is a single YAML string and cannot be merged at
9+
# field granularity by server-side apply.
10+
apiVersion: v1
11+
kind: ConfigMap
12+
metadata:
13+
name: envoy-gateway-config
14+
namespace: envoy-gateway-system
15+
data:
16+
envoy-gateway.yaml: |
17+
apiVersion: gateway.envoyproxy.io/v1alpha1
18+
kind: EnvoyGateway
19+
extensionApis:
20+
enableBackend: true
21+
enableEnvoyPatchPolicy: true
22+
gateway:
23+
controllerName: gateway.envoyproxy.io/gatewayclass-controller
24+
logging:
25+
level:
26+
default: info
27+
provider:
28+
kubernetes:
29+
rateLimitDeployment:
30+
container:
31+
image: docker.io/envoyproxy/ratelimit:ff287602
32+
patch:
33+
type: StrategicMerge
34+
value:
35+
spec:
36+
template:
37+
spec:
38+
containers:
39+
- imagePullPolicy: IfNotPresent
40+
name: envoy-ratelimit
41+
shutdownManager:
42+
image: docker.io/envoyproxy/gateway:v1.8.1
43+
type: Kubernetes
44+
runtimeFlags:
45+
enabled:
46+
- XDSNameSchemeV2
47+
extensionManager:
48+
service:
49+
fqdn:
50+
hostname: network-services-operator-envoy-gateway-extension-server.network-services-operator-system.svc.cluster.local
51+
port: 5005
52+
tls:
53+
# certificateRef points to the Secret created by the nso-es-tls Certificate.
54+
# EG reads ca.crt from this Secret to verify the extension server's TLS cert.
55+
certificateRef:
56+
name: nso-extension-server-tls
57+
namespace: network-services-operator-system
58+
# EG presents this client cert when dialing the extension server.
59+
clientCertificateRef:
60+
name: envoy-gateway-extension-server-eg-client-tls
61+
namespace: network-services-operator-system
62+
retry:
63+
maxAttempts: 4
64+
initialBackoff: 100ms
65+
maxBackoff: 1s
66+
backoffMultiplier:
67+
numerator: 200
68+
retryableStatusCodes:
69+
- UNAVAILABLE
70+
hooks:
71+
xdsTranslator:
72+
post:
73+
- Translation
74+
translation:
75+
listener:
76+
includeAll: true
77+
route:
78+
includeAll: true
79+
cluster:
80+
includeAll: true
81+
secret:
82+
includeAll: true
83+
failOpen: false

‎config/tools/envoy-gateway/overlays/e2e/kustomization.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
apiVersion: kustomize.config.k8s.io/v1beta1
2+
kind: Kustomization
3+
4+
resources:
5+
- envoy-gateway-extension-manager-config.yaml

0 commit comments

Comments
 (0)