Skip to content

Commit facd950

Browse files
ecvecv
authored
Merge pull request #276 from datum-cloud/test/prod-fidelity-env-rebased
test(infra): prod-fidelity edge test env + e2e suite
2 parents 3b7fce8 + cd6f654 commit facd950

39 files changed

Lines changed: 1173 additions & 933 deletions

‎.github/workflows/test-e2e.yml‎

Lines changed: 43 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ on:
77
pull_request:
88
branches:
99
- main
10+
workflow_dispatch:
1011

1112
jobs:
1213
test-e2e:
@@ -23,70 +24,58 @@ jobs:
2324
with:
2425
go-version: '~1.26'
2526

26-
- name: Verify kind installation
27-
run: kind version
28-
29-
- name: Create upstream kind cluster
30-
run: make kind-standard-cluster
31-
32-
- name: Create downstream kind cluster
33-
run: |
34-
make kind-infra-cluster
35-
kubectl config use-context kind-nso-standard
36-
37-
- name: Prepare e2e
38-
run: |
39-
kubectl config use-context kind-nso-standard
40-
make prepare-e2e
41-
42-
kubectl config use-context kind-nso-infra
43-
make prepare-infra-cluster
27+
- name: Install Task
28+
uses: arduino/setup-task@v2
29+
with:
30+
version: 3.x
31+
repo-token: ${{ secrets.GITHUB_TOKEN }}
4432

45-
kind get kubeconfig --name nso-infra --internal > ${TMPDIR}/.kind-nso-infra-internal.yaml
33+
- name: Install Helm
34+
uses: azure/setup-helm@v4
35+
with:
36+
# kustomize v5.5.0 probes helm with `helm version -c`; the -c shorthand
37+
# was removed in helm 3.13, so pin to the last release that accepts it.
38+
version: v3.12.3
4639

47-
kubectl config use-context kind-nso-standard
48-
kubectl create secret -n network-services-operator-system \
49-
generic downstream-cluster-kubeconfig \
50-
--from-file=kubeconfig=${TMPDIR}/.kind-nso-infra-internal.yaml
40+
- name: Install pinned tools
41+
run: make kind kustomize cmctl chainsaw
5142

52-
# Wait for the operator to be ready
53-
kubectl -n network-services-operator-system \
54-
wait deploy network-services-operator-controller-manager \
55-
--for=condition=Available \
56-
--timeout=120s || \
57-
kubectl -n cert-manager describe pods && \
58-
kubectl logs -n cert-manager -l app.kubernetes.io/name=cert-manager --tail=-1 && \
59-
kubectl -n network-services-operator-system describe issuer && \
60-
kubectl -n network-services-operator-system describe certificates && \
61-
kubectl -n network-services-operator-system describe secrets && \
62-
kubectl -n network-services-operator-system describe pod && \
63-
kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=-1
43+
- name: Bring up the prod-fidelity env
44+
run: task test-infra:up
6445

65-
- name: Running Test e2e
66-
run: make test-e2e
46+
- name: Run e2e suite
47+
run: task test-infra:test-e2e
6748

68-
- name: Dump downstream WAF data plane on failure
49+
- name: Dump the downstream data plane on failure
6950
if: failure()
7051
run: |
71-
kubectl config use-context kind-nso-infra
72-
echo "::group::GatewayClasses"
73-
kubectl get gatewayclass -o wide || true
74-
kubectl describe gatewayclass datum-downstream-gateway || true
52+
set +e
53+
UKC=${TMPDIR}/.kind-nso-upstream.yaml
54+
bin/kind-v0.32.0 get kubeconfig --name nso-upstream > "${UKC}"
55+
echo "::group::Upstream NSO manager"
56+
KUBECONFIG="${UKC}" kubectl -n network-services-operator-system get pods -o wide
57+
KUBECONFIG="${UKC}" kubectl -n network-services-operator-system describe deploy network-services-operator-controller-manager
58+
KUBECONFIG="${UKC}" kubectl -n network-services-operator-system describe pod -l app.kubernetes.io/name=network-services-operator
59+
KUBECONFIG="${UKC}" kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=200 --all-containers
7560
echo "::endgroup::"
76-
echo "::group::EnvoyProxy"
77-
kubectl -n datum-downstream-gateway get envoyproxy -o yaml || true
61+
echo "::group::Upstream cert-manager + webhook cert"
62+
KUBECONFIG="${UKC}" kubectl -n cert-manager get pods -o wide
63+
KUBECONFIG="${UKC}" kubectl -n network-services-operator-system get certificate,secret
7864
echo "::endgroup::"
79-
echo "::group::datum-downstream-gateway namespace + pods"
80-
kubectl get ns datum-downstream-gateway --show-labels || true
81-
kubectl -n datum-downstream-gateway get pods -o wide || true
65+
KC=${TMPDIR}/.kind-nso-downstream.yaml
66+
bin/kind-v0.32.0 get kubeconfig --name nso-downstream > "${KC}"
67+
export KUBECONFIG="${KC}"
68+
echo "::group::GatewayClasses"
69+
kubectl get gatewayclass -o wide
8270
echo "::endgroup::"
83-
echo "::group::Downstream Envoy Gateway logs"
84-
kubectl -n datum-downstream-gateway logs deploy/envoy-gateway --tail=200 || true
71+
echo "::group::Gateways (all namespaces)"
72+
kubectl get gateway -A -o wide
8573
echo "::endgroup::"
86-
echo "::group::Extension server"
87-
kubectl -n datum-downstream-gateway describe deploy envoy-gateway-extension-server || true
88-
kubectl -n datum-downstream-gateway logs -l app.kubernetes.io/component=envoy-gateway-extension-server --tail=200 || true
74+
echo "::group::datum-downstream-gateway pods"
75+
kubectl -n datum-downstream-gateway get pods -o wide
76+
kubectl -n datum-downstream-gateway logs deploy/envoy-gateway --tail=200
8977
echo "::endgroup::"
90-
echo "::group::Certificates"
91-
kubectl -n datum-downstream-gateway get certificate,secret || true
78+
echo "::group::Extension server"
79+
kubectl -n network-services-operator-system get pods -o wide
80+
kubectl -n network-services-operator-system logs -l app.kubernetes.io/component=envoy-gateway-extension-server --tail=200
9281
echo "::endgroup::"

‎CLAUDE.md‎

Lines changed: 31 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -63,27 +63,41 @@ Run `make help` for the full list. Most-used targets:
6363
### Tests
6464
- `make test` — unit/integration tests via **envtest** (excludes `test/e2e`);
6565
runs `manifests generate fmt vet` first, writes `cover.out`
66-
- `make test-e2e` — **Chainsaw** e2e suite in `test/e2e` against the two local
67-
Kind clusters (requires `nso-standard` and `nso-infra` to exist)
66+
- `task test-infra:test-e2e` — **Chainsaw** e2e suite in `test/e2e` against the
67+
prod-fidelity env (below); pass a scenario path after `--` for a single test
6868
- `make test-conformance` — Gateway API conformance suite in
6969
`test/conformance/gatewayapi`
7070

71-
### Local Multi-Cluster Dev Environment (Taskfile)
72-
The dev environment is two Kind clusters: an **upstream** cluster
73-
(`nso-standard`) running the operator, and a **downstream** cluster
74-
(`nso-infra`) running Envoy Gateway, cert-manager, and external-dns.
75-
76-
- `task dev:bootstrap` — full setup: create both clusters, deploy the operator
77-
upstream, prepare downstream, and link them via a kubeconfig secret
78-
- `task dev:redeploy-operator` — rebuild the image, load it into `nso-standard`,
79-
and roll the controller-manager
80-
- `task dev:test [-- <path-or-name>]` — run the e2e suite (optionally a single
81-
test under `test/e2e/`)
82-
- `task dev:destroy` — tear down both clusters
71+
### Prod-Fidelity Test Environment (Taskfile)
72+
The e2e environment is two Kind clusters that mirror how the edge runs in
73+
production: an **upstream** cluster (`nso-upstream`) running the operator, and a
74+
**downstream** cluster (`nso-downstream`) running Envoy Gateway v1.7.4, the
75+
extension server + Coraza WAF data plane, cert-manager, and external-dns. It is
76+
defined in `Taskfile.test-infra.yml` and is what CI runs.
77+
78+
- `task test-infra:up` — build the operator image, create both clusters, install
79+
the downstream data plane, deploy + link the operator
80+
- `task test-infra:test-e2e [-- <path>]` — run the e2e suite against the live env
81+
(JSON report written to `$TMPDIR`); optionally a single scenario
82+
- `task test-infra:down` — tear down both clusters
8383
- `task validate-kustomizations` — `kustomize build` every kustomization
8484
- `task test-prometheus-rules` — `promtool` tests for the alerting rules in
8585
`test/prometheus-rules/`
8686

87+
#### Authoring e2e tests against the env
88+
- Bind to the shared data plane the env stands up: downstream GatewayClass
89+
`datum-downstream-gateway-e2e`, Envoy Gateway namespace
90+
`datum-downstream-gateway`, extension server in
91+
`network-services-operator-system`. Do **not** hardcode the bare
92+
`datum-downstream-gateway` class or self-provision a per-test EnvoyProxy in
93+
`envoy-gateway-system` — that was the retired single-stack bridge's model.
94+
- Every scenario under `test/e2e/` runs against this env in CI. Any setup a
95+
scenario needs (a data-plane component, a namespace) must be installed by
96+
`task test-infra:up` (add it to `Taskfile.test-infra.yml`), not assumed from
97+
outside — a scenario whose dependency isn't provisioned by the env will fail.
98+
- Editing `.github/workflows/*` requires pushing over SSH; the HTTPS token used
99+
by `gh` lacks the `workflow` scope.
100+
87101
## High-Level Architecture
88102

89103
### Multi-Cluster (upstream / downstream)
@@ -139,7 +153,7 @@ project follows the Kubebuilder v4 layout (`PROJECT`, domain `datumapis.com`).
139153
2. Run `make manifests generate` to regenerate CRDs, RBAC, deepcopy, defaulters.
140154
3. Run `make api-docs` if the CRD reference under `docs/api/` should update.
141155
4. Add/extend reconciler logic in `internal/controller/` and a `_test.go`.
142-
5. `make test` (envtest) and, when relevant, `make test-e2e`.
156+
5. `make test` (envtest) and, when relevant, `task test-infra:test-e2e`.
143157

144158
Never hand-edit generated files (`zz_generated.*`, `config/crd/bases/*`,
145159
`docs/api/*`) — change the source and regenerate.
@@ -154,7 +168,8 @@ Operator deployment config is composed from Kustomize bases and overlays under
154168

155169
### CI
156170
GitHub Actions enforce the same commands locally available:
157-
`make test` (test.yml), `make test-e2e` (test-e2e.yml), `golangci-lint`
171+
`make test` (test.yml), the `test-infra` e2e env (test-e2e.yml, via
172+
`task test-infra:up` + `task test-infra:test-e2e`), `golangci-lint`
158173
(lint.yml), and kustomize validation (validate-kustomize.yaml). Run them
159174
locally before pushing.
160175

‎Makefile‎

Lines changed: 4 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -64,27 +64,9 @@ vet: ## Run go vet against code.
6464
test: manifests generate fmt vet envtest ## Run tests.
6565
KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out
6666

67-
# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'.
68-
# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally.
69-
# Prometheus and CertManager are installed by default; skip with:
70-
# - PROMETHEUS_INSTALL_SKIP=true
71-
# - CERT_MANAGER_INSTALL_SKIP=true
72-
.PHONY: test-e2e
73-
test-e2e: chainsaw
74-
@command -v kind >/dev/null 2>&1 || { \
75-
echo "Kind is not installed. Please install Kind manually."; \
76-
exit 1; \
77-
}
78-
@kind get clusters | grep -q 'nso-standard' || { \
79-
echo "No Kind cluster is running. Please start a Kind cluster before running the e2e tests."; \
80-
exit 1; \
81-
}
82-
$(KIND) get kubeconfig --name nso-standard > $(TMPDIR)/.kind-nso-standard.yaml
83-
$(KIND) get kubeconfig --name nso-infra > $(TMPDIR)/.kind-nso-infra.yaml
84-
$(CHAINSAW) test $(or $(TEST_DIR),./test/e2e) \
85-
--parallel 1 \
86-
--cluster nso-standard=$(TMPDIR)/.kind-nso-standard.yaml \
87-
--cluster nso-infra=$(TMPDIR)/.kind-nso-infra.yaml
67+
# The e2e suite runs against the two-cluster prod-fidelity env; bring it up and
68+
# run it with `task test-infra:up` then `task test-infra:test-e2e`
69+
# (Taskfile.test-infra.yml).
8870

8971
GATEWAY_CONFORMANCE_CLASS ?= gateway-conformance
9072
GATEWAY_CONFORMANCE_FLAGS ?=
@@ -153,76 +135,14 @@ build-installer: set-image-controller generate ## Generate a consolidated YAML w
153135
set-image-controller: manifests kustomize
154136
cd config/manager && $(KUSTOMIZE) edit set image ghcr.io/datum-cloud/network-services-operator=${IMG}
155137

156-
.PHONY: prepare-infra-cluster
157-
prepare-infra-cluster: cert-manager envoy-gateway external-dns downstream-crds downstream-waf-dataplane billing-usage-collector load-image-infra extension-server configure-eg-extension-manager
158-
159-
.PHONY: downstream-crds
160-
downstream-crds: ## Install NSO CRDs on the downstream (infra) cluster that the replicator mirrors into it.
161-
$(KUBECTL) apply -f config/crd/bases/networking.datumapis.com_connectors.yaml
162-
$(KUBECTL) apply -f config/crd/bases/networking.datumapis.com_httpproxies.yaml
163-
$(KUBECTL) apply -f config/crd/bases/networking.datumapis.com_trafficprotectionpolicies.yaml
164-
165-
.PHONY: downstream-waf-dataplane
166-
downstream-waf-dataplane: kustomize load-image-infra ## Install the Coraza WAF data plane (downstream EG instance + extension server + datum-downstream-gateway GatewayClass) on the infra cluster.
167-
$(KUSTOMIZE) build --enable-helm config/tools/envoy-gateway-downstream | kubectl apply --server-side=true --force-conflicts -f -
168-
$(KUSTOMIZE) build config/e2e-downstream | kubectl apply --server-side=true --force-conflicts -f -
169-
170-
.PHONY: prepare-e2e
171-
prepare-e2e: chainsaw set-image-controller cert-manager load-image-all deploy-e2e
172-
173138
.PHONY: prepare-dev
174139
prepare-dev: chainsaw set-image-controller cert-manager install
175140

176-
.PHONY: load-image-all
177-
load-image-all: load-image-operator load-image-infra
178-
179-
.PHONY: load-image-operator
180-
load-image-operator: docker-build kind
181-
$(KIND) load docker-image $(IMG) -n nso-standard
182-
183-
.PHONY: load-image-infra
184-
load-image-infra: docker-build kind ## Load operator image into nso-infra kind cluster (needed by the extension server).
185-
$(KIND) load docker-image $(IMG) -n nso-infra
186-
187141
.PHONY: cert-manager
188142
cert-manager: cmctl
189143
$(KUSTOMIZE) build --enable-helm config/tools/cert-manager | kubectl apply --server-side=true --force-conflicts -f -
190144
$(CMCTL) check api --wait=5m
191145

192-
.PHONY: envoy-gateway
193-
envoy-gateway:
194-
$(KUSTOMIZE) build --enable-helm config/tools/envoy-gateway | kubectl apply --server-side=true --force-conflicts -f -
195-
196-
.PHONY: external-dns
197-
external-dns:
198-
$(KUSTOMIZE) build --enable-helm config/tools/external-dns | kubectl apply --server-side=true --force-conflicts -f -
199-
200-
.PHONY: billing-usage-collector
201-
billing-usage-collector:
202-
$(KUSTOMIZE) build --enable-helm config/tools/billing-usage-collector | kubectl apply --server-side=true --force-conflicts -f -
203-
204-
.PHONY: extension-server
205-
extension-server: ## Deploy the NSO extension server to the infra cluster (e2e overlay with cert-manager-issued TLS).
206-
$(KUSTOMIZE) build --enable-helm config/extension-server-e2e | kubectl apply --server-side=true --force-conflicts -f -
207-
kubectl rollout restart deployment/network-services-operator-envoy-gateway-extension-server \
208-
-n network-services-operator-system
209-
kubectl rollout status deployment/network-services-operator-envoy-gateway-extension-server \
210-
-n network-services-operator-system --timeout=5m
211-
212-
.PHONY: configure-eg-extension-manager
213-
configure-eg-extension-manager: ## Patch the EG ConfigMap to enable extensionManager and restart the EG controller.
214-
$(KUSTOMIZE) build --enable-helm config/tools/envoy-gateway/overlays/e2e | kubectl apply --server-side=true --force-conflicts -f -
215-
kubectl rollout restart deployment/envoy-gateway -n envoy-gateway-system
216-
kubectl rollout status deployment/envoy-gateway -n envoy-gateway-system --timeout=3m
217-
218-
.PHONY: kind-standard-cluster
219-
kind-standard-cluster: kind
220-
$(KIND) create cluster --config=config/tools/kind/standard-cluster.yaml
221-
222-
.PHONY: kind-infra-cluster
223-
kind-infra-cluster: kind
224-
$(KIND) create cluster --config=config/tools/kind/infra-cluster.yaml
225-
226146
##@ Deployment
227147

228148
ifndef ignore-not-found
@@ -241,9 +161,6 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified
241161
deploy: set-image-controller ## Deploy controller to the K8s cluster specified in ~/.kube/config.
242162
$(KUSTOMIZE) build config/default | $(KUBECTL) apply -f -
243163

244-
.PHONY: deploy-e2e
245-
deploy-e2e: set-image-controller
246-
$(KUSTOMIZE) build config/e2e | $(KUBECTL) apply -f -
247164

248165
.PHONY: undeploy
249166
undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
@@ -284,7 +201,7 @@ CERTMANAGER_VERSION ?= 1.17.1
284201
CRDOC_VERSION ?= v0.6.4
285202

286203
# renovate: datasource=go depName=sigs.k8s.io/kind
287-
KIND_VERSION ?= v0.27.0
204+
KIND_VERSION ?= v0.32.0
288205

289206
# renovate: datasource=go depName=github.com/kyverno/chainsaw
290207
CHAINSAW_VERSION ?= v0.2.15

0 commit comments

Comments
 (0)