diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index badc3109..420fe171 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -7,6 +7,7 @@ on: pull_request: branches: - main + workflow_dispatch: jobs: test-e2e: @@ -23,70 +24,58 @@ jobs: with: go-version: '~1.26' - - name: Verify kind installation - run: kind version - - - name: Create upstream kind cluster - run: make kind-standard-cluster - - - name: Create downstream kind cluster - run: | - make kind-infra-cluster - kubectl config use-context kind-nso-standard - - - name: Prepare e2e - run: | - kubectl config use-context kind-nso-standard - make prepare-e2e - - kubectl config use-context kind-nso-infra - make prepare-infra-cluster + - name: Install Task + uses: arduino/setup-task@v2 + with: + version: 3.x + repo-token: ${{ secrets.GITHUB_TOKEN }} - kind get kubeconfig --name nso-infra --internal > ${TMPDIR}/.kind-nso-infra-internal.yaml + - name: Install Helm + uses: azure/setup-helm@v4 + with: + # kustomize v5.5.0 probes helm with `helm version -c`; the -c shorthand + # was removed in helm 3.13, so pin to the last release that accepts it. + version: v3.12.3 - kubectl config use-context kind-nso-standard - kubectl create secret -n network-services-operator-system \ - generic downstream-cluster-kubeconfig \ - --from-file=kubeconfig=${TMPDIR}/.kind-nso-infra-internal.yaml + - name: Install pinned tools + run: make kind kustomize cmctl chainsaw - # Wait for the operator to be ready - kubectl -n network-services-operator-system \ - wait deploy network-services-operator-controller-manager \ - --for=condition=Available \ - --timeout=120s || \ - kubectl -n cert-manager describe pods && \ - kubectl logs -n cert-manager -l app.kubernetes.io/name=cert-manager --tail=-1 && \ - kubectl -n network-services-operator-system describe issuer && \ - kubectl -n network-services-operator-system describe certificates && \ - kubectl -n network-services-operator-system describe secrets && \ - kubectl -n network-services-operator-system describe pod && \ - kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=-1 + - name: Bring up the prod-fidelity env + run: task test-infra:up - - name: Running Test e2e - run: make test-e2e + - name: Run e2e suite + run: task test-infra:test-e2e - - name: Dump downstream WAF data plane on failure + - name: Dump the downstream data plane on failure if: failure() run: | - kubectl config use-context kind-nso-infra - echo "::group::GatewayClasses" - kubectl get gatewayclass -o wide || true - kubectl describe gatewayclass datum-downstream-gateway || true + set +e + UKC=${TMPDIR}/.kind-nso-upstream.yaml + bin/kind-v0.32.0 get kubeconfig --name nso-upstream > "${UKC}" + echo "::group::Upstream NSO manager" + KUBECONFIG="${UKC}" kubectl -n network-services-operator-system get pods -o wide + KUBECONFIG="${UKC}" kubectl -n network-services-operator-system describe deploy network-services-operator-controller-manager + KUBECONFIG="${UKC}" kubectl -n network-services-operator-system describe pod -l app.kubernetes.io/name=network-services-operator + KUBECONFIG="${UKC}" kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=200 --all-containers echo "::endgroup::" - echo "::group::EnvoyProxy" - kubectl -n datum-downstream-gateway get envoyproxy -o yaml || true + echo "::group::Upstream cert-manager + webhook cert" + KUBECONFIG="${UKC}" kubectl -n cert-manager get pods -o wide + KUBECONFIG="${UKC}" kubectl -n network-services-operator-system get certificate,secret echo "::endgroup::" - echo "::group::datum-downstream-gateway namespace + pods" - kubectl get ns datum-downstream-gateway --show-labels || true - kubectl -n datum-downstream-gateway get pods -o wide || true + KC=${TMPDIR}/.kind-nso-downstream.yaml + bin/kind-v0.32.0 get kubeconfig --name nso-downstream > "${KC}" + export KUBECONFIG="${KC}" + echo "::group::GatewayClasses" + kubectl get gatewayclass -o wide echo "::endgroup::" - echo "::group::Downstream Envoy Gateway logs" - kubectl -n datum-downstream-gateway logs deploy/envoy-gateway --tail=200 || true + echo "::group::Gateways (all namespaces)" + kubectl get gateway -A -o wide echo "::endgroup::" - echo "::group::Extension server" - kubectl -n datum-downstream-gateway describe deploy envoy-gateway-extension-server || true - kubectl -n datum-downstream-gateway logs -l app.kubernetes.io/component=envoy-gateway-extension-server --tail=200 || true + echo "::group::datum-downstream-gateway pods" + kubectl -n datum-downstream-gateway get pods -o wide + kubectl -n datum-downstream-gateway logs deploy/envoy-gateway --tail=200 echo "::endgroup::" - echo "::group::Certificates" - kubectl -n datum-downstream-gateway get certificate,secret || true + echo "::group::Extension server" + kubectl -n network-services-operator-system get pods -o wide + kubectl -n network-services-operator-system logs -l app.kubernetes.io/component=envoy-gateway-extension-server --tail=200 echo "::endgroup::" diff --git a/CLAUDE.md b/CLAUDE.md index 7bfabd48..e9a8034d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -63,27 +63,41 @@ Run `make help` for the full list. Most-used targets: ### Tests - `make test` — unit/integration tests via **envtest** (excludes `test/e2e`); runs `manifests generate fmt vet` first, writes `cover.out` -- `make test-e2e` — **Chainsaw** e2e suite in `test/e2e` against the two local - Kind clusters (requires `nso-standard` and `nso-infra` to exist) +- `task test-infra:test-e2e` — **Chainsaw** e2e suite in `test/e2e` against the + prod-fidelity env (below); pass a scenario path after `--` for a single test - `make test-conformance` — Gateway API conformance suite in `test/conformance/gatewayapi` -### Local Multi-Cluster Dev Environment (Taskfile) -The dev environment is two Kind clusters: an **upstream** cluster -(`nso-standard`) running the operator, and a **downstream** cluster -(`nso-infra`) running Envoy Gateway, cert-manager, and external-dns. - -- `task dev:bootstrap` — full setup: create both clusters, deploy the operator - upstream, prepare downstream, and link them via a kubeconfig secret -- `task dev:redeploy-operator` — rebuild the image, load it into `nso-standard`, - and roll the controller-manager -- `task dev:test [-- ]` — run the e2e suite (optionally a single - test under `test/e2e/`) -- `task dev:destroy` — tear down both clusters +### Prod-Fidelity Test Environment (Taskfile) +The e2e environment is two Kind clusters that mirror how the edge runs in +production: an **upstream** cluster (`nso-upstream`) running the operator, and a +**downstream** cluster (`nso-downstream`) running Envoy Gateway v1.7.4, the +extension server + Coraza WAF data plane, cert-manager, and external-dns. It is +defined in `Taskfile.test-infra.yml` and is what CI runs. + +- `task test-infra:up` — build the operator image, create both clusters, install + the downstream data plane, deploy + link the operator +- `task test-infra:test-e2e [-- ]` — run the e2e suite against the live env + (JSON report written to `$TMPDIR`); optionally a single scenario +- `task test-infra:down` — tear down both clusters - `task validate-kustomizations` — `kustomize build` every kustomization - `task test-prometheus-rules` — `promtool` tests for the alerting rules in `test/prometheus-rules/` +#### Authoring e2e tests against the env +- Bind to the shared data plane the env stands up: downstream GatewayClass + `datum-downstream-gateway-e2e`, Envoy Gateway namespace + `datum-downstream-gateway`, extension server in + `network-services-operator-system`. Do **not** hardcode the bare + `datum-downstream-gateway` class or self-provision a per-test EnvoyProxy in + `envoy-gateway-system` — that was the retired single-stack bridge's model. +- Every scenario under `test/e2e/` runs against this env in CI. Any setup a + scenario needs (a data-plane component, a namespace) must be installed by + `task test-infra:up` (add it to `Taskfile.test-infra.yml`), not assumed from + outside — a scenario whose dependency isn't provisioned by the env will fail. +- Editing `.github/workflows/*` requires pushing over SSH; the HTTPS token used + by `gh` lacks the `workflow` scope. + ## High-Level Architecture ### Multi-Cluster (upstream / downstream) @@ -139,7 +153,7 @@ project follows the Kubebuilder v4 layout (`PROJECT`, domain `datumapis.com`). 2. Run `make manifests generate` to regenerate CRDs, RBAC, deepcopy, defaulters. 3. Run `make api-docs` if the CRD reference under `docs/api/` should update. 4. Add/extend reconciler logic in `internal/controller/` and a `_test.go`. -5. `make test` (envtest) and, when relevant, `make test-e2e`. +5. `make test` (envtest) and, when relevant, `task test-infra:test-e2e`. Never hand-edit generated files (`zz_generated.*`, `config/crd/bases/*`, `docs/api/*`) — change the source and regenerate. @@ -154,7 +168,8 @@ Operator deployment config is composed from Kustomize bases and overlays under ### CI GitHub Actions enforce the same commands locally available: -`make test` (test.yml), `make test-e2e` (test-e2e.yml), `golangci-lint` +`make test` (test.yml), the `test-infra` e2e env (test-e2e.yml, via +`task test-infra:up` + `task test-infra:test-e2e`), `golangci-lint` (lint.yml), and kustomize validation (validate-kustomize.yaml). Run them locally before pushing. diff --git a/Makefile b/Makefile index 5d9a38b6..714b033d 100644 --- a/Makefile +++ b/Makefile @@ -64,27 +64,9 @@ vet: ## Run go vet against code. test: manifests generate fmt vet envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out -# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'. -# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. -# Prometheus and CertManager are installed by default; skip with: -# - PROMETHEUS_INSTALL_SKIP=true -# - CERT_MANAGER_INSTALL_SKIP=true -.PHONY: test-e2e -test-e2e: chainsaw - @command -v kind >/dev/null 2>&1 || { \ - echo "Kind is not installed. Please install Kind manually."; \ - exit 1; \ - } - @kind get clusters | grep -q 'nso-standard' || { \ - echo "No Kind cluster is running. Please start a Kind cluster before running the e2e tests."; \ - exit 1; \ - } - $(KIND) get kubeconfig --name nso-standard > $(TMPDIR)/.kind-nso-standard.yaml - $(KIND) get kubeconfig --name nso-infra > $(TMPDIR)/.kind-nso-infra.yaml - $(CHAINSAW) test $(or $(TEST_DIR),./test/e2e) \ - --parallel 1 \ - --cluster nso-standard=$(TMPDIR)/.kind-nso-standard.yaml \ - --cluster nso-infra=$(TMPDIR)/.kind-nso-infra.yaml +# The e2e suite runs against the two-cluster prod-fidelity env; bring it up and +# run it with `task test-infra:up` then `task test-infra:test-e2e` +# (Taskfile.test-infra.yml). GATEWAY_CONFORMANCE_CLASS ?= gateway-conformance GATEWAY_CONFORMANCE_FLAGS ?= @@ -153,76 +135,14 @@ build-installer: set-image-controller generate ## Generate a consolidated YAML w set-image-controller: manifests kustomize cd config/manager && $(KUSTOMIZE) edit set image ghcr.io/datum-cloud/network-services-operator=${IMG} -.PHONY: prepare-infra-cluster -prepare-infra-cluster: cert-manager envoy-gateway external-dns downstream-crds downstream-waf-dataplane billing-usage-collector load-image-infra extension-server configure-eg-extension-manager - -.PHONY: downstream-crds -downstream-crds: ## Install NSO CRDs on the downstream (infra) cluster that the replicator mirrors into it. - $(KUBECTL) apply -f config/crd/bases/networking.datumapis.com_connectors.yaml - $(KUBECTL) apply -f config/crd/bases/networking.datumapis.com_httpproxies.yaml - $(KUBECTL) apply -f config/crd/bases/networking.datumapis.com_trafficprotectionpolicies.yaml - -.PHONY: downstream-waf-dataplane -downstream-waf-dataplane: kustomize load-image-infra ## Install the Coraza WAF data plane (downstream EG instance + extension server + datum-downstream-gateway GatewayClass) on the infra cluster. - $(KUSTOMIZE) build --enable-helm config/tools/envoy-gateway-downstream | kubectl apply --server-side=true --force-conflicts -f - - $(KUSTOMIZE) build config/e2e-downstream | kubectl apply --server-side=true --force-conflicts -f - - -.PHONY: prepare-e2e -prepare-e2e: chainsaw set-image-controller cert-manager load-image-all deploy-e2e - .PHONY: prepare-dev prepare-dev: chainsaw set-image-controller cert-manager install -.PHONY: load-image-all -load-image-all: load-image-operator load-image-infra - -.PHONY: load-image-operator -load-image-operator: docker-build kind - $(KIND) load docker-image $(IMG) -n nso-standard - -.PHONY: load-image-infra -load-image-infra: docker-build kind ## Load operator image into nso-infra kind cluster (needed by the extension server). - $(KIND) load docker-image $(IMG) -n nso-infra - .PHONY: cert-manager cert-manager: cmctl $(KUSTOMIZE) build --enable-helm config/tools/cert-manager | kubectl apply --server-side=true --force-conflicts -f - $(CMCTL) check api --wait=5m -.PHONY: envoy-gateway -envoy-gateway: - $(KUSTOMIZE) build --enable-helm config/tools/envoy-gateway | kubectl apply --server-side=true --force-conflicts -f - - -.PHONY: external-dns -external-dns: - $(KUSTOMIZE) build --enable-helm config/tools/external-dns | kubectl apply --server-side=true --force-conflicts -f - - -.PHONY: billing-usage-collector -billing-usage-collector: - $(KUSTOMIZE) build --enable-helm config/tools/billing-usage-collector | kubectl apply --server-side=true --force-conflicts -f - - -.PHONY: extension-server -extension-server: ## Deploy the NSO extension server to the infra cluster (e2e overlay with cert-manager-issued TLS). - $(KUSTOMIZE) build --enable-helm config/extension-server-e2e | kubectl apply --server-side=true --force-conflicts -f - - kubectl rollout restart deployment/network-services-operator-envoy-gateway-extension-server \ - -n network-services-operator-system - kubectl rollout status deployment/network-services-operator-envoy-gateway-extension-server \ - -n network-services-operator-system --timeout=5m - -.PHONY: configure-eg-extension-manager -configure-eg-extension-manager: ## Patch the EG ConfigMap to enable extensionManager and restart the EG controller. - $(KUSTOMIZE) build --enable-helm config/tools/envoy-gateway/overlays/e2e | kubectl apply --server-side=true --force-conflicts -f - - kubectl rollout restart deployment/envoy-gateway -n envoy-gateway-system - kubectl rollout status deployment/envoy-gateway -n envoy-gateway-system --timeout=3m - -.PHONY: kind-standard-cluster -kind-standard-cluster: kind - $(KIND) create cluster --config=config/tools/kind/standard-cluster.yaml - -.PHONY: kind-infra-cluster -kind-infra-cluster: kind - $(KIND) create cluster --config=config/tools/kind/infra-cluster.yaml - ##@ Deployment ifndef ignore-not-found @@ -241,9 +161,6 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified deploy: set-image-controller ## Deploy controller to the K8s cluster specified in ~/.kube/config. $(KUSTOMIZE) build config/default | $(KUBECTL) apply -f - -.PHONY: deploy-e2e -deploy-e2e: set-image-controller - $(KUSTOMIZE) build config/e2e | $(KUBECTL) apply -f - .PHONY: undeploy undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. @@ -284,7 +201,7 @@ CERTMANAGER_VERSION ?= 1.17.1 CRDOC_VERSION ?= v0.6.4 # renovate: datasource=go depName=sigs.k8s.io/kind -KIND_VERSION ?= v0.27.0 +KIND_VERSION ?= v0.32.0 # renovate: datasource=go depName=github.com/kyverno/chainsaw CHAINSAW_VERSION ?= v0.2.15 diff --git a/Taskfile.dev.yaml b/Taskfile.dev.yaml deleted file mode 100644 index 257b05ec..00000000 --- a/Taskfile.dev.yaml +++ /dev/null @@ -1,103 +0,0 @@ -version: '3' - -vars: - TMP_DIR: - sh: echo "${TMPDIR:-/tmp}" - -tasks: - bootstrap: - desc: "Bootstrap the multi-cluster dev environment (nso-standard and nso-infra) for local testing" - cmds: - - echo "🚀 Bootstrapping dev environment..." - - task: create-clusters - - task: prep-upstream - - task: prep-downstream - - task: link-clusters - - echo "🎉 Dev environment bootstrapped successfully! Context is now kind-nso-standard." - - create-clusters: - desc: "Create Kind upstream (standard) and downstream (infra) clusters" - cmds: - - echo "🧹 Cleaning up any existing clusters..." - - kind delete cluster --name nso-standard || true - - kind delete cluster --name nso-infra || true - - echo "🏗️ Creating upstream (nso-standard) cluster..." - - make kind-standard-cluster - - echo "🏗️ Creating downstream (nso-infra) cluster..." - - make kind-infra-cluster - - prep-upstream: - desc: "Prepare the upstream cluster with Operator and cert-manager" - cmds: - - echo "🔧 Preparing upstream (nso-standard)..." - - kubectl config use-context kind-nso-standard - - make prepare-e2e - - prep-downstream: - desc: "Prepare the downstream cluster with cert-manager, envoy-gateway, and external-dns" - cmds: - - echo "🔧 Preparing downstream (nso-infra)..." - - kubectl config use-context kind-nso-infra - - make prepare-infra-cluster - - link-clusters: - desc: "Link upstream and downstream clusters using kubeconfig secret" - cmds: - - echo "🔗 Linking clusters..." - - kind get kubeconfig --name nso-infra --internal > {{.TMP_DIR}}/.kind-nso-infra-internal.yaml - - kubectl config use-context kind-nso-standard - - | - kubectl create namespace network-services-operator-system --dry-run=client -o yaml | kubectl apply -f - - kubectl create secret -n network-services-operator-system \ - generic downstream-cluster-kubeconfig \ - --save-config \ - --dry-run=client -o yaml \ - --from-file=kubeconfig={{.TMP_DIR}}/.kind-nso-infra-internal.yaml | kubectl apply -f - - - echo "⏳ Waiting for operator controller manager deployment to be ready..." - - | - kubectl -n network-services-operator-system \ - wait deploy network-services-operator-controller-manager \ - --for=condition=Available \ - --timeout=180s - - redeploy-operator: - desc: "Rebuild the operator image, load it into nso-standard, and roll out the deployed controller" - cmds: - - echo "🔨 Building operator image and loading it into nso-standard..." - # docker-build + kind load docker-image $(IMG) -n nso-standard - - make load-image-operator - - echo "♻️ Restarting the controller-manager to pick up the new image..." - - kubectl config use-context kind-nso-standard - - | - kubectl -n network-services-operator-system \ - rollout restart deploy network-services-operator-controller-manager - - echo "⏳ Waiting for the new controller-manager rollout to complete..." - - | - kubectl -n network-services-operator-system \ - rollout status deploy network-services-operator-controller-manager \ - --timeout=180s - - echo "✅ Operator redeployed with the freshly built image." - - destroy: - desc: "Tear down the multi-cluster dev environment" - cmds: - - echo "💥 Destroying clusters..." - - kind delete cluster --name nso-standard || true - - kind delete cluster --name nso-infra || true - - rm -f {{.TMP_DIR}}/.kind-nso-infra-internal.yaml - - echo "✨ Cleanup finished." - - test: - desc: "Run E2E tests using chainsaw on the local multi-cluster setup" - cmds: - - echo "🧪 Running E2E tests..." - - | - if [ -n "{{.CLI_ARGS}}" ]; then - if [[ "{{.CLI_ARGS}}" == test/e2e/* || "{{.CLI_ARGS}}" == ./test/e2e/* ]]; then - make test-e2e TEST_DIR="{{.CLI_ARGS}}" - else - make test-e2e TEST_DIR="./test/e2e/{{.CLI_ARGS}}" - fi - else - make test-e2e - fi diff --git a/Taskfile.test-infra.yml b/Taskfile.test-infra.yml new file mode 100644 index 00000000..78b070f5 --- /dev/null +++ b/Taskfile.test-infra.yml @@ -0,0 +1,470 @@ +version: '3' + +# Production-fidelity test environment. Two kind clusters: nso-upstream (the +# control plane) and nso-downstream (the edge). Consumed by both the e2e and perf +# suites. +# +# Bring up: task test-infra:up +# Tear down: task test-infra:down +# Smoke: task test-infra:smoke + +# kustomize fetches some CRDs over https. A developer's global git config can +# rewrite https to ssh, which fails in a non-interactive fetch. Neutralize the +# global git config for kustomize's child git so the https fetches succeed; the +# repo-local config is untouched. +env: + GIT_CONFIG_GLOBAL: /dev/null + +vars: + TMP_DIR: + sh: echo "${TMPDIR:-/tmp}" + REPO: + sh: pwd + # Pinned-to-prod tooling. We use an explicit kind v0.32.0 binary because it + # ships the node image production runs, which the checked-in bin/kind predates. + KIND: '{{.REPO}}/bin/kind-v0.32.0' + KUSTOMIZE: '{{.REPO}}/bin/kustomize' + CMCTL: '{{.REPO}}/bin/cmctl' + KARMADACTL: '{{.REPO}}/bin/karmadactl' + CHAINSAW: '{{.REPO}}/bin/chainsaw' + + UPSTREAM_CLUSTER: nso-upstream + DOWNSTREAM_CLUSTER: nso-downstream + UPSTREAM_CTX: kind-nso-upstream + DOWNSTREAM_CTX: kind-nso-downstream + + # The exact node image the production edge runs. + K8S_NODE_IMAGE: kindest/node:v1.35.5@sha256:ce977ae6d65918d0b58a5f8b5e940429c2ce42fa3a5619ec2bbc60b949c0ac95 + # Gateway control-plane version, matching production. The extension-server SDK + # version is paired with it deliberately, also matching production. + ENVOY_GATEWAY_VERSION: v1.7.4 + EG_SDK_VERSION: v1.8.1 + # Multi-arch WAF image — the same filter as the production edge, but it also + # loads natively on arm64 dev hosts. The data plane references this image. + CORAZA_WAF_IMAGE: ghcr.io/datum-labs/coraza-envoy-go-filter/coraza-waf:v1.3.0-multiarch.1 + # CONNECT-proxy stand-in for the connector-tunnel scenario, built from + # test/e2e-edge/_fixtures/connector-tunnel and loaded into the downstream cluster. + CONNECT_PROXY_IMAGE: connect-proxy:e2e + # The WAF is enabled by default since the multi-arch image loads on every + # architecture. Set CORAZA_DISABLED=true only to deliberately skip the WAF. + CORAZA_DISABLED: '{{.CORAZA_DISABLED | default "false"}}' + # Karmada-bundled kube-apiserver, pinned to production. + KARMADA_APISERVER_VERSION: v1.33.2 + # NSO image pinned by git SHA so the loaded image is used deterministically. + IMG: + sh: echo "ghcr.io/datum-cloud/network-services-operator:$(git rev-parse --short HEAD)" + +tasks: + + up: + desc: "Bring the canonical two-cluster prod-fidelity test env ONLINE (clusters + EG v1.7.4 + ext-server + Coraza data plane + NSO manager)." + cmds: + - echo "🚀 test-infra:up — building nso-upstream + nso-downstream" + - task: clusters + # The Gateway-API and gateway CRDs must land on the downstream cluster + # before cert-manager (it probes the Gateway-API CRDs at startup) and before + # the downstream gateway (it needs its own CRDs present). + - task: eg-crds + - task: downstream-crds + - task: downstream-namespaces + - task: cert-manager-upstream + - task: cert-manager-downstream + - task: nso-image + - task: prepare-upstream + - task: eg-downstream + - task: extension-server + - task: billing-usage-collector + - task: external-dns + - task: link-clusters + - task: wait-ready + - echo "🎉 test-infra:up complete. Run 'task test-infra:smoke' to confirm M2." + + down: + desc: "Tear down the test-infra clusters (and any Karmada host)." + cmds: + - echo "💥 test-infra:down" + # Deleting both kind clusters removes all federation state, so this unjoin + # is only a best-effort courtesy. + - '{{.KARMADACTL}} deinit --context {{.UPSTREAM_CTX}} --force --purge-namespace 2>/dev/null || true' + - rm -rf {{.TMP_DIR}}/karmada-{{.UPSTREAM_CLUSTER}} + - '{{.KIND}} delete cluster --name {{.UPSTREAM_CLUSTER}} || true' + - '{{.KIND}} delete cluster --name {{.DOWNSTREAM_CLUSTER}} || true' + - rm -f {{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}-internal.yaml {{.TMP_DIR}}/.kind-{{.UPSTREAM_CLUSTER}}.yaml {{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}.yaml + - echo "✨ done." + + clusters: + desc: "Create the upstream + downstream kind clusters pinned to the prod edge node image." + cmds: + - echo "🏗️ creating clusters (node {{.K8S_NODE_IMAGE}})" + - '{{.KIND}} delete cluster --name {{.UPSTREAM_CLUSTER}} 2>/dev/null || true' + - '{{.KIND}} delete cluster --name {{.DOWNSTREAM_CLUSTER}} 2>/dev/null || true' + - '{{.KIND}} create cluster --image {{.K8S_NODE_IMAGE}} --config=config/tools/kind/upstream-cluster.yaml' + - '{{.KIND}} create cluster --image {{.K8S_NODE_IMAGE}} --config=config/tools/kind/downstream-cluster.yaml' + # Running the full federation control plane alongside the rest of the stack + # in one kind node exhausts the default file-watch limits on the macOS + # Docker VM, which crashloops a component with "too many open files" and + # blocks the member join. Raise the limits on both nodes. + - docker exec {{.UPSTREAM_CLUSTER}}-control-plane sysctl -w fs.inotify.max_user_instances=8192 fs.inotify.max_user_watches=1048576 + - docker exec {{.DOWNSTREAM_CLUSTER}}-control-plane sysctl -w fs.inotify.max_user_instances=8192 fs.inotify.max_user_watches=1048576 + + cert-manager-upstream: + desc: "Install cert-manager (+ CSI driver) on the upstream cluster." + cmds: + - '{{.KUSTOMIZE}} build --enable-helm config/tools/cert-manager | kubectl --context {{.UPSTREAM_CTX}} apply --server-side=true --force-conflicts -f -' + - '{{.CMCTL}} check api --context {{.UPSTREAM_CTX}} --wait=5m' + + cert-manager-downstream: + desc: "Install cert-manager (+ CSI driver) on the downstream cluster." + cmds: + - '{{.KUSTOMIZE}} build --enable-helm config/tools/cert-manager | kubectl --context {{.DOWNSTREAM_CTX}} apply --server-side=true --force-conflicts -f -' + - '{{.CMCTL}} check api --context {{.DOWNSTREAM_CTX}} --wait=5m' + + nso-image: + desc: "Build the NSO operator image (git-SHA tag) and load it into both clusters." + cmds: + - echo "🔨 building {{.IMG}}" + - docker build -t {{.IMG}} . + - cd config/manager && {{.KUSTOMIZE}} edit set image ghcr.io/datum-cloud/network-services-operator={{.IMG}} + - '{{.KIND}} load docker-image {{.IMG}} --name {{.UPSTREAM_CLUSTER}}' + - '{{.KIND}} load docker-image {{.IMG}} --name {{.DOWNSTREAM_CLUSTER}}' + + prepare-upstream: + desc: "Deploy the NSO manager + webhook (config/e2e) on the upstream cluster, with the prod-base memory profile." + cmds: + - echo "🔧 deploying NSO manager (upstream)" + - '{{.KUSTOMIZE}} build config/e2e | kubectl --context {{.UPSTREAM_CTX}} apply --server-side=true --force-conflicts -f -' + # Match production: with the extension server handling proxy configuration, + # NSO must not also emit its own patch policies. The shared config file is + # also used by an older path that still relies on them, so we patch the live + # config here rather than editing that file. + - | + CFG=$(kubectl --context {{.UPSTREAM_CTX}} -n network-services-operator-system get cm network-services-operator-config -o jsonpath='{.data.config\.yaml}') + if ! echo "$CFG" | grep -q "eppEmissionEnabled"; then + NEW=$(echo "$CFG" | sed 's|^ downstreamGatewayClassName: .*|&\n eppEmissionEnabled: false|') + kubectl --context {{.UPSTREAM_CTX}} -n network-services-operator-system create cm network-services-operator-config \ + --from-literal=config.yaml="$NEW" --dry-run=client -o yaml | kubectl --context {{.UPSTREAM_CTX}} apply -f - + fi + # Raise the manager's memory limit to the production base so perf sweeps and + # webhook-under-load reflect production headroom. + - | + kubectl --context {{.UPSTREAM_CTX}} -n network-services-operator-system patch deploy network-services-operator-controller-manager \ + --type=json -p '[{"op":"replace","path":"/spec/template/spec/containers/0/resources/limits/memory","value":"2Gi"},{"op":"replace","path":"/spec/template/spec/containers/0/resources/requests/memory","value":"512Mi"}]' || true + + eg-crds: + desc: "Install the full Gateway-API + Envoy Gateway CRD set (incl ReferenceGrant + EnvoyProxy) on the downstream cluster. The downstream EG chart sets includeCRDs:false, so CRDs are applied separately (matches prod, where CRDs are managed out-of-band)." + vars: + EG_CRD_DIR: config/tools/envoy-gateway-downstream/charts/gateway-helm-{{.ENVOY_GATEWAY_VERSION}}/gateway-helm/crds + cmds: + # The gateway-helm chart is gitignored (config/**/charts); vendor it first + # so the CRD files below exist on a fresh checkout (e.g. CI). Output is + # discarded — this runs for the side effect of pulling the chart. + - '{{.KUSTOMIZE}} build --enable-helm config/tools/envoy-gateway-downstream >/dev/null' + - kubectl --context {{.DOWNSTREAM_CTX}} apply --server-side=true --force-conflicts -f {{.EG_CRD_DIR}}/gatewayapi-crds.yaml + - kubectl --context {{.DOWNSTREAM_CTX}} apply --server-side=true --force-conflicts -f {{.EG_CRD_DIR}}/generated/ + + downstream-namespaces: + desc: "Create the downstream gateway + hostname-accounting namespaces, plus the EG-watched e2e-direct namespace for hand-delivered fixtures (D1/D2). Runs BEFORE eg-downstream so e2e-direct carries its watch label before EG establishes its informer." + cmds: + - kubectl --context {{.DOWNSTREAM_CTX}} apply -f config/dev/downstream_resources/namespaces.yaml + # e2e-direct carries the gateway watch label from creation so gateways + # applied directly here reconcile deterministically. The gateway only + # reliably watches a namespace labeled at creation time, so the label must + # be in the manifest, not added afterward. + - kubectl --context {{.DOWNSTREAM_CTX}} apply -f config/e2e-downstream/direct-namespace.yaml + + billing-usage-collector: + desc: "Install the billing usage collector (Vector agent) that tails the downstream Envoy access logs and emits usage CloudEvents." + cmds: + - echo "🔧 installing billing-usage-collector (Vector agent, tailing datum-downstream-gateway)" + - '{{.KUSTOMIZE}} build --enable-helm config/tools/billing-usage-collector | kubectl --context {{.DOWNSTREAM_CTX}} apply --server-side=true --force-conflicts -f -' + + eg-downstream: + desc: "Install the dedicated downstream Envoy Gateway (v1.7.4) with the ext-server extensionManager wiring on the downstream cluster." + cmds: + - echo "🔧 installing downstream EG {{.ENVOY_GATEWAY_VERSION}} (+ extensionManager, failOpen:false, maxMessageSize:256Mi)" + # Build from the untracked config/e2e-downstream/eg-downstream copy, which + # bakes in the e2e pins, so a hard reset on the shared branch cannot revert + # them — it kept clobbering the tracked copy. + - '{{.KUSTOMIZE}} build --enable-helm config/e2e-downstream/eg-downstream | kubectl --context {{.DOWNSTREAM_CTX}} apply --server-side=true --force-conflicts -f -' + + load-coraza-waf: + desc: "Fallback: pull the Coraza WAF image and kind-load it into the downstream cluster (offline-CI escape hatch)." + cmds: + - docker pull {{.CORAZA_WAF_IMAGE}} + - '{{.KIND}} load docker-image {{.CORAZA_WAF_IMAGE}} --name {{.DOWNSTREAM_CLUSTER}}' + + load-fixtures: + desc: "Build + load the e2e fixture images into the downstream cluster (CONNECT-proxy stand-in for the connector-tunnel scenario)." + cmds: + - echo "🔧 building + loading fixture image {{.CONNECT_PROXY_IMAGE}}" + - docker build -t {{.CONNECT_PROXY_IMAGE}} test/e2e-edge/_fixtures/connector-tunnel + - '{{.KIND}} load docker-image {{.CONNECT_PROXY_IMAGE}} --name {{.DOWNSTREAM_CLUSTER}}' + + d1-mint-expired-secret: + desc: "D1 helper: mint an ALREADY-EXPIRED kubernetes.io/tls Secret and apply it to the downstream cluster, bypassing the upstream #212 cert-health gate so the ext-server prune backstop can be tested in isolation. Vars: NAMESPACE, SECRET, HOSTNAME." + vars: + # Default to the pre-provisioned, gateway-watched namespace so a gateway + # applied alongside this secret reconciles deterministically. + NAMESPACE: '{{.NAMESPACE | default "e2e-direct"}}' + SECRET: '{{.SECRET | default "d1-expired-tls"}}' + HOSTNAME: '{{.HOSTNAME | default "d1-bad.e2e.env.datum.net"}}' + cmds: + - | + config/e2e-downstream/d1-cert-bypass/mint-expired-secret.sh \ + "{{.NAMESPACE}}" "{{.SECRET}}" "{{.HOSTNAME}}" \ + | kubectl --context {{.DOWNSTREAM_CTX}} apply -f - + - echo "✅ applied expired TLS Secret {{.NAMESPACE}}/{{.SECRET}} (host {{.HOSTNAME}}) to {{.DOWNSTREAM_CLUSTER}}" + + extension-server: + desc: "Deploy the ext-server (2 replicas + PDB + mTLS) + e2e cert chain + Coraza/branded-page config + test EnvoyProxy (Coraza+admin:19000) on the downstream cluster." + cmds: + - echo "🔧 deploying ext-server + e2e issuer chain + test EnvoyProxy (config/e2e-downstream)" + # Pin the ext-server image to the git-SHA build (same image as the manager). + - cd config/e2e-downstream && {{.KUSTOMIZE}} edit set image ghcr.io/datum-cloud/network-services-operator={{.IMG}} 2>/dev/null || true + - '{{.KUSTOMIZE}} build config/e2e-downstream | kubectl --context {{.DOWNSTREAM_CTX}} apply --server-side=true --force-conflicts -f -' + # WAF disable toggle. The overlay ships the WAF enabled for production + # fidelity; when CORAZA_DISABLED is set we flip the live config so listeners + # program without it. + - | + if [ "{{.CORAZA_DISABLED}}" = "true" ]; then + echo "⚠️ Coraza WAF disabled (host arch {{OS}}/$(uname -m); WAF .so is amd64-only)" + kubectl --context {{.DOWNSTREAM_CTX}} -n network-services-operator-system get cm extension-server-config -o yaml \ + | sed 's/disabled: false/disabled: true/' | kubectl --context {{.DOWNSTREAM_CTX}} apply -f - + fi + # The extension server's server cert and the gateway's client cert are both + # issued from the e2e certificate authority. Wait for that authority's cert, + # then publish it where each side reads it so they can verify each other. + - task: extserver-ca-bundle + + extserver-ca-bundle: + desc: "Publish the e2e CA's ca.crt into the ext-server CA-bundle ConfigMap and the EG certificateRef Secret (both in network-services-operator-system)." + cmds: + - echo "⏳ waiting for the e2e CA certificate to be issued" + - | + kubectl --context {{.DOWNSTREAM_CTX}} -n cert-manager wait certificate e2e-extension-server-ca \ + --for=condition=Ready --timeout=120s + - kubectl --context {{.DOWNSTREAM_CTX}} create namespace network-services-operator-system --dry-run=client -o yaml | kubectl --context {{.DOWNSTREAM_CTX}} apply -f - + - | + CA_CRT=$(kubectl --context {{.DOWNSTREAM_CTX}} -n cert-manager get secret e2e-extension-server-ca -o jsonpath='{.data.ca\.crt}' | base64 -d) + # The certificate-authority bundle the extension server mounts. + kubectl --context {{.DOWNSTREAM_CTX}} -n network-services-operator-system create configmap extension-server-ca-bundle \ + --from-literal=ca.crt="$CA_CRT" --dry-run=client -o yaml | kubectl --context {{.DOWNSTREAM_CTX}} apply -f - + # The same authority cert the downstream gateway references. The gateway + # reads it from the tls.crt key, not ca.crt. + kubectl --context {{.DOWNSTREAM_CTX}} -n network-services-operator-system create secret generic e2e-extension-server-ca \ + --from-literal=tls.crt="$CA_CRT" --dry-run=client -o yaml | kubectl --context {{.DOWNSTREAM_CTX}} apply -f - + # Restart the extension server so it picks up the freshly-published bundle. + - kubectl --context {{.DOWNSTREAM_CTX}} -n network-services-operator-system rollout restart deploy network-services-operator-envoy-gateway-extension-server || true + + downstream-crds: + desc: "Install the NSO CRDs the replicator mirrors into the downstream cluster (the Gateway-API/EG CRDs come from eg-crds)." + cmds: + - kubectl --context {{.DOWNSTREAM_CTX}} apply -f config/crd/bases/networking.datumapis.com_connectors.yaml + - kubectl --context {{.DOWNSTREAM_CTX}} apply -f config/crd/bases/networking.datumapis.com_httpproxies.yaml + - kubectl --context {{.DOWNSTREAM_CTX}} apply -f config/crd/bases/networking.datumapis.com_trafficprotectionpolicies.yaml + + external-dns: + desc: "Install external-dns CRDs (DNSEndpoint) on the downstream cluster." + cmds: + - '{{.KUSTOMIZE}} build --enable-helm config/tools/external-dns | kubectl --context {{.DOWNSTREAM_CTX}} apply --server-side=true --force-conflicts -f -' + + link-clusters: + desc: "Wire the NSO manager's downstream client to the downstream cluster via the downstream-cluster-kubeconfig secret on the upstream cluster." + cmds: + - echo "🔗 linking upstream -> downstream" + - '{{.KIND}} get kubeconfig --name {{.DOWNSTREAM_CLUSTER}} --internal > {{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}-internal.yaml' + - kubectl --context {{.UPSTREAM_CTX}} create namespace network-services-operator-system --dry-run=client -o yaml | kubectl --context {{.UPSTREAM_CTX}} apply -f - + - | + kubectl --context {{.UPSTREAM_CTX}} create secret -n network-services-operator-system \ + generic downstream-cluster-kubeconfig --save-config --dry-run=client -o yaml \ + --from-file=kubeconfig={{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}-internal.yaml | kubectl --context {{.UPSTREAM_CTX}} apply -f - + + wait-ready: + desc: "Wait for the core components (NSO manager, downstream EG, ext-server) to be Available." + cmds: + - echo "⏳ waiting for NSO manager (upstream)" + - kubectl --context {{.UPSTREAM_CTX}} -n network-services-operator-system wait deploy network-services-operator-controller-manager --for=condition=Available --timeout=360s + - echo "⏳ waiting for downstream EG" + # The gateway control-plane Deployment is named `envoy-gateway` by its chart, + # not after the release name. + - kubectl --context {{.DOWNSTREAM_CTX}} -n datum-downstream-gateway wait deploy envoy-gateway --for=condition=Available --timeout=240s + - echo "⏳ waiting for ext-server" + - kubectl --context {{.DOWNSTREAM_CTX}} -n network-services-operator-system wait deploy network-services-operator-envoy-gateway-extension-server --for=condition=Available --timeout=240s + # Deployment Available != webhook serving: cert-manager injects the webhook + # cert after the pod is up, so early Domain/Gateway creates race it and time + # out ("context deadline exceeded"). Block until a server-side dry-run is + # actually admitted. + - echo "⏳ waiting for the NSO admission webhook to admit" + - | + DOMAIN_JSON='{"apiVersion":"networking.datumapis.com/v1alpha","kind":"Domain","metadata":{"name":"warmup-probe"},"spec":{"domainName":"warmup.e2e.env.datum.net"}}' + for i in $(seq 1 60); do + if echo "$DOMAIN_JSON" | kubectl --context {{.UPSTREAM_CTX}} -n default create --dry-run=server -f - >/dev/null 2>&1; then + echo "✅ webhook admitting"; break + fi + if [ "$i" -eq 60 ]; then echo "❌ webhook never became ready"; exit 1; fi + sleep 2 + done + # Ensure the downstream GatewayClass is Accepted so the first test gateway + # programs promptly instead of racing a 30s assert on a cold edge. + - echo "⏳ waiting for the downstream GatewayClass to be Accepted" + - kubectl --context {{.DOWNSTREAM_CTX}} wait gatewayclass datum-downstream-gateway-e2e --for=condition=Accepted --timeout=120s + - echo "✅ core components ready." + + # ---- Karmada ------------------------------------------------------------ + + karmada-up: + desc: "Stand up a real Karmada host on the upstream cluster (apiserver v1.33.2), join the downstream member, and apply the prod federation artifacts." + vars: + KARMADA_KUBECONFIG: '{{.TMP_DIR}}/karmada-{{.UPSTREAM_CLUSTER}}/karmada-apiserver.config' + # The upstream node's docker-network IP, reachable from the downstream node + # on the same network, used to sign the Karmada apiserver cert. + HOST_IP: + sh: docker inspect {{.UPSTREAM_CLUSTER}}-control-plane -f '{{"{{"}}.NetworkSettings.Networks.kind.IPAddress{{"}}"}}' + # The downstream member's docker-network IP — reachable from the upstream + # control-plane pods on the same network, but not from the macOS host. + MEMBER_IP: + sh: docker inspect {{.DOWNSTREAM_CLUSTER}}-control-plane -f '{{"{{"}}.NetworkSettings.Networks.kind.IPAddress{{"}}"}}' + cmds: + # Advertise 127.0.0.1: karmadactl runs on the macOS host, and init's own + # post-deploy steps dial the advertise address. The docker-network IP is not + # routable from the macOS host, so advertising it would make init time out; + # the upstream cluster maps a host port through to the apiserver, so + # 127.0.0.1 reaches it. We still sign the cert for the docker IP for any + # in-cluster path. The member is joined in push mode, so it never needs to + # reach the apiserver address — advertising 127.0.0.1 is safe for the join. + - echo "🌐 karmada init (apiserver {{.KARMADA_APISERVER_VERSION}}, advertise 127.0.0.1:32443, cert-ip incl {{.HOST_IP}}) on {{.UPSTREAM_CLUSTER}}" + - mkdir -p {{.TMP_DIR}}/karmada-{{.UPSTREAM_CLUSTER}} + - | + {{.KARMADACTL}} init --context {{.UPSTREAM_CTX}} \ + --kube-image-tag {{.KARMADA_APISERVER_VERSION}} \ + --karmada-apiserver-advertise-address 127.0.0.1 \ + --cert-external-ip "127.0.0.1,{{.HOST_IP}}" \ + --etcd-storage-mode hostPath \ + --karmada-data {{.TMP_DIR}}/karmada-{{.UPSTREAM_CLUSTER}} \ + --karmada-pki {{.TMP_DIR}}/karmada-{{.UPSTREAM_CLUSTER}}/pki + - echo "🔗 joining {{.DOWNSTREAM_CLUSTER}} as a Karmada member" + # karmadactl join runs on the host and connects to the member to install its + # agent, so it needs a host-reachable member kubeconfig, not the internal + # docker-hostname form the host cannot resolve. + - '{{.KIND}} get kubeconfig --name {{.DOWNSTREAM_CLUSTER}} > {{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}-host.yaml' + - | + {{.KARMADACTL}} join {{.DOWNSTREAM_CLUSTER}} \ + --karmada-context karmada-apiserver \ + --kubeconfig {{.KARMADA_KUBECONFIG}} \ + --cluster-kubeconfig {{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}-host.yaml \ + --cluster-context {{.DOWNSTREAM_CTX}} + # The join stores the host address as the member's endpoint, which the + # control-plane pods running inside the upstream cluster cannot reach. + # Repoint it to the member's docker IP, which they can reach on the shared + # network, and the member then goes Ready. + - | + kubectl --kubeconfig {{.KARMADA_KUBECONFIG}} --context karmada-apiserver \ + patch cluster {{.DOWNSTREAM_CLUSTER}} --type=merge \ + -p '{"spec":{"apiEndpoint":"https://{{.MEMBER_IP}}:6443"}}' + # Label the member so the production propagation policy places resources onto + # it. + - | + kubectl --kubeconfig {{.KARMADA_KUBECONFIG}} --context karmada-apiserver \ + label cluster {{.DOWNSTREAM_CLUSTER}} infra.datum.net/gateways=enabled --overwrite + - echo "⏳ waiting for the member cluster to become Ready" + - | + kubectl --kubeconfig {{.KARMADA_KUBECONFIG}} --context karmada-apiserver \ + wait cluster {{.DOWNSTREAM_CLUSTER}} --for=condition=Ready --timeout=120s + - echo "📜 applying federation artifacts (config/federation)" + - | + kubectl --kubeconfig {{.KARMADA_KUBECONFIG}} --context karmada-apiserver apply -f config/federation/ + - echo "✅ karmada-up complete. Karmada apiserver kubeconfig at {{.KARMADA_KUBECONFIG}}" + + smoke: + desc: "M2 functional confirmation: drive an upstream Gateway+HTTPRoute through the ext-server path and curl a real 200." + cmds: + - '{{.CHAINSAW}} test ./test/e2e-edge/extension-server-smoke --cluster {{.UPSTREAM_CLUSTER}}={{.TMP_DIR}}/.kind-{{.UPSTREAM_CLUSTER}}.yaml --cluster {{.DOWNSTREAM_CLUSTER}}={{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}.yaml' + + e2e: + desc: "Run chainsaw e2e scenarios against the live two-cluster env. Pass a scenario name or path after -- (e.g. `task test-infra:e2e -- waf-enforcement`); with no arg, runs every ext-server-path scenario that targets nso-upstream/nso-downstream. Use SCENARIOS=... to override the default set." + vars: + # Scenarios authored against this env's cluster names and downstream path. + # Older fixtures targeting the previous cluster names are excluded here. + DEFAULT_SCENARIOS: extension-server-smoke waf-enforcement branded-error-page connector-offline-503 atomic-reject-isolation + # CLI_ARGS (after --) wins; else SCENARIOS env; else the default set. + SELECTED: '{{.CLI_ARGS | default .SCENARIOS | default .DEFAULT_SCENARIOS}}' + deps: + - kubeconfigs + cmds: + - | + set -e + for s in {{.SELECTED}}; do + # Accept either a bare scenario name or a full/relative path. + case "$s" in + test/e2e-edge/*|./test/e2e-edge/*) dir="$s" ;; + */*) dir="$s" ;; + *) dir="./test/e2e-edge/$s" ;; + esac + echo "🧪 chainsaw: $dir" + {{.CHAINSAW}} test "$dir" \ + --cluster {{.UPSTREAM_CLUSTER}}={{.TMP_DIR}}/.kind-{{.UPSTREAM_CLUSTER}}.yaml \ + --cluster {{.DOWNSTREAM_CLUSTER}}={{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}.yaml + done + + test-e2e: + desc: "Run the full control-plane e2e suite (test/e2e) against the live two-cluster env. The suite's chainsaw aliases nso-standard/nso-infra are mapped to this env's upstream/downstream kubeconfigs at invocation. Pass a scenario path after -- to run a single test." + vars: + # CLI_ARGS (after --) wins so a single test can be selected; else the whole + # suite directory. + TARGET: '{{.CLI_ARGS | default "./test/e2e"}}' + # Persist a machine-readable run report so a hang or kill still leaves a + # forensic trail on disk (chainsaw writes nothing by default). + REPORT_PATH: '{{.REPORT_PATH | default .TMP_DIR}}' + REPORT_NAME: '{{.REPORT_NAME | default "chainsaw-report"}}' + deps: + - kubeconfigs + cmds: + - echo "🧪 chainsaw suite {{.TARGET}} (nso-standard={{.UPSTREAM_CLUSTER}}, nso-infra={{.DOWNSTREAM_CLUSTER}}); report {{.REPORT_PATH}}/{{.REPORT_NAME}}.json" + - | + {{.CHAINSAW}} test {{.TARGET}} \ + --parallel 1 \ + --cluster nso-standard={{.TMP_DIR}}/.kind-{{.UPSTREAM_CLUSTER}}.yaml \ + --cluster nso-infra={{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}.yaml \ + --report-format JSON \ + --report-path {{.REPORT_PATH}} \ + --report-name {{.REPORT_NAME}} + + kubeconfigs: + desc: "Export per-cluster kubeconfigs to TMPDIR for chainsaw." + cmds: + - '{{.KIND}} get kubeconfig --name {{.UPSTREAM_CLUSTER}} > {{.TMP_DIR}}/.kind-{{.UPSTREAM_CLUSTER}}.yaml' + - '{{.KIND}} get kubeconfig --name {{.DOWNSTREAM_CLUSTER}} > {{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}.yaml' + + parity:check: + desc: "Run the config-dump parity gate against the live downstream Envoy + ext-server. Pass CLI flags after -- (PARITY owns the CLI). Exit 0 PASS / 1 parity FAIL / 2 tool error." + cmds: + - go build -o bin/parity-check ./cmd/parity-check + - ./bin/parity-check {{.CLI_ARGS}} + + parity:check-live: + desc: "Convenience: resolve the live data-plane Envoy pod from this env's labels and run parity:check in kubectl-exec mode (ext-server via selector across replicas). Extra flags pass through after --." + vars: + # In this env the data-plane proxy and the extension server live in these + # namespaces. + DP_NS: datum-downstream-gateway + DP_SELECTOR: gateway.envoyproxy.io/owning-gatewayclass=datum-downstream-gateway-e2e + EXT_NS: network-services-operator-system + EXT_SELECTOR: app.kubernetes.io/component=envoy-gateway-extension-server + # The admin side wants a single exact proxy pod name, resolved here. The + # extension-server side takes a selector and picks the authoritative replica + # itself. + DP_POD: + sh: kubectl --context {{.DOWNSTREAM_CTX}} -n datum-downstream-gateway get pods -l gateway.envoyproxy.io/owning-gatewayclass=datum-downstream-gateway-e2e -o jsonpath='{.items[0].metadata.name}' + cmds: + - go build -o bin/parity-check ./cmd/parity-check + - | + if [ -z "{{.DP_POD}}" ]; then + echo "no data-plane Envoy pod found (label {{.DP_SELECTOR}} in {{.DP_NS}}); a Gateway must exist in a watched namespace for the merged data plane to be provisioned" >&2 + exit 2 + fi + - | + ./bin/parity-check \ + --coraza-filter=coraza-waf \ + --admin-exec-pod={{.DP_POD}} --admin-exec-namespace={{.DP_NS}} --admin-exec-container=envoy --admin-exec-context={{.DOWNSTREAM_CTX}} \ + --ext-exec-selector={{.EXT_SELECTOR}} --ext-exec-namespace={{.EXT_NS}} --ext-exec-context={{.DOWNSTREAM_CTX}} \ + {{.CLI_ARGS}} diff --git a/Taskfile.yaml b/Taskfile.yaml index a34a3994..ae4b2d6e 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -5,8 +5,8 @@ includes: docs: taskfile: ./docs/Taskfile.yaml dir: ./docs - dev: - taskfile: ./Taskfile.dev.yaml + test-infra: + taskfile: ./Taskfile.test-infra.yml tasks: validate-kustomizations: diff --git a/config/e2e-downstream/d1-cert-bypass/README.md b/config/e2e-downstream/d1-cert-bypass/README.md new file mode 100644 index 00000000..1923679c --- /dev/null +++ b/config/e2e-downstream/d1-cert-bypass/README.md @@ -0,0 +1,28 @@ +# Expired-certificate isolation fixture (test-env-only) + +In production, an expired or otherwise unusable TLS certificate is caught early: +the platform withholds that listener from the edge before it is ever delivered. +The extension server *also* removes unusable certificates at the edge, as a +second line of defense — but because the earlier check normally catches the +problem first, that edge-side removal rarely gets exercised on the normal path. + +This fixture lets a test exercise it directly, by handing the edge a genuinely +expired certificate and bypassing the earlier check. + +1. `mint-expired-secret.sh ` writes a + self-signed, already-expired certificate as a TLS Secret to stdout. Apply it + into the `e2e-direct` namespace on the edge cluster. +2. The test then applies a gateway directly to the edge whose HTTPS listener + uses that certificate. The extension server removes the bad listener while a + healthy sibling keeps serving — which is what the test asserts. + +> **Use the `e2e-direct` namespace.** The gateway controller only watches +> namespaces that already carry the `meta.datumapis.com/upstream-cluster-name` +> label when they are created; a label added afterward is not reliably picked +> up, and a gateway there can stay unprogrammed. The `e2e-direct` namespace is +> created with the label up front for exactly this reason. If you must create a +> namespace inline, set the label at creation time. + +`task -t Taskfile.test-infra.yml d1-mint-expired-secret` is a thin wrapper around +the script (defaults: `NAMESPACE=e2e-direct`, `SECRET=d1-expired-tls`, +`HOSTNAME=d1-bad.e2e.env.datum.net`). diff --git a/config/e2e-downstream/d1-cert-bypass/mint-expired-secret.sh b/config/e2e-downstream/d1-cert-bypass/mint-expired-secret.sh new file mode 100755 index 00000000..8ea5e705 --- /dev/null +++ b/config/e2e-downstream/d1-cert-bypass/mint-expired-secret.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Mint an already-expired self-signed TLS certificate and emit it as a +# kubernetes.io/tls Secret on stdout. Test-env-only: it hands the gateway an +# expired certificate directly, so the extension server's removal of unusable +# certificates can be exercised on its own, without the earlier check rejecting +# it first. +# +# Usage: mint-expired-secret.sh +set -euo pipefail + +NS="${1:?namespace required}" +SECRET="${2:?secret name required}" +HOST="${3:?hostname required}" + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +# Generate a key + a self-signed cert dated entirely in the past so it is expired +# the moment it is created. openssl's -not_before/-not_after (LibreSSL/OpenSSL 3) +# set an explicit validity window; fall back to a 1-second window via -days 0 if +# the flags are unavailable. +openssl req -x509 -newkey rsa:2048 -nodes \ + -keyout "$WORK/tls.key" -out "$WORK/tls.crt" \ + -subj "/CN=${HOST}" \ + -addext "subjectAltName=DNS:${HOST}" \ + -not_before 20200101000000Z -not_after 20200102000000Z 2>/dev/null \ + || openssl req -x509 -newkey rsa:2048 -nodes \ + -keyout "$WORK/tls.key" -out "$WORK/tls.crt" \ + -subj "/CN=${HOST}" -addext "subjectAltName=DNS:${HOST}" -days 1 2>/dev/null + +CRT_B64="$(base64 < "$WORK/tls.crt" | tr -d '\n')" +KEY_B64="$(base64 < "$WORK/tls.key" | tr -d '\n')" + +cat < + + + Service Unavailable +

This service is temporarily unavailable.

+ diff --git a/config/e2e-downstream/extension-server-config.yaml b/config/e2e-downstream/extension-server-config.yaml deleted file mode 100644 index f4c00b6d..00000000 --- a/config/e2e-downstream/extension-server-config.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: apiserver.config.datumapis.com/v1alpha1 -kind: NetworkServicesOperator -gateway: - coraza: - routeBaseDirectives: - - Include @crs-setup-conf - - Include @recommended-conf - - SecResponseBodyLimitAction ProcessPartial - - SecResponseBodyLimit 32000 - errorPage: - enabled: true - bodyPath: /etc/datum/error-pages/error-5xx.html diff --git a/config/e2e-downstream/extserver-base/kustomization.yaml b/config/e2e-downstream/extserver-base/kustomization.yaml new file mode 100644 index 00000000..14e55681 --- /dev/null +++ b/config/e2e-downstream/extserver-base/kustomization.yaml @@ -0,0 +1,32 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Ext-server, namespaced + prefixed so the Service FQDN matches the EG +# extensionManager fqdn and the CSI dns-names baked into the base deployment. +namespace: network-services-operator-system +namePrefix: network-services-operator- + +resources: + - ../../extension-server + +patches: + - path: patches/extserver-tls.yaml + target: + kind: Deployment + name: envoy-gateway-extension-server + - path: patches/extserver-serverconfig.yaml + target: + kind: Deployment + name: envoy-gateway-extension-server + - path: patches/extserver-clientcert-issuer.yaml + target: + kind: Certificate + name: envoy-gateway-extension-server-eg-client-tls + - path: patches/extserver-ca-bundle.yaml + target: + kind: Deployment + name: envoy-gateway-extension-server + - path: patches/extserver-programmed-set.yaml + target: + kind: Deployment + name: envoy-gateway-extension-server diff --git a/config/e2e-downstream/extserver-base/patches/extserver-ca-bundle.yaml b/config/e2e-downstream/extserver-base/patches/extserver-ca-bundle.yaml new file mode 100644 index 00000000..6e85fc0f --- /dev/null +++ b/config/e2e-downstream/extserver-base/patches/extserver-ca-bundle.yaml @@ -0,0 +1,18 @@ +# Point the ext-server CA bundle volume at the e2e CA ConfigMap (carrying the +# ca.crt that signed the EG client cert), replacing placeholder-ca-bundle. The +# ConfigMap is published by the bring-up (test-infra:extserver-ca-bundle) from +# the e2e-extension-server-ca cert-manager secret. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: envoy-gateway-extension-server +spec: + template: + spec: + volumes: + - name: tls-ca + configMap: + name: extension-server-ca-bundle + items: + - key: ca.crt + path: ca.crt diff --git a/config/e2e-downstream/extserver-base/patches/extserver-clientcert-issuer.yaml b/config/e2e-downstream/extserver-base/patches/extserver-clientcert-issuer.yaml new file mode 100644 index 00000000..9710265e --- /dev/null +++ b/config/e2e-downstream/extserver-base/patches/extserver-clientcert-issuer.yaml @@ -0,0 +1,11 @@ +# Point the EG client cert (CN=envoy-gateway) at the e2e CA ClusterIssuer, +# replacing the base certificate's placeholder-issuer. +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: envoy-gateway-extension-server-eg-client-tls +spec: + issuerRef: + name: e2e-extension-server-ca + kind: ClusterIssuer + group: cert-manager.io diff --git a/config/e2e-downstream/extserver-base/patches/extserver-programmed-set.yaml b/config/e2e-downstream/extserver-base/patches/extserver-programmed-set.yaml new file mode 100644 index 00000000..aa4ebfc4 --- /dev/null +++ b/config/e2e-downstream/extserver-base/patches/extserver-programmed-set.yaml @@ -0,0 +1,17 @@ +# Turn on the read-only /debug/programmed-set endpoint so the parity test can +# confirm the proxy is running exactly the set the build intended. The base +# deployment reads --enable-programmed-set from this env var, defaulting off in +# production; flip it to "true" here for the test environment. Strategic-merge +# on env (matched by name). +apiVersion: apps/v1 +kind: Deployment +metadata: + name: envoy-gateway-extension-server +spec: + template: + spec: + containers: + - name: envoy-gateway-extension-server + env: + - name: ENABLE_PROGRAMMED_SET + value: "true" diff --git a/config/e2e-downstream/extserver-base/patches/extserver-serverconfig.yaml b/config/e2e-downstream/extserver-base/patches/extserver-serverconfig.yaml new file mode 100644 index 00000000..2c9701d9 --- /dev/null +++ b/config/e2e-downstream/extserver-base/patches/extserver-serverconfig.yaml @@ -0,0 +1,26 @@ +# Mount the operator config ConfigMap and set SERVER_CONFIG to its path so the +# ext-server loads Coraza + branded-error-page settings. Strategic-merge on env +# (matched by name) and on volumes/volumeMounts. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: envoy-gateway-extension-server +spec: + template: + spec: + containers: + - name: envoy-gateway-extension-server + env: + - name: SERVER_CONFIG + value: /etc/datum/server-config/config.yaml + volumeMounts: + - name: server-config + mountPath: /etc/datum/server-config + readOnly: true + volumes: + - name: server-config + configMap: + name: extension-server-config + items: + - key: config.yaml + path: config.yaml diff --git a/config/e2e-downstream/extserver-base/patches/extserver-tls.yaml b/config/e2e-downstream/extserver-base/patches/extserver-tls.yaml new file mode 100644 index 00000000..c190ac81 --- /dev/null +++ b/config/e2e-downstream/extserver-base/patches/extserver-tls.yaml @@ -0,0 +1,21 @@ +# Point the ext-server server-cert CSI volume at the e2e CA-backed ClusterIssuer, +# replacing the base deployment's placeholder-issuer. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: envoy-gateway-extension-server +spec: + template: + spec: + volumes: + - name: tls + csi: + driver: csi.cert-manager.io + readOnly: true + volumeAttributes: + csi.cert-manager.io/issuer-kind: ClusterIssuer + csi.cert-manager.io/issuer-name: e2e-extension-server-ca + csi.cert-manager.io/common-name: envoy-gateway-extension-server + csi.cert-manager.io/dns-names: "network-services-operator-envoy-gateway-extension-server.network-services-operator-system.svc,network-services-operator-envoy-gateway-extension-server.network-services-operator-system.svc.cluster.local" + csi.cert-manager.io/key-usages: server auth + csi.cert-manager.io/fs-group: "65532" diff --git a/config/e2e-downstream/extserver-config.yaml b/config/e2e-downstream/extserver-config.yaml new file mode 100644 index 00000000..f5828a0c --- /dev/null +++ b/config/e2e-downstream/extserver-config.yaml @@ -0,0 +1,21 @@ +# Extension server config for the e2e edge. +# +# The WAF is enabled so its rules reach the proxy, and the branded 5xx page is +# pointed at the mounted error-pages volume so the suite can assert the branded +# body by content. The connector-tunnel listener name is left at its default so +# the connector fixtures find the listener by the same name production uses. +apiVersion: v1 +kind: ConfigMap +metadata: + name: extension-server-config + namespace: network-services-operator-system +data: + config.yaml: | + apiVersion: apiserver.config.datumapis.com/v1alpha1 + kind: NetworkServicesOperator + gateway: + coraza: + disabled: false + errorPage: + enabled: true + bodyPath: /etc/datum/error-pages/error-5xx.html diff --git a/config/e2e-downstream/issuer.yaml b/config/e2e-downstream/issuer.yaml new file mode 100644 index 00000000..35fc7a62 --- /dev/null +++ b/config/e2e-downstream/issuer.yaml @@ -0,0 +1,41 @@ +# Self-signed root + CA-backed ClusterIssuer for the e2e ext-server mTLS chain. +# +# This issues both sides of the EG <-> ext-server handshake: +# - the ext-server SERVER cert (via the CSI driver, see patches/extserver-tls.yaml) +# - the EG CLIENT cert (envoy-gateway-extension-server-eg-client-tls, CN=envoy-gateway) +# The CA's ca.crt is also published into the ext-server CA bundle ConfigMap +# (so the ext-server can verify the EG client) and into the EG certificateRef +# secret (so EG can verify the ext-server server cert). +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: e2e-extension-server-selfsigned +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: e2e-extension-server-ca + namespace: cert-manager +spec: + isCA: true + commonName: e2e-extension-server-ca + secretName: e2e-extension-server-ca + duration: 8760h + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: e2e-extension-server-selfsigned + kind: ClusterIssuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: e2e-extension-server-ca +spec: + ca: + secretName: e2e-extension-server-ca diff --git a/config/e2e-downstream/kustomization.yaml b/config/e2e-downstream/kustomization.yaml index b00e83e0..6ab183ae 100644 --- a/config/e2e-downstream/kustomization.yaml +++ b/config/e2e-downstream/kustomization.yaml @@ -1,76 +1,38 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -namespace: datum-downstream-gateway - +# The e2e edge overlay applied to nso-downstream. Composes the ext-server +# (prefixed/namespaced via extserver-base) with the e2e mTLS issuer chain, the +# Coraza/branded-page server-config, the branded error page, and the test +# EnvoyProxy (real Coraza WAF image + admin :19000) and its GatewayClass. +# +# The ConfigMaps / issuer / EnvoyProxy / GatewayClass are kept OUT of the +# name-prefix so the deployment's literal references (envoy-error-pages, +# extension-server-config) and the EG fqdn resolve unchanged. resources: - - namespace.yaml - - ../extension-server - - pki.yaml - - waf-gateway.yaml +- namespace.yaml +- extserver-base +- issuer.yaml +- extserver-config.yaml +- error-pages.yaml +- envoyproxy.yaml -configMapGenerator: - - name: nso-extension-server-config - files: - - config.yaml=extension-server-config.yaml - options: - disableNameSuffixHash: true +# The NSO image tag is set by the bring-up (test-infra:extension-server) to the +# git-SHA-built image. +images: +- name: ghcr.io/datum-cloud/network-services-operator + newName: ghcr.io/datum-cloud/network-services-operator + newTag: 63fa912 +# The default e2e path has no Prometheus operator; drop the ServiceMonitor so +# the apply doesn't fail on the missing monitoring.coreos.com CRD. (Re-add via +# OBSERVABILITY=1 when the Flux observability stack is enabled.) patches: - - target: - kind: ServiceMonitor +- patch: | + $patch: delete + apiVersion: monitoring.coreos.com/v1 + kind: ServiceMonitor + metadata: name: envoy-gateway-extension-server-metrics - patch: | - $patch: delete - apiVersion: monitoring.coreos.com/v1 - kind: ServiceMonitor - metadata: - name: envoy-gateway-extension-server-metrics - - target: - kind: Certificate - name: envoy-gateway-extension-server-eg-client-tls - patch: | - - op: replace - path: /spec/issuerRef/kind - value: Issuer - - op: replace - path: /spec/issuerRef/name - value: envoy-gateway-extension-server - - target: - kind: Deployment - name: envoy-gateway-extension-server - patch: | - apiVersion: apps/v1 - kind: Deployment - metadata: - name: envoy-gateway-extension-server - spec: - replicas: 1 - template: - spec: - containers: - - name: envoy-gateway-extension-server - env: - - name: SERVER_CONFIG - value: /config/config.yaml - volumeMounts: - - name: server-config - mountPath: /config - readOnly: true - volumes: - - name: tls - csi: - volumeAttributes: - csi.cert-manager.io/issuer-kind: Issuer - csi.cert-manager.io/issuer-name: envoy-gateway-extension-server - csi.cert-manager.io/dns-names: "envoy-gateway-extension-server.datum-downstream-gateway.svc,envoy-gateway-extension-server.datum-downstream-gateway.svc.cluster.local" - - name: tls-ca - configMap: null - secret: - secretName: envoy-gateway-extension-server-ca - items: - - key: ca.crt - path: ca.crt - - name: server-config - configMap: - name: nso-extension-server-config + target: + kind: ServiceMonitor diff --git a/config/e2e-downstream/namespace.yaml b/config/e2e-downstream/namespace.yaml index 89c7d150..b27c33c1 100644 --- a/config/e2e-downstream/namespace.yaml +++ b/config/e2e-downstream/namespace.yaml @@ -1,6 +1,6 @@ +# The ext-server namespace on the edge cluster. The base ext-server resources +# target this namespace but do not create it. apiVersion: v1 kind: Namespace metadata: - name: datum-downstream-gateway - labels: - meta.datumapis.com/upstream-cluster-name: 0-datum-internal-anchor + name: network-services-operator-system diff --git a/config/e2e-downstream/pki.yaml b/config/e2e-downstream/pki.yaml deleted file mode 100644 index 666bb0b3..00000000 --- a/config/e2e-downstream/pki.yaml +++ /dev/null @@ -1,31 +0,0 @@ ---- -apiVersion: cert-manager.io/v1 -kind: ClusterIssuer -metadata: - name: envoy-gateway-extension-server-selfsigned -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: envoy-gateway-extension-server-ca -spec: - isCA: true - commonName: envoy-gateway-extension-server-ca - secretName: envoy-gateway-extension-server-ca - privateKey: - algorithm: ECDSA - size: 256 - issuerRef: - name: envoy-gateway-extension-server-selfsigned - kind: ClusterIssuer - group: cert-manager.io ---- -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: envoy-gateway-extension-server -spec: - ca: - secretName: envoy-gateway-extension-server-ca diff --git a/config/e2e-downstream/waf-gateway.yaml b/config/e2e-downstream/waf-gateway.yaml deleted file mode 100644 index ef6a6265..00000000 --- a/config/e2e-downstream/waf-gateway.yaml +++ /dev/null @@ -1,59 +0,0 @@ ---- -apiVersion: gateway.envoyproxy.io/v1alpha1 -kind: EnvoyProxy -metadata: - name: datum-downstream-gateway -spec: - mergeGateways: true - ipFamily: DualStack - provider: - type: Kubernetes - kubernetes: - envoyDeployment: - pod: - volumes: - - name: coraza-waf - emptyDir: {} - container: - image: envoyproxy/envoy:contrib-v1.35.0 - volumeMounts: - - name: coraza-waf - mountPath: /opt/coraza-waf - initContainers: - - name: coraza-waf - image: ghcr.io/datum-labs/coraza-envoy-go-filter/coraza-waf:v1.3.0-alpha10 - imagePullPolicy: IfNotPresent - command: - - cp - - /coraza-waf.so - - /opt/coraza-waf/ - volumeMounts: - - name: coraza-waf - mountPath: /opt/coraza-waf - envoyService: - externalTrafficPolicy: Cluster - type: NodePort - patch: - type: StrategicMerge - value: - spec: - ipFamilyPolicy: RequireDualStack - ports: - - name: http-80 - nodePort: 30080 - port: 80 - - name: https-443 - nodePort: 30443 - port: 443 ---- -apiVersion: gateway.networking.k8s.io/v1 -kind: GatewayClass -metadata: - name: datum-downstream-gateway -spec: - controllerName: gateway.envoyproxy.io/datum-downstream-gateway - parametersRef: - group: gateway.envoyproxy.io - kind: EnvoyProxy - name: datum-downstream-gateway - namespace: datum-downstream-gateway diff --git a/config/tools/billing-usage-collector/kustomization.yaml b/config/tools/billing-usage-collector/kustomization.yaml index 42913678..7a3e7272 100644 --- a/config/tools/billing-usage-collector/kustomization.yaml +++ b/config/tools/billing-usage-collector/kustomization.yaml @@ -50,7 +50,7 @@ helmCharts: # Only tail the Envoy proxy pods; their access logs live on the # `envoy` container's stdout. Other namespaces/containers are # ignored so the agent does not parse unrelated cluster logs. - extra_field_selector: "metadata.namespace=envoy-gateway-system" + extra_field_selector: "metadata.namespace=datum-downstream-gateway" # Re-scan for new container log files every 2s instead of the 60s # default so the agent picks up freshly-created Envoy proxy pods # quickly -- the e2e test provisions a gateway and verifies within a diff --git a/config/tools/envoy-gateway-downstream/kustomization.yaml b/config/tools/envoy-gateway-downstream/kustomization.yaml index f95b40f2..9f5a2a92 100644 --- a/config/tools/envoy-gateway-downstream/kustomization.yaml +++ b/config/tools/envoy-gateway-downstream/kustomization.yaml @@ -6,10 +6,6 @@ helmCharts: includeCRDs: false namespace: datum-downstream-gateway releaseName: envoy-datum-downstream-gateway - # Pinned to match production: v1.8.x does not program merged gateways served - # through the extension manager (waf-gw stays with an empty status), so the - # downstream WAF control plane runs v1.7.4. CRDs come from the base - # envoy-gateway install (v1.8.1), which is forward-compatible. version: v1.7.4 repo: oci://docker.io/envoyproxy valuesInline: @@ -33,9 +29,9 @@ helmCharts: - key: meta.datumapis.com/upstream-cluster-name operator: Exists extensionManager: - # The WAF-augmented merged xDS response exceeds gRPC's 4 MiB default, - # so EG rejects it and stops publishing config with failOpen:false. - # Match the ext-server's 256 MiB limit; both sides must agree. + # Match the ext-server's compiled-in 256 MiB gRPC ceiling. Without + # this the EG side defaults to ~4 MiB and silently freezes xDS once a + # translated snapshot exceeds it (~540 gateways in prod). maxMessageSize: 256Mi policyResources: - group: networking.datumapis.com @@ -47,17 +43,18 @@ helmCharts: kind: Connector service: fqdn: - hostname: envoy-gateway-extension-server.datum-downstream-gateway.svc.cluster.local + hostname: network-services-operator-envoy-gateway-extension-server.network-services-operator-system.svc.cluster.local port: 5005 tls: - # EG reads the CA from tls.crt of this Secret; the self-signed CA - # Certificate's Secret carries the CA cert there. certificateRef: - name: envoy-gateway-extension-server-ca - namespace: datum-downstream-gateway + # e2e: the Secret holding the CA that issued the ext-server + # server cert. Published by the bring-up (test-infra:extserver-ca-bundle) + # from the e2e-extension-server-ca cert-manager secret. + name: e2e-extension-server-ca + namespace: network-services-operator-system clientCertificateRef: name: envoy-gateway-extension-server-eg-client-tls - namespace: datum-downstream-gateway + namespace: network-services-operator-system retry: maxAttempts: 4 initialBackoff: 100ms diff --git a/config/tools/kind/infra-cluster.yaml b/config/tools/kind/downstream-cluster.yaml similarity index 92% rename from config/tools/kind/infra-cluster.yaml rename to config/tools/kind/downstream-cluster.yaml index 04531eb9..6339242a 100644 --- a/config/tools/kind/infra-cluster.yaml +++ b/config/tools/kind/downstream-cluster.yaml @@ -1,6 +1,6 @@ kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 -name: nso-infra +name: nso-downstream networking: ipFamily: dual nodes: diff --git a/config/tools/kind/standard-cluster.yaml b/config/tools/kind/standard-cluster.yaml deleted file mode 100644 index 47bf2f3e..00000000 --- a/config/tools/kind/standard-cluster.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -name: nso-standard -networking: - ipFamily: dual -nodes: -- role: control-plane diff --git a/config/tools/kind/upstream-cluster.yaml b/config/tools/kind/upstream-cluster.yaml new file mode 100644 index 00000000..be723669 --- /dev/null +++ b/config/tools/kind/upstream-cluster.yaml @@ -0,0 +1,15 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: nso-upstream +networking: + ipFamily: dual +nodes: +- role: control-plane + # Expose the Karmada apiserver NodePort (32443) to the host so karmadactl — + # which runs on the host — can reach the Karmada apiserver during init/join. + # On macOS the kind docker-network IP is not host-routable, so without this + # mapping karmadactl times out dialing the advertised node IP:32443. + extraPortMappings: + - containerPort: 32443 + hostPort: 32443 + protocol: TCP diff --git a/docs/enhancements/envoy-gateway-extension-server/README.md b/docs/enhancements/envoy-gateway-extension-server/README.md index 3957e68e..9f900035 100644 --- a/docs/enhancements/envoy-gateway-extension-server/README.md +++ b/docs/enhancements/envoy-gateway-extension-server/README.md @@ -358,9 +358,10 @@ The dedicated EG is configured with: - `extensionManager` baked into the helm values — no runtime ConfigMap patch or EG rollout restart is required. -The local/CI e2e installs the dedicated EG via the second `helmCharts:` entry in -`config/tools/envoy-gateway/kustomization.yaml` (applied by `make prepare-infra-cluster`). -In production (`datum-cloud/infra`) it is a FluxCD `HelmRelease` with the same values. +The local/CI e2e installs the dedicated downstream EG via +`config/tools/envoy-gateway-downstream` (applied by the `test-infra:eg-downstream` +task during `task test-infra:up`). In production (`datum-cloud/infra`) it is a +FluxCD `HelmRelease` with the same values. #### extensionManager registration @@ -819,7 +820,7 @@ meaning of "parity" in the Transition Plan below. **3. Real-traffic end-to-end (two kind clusters, live requests).** The [Chainsaw e2e suite](https://github.com/datum-cloud/network-services-operator/blob/main/test/e2e/gateway/chainsaw-test.yaml) (run by -`make test-e2e`, gated in CI) stands up the full upstream/downstream split, +`task test-infra:test-e2e`, gated in CI) stands up the full upstream/downstream split, provisions Gateways and HTTPRoutes, and sends real HTTP through Envoy with `curl` — confirming plaintext routing, TLS termination, and certificate issuance actually work, not just that the config looks right. The diff --git a/test/e2e/billing/chainsaw-test.yaml b/test/e2e/billing/chainsaw-test.yaml index b0c9918a..c03d9f13 100644 --- a/test/e2e/billing/chainsaw-test.yaml +++ b/test/e2e/billing/chainsaw-test.yaml @@ -54,16 +54,6 @@ spec: ca: secretName: ($clusterIssuerName) - - script: - cluster: nso-infra - env: - - name: CLUSTER_ISSUER_NAME - value: ($clusterIssuerName) - content: | - kubectl get secret -n cert-manager $CLUSTER_ISSUER_NAME -o yaml | \ - sed 's/namespace: .*/namespace: envoy-gateway-system/' | \ - kubectl apply -n envoy-gateway-system -f - - - name: Deploy Mock Billing Gateway try: - create: @@ -142,100 +132,6 @@ spec: spec: controllerName: gateway.networking.datumapis.com/external-global-proxy-controller - # Ensure the downstream environment's standard gateway class and config exists - - apply: - cluster: nso-infra - resource: - apiVersion: gateway.envoyproxy.io/v1alpha1 - kind: EnvoyProxy - metadata: - name: custom-proxy-config - namespace: envoy-gateway-system - spec: - telemetry: - accessLog: - settings: - - sinks: - # File sink only -- identical to the production - # downstream gateway EnvoyProxy. Envoy writes the JSON - # access log to stdout; the node-local Vector agent - # tails the container log file (kubernetes_logs source) - # and emits the CloudEvents. This exercises the real, - # file-based production ingestion path -- no OTLP push. - # The test also scrapes the same stdout to derive the - # expected metric values (bytes, duration, project_name). - - type: File - file: - path: /dev/stdout - format: - type: JSON - json: - start_time: "%START_TIME%" - method: "%REQ(:METHOD)%" - path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" - protocol: "%PROTOCOL%" - response_code: "%RESPONSE_CODE%" - bytes_received: "%BYTES_RECEIVED%" - bytes_sent: "%BYTES_SENT%" - duration: "%DURATION%" - route_name: "%ROUTE_NAME%" - project_name: "%METADATA(ROUTE:datum-gateway:project_name)%" - request_id: "%REQ(X-REQUEST-ID)%" - provider: - type: Kubernetes - kubernetes: - envoyService: - type: ClusterIP - patch: - type: StrategicMerge - value: - spec: - ipFamilyPolicy: RequireDualStack - envoyDeployment: - patch: - type: StrategicMerge - value: - spec: - template: - spec: - containers: - - name: envoy - volumeMounts: - - mountPath: /etc/ssl/certs/ - name: selfsigned-ca - readOnly: true - volumes: - - name: selfsigned-ca - secret: - secretName: ($clusterIssuerName) - items: - - key: ca.crt - path: ca-certificates.crt - mergeGateways: true - - - apply: - cluster: nso-infra - resource: - apiVersion: gateway.networking.k8s.io/v1 - kind: GatewayClass - metadata: - name: datum-downstream-gateway-e2e - spec: - controllerName: gateway.envoyproxy.io/gatewayclass-controller - parametersRef: - group: gateway.envoyproxy.io - kind: EnvoyProxy - name: custom-proxy-config - namespace: envoy-gateway-system - - - apply: - cluster: nso-infra - resource: - apiVersion: v1 - kind: Namespace - metadata: - name: datum-downstream-gateway-hostnames - # Get downstream namespace name by retrieving standard namespace UID - script: cluster: nso-standard @@ -359,6 +255,9 @@ spec: namespaces: from: Same - assert: + # Cold edge: the first gateway waits on the merged Envoy proxy to + # spin up, which overruns the 30s default. + timeout: 3m resource: apiVersion: gateway.networking.k8s.io/v1 kind: Gateway @@ -456,7 +355,7 @@ spec: content: | # Wait up to 10s for the envoy service to be created for i in $(seq 1 10); do - SVC=$(kubectl get svc -n envoy-gateway-system -l gateway.envoyproxy.io/owning-gatewayclass=datum-downstream-gateway-e2e -o json || true) + SVC=$(kubectl get svc -n datum-downstream-gateway -l gateway.envoyproxy.io/owning-gatewayclass=datum-downstream-gateway-e2e -o json || true) if [ ! -z "$SVC" ] && echo "$SVC" | grep -q "metadata"; then echo "$SVC" exit 0 @@ -485,7 +384,7 @@ spec: - name: GATEWAY_SERVICE_NAME value: ($gatewayService.metadata.name) - name: GATEWAY_SERVICE_NAMESPACE - value: envoy-gateway-system + value: datum-downstream-gateway lifecycle: postStart: exec: @@ -568,7 +467,7 @@ spec: # captured here (STEP 1-2) mock-billing-gateway (STEP 4) # # Envoy writes the access log to stdout (File sink on - # custom-proxy-config, path: /dev/stdout) when the request is + # the env's shared EnvoyProxy, path: /dev/stdout) when the request is # served; the node-local Vector agent tails the container log file # automatically -- the test does NOT forward it manually. This is # the same file-based ingestion path as the production gateway. @@ -596,7 +495,7 @@ spec: # --- STEP 1: capture the Envoy access log line for our HTTPRoute ----- # The route_name embeds the downstream namespace, so grepping for # 'httproute//' isolates this test's request. - ENVOY_LOG=$(kubectl -n envoy-gateway-system logs -l gateway.envoyproxy.io/owning-gatewayclass=datum-downstream-gateway-e2e -c envoy --tail=500 | grep "httproute/$DOWNSTREAM_NAMESPACE/" | tail -n 1 || true) + ENVOY_LOG=$(kubectl -n datum-downstream-gateway logs -l gateway.envoyproxy.io/owning-gatewayclass=datum-downstream-gateway-e2e -c envoy --tail=500 | grep "httproute/$DOWNSTREAM_NAMESPACE/" | tail -n 1 || true) if [ -z "$ENVOY_LOG" ]; then echo "ERROR: No Envoy access log found containing 'httproute/'" >&2 exit 1 @@ -631,7 +530,7 @@ spec: # --- STEP 3: (no manual forwarding) -------------------------------- # The request above already caused Envoy to write this access log to - # stdout via its File sink (custom-proxy-config, path: /dev/stdout). + # stdout via its File sink (env shared EnvoyProxy, path: /dev/stdout). # The node-local Vector agent tails the Envoy container log file via # its kubernetes_logs source, and Vector's VRL transform parses it # and emits the four CloudEvents to the mock-billing-gateway sink, so diff --git a/test/e2e/gateway-invalid-certificate/chainsaw-test.yaml b/test/e2e/gateway-invalid-certificate/chainsaw-test.yaml index 5ab2e026..4ac1016e 100644 --- a/test/e2e/gateway-invalid-certificate/chainsaw-test.yaml +++ b/test/e2e/gateway-invalid-certificate/chainsaw-test.yaml @@ -4,12 +4,10 @@ # the edge, and the customer must be told why. One bad certificate must never # take HTTPS down for the other hostnames that share the gateway. # -# The test follows the whole story: -# - the bad hostname is held back while the good one keeps working; -# - the customer sees a plain-language reason on the bad hostname; -# - a real HTTPS request to the good hostname returns 200, and the bad one is -# not served; -# - once the certificate can be issued, the bad hostname recovers and serves. +# The test follows the whole story at the gateway-status level: +# - the bad listener is held back from the edge while the good one is programmed; +# - the customer sees a plain-language reason on the bad listener; +# - once the certificate can be issued, the bad listener is programmed at the edge. # # To make the failure happen reliably, the bad hostname points at a certificate # issuer that doesn't exist yet, so its certificate is never issued. Expired and @@ -103,52 +101,6 @@ spec: spec: controllerName: gateway.networking.datumapis.com/external-global-proxy-controller - # Shared cluster-scoped downstream config — applied (not created) so this - # test coexists with gateway-accepted regardless of run order. - - apply: - cluster: nso-infra - resource: - apiVersion: gateway.envoyproxy.io/v1alpha1 - kind: EnvoyProxy - metadata: - name: custom-proxy-config - namespace: envoy-gateway-system - spec: - provider: - type: Kubernetes - kubernetes: - envoyService: - type: ClusterIP - patch: - type: StrategicMerge - value: - spec: - ipFamilyPolicy: RequireDualStack - mergeGateways: true - - - apply: - cluster: nso-infra - resource: - apiVersion: gateway.networking.k8s.io/v1 - kind: GatewayClass - metadata: - name: datum-downstream-gateway-e2e - spec: - controllerName: gateway.envoyproxy.io/gatewayclass-controller - parametersRef: - group: gateway.envoyproxy.io - kind: EnvoyProxy - name: custom-proxy-config - namespace: envoy-gateway-system - - - apply: - cluster: nso-infra - resource: - apiVersion: v1 - kind: Namespace - metadata: - name: datum-downstream-gateway-hostnames - - name: Create a backend to route to try: # Plain HTTP backend on the downstream cluster. The gateway terminates @@ -410,7 +362,7 @@ spec: kubectl -n $DOWNSTREAM_NAMESPACE get httproute -o yaml kubectl -n $DOWNSTREAM_NAMESPACE get certificates -o yaml - - name: Confirm Envoy programmed and serves the good listener + - name: Confirm the edge programmed the good listener try: - script: cluster: nso-standard @@ -455,92 +407,6 @@ spec: - status: "True" (conditions[?type == 'ResolvedRefs']): - status: "True" - - - assert: - cluster: nso-infra - resource: - apiVersion: v1 - kind: Pod - metadata: - name: backend-pod - namespace: default - status: - phase: Running - - # A long-lived client pod with curl preinstalled. - - create: - cluster: nso-infra - resource: - apiVersion: v1 - kind: Pod - metadata: - name: conntest - namespace: default - spec: - containers: - - name: curl - image: curlimages/curl:8.11.1 - command: ["sleep", "infinity"] - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 100m - memory: 128Mi - terminationGracePeriodSeconds: 0 - - - assert: - cluster: nso-infra - resource: - apiVersion: v1 - kind: Pod - metadata: - name: conntest - namespace: default - status: - phase: Running - - # A real HTTPS request to the good hostname must return 200, and the bad - # hostname must not be served. Retries give the edge a moment to pick up - # the configuration. - # Explicit timeout: the chainsaw default for script steps is 5s, too short - # for the retry loop below. - - script: - timeout: 240s - cluster: nso-infra - content: | - set -eu - IP=$(kubectl get svc -n envoy-gateway-system \ - -l gateway.envoyproxy.io/owning-gatewayclass=datum-downstream-gateway-e2e \ - -o jsonpath='{.items[0].spec.clusterIP}') - echo "gateway service clusterIP: ${IP}" - - echo "--- good hostname must serve 200 ---" - ok="" - for i in $(seq 1 24); do - code=$(kubectl -n default exec conntest -- \ - curl -ksS -o /dev/null -w '%{http_code}' --max-time 10 \ - --resolve good.e2e.env.datum.net:443:"${IP}" \ - https://good.e2e.env.datum.net/status/200 || true) - echo "attempt ${i}: good -> ${code}" - if [ "${code}" = "200" ]; then ok="yes"; break; fi - sleep 5 - done - if [ -z "${ok}" ]; then - echo "ERROR: good hostname never returned 200"; exit 1 - fi - - echo "--- bad hostname must not serve (no matching listener) ---" - code=$(kubectl -n default exec conntest -- \ - curl -ksS -o /dev/null -w '%{http_code}' --max-time 10 \ - --resolve bad.e2e.env.datum.net:443:"${IP}" \ - https://bad.e2e.env.datum.net/status/200 || true) - echo "bad -> ${code}" - if [ "${code}" = "200" ]; then - echo "ERROR: bad hostname was served despite an invalid certificate"; exit 1 - fi - echo "OK: good served, bad not served" catch: - script: cluster: nso-infra @@ -550,8 +416,6 @@ spec: content: | kubectl -n $DOWNSTREAM_NAMESPACE get gateway test-gateway -o yaml kubectl -n $DOWNSTREAM_NAMESPACE get httproute -o yaml - kubectl get svc -n envoy-gateway-system -o wide - kubectl -n envoy-gateway-system logs -l gateway.envoyproxy.io/owning-gatewayclass=datum-downstream-gateway-e2e -c envoy --tail=100 - name: Recover the bad listener by creating its issuer try: @@ -617,31 +481,6 @@ spec: - status: "True" (conditions[?type == 'ResolvedRefs']): - status: "True" - - # The recovered hostname now actually serves 200. - - script: - timeout: 240s - cluster: nso-infra - content: | - set -eu - IP=$(kubectl get svc -n envoy-gateway-system \ - -l gateway.envoyproxy.io/owning-gatewayclass=datum-downstream-gateway-e2e \ - -o jsonpath='{.items[0].spec.clusterIP}') - echo "--- recovered hostname must now serve 200 ---" - ok="" - for i in $(seq 1 24); do - code=$(kubectl -n default exec conntest -- \ - curl -ksS -o /dev/null -w '%{http_code}' --max-time 10 \ - --resolve bad.e2e.env.datum.net:443:"${IP}" \ - https://bad.e2e.env.datum.net/status/200 || true) - echo "attempt ${i}: recovered bad -> ${code}" - if [ "${code}" = "200" ]; then ok="yes"; break; fi - sleep 5 - done - if [ -z "${ok}" ]; then - echo "ERROR: recovered hostname never returned 200"; exit 1 - fi - echo "OK: recovered hostname serves 200" catch: - script: cluster: nso-infra diff --git a/test/e2e/gateway/chainsaw-test.yaml b/test/e2e/gateway/chainsaw-test.yaml index 184fff9c..1253d66d 100644 --- a/test/e2e/gateway/chainsaw-test.yaml +++ b/test/e2e/gateway/chainsaw-test.yaml @@ -85,18 +85,6 @@ spec: - type: Ready status: "True" - - script: - # skipCommandOutput: true - # skipLogOutput: true - cluster: nso-infra - env: - - name: CLUSTER_ISSUER_NAME - value: ($clusterIssuerName) - content: | - kubectl get secret -n cert-manager $CLUSTER_ISSUER_NAME -o yaml | \ - sed 's/namespace: .*/namespace: envoy-gateway-system/' | \ - kubectl apply -n envoy-gateway-system -f - - - name: Create GatewayClass for the upstream gateways try: - create: @@ -109,73 +97,6 @@ spec: spec: controllerName: gateway.networking.datumapis.com/external-global-proxy-controller - - name: Create GatewayClass for the downstream gateways - try: - - create: - cluster: nso-infra - resource: - apiVersion: gateway.envoyproxy.io/v1alpha1 - kind: EnvoyProxy - metadata: - name: custom-proxy-config - namespace: envoy-gateway-system - spec: - provider: - type: Kubernetes - kubernetes: - envoyService: - type: ClusterIP - patch: - type: StrategicMerge - value: - spec: - ipFamilyPolicy: RequireDualStack - envoyDeployment: - patch: - type: StrategicMerge - value: - spec: - template: - spec: - containers: - - name: envoy - volumeMounts: - - mountPath: /etc/ssl/certs/ - name: selfsigned-ca - readOnly: true - volumes: - - name: selfsigned-ca - secret: - secretName: ($clusterIssuerName) - items: - - key: ca.crt - path: ca-certificates.crt - mergeGateways: true - - - create: - cluster: nso-infra - resource: - apiVersion: gateway.networking.k8s.io/v1 - kind: GatewayClass - metadata: - name: datum-downstream-gateway-e2e - spec: - controllerName: gateway.envoyproxy.io/gatewayclass-controller - parametersRef: - group: gateway.envoyproxy.io - kind: EnvoyProxy - name: custom-proxy-config - namespace: envoy-gateway-system - - - description: Create namespace for hostname tracking - apply: - cluster: nso-infra - resource: - apiVersion: v1 - kind: Namespace - metadata: - name: datum-downstream-gateway-hostnames - - name: Create pod and service to serve as a backend for the HTTPRoute try: - create: @@ -792,107 +713,3 @@ spec: kubectl get endpointslices -n $NAMESPACE -o yaml kubectl get dnsendpoints -n $NAMESPACE -o yaml kubectl get svc -n $NAMESPACE -o yaml - - - name: Provision Pod to test connectivity - try: - - script: - skipCommandOutput: true - skipLogOutput: true - cluster: nso-standard - content: | - kubectl -n $NAMESPACE get gateway test-gateway -o json - outputs: - - name: upstreamGateway - value: (json_parse($stdout)) - # Re-derive in this step's scope; chainsaw bindings don't carry - # across steps, and the operator-published canonical hostname is - # word-derived from the gateway UID and can't be reconstructed - # inline. - - name: primaryHostname - value: ($upstreamGateway.status.addresses[?type == 'Hostname'].value | [0]) - - - script: - skipCommandOutput: true - skipLogOutput: true - cluster: nso-infra - content: | - kubectl get svc -n envoy-gateway-system -l gateway.envoyproxy.io/owning-gatewayclass=datum-downstream-gateway-e2e -o json - outputs: - - name: gatewayService - value: (json_parse($stdout).items | [0]) - - - create: - cluster: nso-infra - resource: - apiVersion: v1 - kind: Pod - metadata: - name: test-pod - namespace: default - spec: - containers: - - name: connectivity-test - image: alpine:3.21.3 - command: ["/bin/sh", "-c", "sleep infinity"] - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 100m - memory: 1Gi - env: - - name: GATEWAY_SERVICE_NAME - value: ($gatewayService.metadata.name) - - name: GATEWAY_SERVICE_NAMESPACE - value: ($gatewayService.metadata.namespace) - - name: PRIMARY_HOSTNAME - value: ($primaryHostname) - lifecycle: - postStart: - exec: - command: ["apk", "add", "curl"] - terminationGracePeriodSeconds: 0 - - - assert: - cluster: nso-infra - resource: - apiVersion: v1 - kind: Pod - metadata: - name: backend-pod - namespace: default - status: - phase: Running - - - assert: - cluster: nso-infra - resource: - apiVersion: v1 - kind: Pod - metadata: - name: test-pod - namespace: default - status: - phase: Running - - # Sleep for a while to ensure that the Envoy replicas have received the - # latest xDS config. We should remove this once we find a reliable method - # to determine if the Envoy replicas have the latest xDS config. - - sleep: - duration: 10s - - - script: - cluster: nso-infra - content: | - kubectl -n default exec -i test-pod -- sh -c " \ - set -x; \ - curl -kvf -H \"Host: \${PRIMARY_HOSTNAME}\" http://\${GATEWAY_SERVICE_NAME}.\${GATEWAY_SERVICE_NAMESPACE}.svc.cluster.local/plaintext/status/200; \ - curl -kvf -H \"Host: \${PRIMARY_HOSTNAME}\" http://\${GATEWAY_SERVICE_NAME}.\${GATEWAY_SERVICE_NAMESPACE}.svc.cluster.local/tls/status/200; \ - " - catch: - - script: - cluster: nso-infra - content: | - kubectl -n envoy-gateway-system logs -l gateway.envoyproxy.io/owning-gatewayclass=datum-downstream-gateway-e2e -c envoy - kubectl -n envoy-gateway-system logs -l app.kubernetes.io/instance=envoy-gateway diff --git a/test/e2e/trafficprotectionpolicy-enforce-attack/chainsaw-test.yaml b/test/e2e/trafficprotectionpolicy-enforce-attack/chainsaw-test.yaml index 9e6dca69..df627fa3 100644 --- a/test/e2e/trafficprotectionpolicy-enforce-attack/chainsaw-test.yaml +++ b/test/e2e/trafficprotectionpolicy-enforce-attack/chainsaw-test.yaml @@ -17,7 +17,7 @@ metadata: # # Precondition: the downstream (nso-infra) must have the WAF data plane wired up # (extension-server + the extensionManager Envoy Gateway registered on the -# `datum-downstream-gateway` GatewayClass, with the Coraza filter). +# `datum-downstream-gateway-e2e` GatewayClass, with the Coraza filter). # `make prepare-infra-cluster` installs this via `make downstream-waf-dataplane`. spec: cluster: nso-infra @@ -84,7 +84,7 @@ spec: metadata: name: waf-gw spec: - gatewayClassName: datum-downstream-gateway + gatewayClassName: datum-downstream-gateway-e2e listeners: - name: http protocol: HTTP @@ -144,7 +144,7 @@ spec: set -x kubectl get gateway -A -o yaml kubectl describe gateway -A - kubectl get gatewayclass datum-downstream-gateway -o yaml + kubectl get gatewayclass datum-downstream-gateway-e2e -o yaml kubectl -n datum-downstream-gateway get pods -o wide kubectl -n datum-downstream-gateway describe pods kubectl -n datum-downstream-gateway logs deploy/envoy-gateway --tail=-1 diff --git a/test/e2e/trafficprotectionpolicy-enforce/chainsaw-test.yaml b/test/e2e/trafficprotectionpolicy-enforce/chainsaw-test.yaml index 9e42f2e2..0619031b 100644 --- a/test/e2e/trafficprotectionpolicy-enforce/chainsaw-test.yaml +++ b/test/e2e/trafficprotectionpolicy-enforce/chainsaw-test.yaml @@ -12,7 +12,7 @@ metadata: # # Precondition: the downstream (nso-infra) must have the WAF data plane wired up # (extension-server + the extensionManager Envoy Gateway registered on the -# `datum-downstream-gateway` GatewayClass, with the Coraza filter). +# `datum-downstream-gateway-e2e` GatewayClass, with the Coraza filter). # `make prepare-infra-cluster` installs this via `make downstream-waf-dataplane`. spec: cluster: nso-infra @@ -79,7 +79,7 @@ spec: metadata: name: waf-gw spec: - gatewayClassName: datum-downstream-gateway + gatewayClassName: datum-downstream-gateway-e2e listeners: - name: http protocol: HTTP @@ -143,7 +143,7 @@ spec: set -x kubectl get gateway -A -o yaml kubectl describe gateway -A - kubectl get gatewayclass datum-downstream-gateway -o yaml + kubectl get gatewayclass datum-downstream-gateway-e2e -o yaml kubectl -n datum-downstream-gateway get pods -o wide kubectl -n datum-downstream-gateway describe pods kubectl -n datum-downstream-gateway logs deploy/envoy-gateway --tail=-1 diff --git a/test/e2e/trafficprotectionpolicy-neutralize-inverted/chainsaw-test.yaml b/test/e2e/trafficprotectionpolicy-neutralize-inverted/chainsaw-test.yaml index 5482d51f..930a1387 100644 --- a/test/e2e/trafficprotectionpolicy-neutralize-inverted/chainsaw-test.yaml +++ b/test/e2e/trafficprotectionpolicy-neutralize-inverted/chainsaw-test.yaml @@ -94,7 +94,7 @@ spec: metadata: name: waf-gw spec: - gatewayClassName: datum-downstream-gateway + gatewayClassName: datum-downstream-gateway-e2e listeners: - name: http protocol: HTTP @@ -147,13 +147,15 @@ spec: crd=trafficprotectionpolicies.networking.datumapis.com path=/spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/ruleSets/items/properties/owaspCoreRuleSet/properties/paranoiaLevels/x-kubernetes-validations - kubectl patch crd "${crd}" --type=json \ - -p "[{\"op\":\"remove\",\"path\":\"${path}\"}]" - - restore() { - kubectl patch crd "${crd}" --type=json -p "[{\"op\":\"add\",\"path\":\"${path}\",\"value\":[{\"message\":\"detection paranoia level must be greater than or equal to blocking paranoia level\",\"rule\":\"self.detection >= self.blocking\"}]}]" - } - trap restore EXIT + cel=".spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.ruleSets.items.properties.owaspCoreRuleSet.properties.paranoiaLevels.x-kubernetes-validations" + if kubectl get crd "${crd}" -o jsonpath="{${cel}}" | grep -q .; then + kubectl patch crd "${crd}" --type=json \ + -p "[{\"op\":\"remove\",\"path\":\"${path}\"}]" + restore() { + kubectl patch crd "${crd}" --type=json -p "[{\"op\":\"add\",\"path\":\"${path}\",\"value\":[{\"message\":\"detection paranoia level must be greater than or equal to blocking paranoia level\",\"rule\":\"self.detection >= self.blocking\"}]}]" + } + trap restore EXIT + fi created=false for i in $(seq 1 20); do