From d6b6cc4b34e868607648dca0ffa916e4c6ce1085 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 25 Jun 2026 13:29:36 -0500 Subject: [PATCH 01/20] test(infra): stand up a production-fidelity edge test environment Brings up a two-cluster environment that mirrors how the edge really runs: the production gateway version, the firewall data plane real traffic passes through, and the actual path configuration travels to the edge. Lets tests exercise the route customer traffic takes instead of a simplified stand-in. Test scaffolding only; no change to how the edge runs for customers. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01JbCy8vy66RdNYzGSgqH6P6 --- Taskfile.test-infra.yml | 419 ++++++++++++++++++ Taskfile.yaml | 2 + .../e2e-downstream/d1-cert-bypass/README.md | 28 ++ .../d1-cert-bypass/mint-expired-secret.sh | 44 ++ config/e2e-downstream/direct-namespace.yaml | 14 + .../eg-downstream/kustomization.yaml | 92 ++++ .../eg-downstream/namespace.yaml | 4 + .../eg-downstream/nso-crd-rbac.yaml | 37 ++ config/e2e-downstream/envoyproxy.yaml | 87 ++++ config/e2e-downstream/error-pages.yaml | 16 + .../extserver-base/kustomization.yaml | 28 ++ .../patches/extserver-ca-bundle.yaml | 18 + .../patches/extserver-clientcert-issuer.yaml | 11 + .../patches/extserver-serverconfig.yaml | 26 ++ .../extserver-base/patches/extserver-tls.yaml | 21 + config/e2e-downstream/extserver-config.yaml | 21 + config/e2e-downstream/issuer.yaml | 41 ++ config/e2e-downstream/kustomization.yaml | 98 ++-- config/e2e-downstream/namespace.yaml | 6 +- .../kustomization.yaml | 23 +- config/tools/kind/downstream-cluster.yaml | 14 + config/tools/kind/upstream-cluster.yaml | 15 + 22 files changed, 981 insertions(+), 84 deletions(-) create mode 100644 Taskfile.test-infra.yml create mode 100644 config/e2e-downstream/d1-cert-bypass/README.md create mode 100755 config/e2e-downstream/d1-cert-bypass/mint-expired-secret.sh create mode 100644 config/e2e-downstream/direct-namespace.yaml create mode 100644 config/e2e-downstream/eg-downstream/kustomization.yaml create mode 100644 config/e2e-downstream/eg-downstream/namespace.yaml create mode 100644 config/e2e-downstream/eg-downstream/nso-crd-rbac.yaml create mode 100644 config/e2e-downstream/envoyproxy.yaml create mode 100644 config/e2e-downstream/error-pages.yaml create mode 100644 config/e2e-downstream/extserver-base/kustomization.yaml create mode 100644 config/e2e-downstream/extserver-base/patches/extserver-ca-bundle.yaml create mode 100644 config/e2e-downstream/extserver-base/patches/extserver-clientcert-issuer.yaml create mode 100644 config/e2e-downstream/extserver-base/patches/extserver-serverconfig.yaml create mode 100644 config/e2e-downstream/extserver-base/patches/extserver-tls.yaml create mode 100644 config/e2e-downstream/extserver-config.yaml create mode 100644 config/e2e-downstream/issuer.yaml create mode 100644 config/tools/kind/downstream-cluster.yaml create mode 100644 config/tools/kind/upstream-cluster.yaml diff --git a/Taskfile.test-infra.yml b/Taskfile.test-infra.yml new file mode 100644 index 00000000..ae577cf1 --- /dev/null +++ b/Taskfile.test-infra.yml @@ -0,0 +1,419 @@ +version: '3' + +# Production-fidelity test environment. Two kind clusters: nso-upstream (the +# control plane) and nso-downstream (the edge). Consumed by both the e2e and perf +# suites. +# +# Bring up: task test-infra:up +# Tear down: task test-infra:down +# Smoke: task test-infra:smoke + +# kustomize fetches some CRDs over https. A developer's global git config can +# rewrite https to ssh, which fails in a non-interactive fetch. Neutralize the +# global git config for kustomize's child git so the https fetches succeed; the +# repo-local config is untouched. +env: + GIT_CONFIG_GLOBAL: /dev/null + +vars: + TMP_DIR: + sh: echo "${TMPDIR:-/tmp}" + REPO: + sh: pwd + # Pinned-to-prod tooling. We use an explicit kind v0.32.0 binary because it + # ships the node image production runs, which the checked-in bin/kind predates. + KIND: '{{.REPO}}/bin/kind-v0.32.0' + KUSTOMIZE: '{{.REPO}}/bin/kustomize' + CMCTL: '{{.REPO}}/bin/cmctl' + KARMADACTL: '{{.REPO}}/bin/karmadactl' + CHAINSAW: '{{.REPO}}/bin/chainsaw' + + UPSTREAM_CLUSTER: nso-upstream + DOWNSTREAM_CLUSTER: nso-downstream + UPSTREAM_CTX: kind-nso-upstream + DOWNSTREAM_CTX: kind-nso-downstream + + # The exact node image the production edge runs. + K8S_NODE_IMAGE: kindest/node:v1.35.5@sha256:ce977ae6d65918d0b58a5f8b5e940429c2ce42fa3a5619ec2bbc60b949c0ac95 + # Gateway control-plane version, matching production. The extension-server SDK + # version is paired with it deliberately, also matching production. + ENVOY_GATEWAY_VERSION: v1.7.4 + EG_SDK_VERSION: v1.8.1 + # Multi-arch WAF image โ€” the same filter as the production edge, but it also + # loads natively on arm64 dev hosts. The data plane references this image. + CORAZA_WAF_IMAGE: ghcr.io/datum-labs/coraza-envoy-go-filter/coraza-waf:v1.3.0-multiarch.1 + # CONNECT-proxy stand-in for the connector-tunnel scenario, built from + # test/e2e-edge/_fixtures/connector-tunnel and loaded into the downstream cluster. + CONNECT_PROXY_IMAGE: connect-proxy:e2e + # The WAF is enabled by default since the multi-arch image loads on every + # architecture. Set CORAZA_DISABLED=true only to deliberately skip the WAF. + CORAZA_DISABLED: '{{.CORAZA_DISABLED | default "false"}}' + # Karmada-bundled kube-apiserver, pinned to production. + KARMADA_APISERVER_VERSION: v1.33.2 + # NSO image pinned by git SHA so the loaded image is used deterministically. + IMG: + sh: echo "ghcr.io/datum-cloud/network-services-operator:$(git rev-parse --short HEAD)" + +tasks: + + up: + desc: "Bring the canonical two-cluster prod-fidelity test env ONLINE (clusters + EG v1.7.4 + ext-server + Coraza data plane + NSO manager)." + cmds: + - echo "๐Ÿš€ test-infra:up โ€” building nso-upstream + nso-downstream" + - task: clusters + # The Gateway-API and gateway CRDs must land on the downstream cluster + # before cert-manager (it probes the Gateway-API CRDs at startup) and before + # the downstream gateway (it needs its own CRDs present). + - task: eg-crds + - task: downstream-crds + - task: downstream-namespaces + - task: cert-manager-upstream + - task: cert-manager-downstream + - task: nso-image + - task: load-fixtures + - task: prepare-upstream + - task: eg-downstream + - task: extension-server + - task: external-dns + - task: link-clusters + - task: wait-ready + - echo "๐ŸŽ‰ test-infra:up complete. Run 'task test-infra:smoke' to confirm M2." + + down: + desc: "Tear down the test-infra clusters (and any Karmada host)." + cmds: + - echo "๐Ÿ’ฅ test-infra:down" + # Deleting both kind clusters removes all federation state, so this unjoin + # is only a best-effort courtesy. + - '{{.KARMADACTL}} deinit --context {{.UPSTREAM_CTX}} --force --purge-namespace 2>/dev/null || true' + - rm -rf {{.TMP_DIR}}/karmada-{{.UPSTREAM_CLUSTER}} + - '{{.KIND}} delete cluster --name {{.UPSTREAM_CLUSTER}} || true' + - '{{.KIND}} delete cluster --name {{.DOWNSTREAM_CLUSTER}} || true' + - rm -f {{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}-internal.yaml {{.TMP_DIR}}/.kind-{{.UPSTREAM_CLUSTER}}.yaml {{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}.yaml + - echo "โœจ done." + + clusters: + desc: "Create the upstream + downstream kind clusters pinned to the prod edge node image." + cmds: + - echo "๐Ÿ—๏ธ creating clusters (node {{.K8S_NODE_IMAGE}})" + - '{{.KIND}} delete cluster --name {{.UPSTREAM_CLUSTER}} 2>/dev/null || true' + - '{{.KIND}} delete cluster --name {{.DOWNSTREAM_CLUSTER}} 2>/dev/null || true' + - '{{.KIND}} create cluster --image {{.K8S_NODE_IMAGE}} --config=config/tools/kind/upstream-cluster.yaml' + - '{{.KIND}} create cluster --image {{.K8S_NODE_IMAGE}} --config=config/tools/kind/downstream-cluster.yaml' + # Running the full federation control plane alongside the rest of the stack + # in one kind node exhausts the default file-watch limits on the macOS + # Docker VM, which crashloops a component with "too many open files" and + # blocks the member join. Raise the limits on both nodes. + - docker exec {{.UPSTREAM_CLUSTER}}-control-plane sysctl -w fs.inotify.max_user_instances=8192 fs.inotify.max_user_watches=1048576 + - docker exec {{.DOWNSTREAM_CLUSTER}}-control-plane sysctl -w fs.inotify.max_user_instances=8192 fs.inotify.max_user_watches=1048576 + + cert-manager-upstream: + desc: "Install cert-manager (+ CSI driver) on the upstream cluster." + cmds: + - '{{.KUSTOMIZE}} build --enable-helm config/tools/cert-manager | kubectl --context {{.UPSTREAM_CTX}} apply --server-side=true --force-conflicts -f -' + - '{{.CMCTL}} check api --context {{.UPSTREAM_CTX}} --wait=5m' + + cert-manager-downstream: + desc: "Install cert-manager (+ CSI driver) on the downstream cluster." + cmds: + - '{{.KUSTOMIZE}} build --enable-helm config/tools/cert-manager | kubectl --context {{.DOWNSTREAM_CTX}} apply --server-side=true --force-conflicts -f -' + - '{{.CMCTL}} check api --context {{.DOWNSTREAM_CTX}} --wait=5m' + + nso-image: + desc: "Build the NSO operator image (git-SHA tag) and load it into both clusters." + cmds: + - echo "๐Ÿ”จ building {{.IMG}}" + - docker build -t {{.IMG}} . + - cd config/manager && {{.KUSTOMIZE}} edit set image ghcr.io/datum-cloud/network-services-operator={{.IMG}} + - '{{.KIND}} load docker-image {{.IMG}} --name {{.UPSTREAM_CLUSTER}}' + - '{{.KIND}} load docker-image {{.IMG}} --name {{.DOWNSTREAM_CLUSTER}}' + + prepare-upstream: + desc: "Deploy the NSO manager + webhook (config/e2e) on the upstream cluster, with the prod-base memory profile." + cmds: + - echo "๐Ÿ”ง deploying NSO manager (upstream)" + - '{{.KUSTOMIZE}} build config/e2e | kubectl --context {{.UPSTREAM_CTX}} apply --server-side=true --force-conflicts -f -' + # Match production: with the extension server handling proxy configuration, + # NSO must not also emit its own patch policies. The shared config file is + # also used by an older path that still relies on them, so we patch the live + # config here rather than editing that file. + - | + CFG=$(kubectl --context {{.UPSTREAM_CTX}} -n network-services-operator-system get cm network-services-operator-config -o jsonpath='{.data.config\.yaml}') + if ! echo "$CFG" | grep -q "eppEmissionEnabled"; then + NEW=$(echo "$CFG" | sed 's|^ downstreamGatewayClassName: .*|&\n eppEmissionEnabled: false|') + kubectl --context {{.UPSTREAM_CTX}} -n network-services-operator-system create cm network-services-operator-config \ + --from-literal=config.yaml="$NEW" --dry-run=client -o yaml | kubectl --context {{.UPSTREAM_CTX}} apply -f - + fi + # Raise the manager's memory limit to the production base so perf sweeps and + # webhook-under-load reflect production headroom. + - | + kubectl --context {{.UPSTREAM_CTX}} -n network-services-operator-system patch deploy network-services-operator-controller-manager \ + --type=json -p '[{"op":"replace","path":"/spec/template/spec/containers/0/resources/limits/memory","value":"2Gi"},{"op":"replace","path":"/spec/template/spec/containers/0/resources/requests/memory","value":"512Mi"}]' || true + + eg-crds: + desc: "Install the full Gateway-API + Envoy Gateway CRD set (incl ReferenceGrant + EnvoyProxy) on the downstream cluster. The downstream EG chart sets includeCRDs:false, so CRDs are applied separately (matches prod, where CRDs are managed out-of-band)." + vars: + EG_CRD_DIR: config/tools/envoy-gateway-downstream/charts/gateway-helm-{{.ENVOY_GATEWAY_VERSION}}/gateway-helm/crds + cmds: + - kubectl --context {{.DOWNSTREAM_CTX}} apply --server-side=true --force-conflicts -f {{.EG_CRD_DIR}}/gatewayapi-crds.yaml + - kubectl --context {{.DOWNSTREAM_CTX}} apply --server-side=true --force-conflicts -f {{.EG_CRD_DIR}}/generated/ + + downstream-namespaces: + desc: "Create the downstream gateway + hostname-accounting namespaces, plus the EG-watched e2e-direct namespace for hand-delivered fixtures (D1/D2). Runs BEFORE eg-downstream so e2e-direct carries its watch label before EG establishes its informer." + cmds: + - kubectl --context {{.DOWNSTREAM_CTX}} apply -f config/dev/downstream_resources/namespaces.yaml + # e2e-direct carries the gateway watch label from creation so gateways + # applied directly here reconcile deterministically. The gateway only + # reliably watches a namespace labeled at creation time, so the label must + # be in the manifest, not added afterward. + - kubectl --context {{.DOWNSTREAM_CTX}} apply -f config/e2e-downstream/direct-namespace.yaml + + eg-downstream: + desc: "Install the dedicated downstream Envoy Gateway (v1.7.4) with the ext-server extensionManager wiring on the downstream cluster." + cmds: + - echo "๐Ÿ”ง installing downstream EG {{.ENVOY_GATEWAY_VERSION}} (+ extensionManager, failOpen:false, maxMessageSize:256Mi)" + # Build from the untracked config/e2e-downstream/eg-downstream copy, which + # bakes in the e2e pins, so a hard reset on the shared branch cannot revert + # them โ€” it kept clobbering the tracked copy. + - '{{.KUSTOMIZE}} build --enable-helm config/e2e-downstream/eg-downstream | kubectl --context {{.DOWNSTREAM_CTX}} apply --server-side=true --force-conflicts -f -' + + load-coraza-waf: + desc: "Fallback: pull the Coraza WAF image and kind-load it into the downstream cluster (offline-CI escape hatch)." + cmds: + - docker pull {{.CORAZA_WAF_IMAGE}} + - '{{.KIND}} load docker-image {{.CORAZA_WAF_IMAGE}} --name {{.DOWNSTREAM_CLUSTER}}' + + load-fixtures: + desc: "Build + load the e2e fixture images into the downstream cluster (CONNECT-proxy stand-in for the connector-tunnel scenario)." + cmds: + - echo "๐Ÿ”ง building + loading fixture image {{.CONNECT_PROXY_IMAGE}}" + - docker build -t {{.CONNECT_PROXY_IMAGE}} test/e2e-edge/_fixtures/connector-tunnel + - '{{.KIND}} load docker-image {{.CONNECT_PROXY_IMAGE}} --name {{.DOWNSTREAM_CLUSTER}}' + + d1-mint-expired-secret: + desc: "D1 helper: mint an ALREADY-EXPIRED kubernetes.io/tls Secret and apply it to the downstream cluster, bypassing the upstream #212 cert-health gate so the ext-server prune backstop can be tested in isolation. Vars: NAMESPACE, SECRET, HOSTNAME." + vars: + # Default to the pre-provisioned, gateway-watched namespace so a gateway + # applied alongside this secret reconciles deterministically. + NAMESPACE: '{{.NAMESPACE | default "e2e-direct"}}' + SECRET: '{{.SECRET | default "d1-expired-tls"}}' + HOSTNAME: '{{.HOSTNAME | default "d1-bad.e2e.env.datum.net"}}' + cmds: + - | + config/e2e-downstream/d1-cert-bypass/mint-expired-secret.sh \ + "{{.NAMESPACE}}" "{{.SECRET}}" "{{.HOSTNAME}}" \ + | kubectl --context {{.DOWNSTREAM_CTX}} apply -f - + - echo "โœ… applied expired TLS Secret {{.NAMESPACE}}/{{.SECRET}} (host {{.HOSTNAME}}) to {{.DOWNSTREAM_CLUSTER}}" + + extension-server: + desc: "Deploy the ext-server (2 replicas + PDB + mTLS) + e2e cert chain + Coraza/branded-page config + test EnvoyProxy (Coraza+admin:19000) on the downstream cluster." + cmds: + - echo "๐Ÿ”ง deploying ext-server + e2e issuer chain + test EnvoyProxy (config/e2e-downstream)" + # Pin the ext-server image to the git-SHA build (same image as the manager). + - cd config/e2e-downstream && {{.KUSTOMIZE}} edit set image ghcr.io/datum-cloud/network-services-operator={{.IMG}} 2>/dev/null || true + - '{{.KUSTOMIZE}} build config/e2e-downstream | kubectl --context {{.DOWNSTREAM_CTX}} apply --server-side=true --force-conflicts -f -' + # WAF disable toggle. The overlay ships the WAF enabled for production + # fidelity; when CORAZA_DISABLED is set we flip the live config so listeners + # program without it. + - | + if [ "{{.CORAZA_DISABLED}}" = "true" ]; then + echo "โš ๏ธ Coraza WAF disabled (host arch {{OS}}/$(uname -m); WAF .so is amd64-only)" + kubectl --context {{.DOWNSTREAM_CTX}} -n network-services-operator-system get cm extension-server-config -o yaml \ + | sed 's/disabled: false/disabled: true/' | kubectl --context {{.DOWNSTREAM_CTX}} apply -f - + fi + # The extension server's server cert and the gateway's client cert are both + # issued from the e2e certificate authority. Wait for that authority's cert, + # then publish it where each side reads it so they can verify each other. + - task: extserver-ca-bundle + + extserver-ca-bundle: + desc: "Publish the e2e CA's ca.crt into the ext-server CA-bundle ConfigMap and the EG certificateRef Secret (both in network-services-operator-system)." + cmds: + - echo "โณ waiting for the e2e CA certificate to be issued" + - | + kubectl --context {{.DOWNSTREAM_CTX}} -n cert-manager wait certificate e2e-extension-server-ca \ + --for=condition=Ready --timeout=120s + - kubectl --context {{.DOWNSTREAM_CTX}} create namespace network-services-operator-system --dry-run=client -o yaml | kubectl --context {{.DOWNSTREAM_CTX}} apply -f - + - | + CA_CRT=$(kubectl --context {{.DOWNSTREAM_CTX}} -n cert-manager get secret e2e-extension-server-ca -o jsonpath='{.data.ca\.crt}' | base64 -d) + # The certificate-authority bundle the extension server mounts. + kubectl --context {{.DOWNSTREAM_CTX}} -n network-services-operator-system create configmap extension-server-ca-bundle \ + --from-literal=ca.crt="$CA_CRT" --dry-run=client -o yaml | kubectl --context {{.DOWNSTREAM_CTX}} apply -f - + # The same authority cert the downstream gateway references. The gateway + # reads it from the tls.crt key, not ca.crt. + kubectl --context {{.DOWNSTREAM_CTX}} -n network-services-operator-system create secret generic e2e-extension-server-ca \ + --from-literal=tls.crt="$CA_CRT" --dry-run=client -o yaml | kubectl --context {{.DOWNSTREAM_CTX}} apply -f - + # Restart the extension server so it picks up the freshly-published bundle. + - kubectl --context {{.DOWNSTREAM_CTX}} -n network-services-operator-system rollout restart deploy network-services-operator-envoy-gateway-extension-server || true + + downstream-crds: + desc: "Install the NSO CRDs the replicator mirrors into the downstream cluster (the Gateway-API/EG CRDs come from eg-crds)." + cmds: + - kubectl --context {{.DOWNSTREAM_CTX}} apply -f config/crd/bases/networking.datumapis.com_connectors.yaml + - kubectl --context {{.DOWNSTREAM_CTX}} apply -f config/crd/bases/networking.datumapis.com_httpproxies.yaml + - kubectl --context {{.DOWNSTREAM_CTX}} apply -f config/crd/bases/networking.datumapis.com_trafficprotectionpolicies.yaml + + external-dns: + desc: "Install external-dns CRDs (DNSEndpoint) on the downstream cluster." + cmds: + - '{{.KUSTOMIZE}} build --enable-helm config/tools/external-dns | kubectl --context {{.DOWNSTREAM_CTX}} apply --server-side=true --force-conflicts -f -' + + link-clusters: + desc: "Wire the NSO manager's downstream client to the downstream cluster via the downstream-cluster-kubeconfig secret on the upstream cluster." + cmds: + - echo "๐Ÿ”— linking upstream -> downstream" + - '{{.KIND}} get kubeconfig --name {{.DOWNSTREAM_CLUSTER}} --internal > {{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}-internal.yaml' + - kubectl --context {{.UPSTREAM_CTX}} create namespace network-services-operator-system --dry-run=client -o yaml | kubectl --context {{.UPSTREAM_CTX}} apply -f - + - | + kubectl --context {{.UPSTREAM_CTX}} create secret -n network-services-operator-system \ + generic downstream-cluster-kubeconfig --save-config --dry-run=client -o yaml \ + --from-file=kubeconfig={{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}-internal.yaml | kubectl --context {{.UPSTREAM_CTX}} apply -f - + + wait-ready: + desc: "Wait for the core components (NSO manager, downstream EG, ext-server) to be Available." + cmds: + - echo "โณ waiting for NSO manager (upstream)" + - kubectl --context {{.UPSTREAM_CTX}} -n network-services-operator-system wait deploy network-services-operator-controller-manager --for=condition=Available --timeout=240s + - echo "โณ waiting for downstream EG" + # The gateway control-plane Deployment is named `envoy-gateway` by its chart, + # not after the release name. + - kubectl --context {{.DOWNSTREAM_CTX}} -n datum-downstream-gateway wait deploy envoy-gateway --for=condition=Available --timeout=240s + - echo "โณ waiting for ext-server" + - kubectl --context {{.DOWNSTREAM_CTX}} -n network-services-operator-system wait deploy network-services-operator-envoy-gateway-extension-server --for=condition=Available --timeout=240s + - echo "โœ… core components ready." + + # ---- Karmada ------------------------------------------------------------ + + karmada-up: + desc: "Stand up a real Karmada host on the upstream cluster (apiserver v1.33.2), join the downstream member, and apply the prod federation artifacts." + vars: + KARMADA_KUBECONFIG: '{{.TMP_DIR}}/karmada-{{.UPSTREAM_CLUSTER}}/karmada-apiserver.config' + # The upstream node's docker-network IP, reachable from the downstream node + # on the same network, used to sign the Karmada apiserver cert. + HOST_IP: + sh: docker inspect {{.UPSTREAM_CLUSTER}}-control-plane -f '{{"{{"}}.NetworkSettings.Networks.kind.IPAddress{{"}}"}}' + # The downstream member's docker-network IP โ€” reachable from the upstream + # control-plane pods on the same network, but not from the macOS host. + MEMBER_IP: + sh: docker inspect {{.DOWNSTREAM_CLUSTER}}-control-plane -f '{{"{{"}}.NetworkSettings.Networks.kind.IPAddress{{"}}"}}' + cmds: + # Advertise 127.0.0.1: karmadactl runs on the macOS host, and init's own + # post-deploy steps dial the advertise address. The docker-network IP is not + # routable from the macOS host, so advertising it would make init time out; + # the upstream cluster maps a host port through to the apiserver, so + # 127.0.0.1 reaches it. We still sign the cert for the docker IP for any + # in-cluster path. The member is joined in push mode, so it never needs to + # reach the apiserver address โ€” advertising 127.0.0.1 is safe for the join. + - echo "๐ŸŒ karmada init (apiserver {{.KARMADA_APISERVER_VERSION}}, advertise 127.0.0.1:32443, cert-ip incl {{.HOST_IP}}) on {{.UPSTREAM_CLUSTER}}" + - mkdir -p {{.TMP_DIR}}/karmada-{{.UPSTREAM_CLUSTER}} + - | + {{.KARMADACTL}} init --context {{.UPSTREAM_CTX}} \ + --kube-image-tag {{.KARMADA_APISERVER_VERSION}} \ + --karmada-apiserver-advertise-address 127.0.0.1 \ + --cert-external-ip "127.0.0.1,{{.HOST_IP}}" \ + --etcd-storage-mode hostPath \ + --karmada-data {{.TMP_DIR}}/karmada-{{.UPSTREAM_CLUSTER}} \ + --karmada-pki {{.TMP_DIR}}/karmada-{{.UPSTREAM_CLUSTER}}/pki + - echo "๐Ÿ”— joining {{.DOWNSTREAM_CLUSTER}} as a Karmada member" + # karmadactl join runs on the host and connects to the member to install its + # agent, so it needs a host-reachable member kubeconfig, not the internal + # docker-hostname form the host cannot resolve. + - '{{.KIND}} get kubeconfig --name {{.DOWNSTREAM_CLUSTER}} > {{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}-host.yaml' + - | + {{.KARMADACTL}} join {{.DOWNSTREAM_CLUSTER}} \ + --karmada-context karmada-apiserver \ + --kubeconfig {{.KARMADA_KUBECONFIG}} \ + --cluster-kubeconfig {{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}-host.yaml \ + --cluster-context {{.DOWNSTREAM_CTX}} + # The join stores the host address as the member's endpoint, which the + # control-plane pods running inside the upstream cluster cannot reach. + # Repoint it to the member's docker IP, which they can reach on the shared + # network, and the member then goes Ready. + - | + kubectl --kubeconfig {{.KARMADA_KUBECONFIG}} --context karmada-apiserver \ + patch cluster {{.DOWNSTREAM_CLUSTER}} --type=merge \ + -p '{"spec":{"apiEndpoint":"https://{{.MEMBER_IP}}:6443"}}' + # Label the member so the production propagation policy places resources onto + # it. + - | + kubectl --kubeconfig {{.KARMADA_KUBECONFIG}} --context karmada-apiserver \ + label cluster {{.DOWNSTREAM_CLUSTER}} infra.datum.net/gateways=enabled --overwrite + - echo "โณ waiting for the member cluster to become Ready" + - | + kubectl --kubeconfig {{.KARMADA_KUBECONFIG}} --context karmada-apiserver \ + wait cluster {{.DOWNSTREAM_CLUSTER}} --for=condition=Ready --timeout=120s + - echo "๐Ÿ“œ applying federation artifacts (config/federation)" + - | + kubectl --kubeconfig {{.KARMADA_KUBECONFIG}} --context karmada-apiserver apply -f config/federation/ + - echo "โœ… karmada-up complete. Karmada apiserver kubeconfig at {{.KARMADA_KUBECONFIG}}" + + smoke: + desc: "M2 functional confirmation: drive an upstream Gateway+HTTPRoute through the ext-server path and curl a real 200." + cmds: + - '{{.CHAINSAW}} test ./test/e2e-edge/extension-server-smoke --cluster {{.UPSTREAM_CLUSTER}}={{.TMP_DIR}}/.kind-{{.UPSTREAM_CLUSTER}}.yaml --cluster {{.DOWNSTREAM_CLUSTER}}={{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}.yaml' + + e2e: + desc: "Run chainsaw e2e scenarios against the live two-cluster env. Pass a scenario name or path after -- (e.g. `task test-infra:e2e -- waf-enforcement`); with no arg, runs every ext-server-path scenario that targets nso-upstream/nso-downstream. Use SCENARIOS=... to override the default set." + vars: + # Scenarios authored against this env's cluster names and downstream path. + # Older fixtures targeting the previous cluster names are excluded here. + DEFAULT_SCENARIOS: extension-server-smoke waf-enforcement branded-error-page connector-offline-503 atomic-reject-isolation + # CLI_ARGS (after --) wins; else SCENARIOS env; else the default set. + SELECTED: '{{.CLI_ARGS | default .SCENARIOS | default .DEFAULT_SCENARIOS}}' + deps: + - kubeconfigs + cmds: + - | + set -e + for s in {{.SELECTED}}; do + # Accept either a bare scenario name or a full/relative path. + case "$s" in + test/e2e-edge/*|./test/e2e-edge/*) dir="$s" ;; + */*) dir="$s" ;; + *) dir="./test/e2e-edge/$s" ;; + esac + echo "๐Ÿงช chainsaw: $dir" + {{.CHAINSAW}} test "$dir" \ + --cluster {{.UPSTREAM_CLUSTER}}={{.TMP_DIR}}/.kind-{{.UPSTREAM_CLUSTER}}.yaml \ + --cluster {{.DOWNSTREAM_CLUSTER}}={{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}.yaml + done + + kubeconfigs: + desc: "Export per-cluster kubeconfigs to TMPDIR for chainsaw." + cmds: + - '{{.KIND}} get kubeconfig --name {{.UPSTREAM_CLUSTER}} > {{.TMP_DIR}}/.kind-{{.UPSTREAM_CLUSTER}}.yaml' + - '{{.KIND}} get kubeconfig --name {{.DOWNSTREAM_CLUSTER}} > {{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}.yaml' + + parity:check: + desc: "Run the config-dump parity gate against the live downstream Envoy + ext-server. Pass CLI flags after -- (PARITY owns the CLI). Exit 0 PASS / 1 parity FAIL / 2 tool error." + cmds: + - go build -o bin/parity-check ./cmd/parity-check + - ./bin/parity-check {{.CLI_ARGS}} + + parity:check-live: + desc: "Convenience: resolve the live data-plane Envoy pod from this env's labels and run parity:check in kubectl-exec mode (ext-server via selector across replicas). Extra flags pass through after --." + vars: + # In this env the data-plane proxy and the extension server live in these + # namespaces. + DP_NS: datum-downstream-gateway + DP_SELECTOR: gateway.envoyproxy.io/owning-gatewayclass=datum-downstream-gateway-e2e + EXT_NS: network-services-operator-system + EXT_SELECTOR: app.kubernetes.io/component=envoy-gateway-extension-server + # The admin side wants a single exact proxy pod name, resolved here. The + # extension-server side takes a selector and picks the authoritative replica + # itself. + DP_POD: + sh: kubectl --context {{.DOWNSTREAM_CTX}} -n datum-downstream-gateway get pods -l gateway.envoyproxy.io/owning-gatewayclass=datum-downstream-gateway-e2e -o jsonpath='{.items[0].metadata.name}' + cmds: + - go build -o bin/parity-check ./cmd/parity-check + - | + if [ -z "{{.DP_POD}}" ]; then + echo "no data-plane Envoy pod found (label {{.DP_SELECTOR}} in {{.DP_NS}}); a Gateway must exist in a watched namespace for the merged data plane to be provisioned" >&2 + exit 2 + fi + - | + ./bin/parity-check \ + --coraza-filter=coraza-waf \ + --admin-exec-pod={{.DP_POD}} --admin-exec-namespace={{.DP_NS}} --admin-exec-container=envoy --admin-exec-context={{.DOWNSTREAM_CTX}} \ + --ext-exec-selector={{.EXT_SELECTOR}} --ext-exec-namespace={{.EXT_NS}} --ext-exec-context={{.DOWNSTREAM_CTX}} \ + {{.CLI_ARGS}} diff --git a/Taskfile.yaml b/Taskfile.yaml index a34a3994..890a88f7 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -7,6 +7,8 @@ includes: dir: ./docs dev: taskfile: ./Taskfile.dev.yaml + test-infra: + taskfile: ./Taskfile.test-infra.yml tasks: validate-kustomizations: diff --git a/config/e2e-downstream/d1-cert-bypass/README.md b/config/e2e-downstream/d1-cert-bypass/README.md new file mode 100644 index 00000000..1923679c --- /dev/null +++ b/config/e2e-downstream/d1-cert-bypass/README.md @@ -0,0 +1,28 @@ +# Expired-certificate isolation fixture (test-env-only) + +In production, an expired or otherwise unusable TLS certificate is caught early: +the platform withholds that listener from the edge before it is ever delivered. +The extension server *also* removes unusable certificates at the edge, as a +second line of defense โ€” but because the earlier check normally catches the +problem first, that edge-side removal rarely gets exercised on the normal path. + +This fixture lets a test exercise it directly, by handing the edge a genuinely +expired certificate and bypassing the earlier check. + +1. `mint-expired-secret.sh ` writes a + self-signed, already-expired certificate as a TLS Secret to stdout. Apply it + into the `e2e-direct` namespace on the edge cluster. +2. The test then applies a gateway directly to the edge whose HTTPS listener + uses that certificate. The extension server removes the bad listener while a + healthy sibling keeps serving โ€” which is what the test asserts. + +> **Use the `e2e-direct` namespace.** The gateway controller only watches +> namespaces that already carry the `meta.datumapis.com/upstream-cluster-name` +> label when they are created; a label added afterward is not reliably picked +> up, and a gateway there can stay unprogrammed. The `e2e-direct` namespace is +> created with the label up front for exactly this reason. If you must create a +> namespace inline, set the label at creation time. + +`task -t Taskfile.test-infra.yml d1-mint-expired-secret` is a thin wrapper around +the script (defaults: `NAMESPACE=e2e-direct`, `SECRET=d1-expired-tls`, +`HOSTNAME=d1-bad.e2e.env.datum.net`). diff --git a/config/e2e-downstream/d1-cert-bypass/mint-expired-secret.sh b/config/e2e-downstream/d1-cert-bypass/mint-expired-secret.sh new file mode 100755 index 00000000..8ea5e705 --- /dev/null +++ b/config/e2e-downstream/d1-cert-bypass/mint-expired-secret.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Mint an already-expired self-signed TLS certificate and emit it as a +# kubernetes.io/tls Secret on stdout. Test-env-only: it hands the gateway an +# expired certificate directly, so the extension server's removal of unusable +# certificates can be exercised on its own, without the earlier check rejecting +# it first. +# +# Usage: mint-expired-secret.sh +set -euo pipefail + +NS="${1:?namespace required}" +SECRET="${2:?secret name required}" +HOST="${3:?hostname required}" + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +# Generate a key + a self-signed cert dated entirely in the past so it is expired +# the moment it is created. openssl's -not_before/-not_after (LibreSSL/OpenSSL 3) +# set an explicit validity window; fall back to a 1-second window via -days 0 if +# the flags are unavailable. +openssl req -x509 -newkey rsa:2048 -nodes \ + -keyout "$WORK/tls.key" -out "$WORK/tls.crt" \ + -subj "/CN=${HOST}" \ + -addext "subjectAltName=DNS:${HOST}" \ + -not_before 20200101000000Z -not_after 20200102000000Z 2>/dev/null \ + || openssl req -x509 -newkey rsa:2048 -nodes \ + -keyout "$WORK/tls.key" -out "$WORK/tls.crt" \ + -subj "/CN=${HOST}" -addext "subjectAltName=DNS:${HOST}" -days 1 2>/dev/null + +CRT_B64="$(base64 < "$WORK/tls.crt" | tr -d '\n')" +KEY_B64="$(base64 < "$WORK/tls.key" | tr -d '\n')" + +cat < + + + Service Unavailable +

This service is temporarily unavailable.

+ diff --git a/config/e2e-downstream/extserver-base/kustomization.yaml b/config/e2e-downstream/extserver-base/kustomization.yaml new file mode 100644 index 00000000..2ba7af04 --- /dev/null +++ b/config/e2e-downstream/extserver-base/kustomization.yaml @@ -0,0 +1,28 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Ext-server, namespaced + prefixed so the Service FQDN matches the EG +# extensionManager fqdn and the CSI dns-names baked into the base deployment. +namespace: network-services-operator-system +namePrefix: network-services-operator- + +resources: + - ../../extension-server + +patches: + - path: patches/extserver-tls.yaml + target: + kind: Deployment + name: envoy-gateway-extension-server + - path: patches/extserver-serverconfig.yaml + target: + kind: Deployment + name: envoy-gateway-extension-server + - path: patches/extserver-clientcert-issuer.yaml + target: + kind: Certificate + name: envoy-gateway-extension-server-eg-client-tls + - path: patches/extserver-ca-bundle.yaml + target: + kind: Deployment + name: envoy-gateway-extension-server diff --git a/config/e2e-downstream/extserver-base/patches/extserver-ca-bundle.yaml b/config/e2e-downstream/extserver-base/patches/extserver-ca-bundle.yaml new file mode 100644 index 00000000..6e85fc0f --- /dev/null +++ b/config/e2e-downstream/extserver-base/patches/extserver-ca-bundle.yaml @@ -0,0 +1,18 @@ +# Point the ext-server CA bundle volume at the e2e CA ConfigMap (carrying the +# ca.crt that signed the EG client cert), replacing placeholder-ca-bundle. The +# ConfigMap is published by the bring-up (test-infra:extserver-ca-bundle) from +# the e2e-extension-server-ca cert-manager secret. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: envoy-gateway-extension-server +spec: + template: + spec: + volumes: + - name: tls-ca + configMap: + name: extension-server-ca-bundle + items: + - key: ca.crt + path: ca.crt diff --git a/config/e2e-downstream/extserver-base/patches/extserver-clientcert-issuer.yaml b/config/e2e-downstream/extserver-base/patches/extserver-clientcert-issuer.yaml new file mode 100644 index 00000000..9710265e --- /dev/null +++ b/config/e2e-downstream/extserver-base/patches/extserver-clientcert-issuer.yaml @@ -0,0 +1,11 @@ +# Point the EG client cert (CN=envoy-gateway) at the e2e CA ClusterIssuer, +# replacing the base certificate's placeholder-issuer. +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: envoy-gateway-extension-server-eg-client-tls +spec: + issuerRef: + name: e2e-extension-server-ca + kind: ClusterIssuer + group: cert-manager.io diff --git a/config/e2e-downstream/extserver-base/patches/extserver-serverconfig.yaml b/config/e2e-downstream/extserver-base/patches/extserver-serverconfig.yaml new file mode 100644 index 00000000..2c9701d9 --- /dev/null +++ b/config/e2e-downstream/extserver-base/patches/extserver-serverconfig.yaml @@ -0,0 +1,26 @@ +# Mount the operator config ConfigMap and set SERVER_CONFIG to its path so the +# ext-server loads Coraza + branded-error-page settings. Strategic-merge on env +# (matched by name) and on volumes/volumeMounts. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: envoy-gateway-extension-server +spec: + template: + spec: + containers: + - name: envoy-gateway-extension-server + env: + - name: SERVER_CONFIG + value: /etc/datum/server-config/config.yaml + volumeMounts: + - name: server-config + mountPath: /etc/datum/server-config + readOnly: true + volumes: + - name: server-config + configMap: + name: extension-server-config + items: + - key: config.yaml + path: config.yaml diff --git a/config/e2e-downstream/extserver-base/patches/extserver-tls.yaml b/config/e2e-downstream/extserver-base/patches/extserver-tls.yaml new file mode 100644 index 00000000..c190ac81 --- /dev/null +++ b/config/e2e-downstream/extserver-base/patches/extserver-tls.yaml @@ -0,0 +1,21 @@ +# Point the ext-server server-cert CSI volume at the e2e CA-backed ClusterIssuer, +# replacing the base deployment's placeholder-issuer. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: envoy-gateway-extension-server +spec: + template: + spec: + volumes: + - name: tls + csi: + driver: csi.cert-manager.io + readOnly: true + volumeAttributes: + csi.cert-manager.io/issuer-kind: ClusterIssuer + csi.cert-manager.io/issuer-name: e2e-extension-server-ca + csi.cert-manager.io/common-name: envoy-gateway-extension-server + csi.cert-manager.io/dns-names: "network-services-operator-envoy-gateway-extension-server.network-services-operator-system.svc,network-services-operator-envoy-gateway-extension-server.network-services-operator-system.svc.cluster.local" + csi.cert-manager.io/key-usages: server auth + csi.cert-manager.io/fs-group: "65532" diff --git a/config/e2e-downstream/extserver-config.yaml b/config/e2e-downstream/extserver-config.yaml new file mode 100644 index 00000000..f5828a0c --- /dev/null +++ b/config/e2e-downstream/extserver-config.yaml @@ -0,0 +1,21 @@ +# Extension server config for the e2e edge. +# +# The WAF is enabled so its rules reach the proxy, and the branded 5xx page is +# pointed at the mounted error-pages volume so the suite can assert the branded +# body by content. The connector-tunnel listener name is left at its default so +# the connector fixtures find the listener by the same name production uses. +apiVersion: v1 +kind: ConfigMap +metadata: + name: extension-server-config + namespace: network-services-operator-system +data: + config.yaml: | + apiVersion: apiserver.config.datumapis.com/v1alpha1 + kind: NetworkServicesOperator + gateway: + coraza: + disabled: false + errorPage: + enabled: true + bodyPath: /etc/datum/error-pages/error-5xx.html diff --git a/config/e2e-downstream/issuer.yaml b/config/e2e-downstream/issuer.yaml new file mode 100644 index 00000000..35fc7a62 --- /dev/null +++ b/config/e2e-downstream/issuer.yaml @@ -0,0 +1,41 @@ +# Self-signed root + CA-backed ClusterIssuer for the e2e ext-server mTLS chain. +# +# This issues both sides of the EG <-> ext-server handshake: +# - the ext-server SERVER cert (via the CSI driver, see patches/extserver-tls.yaml) +# - the EG CLIENT cert (envoy-gateway-extension-server-eg-client-tls, CN=envoy-gateway) +# The CA's ca.crt is also published into the ext-server CA bundle ConfigMap +# (so the ext-server can verify the EG client) and into the EG certificateRef +# secret (so EG can verify the ext-server server cert). +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: e2e-extension-server-selfsigned +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: e2e-extension-server-ca + namespace: cert-manager +spec: + isCA: true + commonName: e2e-extension-server-ca + secretName: e2e-extension-server-ca + duration: 8760h + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: e2e-extension-server-selfsigned + kind: ClusterIssuer + group: cert-manager.io +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: e2e-extension-server-ca +spec: + ca: + secretName: e2e-extension-server-ca diff --git a/config/e2e-downstream/kustomization.yaml b/config/e2e-downstream/kustomization.yaml index b00e83e0..6ab183ae 100644 --- a/config/e2e-downstream/kustomization.yaml +++ b/config/e2e-downstream/kustomization.yaml @@ -1,76 +1,38 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -namespace: datum-downstream-gateway - +# The e2e edge overlay applied to nso-downstream. Composes the ext-server +# (prefixed/namespaced via extserver-base) with the e2e mTLS issuer chain, the +# Coraza/branded-page server-config, the branded error page, and the test +# EnvoyProxy (real Coraza WAF image + admin :19000) and its GatewayClass. +# +# The ConfigMaps / issuer / EnvoyProxy / GatewayClass are kept OUT of the +# name-prefix so the deployment's literal references (envoy-error-pages, +# extension-server-config) and the EG fqdn resolve unchanged. resources: - - namespace.yaml - - ../extension-server - - pki.yaml - - waf-gateway.yaml +- namespace.yaml +- extserver-base +- issuer.yaml +- extserver-config.yaml +- error-pages.yaml +- envoyproxy.yaml -configMapGenerator: - - name: nso-extension-server-config - files: - - config.yaml=extension-server-config.yaml - options: - disableNameSuffixHash: true +# The NSO image tag is set by the bring-up (test-infra:extension-server) to the +# git-SHA-built image. +images: +- name: ghcr.io/datum-cloud/network-services-operator + newName: ghcr.io/datum-cloud/network-services-operator + newTag: 63fa912 +# The default e2e path has no Prometheus operator; drop the ServiceMonitor so +# the apply doesn't fail on the missing monitoring.coreos.com CRD. (Re-add via +# OBSERVABILITY=1 when the Flux observability stack is enabled.) patches: - - target: - kind: ServiceMonitor +- patch: | + $patch: delete + apiVersion: monitoring.coreos.com/v1 + kind: ServiceMonitor + metadata: name: envoy-gateway-extension-server-metrics - patch: | - $patch: delete - apiVersion: monitoring.coreos.com/v1 - kind: ServiceMonitor - metadata: - name: envoy-gateway-extension-server-metrics - - target: - kind: Certificate - name: envoy-gateway-extension-server-eg-client-tls - patch: | - - op: replace - path: /spec/issuerRef/kind - value: Issuer - - op: replace - path: /spec/issuerRef/name - value: envoy-gateway-extension-server - - target: - kind: Deployment - name: envoy-gateway-extension-server - patch: | - apiVersion: apps/v1 - kind: Deployment - metadata: - name: envoy-gateway-extension-server - spec: - replicas: 1 - template: - spec: - containers: - - name: envoy-gateway-extension-server - env: - - name: SERVER_CONFIG - value: /config/config.yaml - volumeMounts: - - name: server-config - mountPath: /config - readOnly: true - volumes: - - name: tls - csi: - volumeAttributes: - csi.cert-manager.io/issuer-kind: Issuer - csi.cert-manager.io/issuer-name: envoy-gateway-extension-server - csi.cert-manager.io/dns-names: "envoy-gateway-extension-server.datum-downstream-gateway.svc,envoy-gateway-extension-server.datum-downstream-gateway.svc.cluster.local" - - name: tls-ca - configMap: null - secret: - secretName: envoy-gateway-extension-server-ca - items: - - key: ca.crt - path: ca.crt - - name: server-config - configMap: - name: nso-extension-server-config + target: + kind: ServiceMonitor diff --git a/config/e2e-downstream/namespace.yaml b/config/e2e-downstream/namespace.yaml index 89c7d150..b27c33c1 100644 --- a/config/e2e-downstream/namespace.yaml +++ b/config/e2e-downstream/namespace.yaml @@ -1,6 +1,6 @@ +# The ext-server namespace on the edge cluster. The base ext-server resources +# target this namespace but do not create it. apiVersion: v1 kind: Namespace metadata: - name: datum-downstream-gateway - labels: - meta.datumapis.com/upstream-cluster-name: 0-datum-internal-anchor + name: network-services-operator-system diff --git a/config/tools/envoy-gateway-downstream/kustomization.yaml b/config/tools/envoy-gateway-downstream/kustomization.yaml index f95b40f2..9f5a2a92 100644 --- a/config/tools/envoy-gateway-downstream/kustomization.yaml +++ b/config/tools/envoy-gateway-downstream/kustomization.yaml @@ -6,10 +6,6 @@ helmCharts: includeCRDs: false namespace: datum-downstream-gateway releaseName: envoy-datum-downstream-gateway - # Pinned to match production: v1.8.x does not program merged gateways served - # through the extension manager (waf-gw stays with an empty status), so the - # downstream WAF control plane runs v1.7.4. CRDs come from the base - # envoy-gateway install (v1.8.1), which is forward-compatible. version: v1.7.4 repo: oci://docker.io/envoyproxy valuesInline: @@ -33,9 +29,9 @@ helmCharts: - key: meta.datumapis.com/upstream-cluster-name operator: Exists extensionManager: - # The WAF-augmented merged xDS response exceeds gRPC's 4 MiB default, - # so EG rejects it and stops publishing config with failOpen:false. - # Match the ext-server's 256 MiB limit; both sides must agree. + # Match the ext-server's compiled-in 256 MiB gRPC ceiling. Without + # this the EG side defaults to ~4 MiB and silently freezes xDS once a + # translated snapshot exceeds it (~540 gateways in prod). maxMessageSize: 256Mi policyResources: - group: networking.datumapis.com @@ -47,17 +43,18 @@ helmCharts: kind: Connector service: fqdn: - hostname: envoy-gateway-extension-server.datum-downstream-gateway.svc.cluster.local + hostname: network-services-operator-envoy-gateway-extension-server.network-services-operator-system.svc.cluster.local port: 5005 tls: - # EG reads the CA from tls.crt of this Secret; the self-signed CA - # Certificate's Secret carries the CA cert there. certificateRef: - name: envoy-gateway-extension-server-ca - namespace: datum-downstream-gateway + # e2e: the Secret holding the CA that issued the ext-server + # server cert. Published by the bring-up (test-infra:extserver-ca-bundle) + # from the e2e-extension-server-ca cert-manager secret. + name: e2e-extension-server-ca + namespace: network-services-operator-system clientCertificateRef: name: envoy-gateway-extension-server-eg-client-tls - namespace: datum-downstream-gateway + namespace: network-services-operator-system retry: maxAttempts: 4 initialBackoff: 100ms diff --git a/config/tools/kind/downstream-cluster.yaml b/config/tools/kind/downstream-cluster.yaml new file mode 100644 index 00000000..6339242a --- /dev/null +++ b/config/tools/kind/downstream-cluster.yaml @@ -0,0 +1,14 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: nso-downstream +networking: + ipFamily: dual +nodes: +- role: control-plane + extraPortMappings: + - containerPort: 30080 + hostPort: 30080 + protocol: TCP + - containerPort: 30443 + hostPort: 30443 + protocol: TCP diff --git a/config/tools/kind/upstream-cluster.yaml b/config/tools/kind/upstream-cluster.yaml new file mode 100644 index 00000000..be723669 --- /dev/null +++ b/config/tools/kind/upstream-cluster.yaml @@ -0,0 +1,15 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: nso-upstream +networking: + ipFamily: dual +nodes: +- role: control-plane + # Expose the Karmada apiserver NodePort (32443) to the host so karmadactl โ€” + # which runs on the host โ€” can reach the Karmada apiserver during init/join. + # On macOS the kind docker-network IP is not host-routable, so without this + # mapping karmadactl times out dialing the advertised node IP:32443. + extraPortMappings: + - containerPort: 32443 + hostPort: 32443 + protocol: TCP From a824579fae8ceb7b9359e3e4bbd65f9b6da66c8a Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 25 Jun 2026 16:41:26 -0500 Subject: [PATCH 02/20] Enable the programmed-set endpoint in the test environment The endpoint that lets the parity test confirm the proxy is running exactly the set the build intended is off by default and served only when asked for. The base deployment reads --enable-programmed-set from the ENABLE_PROGRAMMED_SET env var; flip it to "true" in the downstream edge overlay with a strategic-merge patch on env so the test environment keeps that visibility while production exposes nothing. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01JbCy8vy66RdNYzGSgqH6P6 --- .../extserver-base/kustomization.yaml | 4 ++++ .../patches/extserver-programmed-set.yaml | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 config/e2e-downstream/extserver-base/patches/extserver-programmed-set.yaml diff --git a/config/e2e-downstream/extserver-base/kustomization.yaml b/config/e2e-downstream/extserver-base/kustomization.yaml index 2ba7af04..14e55681 100644 --- a/config/e2e-downstream/extserver-base/kustomization.yaml +++ b/config/e2e-downstream/extserver-base/kustomization.yaml @@ -26,3 +26,7 @@ patches: target: kind: Deployment name: envoy-gateway-extension-server + - path: patches/extserver-programmed-set.yaml + target: + kind: Deployment + name: envoy-gateway-extension-server diff --git a/config/e2e-downstream/extserver-base/patches/extserver-programmed-set.yaml b/config/e2e-downstream/extserver-base/patches/extserver-programmed-set.yaml new file mode 100644 index 00000000..aa4ebfc4 --- /dev/null +++ b/config/e2e-downstream/extserver-base/patches/extserver-programmed-set.yaml @@ -0,0 +1,17 @@ +# Turn on the read-only /debug/programmed-set endpoint so the parity test can +# confirm the proxy is running exactly the set the build intended. The base +# deployment reads --enable-programmed-set from this env var, defaulting off in +# production; flip it to "true" here for the test environment. Strategic-merge +# on env (matched by name). +apiVersion: apps/v1 +kind: Deployment +metadata: + name: envoy-gateway-extension-server +spec: + template: + spec: + containers: + - name: envoy-gateway-extension-server + env: + - name: ENABLE_PROGRAMMED_SET + value: "true" From 90b2b12b55b3844805a0f8159ef728f34eba986b Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Mon, 13 Jul 2026 20:33:22 -0400 Subject: [PATCH 03/20] test(infra): run e2e suite against the two-cluster prod-fidelity env Add a test-infra:test-e2e task that runs the control-plane e2e suite against the live #226 two-cluster environment, mapping the suite's chainsaw aliases (nso-standard/nso-infra) onto this env's upstream and downstream kubeconfigs at invocation. Bring the hardened suite from main onto this branch and pin the operator image to the env's build. Key changes: - add test-infra:test-e2e task; CLI_ARGS selects a single scenario - pin manager and e2e-downstream image tags to d2e62bf - assert the ClusterIssuer CA Certificate is Ready before gateway steps - carry the listener-hostname-collision, tpp-enforce, and tpp-paranoia-validation scenarios from main --- Taskfile.test-infra.yml | 15 +++++++++++++++ config/e2e-downstream/kustomization.yaml | 2 +- config/manager/kustomization.yaml | 2 +- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/Taskfile.test-infra.yml b/Taskfile.test-infra.yml index ae577cf1..34de268b 100644 --- a/Taskfile.test-infra.yml +++ b/Taskfile.test-infra.yml @@ -378,6 +378,21 @@ tasks: --cluster {{.DOWNSTREAM_CLUSTER}}={{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}.yaml done + test-e2e: + desc: "Run the full control-plane e2e suite (test/e2e) against the live two-cluster env. The suite's chainsaw aliases nso-standard/nso-infra are mapped to this env's upstream/downstream kubeconfigs at invocation. Pass a scenario path after -- to run a single test." + vars: + # CLI_ARGS (after --) wins so a single test can be selected; else the whole + # suite directory. + TARGET: '{{.CLI_ARGS | default "./test/e2e"}}' + deps: + - kubeconfigs + cmds: + - echo "๐Ÿงช chainsaw suite {{.TARGET}} (nso-standard={{.UPSTREAM_CLUSTER}}, nso-infra={{.DOWNSTREAM_CLUSTER}})" + - | + {{.CHAINSAW}} test {{.TARGET}} \ + --cluster nso-standard={{.TMP_DIR}}/.kind-{{.UPSTREAM_CLUSTER}}.yaml \ + --cluster nso-infra={{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}.yaml + kubeconfigs: desc: "Export per-cluster kubeconfigs to TMPDIR for chainsaw." cmds: diff --git a/config/e2e-downstream/kustomization.yaml b/config/e2e-downstream/kustomization.yaml index 6ab183ae..39cbe731 100644 --- a/config/e2e-downstream/kustomization.yaml +++ b/config/e2e-downstream/kustomization.yaml @@ -22,7 +22,7 @@ resources: images: - name: ghcr.io/datum-cloud/network-services-operator newName: ghcr.io/datum-cloud/network-services-operator - newTag: 63fa912 + newTag: d2e62bf # The default e2e path has no Prometheus operator; drop the ServiceMonitor so # the apply doesn't fail on the missing monitoring.coreos.com CRD. (Re-add via diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index ff5b1ecf..8f2fd385 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -6,7 +6,7 @@ resources: images: - name: ghcr.io/datum-cloud/network-services-operator newName: ghcr.io/datum-cloud/network-services-operator - newTag: latest + newTag: d2e62bf configMapGenerator: - files: - config.yaml From 86f3c45fe0def93eec40ac81a92e2af5e3e31537 Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Mon, 13 Jul 2026 21:26:16 -0400 Subject: [PATCH 04/20] test(e2e): target the env GatewayClass in the WAF enforce test The prod-fidelity env installs the downstream GatewayClass as datum-downstream-gateway-e2e; the enforce test hardcoded the bare datum-downstream-gateway, so waf-gw never bound a class and timed out before Programmed. Point the Gateway, diagnostics, and comment at the env's class. waf-gw now programs and benign traffic returns 200. --- test/e2e/trafficprotectionpolicy-enforce/chainsaw-test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e/trafficprotectionpolicy-enforce/chainsaw-test.yaml b/test/e2e/trafficprotectionpolicy-enforce/chainsaw-test.yaml index 9e42f2e2..0619031b 100644 --- a/test/e2e/trafficprotectionpolicy-enforce/chainsaw-test.yaml +++ b/test/e2e/trafficprotectionpolicy-enforce/chainsaw-test.yaml @@ -12,7 +12,7 @@ metadata: # # Precondition: the downstream (nso-infra) must have the WAF data plane wired up # (extension-server + the extensionManager Envoy Gateway registered on the -# `datum-downstream-gateway` GatewayClass, with the Coraza filter). +# `datum-downstream-gateway-e2e` GatewayClass, with the Coraza filter). # `make prepare-infra-cluster` installs this via `make downstream-waf-dataplane`. spec: cluster: nso-infra @@ -79,7 +79,7 @@ spec: metadata: name: waf-gw spec: - gatewayClassName: datum-downstream-gateway + gatewayClassName: datum-downstream-gateway-e2e listeners: - name: http protocol: HTTP @@ -143,7 +143,7 @@ spec: set -x kubectl get gateway -A -o yaml kubectl describe gateway -A - kubectl get gatewayclass datum-downstream-gateway -o yaml + kubectl get gatewayclass datum-downstream-gateway-e2e -o yaml kubectl -n datum-downstream-gateway get pods -o wide kubectl -n datum-downstream-gateway describe pods kubectl -n datum-downstream-gateway logs deploy/envoy-gateway --tail=-1 From c4eaa13c1558d0d1d2f2cee1d8b8fa7601d95d2f Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Tue, 14 Jul 2026 00:05:02 -0400 Subject: [PATCH 05/20] test(e2e): ride the shared data plane in the cert tests The gateway-accepted and gateway-invalid-certificate scenarios self- provisioned a per-test EnvoyProxy + GatewayClass in envoy-gateway-system, the single-stack bridge topology the prod-fidelity env retires. That namespace and controller don't exist on the shared env, so both timed out. Point them at the env's shared GatewayClass and assert the cert-health behavior (#212) at the gateway-status level, where the upstream controller expresses it: bad listener withheld from the edge, good programmed, plain-language reason surfaced, recovery propagated. Drop the bespoke proxy and the backend-re-encrypt wire checks; shared-plane serving is already covered by the WAF enforce test. Key changes: - remove self-provisioned EnvoyProxy/GatewayClass; use the env class - keep the full #212 status arc, incl. recovery - drop the envoy-gateway-system connectivity probes --- .../chainsaw-test.yaml | 171 +--------------- test/e2e/gateway/chainsaw-test.yaml | 183 ------------------ 2 files changed, 5 insertions(+), 349 deletions(-) diff --git a/test/e2e/gateway-invalid-certificate/chainsaw-test.yaml b/test/e2e/gateway-invalid-certificate/chainsaw-test.yaml index 5ab2e026..4ac1016e 100644 --- a/test/e2e/gateway-invalid-certificate/chainsaw-test.yaml +++ b/test/e2e/gateway-invalid-certificate/chainsaw-test.yaml @@ -4,12 +4,10 @@ # the edge, and the customer must be told why. One bad certificate must never # take HTTPS down for the other hostnames that share the gateway. # -# The test follows the whole story: -# - the bad hostname is held back while the good one keeps working; -# - the customer sees a plain-language reason on the bad hostname; -# - a real HTTPS request to the good hostname returns 200, and the bad one is -# not served; -# - once the certificate can be issued, the bad hostname recovers and serves. +# The test follows the whole story at the gateway-status level: +# - the bad listener is held back from the edge while the good one is programmed; +# - the customer sees a plain-language reason on the bad listener; +# - once the certificate can be issued, the bad listener is programmed at the edge. # # To make the failure happen reliably, the bad hostname points at a certificate # issuer that doesn't exist yet, so its certificate is never issued. Expired and @@ -103,52 +101,6 @@ spec: spec: controllerName: gateway.networking.datumapis.com/external-global-proxy-controller - # Shared cluster-scoped downstream config โ€” applied (not created) so this - # test coexists with gateway-accepted regardless of run order. - - apply: - cluster: nso-infra - resource: - apiVersion: gateway.envoyproxy.io/v1alpha1 - kind: EnvoyProxy - metadata: - name: custom-proxy-config - namespace: envoy-gateway-system - spec: - provider: - type: Kubernetes - kubernetes: - envoyService: - type: ClusterIP - patch: - type: StrategicMerge - value: - spec: - ipFamilyPolicy: RequireDualStack - mergeGateways: true - - - apply: - cluster: nso-infra - resource: - apiVersion: gateway.networking.k8s.io/v1 - kind: GatewayClass - metadata: - name: datum-downstream-gateway-e2e - spec: - controllerName: gateway.envoyproxy.io/gatewayclass-controller - parametersRef: - group: gateway.envoyproxy.io - kind: EnvoyProxy - name: custom-proxy-config - namespace: envoy-gateway-system - - - apply: - cluster: nso-infra - resource: - apiVersion: v1 - kind: Namespace - metadata: - name: datum-downstream-gateway-hostnames - - name: Create a backend to route to try: # Plain HTTP backend on the downstream cluster. The gateway terminates @@ -410,7 +362,7 @@ spec: kubectl -n $DOWNSTREAM_NAMESPACE get httproute -o yaml kubectl -n $DOWNSTREAM_NAMESPACE get certificates -o yaml - - name: Confirm Envoy programmed and serves the good listener + - name: Confirm the edge programmed the good listener try: - script: cluster: nso-standard @@ -455,92 +407,6 @@ spec: - status: "True" (conditions[?type == 'ResolvedRefs']): - status: "True" - - - assert: - cluster: nso-infra - resource: - apiVersion: v1 - kind: Pod - metadata: - name: backend-pod - namespace: default - status: - phase: Running - - # A long-lived client pod with curl preinstalled. - - create: - cluster: nso-infra - resource: - apiVersion: v1 - kind: Pod - metadata: - name: conntest - namespace: default - spec: - containers: - - name: curl - image: curlimages/curl:8.11.1 - command: ["sleep", "infinity"] - resources: - requests: - cpu: 50m - memory: 64Mi - limits: - cpu: 100m - memory: 128Mi - terminationGracePeriodSeconds: 0 - - - assert: - cluster: nso-infra - resource: - apiVersion: v1 - kind: Pod - metadata: - name: conntest - namespace: default - status: - phase: Running - - # A real HTTPS request to the good hostname must return 200, and the bad - # hostname must not be served. Retries give the edge a moment to pick up - # the configuration. - # Explicit timeout: the chainsaw default for script steps is 5s, too short - # for the retry loop below. - - script: - timeout: 240s - cluster: nso-infra - content: | - set -eu - IP=$(kubectl get svc -n envoy-gateway-system \ - -l gateway.envoyproxy.io/owning-gatewayclass=datum-downstream-gateway-e2e \ - -o jsonpath='{.items[0].spec.clusterIP}') - echo "gateway service clusterIP: ${IP}" - - echo "--- good hostname must serve 200 ---" - ok="" - for i in $(seq 1 24); do - code=$(kubectl -n default exec conntest -- \ - curl -ksS -o /dev/null -w '%{http_code}' --max-time 10 \ - --resolve good.e2e.env.datum.net:443:"${IP}" \ - https://good.e2e.env.datum.net/status/200 || true) - echo "attempt ${i}: good -> ${code}" - if [ "${code}" = "200" ]; then ok="yes"; break; fi - sleep 5 - done - if [ -z "${ok}" ]; then - echo "ERROR: good hostname never returned 200"; exit 1 - fi - - echo "--- bad hostname must not serve (no matching listener) ---" - code=$(kubectl -n default exec conntest -- \ - curl -ksS -o /dev/null -w '%{http_code}' --max-time 10 \ - --resolve bad.e2e.env.datum.net:443:"${IP}" \ - https://bad.e2e.env.datum.net/status/200 || true) - echo "bad -> ${code}" - if [ "${code}" = "200" ]; then - echo "ERROR: bad hostname was served despite an invalid certificate"; exit 1 - fi - echo "OK: good served, bad not served" catch: - script: cluster: nso-infra @@ -550,8 +416,6 @@ spec: content: | kubectl -n $DOWNSTREAM_NAMESPACE get gateway test-gateway -o yaml kubectl -n $DOWNSTREAM_NAMESPACE get httproute -o yaml - kubectl get svc -n envoy-gateway-system -o wide - kubectl -n envoy-gateway-system logs -l gateway.envoyproxy.io/owning-gatewayclass=datum-downstream-gateway-e2e -c envoy --tail=100 - name: Recover the bad listener by creating its issuer try: @@ -617,31 +481,6 @@ spec: - status: "True" (conditions[?type == 'ResolvedRefs']): - status: "True" - - # The recovered hostname now actually serves 200. - - script: - timeout: 240s - cluster: nso-infra - content: | - set -eu - IP=$(kubectl get svc -n envoy-gateway-system \ - -l gateway.envoyproxy.io/owning-gatewayclass=datum-downstream-gateway-e2e \ - -o jsonpath='{.items[0].spec.clusterIP}') - echo "--- recovered hostname must now serve 200 ---" - ok="" - for i in $(seq 1 24); do - code=$(kubectl -n default exec conntest -- \ - curl -ksS -o /dev/null -w '%{http_code}' --max-time 10 \ - --resolve bad.e2e.env.datum.net:443:"${IP}" \ - https://bad.e2e.env.datum.net/status/200 || true) - echo "attempt ${i}: recovered bad -> ${code}" - if [ "${code}" = "200" ]; then ok="yes"; break; fi - sleep 5 - done - if [ -z "${ok}" ]; then - echo "ERROR: recovered hostname never returned 200"; exit 1 - fi - echo "OK: recovered hostname serves 200" catch: - script: cluster: nso-infra diff --git a/test/e2e/gateway/chainsaw-test.yaml b/test/e2e/gateway/chainsaw-test.yaml index 184fff9c..1253d66d 100644 --- a/test/e2e/gateway/chainsaw-test.yaml +++ b/test/e2e/gateway/chainsaw-test.yaml @@ -85,18 +85,6 @@ spec: - type: Ready status: "True" - - script: - # skipCommandOutput: true - # skipLogOutput: true - cluster: nso-infra - env: - - name: CLUSTER_ISSUER_NAME - value: ($clusterIssuerName) - content: | - kubectl get secret -n cert-manager $CLUSTER_ISSUER_NAME -o yaml | \ - sed 's/namespace: .*/namespace: envoy-gateway-system/' | \ - kubectl apply -n envoy-gateway-system -f - - - name: Create GatewayClass for the upstream gateways try: - create: @@ -109,73 +97,6 @@ spec: spec: controllerName: gateway.networking.datumapis.com/external-global-proxy-controller - - name: Create GatewayClass for the downstream gateways - try: - - create: - cluster: nso-infra - resource: - apiVersion: gateway.envoyproxy.io/v1alpha1 - kind: EnvoyProxy - metadata: - name: custom-proxy-config - namespace: envoy-gateway-system - spec: - provider: - type: Kubernetes - kubernetes: - envoyService: - type: ClusterIP - patch: - type: StrategicMerge - value: - spec: - ipFamilyPolicy: RequireDualStack - envoyDeployment: - patch: - type: StrategicMerge - value: - spec: - template: - spec: - containers: - - name: envoy - volumeMounts: - - mountPath: /etc/ssl/certs/ - name: selfsigned-ca - readOnly: true - volumes: - - name: selfsigned-ca - secret: - secretName: ($clusterIssuerName) - items: - - key: ca.crt - path: ca-certificates.crt - mergeGateways: true - - - create: - cluster: nso-infra - resource: - apiVersion: gateway.networking.k8s.io/v1 - kind: GatewayClass - metadata: - name: datum-downstream-gateway-e2e - spec: - controllerName: gateway.envoyproxy.io/gatewayclass-controller - parametersRef: - group: gateway.envoyproxy.io - kind: EnvoyProxy - name: custom-proxy-config - namespace: envoy-gateway-system - - - description: Create namespace for hostname tracking - apply: - cluster: nso-infra - resource: - apiVersion: v1 - kind: Namespace - metadata: - name: datum-downstream-gateway-hostnames - - name: Create pod and service to serve as a backend for the HTTPRoute try: - create: @@ -792,107 +713,3 @@ spec: kubectl get endpointslices -n $NAMESPACE -o yaml kubectl get dnsendpoints -n $NAMESPACE -o yaml kubectl get svc -n $NAMESPACE -o yaml - - - name: Provision Pod to test connectivity - try: - - script: - skipCommandOutput: true - skipLogOutput: true - cluster: nso-standard - content: | - kubectl -n $NAMESPACE get gateway test-gateway -o json - outputs: - - name: upstreamGateway - value: (json_parse($stdout)) - # Re-derive in this step's scope; chainsaw bindings don't carry - # across steps, and the operator-published canonical hostname is - # word-derived from the gateway UID and can't be reconstructed - # inline. - - name: primaryHostname - value: ($upstreamGateway.status.addresses[?type == 'Hostname'].value | [0]) - - - script: - skipCommandOutput: true - skipLogOutput: true - cluster: nso-infra - content: | - kubectl get svc -n envoy-gateway-system -l gateway.envoyproxy.io/owning-gatewayclass=datum-downstream-gateway-e2e -o json - outputs: - - name: gatewayService - value: (json_parse($stdout).items | [0]) - - - create: - cluster: nso-infra - resource: - apiVersion: v1 - kind: Pod - metadata: - name: test-pod - namespace: default - spec: - containers: - - name: connectivity-test - image: alpine:3.21.3 - command: ["/bin/sh", "-c", "sleep infinity"] - resources: - requests: - cpu: 100m - memory: 128Mi - limits: - cpu: 100m - memory: 1Gi - env: - - name: GATEWAY_SERVICE_NAME - value: ($gatewayService.metadata.name) - - name: GATEWAY_SERVICE_NAMESPACE - value: ($gatewayService.metadata.namespace) - - name: PRIMARY_HOSTNAME - value: ($primaryHostname) - lifecycle: - postStart: - exec: - command: ["apk", "add", "curl"] - terminationGracePeriodSeconds: 0 - - - assert: - cluster: nso-infra - resource: - apiVersion: v1 - kind: Pod - metadata: - name: backend-pod - namespace: default - status: - phase: Running - - - assert: - cluster: nso-infra - resource: - apiVersion: v1 - kind: Pod - metadata: - name: test-pod - namespace: default - status: - phase: Running - - # Sleep for a while to ensure that the Envoy replicas have received the - # latest xDS config. We should remove this once we find a reliable method - # to determine if the Envoy replicas have the latest xDS config. - - sleep: - duration: 10s - - - script: - cluster: nso-infra - content: | - kubectl -n default exec -i test-pod -- sh -c " \ - set -x; \ - curl -kvf -H \"Host: \${PRIMARY_HOSTNAME}\" http://\${GATEWAY_SERVICE_NAME}.\${GATEWAY_SERVICE_NAMESPACE}.svc.cluster.local/plaintext/status/200; \ - curl -kvf -H \"Host: \${PRIMARY_HOSTNAME}\" http://\${GATEWAY_SERVICE_NAME}.\${GATEWAY_SERVICE_NAMESPACE}.svc.cluster.local/tls/status/200; \ - " - catch: - - script: - cluster: nso-infra - content: | - kubectl -n envoy-gateway-system logs -l gateway.envoyproxy.io/owning-gatewayclass=datum-downstream-gateway-e2e -c envoy - kubectl -n envoy-gateway-system logs -l app.kubernetes.io/instance=envoy-gateway From 2565543ad364a2426ca18dc8fdd7e816d78afc7e Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Tue, 14 Jul 2026 00:05:21 -0400 Subject: [PATCH 06/20] test(infra): persist a JSON run report from test-e2e chainsaw writes no report by default, so a hung or killed run left no trail on disk. Emit a JSON report (path/name overridable) so a stall is diagnosable after the fact instead of only in lost terminal scrollback. --- Taskfile.test-infra.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Taskfile.test-infra.yml b/Taskfile.test-infra.yml index 34de268b..d6f3a260 100644 --- a/Taskfile.test-infra.yml +++ b/Taskfile.test-infra.yml @@ -384,14 +384,21 @@ tasks: # CLI_ARGS (after --) wins so a single test can be selected; else the whole # suite directory. TARGET: '{{.CLI_ARGS | default "./test/e2e"}}' + # Persist a machine-readable run report so a hang or kill still leaves a + # forensic trail on disk (chainsaw writes nothing by default). + REPORT_PATH: '{{.REPORT_PATH | default .TMP_DIR}}' + REPORT_NAME: '{{.REPORT_NAME | default "chainsaw-report"}}' deps: - kubeconfigs cmds: - - echo "๐Ÿงช chainsaw suite {{.TARGET}} (nso-standard={{.UPSTREAM_CLUSTER}}, nso-infra={{.DOWNSTREAM_CLUSTER}})" + - echo "๐Ÿงช chainsaw suite {{.TARGET}} (nso-standard={{.UPSTREAM_CLUSTER}}, nso-infra={{.DOWNSTREAM_CLUSTER}}); report {{.REPORT_PATH}}/{{.REPORT_NAME}}.json" - | {{.CHAINSAW}} test {{.TARGET}} \ --cluster nso-standard={{.TMP_DIR}}/.kind-{{.UPSTREAM_CLUSTER}}.yaml \ - --cluster nso-infra={{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}.yaml + --cluster nso-infra={{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}.yaml \ + --report-format JSON \ + --report-path {{.REPORT_PATH}} \ + --report-name {{.REPORT_NAME}} kubeconfigs: desc: "Export per-cluster kubeconfigs to TMPDIR for chainsaw." From 4466cf501ba4f2da5a8235324f3fd441ba10054f Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Tue, 14 Jul 2026 00:12:57 -0400 Subject: [PATCH 07/20] test(infra): keep env image placeholders in the suite fold The suite commits carried a fixed image tag from the standalone branch; the env sets the operator tag at deploy time (nso-image / extension-server), so restore the placeholder tags to avoid committing a stale build reference. --- config/e2e-downstream/kustomization.yaml | 2 +- config/manager/kustomization.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/e2e-downstream/kustomization.yaml b/config/e2e-downstream/kustomization.yaml index 39cbe731..6ab183ae 100644 --- a/config/e2e-downstream/kustomization.yaml +++ b/config/e2e-downstream/kustomization.yaml @@ -22,7 +22,7 @@ resources: images: - name: ghcr.io/datum-cloud/network-services-operator newName: ghcr.io/datum-cloud/network-services-operator - newTag: d2e62bf + newTag: 63fa912 # The default e2e path has no Prometheus operator; drop the ServiceMonitor so # the apply doesn't fail on the missing monitoring.coreos.com CRD. (Re-add via diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 8f2fd385..ff5b1ecf 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -6,7 +6,7 @@ resources: images: - name: ghcr.io/datum-cloud/network-services-operator newName: ghcr.io/datum-cloud/network-services-operator - newTag: d2e62bf + newTag: latest configMapGenerator: - files: - config.yaml From 4aecfe027d973912a3a7e843bfc12572fe6815c9 Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Tue, 14 Jul 2026 00:33:56 -0400 Subject: [PATCH 08/20] test(e2e): adapt the WAF attack/neutralize tests to the env Two main-carried WAF scenarios failed on the two-cluster env: - enforce-blocks-attack and neutralize-inverted hardcoded the bare datum-downstream-gateway class; the env installs datum-downstream-gateway-e2e, so waf-gw never programmed. - neutralize-inverted stripped the paranoia CEL off the CRD to inject a grandfathered inverted policy, but it runs against the downstream cluster whose replicated CRD carries no such validation, so the remove patch failed on a missing path. Guard the strip/restore on the CEL actually being present; when absent the inverted policy applies directly. --- .../chainsaw-test.yaml | 6 +++--- .../chainsaw-test.yaml | 18 ++++++++++-------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/test/e2e/trafficprotectionpolicy-enforce-attack/chainsaw-test.yaml b/test/e2e/trafficprotectionpolicy-enforce-attack/chainsaw-test.yaml index 9e6dca69..df627fa3 100644 --- a/test/e2e/trafficprotectionpolicy-enforce-attack/chainsaw-test.yaml +++ b/test/e2e/trafficprotectionpolicy-enforce-attack/chainsaw-test.yaml @@ -17,7 +17,7 @@ metadata: # # Precondition: the downstream (nso-infra) must have the WAF data plane wired up # (extension-server + the extensionManager Envoy Gateway registered on the -# `datum-downstream-gateway` GatewayClass, with the Coraza filter). +# `datum-downstream-gateway-e2e` GatewayClass, with the Coraza filter). # `make prepare-infra-cluster` installs this via `make downstream-waf-dataplane`. spec: cluster: nso-infra @@ -84,7 +84,7 @@ spec: metadata: name: waf-gw spec: - gatewayClassName: datum-downstream-gateway + gatewayClassName: datum-downstream-gateway-e2e listeners: - name: http protocol: HTTP @@ -144,7 +144,7 @@ spec: set -x kubectl get gateway -A -o yaml kubectl describe gateway -A - kubectl get gatewayclass datum-downstream-gateway -o yaml + kubectl get gatewayclass datum-downstream-gateway-e2e -o yaml kubectl -n datum-downstream-gateway get pods -o wide kubectl -n datum-downstream-gateway describe pods kubectl -n datum-downstream-gateway logs deploy/envoy-gateway --tail=-1 diff --git a/test/e2e/trafficprotectionpolicy-neutralize-inverted/chainsaw-test.yaml b/test/e2e/trafficprotectionpolicy-neutralize-inverted/chainsaw-test.yaml index 5482d51f..930a1387 100644 --- a/test/e2e/trafficprotectionpolicy-neutralize-inverted/chainsaw-test.yaml +++ b/test/e2e/trafficprotectionpolicy-neutralize-inverted/chainsaw-test.yaml @@ -94,7 +94,7 @@ spec: metadata: name: waf-gw spec: - gatewayClassName: datum-downstream-gateway + gatewayClassName: datum-downstream-gateway-e2e listeners: - name: http protocol: HTTP @@ -147,13 +147,15 @@ spec: crd=trafficprotectionpolicies.networking.datumapis.com path=/spec/versions/0/schema/openAPIV3Schema/properties/spec/properties/ruleSets/items/properties/owaspCoreRuleSet/properties/paranoiaLevels/x-kubernetes-validations - kubectl patch crd "${crd}" --type=json \ - -p "[{\"op\":\"remove\",\"path\":\"${path}\"}]" - - restore() { - kubectl patch crd "${crd}" --type=json -p "[{\"op\":\"add\",\"path\":\"${path}\",\"value\":[{\"message\":\"detection paranoia level must be greater than or equal to blocking paranoia level\",\"rule\":\"self.detection >= self.blocking\"}]}]" - } - trap restore EXIT + cel=".spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.ruleSets.items.properties.owaspCoreRuleSet.properties.paranoiaLevels.x-kubernetes-validations" + if kubectl get crd "${crd}" -o jsonpath="{${cel}}" | grep -q .; then + kubectl patch crd "${crd}" --type=json \ + -p "[{\"op\":\"remove\",\"path\":\"${path}\"}]" + restore() { + kubectl patch crd "${crd}" --type=json -p "[{\"op\":\"add\",\"path\":\"${path}\",\"value\":[{\"message\":\"detection paranoia level must be greater than or equal to blocking paranoia level\",\"rule\":\"self.detection >= self.blocking\"}]}]" + } + trap restore EXIT + fi created=false for i in $(seq 1 20); do From e2c129636ecd3993a9162307d57b863d5c46b4b1 Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Tue, 14 Jul 2026 12:14:34 -0400 Subject: [PATCH 09/20] ci: run e2e against the prod-fidelity env, retire the bridge The E2E Tests workflow stood up a single-stack bridge (nso-standard / nso-infra via make kind-*-cluster + prepare-*), which diverged from how the edge runs in production and from the two-cluster prod-fidelity env. Point CI at that env instead: install task + helm + the pinned tools, then `task test-infra:up` and `task test-infra:test-e2e`. Retire the now-unused bridge: the dev Taskfile, its Makefile targets (test-e2e, prepare-e2e, prepare-infra-cluster, downstream-crds, downstream-waf-dataplane, load-image-*, kind-*-cluster, deploy-e2e, envoy-gateway, external-dns), the bridge kind cluster configs, and the orphaned config/e2e-downstream manifests. Key changes: - rewrite test-e2e.yml to bring up and run the test-infra env - bump pinned kind to v0.32.0 to match the env's node image - delete Taskfile.dev.yaml and drop its include - remove bridge Makefile targets and config/tools/kind/{standard,infra} - delete config/e2e-downstream/{pki,waf-gateway,extension-server-config} --- .github/workflows/test-e2e.yml | 80 +++++--------- Makefile | 91 +--------------- Taskfile.dev.yaml | 103 ------------------ Taskfile.yaml | 2 - .../extension-server-config.yaml | 12 -- config/e2e-downstream/pki.yaml | 31 ------ config/e2e-downstream/waf-gateway.yaml | 59 ---------- config/tools/kind/infra-cluster.yaml | 14 --- config/tools/kind/standard-cluster.yaml | 7 -- 9 files changed, 30 insertions(+), 369 deletions(-) delete mode 100644 Taskfile.dev.yaml delete mode 100644 config/e2e-downstream/extension-server-config.yaml delete mode 100644 config/e2e-downstream/pki.yaml delete mode 100644 config/e2e-downstream/waf-gateway.yaml delete mode 100644 config/tools/kind/infra-cluster.yaml delete mode 100644 config/tools/kind/standard-cluster.yaml diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index badc3109..7bbe1200 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -23,70 +23,42 @@ jobs: with: go-version: '~1.26' - - name: Verify kind installation - run: kind version - - - name: Create upstream kind cluster - run: make kind-standard-cluster - - - name: Create downstream kind cluster - run: | - make kind-infra-cluster - kubectl config use-context kind-nso-standard - - - name: Prepare e2e - run: | - kubectl config use-context kind-nso-standard - make prepare-e2e - - kubectl config use-context kind-nso-infra - make prepare-infra-cluster + - name: Install Task + uses: arduino/setup-task@v2 + with: + version: 3.x + repo-token: ${{ secrets.GITHUB_TOKEN }} - kind get kubeconfig --name nso-infra --internal > ${TMPDIR}/.kind-nso-infra-internal.yaml + - name: Install Helm + uses: azure/setup-helm@v4 - kubectl config use-context kind-nso-standard - kubectl create secret -n network-services-operator-system \ - generic downstream-cluster-kubeconfig \ - --from-file=kubeconfig=${TMPDIR}/.kind-nso-infra-internal.yaml + - name: Install pinned tools + run: make kind kustomize cmctl chainsaw - # Wait for the operator to be ready - kubectl -n network-services-operator-system \ - wait deploy network-services-operator-controller-manager \ - --for=condition=Available \ - --timeout=120s || \ - kubectl -n cert-manager describe pods && \ - kubectl logs -n cert-manager -l app.kubernetes.io/name=cert-manager --tail=-1 && \ - kubectl -n network-services-operator-system describe issuer && \ - kubectl -n network-services-operator-system describe certificates && \ - kubectl -n network-services-operator-system describe secrets && \ - kubectl -n network-services-operator-system describe pod && \ - kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=-1 + - name: Bring up the prod-fidelity env + run: task test-infra:up - - name: Running Test e2e - run: make test-e2e + - name: Run e2e suite + run: task test-infra:test-e2e - - name: Dump downstream WAF data plane on failure + - name: Dump the downstream data plane on failure if: failure() run: | - kubectl config use-context kind-nso-infra + set +e + KC=${TMPDIR}/.kind-nso-downstream.yaml + bin/kind-v0.32.0 get kubeconfig --name nso-downstream > "${KC}" + export KUBECONFIG="${KC}" echo "::group::GatewayClasses" - kubectl get gatewayclass -o wide || true - kubectl describe gatewayclass datum-downstream-gateway || true + kubectl get gatewayclass -o wide echo "::endgroup::" - echo "::group::EnvoyProxy" - kubectl -n datum-downstream-gateway get envoyproxy -o yaml || true + echo "::group::Gateways (all namespaces)" + kubectl get gateway -A -o wide echo "::endgroup::" - echo "::group::datum-downstream-gateway namespace + pods" - kubectl get ns datum-downstream-gateway --show-labels || true - kubectl -n datum-downstream-gateway get pods -o wide || true - echo "::endgroup::" - echo "::group::Downstream Envoy Gateway logs" - kubectl -n datum-downstream-gateway logs deploy/envoy-gateway --tail=200 || true + echo "::group::datum-downstream-gateway pods" + kubectl -n datum-downstream-gateway get pods -o wide + kubectl -n datum-downstream-gateway logs deploy/envoy-gateway --tail=200 echo "::endgroup::" echo "::group::Extension server" - kubectl -n datum-downstream-gateway describe deploy envoy-gateway-extension-server || true - kubectl -n datum-downstream-gateway logs -l app.kubernetes.io/component=envoy-gateway-extension-server --tail=200 || true - echo "::endgroup::" - echo "::group::Certificates" - kubectl -n datum-downstream-gateway get certificate,secret || true + kubectl -n network-services-operator-system get pods -o wide + kubectl -n network-services-operator-system logs -l app.kubernetes.io/component=envoy-gateway-extension-server --tail=200 echo "::endgroup::" diff --git a/Makefile b/Makefile index 5d9a38b6..714b033d 100644 --- a/Makefile +++ b/Makefile @@ -64,27 +64,9 @@ vet: ## Run go vet against code. test: manifests generate fmt vet envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out -# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'. -# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. -# Prometheus and CertManager are installed by default; skip with: -# - PROMETHEUS_INSTALL_SKIP=true -# - CERT_MANAGER_INSTALL_SKIP=true -.PHONY: test-e2e -test-e2e: chainsaw - @command -v kind >/dev/null 2>&1 || { \ - echo "Kind is not installed. Please install Kind manually."; \ - exit 1; \ - } - @kind get clusters | grep -q 'nso-standard' || { \ - echo "No Kind cluster is running. Please start a Kind cluster before running the e2e tests."; \ - exit 1; \ - } - $(KIND) get kubeconfig --name nso-standard > $(TMPDIR)/.kind-nso-standard.yaml - $(KIND) get kubeconfig --name nso-infra > $(TMPDIR)/.kind-nso-infra.yaml - $(CHAINSAW) test $(or $(TEST_DIR),./test/e2e) \ - --parallel 1 \ - --cluster nso-standard=$(TMPDIR)/.kind-nso-standard.yaml \ - --cluster nso-infra=$(TMPDIR)/.kind-nso-infra.yaml +# The e2e suite runs against the two-cluster prod-fidelity env; bring it up and +# run it with `task test-infra:up` then `task test-infra:test-e2e` +# (Taskfile.test-infra.yml). GATEWAY_CONFORMANCE_CLASS ?= gateway-conformance GATEWAY_CONFORMANCE_FLAGS ?= @@ -153,76 +135,14 @@ build-installer: set-image-controller generate ## Generate a consolidated YAML w set-image-controller: manifests kustomize cd config/manager && $(KUSTOMIZE) edit set image ghcr.io/datum-cloud/network-services-operator=${IMG} -.PHONY: prepare-infra-cluster -prepare-infra-cluster: cert-manager envoy-gateway external-dns downstream-crds downstream-waf-dataplane billing-usage-collector load-image-infra extension-server configure-eg-extension-manager - -.PHONY: downstream-crds -downstream-crds: ## Install NSO CRDs on the downstream (infra) cluster that the replicator mirrors into it. - $(KUBECTL) apply -f config/crd/bases/networking.datumapis.com_connectors.yaml - $(KUBECTL) apply -f config/crd/bases/networking.datumapis.com_httpproxies.yaml - $(KUBECTL) apply -f config/crd/bases/networking.datumapis.com_trafficprotectionpolicies.yaml - -.PHONY: downstream-waf-dataplane -downstream-waf-dataplane: kustomize load-image-infra ## Install the Coraza WAF data plane (downstream EG instance + extension server + datum-downstream-gateway GatewayClass) on the infra cluster. - $(KUSTOMIZE) build --enable-helm config/tools/envoy-gateway-downstream | kubectl apply --server-side=true --force-conflicts -f - - $(KUSTOMIZE) build config/e2e-downstream | kubectl apply --server-side=true --force-conflicts -f - - -.PHONY: prepare-e2e -prepare-e2e: chainsaw set-image-controller cert-manager load-image-all deploy-e2e - .PHONY: prepare-dev prepare-dev: chainsaw set-image-controller cert-manager install -.PHONY: load-image-all -load-image-all: load-image-operator load-image-infra - -.PHONY: load-image-operator -load-image-operator: docker-build kind - $(KIND) load docker-image $(IMG) -n nso-standard - -.PHONY: load-image-infra -load-image-infra: docker-build kind ## Load operator image into nso-infra kind cluster (needed by the extension server). - $(KIND) load docker-image $(IMG) -n nso-infra - .PHONY: cert-manager cert-manager: cmctl $(KUSTOMIZE) build --enable-helm config/tools/cert-manager | kubectl apply --server-side=true --force-conflicts -f - $(CMCTL) check api --wait=5m -.PHONY: envoy-gateway -envoy-gateway: - $(KUSTOMIZE) build --enable-helm config/tools/envoy-gateway | kubectl apply --server-side=true --force-conflicts -f - - -.PHONY: external-dns -external-dns: - $(KUSTOMIZE) build --enable-helm config/tools/external-dns | kubectl apply --server-side=true --force-conflicts -f - - -.PHONY: billing-usage-collector -billing-usage-collector: - $(KUSTOMIZE) build --enable-helm config/tools/billing-usage-collector | kubectl apply --server-side=true --force-conflicts -f - - -.PHONY: extension-server -extension-server: ## Deploy the NSO extension server to the infra cluster (e2e overlay with cert-manager-issued TLS). - $(KUSTOMIZE) build --enable-helm config/extension-server-e2e | kubectl apply --server-side=true --force-conflicts -f - - kubectl rollout restart deployment/network-services-operator-envoy-gateway-extension-server \ - -n network-services-operator-system - kubectl rollout status deployment/network-services-operator-envoy-gateway-extension-server \ - -n network-services-operator-system --timeout=5m - -.PHONY: configure-eg-extension-manager -configure-eg-extension-manager: ## Patch the EG ConfigMap to enable extensionManager and restart the EG controller. - $(KUSTOMIZE) build --enable-helm config/tools/envoy-gateway/overlays/e2e | kubectl apply --server-side=true --force-conflicts -f - - kubectl rollout restart deployment/envoy-gateway -n envoy-gateway-system - kubectl rollout status deployment/envoy-gateway -n envoy-gateway-system --timeout=3m - -.PHONY: kind-standard-cluster -kind-standard-cluster: kind - $(KIND) create cluster --config=config/tools/kind/standard-cluster.yaml - -.PHONY: kind-infra-cluster -kind-infra-cluster: kind - $(KIND) create cluster --config=config/tools/kind/infra-cluster.yaml - ##@ Deployment ifndef ignore-not-found @@ -241,9 +161,6 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified deploy: set-image-controller ## Deploy controller to the K8s cluster specified in ~/.kube/config. $(KUSTOMIZE) build config/default | $(KUBECTL) apply -f - -.PHONY: deploy-e2e -deploy-e2e: set-image-controller - $(KUSTOMIZE) build config/e2e | $(KUBECTL) apply -f - .PHONY: undeploy undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. @@ -284,7 +201,7 @@ CERTMANAGER_VERSION ?= 1.17.1 CRDOC_VERSION ?= v0.6.4 # renovate: datasource=go depName=sigs.k8s.io/kind -KIND_VERSION ?= v0.27.0 +KIND_VERSION ?= v0.32.0 # renovate: datasource=go depName=github.com/kyverno/chainsaw CHAINSAW_VERSION ?= v0.2.15 diff --git a/Taskfile.dev.yaml b/Taskfile.dev.yaml deleted file mode 100644 index 257b05ec..00000000 --- a/Taskfile.dev.yaml +++ /dev/null @@ -1,103 +0,0 @@ -version: '3' - -vars: - TMP_DIR: - sh: echo "${TMPDIR:-/tmp}" - -tasks: - bootstrap: - desc: "Bootstrap the multi-cluster dev environment (nso-standard and nso-infra) for local testing" - cmds: - - echo "๐Ÿš€ Bootstrapping dev environment..." - - task: create-clusters - - task: prep-upstream - - task: prep-downstream - - task: link-clusters - - echo "๐ŸŽ‰ Dev environment bootstrapped successfully! Context is now kind-nso-standard." - - create-clusters: - desc: "Create Kind upstream (standard) and downstream (infra) clusters" - cmds: - - echo "๐Ÿงน Cleaning up any existing clusters..." - - kind delete cluster --name nso-standard || true - - kind delete cluster --name nso-infra || true - - echo "๐Ÿ—๏ธ Creating upstream (nso-standard) cluster..." - - make kind-standard-cluster - - echo "๐Ÿ—๏ธ Creating downstream (nso-infra) cluster..." - - make kind-infra-cluster - - prep-upstream: - desc: "Prepare the upstream cluster with Operator and cert-manager" - cmds: - - echo "๐Ÿ”ง Preparing upstream (nso-standard)..." - - kubectl config use-context kind-nso-standard - - make prepare-e2e - - prep-downstream: - desc: "Prepare the downstream cluster with cert-manager, envoy-gateway, and external-dns" - cmds: - - echo "๐Ÿ”ง Preparing downstream (nso-infra)..." - - kubectl config use-context kind-nso-infra - - make prepare-infra-cluster - - link-clusters: - desc: "Link upstream and downstream clusters using kubeconfig secret" - cmds: - - echo "๐Ÿ”— Linking clusters..." - - kind get kubeconfig --name nso-infra --internal > {{.TMP_DIR}}/.kind-nso-infra-internal.yaml - - kubectl config use-context kind-nso-standard - - | - kubectl create namespace network-services-operator-system --dry-run=client -o yaml | kubectl apply -f - - kubectl create secret -n network-services-operator-system \ - generic downstream-cluster-kubeconfig \ - --save-config \ - --dry-run=client -o yaml \ - --from-file=kubeconfig={{.TMP_DIR}}/.kind-nso-infra-internal.yaml | kubectl apply -f - - - echo "โณ Waiting for operator controller manager deployment to be ready..." - - | - kubectl -n network-services-operator-system \ - wait deploy network-services-operator-controller-manager \ - --for=condition=Available \ - --timeout=180s - - redeploy-operator: - desc: "Rebuild the operator image, load it into nso-standard, and roll out the deployed controller" - cmds: - - echo "๐Ÿ”จ Building operator image and loading it into nso-standard..." - # docker-build + kind load docker-image $(IMG) -n nso-standard - - make load-image-operator - - echo "โ™ป๏ธ Restarting the controller-manager to pick up the new image..." - - kubectl config use-context kind-nso-standard - - | - kubectl -n network-services-operator-system \ - rollout restart deploy network-services-operator-controller-manager - - echo "โณ Waiting for the new controller-manager rollout to complete..." - - | - kubectl -n network-services-operator-system \ - rollout status deploy network-services-operator-controller-manager \ - --timeout=180s - - echo "โœ… Operator redeployed with the freshly built image." - - destroy: - desc: "Tear down the multi-cluster dev environment" - cmds: - - echo "๐Ÿ’ฅ Destroying clusters..." - - kind delete cluster --name nso-standard || true - - kind delete cluster --name nso-infra || true - - rm -f {{.TMP_DIR}}/.kind-nso-infra-internal.yaml - - echo "โœจ Cleanup finished." - - test: - desc: "Run E2E tests using chainsaw on the local multi-cluster setup" - cmds: - - echo "๐Ÿงช Running E2E tests..." - - | - if [ -n "{{.CLI_ARGS}}" ]; then - if [[ "{{.CLI_ARGS}}" == test/e2e/* || "{{.CLI_ARGS}}" == ./test/e2e/* ]]; then - make test-e2e TEST_DIR="{{.CLI_ARGS}}" - else - make test-e2e TEST_DIR="./test/e2e/{{.CLI_ARGS}}" - fi - else - make test-e2e - fi diff --git a/Taskfile.yaml b/Taskfile.yaml index 890a88f7..ae4b2d6e 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -5,8 +5,6 @@ includes: docs: taskfile: ./docs/Taskfile.yaml dir: ./docs - dev: - taskfile: ./Taskfile.dev.yaml test-infra: taskfile: ./Taskfile.test-infra.yml diff --git a/config/e2e-downstream/extension-server-config.yaml b/config/e2e-downstream/extension-server-config.yaml deleted file mode 100644 index f4c00b6d..00000000 --- a/config/e2e-downstream/extension-server-config.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: apiserver.config.datumapis.com/v1alpha1 -kind: NetworkServicesOperator -gateway: - coraza: - routeBaseDirectives: - - Include @crs-setup-conf - - Include @recommended-conf - - SecResponseBodyLimitAction ProcessPartial - - SecResponseBodyLimit 32000 - errorPage: - enabled: true - bodyPath: /etc/datum/error-pages/error-5xx.html diff --git a/config/e2e-downstream/pki.yaml b/config/e2e-downstream/pki.yaml deleted file mode 100644 index 666bb0b3..00000000 --- a/config/e2e-downstream/pki.yaml +++ /dev/null @@ -1,31 +0,0 @@ ---- -apiVersion: cert-manager.io/v1 -kind: ClusterIssuer -metadata: - name: envoy-gateway-extension-server-selfsigned -spec: - selfSigned: {} ---- -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: envoy-gateway-extension-server-ca -spec: - isCA: true - commonName: envoy-gateway-extension-server-ca - secretName: envoy-gateway-extension-server-ca - privateKey: - algorithm: ECDSA - size: 256 - issuerRef: - name: envoy-gateway-extension-server-selfsigned - kind: ClusterIssuer - group: cert-manager.io ---- -apiVersion: cert-manager.io/v1 -kind: Issuer -metadata: - name: envoy-gateway-extension-server -spec: - ca: - secretName: envoy-gateway-extension-server-ca diff --git a/config/e2e-downstream/waf-gateway.yaml b/config/e2e-downstream/waf-gateway.yaml deleted file mode 100644 index ef6a6265..00000000 --- a/config/e2e-downstream/waf-gateway.yaml +++ /dev/null @@ -1,59 +0,0 @@ ---- -apiVersion: gateway.envoyproxy.io/v1alpha1 -kind: EnvoyProxy -metadata: - name: datum-downstream-gateway -spec: - mergeGateways: true - ipFamily: DualStack - provider: - type: Kubernetes - kubernetes: - envoyDeployment: - pod: - volumes: - - name: coraza-waf - emptyDir: {} - container: - image: envoyproxy/envoy:contrib-v1.35.0 - volumeMounts: - - name: coraza-waf - mountPath: /opt/coraza-waf - initContainers: - - name: coraza-waf - image: ghcr.io/datum-labs/coraza-envoy-go-filter/coraza-waf:v1.3.0-alpha10 - imagePullPolicy: IfNotPresent - command: - - cp - - /coraza-waf.so - - /opt/coraza-waf/ - volumeMounts: - - name: coraza-waf - mountPath: /opt/coraza-waf - envoyService: - externalTrafficPolicy: Cluster - type: NodePort - patch: - type: StrategicMerge - value: - spec: - ipFamilyPolicy: RequireDualStack - ports: - - name: http-80 - nodePort: 30080 - port: 80 - - name: https-443 - nodePort: 30443 - port: 443 ---- -apiVersion: gateway.networking.k8s.io/v1 -kind: GatewayClass -metadata: - name: datum-downstream-gateway -spec: - controllerName: gateway.envoyproxy.io/datum-downstream-gateway - parametersRef: - group: gateway.envoyproxy.io - kind: EnvoyProxy - name: datum-downstream-gateway - namespace: datum-downstream-gateway diff --git a/config/tools/kind/infra-cluster.yaml b/config/tools/kind/infra-cluster.yaml deleted file mode 100644 index 04531eb9..00000000 --- a/config/tools/kind/infra-cluster.yaml +++ /dev/null @@ -1,14 +0,0 @@ -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -name: nso-infra -networking: - ipFamily: dual -nodes: -- role: control-plane - extraPortMappings: - - containerPort: 30080 - hostPort: 30080 - protocol: TCP - - containerPort: 30443 - hostPort: 30443 - protocol: TCP diff --git a/config/tools/kind/standard-cluster.yaml b/config/tools/kind/standard-cluster.yaml deleted file mode 100644 index 47bf2f3e..00000000 --- a/config/tools/kind/standard-cluster.yaml +++ /dev/null @@ -1,7 +0,0 @@ -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -name: nso-standard -networking: - ipFamily: dual -nodes: -- role: control-plane From c2a2126cea9722b50981eabfe72a66bf3e27e23d Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Tue, 14 Jul 2026 12:14:40 -0400 Subject: [PATCH 10/20] docs: point e2e docs at the prod-fidelity env Update CLAUDE.md and the extension-server enhancement to describe the two-cluster test-infra env (nso-upstream / nso-downstream) and its task-based entrypoints, replacing the retired bridge references (make test-e2e, task dev:*, make prepare-infra-cluster). --- CLAUDE.md | 33 ++++++++++--------- .../envoy-gateway-extension-server/README.md | 9 ++--- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 7bfabd48..f46531e8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -63,23 +63,23 @@ Run `make help` for the full list. Most-used targets: ### Tests - `make test` โ€” unit/integration tests via **envtest** (excludes `test/e2e`); runs `manifests generate fmt vet` first, writes `cover.out` -- `make test-e2e` โ€” **Chainsaw** e2e suite in `test/e2e` against the two local - Kind clusters (requires `nso-standard` and `nso-infra` to exist) +- `task test-infra:test-e2e` โ€” **Chainsaw** e2e suite in `test/e2e` against the + prod-fidelity env (below); pass a scenario path after `--` for a single test - `make test-conformance` โ€” Gateway API conformance suite in `test/conformance/gatewayapi` -### Local Multi-Cluster Dev Environment (Taskfile) -The dev environment is two Kind clusters: an **upstream** cluster -(`nso-standard`) running the operator, and a **downstream** cluster -(`nso-infra`) running Envoy Gateway, cert-manager, and external-dns. - -- `task dev:bootstrap` โ€” full setup: create both clusters, deploy the operator - upstream, prepare downstream, and link them via a kubeconfig secret -- `task dev:redeploy-operator` โ€” rebuild the image, load it into `nso-standard`, - and roll the controller-manager -- `task dev:test [-- ]` โ€” run the e2e suite (optionally a single - test under `test/e2e/`) -- `task dev:destroy` โ€” tear down both clusters +### Prod-Fidelity Test Environment (Taskfile) +The e2e environment is two Kind clusters that mirror how the edge runs in +production: an **upstream** cluster (`nso-upstream`) running the operator, and a +**downstream** cluster (`nso-downstream`) running Envoy Gateway v1.7.4, the +extension server + Coraza WAF data plane, cert-manager, and external-dns. It is +defined in `Taskfile.test-infra.yml` and is what CI runs. + +- `task test-infra:up` โ€” build the operator image, create both clusters, install + the downstream data plane, deploy + link the operator +- `task test-infra:test-e2e [-- ]` โ€” run the e2e suite against the live env + (JSON report written to `$TMPDIR`); optionally a single scenario +- `task test-infra:down` โ€” tear down both clusters - `task validate-kustomizations` โ€” `kustomize build` every kustomization - `task test-prometheus-rules` โ€” `promtool` tests for the alerting rules in `test/prometheus-rules/` @@ -139,7 +139,7 @@ project follows the Kubebuilder v4 layout (`PROJECT`, domain `datumapis.com`). 2. Run `make manifests generate` to regenerate CRDs, RBAC, deepcopy, defaulters. 3. Run `make api-docs` if the CRD reference under `docs/api/` should update. 4. Add/extend reconciler logic in `internal/controller/` and a `_test.go`. -5. `make test` (envtest) and, when relevant, `make test-e2e`. +5. `make test` (envtest) and, when relevant, `task test-infra:test-e2e`. Never hand-edit generated files (`zz_generated.*`, `config/crd/bases/*`, `docs/api/*`) โ€” change the source and regenerate. @@ -154,7 +154,8 @@ Operator deployment config is composed from Kustomize bases and overlays under ### CI GitHub Actions enforce the same commands locally available: -`make test` (test.yml), `make test-e2e` (test-e2e.yml), `golangci-lint` +`make test` (test.yml), the `test-infra` e2e env (test-e2e.yml, via +`task test-infra:up` + `task test-infra:test-e2e`), `golangci-lint` (lint.yml), and kustomize validation (validate-kustomize.yaml). Run them locally before pushing. diff --git a/docs/enhancements/envoy-gateway-extension-server/README.md b/docs/enhancements/envoy-gateway-extension-server/README.md index 3957e68e..9f900035 100644 --- a/docs/enhancements/envoy-gateway-extension-server/README.md +++ b/docs/enhancements/envoy-gateway-extension-server/README.md @@ -358,9 +358,10 @@ The dedicated EG is configured with: - `extensionManager` baked into the helm values โ€” no runtime ConfigMap patch or EG rollout restart is required. -The local/CI e2e installs the dedicated EG via the second `helmCharts:` entry in -`config/tools/envoy-gateway/kustomization.yaml` (applied by `make prepare-infra-cluster`). -In production (`datum-cloud/infra`) it is a FluxCD `HelmRelease` with the same values. +The local/CI e2e installs the dedicated downstream EG via +`config/tools/envoy-gateway-downstream` (applied by the `test-infra:eg-downstream` +task during `task test-infra:up`). In production (`datum-cloud/infra`) it is a +FluxCD `HelmRelease` with the same values. #### extensionManager registration @@ -819,7 +820,7 @@ meaning of "parity" in the Transition Plan below. **3. Real-traffic end-to-end (two kind clusters, live requests).** The [Chainsaw e2e suite](https://github.com/datum-cloud/network-services-operator/blob/main/test/e2e/gateway/chainsaw-test.yaml) (run by -`make test-e2e`, gated in CI) stands up the full upstream/downstream split, +`task test-infra:test-e2e`, gated in CI) stands up the full upstream/downstream split, provisions Gateways and HTTPRoutes, and sends real HTTP through Envoy with `curl` โ€” confirming plaintext routing, TLS termination, and certificate issuance actually work, not just that the config looks right. The From ff1c2cf03ef451ef7e7a5238fbd00cfdbbbc5e79 Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Tue, 14 Jul 2026 12:23:27 -0400 Subject: [PATCH 11/20] ci: nudge e2e workflow run Empty commit to re-fire the pull_request synchronize event after the workflow-file change gated the prior run. From 14433d32090047ecae7fb854b8344b879f8a20d7 Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Tue, 14 Jul 2026 12:30:35 -0400 Subject: [PATCH 12/20] docs: guide e2e test authoring against the env Capture the conventions that the CI-on-env switch makes load-bearing: bind to the env's shared data plane (datum-downstream-gateway-e2e, not the bare bridge class or a self-provisioned envoy-gateway-system proxy), provision any scenario dependency in test-infra:up so it exists on the env, and push workflow edits over SSH (the gh HTTPS token lacks the workflow scope). --- CLAUDE.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index f46531e8..e9a8034d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -84,6 +84,20 @@ defined in `Taskfile.test-infra.yml` and is what CI runs. - `task test-prometheus-rules` โ€” `promtool` tests for the alerting rules in `test/prometheus-rules/` +#### Authoring e2e tests against the env +- Bind to the shared data plane the env stands up: downstream GatewayClass + `datum-downstream-gateway-e2e`, Envoy Gateway namespace + `datum-downstream-gateway`, extension server in + `network-services-operator-system`. Do **not** hardcode the bare + `datum-downstream-gateway` class or self-provision a per-test EnvoyProxy in + `envoy-gateway-system` โ€” that was the retired single-stack bridge's model. +- Every scenario under `test/e2e/` runs against this env in CI. Any setup a + scenario needs (a data-plane component, a namespace) must be installed by + `task test-infra:up` (add it to `Taskfile.test-infra.yml`), not assumed from + outside โ€” a scenario whose dependency isn't provisioned by the env will fail. +- Editing `.github/workflows/*` requires pushing over SSH; the HTTPS token used + by `gh` lacks the `workflow` scope. + ## High-Level Architecture ### Multi-Cluster (upstream / downstream) From 159ccb7b7d05ad7aeb9419be8ce0d337727e3499 Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Tue, 14 Jul 2026 13:29:56 -0400 Subject: [PATCH 13/20] test(infra): port the billing usage collector into the env CI now runs test/e2e (incl. test/e2e/billing) on the prod-fidelity env, but #239 wired the Vector collector into the retired bridge's prepare-infra-cluster. Install it from the env instead, and point its kubernetes_logs source at the env's Envoy namespace (datum-downstream-gateway, not the bridge's envoy-gateway-system). Also add workflow_dispatch to the e2e workflow so the run can be triggered directly instead of only via the gated pull_request event. --- .github/workflows/test-e2e.yml | 1 + Taskfile.test-infra.yml | 7 +++++++ config/tools/billing-usage-collector/kustomization.yaml | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 7bbe1200..962655bd 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -7,6 +7,7 @@ on: pull_request: branches: - main + workflow_dispatch: jobs: test-e2e: diff --git a/Taskfile.test-infra.yml b/Taskfile.test-infra.yml index d6f3a260..37d1932c 100644 --- a/Taskfile.test-infra.yml +++ b/Taskfile.test-infra.yml @@ -74,6 +74,7 @@ tasks: - task: prepare-upstream - task: eg-downstream - task: extension-server + - task: billing-usage-collector - task: external-dns - task: link-clusters - task: wait-ready @@ -168,6 +169,12 @@ tasks: # be in the manifest, not added afterward. - kubectl --context {{.DOWNSTREAM_CTX}} apply -f config/e2e-downstream/direct-namespace.yaml + billing-usage-collector: + desc: "Install the billing usage collector (Vector agent) that tails the downstream Envoy access logs and emits usage CloudEvents." + cmds: + - echo "๐Ÿ”ง installing billing-usage-collector (Vector agent, tailing datum-downstream-gateway)" + - '{{.KUSTOMIZE}} build --enable-helm config/tools/billing-usage-collector | kubectl --context {{.DOWNSTREAM_CTX}} apply --server-side=true --force-conflicts -f -' + eg-downstream: desc: "Install the dedicated downstream Envoy Gateway (v1.7.4) with the ext-server extensionManager wiring on the downstream cluster." cmds: diff --git a/config/tools/billing-usage-collector/kustomization.yaml b/config/tools/billing-usage-collector/kustomization.yaml index 42913678..7a3e7272 100644 --- a/config/tools/billing-usage-collector/kustomization.yaml +++ b/config/tools/billing-usage-collector/kustomization.yaml @@ -50,7 +50,7 @@ helmCharts: # Only tail the Envoy proxy pods; their access logs live on the # `envoy` container's stdout. Other namespaces/containers are # ignored so the agent does not parse unrelated cluster logs. - extra_field_selector: "metadata.namespace=envoy-gateway-system" + extra_field_selector: "metadata.namespace=datum-downstream-gateway" # Re-scan for new container log files every 2s instead of the 60s # default so the agent picks up freshly-created Envoy proxy pods # quickly -- the e2e test provisions a gateway and verifies within a From 4b0da7ace80eeb18fa7f898bc46ba7fa2d1cb0ef Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Tue, 14 Jul 2026 13:45:41 -0400 Subject: [PATCH 14/20] fix(test-infra): vendor the EG chart before eg-crds The gateway-helm chart is gitignored (config/**/charts), so on a fresh checkout (CI) the eg-crds task's static CRD path does not exist and test-infra:up fails. Build the kustomization with --enable-helm first so the chart is pulled, then apply the CRDs. --- Taskfile.test-infra.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Taskfile.test-infra.yml b/Taskfile.test-infra.yml index 37d1932c..3f708f9e 100644 --- a/Taskfile.test-infra.yml +++ b/Taskfile.test-infra.yml @@ -156,6 +156,10 @@ tasks: vars: EG_CRD_DIR: config/tools/envoy-gateway-downstream/charts/gateway-helm-{{.ENVOY_GATEWAY_VERSION}}/gateway-helm/crds cmds: + # The gateway-helm chart is gitignored (config/**/charts); vendor it first + # so the CRD files below exist on a fresh checkout (e.g. CI). Output is + # discarded โ€” this runs for the side effect of pulling the chart. + - '{{.KUSTOMIZE}} build --enable-helm config/tools/envoy-gateway-downstream >/dev/null' - kubectl --context {{.DOWNSTREAM_CTX}} apply --server-side=true --force-conflicts -f {{.EG_CRD_DIR}}/gatewayapi-crds.yaml - kubectl --context {{.DOWNSTREAM_CTX}} apply --server-side=true --force-conflicts -f {{.EG_CRD_DIR}}/generated/ From 662244f99c311a1c041eb17942c66ab205711320 Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Tue, 14 Jul 2026 13:45:41 -0400 Subject: [PATCH 15/20] test(e2e): run the billing metering test on the shared data plane MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The billing scenario self-provisioned its own EnvoyProxy (carrying the JSON access-log File sink the Vector pipeline tails) + GatewayClass in envoy-gateway-system โ€” the retired bridge model. Move the access-log sink onto the env's shared downstream EnvoyProxy (matching production, which emits it) and rework the test to bind to the env's shared class, dropping the self-provisioned proxy/class and the CA copy. Retarget the wire and log-scrape steps from envoy-gateway-system to the env's Envoy namespace, datum-downstream-gateway. --- config/e2e-downstream/envoyproxy.yaml | 20 +++++ test/e2e/billing/chainsaw-test.yaml | 114 ++------------------------ 2 files changed, 25 insertions(+), 109 deletions(-) diff --git a/config/e2e-downstream/envoyproxy.yaml b/config/e2e-downstream/envoyproxy.yaml index 283bd01b..60032293 100644 --- a/config/e2e-downstream/envoyproxy.yaml +++ b/config/e2e-downstream/envoyproxy.yaml @@ -70,6 +70,26 @@ spec: address: 0.0.0.0 port_value: 19000 telemetry: + accessLog: + settings: + - sinks: + - type: File + file: + path: /dev/stdout + format: + type: JSON + json: + start_time: "%START_TIME%" + method: "%REQ(:METHOD)%" + path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" + protocol: "%PROTOCOL%" + response_code: "%RESPONSE_CODE%" + bytes_received: "%BYTES_RECEIVED%" + bytes_sent: "%BYTES_SENT%" + duration: "%DURATION%" + route_name: "%ROUTE_NAME%" + project_name: "%METADATA(ROUTE:datum-gateway:project_name)%" + request_id: "%REQ(X-REQUEST-ID)%" metrics: prometheus: disable: false diff --git a/test/e2e/billing/chainsaw-test.yaml b/test/e2e/billing/chainsaw-test.yaml index b0c9918a..eb5dd80b 100644 --- a/test/e2e/billing/chainsaw-test.yaml +++ b/test/e2e/billing/chainsaw-test.yaml @@ -54,16 +54,6 @@ spec: ca: secretName: ($clusterIssuerName) - - script: - cluster: nso-infra - env: - - name: CLUSTER_ISSUER_NAME - value: ($clusterIssuerName) - content: | - kubectl get secret -n cert-manager $CLUSTER_ISSUER_NAME -o yaml | \ - sed 's/namespace: .*/namespace: envoy-gateway-system/' | \ - kubectl apply -n envoy-gateway-system -f - - - name: Deploy Mock Billing Gateway try: - create: @@ -142,100 +132,6 @@ spec: spec: controllerName: gateway.networking.datumapis.com/external-global-proxy-controller - # Ensure the downstream environment's standard gateway class and config exists - - apply: - cluster: nso-infra - resource: - apiVersion: gateway.envoyproxy.io/v1alpha1 - kind: EnvoyProxy - metadata: - name: custom-proxy-config - namespace: envoy-gateway-system - spec: - telemetry: - accessLog: - settings: - - sinks: - # File sink only -- identical to the production - # downstream gateway EnvoyProxy. Envoy writes the JSON - # access log to stdout; the node-local Vector agent - # tails the container log file (kubernetes_logs source) - # and emits the CloudEvents. This exercises the real, - # file-based production ingestion path -- no OTLP push. - # The test also scrapes the same stdout to derive the - # expected metric values (bytes, duration, project_name). - - type: File - file: - path: /dev/stdout - format: - type: JSON - json: - start_time: "%START_TIME%" - method: "%REQ(:METHOD)%" - path: "%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%" - protocol: "%PROTOCOL%" - response_code: "%RESPONSE_CODE%" - bytes_received: "%BYTES_RECEIVED%" - bytes_sent: "%BYTES_SENT%" - duration: "%DURATION%" - route_name: "%ROUTE_NAME%" - project_name: "%METADATA(ROUTE:datum-gateway:project_name)%" - request_id: "%REQ(X-REQUEST-ID)%" - provider: - type: Kubernetes - kubernetes: - envoyService: - type: ClusterIP - patch: - type: StrategicMerge - value: - spec: - ipFamilyPolicy: RequireDualStack - envoyDeployment: - patch: - type: StrategicMerge - value: - spec: - template: - spec: - containers: - - name: envoy - volumeMounts: - - mountPath: /etc/ssl/certs/ - name: selfsigned-ca - readOnly: true - volumes: - - name: selfsigned-ca - secret: - secretName: ($clusterIssuerName) - items: - - key: ca.crt - path: ca-certificates.crt - mergeGateways: true - - - apply: - cluster: nso-infra - resource: - apiVersion: gateway.networking.k8s.io/v1 - kind: GatewayClass - metadata: - name: datum-downstream-gateway-e2e - spec: - controllerName: gateway.envoyproxy.io/gatewayclass-controller - parametersRef: - group: gateway.envoyproxy.io - kind: EnvoyProxy - name: custom-proxy-config - namespace: envoy-gateway-system - - - apply: - cluster: nso-infra - resource: - apiVersion: v1 - kind: Namespace - metadata: - name: datum-downstream-gateway-hostnames - # Get downstream namespace name by retrieving standard namespace UID - script: cluster: nso-standard @@ -456,7 +352,7 @@ spec: content: | # Wait up to 10s for the envoy service to be created for i in $(seq 1 10); do - SVC=$(kubectl get svc -n envoy-gateway-system -l gateway.envoyproxy.io/owning-gatewayclass=datum-downstream-gateway-e2e -o json || true) + SVC=$(kubectl get svc -n datum-downstream-gateway -l gateway.envoyproxy.io/owning-gatewayclass=datum-downstream-gateway-e2e -o json || true) if [ ! -z "$SVC" ] && echo "$SVC" | grep -q "metadata"; then echo "$SVC" exit 0 @@ -485,7 +381,7 @@ spec: - name: GATEWAY_SERVICE_NAME value: ($gatewayService.metadata.name) - name: GATEWAY_SERVICE_NAMESPACE - value: envoy-gateway-system + value: datum-downstream-gateway lifecycle: postStart: exec: @@ -568,7 +464,7 @@ spec: # captured here (STEP 1-2) mock-billing-gateway (STEP 4) # # Envoy writes the access log to stdout (File sink on - # custom-proxy-config, path: /dev/stdout) when the request is + # the env's shared EnvoyProxy, path: /dev/stdout) when the request is # served; the node-local Vector agent tails the container log file # automatically -- the test does NOT forward it manually. This is # the same file-based ingestion path as the production gateway. @@ -596,7 +492,7 @@ spec: # --- STEP 1: capture the Envoy access log line for our HTTPRoute ----- # The route_name embeds the downstream namespace, so grepping for # 'httproute//' isolates this test's request. - ENVOY_LOG=$(kubectl -n envoy-gateway-system logs -l gateway.envoyproxy.io/owning-gatewayclass=datum-downstream-gateway-e2e -c envoy --tail=500 | grep "httproute/$DOWNSTREAM_NAMESPACE/" | tail -n 1 || true) + ENVOY_LOG=$(kubectl -n datum-downstream-gateway logs -l gateway.envoyproxy.io/owning-gatewayclass=datum-downstream-gateway-e2e -c envoy --tail=500 | grep "httproute/$DOWNSTREAM_NAMESPACE/" | tail -n 1 || true) if [ -z "$ENVOY_LOG" ]; then echo "ERROR: No Envoy access log found containing 'httproute/'" >&2 exit 1 @@ -631,7 +527,7 @@ spec: # --- STEP 3: (no manual forwarding) -------------------------------- # The request above already caused Envoy to write this access log to - # stdout via its File sink (custom-proxy-config, path: /dev/stdout). + # stdout via its File sink (env shared EnvoyProxy, path: /dev/stdout). # The node-local Vector agent tails the Envoy container log file via # its kubernetes_logs source, and Vector's VRL transform parses it # and emits the four CloudEvents to the mock-billing-gateway sink, so From 59aabf43cf661cb322252062b830ca90eb0380b5 Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Tue, 14 Jul 2026 14:09:47 -0400 Subject: [PATCH 16/20] ci: pin helm to v3.12.3 for kustomize helm builds kustomize v5.5.0 probes helm via `helm version -c`; the -c shorthand was removed in helm 3.13, so the latest helm from setup-helm broke every --enable-helm build in test-infra:up. Pin helm to the last release that still accepts it. --- .github/workflows/test-e2e.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 962655bd..262c0443 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -32,6 +32,10 @@ jobs: - name: Install Helm uses: azure/setup-helm@v4 + with: + # kustomize v5.5.0 probes helm with `helm version -c`; the -c shorthand + # was removed in helm 3.13, so pin to the last release that accepts it. + version: v3.12.3 - name: Install pinned tools run: make kind kustomize cmctl chainsaw From 3e0392ae43fd608160f85e435ce93937b63dc86b Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Tue, 14 Jul 2026 14:22:34 -0400 Subject: [PATCH 17/20] test(infra): drop load-fixtures from env bring-up load-fixtures builds a connector-tunnel fixture from test/e2e-edge, which is not present on this branch and whose scenario is not in the test/e2e suite CI runs. The step just fails the bring-up on a missing path. Remove it from up; re-add if the e2e-edge suite lands here. --- Taskfile.test-infra.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/Taskfile.test-infra.yml b/Taskfile.test-infra.yml index 3f708f9e..6b16fa10 100644 --- a/Taskfile.test-infra.yml +++ b/Taskfile.test-infra.yml @@ -70,7 +70,6 @@ tasks: - task: cert-manager-upstream - task: cert-manager-downstream - task: nso-image - - task: load-fixtures - task: prepare-upstream - task: eg-downstream - task: extension-server From 08d622328eba2a77badcf8fb603f9933abec788d Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Tue, 14 Jul 2026 14:59:18 -0400 Subject: [PATCH 18/20] test(infra): harden wait-ready against cold-start flakes On a freshly brought-up env the suite raced fixed timeouts: the NSO webhook cert is injected after the manager is Available, so early Domain creates hit "context deadline exceeded"; and the first gateway waits on the merged Envoy proxy to spin up, overrunning a 30s assert. Block wait-ready until the webhook actually admits a server-side dry-run and the downstream GatewayClass is Accepted, and give the billing test's first-gateway Programmed assert the same 3m budget the other gateway tests use. Fixes #279. --- Taskfile.test-infra.yml | 18 ++++++++++++++++++ test/e2e/billing/chainsaw-test.yaml | 3 +++ 2 files changed, 21 insertions(+) diff --git a/Taskfile.test-infra.yml b/Taskfile.test-infra.yml index 6b16fa10..4cca3e9d 100644 --- a/Taskfile.test-infra.yml +++ b/Taskfile.test-infra.yml @@ -290,6 +290,24 @@ tasks: - kubectl --context {{.DOWNSTREAM_CTX}} -n datum-downstream-gateway wait deploy envoy-gateway --for=condition=Available --timeout=240s - echo "โณ waiting for ext-server" - kubectl --context {{.DOWNSTREAM_CTX}} -n network-services-operator-system wait deploy network-services-operator-envoy-gateway-extension-server --for=condition=Available --timeout=240s + # Deployment Available != webhook serving: cert-manager injects the webhook + # cert after the pod is up, so early Domain/Gateway creates race it and time + # out ("context deadline exceeded"). Block until a server-side dry-run is + # actually admitted. + - echo "โณ waiting for the NSO admission webhook to admit" + - | + DOMAIN_JSON='{"apiVersion":"networking.datumapis.com/v1alpha","kind":"Domain","metadata":{"name":"warmup-probe"},"spec":{"domainName":"warmup.e2e.env.datum.net"}}' + for i in $(seq 1 60); do + if echo "$DOMAIN_JSON" | kubectl --context {{.UPSTREAM_CTX}} -n default create --dry-run=server -f - >/dev/null 2>&1; then + echo "โœ… webhook admitting"; break + fi + if [ "$i" -eq 60 ]; then echo "โŒ webhook never became ready"; exit 1; fi + sleep 2 + done + # Ensure the downstream GatewayClass is Accepted so the first test gateway + # programs promptly instead of racing a 30s assert on a cold edge. + - echo "โณ waiting for the downstream GatewayClass to be Accepted" + - kubectl --context {{.DOWNSTREAM_CTX}} wait gatewayclass datum-downstream-gateway-e2e --for=condition=Accepted --timeout=120s - echo "โœ… core components ready." # ---- Karmada ------------------------------------------------------------ diff --git a/test/e2e/billing/chainsaw-test.yaml b/test/e2e/billing/chainsaw-test.yaml index eb5dd80b..c03d9f13 100644 --- a/test/e2e/billing/chainsaw-test.yaml +++ b/test/e2e/billing/chainsaw-test.yaml @@ -255,6 +255,9 @@ spec: namespaces: from: Same - assert: + # Cold edge: the first gateway waits on the merged Envoy proxy to + # spin up, which overruns the 30s default. + timeout: 3m resource: apiVersion: gateway.networking.k8s.io/v1 kind: Gateway From 7f208a1e905f803e2d162b5c9dd515f025f3d9a0 Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Tue, 14 Jul 2026 15:13:11 -0400 Subject: [PATCH 19/20] ci: give the manager more cold-start budget + dump it on failure The manager Deployment intermittently misses the 240s Available wait on a loaded CI runner (heavy two-cluster env). Bump the budget to 360s, and dump the upstream manager + cert-manager state on failure so a genuine stuck manager (vs slow start) is diagnosable. Refs #279. --- .github/workflows/test-e2e.yml | 12 ++++++++++++ Taskfile.test-infra.yml | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 262c0443..420fe171 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -50,6 +50,18 @@ jobs: if: failure() run: | set +e + UKC=${TMPDIR}/.kind-nso-upstream.yaml + bin/kind-v0.32.0 get kubeconfig --name nso-upstream > "${UKC}" + echo "::group::Upstream NSO manager" + KUBECONFIG="${UKC}" kubectl -n network-services-operator-system get pods -o wide + KUBECONFIG="${UKC}" kubectl -n network-services-operator-system describe deploy network-services-operator-controller-manager + KUBECONFIG="${UKC}" kubectl -n network-services-operator-system describe pod -l app.kubernetes.io/name=network-services-operator + KUBECONFIG="${UKC}" kubectl -n network-services-operator-system logs -l app.kubernetes.io/name=network-services-operator --tail=200 --all-containers + echo "::endgroup::" + echo "::group::Upstream cert-manager + webhook cert" + KUBECONFIG="${UKC}" kubectl -n cert-manager get pods -o wide + KUBECONFIG="${UKC}" kubectl -n network-services-operator-system get certificate,secret + echo "::endgroup::" KC=${TMPDIR}/.kind-nso-downstream.yaml bin/kind-v0.32.0 get kubeconfig --name nso-downstream > "${KC}" export KUBECONFIG="${KC}" diff --git a/Taskfile.test-infra.yml b/Taskfile.test-infra.yml index 4cca3e9d..aea687ad 100644 --- a/Taskfile.test-infra.yml +++ b/Taskfile.test-infra.yml @@ -283,7 +283,7 @@ tasks: desc: "Wait for the core components (NSO manager, downstream EG, ext-server) to be Available." cmds: - echo "โณ waiting for NSO manager (upstream)" - - kubectl --context {{.UPSTREAM_CTX}} -n network-services-operator-system wait deploy network-services-operator-controller-manager --for=condition=Available --timeout=240s + - kubectl --context {{.UPSTREAM_CTX}} -n network-services-operator-system wait deploy network-services-operator-controller-manager --for=condition=Available --timeout=360s - echo "โณ waiting for downstream EG" # The gateway control-plane Deployment is named `envoy-gateway` by its chart, # not after the release name. From cd6f654af2fa3d2b38c23d27142cb9db1e39ef7a Mon Sep 17 00:00:00 2001 From: Evan Vetere Date: Tue, 14 Jul 2026 15:29:21 -0400 Subject: [PATCH 20/20] test(infra): run the e2e suite serially Parallel execution on one CI runner maximized cold-start races and resource pressure: the billing test's Envoy access-log scrape ran before the merged proxy had served its route. Run chainsaw with --parallel 1 (as the retired bridge harness did) so the shared data plane warms progressively and each scenario runs against a ready proxy. Refs #279. --- Taskfile.test-infra.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/Taskfile.test-infra.yml b/Taskfile.test-infra.yml index aea687ad..78b070f5 100644 --- a/Taskfile.test-infra.yml +++ b/Taskfile.test-infra.yml @@ -422,6 +422,7 @@ tasks: - echo "๐Ÿงช chainsaw suite {{.TARGET}} (nso-standard={{.UPSTREAM_CLUSTER}}, nso-infra={{.DOWNSTREAM_CLUSTER}}); report {{.REPORT_PATH}}/{{.REPORT_NAME}}.json" - | {{.CHAINSAW}} test {{.TARGET}} \ + --parallel 1 \ --cluster nso-standard={{.TMP_DIR}}/.kind-{{.UPSTREAM_CLUSTER}}.yaml \ --cluster nso-infra={{.TMP_DIR}}/.kind-{{.DOWNSTREAM_CLUSTER}}.yaml \ --report-format JSON \