Skip to content

Commit 14930e9

Browse files
Foo BarCopilot
andcommitted
feat: enable ginkgo parallel execution for API7EE E2E tests
- Convert BeforeSuite/AfterSuite to SynchronizedBeforeSuite/SynchronizedAfterSuite so the API7EE control plane is deployed only once (node 1) while each ginkgo parallel node creates its own dashboard port-forward tunnel - Fix newDashboardTunnel to use auto-assigned local ports (findFreePort) instead of the fixed NodePort value, preventing port conflicts between parallel nodes - Add ginkgo-api7ee-e2e-test Makefile target for ginkgo CLI invocation - Update e2e-test.yml to install ginkgo and run with E2E_NODES=2 for non-webhook matrix jobs (webhook remains serial with E2E_NODES=1) Expected: CI time reduced from ~60 min to ~22-25 min on the two heavy shards. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent a5c8d8d commit 14930e9

4 files changed

Lines changed: 109 additions & 11 deletions

File tree

.github/workflows/e2e-test.yml

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,10 @@ jobs:
6161
curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
6262
chmod 700 get_helm.sh
6363
./get_helm.sh
64+
65+
- name: Install ginkgo
66+
run: make install-ginkgo
67+
6468
- name: Login to Registry
6569
uses: docker/login-action@v3
6670
with:
@@ -121,4 +125,8 @@ jobs:
121125
TEST_LABEL: ${{ matrix.cases_subset }}
122126
TEST_ENV: CI
123127
run: |
124-
make e2e-test
128+
if [[ "${{ matrix.cases_subset }}" == "webhook" ]]; then
129+
E2E_NODES=1 make ginkgo-api7ee-e2e-test
130+
else
131+
E2E_NODES=2 make ginkgo-api7ee-e2e-test
132+
fi

Makefile

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,13 @@ download-api7ee3-chart:
154154
ginkgo-e2e-test: adc
155155
@ginkgo -cover -coverprofile=coverage.txt -r --randomize-all --randomize-suites --trace --focus=$(E2E_FOCUS) --nodes=$(E2E_NODES) --label-filter="$(TEST_LABEL)" $(TEST_DIR)
156156

157+
.PHONY: ginkgo-api7ee-e2e-test
158+
ginkgo-api7ee-e2e-test: adc
159+
@DASHBOARD_VERSION=$(DASHBOARD_VERSION) ginkgo -cover -coverprofile=coverage.txt \
160+
--randomize-all --randomize-suites --trace \
161+
--timeout=$(TEST_TIMEOUT) --nodes=$(E2E_NODES) \
162+
--label-filter="$(TEST_LABEL)" ./test/e2e/
163+
157164
.PHONY: install-ginkgo
158165
install-ginkgo:
159166
@go install github.com/onsi/ginkgo/v2/ginkgo@v$(GINKGO_VERSION)

test/e2e/e2e_test.go

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,13 @@ func TestE2E(t *testing.T) {
4242
// init newDeployer function
4343
scaffold.NewDeployer = scaffold.NewAPI7Deployer
4444

45-
BeforeSuite(f.BeforeSuite)
46-
AfterSuite(f.AfterSuite)
45+
// DeployAPI7EE runs only on ginkgo node 1 and deploys the shared API7EE control plane.
46+
// InitNodeConnections runs on every node to set up per-node dashboard connections.
47+
SynchronizedBeforeSuite(f.DeployAPI7EE, f.InitNodeConnections)
48+
49+
// CloseNodeConnections runs on every node to close per-node dashboard tunnels.
50+
// TeardownInfrastructure runs only on node 1 for any suite-level cleanup.
51+
SynchronizedAfterSuite(f.CloseNodeConnections, f.TeardownInfrastructure)
4752

4853
_, _ = fmt.Fprintf(GinkgoWriter, "Starting apisix-ingress suite\n")
4954
RunSpecs(t, "e2e suite")

test/e2e/framework/api7_framework.go

Lines changed: 86 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"bytes"
2222
"encoding/json"
2323
"fmt"
24+
"net"
2425
"os"
2526
"time"
2627

@@ -83,6 +84,74 @@ func (f *Framework) AfterSuite() {
8384
f.shutdownDashboardTunnel()
8485
}
8586

87+
// DeployAPI7EE deploys the API7EE control plane once (runs on ginkgo node 1 only).
88+
// It returns a ready signal consumed by InitNodeConnections on all nodes.
89+
func (f *Framework) DeployAPI7EE() []byte {
90+
API7EELicense = os.Getenv("API7_EE_LICENSE")
91+
if API7EELicense == "" {
92+
panic("env {API7_EE_LICENSE} is required")
93+
}
94+
95+
dashboardVersion = os.Getenv("DASHBOARD_VERSION")
96+
if dashboardVersion == "" {
97+
dashboardVersion = "dev"
98+
}
99+
100+
_ = k8s.DeleteNamespaceE(GinkgoT(), f.kubectlOpts, _namespace)
101+
102+
Eventually(func() error {
103+
_, err := k8s.GetNamespaceE(GinkgoT(), f.kubectlOpts, _namespace)
104+
if k8serrors.IsNotFound(err) {
105+
return nil
106+
}
107+
return fmt.Errorf("namespace %s still exists", _namespace)
108+
}, "1m", "2s").Should(Succeed())
109+
110+
k8s.CreateNamespace(GinkgoT(), f.kubectlOpts, _namespace)
111+
112+
f.DeployComponents()
113+
114+
time.Sleep(1 * time.Minute)
115+
116+
// Create a temporary tunnel for one-time setup operations.
117+
// Each node will create its own persistent tunnel in InitNodeConnections.
118+
err := f.newDashboardTunnel()
119+
Expect(err).ShouldNot(HaveOccurred(), "creating temporary dashboard tunnel")
120+
f.Logf("Temporary dashboard tunnel: %s", _dashboardHTTPTunnel.Endpoint())
121+
122+
f.UploadLicense()
123+
f.setDpManagerEndpoints()
124+
125+
// Close the temporary tunnel; each node creates its own in InitNodeConnections.
126+
f.shutdownDashboardTunnel()
127+
128+
return []byte("ready")
129+
}
130+
131+
// InitNodeConnections initializes per-node connections to the shared API7EE control plane.
132+
// It runs on every ginkgo parallel node after DeployAPI7EE completes.
133+
func (f *Framework) InitNodeConnections(_ []byte) {
134+
API7EELicense = os.Getenv("API7_EE_LICENSE")
135+
136+
dashboardVersion = os.Getenv("DASHBOARD_VERSION")
137+
if dashboardVersion == "" {
138+
dashboardVersion = "dev"
139+
}
140+
141+
err := f.newDashboardTunnel()
142+
Expect(err).ShouldNot(HaveOccurred(), "creating dashboard tunnel for node")
143+
f.Logf("Dashboard HTTP Tunnel: %s", _dashboardHTTPTunnel.Endpoint())
144+
}
145+
146+
// CloseNodeConnections closes per-node connections. Runs on every ginkgo parallel node.
147+
func (f *Framework) CloseNodeConnections() {
148+
f.shutdownDashboardTunnel()
149+
}
150+
151+
// TeardownInfrastructure cleans up suite-level resources. Runs on ginkgo node 1 only.
152+
// The Kind cluster is deleted by CI after the job, so this is a no-op.
153+
func (f *Framework) TeardownInfrastructure() {}
154+
86155
// DeployComponents deploy necessary components
87156
func (f *Framework) DeployComponents() {
88157
f.deploy()
@@ -167,31 +236,40 @@ var (
167236
_dashboardHTTPSTunnel *k8s.Tunnel
168237
)
169238

239+
// findFreePort returns an available TCP port on the local machine.
240+
// Using port 0 with net.Listen lets the OS pick a free port.
241+
func findFreePort() int {
242+
ln, err := net.Listen("tcp", ":0")
243+
if err != nil {
244+
panic(fmt.Sprintf("find free port: %v", err))
245+
}
246+
port := ln.Addr().(*net.TCPAddr).Port
247+
_ = ln.Close()
248+
return port
249+
}
250+
170251
func (f *Framework) newDashboardTunnel() error {
171252
var (
172-
httpNodePort int
173-
httpsNodePort int
174-
httpPort int
175-
httpsPort int
253+
httpPort int
254+
httpsPort int
176255
)
177256

178257
service := k8s.GetService(f.GinkgoT, f.kubectlOpts, "api7ee3-dashboard")
179258

180259
for _, port := range service.Spec.Ports {
181260
switch port.Name {
182261
case "http":
183-
httpNodePort = int(port.NodePort)
184262
httpPort = int(port.Port)
185263
case "https":
186-
httpsNodePort = int(port.NodePort)
187264
httpsPort = int(port.Port)
188265
}
189266
}
190267

268+
// Use auto-assigned local ports so parallel ginkgo nodes don't conflict.
191269
_dashboardHTTPTunnel = k8s.NewTunnel(f.kubectlOpts, k8s.ResourceTypeService, "api7ee3-dashboard",
192-
httpNodePort, httpPort)
270+
findFreePort(), httpPort)
193271
_dashboardHTTPSTunnel = k8s.NewTunnel(f.kubectlOpts, k8s.ResourceTypeService, "api7ee3-dashboard",
194-
httpsNodePort, httpsPort)
272+
findFreePort(), httpsPort)
195273

196274
if err := _dashboardHTTPTunnel.ForwardPortE(f.GinkgoT); err != nil {
197275
return err

0 commit comments

Comments
 (0)