Skip to content

Commit f182ea3

Browse files
Foo BarCopilot
andcommitted
feat: enable ginkgo parallel execution for API7EE E2E tests
- Convert BeforeSuite/AfterSuite to SynchronizedBeforeSuite/SynchronizedAfterSuite so the API7EE control plane is deployed only once (node 1) while each ginkgo parallel node creates its own dashboard port-forward tunnel - Fix newDashboardTunnel to use auto-assigned local ports (findFreePort) instead of the fixed NodePort value, preventing port conflicts between parallel nodes - Add ginkgo-api7ee-e2e-test Makefile target for ginkgo CLI invocation - Update e2e-test.yml to install ginkgo and run with E2E_NODES=2 for non-webhook matrix jobs (webhook remains serial with E2E_NODES=1) Expected: CI time reduced from ~60 min to ~22-25 min on the two heavy shards. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 39e1c91 commit f182ea3

4 files changed

Lines changed: 110 additions & 11 deletions

File tree

.github/workflows/e2e-test.yml

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,10 @@ jobs:
6161
curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
6262
chmod 700 get_helm.sh
6363
./get_helm.sh
64+
65+
- name: Install ginkgo
66+
run: make install-ginkgo
67+
6468
- name: Login to Registry
6569
uses: docker/login-action@v3
6670
with:
@@ -121,4 +125,8 @@ jobs:
121125
TEST_LABEL: ${{ matrix.cases_subset }}
122126
TEST_ENV: CI
123127
run: |
124-
make e2e-test
128+
if [[ "${{ matrix.cases_subset }}" == "webhook" ]]; then
129+
E2E_NODES=1 make ginkgo-api7ee-e2e-test
130+
else
131+
E2E_NODES=2 make ginkgo-api7ee-e2e-test
132+
fi

Makefile

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,13 @@ download-api7ee3-chart:
154154
ginkgo-e2e-test: adc
155155
@ginkgo -cover -coverprofile=coverage.txt -r --randomize-all --randomize-suites --trace --focus=$(E2E_FOCUS) --nodes=$(E2E_NODES) --label-filter="$(TEST_LABEL)" $(TEST_DIR)
156156

157+
.PHONY: ginkgo-api7ee-e2e-test
158+
ginkgo-api7ee-e2e-test: adc
159+
@DASHBOARD_VERSION=$(DASHBOARD_VERSION) ginkgo -cover -coverprofile=coverage.txt \
160+
--randomize-all --randomize-suites --trace \
161+
--timeout=$(TEST_TIMEOUT) --nodes=$(E2E_NODES) \
162+
--label-filter="$(TEST_LABEL)" ./test/e2e/
163+
157164
.PHONY: install-ginkgo
158165
install-ginkgo:
159166
@go install github.com/onsi/ginkgo/v2/ginkgo@v$(GINKGO_VERSION)

test/e2e/e2e_test.go

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,13 @@ func TestE2E(t *testing.T) {
4242
// init newDeployer function
4343
scaffold.NewDeployer = scaffold.NewAPI7Deployer
4444

45-
BeforeSuite(f.BeforeSuite)
46-
AfterSuite(f.AfterSuite)
45+
// DeployAPI7EE runs only on ginkgo node 1 and deploys the shared API7EE control plane.
46+
// InitNodeConnections runs on every node to set up per-node dashboard connections.
47+
SynchronizedBeforeSuite(f.DeployAPI7EE, f.InitNodeConnections)
48+
49+
// CloseNodeConnections runs on every node to close per-node dashboard tunnels.
50+
// TeardownInfrastructure runs only on node 1 for any suite-level cleanup.
51+
SynchronizedAfterSuite(f.CloseNodeConnections, f.TeardownInfrastructure)
4752

4853
_, _ = fmt.Fprintf(GinkgoWriter, "Starting apisix-ingress suite\n")
4954
RunSpecs(t, "e2e suite")

test/e2e/framework/api7_framework.go

Lines changed: 87 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,74 @@ func (f *Framework) AfterSuite() {
8383
f.shutdownDashboardTunnel()
8484
}
8585

86+
// DeployAPI7EE deploys the API7EE control plane once (runs on ginkgo node 1 only).
87+
// It returns a ready signal consumed by InitNodeConnections on all nodes.
88+
func (f *Framework) DeployAPI7EE() []byte {
89+
API7EELicense = os.Getenv("API7_EE_LICENSE")
90+
if API7EELicense == "" {
91+
panic("env {API7_EE_LICENSE} is required")
92+
}
93+
94+
dashboardVersion = os.Getenv("DASHBOARD_VERSION")
95+
if dashboardVersion == "" {
96+
dashboardVersion = "dev"
97+
}
98+
99+
_ = k8s.DeleteNamespaceE(GinkgoT(), f.kubectlOpts, _namespace)
100+
101+
Eventually(func() error {
102+
_, err := k8s.GetNamespaceE(GinkgoT(), f.kubectlOpts, _namespace)
103+
if k8serrors.IsNotFound(err) {
104+
return nil
105+
}
106+
return fmt.Errorf("namespace %s still exists", _namespace)
107+
}, "1m", "2s").Should(Succeed())
108+
109+
k8s.CreateNamespace(GinkgoT(), f.kubectlOpts, _namespace)
110+
111+
f.DeployComponents()
112+
113+
time.Sleep(1 * time.Minute)
114+
115+
// Create a temporary tunnel for one-time setup operations.
116+
// Each node will create its own persistent tunnel in InitNodeConnections.
117+
err := f.newDashboardTunnel()
118+
Expect(err).ShouldNot(HaveOccurred(), "creating temporary dashboard tunnel")
119+
f.Logf("Temporary dashboard tunnel: %s", _dashboardHTTPTunnel.Endpoint())
120+
121+
f.UploadLicense()
122+
f.setDpManagerEndpoints()
123+
124+
// Close the temporary tunnel; each node creates its own in InitNodeConnections.
125+
f.shutdownDashboardTunnel()
126+
127+
return []byte("ready")
128+
}
129+
130+
// InitNodeConnections initializes per-node connections to the shared API7EE control plane.
131+
// It runs on every ginkgo parallel node after DeployAPI7EE completes.
132+
func (f *Framework) InitNodeConnections(_ []byte) {
133+
API7EELicense = os.Getenv("API7_EE_LICENSE")
134+
135+
dashboardVersion = os.Getenv("DASHBOARD_VERSION")
136+
if dashboardVersion == "" {
137+
dashboardVersion = "dev"
138+
}
139+
140+
err := f.newDashboardTunnel()
141+
Expect(err).ShouldNot(HaveOccurred(), "creating dashboard tunnel for node")
142+
f.Logf("Dashboard HTTP Tunnel: %s", _dashboardHTTPTunnel.Endpoint())
143+
}
144+
145+
// CloseNodeConnections closes per-node connections. Runs on every ginkgo parallel node.
146+
func (f *Framework) CloseNodeConnections() {
147+
f.shutdownDashboardTunnel()
148+
}
149+
150+
// TeardownInfrastructure cleans up suite-level resources. Runs on ginkgo node 1 only.
151+
// The Kind cluster is deleted by CI after the job, so this is a no-op.
152+
func (f *Framework) TeardownInfrastructure() {}
153+
86154
// DeployComponents deploy necessary components
87155
func (f *Framework) DeployComponents() {
88156
f.deploy()
@@ -167,31 +235,42 @@ var (
167235
_dashboardHTTPSTunnel *k8s.Tunnel
168236
)
169237

238+
// dashboardLocalPorts returns the local port pair to use for the dashboard HTTP
239+
// and HTTPS tunnels. Each ginkgo parallel process gets a unique, non-overlapping
240+
// range based on its 1-indexed process number, eliminating port conflicts without
241+
// any TOCTOU race.
242+
//
243+
// Process 1 → 18100 / 18101
244+
// Process 2 → 18200 / 18201
245+
// Process N → 18N00 / 18N01
246+
func dashboardLocalPorts() (httpLocal, httpsLocal int) {
247+
node := GinkgoParallelProcess() // 1-indexed
248+
base := 18000 + node*100
249+
return base, base + 1
250+
}
251+
170252
func (f *Framework) newDashboardTunnel() error {
171253
var (
172-
httpNodePort int
173-
httpsNodePort int
174-
httpPort int
175-
httpsPort int
254+
httpPort int
255+
httpsPort int
176256
)
177257

178258
service := k8s.GetService(f.GinkgoT, f.kubectlOpts, "api7ee3-dashboard")
179259

180260
for _, port := range service.Spec.Ports {
181261
switch port.Name {
182262
case "http":
183-
httpNodePort = int(port.NodePort)
184263
httpPort = int(port.Port)
185264
case "https":
186-
httpsNodePort = int(port.NodePort)
187265
httpsPort = int(port.Port)
188266
}
189267
}
190268

269+
httpLocal, httpsLocal := dashboardLocalPorts()
191270
_dashboardHTTPTunnel = k8s.NewTunnel(f.kubectlOpts, k8s.ResourceTypeService, "api7ee3-dashboard",
192-
httpNodePort, httpPort)
271+
httpLocal, httpPort)
193272
_dashboardHTTPSTunnel = k8s.NewTunnel(f.kubectlOpts, k8s.ResourceTypeService, "api7ee3-dashboard",
194-
httpsNodePort, httpsPort)
273+
httpsLocal, httpsPort)
195274

196275
if err := _dashboardHTTPTunnel.ForwardPortE(f.GinkgoT); err != nil {
197276
return err

0 commit comments

Comments
 (0)