Skip to content

Commit 8a50eb5

Browse files
AlinsRanbackport-bot[bot]
authored andcommitted
fix: wss related tests are unstable (#2675)
(cherry picked from commit b70e22e)
1 parent 133ed58 commit 8a50eb5

9 files changed

Lines changed: 200 additions & 21 deletions

File tree

internal/controller/gateway_controller.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,10 @@ func (r *GatewayReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
120120
}
121121
return ctrl.Result{}, err
122122
}
123+
if !r.checkGatewayClass(gateway) {
124+
return ctrl.Result{}, nil
125+
}
126+
123127
conditionProgrammedStatus, conditionProgrammedMsg := true, "Programmed"
124128

125129
r.Log.Info("gateway has been accepted", "gateway", gateway.GetName())

internal/controller/indexer/ssl_host.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,9 +69,11 @@ func GatewayTLSHostIndexFunc(rawObj client.Object) []string {
6969
}
7070
}
7171

72-
tlsHostIndexLogger.Info("GatewayTLSHostIndexFunc", "hosts", hostSetToSlice(hosts), "len", len(hostSetToSlice(hosts)))
72+
hostsSlice := hostSetToSlice(hosts)
7373

74-
return hostSetToSlice(hosts)
74+
tlsHostIndexLogger.V(1).Info("GatewayTLSHostIndexFunc", "hosts", hostsSlice)
75+
76+
return hostsSlice
7577
}
7678

7779
// IngressTLSHostIndexFunc indexes Ingresses by their TLS SNI hosts.

test/e2e/crds/v2/route.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2323,6 +2323,7 @@ spec:
23232323
applier.MustApplyAPIv2(types.NamespacedName{Namespace: s.Namespace(), Name: "default"},
23242324
new(apiv2.ApisixRoute), fmt.Sprintf(apisixRouteWithBackendWSS, s.Namespace()))
23252325

2326+
<<<<<<< HEAD
23262327
By("verify wss connection with retry")
23272328
u := url.URL{
23282329
Scheme: "wss",
@@ -2345,6 +2346,14 @@ spec:
23452346
return dialErr
23462347
}).WithTimeout(30*time.Second).WithPolling(2*time.Second).Should(Succeed(), "WebSocket handshake should succeed")
23472348
Expect(resp.StatusCode).Should(Equal(http.StatusSwitchingProtocols))
2349+
=======
2350+
By("verify wss connection")
2351+
hostname := "api6.com"
2352+
conn := s.NewWebsocketClient(&tls.Config{
2353+
InsecureSkipVerify: true,
2354+
ServerName: hostname,
2355+
}, "/ws", http.Header{"Host": []string{hostname}})
2356+
>>>>>>> b70e22eb (fix: wss related tests are unstable (#2675))
23482357

23492358
defer func() {
23502359
_ = conn.Close()

test/e2e/framework/manifests/apisix.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,15 +63,18 @@ metadata:
6363
name: apisix
6464
labels:
6565
app.kubernetes.io/name: apisix
66+
app: apisix
6667
spec:
6768
replicas: {{ default 1 .Replicas }}
6869
selector:
6970
matchLabels:
7071
app.kubernetes.io/name: apisix
72+
app: apisix
7173
template:
7274
metadata:
7375
labels:
7476
app.kubernetes.io/name: apisix
77+
app: apisix
7578
spec:
7679
initContainers:
7780
- name: config-setup

test/e2e/framework/manifests/nginx.yaml

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -124,13 +124,19 @@ spec:
124124
containers:
125125
- livenessProbe:
126126
failureThreshold: 3
127-
initialDelaySeconds: 1
128-
periodSeconds: 5
127+
initialDelaySeconds: 10
128+
periodSeconds: 15
129129
successThreshold: 1
130130
httpGet:
131131
path: /healthz
132132
port: 80
133133
timeoutSeconds: 2
134+
readinessProbe:
135+
httpGet:
136+
path: /healthz
137+
port: 80
138+
initialDelaySeconds: 5
139+
periodSeconds: 5
134140
image: "openresty/openresty:1.27.1.2-4-bullseye-fat"
135141
imagePullPolicy: IfNotPresent
136142
name: nginx

test/e2e/gatewayapi/httproute.go

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ import (
2222
"crypto/tls"
2323
"fmt"
2424
"net/http"
25-
"net/url"
2625
"strings"
2726
"time"
2827

@@ -2507,17 +2506,10 @@ spec:
25072506

25082507
It("WSS backend", func() {
25092508
s.ResourceApplied("HTTPRoute", "nginx-wss", fmt.Sprintf(httprouteWithWSS, s.Namespace()), 1)
2510-
time.Sleep(6 * time.Second)
25112509

25122510
By("verify wss connection")
2513-
u := url.URL{
2514-
Scheme: "wss",
2515-
Host: s.GetAPISIXHTTPSEndpoint(),
2516-
Path: "/ws",
2517-
}
2518-
headers := http.Header{"Host": []string{"api6.com"}}
2519-
25202511
hostname := "api6.com"
2512+
<<<<<<< HEAD
25212513

25222514
dialer := websocket.Dialer{
25232515
TLSClientConfig: &tls.Config{
@@ -2534,6 +2526,12 @@ spec:
25342526
return dialErr
25352527
}).WithTimeout(30*time.Second).WithPolling(2*time.Second).Should(Succeed(), "WebSocket handshake should succeed")
25362528
Expect(resp.StatusCode).Should(Equal(http.StatusSwitchingProtocols))
2529+
=======
2530+
conn := s.NewWebsocketClient(&tls.Config{
2531+
InsecureSkipVerify: true,
2532+
ServerName: hostname,
2533+
}, "/ws", http.Header{"Host": []string{hostname}})
2534+
>>>>>>> b70e22eb (fix: wss related tests are unstable (#2675))
25372535

25382536
defer func() {
25392537
_ = conn.Close()

test/e2e/ingress/ingress.go

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ import (
2323
"encoding/base64"
2424
"fmt"
2525
"net/http"
26-
"net/url"
2726
"strings"
2827
"time"
2928

@@ -992,14 +991,13 @@ spec:
992991
By("create Ingress")
993992
err := s.CreateResourceFromStringWithNamespace(fmt.Sprintf(ingress, s.Namespace()), s.Namespace())
994993
Expect(err).NotTo(HaveOccurred(), "creating Ingress")
995-
time.Sleep(5 * time.Second)
996994

997995
By("verify Ingress works")
998-
s.NewAPISIXClient().
999-
GET("/get").
1000-
WithHost("ingress.example.com").
1001-
Expect().
1002-
Status(200)
996+
s.RequestAssert(&scaffold.RequestAssert{
997+
Method: "GET",
998+
Host: "ingress.example.com",
999+
Check: scaffold.WithExpectedStatus(http.StatusOK),
1000+
})
10031001

10041002
By("create additional gateway group to get new admin key")
10051003
additionalGatewayGroupID, _, err = s.Deployer.CreateAdditionalGateway("gateway-proxy-update")
@@ -1121,9 +1119,9 @@ spec:
11211119
createSecret(s, _secretName)
11221120
By("create Ingress")
11231121
Expect(s.CreateResourceFromString(fmt.Sprintf(ingressWithWSS, s.Namespace()))).ShouldNot(HaveOccurred(), "creating Ingress")
1124-
time.Sleep(6 * time.Second)
11251122

11261123
By("verify wss connection")
1124+
<<<<<<< HEAD
11271125
u := url.URL{
11281126
Scheme: "wss",
11291127
Host: s.GetAPISIXHTTPSEndpoint(),
@@ -1145,6 +1143,13 @@ spec:
11451143
return dialErr
11461144
}).WithTimeout(30*time.Second).WithPolling(2*time.Second).Should(Succeed(), "WebSocket handshake should succeed")
11471145
Expect(resp.StatusCode).Should(Equal(http.StatusSwitchingProtocols))
1146+
=======
1147+
hostname := "api6.com"
1148+
conn := s.NewWebsocketClient(&tls.Config{
1149+
InsecureSkipVerify: true,
1150+
ServerName: hostname,
1151+
}, "/ws", http.Header{"Host": []string{hostname}})
1152+
>>>>>>> b70e22eb (fix: wss related tests are unstable (#2675))
11481153

11491154
defer func() {
11501155
_ = conn.Close()

test/e2e/scaffold/apisix_deployer.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,19 @@ func (s *APISIXDeployer) AfterEach() {
111111
if output != "" {
112112
_, _ = fmt.Fprintln(GinkgoWriter, output)
113113
}
114+
115+
_, _ = fmt.Fprintln(GinkgoWriter, "Dumping APISIX logs:")
116+
output = s.GetDeploymentLogs("apisix")
117+
if output != "" {
118+
_, _ = fmt.Fprintln(GinkgoWriter, output)
119+
}
120+
if framework.ProviderType == framework.ProviderTypeAPISIXStandalone && s.adminTunnel != nil {
121+
client := NewClient("http", s.adminTunnel.Endpoint())
122+
reporter := &ErrorReporter{}
123+
body := client.GET("/apisix/admin/configs").WithHeader("X-API-KEY", s.AdminKey()).WithReporter(reporter).Expect().Body().Raw()
124+
_, _ = fmt.Fprintln(GinkgoWriter, "Dumping APISIX configs:")
125+
_, _ = fmt.Fprintln(GinkgoWriter, body)
126+
}
114127
}
115128

116129
// Delete all additional gateways

test/e2e/scaffold/scaffold.go

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929

3030
"github.com/api7/gopkg/pkg/log"
3131
"github.com/gavv/httpexpect/v2"
32+
"github.com/gorilla/websocket"
3233
"github.com/gruntwork-io/terratest/modules/k8s"
3334
"github.com/gruntwork-io/terratest/modules/testing"
3435
. "github.com/onsi/ginkgo/v2" //nolint:staticcheck
@@ -579,3 +580,141 @@ func (s *Scaffold) GetMetricsEndpoint() string {
579580
s.addFinalizers(tunnel.Close)
580581
return fmt.Sprintf("http://%s/metrics", tunnel.Endpoint())
581582
}
583+
<<<<<<< HEAD
584+
=======
585+
586+
func (s *Scaffold) NewWebsocketClient(tls *tls.Config, path string, headers http.Header) *websocket.Conn {
587+
var host = s.ApisixHTTPEndpoint()
588+
var scheme = "ws"
589+
if tls != nil {
590+
scheme = "wss"
591+
host = s.GetAPISIXHTTPSEndpoint()
592+
}
593+
594+
dialer := websocket.Dialer{
595+
TLSClientConfig: tls,
596+
}
597+
598+
u := url.URL{
599+
Scheme: scheme,
600+
Host: host,
601+
Path: path,
602+
}
603+
var conn *websocket.Conn
604+
605+
s.RetryAssertion(func() error {
606+
c, resp, err := dialer.Dial(u.String(), headers)
607+
if err != nil {
608+
return err
609+
}
610+
if resp == nil || resp.StatusCode != http.StatusSwitchingProtocols {
611+
_ = c.Close()
612+
return fmt.Errorf("unexpected status code: %d", resp.StatusCode)
613+
}
614+
conn = c
615+
return nil
616+
}).ShouldNot(HaveOccurred(), "establishing websocket connection")
617+
return conn
618+
}
619+
620+
func (s *Scaffold) ControlAPIClient() (ControlAPIClient, error) {
621+
tunnel := k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, "apisix-control-api", 9090, 9090)
622+
if err := tunnel.ForwardPortE(s.t); err != nil {
623+
return nil, err
624+
}
625+
s.addFinalizers(tunnel.Close)
626+
627+
return &controlAPI{
628+
client: NewClient("http", tunnel.Endpoint()),
629+
}, nil
630+
}
631+
632+
func (s *Scaffold) EnsureNumService(controlAPIClient ControlAPIClient, matcher func(result int) bool) error {
633+
times := 0
634+
return wait.PollUntilContextTimeout(context.Background(), 100*time.Millisecond, 10*time.Minute, true, func(ctx context.Context) (done bool, err error) {
635+
times++
636+
results, _, err := controlAPIClient.ListServices()
637+
if err != nil {
638+
log.Errorw("failed to ListServices", zap.Error(err))
639+
return false, nil
640+
}
641+
if !matcher(len(results)) {
642+
log.Debugw("number of effective services", zap.Int("number", len(results)), zap.Int("times", times))
643+
return false, nil
644+
}
645+
return true, nil
646+
})
647+
}
648+
649+
func (s *Scaffold) ExpectUpstream(controlAPIClient ControlAPIClient, name string, matcher func(upstream adctypes.Upstream) bool) error {
650+
times := 0
651+
return wait.PollUntilContextTimeout(context.Background(), 1*time.Second, 10*time.Minute, true, func(ctx context.Context) (done bool, err error) {
652+
times++
653+
upstreams, _, err := controlAPIClient.ListUpstreams()
654+
if err != nil {
655+
log.Errorw("failed to ListServices", zap.Error(err))
656+
return false, nil
657+
}
658+
for _, upstream := range upstreams {
659+
upsValue := upstream.(map[string]any)
660+
data, err := json.Marshal(upsValue["value"])
661+
if err != nil {
662+
return false, fmt.Errorf("failed to marshal upstream: %v", err)
663+
}
664+
665+
var ups adctypes.Upstream
666+
if err := json.Unmarshal(data, &ups); err != nil {
667+
return false, fmt.Errorf("failed to unmarshal upstream: %v", err)
668+
}
669+
if name != "" && ups.Name != name {
670+
continue
671+
}
672+
if ok := matcher(ups); !ok {
673+
return false, nil
674+
}
675+
}
676+
return true, nil
677+
})
678+
}
679+
680+
func (s *Scaffold) EnsureNumUpstreamNodes(controlAPIClient ControlAPIClient, name string, number int) error {
681+
return s.ExpectUpstream(controlAPIClient, name, func(upstream adctypes.Upstream) bool {
682+
if len(upstream.Nodes) != number {
683+
log.Warnf("expect upstream: [%s] nodes num to be %d, but got %d", upstream.Name, number, len(upstream.Nodes))
684+
return false
685+
}
686+
return true
687+
})
688+
}
689+
690+
type ControlAPIClient interface {
691+
ListServices() ([]any, int64, error)
692+
ListUpstreams() ([]any, int64, error)
693+
}
694+
695+
type controlAPI struct {
696+
client *httpexpect.Expect
697+
}
698+
699+
func (c *controlAPI) ListUpstreams() (result []any, total int64, err error) {
700+
resp := c.client.Request(http.MethodGet, "/v1/upstreams").Expect()
701+
if resp.Raw().StatusCode != http.StatusOK {
702+
return nil, 0, fmt.Errorf("unexpected status code: %v, message: %s", resp.Raw().StatusCode, resp.Body().Raw())
703+
}
704+
if err = json.Unmarshal([]byte(resp.Body().Raw()), &result); err != nil {
705+
return nil, 0, fmt.Errorf("failed to unmarshal response body: %w", err)
706+
}
707+
return result, int64(len(result)), err
708+
}
709+
710+
func (c *controlAPI) ListServices() (result []any, total int64, err error) {
711+
resp := c.client.Request(http.MethodGet, "/v1/services").Expect()
712+
if resp.Raw().StatusCode != http.StatusOK {
713+
return nil, 0, fmt.Errorf("unexpected status code: %v, message: %s", resp.Raw().StatusCode, resp.Body().Raw())
714+
}
715+
if err = json.Unmarshal([]byte(resp.Body().Raw()), &result); err != nil {
716+
return nil, 0, fmt.Errorf("failed to unmarshal response body: %w", err)
717+
}
718+
return result, int64(len(result)), err
719+
}
720+
>>>>>>> b70e22eb (fix: wss related tests are unstable (#2675))

0 commit comments

Comments
 (0)