Skip to content

Commit ab9215f

Browse files
teqwveGopher Bot
teqwve
authored and
Gopher Bot
committed
BUG/MEDIUM: handle different endpoint ports
Before the change the first port encountered when gathering service endpoints were used for all endpoints even if the ports were different. Fixes: #737
1 parent ee4a12d commit ab9215f

9 files changed

Lines changed: 298 additions & 46 deletions

File tree

‎deploy/tests/e2e/different-ports/config/deploy.yaml.tmpl‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
{{ range $instanceIndex, $instance := .Instances }}
2+
---
3+
kind: Deployment
4+
apiVersion: apps/v1
5+
metadata:
6+
name: http-echo-{{ $instanceIndex }}
7+
spec:
8+
replicas: {{ $instance.Replicas }}
9+
selector:
10+
matchLabels:
11+
app.kubernetes.io/name: http-echo
12+
app.kubernetes.io/instance: http-echo-{{ $instanceIndex }}
13+
template:
14+
metadata:
15+
labels:
16+
app.kubernetes.io/name: http-echo
17+
app.kubernetes.io/instance: http-echo-{{ $instanceIndex }}
18+
spec:
19+
containers:
20+
- name: http-echo
21+
image: haproxytech/http-echo:latest
22+
imagePullPolicy: Never
23+
args:
24+
- --http={{ $instance.Port }}
25+
- --default-response=hostname
26+
ports:
27+
- name: http
28+
containerPort: {{ $instance.Port }}
29+
protocol: TCP
30+
{{- end }}
31+
---
32+
kind: Service
33+
apiVersion: v1
34+
metadata:
35+
name: http-echo
36+
spec:
37+
ipFamilyPolicy: RequireDualStack
38+
ports:
39+
- name: http
40+
protocol: TCP
41+
port: 80
42+
targetPort: http
43+
selector:
44+
app.kubernetes.io/name: http-echo

‎deploy/tests/e2e/different-ports/config/ingress.yaml.tmpl‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
---
2+
kind: Ingress
3+
apiVersion: networking.k8s.io/v1
4+
metadata:
5+
name: http-echo
6+
annotations:
7+
{{range .IngAnnotations}}
8+
"{{ .Key }}": "{{ .Value }}"
9+
{{end}}
10+
spec:
11+
ingressClassName: haproxy
12+
rules:
13+
- host: {{ .Host }}
14+
http:
15+
paths:
16+
- path: /
17+
pathType: Prefix
18+
backend:
19+
service:
20+
name: http-echo
21+
port:
22+
name: http

‎deploy/tests/e2e/different-ports/different_ports_test.go‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
// Copyright 2026 HAProxy Technologies LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
//go:build e2e_parallel
16+
17+
package differentports
18+
19+
import (
20+
"io"
21+
"strconv"
22+
"strings"
23+
24+
"github.com/haproxytech/kubernetes-ingress/deploy/tests/e2e"
25+
)
26+
27+
func (suite *DifferentPortsSuite) Test_Different_Ports() {
28+
suite.Run("different port added without scaling", func() {
29+
suite.tmplData.Instances = append(suite.tmplData.Instances, instanceTmplData{Port: 8001, Replicas: 1})
30+
suite.Require().NoError(suite.test.Apply("config/deploy.yaml.tmpl", suite.test.GetNS(), suite.tmplData))
31+
suite.Eventually(suite.everyReplicaReachable, e2e.WaitDuration, e2e.TickDuration)
32+
})
33+
34+
suite.Run("different port added with scaling", func() {
35+
suite.tmplData.Instances = append(suite.tmplData.Instances, instanceTmplData{Port: 8002, Replicas: 8})
36+
suite.Require().NoError(suite.test.Apply("config/deploy.yaml.tmpl", suite.test.GetNS(), suite.tmplData))
37+
suite.Eventually(suite.everyReplicaReachable, e2e.WaitDuration, e2e.TickDuration)
38+
})
39+
40+
suite.Run("different port added while previous port removed", func() {
41+
suite.tmplData.Instances = append(suite.tmplData.Instances, instanceTmplData{Port: 8003, Replicas: 1})
42+
suite.tmplData.Instances[len(suite.tmplData.Instances)-2].Replicas = 0
43+
suite.Require().NoError(suite.test.Apply("config/deploy.yaml.tmpl", suite.test.GetNS(), suite.tmplData))
44+
suite.Eventually(suite.everyReplicaReachable, e2e.WaitDuration, e2e.TickDuration)
45+
})
46+
47+
suite.Run("standalone different ports", func() {
48+
suite.tmplData.Instances = append(suite.tmplData.Instances, instanceTmplData{Port: 8004, Replicas: 1})
49+
suite.tmplData.IngAnnotations = []struct{ Key, Value string }{{Key: "haproxy.org/standalone-backend", Value: "true"}}
50+
suite.Require().NoError(suite.test.Apply("config/deploy.yaml.tmpl", suite.test.GetNS(), suite.tmplData))
51+
suite.Require().NoError(suite.test.Apply("config/ingress.yaml.tmpl", suite.test.GetNS(), suite.tmplData))
52+
suite.Eventually(suite.everyReplicaReachable, e2e.WaitDuration, e2e.TickDuration)
53+
})
54+
}
55+
56+
func (suite *DifferentPortsSuite) everyReplicaReachable() bool {
57+
totalReplicas := 0
58+
counter := map[int]map[string]int{}
59+
for instanceIndex, instance := range suite.tmplData.Instances {
60+
totalReplicas += instance.Replicas
61+
counter[instanceIndex] = make(map[string]int)
62+
}
63+
64+
for i := 0; i < 2*totalReplicas; i++ {
65+
func() {
66+
res, cls, err := suite.client.Do()
67+
if err != nil {
68+
suite.T().Log(err.Error())
69+
}
70+
defer cls()
71+
if res.StatusCode == 200 {
72+
body, err := io.ReadAll(res.Body)
73+
if err != nil {
74+
suite.T().Log(err.Error())
75+
return
76+
}
77+
78+
pod := strings.TrimSpace(string(body))
79+
instanceIndex, err := strconv.Atoi(strings.Split(pod, "-")[2]) // http-echo-<index>-<hash>-<random>
80+
if err != nil {
81+
suite.T().Log(err.Error())
82+
return
83+
}
84+
counter[instanceIndex][pod]++
85+
}
86+
}()
87+
}
88+
89+
for instanceIndex, instance := range suite.tmplData.Instances {
90+
if len(counter[instanceIndex]) != instance.Replicas {
91+
return false
92+
}
93+
for _, v := range counter[instanceIndex] {
94+
if v != 2 {
95+
return false
96+
}
97+
}
98+
}
99+
return true
100+
}

‎deploy/tests/e2e/different-ports/suite_test.go‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
// Copyright 2026 HAProxy Technologies LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
//go:build e2e_parallel
16+
17+
package differentports
18+
19+
import (
20+
"net/http"
21+
"testing"
22+
23+
"github.com/stretchr/testify/suite"
24+
25+
"github.com/haproxytech/kubernetes-ingress/deploy/tests/e2e"
26+
)
27+
28+
type DifferentPortsSuite struct {
29+
suite.Suite
30+
test e2e.Test
31+
client *e2e.Client
32+
tmplData tmplData
33+
}
34+
35+
type instanceTmplData struct {
36+
Port int
37+
Replicas int
38+
}
39+
40+
type tmplData struct {
41+
IngAnnotations []struct{ Key, Value string }
42+
Host string
43+
Instances []instanceTmplData
44+
}
45+
46+
func (suite *DifferentPortsSuite) SetupSuite() {
47+
var err error
48+
suite.test, err = e2e.NewTest()
49+
suite.Require().NoError(err)
50+
suite.tmplData = tmplData{
51+
Host: suite.test.GetNS() + ".test",
52+
Instances: []instanceTmplData{{Port: 8000, Replicas: 1}},
53+
}
54+
suite.client, err = e2e.NewHTTPClient(suite.tmplData.Host)
55+
suite.Require().NoError(err)
56+
suite.Require().NoError(suite.test.Apply("config/deploy.yaml.tmpl", suite.test.GetNS(), suite.tmplData))
57+
suite.Require().NoError(suite.test.Apply("config/ingress.yaml.tmpl", suite.test.GetNS(), suite.tmplData))
58+
suite.Require().Eventually(func() bool {
59+
res, cls, err := suite.client.Do()
60+
if res == nil {
61+
suite.T().Log(err)
62+
return false
63+
}
64+
defer cls()
65+
return res.StatusCode == http.StatusOK
66+
}, e2e.WaitDuration, e2e.TickDuration)
67+
}
68+
69+
func (suite *DifferentPortsSuite) TearDownSuite() {
70+
suite.test.TearDown()
71+
}
72+
73+
func TestDifferentPortsSuite(t *testing.T) {
74+
suite.Run(t, new(DifferentPortsSuite))
75+
}

‎pkg/haproxy/api/api.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ type HAProxyClient interface { //nolint:interfacebloat
9090
SetMapContent(mapFile string, payload []string) error
9191
SetServerAddrAndState([]RuntimeServerData) error
9292
SetAuxCfgFile(auxCfgFile string)
93-
SyncBackendSrvs(backend *store.RuntimeBackend, portUpdated bool) error
93+
SyncBackendSrvs(backend *store.RuntimeBackend) error
9494
UserListDeleteAll() error
9595
UserListExistsByGroup(group string) (bool, error)
9696
UserListCreateByGroup(group string, userPasswordMap map[string][]byte) error

‎pkg/haproxy/api/runtime.go‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -160,43 +160,45 @@ func (c *clientNative) GetMap(mapFile string) (*models.Map, error) {
160160
}
161161

162162
// SyncBackendSrvs syncs states and addresses of a backend servers with corresponding endpoints.
163-
func (c *clientNative) SyncBackendSrvs(backend *store.RuntimeBackend, portUpdated bool) error {
163+
func (c *clientNative) SyncBackendSrvs(backend *store.RuntimeBackend) error {
164164
logger := utils.GetLogger()
165165
if backend.Name == "" {
166166
return nil
167167
}
168168
logger.Tracef("[RUNTIME] [BACKEND] [SERVER] updating backend %s for haproxy servers update (address and state) through socket", backend.Name)
169169
haproxySrvs := backend.HAProxySrvs
170-
addresses := backend.Endpoints.Addresses
170+
endpoints := backend.Endpoints
171171
logger.Tracef("[RUNTIME] [BACKEND] [SERVER] backend %s: list of servers %+v", backend.Name, haproxySrvs)
172-
logger.Tracef("[RUNTIME] [BACKEND] [SERVER] backend %s: list of endpoints addresses %+v", backend.Name, addresses)
172+
logger.Tracef("[RUNTIME] [BACKEND] [SERVER] backend %s: list of endpoints %+v", backend.Name, endpoints)
173173
// Disable stale entries from HAProxySrvs
174174
// and provide list of Disabled Srvs
175175
var disabled []*store.HAProxySrv
176176
for i, srv := range haproxySrvs {
177-
srv.Modified = srv.Modified || portUpdated
178-
if _, ok := addresses[srv.Address]; ok {
179-
delete(addresses, srv.Address)
177+
srvEndpoint := store.RuntimeEndpoint{Address: srv.Address, Port: srv.Port}
178+
if _, ok := endpoints[srvEndpoint]; ok {
179+
delete(endpoints, srvEndpoint)
180180
} else {
181181
haproxySrvs[i].Address = ""
182+
haproxySrvs[i].Port = 1
182183
haproxySrvs[i].Modified = true
183184
disabled = append(disabled, srv)
184185
}
185186
}
186187

187-
// Configure new Addresses in available HAProxySrvs
188-
for newAddr := range addresses {
188+
// Configure new Endpoints in available HAProxySrvs
189+
for newEndpoint := range endpoints {
189190
if len(disabled) == 0 {
190191
break
191192
}
192-
disabled[0].Address = newAddr
193+
disabled[0].Address = newEndpoint.Address
194+
disabled[0].Port = newEndpoint.Port
193195
disabled[0].Modified = true
194196
disabled = disabled[1:]
195-
delete(addresses, newAddr)
197+
delete(endpoints, newEndpoint)
196198
}
197199

198200
logger.Tracef("[RUNTIME] [BACKEND] [SERVER] backend %s: list of servers after treatment %+v", backend.Name, haproxySrvs)
199-
logger.Tracef("[RUNTIME] [BACKEND] [SERVER] backend %s: list of endpoints addresses after treatment %+v", backend.Name, addresses)
201+
logger.Tracef("[RUNTIME] [BACKEND] [SERVER] backend %s: list of endpoints after treatment %+v", backend.Name, endpoints)
200202

201203
// Dynamically updates HAProxy backend servers with HAProxySrvs content
202204
runtimeServerData := make([]RuntimeServerData, 0, len(haproxySrvs))
@@ -219,7 +221,7 @@ func (c *clientNative) SyncBackendSrvs(backend *store.RuntimeBackend, portUpdate
219221
BackendName: backend.Name,
220222
ServerName: srv.Name,
221223
IP: srv.Address,
222-
Port: int(backend.Endpoints.Port),
224+
Port: int(srv.Port),
223225
State: "ready",
224226
})
225227
}

0 commit comments

Comments
 (0)