Skip to content

Commit f7bd1a7

Browse files
liangcdliangcd
authored
Merge pull request #129 from qiangzii/master
support config lb backend nodes count for service;
2 parents 5fa9556 + c41b6ab commit f7bd1a7

7 files changed

Lines changed: 189 additions & 35 deletions

File tree

‎docs/configure.md‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,13 @@ spec:
155155
- `Cluster`: 如果`service`中不显式指定 `externalTrafficPolicy` 字段的值，则默认为`Cluster`；这种模式下，可以通过给服务添加相关注解来指定LB监听器backend的添加规则
156156

157157
`Cluster`模式下，目前支持的 `service` 注解有：
158-
- 使用指定Label的Worker节点作为后端服务器， `service.beta.kubernetes.io/qingcloud-lb-backend-label`，可以指定多个Label，多个Label以逗号分隔。例如：`key1=value1,key2=value2`，多个Label之间是And关系。同时，在需要成为后端服务器的Worker节点打上`key1=value1,key2=value2`的Label；只有服务指定的所有Label的key和value都和Worker节点匹配时，Worker节点会被选为服务的后端服务器；没有此注解则添加所有Worker节点为backend；通过注解过滤节点后，如果没有满足条件的节点，为了避免服务中断，会添加所有Worker节点为后端服务器；
158+
> 以下各种过滤节点的方式不能结合使用，如果指定了多个，则会按照以下注解的说明顺序选用第一个匹配到的注解，并按照该方法过滤节点；
159+
- 使用指定Label的Worker节点作为后端服务器， `service.beta.kubernetes.io/qingcloud-lb-backend-label`，可以指定多个Label，多个Label以逗号分隔。例如：`key1=value1,key2=value2`，多个Label之间是And关系。同时，在需要成为后端服务器的Worker节点打上`key1=value1,key2=value2`的Label；只有服务指定的所有Label的key和value都和Worker节点匹配时，Worker节点会被选为服务的后端服务器；特殊情况说明：
160+
- 没有此注解则添加所有Worker节点为backend；
161+
- 通过注解过滤节点后，如果没有满足条件的节点，为了避免服务中断，会添加所有Worker节点为后端服务器；
162+
- 使用指定数量的Worker节点作为后端服务器，`service.beta.kubernetes.io/qingcloud-lb-backend-count`，通过此注解指定该服务使用的lb backend节点数量；如果服务添加了该注解，就认为用户想使用该特性；如果指定的数量为0或者不在可用节点数量的范围内，则使用默认值：集群所有节点的1/3；如果集群中节点状态发生变化，但是当前服务的lb后端数量就是用户指定的数量，并且已添加的所有lb后端节点在集群中都是ready状态的，则不会更新lb的backend；默认值特殊情况说明：
163+
- 如果集群总worker节点数少于3个，则添加所有worker节点为backend，不再按照比例计算节点数；
164+
- 如果集群总worker节点数多于3个，若按照比例计算后少于3个，则设置为3个;
159165

160166
> 本章节所说的"所有Worker节点"特指所有 `Ready` 状态的Worker节点；
161167

@@ -184,6 +190,7 @@ spec:
184190
```
185191

186192
#### Cluster模式
193+
##### 使用指定Label的Worker节点作为后端服务器
187194
将服务的`externalTrafficPolicy`指定为`Cluster`，并在服务的注解`service.beta.kubernetes.io/qingcloud-lb-backend-label`中指定要添加为backend的worker节点的label:
188195

189196
```yaml
@@ -232,6 +239,28 @@ status:
232239
...
233240
```
234241

242+
##### 使用指定数量的Worker节点作为后端服务器
243+
244+
```yaml
245+
kind: Service
246+
apiVersion: v1
247+
metadata:
248+
name: reuse-lb
249+
annotations:
250+
service.beta.kubernetes.io/qingcloud-load-balancer-eip-strategy: "reuse-lb"
251+
service.beta.kubernetes.io/qingcloud-load-balancer-id: "lb-oglqftju"
252+
service.beta.kubernetes.io/qingcloud-lb-backend-count: "3"
253+
spec:
254+
externalTrafficPolicy: Cluster
255+
selector:
256+
app: mylbapp
257+
type: LoadBalancer
258+
ports:
259+
- name: http
260+
port: 8090
261+
protocol: TCP
262+
targetPort: 80
263+
```
235264

236265
## 配置内网负载均衡器
237266
### 已知问题

‎pkg/qingcloud/annotations.go‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,10 +80,14 @@ const (
8080
ServiceAnnotationListenerServerCertificate = "service.beta.kubernetes.io/qingcloud-lb-listener-cert"
8181
// port:protocol, such as "443:https,80:http"
8282
ServiceAnnotationListenerProtocol = "service.beta.kubernetes.io/qingcloud-lb-listener-protocol"
83+
// port:timeout, such as "443:50", the value must in range 10 ～ 86400
84+
ServiceAnnotationListenerTimeout = "service.beta.kubernetes.io/qingcloud-lb-listener-timeout"
8385

8486
// 5. Configure backend
8587
// backend label, such as "key1=value1,key2=value2"
8688
ServiceAnnotationBackendLabel = "service.beta.kubernetes.io/qingcloud-lb-backend-label"
89+
// backend count limit, if value is 0 or greater than cluster ready worker, will use default value : 1/3 of cluster ready worker
90+
ServiceAnnotationBackendCount = "service.beta.kubernetes.io/qingcloud-lb-backend-count"
8791
)
8892

8993
type LoadBalancerConfig struct {
@@ -105,7 +109,9 @@ type LoadBalancerConfig struct {
105109
Protocol *string
106110

107111
//backend
108-
BackendLabel string
112+
BackendLabel string
113+
BackendCountConfig string
114+
BackendCountResult int
109115

110116
//It's just for defining names, nothing more.
111117
NetworkType string
@@ -171,6 +177,13 @@ func (qc *QingCloud) ParseServiceLBConfig(cluster string, service *v1.Service) (
171177
if backendLabel, ok := annotation[ServiceAnnotationBackendLabel]; ok {
172178
config.BackendLabel = backendLabel
173179
}
180+
if backendCount, ok := annotation[ServiceAnnotationBackendCount]; ok {
181+
_, err := strconv.Atoi(backendCount)
182+
if err != nil {
183+
return nil, fmt.Errorf("please spec a valid value of loadBalancer backend count")
184+
}
185+
config.BackendCountConfig = backendCount
186+
}
174187

175188
networkType := annotation[ServiceAnnotationLoadBalancerNetworkType]
176189
if config.VxNetID == nil && qc.Config.DefaultVxNetForLB != "" {

‎pkg/qingcloud/loadbalancer_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -400,7 +400,7 @@ func TestDiffListeners(t *testing.T) {
400400
}
401401

402402
for _, tc := range testCases {
403-
toDelete, toAdd := diffListeners(tc.listeners, tc.conf, tc.ports)
403+
toDelete, toAdd, _ := diffListeners(tc.listeners, tc.conf, tc.ports)
404404
// fmt.Printf("delete=%s, add=%s", spew.Sdump(toDelete), spew.Sdump(toAdd))
405405
if !reflect.DeepEqual(toDelete, tc.toDelete) || !reflect.DeepEqual(toAdd, tc.toAdd) {
406406
t.Fail()

‎pkg/qingcloud/loadbalancer_utils.go‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
package qingcloud
22

33
import (
4+
"crypto/rand"
45
"fmt"
6+
"math/big"
57
"strconv"
68
"strings"
79

@@ -152,7 +154,7 @@ func getProtocol(annotationConf map[int]string, port int) *string {
152154
}
153155
}
154156

155-
func diffListeners(listeners []*apis.LoadBalancerListener, conf *LoadBalancerConfig, ports []v1.ServicePort) (toDelete []*string, toAdd []v1.ServicePort) {
157+
func diffListeners(listeners []*apis.LoadBalancerListener, conf *LoadBalancerConfig, ports []v1.ServicePort) (toDelete []*string, toAdd []v1.ServicePort, toKeep []*apis.LoadBalancerListener) {
156158
svcNodePort := make(map[string]int)
157159
for _, listener := range listeners {
158160
if len(listener.Status.LoadBalancerBackends) > 0 {
@@ -197,6 +199,8 @@ func diffListeners(listeners []*apis.LoadBalancerListener, conf *LoadBalancerCon
197199
}
198200
if delete {
199201
toDelete = append(toDelete, listener.Status.LoadBalancerListenerID)
202+
} else {
203+
toKeep = append(toKeep, listener)
200204
}
201205
}
202206

@@ -521,3 +525,18 @@ func equalProtocol(listener *apis.LoadBalancerListener, conf *LoadBalancerConfig
521525
}
522526
return false
523527
}
528+
529+
func getRandomNodes(nodes []*v1.Node, count int) (result []*v1.Node) {
530+
resultMap := make(map[int64]bool)
531+
length := int64(len(nodes))
532+
533+
for i := 0; i < count; {
534+
r, _ := rand.Int(rand.Reader, big.NewInt(length))
535+
if !resultMap[r.Int64()] {
536+
result = append(result, nodes[r.Int64()])
537+
resultMap[r.Int64()] = true
538+
i++
539+
}
540+
}
541+
return
542+
}

‎pkg/qingcloud/qingcloud.go‎

Lines changed: 68 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"context"
1212
"fmt"
1313
"io"
14+
"strconv"
1415

1516
"github.com/davecgh/go-spew/spew"
1617
yaml "gopkg.in/yaml.v2"
@@ -27,8 +28,9 @@ import (
2728
)
2829

2930
const (
30-
ProviderName = "qingcloud"
31-
QYConfigPath = "/etc/qingcloud/config.yaml"
31+
ProviderName = "qingcloud"
32+
QYConfigPath = "/etc/qingcloud/config.yaml"
33+
DefaultBackendCount = 3
3234
)
3335

3436
type Config struct {
@@ -232,7 +234,7 @@ func (qc *QingCloud) EnsureLoadBalancer(ctx context.Context, _ string, service *
232234
klog.Infof("The loadbalancer %s has the following listeners %s", *lb.Status.LoadBalancerID, spew.Sdump(listenerIDs))
233235
if len(listenerIDs) <= 0 {
234236
klog.Infof("creating listeners for loadbalancers %s, service ports %s", *lb.Status.LoadBalancerID, spew.Sdump(service.Spec.Ports))
235-
if err = qc.createListenersAndBackends(conf, lb, service.Spec.Ports, nodes); err != nil {
237+
if err = qc.createListenersAndBackends(conf, lb, service.Spec.Ports, nodes, service); err != nil {
236238
klog.Errorf("createListenersAndBackends for loadbalancer %s error: %v", *lb.Status.LoadBalancerID, err)
237239
return nil, err
238240
}
@@ -244,7 +246,7 @@ func (qc *QingCloud) EnsureLoadBalancer(ctx context.Context, _ string, service *
244246
}
245247

246248
//update listerner
247-
toDelete, toAdd := diffListeners(listeners, conf, service.Spec.Ports)
249+
toDelete, toAdd, toKeep := diffListeners(listeners, conf, service.Spec.Ports)
248250
if len(toDelete) > 0 {
249251
klog.Infof("listeners %s will be deleted for lb %s", spew.Sdump(toDelete), *lb.Status.LoadBalancerID)
250252
err = qc.Client.DeleteListener(toDelete)
@@ -256,38 +258,37 @@ func (qc *QingCloud) EnsureLoadBalancer(ctx context.Context, _ string, service *
256258

257259
if len(toAdd) > 0 {
258260
klog.Infof("listeners %s will be added for lb %s", spew.Sdump(toAdd), *lb.Status.LoadBalancerID)
259-
err = qc.createListenersAndBackends(conf, lb, toAdd, nodes)
261+
err = qc.createListenersAndBackends(conf, lb, toAdd, nodes, service)
260262
if err != nil {
261263
return nil, err
262264
}
263265
modify = true
264266
}
265267

266268
//update backend; for example, service annotation for backend label changed
267-
if len(toAdd) == 0 && len(toDelete) == 0 {
268-
for _, listener := range listeners {
269-
toDelete, toAdd := diffBackend(listener, nodes)
270-
271-
if len(toDelete) > 0 {
272-
klog.Infof("backends %s will be deleted for listener %s(%s) of lb %s",
273-
spew.Sdump(toDelete), *listener.Spec.LoadBalancerListenerName, *listener.Spec.LoadBalancerListenerID, *lb.Status.LoadBalancerID)
274-
err = qc.Client.DeleteBackends(toDelete)
275-
if err != nil {
276-
return nil, err
277-
}
278-
modify = true
269+
for _, listener := range toKeep {
270+
// toDelete, toAdd := diffBackend(listener, nodes)
271+
toDelete, toAdd := qc.diffBackend(listener, nodes, conf, service)
272+
273+
if len(toDelete) > 0 {
274+
klog.Infof("backends %s will be deleted for listener %s(%s) of lb %s",
275+
spew.Sdump(toDelete), *listener.Spec.LoadBalancerListenerName, *listener.Spec.LoadBalancerListenerID, *lb.Status.LoadBalancerID)
276+
err = qc.Client.DeleteBackends(toDelete)
277+
if err != nil {
278+
return nil, err
279279
}
280+
modify = true
281+
}
280282

281-
toAddBackends := generateLoadBalancerBackends(toAdd, listener, service.Spec.Ports)
282-
if len(toAddBackends) > 0 {
283-
klog.Infof("backends %s will be added for listener %s(%s) of lb %s",
284-
spew.Sdump(toAddBackends), *listener.Spec.LoadBalancerListenerName, *listener.Spec.LoadBalancerListenerID, *lb.Status.LoadBalancerID)
285-
_, err = qc.Client.CreateBackends(toAddBackends)
286-
if err != nil {
287-
return nil, err
288-
}
289-
modify = true
283+
toAddBackends := generateLoadBalancerBackends(toAdd, listener, service.Spec.Ports)
284+
if len(toAddBackends) > 0 {
285+
klog.Infof("backends %s will be added for listener %s(%s) of lb %s",
286+
spew.Sdump(toAddBackends), *listener.Spec.LoadBalancerListenerName, *listener.Spec.LoadBalancerListenerID, *lb.Status.LoadBalancerID)
287+
_, err = qc.Client.CreateBackends(toAddBackends)
288+
if err != nil {
289+
return nil, err
290290
}
291+
modify = true
291292
}
292293
}
293294

@@ -316,6 +317,7 @@ func (qc *QingCloud) EnsureLoadBalancer(ctx context.Context, _ string, service *
316317
//1.2 prepare sg
317318
//default sg set by Client auto
318319
//1.3 create lb
320+
klog.Infof("creating loadbalance for service %s/%s", service.Namespace, service.Name)
319321
lb, err = qc.Client.CreateLB(&apis.LoadBalancer{
320322
Spec: apis.LoadBalancerSpec{
321323
LoadBalancerName: &conf.LoadBalancerName,
@@ -331,7 +333,7 @@ func (qc *QingCloud) EnsureLoadBalancer(ctx context.Context, _ string, service *
331333
}
332334

333335
//create listener
334-
if err = qc.createListenersAndBackends(conf, lb, service.Spec.Ports, nodes); err != nil {
336+
if err = qc.createListenersAndBackends(conf, lb, service.Spec.Ports, nodes, service); err != nil {
335337
return nil, err
336338
}
337339
} else {
@@ -382,7 +384,8 @@ func (qc *QingCloud) UpdateLoadBalancer(ctx context.Context, _ string, service *
382384
}
383385

384386
for _, listener := range listeners {
385-
toDelete, toAdd := diffBackend(listener, nodes)
387+
// toDelete, toAdd := diffBackend(listener, nodes)
388+
toDelete, toAdd := qc.diffBackend(listener, nodes, conf, service)
386389

387390
if len(toDelete) > 0 {
388391
klog.Infof("backends %s will be deleted for listener %s(%s) of lb %s",
@@ -414,7 +417,7 @@ func (qc *QingCloud) UpdateLoadBalancer(ctx context.Context, _ string, service *
414417
return qc.Client.UpdateLB(lb.Status.LoadBalancerID)
415418
}
416419

417-
func (qc *QingCloud) createListenersAndBackends(conf *LoadBalancerConfig, status *apis.LoadBalancer, ports []v1.ServicePort, nodes []*v1.Node) error {
420+
func (qc *QingCloud) createListenersAndBackends(conf *LoadBalancerConfig, status *apis.LoadBalancer, ports []v1.ServicePort, nodes []*v1.Node, svc *v1.Service) error {
418421
listeners, err := generateLoadBalancerListeners(conf, status, ports)
419422
if err != nil {
420423
klog.Errorf("generateLoadBalancerListeners for loadbalancer %s error: %v", *status.Status.LoadBalancerID, err)
@@ -426,7 +429,19 @@ func (qc *QingCloud) createListenersAndBackends(conf *LoadBalancerConfig, status
426429
return err
427430
}
428431

429-
//create backend
432+
// filter backend nodes by count
433+
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster && conf.BackendCountConfig != "" {
434+
klog.Infof("service %s/%s has lb backend count annotation, try to get %d random nodes as backend", svc.Namespace, svc.Name, conf.BackendCountResult)
435+
nodes = getRandomNodes(nodes, conf.BackendCountResult)
436+
437+
var resultNames []string
438+
for _, node := range nodes {
439+
resultNames = append(resultNames, node.Name)
440+
}
441+
klog.Infof("get random nodes result for service %s/%s: %v", svc.Namespace, svc.Name, resultNames)
442+
}
443+
444+
// create backend
430445
for _, listener := range listeners {
431446
backends := generateLoadBalancerBackends(nodes, listener, ports)
432447
_, err = qc.Client.CreateBackends(backends)
@@ -515,7 +530,7 @@ func (qc *QingCloud) filterNodes(ctx context.Context, svc *v1.Service, nodes []*
515530
}
516531
}
517532
} else {
518-
if lbconfog.BackendLabel != "" {
533+
if lbconfog.BackendLabel != "" { // filter by node label
519534
klog.Infof("filter nodes for service %s/%s by backend label: %s", svc.Namespace, svc.Name, lbconfog.BackendLabel)
520535

521536
// filter by label
@@ -543,6 +558,28 @@ func (qc *QingCloud) filterNodes(ctx context.Context, svc *v1.Service, nodes []*
543558
klog.Infof("there are no available nodes for service %s/%s, use all nodes!", svc.Namespace, svc.Name)
544559
newNodes = nodes
545560
}
561+
// clear lb backend count config
562+
lbconfog.BackendCountConfig = ""
563+
} else if lbconfog.BackendCountConfig != "" { //filter by backend count config
564+
var backendCountResult int
565+
566+
backendCountConfig, _ := strconv.Atoi(lbconfog.BackendCountConfig)
567+
if backendCountConfig > 0 && backendCountConfig <= len(nodes) {
568+
backendCountResult = backendCountConfig
569+
} else {
570+
//invalid count config, use default value (1/3 of all nodes)
571+
if len(nodes) <= 3 {
572+
backendCountResult = len(nodes)
573+
} else {
574+
backendCountResult = len(nodes) / 3
575+
if backendCountResult < 3 {
576+
backendCountResult = DefaultBackendCount
577+
}
578+
}
579+
}
580+
581+
lbconfog.BackendCountResult = backendCountResult
582+
newNodes = nodes
546583
} else {
547584
// no need to filter
548585
newNodes = nodes

‎pkg/qingcloud/qingcloud_utils.go‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,11 @@ import (
44
"fmt"
55

66
"github.com/davecgh/go-spew/spew"
7+
v1 "k8s.io/api/core/v1"
78
"k8s.io/klog/v2"
89

910
"github.com/yunify/qingcloud-cloud-controller-manager/pkg/apis"
11+
"github.com/yunify/qingcloud-cloud-controller-manager/pkg/util"
1012
)
1113

1214
func (qc *QingCloud) prepareEip(eipSource *string) (eip *apis.EIP, err error) {
@@ -143,3 +145,40 @@ func (qc *QingCloud) updateLBEip(config *LoadBalancerConfig, lb *apis.LoadBalanc
143145

144146
return nil
145147
}
148+
149+
func (qc *QingCloud) diffBackend(listener *apis.LoadBalancerListener, nodes []*v1.Node, conf *LoadBalancerConfig, svc *v1.Service) (toDelete []*string, toAdd []*v1.Node) {
150+
var backendLeftID []*string
151+
for _, backend := range listener.Status.LoadBalancerBackends {
152+
if !nodesHasBackend(*backend.Spec.LoadBalancerBackendName, nodes) {
153+
toDelete = append(toDelete, backend.Status.LoadBalancerBackendID)
154+
} else {
155+
backendLeftID = append(backendLeftID, backend.Status.LoadBalancerBackendID)
156+
}
157+
}
158+
159+
// filter backend nodes by count
160+
if svc.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeCluster && conf.BackendCountConfig != "" {
161+
backendLeftCount := len(listener.Status.LoadBalancerBackends) - len(toDelete)
162+
if backendLeftCount > conf.BackendCountResult {
163+
// delete some
164+
toDelete = append(toDelete, util.GetRandomItems(backendLeftID, backendLeftCount-conf.BackendCountResult)...)
165+
} else {
166+
// add some
167+
var nodeLeft []*v1.Node
168+
for _, node := range nodes {
169+
if !backendsHasNode(node, listener.Status.LoadBalancerBackends) {
170+
nodeLeft = append(nodeLeft, node)
171+
}
172+
}
173+
toAdd = append(toAdd, getRandomNodes(nodeLeft, conf.BackendCountResult-backendLeftCount)...)
174+
}
175+
} else {
176+
for _, node := range nodes {
177+
if !backendsHasNode(node, listener.Status.LoadBalancerBackends) {
178+
toAdd = append(toAdd, node)
179+
}
180+
}
181+
}
182+
183+
return
184+
}

0 commit comments

Comments
 (0)