tg-node-annotation-patch

Signed-off-by: Ilya Lisitsyn <ilya.lisitsyn@flant.com>

Author: ilia601

.gitignore

@@ -1,3 +1,4 @@
+.DS_Store
 # Program binaries
 *.exe
 *.exe~

README.md

@@ -129,6 +129,11 @@ Due to API limitations, only one subnet from each zone must be present in each N
 * `yandex.cpi.flant.com/listener-subnet-id` – default SubnetID to use for Listeners in created NetworkLoadBalancers. NetworkLoadBalancers will be INTERNAL.
 * `yandex.cpi.flant.com/listener-address-ipv4` – select pre-defined IPv4 address. Works both on internal and external NetworkLoadBalancers.
 * `yandex.cpi.flant.com/loadbalancer-external` – override `YANDEX_CLOUD_DEFAULT_LB_LISTENER_SUBNET_ID` per-service.
+* `yandex.cpi.flant.com/target-group-name-prefix` - set target group for LB to target group with name `yandex.cpi.flant.com/target-group-name-prefix` annotation value + yandex cluster name + `YANDEX_CLOUD_DEFAULT_LB_TARGET_GROUP_NETWORK_ID`.
+
+##### Node annotations
+
+* `yandex.cpi.flant.com/target-group-name-prefix` - set node to the non-default target group add this annotation to the node.  Yandex CCM creates new target groups with name `yandex.cpi.flant.com/target-group-name-prefix` annotation value + yandex cluster name + network id of instance interfaces.
 
 ## Warning
 

pkg/cloudprovider/yandex/load_balancer.go

@@ -13,10 +13,12 @@ import (
 )
 
 const (
-	targetGroupNetworkIdAnnotation = "yandex.cpi.flant.com/target-group-network-id"
-	externalLoadBalancerAnnotation = "yandex.cpi.flant.com/loadbalancer-external"
-	listenerSubnetIdAnnotation     = "yandex.cpi.flant.com/listener-subnet-id"
-	listenerAddressIPv4            = "yandex.cpi.flant.com/listener-address-ipv4"
+	// node annotation to put node to the specific target group
+	customTargetGroupNamePrefixAnnotation = "yandex.cpi.flant.com/target-group-name-prefix"
+	targetGroupNetworkIdAnnotation        = "yandex.cpi.flant.com/target-group-network-id"
+	externalLoadBalancerAnnotation        = "yandex.cpi.flant.com/loadbalancer-external"
+	listenerSubnetIdAnnotation            = "yandex.cpi.flant.com/listener-subnet-id"
+	listenerAddressIPv4                   = "yandex.cpi.flant.com/listener-address-ipv4"
 
 	nodesHealthCheckPath = "/healthz"
 	// NOTE: Please keep the following port in sync with ProxyHealthzPort in pkg/cluster/ports/ports.go
@@ -178,7 +180,8 @@ func (yc *Cloud) ensureLB(ctx context.Context, service *v1.Service, nodes []*v1.
 		},
 	}
 
-	tgName := yc.config.ClusterName + lbParams.targetGroupNetworkID
+	tgName := lbParams.targetGroupNamePrefix + yc.config.ClusterName + lbParams.targetGroupNetworkID
+
 	tg, err := yc.yandexService.LbSvc.GetTgByName(ctx, tgName)
 	if err != nil {
 		return nil, err
@@ -201,10 +204,11 @@ func (yc *Cloud) ensureLB(ctx context.Context, service *v1.Service, nodes []*v1.
 }
 
 type loadBalancerParameters struct {
-	targetGroupNetworkID string
-	listenerSubnetID     string
-	listenerAddressIPv4  string
-	internal             bool
+	targetGroupNetworkID  string
+	targetGroupNamePrefix string
+	listenerSubnetID      string
+	listenerAddressIPv4   string
+	internal              bool
 }
 
 func (yc *Cloud) getLoadBalancerParameters(svc *v1.Service) (lbParams loadBalancerParameters) {
@@ -227,5 +231,9 @@ func (yc *Cloud) getLoadBalancerParameters(svc *v1.Service) (lbParams loadBalanc
 		lbParams.listenerAddressIPv4 = value
 	}
 
+	if value, ok := svc.ObjectMeta.Annotations[customTargetGroupNamePrefixAnnotation]; ok {
+		lbParams.targetGroupNamePrefix = value
+	}
+
 	return
 }

pkg/cloudprovider/yandex/load_balancer_tg_controller.go

@@ -64,7 +64,7 @@ func (ntgs *NodeTargetGroupSyncer) SyncTGs(ctx context.Context, nodes []*corev1.
 	return nil
 }
 
-type networkIdToTargetMap map[string][]*loadbalancer.Target
+type tgNameToTargetMap map[string][]*loadbalancer.Target
 
 func fromNodeToInterfaceSlice(nodes []*corev1.Node) (ret []interface{}) {
 	for _, node := range nodes {
@@ -97,6 +97,11 @@ func (ntgs *NodeTargetGroupSyncer) cleanUpTargetGroups(ctx context.Context) erro
 	return nil
 }
 
+type instanceWithNodeInfo struct {
+	Instance *compute.Instance
+	Node     *corev1.Node
+}
+
 func (ntgs *NodeTargetGroupSyncer) synchronizeNodesWithTargetGroups(ctx context.Context, nodes []*corev1.Node) error {
 	if len(nodes) == 0 {
 		klog.Info("no nodes to synchronize TGs with, skipping...")
@@ -109,7 +114,7 @@ func (ntgs *NodeTargetGroupSyncer) synchronizeNodesWithTargetGroups(ctx context.
 	}
 
 	// TODO: speed up by not performing individual lookups
-	var instances []*compute.Instance
+	var instances []*instanceWithNodeInfo
 	for _, node := range nodes {
 		nodeName := MapNodeNameToInstanceName(types.NodeName(node.Name))
 		log.Printf("Finding Instance by Folder %q and Name %q", ntgs.cloud.config.FolderID, nodeName)
@@ -118,16 +123,16 @@ func (ntgs *NodeTargetGroupSyncer) synchronizeNodesWithTargetGroups(ctx context.
 			return fmt.Errorf("failed to find Instance by its name: %s", err)
 		}
 
-		instances = append(instances, instance)
+		instances = append(instances, &instanceWithNodeInfo{Instance: instance, Node: node})
 	}
 
-	mapping, err := ntgs.constructNetworkIdToTargetMap(ctx, instances)
+	mapping, err := ntgs.constructTgNameToTargetMap(ctx, instances)
 	if err != nil {
-		return fmt.Errorf("failed to construct NetworkIdToTargetMap: %s", err)
+		return fmt.Errorf("failed to construct tgNameToTargetMap: %s", err)
 	}
 
-	for networkID, targets := range mapping {
-		_, err := ntgs.cloud.yandexService.LbSvc.CreateOrUpdateTG(ctx, ntgs.cloud.config.ClusterName+networkID, targets)
+	for tgName, targets := range mapping {
+		_, err := ntgs.cloud.yandexService.LbSvc.CreateOrUpdateTG(ctx, tgName, targets)
 		if err != nil {
 			return err
 		}
@@ -138,18 +143,22 @@ func (ntgs *NodeTargetGroupSyncer) synchronizeNodesWithTargetGroups(ctx context.
 	return nil
 }
 
-func (ntgs *NodeTargetGroupSyncer) constructNetworkIdToTargetMap(ctx context.Context, instances []*compute.Instance) (networkIdToTargetMap, error) {
-	mapping := make(networkIdToTargetMap)
+func (ntgs *NodeTargetGroupSyncer) constructTgNameToTargetMap(ctx context.Context, instances []*instanceWithNodeInfo) (tgNameToTargetMap, error) {
+	mapping := make(tgNameToTargetMap)
 
 	// TODO: Implement simple caching mechanism for subnet-VPC membership lookups
 	for _, instance := range instances {
-		for _, iface := range instance.NetworkInterfaces {
+		for _, iface := range instance.Instance.NetworkInterfaces {
 			subnetInfo, err := ntgs.cloud.yandexService.VPCSvc.SubnetSvc.Get(ctx, &vpc.GetSubnetRequest{SubnetId: iface.SubnetId})
 			if err != nil {
 				return nil, errors.WithStack(err)
 			}
 
-			mapping[subnetInfo.NetworkId] = append(mapping[subnetInfo.NetworkId], &loadbalancer.Target{
+			key := ntgs.cloud.config.ClusterName + subnetInfo.NetworkId
+			if v, ok := instance.Node.Annotations[customTargetGroupNamePrefixAnnotation]; ok {
+				key = truncateAnnotationValue(v) + key
+			}
+			mapping[key] = append(mapping[key], &loadbalancer.Target{
 				SubnetId: iface.SubnetId,
 				Address:  iface.PrimaryV4Address.Address,
 			})
@@ -162,3 +171,12 @@ func (ntgs *NodeTargetGroupSyncer) constructNetworkIdToTargetMap(ctx context.Con
 
 	return mapping, nil
 }
+
+func truncateAnnotationValue(value string) string {
+	// maximum length of annotation values should not exceed 63 - length of cluster uuid(26 symbols) - length of network id(21)
+	if len(value) > 36 {
+		log.Printf("annotation '%s' length should be less than 36 characters, truncate it", value)
+		value = value[:36]
+	}
+	return value
+}

pkg/yapi/loadbalancer.go

@@ -197,7 +197,7 @@ func (ySvc *LoadBalancerService) GetTGsByClusterName(ctx context.Context, cluste
 	}
 
 	for _, tg := range result.TargetGroups {
-		if strings.HasPrefix(tg.Name, clusterName) {
+		if strings.Contains(tg.Name, clusterName) {
 			ret = append(ret, tg)
 		}
 	}