add application profile annotation support

Author: viveksyngh

pkg/ccm/loadbalancer.go

@@ -11,6 +11,9 @@ package ccm
 import (
 	"context"
 	"fmt"
+	"strconv"
+	"strings"
+
 	"github.com/vmware/cloud-provider-for-cloud-director/pkg/cpisdk"
 	"github.com/vmware/cloud-provider-for-cloud-director/pkg/util"
 	"github.com/vmware/cloud-provider-for-cloud-director/pkg/vcdsdk"
@@ -22,13 +25,12 @@ import (
 	"k8s.io/client-go/kubernetes"
 	cloudProvider "k8s.io/cloud-provider"
 	"k8s.io/klog"
-	"strconv"
-	"strings"
 )
 
 const (
 	sslPortsAnnotation              = `service.beta.kubernetes.io/vcloud-avi-ssl-ports`
 	sslCertAliasAnnotation          = `service.beta.kubernetes.io/vcloud-avi-ssl-cert-alias`
+	applicationProfileAnnotation    = `service.beta.kubernetes.io/vcloud-avi-application-profile`
 	skipAviSSLTerminationAnnotation = `service.beta.kubernetes.io/vcloud-avi-ssl-no-termination`
 	// TODO: Update controlPlaneLabel to use default K8s constants if available
 	controlPlaneLabel = `node-role.kubernetes.io/control-plane`
@@ -228,8 +230,9 @@ func (lb *LBManager) UpdateLoadBalancer(ctx context.Context, clusterName string,
 		klog.Infof("Updating pool [%s] with port [%s:%d]", lbPoolName, portName, internalPort)
 		protocol, _ := nameToProtocol[portName]
 		resourcesAllocated := &util.AllocatedResourcesMap{}
+		applicationProfile := getApplicationProfile(service)
 		vip, err := gm.UpdateLoadBalancer(ctx, lbPoolName, virtualServiceName, nodeIps, userSpecifiedLBIP, internalPort,
-			externalPort, lb.OneArm, lb.EnableVirtualServiceSharedIP, protocol, resourcesAllocated)
+			externalPort, lb.OneArm, lb.EnableVirtualServiceSharedIP, protocol, resourcesAllocated, applicationProfile)
 		// TODO: Should we record this error as well?
 		if rdeErr := lb.addLBResourcesToRDE(ctx, resourcesAllocated, vip); rdeErr != nil {
 			return fmt.Errorf("failed to add load balancer resources to RDE [%s]: [%v]", lb.clusterID, err)
@@ -504,6 +507,15 @@ func getSSLCertAlias(service *v1.Service) string {
 	return sslCertAlias
 }
 
+func getApplicationProfile(service *v1.Service) string {
+	applicationProfile, ok := service.Annotations[applicationProfileAnnotation]
+	if !ok {
+		return ""
+	}
+
+	return applicationProfile
+}
+
 func shouldSkipAviSSLTermination(service *v1.Service) bool {
 	shouldSkipAviSSLTerminationStr, ok := service.Annotations[skipAviSSLTerminationAnnotation]
 	if !ok {
@@ -559,6 +571,8 @@ func (lb *LBManager) createLoadBalancer(ctx context.Context, service *v1.Service
 	userSpecifiedLBIP := getUserSpecifiedLoadBalancerIP(service)
 	klog.Infof("createLoadBalancer called with loadBalancerIP [%s] for service [%s]", userSpecifiedLBIP, service.Name)
 
+	applicationProfile := getApplicationProfile(service)
+
 	if lbExists {
 		// Update load balancer if there are changes in service properties
 		typeToInternalPortMap, typeToExternalPortMap, nameToProtocol := lb.getServicePortMap(service)
@@ -570,7 +584,7 @@ func (lb *LBManager) createLoadBalancer(ctx context.Context, service *v1.Service
 			klog.Infof("Updating pool [%s] with port [%s:%d:%d]", lbPoolName, portName, internalPort, externalPort)
 			resourcesAllocated := &util.AllocatedResourcesMap{}
 			vip, err := gm.UpdateLoadBalancer(ctx, lbPoolName, virtualServiceName, nodeIPs, userSpecifiedLBIP, internalPort,
-				externalPort, lb.OneArm, lb.EnableVirtualServiceSharedIP, protocol, resourcesAllocated)
+				externalPort, lb.OneArm, lb.EnableVirtualServiceSharedIP, protocol, resourcesAllocated, applicationProfile)
 			if rdeErr := lb.addLBResourcesToRDE(ctx, resourcesAllocated, vip); rdeErr != nil {
 				return nil, fmt.Errorf("failed to update RDE [%s] with load balancer resources: [%v]", lb.clusterID, err)
 			}
@@ -676,7 +690,7 @@ func (lb *LBManager) createLoadBalancer(ctx context.Context, service *v1.Service
 	// Create using VCD API
 	resourcesAllocated := &util.AllocatedResourcesMap{}
 	lbIP, err := gm.CreateLoadBalancer(ctx, virtualServiceNamePrefix, lbPoolNamePrefix, nodeIPs, portDetailsList,
-		lb.OneArm, lb.EnableVirtualServiceSharedIP, portNameToIPMap, userSpecifiedLBIP, resourcesAllocated)
+		lb.OneArm, lb.EnableVirtualServiceSharedIP, portNameToIPMap, userSpecifiedLBIP, resourcesAllocated, applicationProfile)
 	if rdeErr := lb.addLBResourcesToRDE(ctx, resourcesAllocated, lbIP); rdeErr != nil {
 		return nil, fmt.Errorf("unable to add load balancer pool resources to RDE [%s]: [%v]", lb.clusterID, err)
 	}

pkg/vcdsdk/gateway.go

@@ -10,17 +10,18 @@ package vcdsdk
 import (
 	"context"
 	"fmt"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+
 	"github.com/antihax/optional"
 	"github.com/peterhellberg/link"
 	"github.com/vmware/cloud-provider-for-cloud-director/pkg/util"
 	swaggerClient "github.com/vmware/cloud-provider-for-cloud-director/pkg/vcdswaggerclient_36_0"
 	"github.com/vmware/go-vcloud-director/v2/govcd"
 	"github.com/vmware/go-vcloud-director/v2/types/v56"
 	"k8s.io/klog"
-	"net/http"
-	"net/url"
-	"strconv"
-	"strings"
 )
 
 type OneArm struct {
@@ -1104,7 +1105,7 @@ func (gatewayManager *GatewayManager) checkIfGatewayIsReady(ctx context.Context)
 }
 
 func (gatewayManager *GatewayManager) UpdateVirtualService(ctx context.Context, virtualServiceName string,
-	virtualServiceIP string, externalPort int32, oneArmEnabled bool) (*swaggerClient.EntityReference, error) {
+	virtualServiceIP string, externalPort int32, oneArmEnabled bool, applicationProfile string) (*swaggerClient.EntityReference, error) {
 	client := gatewayManager.Client
 	vsSummary, err := gatewayManager.GetVirtualService(ctx, virtualServiceName)
 	if err != nil {
@@ -1147,6 +1148,11 @@ func (gatewayManager *GatewayManager) UpdateVirtualService(ctx context.Context,
 		// update the virtual IP address of the virtual service when one arm is nil
 		vs.VirtualIpAddress = virtualServiceIP
 	}
+
+	if applicationProfile != "" && vs.ApplicationProfile.Name != applicationProfile {
+		vs.ApplicationProfile.Name = applicationProfile
+	}
+
 	resp, err := client.APIClient.EdgeGatewayLoadBalancerVirtualServiceApi.UpdateVirtualService(ctx, vs, vsSummary.Id, org.Org.ID)
 	if resp != nil && resp.StatusCode != http.StatusAccepted {
 		var responseMessageBytes []byte
@@ -1187,7 +1193,7 @@ func (gatewayManager *GatewayManager) UpdateVirtualService(ctx context.Context,
 func (gatewayManager *GatewayManager) CreateVirtualService(ctx context.Context, virtualServiceName string,
 	lbPoolRef *swaggerClient.EntityReference, segRef *swaggerClient.EntityReference,
 	freeIP string, vsType string, externalPort int32,
-	useSSL bool, certificateAlias string) (*swaggerClient.EntityReference, error) {
+	useSSL bool, certificateAlias, applicationProfile string) (*swaggerClient.EntityReference, error) {
 
 	client := gatewayManager.Client
 	if gatewayManager.GatewayRef == nil {
@@ -1231,9 +1237,7 @@ func (gatewayManager *GatewayManager) CreateVirtualService(ctx context.Context,
 				SslEnabled: useSSL,
 			},
 		},
-		ApplicationProfile: &swaggerClient.EdgeLoadBalancerApplicationProfile{
-			SystemDefined: true,
-		},
+		ApplicationProfile: &swaggerClient.EdgeLoadBalancerApplicationProfile{},
 	}
 	switch vsType {
 	case "TCP":
@@ -1259,6 +1263,10 @@ func (gatewayManager *GatewayManager) CreateVirtualService(ctx context.Context,
 		return nil, fmt.Errorf("unhandled virtual service type [%s]", vsType)
 	}
 
+	if applicationProfile != "" {
+		virtualServiceConfig.ApplicationProfile.Name = applicationProfile
+	}
+
 	clusterOrg, err := client.VCDClient.GetOrgByName(client.ClusterOrgName)
 	if err != nil {
 		return nil, fmt.Errorf("unable to get org for org [%s]: [%v]", client.ClusterOrgName, err)
@@ -1502,7 +1510,7 @@ func (gatewayManager *GatewayManager) GetLoadBalancerPoolMemberIPs(ctx context.C
 
 func (gm *GatewayManager) CreateLoadBalancer(ctx context.Context, virtualServiceNamePrefix string, lbPoolNamePrefix string,
 	ips []string, portDetailsList []PortDetails, oneArm *OneArm, enableVirtualServiceSharedIP bool,
-	portNameToIP map[string]string, providedIP string, resourcesAllocated *util.AllocatedResourcesMap) (string, error) {
+	portNameToIP map[string]string, providedIP string, resourcesAllocated *util.AllocatedResourcesMap, applicationProfile string) (string, error) {
 	if len(portDetailsList) == 0 {
 		// nothing to do here
 		klog.Infof("There is no port specified. Hence nothing to do.")
@@ -1707,7 +1715,7 @@ func (gm *GatewayManager) CreateLoadBalancer(ctx context.Context, virtualService
 
 		virtualServiceRef, err := gm.CreateVirtualService(ctx, virtualServiceName, lbPoolRef, segRef,
 			virtualServiceIP, portDetails.Protocol, portDetails.ExternalPort,
-			portDetails.UseSSL, portDetails.CertAlias)
+			portDetails.UseSSL, portDetails.CertAlias, applicationProfile)
 		if err != nil {
 			// return  plain error if vcdsdk.VirtualServicePendingError is returned. Helps the caller recognize that the
 			// error is because VirtualService is still in Pending state.
@@ -1833,7 +1841,7 @@ func (gm *GatewayManager) DeleteLoadBalancer(ctx context.Context, virtualService
 
 func (gm *GatewayManager) UpdateLoadBalancer(ctx context.Context, lbPoolName string, virtualServiceName string,
 	ips []string, externalIP string, internalPort int32, externalPort int32, oneArm *OneArm, enableVirtualServiceSharedIP bool, protocol string,
-	resourcesAllocated *util.AllocatedResourcesMap) (string, error) {
+	resourcesAllocated *util.AllocatedResourcesMap, applicationProfile string) (string, error) {
 
 	if gm == nil {
 		return "", fmt.Errorf("GatewayManager cannot be nil")
@@ -1852,7 +1860,7 @@ func (gm *GatewayManager) UpdateLoadBalancer(ctx context.Context, lbPoolName str
 		return "", fmt.Errorf("unable to update load balancer pool [%s]: [%v]", lbPoolName, err)
 	}
 	resourcesAllocated.Insert(VcdResourceLoadBalancerPool, lbPoolRef)
-	vsRef, err := gm.UpdateVirtualService(ctx, virtualServiceName, externalIP, externalPort, oneArm != nil)
+	vsRef, err := gm.UpdateVirtualService(ctx, virtualServiceName, externalIP, externalPort, oneArm != nil, applicationProfile)
 	if vsRef != nil {
 		resourcesAllocated.Insert(VcdResourceVirtualService, vsRef)
 	}