Cleanup placement during decomissioning

Usually, there should be no resources on the host
left after having migrated all machines.
But ocassionally, we are still left with some
empty resources.
This change removes all empty resources
before removing the resource-provider as well.

Author: fwiesel

internal/controller/decomission_controller.go

@@ -35,6 +35,7 @@ import (
 	"github.com/gophercloud/gophercloud/v2"
 	"github.com/gophercloud/gophercloud/v2/openstack/compute/v2/hypervisors"
 	"github.com/gophercloud/gophercloud/v2/openstack/compute/v2/services"
+	"github.com/gophercloud/gophercloud/v2/openstack/placement/v1/resourceproviders"
 )
 
 const (
@@ -43,8 +44,9 @@ const (
 
 type NodeDecommissionReconciler struct {
 	k8sclient.Client
-	Scheme        *runtime.Scheme
-	computeClient *gophercloud.ServiceClient
+	Scheme          *runtime.Scheme
+	computeClient   *gophercloud.ServiceClient
+	placementClient *gophercloud.ServiceClient
 }
 
 // The counter-side in gardener is here:
@@ -126,11 +128,21 @@ func (r *NodeDecommissionReconciler) shutdownService(ctx context.Context, node *
 
 	// Deleting and evicted, so better delete the service
 	err = services.Delete(ctx, r.computeClient, hypervisor.Service.ID).ExtractErr()
-	if err == nil || gophercloud.ResponseCodeIs(err, http.StatusNotFound) {
-		return r.removeFinalizer(ctx, node)
+	if err != nil && !gophercloud.ResponseCodeIs(err, http.StatusNotFound) {
+		return ctrl.Result{}, fmt.Errorf("cannot delete service due to %w", err)
+	}
+
+	rp, err := resourceproviders.Get(ctx, r.placementClient, hypervisorID).Extract()
+	if err != nil && !gophercloud.ResponseCodeIs(err, http.StatusNotFound) {
+		return ctrl.Result{}, fmt.Errorf("cannot get resource provider due to %w", err)
+	}
+
+	err = openstack.CleanupResourceProvider(ctx, r.placementClient, rp)
+	if err != nil {
+		return ctrl.Result{}, fmt.Errorf("cannot get resource provider due to %w", err)
 	}
 
-	return ctrl.Result{}, err
+	return r.removeFinalizer(ctx, node)
 }
 
 func (r *NodeDecommissionReconciler) removeFinalizer(ctx context.Context, node *corev1.Node) (ctrl.Result, error) {
@@ -153,6 +165,12 @@ func (r *NodeDecommissionReconciler) SetupWithManager(mgr ctrl.Manager) error {
 
 	r.computeClient.Microversion = "2.93"
 
+	r.placementClient, err = openstack.GetServiceClient(ctx, "placement")
+	if err != nil {
+		return err
+	}
+	r.placementClient.Microversion = "1.39" // yoga, or later
+
 	return ctrl.NewControllerManagedBy(mgr).
 		Named("nodeDecommission").
 		For(&corev1.Node{}).

internal/openstack/placement.go

@@ -19,8 +19,10 @@ package openstack
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/gophercloud/gophercloud/v2"
+	"github.com/gophercloud/gophercloud/v2/openstack/placement/v1/resourceproviders"
 )
 
 // UpdateTraitsResult is the response of a Put traits operations. Call its Extract method
@@ -67,3 +69,93 @@ func UpdateTraits(ctx context.Context, client *gophercloud.ServiceClient, resour
 	_, r.Header, r.Err = gophercloud.ParseResponse(resp, err)
 	return
 }
+
+func getAllocationsURL(client *gophercloud.ServiceClient, consumerID string) string {
+	return client.ServiceURL("allocations", consumerID)
+}
+
+// ListAllocationsResult is the response of a Get allocations operations. Call its Extract method
+// to interpret it as a Allocations.
+type ListAllocationsResult struct {
+	gophercloud.Result
+}
+
+type ConsumerAllocations struct {
+	Allocations        map[string]resourceproviders.Allocation `json:"allocations"`
+	ConsumerGeneration int                                     `json:"consumer_generation"`
+	ProjectID          string                                  `json:"project_id"`
+	UserID             string                                  `json:"user_id"`
+	ConsumerType       string                                  `json:"consumer_type"`
+}
+
+// Extract interprets a ListAllocationsResult as a Allocations.
+func (r ListAllocationsResult) Extract() (*ConsumerAllocations, error) {
+	var s ConsumerAllocations
+	err := r.ExtractInto(&s)
+	return &s, err
+}
+
+// List Allocations for a certain consumer
+func ListAllocations(ctx context.Context, client *gophercloud.ServiceClient, consumerID string) (r ListAllocationsResult) {
+	resp, err := client.Get(ctx, getAllocationsURL(client, consumerID), nil, &gophercloud.RequestOpts{
+		OkCodes: []int{200},
+	})
+	if err != nil {
+		r.Err = err
+		return
+	}
+
+	_, r.Header, r.Err = gophercloud.ParseResponse(resp, err)
+	return
+}
+
+// Delete all Allocations for a certain consumer
+func DeleteConsumerAllocations(ctx context.Context, client *gophercloud.ServiceClient, consumerID string) (r ListAllocationsResult) {
+	resp, err := client.Delete(ctx, getAllocationsURL(client, consumerID), &gophercloud.RequestOpts{
+		OkCodes: []int{204, 404},
+	})
+	if err != nil {
+		r.Err = err
+		return
+	}
+
+	_, r.Header, r.Err = gophercloud.ParseResponse(resp, err)
+	return
+}
+
+// Remove all empty Allocations for a certain provider, and if it is empty, delete it
+func CleanupResourceProvider(ctx context.Context, client *gophercloud.ServiceClient, provider *resourceproviders.ResourceProvider) error {
+	if provider == nil {
+		return nil
+	}
+	result := resourceproviders.GetAllocations(ctx, client, provider.UUID)
+	providerAllocations, err := result.Extract()
+	if err != nil {
+		return err
+	}
+	if providerAllocations == nil {
+		return nil
+	}
+	allocations := len(providerAllocations.Allocations)
+	for consumerID := range providerAllocations.Allocations {
+		result := ListAllocations(ctx, client, consumerID)
+		consumerAllocations, err := result.Extract()
+		if err != nil {
+			return err
+		}
+		if len(consumerAllocations.Allocations) == 0 {
+			DeleteConsumerAllocations(ctx, client, consumerID)
+			allocations -= 1
+			continue
+		}
+		return fmt.Errorf("cannot clean up provider, still has some allocation")
+	}
+
+	if allocations == 0 {
+		result := resourceproviders.Delete(ctx, client, provider.UUID)
+		if result.Err != nil {
+			return result.Err
+		}
+	}
+	return nil
+}