Cleanup placement during decomissioning

Usually, there should be no resources on the host
left after having migrated all machines.
But ocassionally, we are still left with some
empty resources.
This change removes all empty resources
before removing the resource-provider as well.

Author: fwiesel

internal/controller/decomission_controller.go

@@ -35,6 +35,7 @@ import (
 	"github.com/gophercloud/gophercloud/v2"
 	"github.com/gophercloud/gophercloud/v2/openstack/compute/v2/hypervisors"
 	"github.com/gophercloud/gophercloud/v2/openstack/compute/v2/services"
+	"github.com/gophercloud/gophercloud/v2/openstack/placement/v1/resourceproviders"
 )
 
 const (
@@ -43,8 +44,9 @@ const (
 
 type NodeDecommissionReconciler struct {
 	k8sclient.Client
-	Scheme        *runtime.Scheme
-	computeClient *gophercloud.ServiceClient
+	Scheme          *runtime.Scheme
+	computeClient   *gophercloud.ServiceClient
+	placementClient *gophercloud.ServiceClient
 }
 
 // The counter-side in gardener is here:
@@ -126,11 +128,21 @@ func (r *NodeDecommissionReconciler) shutdownService(ctx context.Context, node *
 
 	// Deleting and evicted, so better delete the service
 	err = services.Delete(ctx, r.computeClient, hypervisor.Service.ID).ExtractErr()
-	if err == nil || gophercloud.ResponseCodeIs(err, http.StatusNotFound) {
-		return r.removeFinalizer(ctx, node)
+	if err != nil && !gophercloud.ResponseCodeIs(err, http.StatusNotFound) {
+		return ctrl.Result{}, fmt.Errorf("cannot delete service due to %w", err)
+	}
+
+	rp, err := resourceproviders.Get(ctx, r.placementClient, hypervisorID).Extract()
+	if err != nil && !gophercloud.ResponseCodeIs(err, http.StatusNotFound) {
+		return ctrl.Result{}, fmt.Errorf("cannot get resource provider due to %w", err)
+	}
+
+	err = openstack.CleanupResourceProvider(ctx, r.placementClient, rp)
+	if err != nil {
+		return ctrl.Result{}, fmt.Errorf("cannot clean up resource provider due to %w", err)
 	}
 
-	return ctrl.Result{}, err
+	return r.removeFinalizer(ctx, node)
 }
 
 func (r *NodeDecommissionReconciler) removeFinalizer(ctx context.Context, node *corev1.Node) (ctrl.Result, error) {
@@ -153,6 +165,12 @@ func (r *NodeDecommissionReconciler) SetupWithManager(mgr ctrl.Manager) error {
 
 	r.computeClient.Microversion = "2.93"
 
+	r.placementClient, err = openstack.GetServiceClient(ctx, "placement")
+	if err != nil {
+		return err
+	}
+	r.placementClient.Microversion = "1.39" // yoga, or later
+
 	return ctrl.NewControllerManagedBy(mgr).
 		Named("nodeDecommission").
 		For(&corev1.Node{}).

internal/openstack/placement.go

@@ -19,8 +19,10 @@ package openstack
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/gophercloud/gophercloud/v2"
+	"github.com/gophercloud/gophercloud/v2/openstack/placement/v1/resourceproviders"
 )
 
 // UpdateTraitsResult is the response of a Put traits operations. Call its Extract method
@@ -67,3 +69,96 @@ func UpdateTraits(ctx context.Context, client *gophercloud.ServiceClient, resour
 	_, r.Header, r.Err = gophercloud.ParseResponse(resp, err)
 	return
 }
+
+func getAllocationsURL(client *gophercloud.ServiceClient, consumerID string) string {
+	return client.ServiceURL("allocations", consumerID)
+}
+
+// ListAllocationsResult is the response of a Get allocations operations. Call its Extract method
+// to interpret it as a Allocations.
+type ListAllocationsResult struct {
+	gophercloud.Result
+}
+
+type ConsumerAllocations struct {
+	Allocations        map[string]resourceproviders.Allocation `json:"allocations"`
+	ConsumerGeneration int                                     `json:"consumer_generation"`
+	ProjectID          string                                  `json:"project_id"`
+	UserID             string                                  `json:"user_id"`
+	ConsumerType       string                                  `json:"consumer_type"`
+}
+
+// Extract interprets a ListAllocationsResult as a Allocations.
+func (r ListAllocationsResult) Extract() (*ConsumerAllocations, error) {
+	var s ConsumerAllocations
+	err := r.ExtractInto(&s)
+	return &s, err
+}
+
+// List Allocations for a certain consumer
+func ListAllocations(ctx context.Context, client *gophercloud.ServiceClient, consumerID string) (r ListAllocationsResult) {
+	resp, err := client.Get(ctx, getAllocationsURL(client, consumerID), nil, &gophercloud.RequestOpts{
+		OkCodes: []int{200},
+	})
+	if err != nil {
+		r.Err = err
+		return
+	}
+
+	_, r.Header, r.Err = gophercloud.ParseResponse(resp, err)
+	return
+}
+
+// Delete all Allocations for a certain consumer
+func DeleteConsumerAllocations(ctx context.Context, client *gophercloud.ServiceClient, consumerID string) (r ListAllocationsResult) {
+	resp, err := client.Delete(ctx, getAllocationsURL(client, consumerID), &gophercloud.RequestOpts{
+		OkCodes: []int{204, 404},
+	})
+	if err != nil {
+		r.Err = err
+		return
+	}
+
+	_, r.Header, r.Err = gophercloud.ParseResponse(resp, err)
+	return
+}
+
+// Remove all empty Allocations for a certain provider, and if it is empty, delete it
+func CleanupResourceProvider(ctx context.Context, client *gophercloud.ServiceClient, provider *resourceproviders.ResourceProvider) error {
+	if provider == nil {
+		return nil
+	}
+
+	providerAllocations, err := resourceproviders.GetAllocations(ctx, client, provider.UUID).Extract()
+	if err != nil {
+		return err
+	}
+
+	// It is a map of consumer-ids to their alloctions, we just go over their ids
+	// to cross-check, what is stored for the consumer itself
+	for consumerID := range providerAllocations.Allocations {
+		// Allocations of the consumer mapped by the resource provider, so the
+		// "reverse" of what we got before
+		result := ListAllocations(ctx, client, consumerID)
+		consumerAllocations, err := result.Extract()
+		if err != nil {
+			return err
+		}
+
+		if len(consumerAllocations.Allocations) > 0 {
+			return fmt.Errorf("cannot clean up provider, cannot handle non-empty consumer allocations")
+		}
+
+		// The consumer actually doesn't have *any* allocations, so it is just
+		// inconsistent, and we can drop them all
+		DeleteConsumerAllocations(ctx, client, consumerID)
+	}
+
+	// We are done, let's clean it up
+	err = resourceproviders.Delete(ctx, client, provider.UUID).ExtractErr()
+	if err != nil {
+		return fmt.Errorf("failed to delete after cleanup due to %w", err)
+	}
+
+	return nil
+}