magnum-auto-healer: Remove Node object after Heat stack update (kubernetes#701)

Author: lingxiankong

pkg/autohealing/cloudprovider/cloudprovider.go

@@ -17,6 +17,7 @@ limitations under the License.
 package cloudprovider
 
 import (
+	"k8s.io/client-go/kubernetes"
 	log "k8s.io/klog"
 
 	"k8s.io/cloud-provider-openstack/pkg/autohealing/config"
@@ -43,7 +44,7 @@ type CloudProvider interface {
 	Enabled() bool
 }
 
-type RegisterFunc func(config config.Config) (CloudProvider, error)
+type RegisterFunc func(config config.Config, client kubernetes.Interface) (CloudProvider, error)
 
 // RegisterCloudProvider registers a cloudprovider.Factory by name. This
 // is expected to happen during app startup.
@@ -57,10 +58,10 @@ func RegisterCloudProvider(name string, register RegisterFunc) {
 }
 
 // GetCloudProvider creates an instance of the named cloud provider
-func GetCloudProvider(name string, config config.Config) (CloudProvider, error) {
+func GetCloudProvider(name string, config config.Config, client kubernetes.Interface) (CloudProvider, error) {
 	f, found := providers[name]
 	if !found {
 		return nil, nil
 	}
-	return f(config)
+	return f(config, client)
 }

pkg/autohealing/cloudprovider/openstack/provider.go

@@ -29,8 +29,10 @@ import (
 	"github.com/gophercloud/gophercloud/openstack/orchestration/v1/stackresources"
 	"github.com/gophercloud/gophercloud/openstack/orchestration/v1/stacks"
 	uuid "github.com/pborman/uuid"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
 	log "k8s.io/klog"
 
 	"k8s.io/cloud-provider-openstack/pkg/autohealing/config"
@@ -55,6 +57,7 @@ var statusesPreventingRepair = sets.NewString(
 
 // OpenStack is an implementation of cloud provider Interface for OpenStack.
 type OpenStackCloudProvider struct {
+	KubeClient           kubernetes.Interface
 	Nova                 *gophercloud.ServiceClient
 	Heat                 *gophercloud.ServiceClient
 	Magnum               *gophercloud.ServiceClient
@@ -209,6 +212,13 @@ func (provider OpenStackCloudProvider) Repair(nodes []healthcheck.NodeInfo) erro
 
 	log.Infof("Started Heat stack update to rebuild resources, cluster: %s, stack: %s", clusterName, cluster.StackID)
 
+	// Remove the broken nodes from the cluster
+	for _, n := range nodes {
+		if err := provider.KubeClient.CoreV1().Nodes().Delete(n.KubeNode.Name, &metav1.DeleteOptions{}); err != nil {
+			log.Errorf("Failed to remove the node %s from cluster, error: %v", n.KubeNode.Name, err)
+		}
+	}
+
 	return nil
 }
 

pkg/autohealing/cloudprovider/register/register.go

@@ -26,14 +26,15 @@ import (
 	gopenstack "github.com/gophercloud/gophercloud/openstack"
 	"github.com/gophercloud/gophercloud/openstack/identity/v3/extensions/trusts"
 	netutil "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/client-go/kubernetes"
 	certutil "k8s.io/client-go/util/cert"
 
 	"k8s.io/cloud-provider-openstack/pkg/autohealing/cloudprovider"
 	"k8s.io/cloud-provider-openstack/pkg/autohealing/cloudprovider/openstack"
 	"k8s.io/cloud-provider-openstack/pkg/autohealing/config"
 )
 
-func registerOpenStack(cfg config.Config) (cloudprovider.CloudProvider, error) {
+func registerOpenStack(cfg config.Config, kubeClient kubernetes.Interface) (cloudprovider.CloudProvider, error) {
 	client, err := gopenstack.NewClient(cfg.OpenStack.AuthURL)
 	if err != nil {
 		return nil, err
@@ -93,10 +94,11 @@ func registerOpenStack(cfg config.Config) (cloudprovider.CloudProvider, error) {
 
 	var p cloudprovider.CloudProvider
 	p = openstack.OpenStackCloudProvider{
-		Nova:   novaClient,
-		Heat:   heatClient,
-		Magnum: magnumClient,
-		Config: cfg,
+		KubeClient: kubeClient,
+		Nova:       novaClient,
+		Heat:       heatClient,
+		Magnum:     magnumClient,
+		Config:     cfg,
 	}
 
 	return p, nil

pkg/autohealing/controller/controller.go

@@ -104,20 +104,20 @@ func createKubeClients(apiserverHost string, kubeConfig string) (*kubernetes.Cli
 
 // NewController creates a new autohealer controller.
 func NewController(conf config.Config) *Controller {
+	// initialize k8s clients
+	kubeClient, leaderElectionClient, err := createKubeClients(conf.Kubernetes.ApiserverHost, conf.Kubernetes.KubeConfig)
+	if err != nil {
+		log.Fatalf("failed to initialize kubernetes client, error: %v", err)
+	}
+
 	// initialize cloud provider
-	provider, err := cloudprovider.GetCloudProvider(conf.CloudProvider, conf)
+	provider, err := cloudprovider.GetCloudProvider(conf.CloudProvider, conf, kubeClient)
 	if err != nil {
 		log.Fatalf("Failed to get the cloud provider %s: %v", conf.CloudProvider, err)
 	}
 
 	log.Infof("Using cloud provider: %s", provider.GetName())
 
-	// initialize k8s clients
-	kubeClient, leaderElectionClient, err := createKubeClients(conf.Kubernetes.ApiserverHost, conf.Kubernetes.KubeConfig)
-	if err != nil {
-		log.Fatalf("failed to initialize kubernetes client, error: %v", err)
-	}
-
 	// event
 	eventBroadcaster := record.NewBroadcaster()
 	eventBroadcaster.StartLogging(log.V(4).Infof)
@@ -316,12 +316,6 @@ func (c *Controller) repairNodes(unhealthyNodes []healthcheck.NodeInfo) {
 					} else {
 						log.Infof("Node %s is cordoned", nodeName)
 					}
-
-					// Remove the broken node from the cluster
-					if err := c.kubeClient.CoreV1().Nodes().Delete(nodeName, &metav1.DeleteOptions{}); err != nil {
-						log.Errorf("Failed to remove the node %s from cluster, error: %v", unhealthyNodeNames.List(), err)
-					}
-					// TODO: Deal with the situation that the autohealer is running on a failed node
 				}
 
 				// Start to repair all the unhealthy nodes.