Onboard/Offboard nodes with instance-ha service

Author: fwiesel

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/go-openapi/swag v0.23.1
 	github.com/gophercloud/gophercloud/v2 v2.7.0
 	github.com/gophercloud/utils/v2 v2.0.0-20250711132455-9770683b100a
+	github.com/jarcoal/httpmock v1.4.0
 	github.com/onsi/ginkgo/v2 v2.23.4
 	github.com/onsi/gomega v1.37.0
 	k8s.io/api v0.33.3

go.sum

@@ -74,6 +74,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1 h1:VNqngBF40hVlDloBruUehVYC3Ar
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1/go.mod h1:RBRO7fro65R6tjKzYgLAFo0t1QEXY1Dp+i/bvpRiqiQ=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jarcoal/httpmock v1.4.0 h1:BvhqnH0JAYbNudL2GMJKgOHe2CtKlzJ/5rWKyp+hc2k=
+github.com/jarcoal/httpmock v1.4.0/go.mod h1:ftW1xULwo+j0R0JJkJIIi7UKigZUXCLLanykgjwBXL0=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
@@ -90,6 +92,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
 github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
+github.com/maxatome/go-testdeep v1.14.0 h1:rRlLv1+kI8eOI3OaBXZwb3O7xY3exRzdW5QyX48g9wI=
+github.com/maxatome/go-testdeep v1.14.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=

internal/controller/maintenance_controller_test.go

@@ -20,7 +20,6 @@ package controller
 import (
 	"context"
 
-	"github.com/gophercloud/gophercloud/v2/testhelper"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
@@ -79,9 +78,6 @@ var _ = Describe("Maintenance Controller", func() {
 
 		It("should successfully reconcile the resource", func() {
 			By("Reconciling the created resource")
-			testhelper.SetupHTTP()
-			defer testhelper.TeardownHTTP()
-
 			_, err := reconcileNodeLoop(1)
 			Expect(err).NotTo(HaveOccurred())
 		})

internal/controller/node_eviction_label_controller.go

@@ -99,6 +99,11 @@ func (r *NodeEvictionLabelReconciler) Reconcile(ctx context.Context, req ctrl.Re
 	}
 
 	if value != "" {
+		err = disableInstanceHA(node)
+		if err != nil {
+			return ctrl.Result{}, err
+		}
+
 		newNode := node.DeepCopy()
 		if value == "true" { //nolint:goconst
 			evictAgentsLabels(newNode.Labels)
@@ -128,8 +133,12 @@ func (r *NodeEvictionLabelReconciler) reconcileEviction(ctx context.Context, evi
 			Reason:     fmt.Sprintf("openstack-hypervisor-operator: label %v=%v", labelEvictionRequired, maintenanceValue),
 		}
 
+		if err = enableInstanceHAMissingOkay(node); err != nil {
+			return "", fmt.Errorf("failed to enable instance ha before eviction due to %w", err)
+		}
+
 		if err = r.Create(ctx, eviction); err != nil {
-			return "", err
+			return "", fmt.Errorf("failed to create eviction due to %w", err)
 		}
 	}
 

internal/controller/node_eviction_label_controller_test.go

@@ -19,6 +19,7 @@ package controller
 
 import (
 	"context"
+	"fmt"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -29,13 +30,17 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	kvmv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1"
+	"github.com/jarcoal/httpmock"
 )
 
 var _ = Describe("Node Eviction Label Controller", func() {
 	var nodeReconciler *NodeEvictionLabelReconciler
 
 	Context("When reconciling a node", func() {
-		const nodeName = "node-test"
+		const nodeName = "test-node"
+		const hostName = "test-hostname"
+		const region = "region"
+		const zone = "zone"
 		req := ctrl.Request{
 			NamespacedName: types.NamespacedName{Name: nodeName},
 		}
@@ -53,6 +58,7 @@ var _ = Describe("Node Eviction Label Controller", func() {
 		}
 
 		BeforeEach(func() {
+			httpmock.Activate()
 			nodeReconciler = &NodeEvictionLabelReconciler{
 				Client: k8sClient,
 				Scheme: k8sClient.Scheme(),
@@ -67,9 +73,11 @@ var _ = Describe("Node Eviction Label Controller", func() {
 				ObjectMeta: metav1.ObjectMeta{
 					Name: nodeName,
 					Labels: map[string]string{
-						corev1.LabelHostname:  "test",
-						labelEvictionRequired: "true",
-						labelOnboardingState:  "completed"},
+						corev1.LabelHostname:       hostName,
+						corev1.LabelTopologyRegion: region,
+						corev1.LabelTopologyZone:   zone,
+						labelEvictionRequired:      "true",
+						labelOnboardingState:       "completed"},
 				},
 			}
 			Expect(k8sClient.Create(ctx, resource)).To(Succeed())
@@ -79,16 +87,20 @@ var _ = Describe("Node Eviction Label Controller", func() {
 			node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: nodeName}}
 			By("Cleanup the specific node")
 			Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, node))).To(Succeed())
+			httpmock.DeactivateAndReset()
 		})
 
 		It("should successfully reconcile the resource", func() {
+			url := InstanceHaUrl(region, zone, hostName)
+			httpmock.RegisterResponder("POST", url, httpmock.NewStringResponder(200, ``))
+
 			By("Reconciling the created resource")
 			_, err := reconcileNodeLoop(5)
 			Expect(err).NotTo(HaveOccurred())
 
 			// expect node controller to create an eviction for the node
 			err = k8sClient.Get(ctx, types.NamespacedName{
-				Name:      "maintenance-required-test",
+				Name:      fmt.Sprintf("maintenance-required-%v", hostName),
 				Namespace: "monsoon3",
 			}, &kvmv1.Eviction{})
 			Expect(err).NotTo(HaveOccurred())

internal/controller/onboarding_controller.go

@@ -148,6 +148,7 @@ func (r *OnboardingController) initialOnboarding(ctx context.Context, node *core
 	if !found || serviceId == "" {
 		return fmt.Errorf("empty service-id for label %v on node", labelServiceID)
 	}
+
 	result := services.Update(ctx, r.computeClient, serviceId, services.UpdateOpts{Status: services.ServiceEnabled})
 	if result.Err != nil {
 		return result.Err
@@ -194,6 +195,7 @@ func (r *OnboardingController) smokeTest(ctx context.Context, node *corev1.Node,
 }
 
 func (r *OnboardingController) completeOnboarding(ctx context.Context, host string, node *corev1.Node) (ctrl.Result, error) {
+	log := logger.FromContext(ctx)
 	aggs, err := aggregatesByName(ctx, r.computeClient)
 	if err != nil {
 		return ctrl.Result{}, fmt.Errorf("failed to get aggregates %w", err)
@@ -203,6 +205,13 @@ func (r *OnboardingController) completeOnboarding(ctx context.Context, host stri
 	if err != nil {
 		return ctrl.Result{}, fmt.Errorf("failed to remove from test aggregate %w", err)
 	}
+	log.Info("removed from test-aggregate", "name", testAggregateName)
+
+	err = enableInstanceHA(node)
+	log.Info("enabled instance-ha")
+	if err != nil {
+		return ctrl.Result{}, err
+	}
 
 	_, err = setNodeLabels(ctx, r, node, map[string]string{
 		labelOnboardingState: onboardingValueCompleted,

internal/controller/utils.go

@@ -18,8 +18,12 @@ limitations under the License.
 package controller
 
 import (
+	"bytes"
 	"context"
+	"fmt"
 	"maps"
+	"net/http"
+	"slices"
 
 	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -46,3 +50,45 @@ func setNodeAnnotations(ctx context.Context, writer client.Writer, node *corev1.
 
 	return writer.Patch(ctx, newNode, client.MergeFrom(node))
 }
+
+func InstanceHaUrl(region, zone, hostname string) string {
+	return fmt.Sprintf("https://kvm-ha-service-%v.%v.cloud.sap/api/hypervisors/%v", zone, region, hostname)
+}
+
+func updateInstanceHA(node *corev1.Node, data string, acceptedCodes []int) error {
+	zone, found := node.Labels[corev1.LabelTopologyZone]
+	if !found {
+		return fmt.Errorf("could not find label %v for node", corev1.LabelTopologyZone)
+	}
+	region, found := node.Labels[corev1.LabelTopologyRegion]
+	if !found {
+		return fmt.Errorf("could not find label %v for node", corev1.LabelTopologyRegion)
+	}
+
+	hostname, found := node.Labels[corev1.LabelHostname]
+	if !found {
+		return fmt.Errorf("could not find label %v for node", corev1.LabelHostname)
+	}
+
+	url := InstanceHaUrl(region, zone, hostname)
+	resp, err := http.Post(url, "application/json", bytes.NewBuffer([]byte(data)))
+	if err != nil {
+		return fmt.Errorf("failed to send request to ha service due to %w", err)
+	}
+	if !slices.Contains(acceptedCodes, resp.StatusCode) {
+		return fmt.Errorf("ha service answered with unexpected response %v for %v from %v", resp.StatusCode, data, url)
+	}
+	return nil
+}
+
+func enableInstanceHA(node *corev1.Node) error {
+	return updateInstanceHA(node, `{"enabled": true}`, []int{http.StatusOK})
+}
+
+func enableInstanceHAMissingOkay(node *corev1.Node) error {
+	return updateInstanceHA(node, `{"enabled": true}`, []int{http.StatusOK, http.StatusNotFound})
+}
+
+func disableInstanceHA(node *corev1.Node) error {
+	return updateInstanceHA(node, `{"enabled": false}`, []int{http.StatusOK, http.StatusNotFound})
+}