disable HAC + set noobaa-core term grace to 1

- HAC can be enabled with the env HAC_ENABLED=true
- Also changed the default grace period of the HAC when enabled from 0 to 1. This change avoids force deletion of the pods, to prevent multiple containers running concurrently.
- The grace period can be set to 0 with the env HAC_FORCE_DELETE=true
- Also setting the terminationGracePeriodSeconds of noobaa-core to 1. By default this is 30 seconds. Since we don't have any handler in noobaa-core for SIGETRM, there isn't a need to wait 30 seconds before killing the container.

Signed-off-by: Danny Zaken <dannyzaken@gmail.com>
(cherry picked from commit 38294af)
(cherry picked from commit b7652a4)
(cherry picked from commit 1e0e4ee)

Author: dannyzaken

.github/workflows/run_hac_test.yml

@@ -1,5 +1,5 @@
 name: HAC on KIND cluster Test
-on: [push, pull_request, workflow_dispatch]
+on: [workflow_dispatch]
 
 jobs:
   run-hac-test:

pkg/controller/add_hac.go

@@ -1,10 +1,15 @@
 package controller
 
 import (
+	"os"
+
 	hac "github.com/noobaa/noobaa-operator/v5/pkg/controller/ha"
 )
 
 func init() {
-	// AddToManagerFuncs is a list of functions to create controllers and add them to a manager.
-	AddToManagerFuncs = append(AddToManagerFuncs, hac.Add)
+	hacEnabled := os.Getenv("HAC_ENABLED")
+	if hacEnabled == "true" {
+		// AddToManagerFuncs is a list of functions to create controllers and add them to a manager.
+		AddToManagerFuncs = append(AddToManagerFuncs, hac.Add)
+	}
 }

pkg/controller/ha/ha_controller.go

@@ -71,6 +71,7 @@ func nodeNotReadyPredicate() predicate.Predicate {
 
 // Add creates a nodewatcher Controller and adds it to the Manager.
 func Add(mgr manager.Manager) error {
+	util.Logger().Info("Adding high availability controller (HAC)")
 
 	opts := controller.Options{
 		MaxConcurrentReconciles: 1,

pkg/hac/hac.go

@@ -5,6 +5,7 @@ package hac
 
 import (
 	"context"
+	"os"
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -43,8 +44,12 @@ func (pd *PodDeleter) DeletePodsOnNode() error {
 		return errors.Errorf("failed to list noobaa pods on the node %v in namespace %v", pd.NodeName, options.Namespace)
 	}
 
-	// delete the found pods
-	var gracePeriod int64 = 0
+	// delete the found pods. by default using a 1 second grace period.
+	var gracePeriod int64 = 1
+	if os.Getenv("HAC_FORCE_DELETE") == "true" {
+		// if HAC_FORCE_DELETE is set to true, use a 0 second grace period.
+		gracePeriod = 0
+	}
 	deleteOpts := client.DeleteOptions{GracePeriodSeconds: &gracePeriod}
 	for _, pod := range podList.Items {
 		if err := pd.Client.Delete(context.Background(), &pod, &deleteOpts); err != nil {

pkg/system/phase2_creating.go

@@ -536,6 +536,10 @@ func (r *Reconciler) SetDesiredCoreApp() error {
 	r.CoreApp.Spec.ServiceName = r.ServiceMgmt.Name
 
 	podSpec := &r.CoreApp.Spec.Template.Spec
+	// set the termination grace period for noobaa-core pod.
+	// For now we set it to 1 second. A better approach should be to implement a graceful shutdown for the noobaa-core pod when SIGTERM is received.
+	terminationGracePeriodSeconds := int64(1)
+	podSpec.TerminationGracePeriodSeconds = &terminationGracePeriodSeconds
 	podSpec.ServiceAccountName = "noobaa-core"
 	coreImageChanged := false