feat(threadwinds-ingestion): implement infinite retry with exponential backoff

Author: JocLRojas

threadwinds-ingestion/main.go

@@ -71,11 +71,7 @@ func main() {
 			return nil
 		}
 
-		err = utils.InfiniteRetryIfXError(registerFunc, "404", "Not Found", "connection refused")
-		if err != nil {
-			catcher.Error("failed to register in ThreadWinds Platform after retries", err, nil)
-			os.Exit(1)
-		}
+		utils.InfiniteRetry(registerFunc, "ThreadWinds registration")
 
 		catcher.Info("ThreadWinds registration successful", nil)
 

threadwinds-ingestion/utils/check.go

@@ -1,39 +1,48 @@
 package utils
 
 import (
-	"strings"
+	"fmt"
 	"time"
 
 	"github.com/threatwinds/go-sdk/catcher"
 )
 
 const (
-	wait = 5 * time.Second
+	retryInitialBackoff    = 5 * time.Second
+	retryMaxBackoff        = 2 * time.Minute
+	retryBackoffMultiplier = 2.0
+	retryLogInterval       = 10
 )
 
-func InfiniteRetryIfXError(f func() error, exceptions ...string) error {
-	var xErrorWasLogged bool
+func InfiniteRetry(f func() error, operationName string) {
+	attempt := 0
+	currentBackoff := retryInitialBackoff
+
+	catcher.Info(fmt.Sprintf("Starting %s with infinite retry and exponential backoff", operationName), map[string]any{
+		"initial_backoff": retryInitialBackoff.String(),
+		"max_backoff":     retryMaxBackoff.String(),
+	})
 
 	for {
+		attempt++
 		err := f()
-		if err != nil && is(err, exceptions...) {
-			if !xErrorWasLogged {
-				_ = catcher.Error("An error occurred (%s), will keep retrying indefinitely...", err, nil)
-				xErrorWasLogged = true
-			}
-			time.Sleep(wait)
-			continue
-		}
 
-		return err
-	}
-}
+		if err == nil {
+			catcher.Info(fmt.Sprintf("%s completed successfully", operationName), map[string]any{
+				"attempts": attempt,
+			})
+			return
+		}
 
-func is(e error, args ...string) bool {
-	for _, arg := range args {
-		if strings.Contains(e.Error(), arg) {
-			return true
+		if attempt == 1 || attempt%retryLogInterval == 0 {
+			_ = catcher.Error(fmt.Sprintf("%s failed, will retry indefinitely...", operationName), err, map[string]any{
+				"attempt":       attempt,
+				"next_retry_in": currentBackoff.String(),
+			})
 		}
+
+		time.Sleep(currentBackoff)
+
+		currentBackoff = min(time.Duration(float64(currentBackoff)*retryBackoffMultiplier), retryMaxBackoff)
 	}
-	return false
 }