Merge pull request #45 from appbaseio/hotfix/retry-http-requests

hotfix: add retry logic to ES proxy

Author: bietkul

plugins/elasticsearch/handlers.go

@@ -4,6 +4,7 @@ import (
 	"io"
 	"log"
 	"net/http"
+	"time"
 
 	"github.com/appbaseio/arc/model/acl"
 	"github.com/appbaseio/arc/model/category"
@@ -39,12 +40,19 @@ func (es *elasticsearch) handler() http.HandlerFunc {
 
 		// Forward the request to elasticsearch
 		client := util.HTTPClient()
-		response, err := client.Do(r)
-		if err != nil {
-			log.Printf("%s: error fetching response for %s: %v\n", logTag, r.URL.Path, err)
-			util.WriteBackError(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
+
+		var response *http.Response
+		util.Retry(3, 100*time.Millisecond, func() bool {
+			response, err = client.Do(r)
+			if err != nil {
+				log.Printf("%s: error fetching response for %s: %v\n", logTag, r.URL.Path, err)
+				util.WriteBackError(w, err.Error(), http.StatusInternalServerError)
+				return false
+			}
+			err = nil
+			return true
+		})
+
 		defer response.Body.Close()
 
 		// Copy the headers

util/util.go

@@ -219,6 +219,7 @@ func IntervalForRange(from, to string) (string, error) {
 	return fmt.Sprintf("%ds", int64(intervalInSecs)), nil
 }
 
+// DecodeBase64Key decodes a base64 input
 func DecodeBase64Key(encoded string) ([]byte, error) {
 	decoded, err := base64.StdEncoding.DecodeString(encoded)
 	if err != nil {
@@ -229,3 +230,16 @@ func DecodeBase64Key(encoded string) ([]byte, error) {
 	}
 	return decoded, nil
 }
+
+// Retry is a general purpose retrier for a function call
+func Retry(numberRetries int, sleepInterval time.Duration, testFunc func() bool) {
+	executionPassed := false
+	executionCount := 0
+	for !executionPassed && executionCount < numberRetries {
+		executionPassed = testFunc()
+		if !executionPassed {
+			executionCount++
+			time.Sleep(sleepInterval)
+		}
+	}
+}