feat: use lumberjack for logs + performance optimizations (#119)

* feat: use file system to store logs

* fix: minor changes

* feat: add context for api response

* feat: add request id context

* feat: add response model

* feat: add request context

* feat: add esresponse context

* feat: add profiling

* fix: logs middleware

* fix: logs for RS API

* fix: minor changes

* fix: log middleware read all

* fix: minor changes

* fix: concurrent write on map

* fix: record 500 logs

* feat: cache bcrypt

* fix: map lock

* fix: minor fixes

* fix: minor fixes

* minor changes

* minor fix

* feat: updates the search index template

* adds a new delimiter analyzer
* uses index_prefixes
* uses index_phrases

* remove profiling

* fix: use sync map

* fix: use sync map

* fix: use sync map

* remove save response method

* use new map

* fix: use pointer

* fix: minor fixes

* fix: response body in logs

* fix: minor fixes

* fix: minor fixes

* Update plugins/logs/logs.go

Co-authored-by: Siddharth Kothari <sids.aquarius@gmail.com>

* fix: minor changes

* fix: add order key so a user's own template has a higher priority

* fix: add order key so a user's own template has a higher priority

* experiment with context

* pulic struct

* fix

* fix

* fix: use ctx for response

* fix: minor changes

* fix: remove sync.Map

* fix: add chan

* fix: remove sync wait

* fix: check for passwordExists condition only once

short-circuiting in L->R with && and || operators is guaranteed in golang

* fix: update template to index numeric types as keyword

Co-authored-by: Siddharth Kothari <sids.aquarius@gmail.com>

Author: bietkul

.gitignore

@@ -8,3 +8,4 @@ build
 tags
 
 config
+log

config/manual.env

@@ -8,3 +8,4 @@ HTTPS_CERT=sample/server.crt
 HTTPS_KEY=sample/server.key
 JWT_ROLE_KEY=role
 SET_SNIFFING=false
+LOG_FILE_PATH=log/arc/es.json

go.mod

@@ -9,9 +9,11 @@ require (
 	github.com/google/uuid v1.0.0
 	github.com/gorilla/mux v1.7.1
 	github.com/hashicorp/golang-lru v0.5.0 // indirect
+	github.com/natefinch/lumberjack v2.0.1-0.20190411184413-94d9e492cc53+incompatible
 	github.com/olivere/elastic v6.2.21+incompatible
 	github.com/olivere/elastic/v7 v7.0.17
 	github.com/openzipkin/zipkin-go v0.1.6 // indirect
+	github.com/pkg/profile v1.5.0
 	github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 // indirect
 	github.com/robfig/cron v1.1.0
 	github.com/rogpeppe/go-internal v1.2.2 // indirect

go.sum

@@ -333,6 +333,10 @@ github.com/mitchellh/mapstructure v1.0.0/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/monoculum/formam v0.0.0-20180901015400-4e68be1d79ba/go.mod h1:RKgILGEJq24YyJ2ban8EO0RUVSJlF1pGsEvoLEACr/Q=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM=
+github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk=
+github.com/natefinch/lumberjack v2.0.1-0.20190411184413-94d9e492cc53+incompatible h1:uf1v9aKCMoL0/6eNuB2uaV/oi91/EMGWvlEN27jWe7I=
+github.com/natefinch/lumberjack v2.0.1-0.20190411184413-94d9e492cc53+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk=
 github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q=
 github.com/olivere/elastic v6.2.21+incompatible h1:QnTuofzxOCV5FrYLywjkMxOmOWhAeild1VXxKRksK9Y=
 github.com/olivere/elastic v6.2.21+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8=
@@ -354,6 +358,8 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/profile v1.5.0 h1:042Buzk+NhDI+DeSAA62RwJL8VAuZUMQZUjCsRz1Mug=
+github.com/pkg/profile v1.5.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=

model/queryid/queryid.go

@@ -0,0 +1,21 @@
+package request
+
+import (
+	"context"
+)
+
+type contextKey string
+
+// CtxKey is a key against which api request will get stored in the context.
+const CtxKey = contextKey("request")
+
+// NewContext returns a new context with the given request body.
+func NewContext(ctx context.Context, request interface{}) context.Context {
+	return context.WithValue(ctx, CtxKey, request)
+}
+
+// FromContext retrieves the api request body stored against the request.ctxKey from the context.
+func FromContext(ctx context.Context) (*interface{}, error) {
+	ctxRequest := ctx.Value(CtxKey)
+	return &ctxRequest, nil
+}

model/request/request.go

@@ -0,0 +1,37 @@
+package response
+
+import (
+	"context"
+	"sync"
+
+	"github.com/appbaseio/arc/errors"
+)
+
+type contextKey string
+
+// CtxKey is a key against which api response will get stored in the context.
+const ctxKey = contextKey("response")
+
+type Response struct {
+	L        *sync.RWMutex
+	Command  chan string // use to control the go routine executions
+	Response map[string]interface{}
+}
+
+// NewContext returns a new context with the given response body.
+func NewContext(ctx context.Context, response Response) context.Context {
+	return context.WithValue(ctx, ctxKey, response)
+}
+
+// FromContext retrieves the api response body stored against the response.ctxKey from the context.
+func FromContext(ctx context.Context) (*Response, error) {
+	ctxResponse := ctx.Value(ctxKey)
+	if ctxResponse == nil {
+		return nil, errors.NewNotFoundInContextError("Response")
+	}
+	responseBody, ok := ctxResponse.(Response)
+	if !ok {
+		return nil, errors.NewInvalidCastError("responseBody", "Response")
+	}
+	return &responseBody, nil
+}

model/response/response.go

@@ -161,11 +161,14 @@ func (a *Auth) basicAuth(h http.HandlerFunc) http.HandlerFunc {
 			{
 				// if the request is made to elasticsearch using user credentials, then the user has to be an admin
 				reqUser := obj.(*user.User)
-				if hasBasicAuth && bcrypt.CompareHashAndPassword([]byte(reqUser.Password), []byte(password)) != nil {
+				// No need to validate if already validated before
+				if hasBasicAuth && !IsPasswordExist(reqUser.Username, password) && bcrypt.CompareHashAndPassword([]byte(reqUser.Password), []byte(password)) != nil {
 					w.Header().Set("www-authenticate", "Basic realm=\"Authentication Required\"")
 					util.WriteBackError(w, "invalid password", http.StatusUnauthorized)
 					return
 				}
+				// Save validated username to avoid the bcrypt comparison
+				SavePassword(reqUser.Username, password)
 
 				if reqCategory.IsFromES() || reqCategory.IsFromRS() {
 					authenticated = *reqUser.IsAdmin

plugins/auth/middleware.go

@@ -0,0 +1,36 @@
+package auth
+
+import (
+	"sync"
+)
+
+// UserToPasswordCache represents a map of bcrypt validated users
+var UserToPasswordCache = make(map[string]interface{})
+
+// CurrentProcessMutex to stop concurrent writes on map
+var CurrentProcessMutex = sync.RWMutex{}
+
+// SavePassword saved the password in the cache
+func SavePassword(username string, password string) {
+	CurrentProcessMutex.Lock()
+	UserToPasswordCache[username] = password
+	CurrentProcessMutex.Unlock()
+}
+
+// ClearPassword clears the password in the cache
+func ClearPassword(username string) {
+	CurrentProcessMutex.Lock()
+	delete(UserToPasswordCache, username)
+	CurrentProcessMutex.Unlock()
+}
+
+// IsPasswordExist checks whether the password in the cache or not
+func IsPasswordExist(username string, password string) bool {
+	CurrentProcessMutex.Lock()
+	cachedPassword, ok := UserToPasswordCache[username]
+	CurrentProcessMutex.Unlock()
+	if !ok {
+		return false
+	}
+	return cachedPassword == password
+}

plugins/auth/util.go

@@ -192,15 +192,13 @@ func intercept(h http.HandlerFunc) http.HandlerFunc {
 						modifiedBody := []byte(modifiedBodyString)
 						req.Body = ioutil.NopCloser(bytes.NewReader(modifiedBody))
 					} else {
-						body, err := ioutil.ReadAll(req.Body)
+						reqBody := make(map[string]interface{})
+						err := json.NewDecoder(req.Body).Decode(&reqBody)
 						if err != nil {
 							log.Errorln(logTag, ":", err)
 							util.WriteBackError(w, err.Error(), http.StatusInternalServerError)
 							return
 						}
-						d := json.NewDecoder(ioutil.NopCloser(bytes.NewReader(body)))
-						reqBody := make(map[string]interface{})
-						d.Decode(&reqBody)
 						reqBody["_source"] = sources
 						modifiedBody, _ := json.Marshal(reqBody)
 						req.Body = ioutil.NopCloser(bytes.NewReader(modifiedBody))