Merge branch 'dev' of github.com:appbaseio/arc into feat/size_limit

bietkul · bietkul · commit c33530287d49 · 2020-03-17T15:57:55.000+05:30
diff --git a/.travis.yml b/.travis.yml
@@ -29,9 +29,9 @@ jobs:
         - make clean && make
 
       before_deploy:
-        - mkdir -p pkg/mod/github.com/appbaseio/
-        - sudo mv $GOPATH/pkg/mod/github.com/appbaseio/* pkg/mod/github.com/appbaseio/
-        - zip -r arc-linux.zip build pkg sample
+        - mkdir -p go/pkg/mod/github.com/appbaseio/
+        - sudo mv $GOPATH/pkg/mod/github.com/appbaseio/* go/pkg/mod/github.com/appbaseio/
+        - zip -r arc-linux.zip build go sample
         - export TRAVIS_TAG="preview"
 
       deploy:
@@ -40,5 +40,6 @@ jobs:
         skip_cleanup: true
         file: arc-linux.zip
         draft: true
+        overwrite: true
         on:
           all_branches: true
diff --git a/main.go b/main.go
@@ -192,7 +192,7 @@ func main() {
 	// ES v7 and v6 clients
 	util.NewClient()
 	// map of specific plugins
-	sequencedPlugins := []string{"rules.so", "functions.so", "analytics.so"}
+	sequencedPlugins := []string{"searchsettings.so", "rules.so", "functions.so", "analytics.so"}
 	sequencedPluginsByPath := make(map[string]string)
 
 	var elasticSearchPath, reactiveSearchPath string
diff --git a/middleware/classify/indices.go b/middleware/classify/indices.go
@@ -2,6 +2,8 @@ package classify
 
 import (
 	"net/http"
+	"regexp"
+	"strings"
 
 	"github.com/appbaseio/arc/middleware"
 	"github.com/appbaseio/arc/model/index"
@@ -16,7 +18,50 @@ func Indices() middleware.Middleware {
 func indices(h http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, req *http.Request) {
 		indices := util.IndicesFromRequest(req)
+		currentCache := GetIndexAliasCache()
 
+		for _, index := range indices {
+			// '*' in case of all indices put alias in context
+			if index == "*" {
+				for cachedItem := range currentCache {
+					alias := GetIndexAlias(cachedItem)
+					if alias != "" {
+						indices = append(indices, alias)
+					}
+				}
+				break
+			} else if strings.Contains(index, "*") {
+				// in case of regex check if string contains '*' in naming pattern, if contains and doesn't have '.*' and replace '*' with '.*' because golang regex can match in that pattern. Next match regex patters with existing index names in cache and add those alias to context.
+				regex := index
+
+				if !strings.Contains(index, ".*") {
+					regex = strings.Replace(regex, "*", ".*", -1)
+				}
+				r, _ := regexp.Compile(regex)
+				cachedIndices := []string{}
+
+				for cachedItem := range currentCache {
+					cachedIndices = append(cachedIndices, cachedItem)
+				}
+				for _, val := range cachedIndices {
+					if r.MatchString(val) {
+						alias := GetIndexAlias(val)
+						if alias != "" {
+							indices = append(indices, alias)
+						}
+						break
+					}
+				}
+
+			} else {
+				// get alias for index and put in context
+				alias := GetIndexAlias(index)
+				if alias != "" {
+					indices = append(indices, alias)
+				}
+				break
+			}
+		}
 		ctx := index.NewContext(req.Context(), indices)
 		req = req.WithContext(ctx)
 
diff --git a/middleware/classify/util.go b/middleware/classify/util.go
@@ -0,0 +1,24 @@
+package classify
+
+// IndexAliasCache cache to store index alias map
+var IndexAliasCache = make(map[string]string)
+
+// GetIndexAliasCache get whole cache
+func GetIndexAliasCache() map[string]string {
+	return IndexAliasCache
+}
+
+// GetIndexAlias get alias for specific index
+func GetIndexAlias(index string) string {
+	alias, ok := IndexAliasCache[index]
+
+	if !ok {
+		return ""
+	}
+	return alias
+}
+
+// SetIndexAlias set alias for specific index
+func SetIndexAlias(index, alias string) {
+	IndexAliasCache[index] = alias
+}
diff --git a/model/category/category.go b/model/category/category.go
@@ -35,6 +35,7 @@ const (
 	Auth
 	Functions
 	ReactiveSearch
+	SearchSettings
 )
 
 // String is an implementation of Stringer interface that returns the string representation of category.Categories.
@@ -56,6 +57,7 @@ func (c Category) String() string {
 		"auth",
 		"functions",
 		"reactivesearch",
+		"searchsettings",
 	}[c]
 }
 
@@ -99,6 +101,8 @@ func (c *Category) UnmarshalJSON(bytes []byte) error {
 		*c = Functions
 	case ReactiveSearch.String():
 		*c = ReactiveSearch
+	case SearchSettings.String():
+		*c = SearchSettings
 	default:
 		return fmt.Errorf("invalid category encountered: %v", category)
 	}
@@ -141,6 +145,8 @@ func (c Category) MarshalJSON() ([]byte, error) {
 		category = Functions.String()
 	case ReactiveSearch:
 		category = ReactiveSearch.String()
+	case SearchSettings:
+		category = SearchSettings.String()
 	default:
 		return nil, fmt.Errorf("invalid category encountered: %v" + c.String())
 	}
diff --git a/model/permission/defaults.go b/model/permission/defaults.go
@@ -29,6 +29,7 @@ var (
 		category.Auth,
 		category.Functions,
 		category.ReactiveSearch,
+		category.SearchSettings,
 	}
 
 	defaultOps = []op.Operation{
@@ -59,6 +60,7 @@ var (
 		AuthLimit:           10,
 		FunctionsLimit:      10,
 		ReactiveSearchLimit: 10,
+		SearchSettingsLimit: 10,
 	}
 
 	defaultAdminLimits = Limits{
@@ -79,5 +81,6 @@ var (
 		AuthLimit:           30,
 		FunctionsLimit:      30,
 		ReactiveSearchLimit: 30,
+		SearchSettingsLimit: 30,
 	}
 )
diff --git a/model/permission/permission.go b/model/permission/permission.go
@@ -71,6 +71,7 @@ type Limits struct {
 	AuthLimit           int64 `json:"auth_limit"`
 	FunctionsLimit      int64 `json:"functions_limit"`
 	ReactiveSearchLimit int64 `json:"reactivesearch_limit"`
+	SearchSettingsLimit int64 `json:"searchsettings_limit"`
 }
 
 // Options is a function type used to define a permission's properties.
@@ -471,6 +472,8 @@ func (p *Permission) GetLimitFor(c category.Category) (int64, error) {
 		return p.Limits.FunctionsLimit, nil
 	case category.ReactiveSearch:
 		return p.Limits.ReactiveSearchLimit, nil
+	case category.SearchSettings:
+		return p.Limits.SearchSettingsLimit, nil
 	default:
 		return -1, fmt.Errorf(`we do not rate limit "%s" category`, c)
 	}
@@ -591,6 +594,10 @@ func (p *Permission) GetPatch(rolePatched bool) (map[string]interface{}, error)
 			limits["reactivesearch_limit"] = p.Limits.ReactiveSearchLimit
 		}
 
+		if p.Limits.SearchSettingsLimit != 0 {
+			limits["searchsettings_limit"] = p.Limits.SearchSettingsLimit
+		}
+
 		patch["limits"] = limits
 	}
 	if p.Description != "" {
diff --git a/model/user/defaults.go b/model/user/defaults.go
@@ -28,6 +28,7 @@ var (
 		category.Suggestions,
 		category.Functions,
 		category.ReactiveSearch,
+		category.SearchSettings,
 		category.Auth,
 	}
 
diff --git a/plugins/auth/e2e_test.go b/plugins/auth/e2e_test.go
@@ -28,6 +28,7 @@ var adminCategories = []category.Category{
 	category.Auth,
 	category.Functions,
 	category.ReactiveSearch,
+	category.SearchSettings,
 }
 
 var adminOps = []op.Operation{
@@ -54,6 +55,7 @@ var defaultAdminLimits = permission.Limits{
 	AuthLimit:           30,
 	FunctionsLimit:      30,
 	ReactiveSearchLimit: 30,
+	SearchSettingsLimit: 30,
 }
 
 var createPermissionResponse = map[string]interface{}{
diff --git a/plugins/permissions/e2e_test.go b/plugins/permissions/e2e_test.go
@@ -28,6 +28,7 @@ var adminCategories = []category.Category{
 	category.Suggestions,
 	category.Functions,
 	category.ReactiveSearch,
+	category.SearchSettings,
 	category.Auth,
 }
 
@@ -55,6 +56,7 @@ var defaultAdminLimits = permission.Limits{
 	AuthLimit:           30,
 	FunctionsLimit:      30,
 	ReactiveSearchLimit: 30,
+	SearchSettingsLimit: 30,
 }
 
 var createPermissionResponse = map[string]interface{}{
diff --git a/plugins/reindexer/dao.go b/plugins/reindexer/dao.go
@@ -4,9 +4,11 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"regexp"
 
 	log "github.com/sirupsen/logrus"
 
+	"github.com/appbaseio/arc/middleware/classify"
 	"github.com/appbaseio/arc/util"
 	es7 "github.com/olivere/elastic/v7"
 )
@@ -114,11 +116,18 @@ func reindex(ctx context.Context, sourceIndex string, config *reindexConfig, wai
 
 		if destinationIndex == "" {
 			// Fetch all the aliases of old index
-			aliases, err := aliasesOf(ctx, sourceIndex)
+			alias, err := aliasesOf(ctx, sourceIndex)
+
+			var aliases = []string{}
 			if err != nil {
 				return nil, fmt.Errorf(`error fetching aliases of index "%s": %v`, sourceIndex, err)
 			}
-			aliases = append(aliases, sourceIndex)
+
+			if alias == "" {
+				aliases = append(aliases, sourceIndex)
+			} else {
+				aliases = append(aliases, alias)
+			}
 
 			// Delete old index
 			err = deleteIndex(ctx, sourceIndex)
@@ -210,22 +219,28 @@ func settingsOf(ctx context.Context, indexName string) (map[string]interface{},
 	return settings, nil
 }
 
-func aliasesOf(ctx context.Context, indexName string) ([]string, error) {
+func aliasesOf(ctx context.Context, indexName string) (string, error) {
 	response, err := util.GetClient7().CatAliases().
 		Pretty(true).
 		Do(ctx)
 	if err != nil {
-		return nil, err
+		return "", err
 	}
 
-	var aliases []string
+	var alias = ""
+
+	// set alias for original index name only.
+	regex := ".*reindexed_[0-9]+"
+	r, _ := regexp.Compile(regex)
+
 	for _, row := range response {
-		if row.Index == indexName {
-			aliases = append(aliases, row.Alias)
+		// r.MatchString(indexName) this condition is added to handle existing alias which are created incorrectly
+		if row.Index == indexName && r.MatchString(indexName) {
+			alias = row.Alias
 		}
 	}
 
-	return aliases, nil
+	return alias, nil
 }
 
 func createIndex(ctx context.Context, indexName string, body map[string]interface{}) error {
@@ -276,6 +291,8 @@ func setAlias(ctx context.Context, indexName string, aliases ...string) error {
 		return fmt.Errorf(`unable to set aliases "%v" for index "%s"`, aliases, indexName)
 	}
 
+	// We only have one alias per index.
+	classify.SetIndexAlias(indexName, aliases[0])
 	return nil
 }
 
@@ -288,3 +305,52 @@ func getIndicesByAlias(ctx context.Context, alias string) ([]string, error) {
 	}
 	return response.IndicesByAlias(alias), nil
 }
+
+func getAliasedIndices(ctx context.Context) ([]AliasedIndices, error) {
+	var indicesList []AliasedIndices
+	indices, err := util.GetClient7().CatIndices().
+		Do(ctx)
+	if err != nil {
+		return indicesList, err
+	}
+
+	aliases, err := util.GetClient7().CatAliases().
+		Pretty(true).
+		Do(ctx)
+	if err != nil {
+		return indicesList, err
+	}
+
+	for _, index := range indices {
+		var indexStruct = AliasedIndices{
+			Health:       index.Health,
+			Status:       index.Status,
+			Index:        index.Index,
+			UUID:         index.UUID,
+			Pri:          index.Pri,
+			Rep:          index.Rep,
+			DocsCount:    index.DocsCount,
+			DocsDeleted:  index.DocsDeleted,
+			StoreSize:    index.StoreSize,
+			PriStoreSize: index.PriStoreSize,
+		}
+		var alias string
+		regex := ".*reindexed_[0-9]+"
+		r, _ := regexp.Compile(regex)
+
+		for _, row := range aliases {
+			if row.Index == index.Index && r.MatchString(index.Index) {
+				alias = row.Alias
+				break
+			}
+		}
+		if err == nil && alias != "" {
+			indexStruct.Alias = alias
+		}
+
+		indicesList = append(indicesList, indexStruct)
+
+	}
+
+	return indicesList, nil
+}
diff --git a/plugins/reindexer/handlers.go b/plugins/reindexer/handlers.go
@@ -60,6 +60,19 @@ func (rx *reindexer) reindexSrcToDest() http.HandlerFunc {
 	}
 }
 
+func (rx *reindexer) aliasedIndices() http.HandlerFunc {
+	return func(w http.ResponseWriter, req *http.Request) {
+		res, err := getAliasedIndices(req.Context())
+		if err != nil {
+			util.WriteBackError(w, "Unable to get aliased indices.\n"+err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		response, err := json.Marshal(res)
+		errorHandler(nil, w, response)
+	}
+}
+
 func errorHandler(err error, w http.ResponseWriter, response []byte) {
 	if err != nil {
 		log.Errorln(logTag, ":", err)
diff --git a/plugins/reindexer/reindexer.go b/plugins/reindexer/reindexer.go
@@ -32,6 +32,7 @@ func (rx *reindexer) Name() string {
 }
 
 func (rx *reindexer) InitFunc() error {
+	InitIndexAliasCache()
 	return nil
 }
 
diff --git a/plugins/reindexer/routes.go b/plugins/reindexer/routes.go
diff --git a/plugins/reindexer/util.go b/plugins/reindexer/util.go

Original file line number	Diff line number	Diff line change
`@@ -29,6 +29,7 @@ var (`
`29`	`29`	`category.Auth,`
`30`	`30`	`category.Functions,`
`31`	`31`	`category.ReactiveSearch,`
	`32`	`+ category.SearchSettings,`
`32`	`33`	`}`
`33`	`34`
`34`	`35`	`defaultOps = []op.Operation{`
`@@ -59,6 +60,7 @@ var (`
`59`	`60`	`AuthLimit: 10,`
`60`	`61`	`FunctionsLimit: 10,`
`61`	`62`	`ReactiveSearchLimit: 10,`
	`63`	`+ SearchSettingsLimit: 10,`
`62`	`64`	`}`
`63`	`65`
`64`	`66`	`defaultAdminLimits = Limits{`
`@@ -79,5 +81,6 @@ var (`
`79`	`81`	`AuthLimit: 30,`
`80`	`82`	`FunctionsLimit: 30,`
`81`	`83`	`ReactiveSearchLimit: 30,`
	`84`	`+ SearchSettingsLimit: 30,`
`82`	`85`	`}`
`83`	`86`	`)`
Original file line number	Diff line number	Diff line change
`@@ -71,6 +71,7 @@ type Limits struct {`
`71`	`71`	AuthLimit int64 `json:"auth_limit"`
`72`	`72`	FunctionsLimit int64 `json:"functions_limit"`
`73`	`73`	ReactiveSearchLimit int64 `json:"reactivesearch_limit"`
	`74`	+ SearchSettingsLimit int64 `json:"searchsettings_limit"`
`74`	`75`	`}`
`75`	`76`
`76`	`77`	`// Options is a function type used to define a permission's properties.`
`@@ -471,6 +472,8 @@ func (p *Permission) GetLimitFor(c category.Category) (int64, error) {`
`471`	`472`	`return p.Limits.FunctionsLimit, nil`
`472`	`473`	`case category.ReactiveSearch:`
`473`	`474`	`return p.Limits.ReactiveSearchLimit, nil`
	`475`	`+ case category.SearchSettings:`
	`476`	`+ return p.Limits.SearchSettingsLimit, nil`
`474`	`477`	`default:`
`475`	`478`	return -1, fmt.Errorf(`we do not rate limit "%s" category`, c)
`476`	`479`	`}`
`@@ -591,6 +594,10 @@ func (p *Permission) GetPatch(rolePatched bool) (map[string]interface{}, error)`
`591`	`594`	`limits["reactivesearch_limit"] = p.Limits.ReactiveSearchLimit`
`592`	`595`	`}`
`593`	`596`
	`597`	`+ if p.Limits.SearchSettingsLimit != 0 {`
	`598`	`+ limits["searchsettings_limit"] = p.Limits.SearchSettingsLimit`
	`599`	`+ }`
	`600`	`+`
`594`	`601`	`patch["limits"] = limits`
`595`	`602`	`}`
`596`	`603`	`if p.Description != "" {`
Original file line number	Diff line number	Diff line change
`@@ -28,6 +28,7 @@ var (`
`28`	`28`	`category.Suggestions,`
`29`	`29`	`category.Functions,`
`30`	`30`	`category.ReactiveSearch,`
	`31`	`+ category.SearchSettings,`
`31`	`32`	`category.Auth,`
`32`	`33`	`}`
`33`	`34`
Original file line number	Diff line number	Diff line change
`@@ -28,6 +28,7 @@ var adminCategories = []category.Category{`
`28`	`28`	`category.Auth,`
`29`	`29`	`category.Functions,`
`30`	`30`	`category.ReactiveSearch,`
	`31`	`+ category.SearchSettings,`
`31`	`32`	`}`
`32`	`33`
`33`	`34`	`var adminOps = []op.Operation{`
`@@ -54,6 +55,7 @@ var defaultAdminLimits = permission.Limits{`
`54`	`55`	`AuthLimit: 30,`
`55`	`56`	`FunctionsLimit: 30,`
`56`	`57`	`ReactiveSearchLimit: 30,`
	`58`	`+ SearchSettingsLimit: 30,`
`57`	`59`	`}`
`58`	`60`
`59`	`61`	`var createPermissionResponse = map[string]interface{}{`