Skip to content

Commit 26a099b

Browse files
bietkulbietkul
authored
feat(reindex): add support to copy searchrelevancy and synonyms (#117)
1 parent d658704 commit 26a099b

8 files changed

Lines changed: 144 additions & 10 deletions

File tree

‎go.sum‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,7 @@ github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8l
239239
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
240240
github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
241241
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
242+
github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0=
242243
github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
243244
github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA=
244245
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=

‎plugins/reindexer/actions.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,12 @@ const (
66
Mappings Action = iota
77
Settings
88
Data
9+
SearchRelevancy
10+
Synonyms
911
)
1012

1113
func (o Action) String() string {
12-
return [...]string{"mappings", "settings", "data"}[o]
14+
return [...]string{"mappings", "settings", "data", "search_relevancy", "synonyms"}[o]
1315
}
1416

1517
type ReIndexOperation int

‎plugins/reindexer/dao.go‎

Lines changed: 64 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ func reindex(ctx context.Context, sourceIndex string, config *reindexConfig, wai
8686
// If mappings are not passed, we fetch the mappings of the old index.
8787
if config.Mappings == nil {
8888
found := util.IsExists(Mappings.String(), config.Action)
89-
if config.Action == nil || found {
89+
if len(config.Action) == 0 || found {
9090
config.Mappings, err = mappingsOf(ctx, sourceIndex)
9191
if err != nil {
9292
return nil, fmt.Errorf(`error fetching mappings of index "%s": %v`, sourceIndex, err)
@@ -97,7 +97,7 @@ func reindex(ctx context.Context, sourceIndex string, config *reindexConfig, wai
9797
// If settings are not passed, we fetch the settings of the old index.
9898
if config.Settings == nil {
9999
found := util.IsExists(Settings.String(), config.Action)
100-
if config.Action == nil || found {
100+
if len(config.Action) == 0 || found {
101101
config.Settings, err = settingsOf(ctx, sourceIndex)
102102
if err != nil {
103103
return nil, fmt.Errorf(`error fetching settings of index "%s": %v`, sourceIndex, err)
@@ -132,10 +132,32 @@ func reindex(ctx context.Context, sourceIndex string, config *reindexConfig, wai
132132
return nil, err
133133
}
134134

135+
/* Copy search relevancy settings if
136+
- `search_relevancy_settings` object is present
137+
- and action array has the `search_relevancy` action defined
138+
*/
139+
if config.SearchRelevancySettings != nil && util.IsExists(SearchRelevancy.String(), config.Action) {
140+
// Index a document in .searchrelevancy index for the destination `index`
141+
err := putSearchRelevancySettings(ctx, newIndexName, *config.SearchRelevancySettings)
142+
if err != nil {
143+
return nil, fmt.Errorf(`error while copying search relevancy settings: %v`, err)
144+
}
145+
}
146+
147+
/* Copy Synonyms if `synonyms` action is set in the action array
148+
*/
149+
if util.IsExists(Synonyms.String(), config.Action) {
150+
// Update synonyms by query
151+
err := updateSynonyms(ctx, sourceIndex, newIndexName)
152+
if err != nil {
153+
return nil, fmt.Errorf(`error while updating the synonyms: %v`, err)
154+
}
155+
}
156+
135157
found := util.IsExists(Data.String(), config.Action)
136158

137159
// do not copy data
138-
if !(config.Action == nil || found) {
160+
if !(len(config.Action) == 0 || found) {
139161
return nil, nil
140162
}
141163

@@ -479,3 +501,42 @@ func asyncReIndex(taskID, source, destination string, operation ReIndexOperation
479501
}
480502
}
481503
}
504+
505+
func putSearchRelevancySettings(ctx context.Context, docID string, record map[string]interface{}) error {
506+
_, err := util.GetClient7().
507+
Index().
508+
Refresh("wait_for").
509+
Index(getSearchRelevancyIndex()).
510+
BodyJson(record).
511+
Id(docID).
512+
Do(ctx)
513+
if err != nil {
514+
log.Errorln(logTag, ": error indexing searchrelevancy record for id=", docID, ":", err)
515+
return err
516+
}
517+
return nil
518+
}
519+
520+
func updateSynonyms(ctx context.Context, sourceIndex string, destinationIndex string) error {
521+
script := `
522+
if(ctx._source.index == null) {
523+
ctx._source.index = []
524+
}
525+
if(ctx._source.index instanceof String) {
526+
ctx._source.index = [ctx._source.index]
527+
}
528+
if (params.index != null) {
529+
if (ctx._source.index.indexOf(params.index) == -1) {
530+
ctx._source.index.add(params.index)
531+
}
532+
}`
533+
params := map[string]interface{}{
534+
"index": destinationIndex,
535+
}
536+
switch util.GetVersion() {
537+
case 6:
538+
return updateSynonymsEs6(ctx, script, sourceIndex, params)
539+
default:
540+
return updateSynonymsEs7(ctx, script, sourceIndex, params)
541+
}
542+
}

‎plugins/reindexer/dao_es6.go‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package reindexer
2+
3+
import (
4+
"context"
5+
6+
"github.com/appbaseio/arc/util"
7+
log "github.com/sirupsen/logrus"
8+
es6 "gopkg.in/olivere/elastic.v6"
9+
)
10+
11+
func updateSynonymsEs6(ctx context.Context, script string, index string, params map[string]interface{}) error {
12+
query := es6.NewTermQuery("index.keyword", index)
13+
_, err := util.GetClient6().
14+
UpdateByQuery().
15+
Type(typeName).
16+
Query(query).
17+
Index(getSynonymsIndex()).
18+
Script(es6.NewScript(script).Params(params)).
19+
Do(ctx)
20+
if err != nil {
21+
log.Errorln(logTag, ": error updating synonyms for index=", index, ":", err)
22+
return err
23+
}
24+
return nil
25+
}

‎plugins/reindexer/dao_es7.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package reindexer
2+
3+
import (
4+
"context"
5+
6+
"github.com/appbaseio/arc/util"
7+
es7 "github.com/olivere/elastic/v7"
8+
log "github.com/sirupsen/logrus"
9+
)
10+
11+
func updateSynonymsEs7(ctx context.Context, script string, index string, params map[string]interface{}) error {
12+
query := es7.NewTermQuery("index.keyword", index)
13+
_, err := util.GetClient7().
14+
UpdateByQuery().
15+
Query(query).
16+
Index(getSynonymsIndex()).
17+
Script(es7.NewScript(script).Params(params)).
18+
Do(ctx)
19+
if err != nil {
20+
log.Errorln(logTag, ": error updating synonyms for index=", index, ":", err)
21+
return err
22+
}
23+
return nil
24+
}

‎plugins/reindexer/handlers.go‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,13 @@ import (
1414
)
1515

1616
type reindexConfig struct {
17-
Mappings map[string]interface{} `json:"mappings"`
18-
Settings map[string]interface{} `json:"settings"`
19-
Include []string `json:"include_fields"`
20-
Exclude []string `json:"exclude_fields"`
21-
Types []string `json:"types"`
22-
Action []string `json:"action,omitempty"`
17+
Mappings map[string]interface{} `json:"mappings"`
18+
Settings map[string]interface{} `json:"settings"`
19+
SearchRelevancySettings *map[string]interface{} `json:"search_relevancy_settings"`
20+
Include []string `json:"include_fields"`
21+
Exclude []string `json:"exclude_fields"`
22+
Types []string `json:"types"`
23+
Action []string `json:"action,omitempty"`
2324
}
2425

2526
func (rx *reindexer) reindex() http.HandlerFunc {

‎plugins/reindexer/reindexer.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
const (
1111
logTag = "[reindexer]"
1212
envEsURL = "ES_CLUSTER_URL"
13+
typeName = "_doc"
1314
)
1415

1516
var (

‎plugins/reindexer/util.go‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package reindexer
33
import (
44
"context"
55
"fmt"
6+
"os"
67
"regexp"
78
"strconv"
89
"strings"
@@ -114,3 +115,21 @@ func IsReIndexInProcess(source, destination string) bool {
114115

115116
return false
116117
}
118+
119+
// Returns the index name for search relevancy
120+
func getSearchRelevancyIndex() string {
121+
searchRelevancyIndex := os.Getenv("SEARCH_RELEVANCY_ES_INDEX")
122+
if searchRelevancyIndex == "" {
123+
searchRelevancyIndex = ".searchrelevancy"
124+
}
125+
return searchRelevancyIndex
126+
}
127+
128+
// Returns the index name for synonyms
129+
func getSynonymsIndex() string {
130+
synonymsIndex := os.Getenv("SYNONYMS_ES_INDEX")
131+
if synonymsIndex == "" {
132+
synonymsIndex = ".synonyms"
133+
}
134+
return synonymsIndex
135+
}

0 commit comments

Comments
 (0)