Added contains and suffix search benchmarks (#17)

Author: filipecosta90

.github/workflows/publish.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        goos: [linux, darwin,windows]
+        goos: [linux, darwin]
         goarch: [amd64, arm64]
     steps:
       - uses: actions/checkout@v3

benchmark.go

@@ -30,6 +30,58 @@ func SearchBenchmark(queries []string, field string, idx index.Index, opts inter
 	}
 }
 
+func SuffixBenchmark(terms []string, field string, idx index.Index, prefixMinLen, prefixMaxLen int64, debug int) func() error {
+	counter := 0
+	fixedPrefixSize := false
+	if prefixMinLen == prefixMaxLen {
+		fixedPrefixSize = true
+	}
+	return func() error {
+		term := terms[counter%len(terms)]
+		var prefixSize int64 = prefixMinLen
+		if !fixedPrefixSize {
+			n := rand.Int63n(int64(prefixMaxLen - prefixMinLen))
+			prefixSize = prefixSize + n
+		}
+		for prefixSize > int64(len(term)) {
+			counter++
+			term = terms[counter%len(terms)]
+		}
+		term = term[len(term)-int(prefixSize):]
+		q := query.NewQuery(idx.GetName(), term).Limit(0, 5).SetFlags(query.QueryTypeSuffix).SetField(field)
+		_, _, err := idx.SuffixQuery(*q, debug)
+		counter++
+		return err
+	}
+}
+
+func ContainsBenchmark(terms []string, field string, idx index.Index, prefixMinLen, prefixMaxLen int64, debug int) func() error {
+	counter := 0
+	fixedPrefixSize := false
+	if prefixMinLen == prefixMaxLen {
+		fixedPrefixSize = true
+	}
+	return func() error {
+		term := terms[counter%len(terms)]
+		var prefixSize int64 = prefixMinLen
+		if !fixedPrefixSize {
+			n := rand.Int63n(int64(prefixMaxLen - prefixMinLen))
+			prefixSize = prefixSize + n
+		}
+		for prefixSize > int64(len(term)) {
+			counter++
+			term = terms[counter%len(terms)]
+		}
+		term = term[0:prefixSize]
+		term = "*" + term + "*"
+
+		q := query.NewQuery(idx.GetName(), term).Limit(0, 5).SetField(field)
+		_, _, err := idx.ContainsQuery(*q, debug)
+		counter++
+		return err
+	}
+}
+
 // SearchBenchmark returns a closure of a function for the benchmarker to run, using a given index
 // and options, on a set of queries
 func PrefixBenchmark(terms []string, field string, idx index.Index, prefixMinLen, prefixMaxLen int64, debug int) func() error {

index/elastic/elastic.go

@@ -310,6 +310,42 @@ func elasticParseResponse(r map[string]interface{}, verbose int, res *esapi.Resp
 	return hits
 }
 
+// https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-wildcard-query.html
+func (i *Index) ContainsQuery(q query.Query, verbose int) ([]index.Document, int, error) {
+	es := i.conn
+	query := map[string]interface{}{
+		"from": q.Paging.Offset,
+		"size": q.Paging.Num,
+		"query": map[string]interface{}{
+			"wildcard": map[string]interface{}{
+				q.Field: map[string]interface{}{
+					"value": q.Term,
+				},
+			},
+		},
+	}
+	hits, err := elasticSearchQuery(i.name, es, verbose, query)
+	return nil, hits, err
+}
+
+// https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-wildcard-query.html
+func (i *Index) SuffixQuery(q query.Query, verbose int) ([]index.Document, int, error) {
+	es := i.conn
+	query := map[string]interface{}{
+		"from": q.Paging.Offset,
+		"size": q.Paging.Num,
+		"query": map[string]interface{}{
+			"wildcard": map[string]interface{}{
+				q.Field: map[string]interface{}{
+					"value": q.Term,
+				},
+			},
+		},
+	}
+	hits, err := elasticSearchQuery(i.name, es, verbose, query)
+	return nil, hits, err
+}
+
 // https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-wildcard-query.html
 func (i *Index) WildCardQuery(q query.Query, verbose int) ([]index.Document, int, error) {
 	es := i.conn

index/index.go

@@ -11,7 +11,9 @@ type Index interface {
 	Index(documents []Document, options interface{}) error
 	FullTextQuerySingleField(query.Query, int) (docs []Document, total int, err error)
 	PrefixQuery(query.Query, int) (docs []Document, total int, err error)
+	SuffixQuery(query.Query, int) (docs []Document, total int, err error)
 	WildCardQuery(query.Query, int) (docs []Document, total int, err error)
+	ContainsQuery(q query.Query, debug int) (docs []Document, total int, err error)
 	Drop() error
 	DocumentCount() int64
 	Create() error

index/redisearch/index.go index/redisearch/redisearch.goindex/redisearch/index.go renamed to index/redisearch/redisearch.go

@@ -197,6 +197,14 @@ func (i *Index) Index(docs []index.Document, options interface{}) error {
 	return nil
 }
 
+func (i *Index) SuffixQuery(q query.Query, verbose int) (docs []index.Document, total int, err error) {
+	return i.FullTextQuerySingleField(q, verbose)
+}
+
+func (i *Index) ContainsQuery(q query.Query, verbose int) (docs []index.Document, total int, err error) {
+	return i.FullTextQuerySingleField(q, verbose)
+}
+
 func (i *Index) PrefixQuery(q query.Query, verbose int) (docs []index.Document, total int, err error) {
 	return i.FullTextQuerySingleField(q, verbose)
 }
@@ -213,6 +221,9 @@ func (i *Index) FullTextQuerySingleField(q query.Query, verbose int) (docs []ind
 	if q.Flags&query.QueryTypePrefix != 0 && term[len(term)-1] != '*' {
 		term = fmt.Sprintf("%s*", term)
 	}
+	if q.Flags&query.QueryTypeSuffix != 0 && term[0] != '*' {
+		term = fmt.Sprintf("*%s", term)
+	}
 	queryParam := term
 	if q.Field != "" {
 		queryParam = fmt.Sprintf("@%s:%s", q.Field, term)

main.go

@@ -31,6 +31,8 @@ const (
 	DEFAULT_DATASET           = EN_WIKI_DATASET
 	BENCHMARK_SEARCH          = "search"
 	BENCHMARK_PREFIX          = "prefix"
+	BENCHMARK_CONTAINS        = "contains"
+	BENCHMARK_SUFFIX          = "suffix"
 	BENCHMARK_WILDCARD        = "wildcard"
 	BENCHMARK_DEFAULT         = BENCHMARK_SEARCH
 	ENGINE_REDIS              = "redis"
@@ -93,7 +95,7 @@ func main() {
 	randomSeed := flag.Int64("seed", 12345, "PRNG seed.")
 	termStopWords := flag.String("stopwords", DEFAULT_STOPWORDS, "filtered stopwords for term creation")
 	dataset := flag.String("dataset", DEFAULT_DATASET, fmt.Sprintf("The dataset tp process. One of: [%s]", strings.Join([]string{EN_WIKI_DATASET, REDDIT_DATASET, PMC_DATASET}, "|")))
-	benchmark := flag.String("benchmark", "", fmt.Sprintf("The benchmark to run. One of: [%s]. If empty will not run.", strings.Join([]string{BENCHMARK_SEARCH, BENCHMARK_PREFIX, BENCHMARK_WILDCARD}, "|")))
+	benchmark := flag.String("benchmark", "", fmt.Sprintf("The benchmark to run. One of: [%s]. If empty will not run.", strings.Join([]string{BENCHMARK_SEARCH, BENCHMARK_PREFIX, BENCHMARK_WILDCARD, BENCHMARK_CONTAINS, BENCHMARK_SUFFIX}, "|")))
 
 	tlsSkipVerify := flag.Bool("tls-skip-verify", true, "Skip verification of server certificate.")
 	seconds := flag.Int("duration", 60, "number of seconds to run the benchmark")
@@ -194,6 +196,10 @@ func main() {
 		}
 		returnCode := 0
 		switch *benchmark {
+		case BENCHMARK_CONTAINS:
+			name := fmt.Sprintf("contains: %d terms", len(queries))
+			log.Println("Starting term-level queries benchmark: Type CONTAINS")
+			Benchmark(*conc, duration, &histogramMutex, *engine, name, *outfile, *reportingPeriod, w, ContainsBenchmark(queries, benchmarkQueryField, indexes[0], *termQueryPrefixMinLen, *termQueryPrefixMaxLen, *debugLevel))
 		case BENCHMARK_WILDCARD:
 			name := fmt.Sprintf("wildcard: %d terms", len(queries))
 			log.Println("Starting term-level queries benchmark: Type WILDCARD")
@@ -203,6 +209,10 @@ func main() {
 				log.Println(fmt.Sprintf("%s needs to be at least larger by 2 than min length given we want the wildcard to be present at the midle of the term. Forcing %s=%d", TERM_QUERY_MAX_LEN, TERM_QUERY_MAX_LEN, prefixMaxLen))
 			}
 			Benchmark(*conc, duration, &histogramMutex, *engine, name, *outfile, *reportingPeriod, w, WildcardBenchmark(queries, benchmarkQueryField, indexes[0], *termQueryPrefixMinLen, prefixMaxLen, *debugLevel))
+		case BENCHMARK_SUFFIX:
+			name := fmt.Sprintf("suffix: %d terms", len(queries))
+			log.Println("Starting term-level queries benchmark: Type SUFFIX")
+			Benchmark(*conc, duration, &histogramMutex, *engine, name, *outfile, *reportingPeriod, w, SuffixBenchmark(queries, benchmarkQueryField, indexes[0], *termQueryPrefixMinLen, *termQueryPrefixMaxLen, *debugLevel))
 		case BENCHMARK_PREFIX:
 			name := fmt.Sprintf("prefix: %d terms", len(queries))
 			log.Println("Starting term-level queries benchmark: Type PREFIX")

query/query.go

@@ -7,7 +7,8 @@ const (
 	QueryVerbatim     Flag = 0x1
 	QueryNoContent    Flag = 0x2
 	QueryTypePrefix   Flag = 0x3
-	QueryTypeWildcard Flag = 0x4
+	QueryTypeSuffix   Flag = 0x4
+	QueryTypeWildcard Flag = 0x5
 	// ... more to come!
 
 	DefaultOffset = 0