MB-56946: Introduce AnalyzerV2 and Runtime registry

+ Introduce new analyzer interface (AnalyzerV2) capable of:
  + Generating values other than tokenStream, like embedding vectors
  + Error can be returned during the analysis

+ Update the analyzers registry to allow runtime registration/removal

(*) Note on Backward compatibility
  - Old Analyzers are all wrapped under AnalyzerAdapter to ensure
  backward compatibility.
  - Test cases which register analyzer in the cache and call the Analyze method
  must be updated to deal with error returned by it
  (Refer the test cases updated in the PR)

Author: moshaad7

analysis/benchmark_test.go

@@ -32,7 +32,10 @@ func BenchmarkAnalysis(b *testing.B) {
 			b.Fatal(err)
 		}
 
-		ts := analyzer.Analyze(bleveWikiArticle)
+		ts, err := analysis.AnalyzeForTokens(analyzer, bleveWikiArticle)
+		if err != nil {
+			b.Fatalf("error analyzing text: %v", err)
+		}
 		freqs := analysis.TokenFrequency(ts, nil, index.IncludeTermVectors)
 		if len(freqs) != 511 {
 			b.Errorf("expected %d freqs, got %d", 511, len(freqs))

analysis/lang/ar/analyzer_ar_test.go

@@ -175,7 +175,10 @@ func TestArabicAnalyzer(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if !reflect.DeepEqual(actual, test.output) {
 			t.Errorf("expected %v, got %v", test.output, actual)
 			t.Errorf("expected % x, got % x", test.output[0].Term, actual[0].Term)

analysis/lang/cjk/analyzer_cjk_test.go

@@ -617,7 +617,10 @@ func TestCJKAnalyzer(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if !reflect.DeepEqual(actual, test.output) {
 			t.Errorf("expected %v, got %v", test.output, actual)
 		}

analysis/lang/ckb/analyzer_ckb_test.go

@@ -69,7 +69,10 @@ func TestSoraniAnalyzer(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if !reflect.DeepEqual(actual, test.output) {
 			t.Errorf("expected %v, got %v", test.output, actual)
 		}

analysis/lang/ckb/sorani_stemmer_filter_test.go

@@ -26,7 +26,7 @@ func TestSoraniStemmerFilter(t *testing.T) {
 
 	// in order to match the lucene tests
 	// we will test with an analyzer, not just the stemmer
-	analyzer := analysis.DefaultAnalyzer{
+	analyzer := &analysis.DefaultAnalyzer{
 		Tokenizer: single.NewSingleTokenTokenizer(),
 		TokenFilters: []analysis.TokenFilter{
 			NewSoraniNormalizeFilter(),
@@ -283,7 +283,11 @@ func TestSoraniStemmerFilter(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Errorf("error analyzing input: %v", err)
+		}
+
 		if !reflect.DeepEqual(actual, test.output) {
 			t.Errorf("for input %s(% x)", test.input, test.input)
 			t.Errorf("\texpected:")

analysis/lang/da/analyzer_da_test.go

@@ -63,7 +63,10 @@ func TestDanishAnalyzer(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if !reflect.DeepEqual(actual, test.output) {
 			t.Errorf("expected %v, got %v", test.output, actual)
 		}

analysis/lang/de/analyzer_de_test.go

@@ -147,7 +147,10 @@ func TestGermanAnalyzer(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if !reflect.DeepEqual(actual, test.output) {
 			t.Errorf("expected %v, got %v", test.output, actual)
 		}

analysis/lang/en/analyzer_en_test.go

@@ -97,7 +97,10 @@ func TestEnglishAnalyzer(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if !reflect.DeepEqual(actual, test.output) {
 			t.Errorf("expected %v, got %v", test.output, actual)
 		}

analysis/lang/es/analyzer_es_test.go

@@ -114,7 +114,10 @@ func TestSpanishAnalyzer(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if !reflect.DeepEqual(actual, test.output) {
 			t.Errorf("expected %v, got %v", test.output, actual)
 		}

analysis/lang/fa/analyzer_fa_test.go

@@ -305,7 +305,10 @@ func TestPersianAnalyzerVerbs(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if len(actual) != len(test.output) {
 			t.Fatalf("expected length: %d, got %d", len(test.output), len(actual))
 		}
@@ -600,7 +603,10 @@ func TestPersianAnalyzerVerbsDefective(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if len(actual) != len(test.output) {
 			t.Fatalf("expected length: %d, got %d", len(test.output), len(actual))
 		}
@@ -671,7 +677,10 @@ func TestPersianAnalyzerOthers(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, test := range tests {
-		actual := analyzer.Analyze(test.input)
+		actual, err := analysis.AnalyzeForTokens(analyzer, test.input)
+		if err != nil {
+			t.Fatalf("error analyzing input: %v", err)
+		}
 		if len(actual) != len(test.output) {
 			t.Fatalf("expected length: %d, got %d", len(test.output), len(actual))
 		}