Allow to set custom regex to split phrases into words

Author: cyradin

README.md

@@ -6,12 +6,13 @@
 
 Before running the service, set the following environment variables:
 
-|Variable        | 	Description | Example |
-|----------------|------------- |---------|
-|SPELLCHECKER_DIR| 	Directory to store dictionaries |	/tmp/spellchecker |
-|SPELLCHECKER_AUTOSAVE_INTERVAL| 	Auto-save interval (Go time.Duration) | 5m |
-|HTTP_ADDR| 	HTTP server address and port | localhost:8011 |
-|LOG_LEVEL| 	Logging level |	info, debug |
+|Variable        | 	Description | Example | Default value | Required |
+|----------------|------------- |---------|---------------|----------|
+|SPELLCHECKER_DIR| 	Directory to store dictionaries |	/tmp/spellchecker | none | yes |
+|SPELLCHECKER_AUTOSAVE_INTERVAL| 	Auto-save interval (Go time.Duration) | 5m | none | no |
+|SPELLCHECKER_WORD_SPLIT_REGEXP| Regular expression used to split phrases by words | ['\pL]+ | ['\pL]+| no |
+|SPELLCHECKER_HTTP_ADDR| 	HTTP server address and port | localhost:8011 | localhost:8011 | no |
+|SPELLCHECKER_LOG_LEVEL| 	Logging level |	error | info | no |
 
 ## Swagger Docs
 

cmd/web/main.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"regexp"
 	"syscall"
 	"time"
 
@@ -33,7 +34,7 @@ func main() {
 
 	ctx = logger.WithContext(
 		ctx,
-		logger.New(GitCommit, os.Getenv("LOG_LEVEL")),
+		logger.New(GitCommit, os.Getenv("SPELLCHECKER_LOG_LEVEL")),
 	)
 
 	registry, err := initRegistry(ctx)
@@ -42,12 +43,20 @@ func main() {
 		os.Exit(1)
 	}
 
+	splitter, err := initWordSpliter()
+	if err != nil {
+		logger.FromContext(ctx).Error("init spellchecker error", "error", err)
+		os.Exit(1)
+	}
+
+	fmt.Println(splitter)
+
 	defer registry.SaveAll(ctx)
 
-	server := server.NewServer(ctx, registry)
+	server := server.NewServer(ctx, registry, splitter)
 
 	addr := defaultServerAddr
-	if a := os.Getenv("HTTP_ADDR"); a != "" {
+	if a := os.Getenv("SPELLCHECKER_HTTP_ADDR"); a != "" {
 		addr = a
 	}
 
@@ -116,3 +125,19 @@ func initRegistry(ctx context.Context) (*spellchecker.Registry, error) {
 
 	return result, nil
 }
+
+var defaultRegexp = regexp.MustCompile(`['\pL]+`)
+
+func initWordSpliter() (*regexp.Regexp, error) {
+	value := os.Getenv("SPELLCHECKER_WORD_SPLIT_REGEXP")
+	if value == "" {
+		return defaultRegexp, nil
+	}
+
+	result, err := regexp.Compile(value)
+	if err != nil {
+		return nil, fmt.Errorf("invalid SPELLCHECKER_WORD_SPLIT_REGEXP: %w", err)
+	}
+
+	return result, nil
+}

internal/routes/dictionary_fix.go

@@ -34,9 +34,7 @@ type SpellcheckerSuggestion struct {
 	Score float64 `json:"score" description:"Confidence score of the suggestion."`
 }
 
-var wordSymbols = regexp.MustCompile(`[-\pL]+`)
-
-func dictionaryFix(registry *spellchecker.Registry) usecase.Interactor {
+func dictionaryFix(registry *spellchecker.Registry, splitter *regexp.Regexp) usecase.Interactor {
 	const (
 		errorUnknownWord = "unknown_word"
 		errorInvalidWord = "invalid_word"
@@ -54,7 +52,7 @@ func dictionaryFix(registry *spellchecker.Registry) usecase.Interactor {
 			return nil
 		}
 
-		matches := wordSymbols.FindAllStringIndex(input.Text, -1)
+		matches := splitter.FindAllStringIndex(input.Text, -1)
 		fixes := make([]Fix, 0, len(matches))
 
 		for _, match := range matches {

internal/routes/dictionary_item.go

@@ -3,6 +3,7 @@ package routes
 import (
 	"context"
 	"errors"
+	"regexp"
 
 	"github.com/f1monkey/spellchecker-web/internal/spellchecker"
 	"github.com/swaggest/usecase"
@@ -24,7 +25,7 @@ type DictionaryItemAddResponse struct {
 	Words int `json:"words" description:"Number of phrases successfully added."`
 }
 
-func dictionaryItemAdd(registry *spellchecker.Registry) usecase.Interactor {
+func dictionaryItemAdd(registry *spellchecker.Registry, splitter *regexp.Regexp) usecase.Interactor {
 	u := usecase.NewInteractor(func(ctx context.Context, input DictionaryItemAddRequest, output *DictionaryItemAddResponse) error {
 		sc, err := registry.Get(input.Code)
 		if errors.Is(spellchecker.ErrNotFound, err) {
@@ -37,7 +38,7 @@ func dictionaryItemAdd(registry *spellchecker.Registry) usecase.Interactor {
 
 		for i := range input.Phrases {
 
-			words := wordSymbols.FindAllString(input.Phrases[i].Text, -1)
+			words := splitter.FindAllString(input.Phrases[i].Text, -1)
 			if len(words) == 0 {
 				continue
 			}

internal/routes/routes.go

@@ -2,6 +2,7 @@ package routes
 
 import (
 	"net/http"
+	"regexp"
 
 	"github.com/f1monkey/spellchecker-web/internal/spellchecker"
 	"github.com/go-chi/chi/v5"
@@ -10,32 +11,32 @@ import (
 
 type EmptyResponse struct{}
 
-func Routes(registry *spellchecker.Registry) func(r chi.Router) {
+func Routes(registry *spellchecker.Registry, splitter *regexp.Regexp) func(r chi.Router) {
 	return func(r chi.Router) {
-		r.Route("/dictionaries", dictionaryRoutes(registry))
+		r.Route("/dictionaries", dictionaryRoutes(registry, splitter))
 	}
 }
 
-func dictionaryRoutes(registry *spellchecker.Registry) func(r chi.Router) {
+func dictionaryRoutes(registry *spellchecker.Registry, splitter *regexp.Regexp) func(r chi.Router) {
 	return func(r chi.Router) {
 		r.Method(http.MethodPost, "/{code}", nethttp.NewHandler(
-			dictionaryCreate(registry)),
-		)
+			dictionaryCreate(registry),
+		))
 
 		r.Method(http.MethodDelete, "/{code}", nethttp.NewHandler(
-			dictionaryDelete(registry)),
-		)
+			dictionaryDelete(registry),
+		))
 
 		r.Method(http.MethodPost, "/{code}/save", nethttp.NewHandler(
-			dictionarySave(registry)),
-		)
+			dictionarySave(registry),
+		))
 
 		r.Method(http.MethodPost, "/{code}/add", nethttp.NewHandler(
-			dictionaryItemAdd(registry)),
-		)
+			dictionaryItemAdd(registry, splitter),
+		))
 
 		r.Method(http.MethodPost, "/{code}/fix", nethttp.NewHandler(
-			dictionaryFix(registry)),
-		)
+			dictionaryFix(registry, splitter),
+		))
 	}
 }

internal/spellchecker/registry.go

@@ -36,33 +36,26 @@ func NewRegistry(ctx context.Context, dir string) (*Registry, error) {
 		return nil, err
 	}
 
-	items := make(map[string]RegistryItem)
+	result := &Registry{
+		dir:   dir,
+		items: make(map[string]RegistryItem),
+	}
 
 	for _, f := range files {
-		buf, err := os.ReadFile(path.Join(dir, f.Name()))
-		if err != nil {
-			logger.FromContext(ctx).Error("registry: read file err", "file", f, "error", err)
-			continue
-		}
-
-		var item RegistryItem
+		code, _ := strings.CutSuffix(f.Name(), extension)
 
-		if err := json.Unmarshal(buf, &item); err != nil {
-			logger.FromContext(ctx).Error("registry: unable to initalize registry item", "file", f, "error", err)
+		item, err := result.doLoad(code)
+		if err != nil {
+			logger.FromContext(ctx).Error("registry: dictionary load error", "code", code, "error", err)
 			continue
 		}
 
-		code, _ := strings.CutSuffix(f.Name(), extension)
-
 		logger.FromContext(ctx).Info("registry: loaded dictionary", "dictionary", code)
 
-		items[code] = item
+		result.items[code] = item
 	}
 
-	return &Registry{
-		dir:   dir,
-		items: items,
-	}, nil
+	return result, nil
 }
 
 func (r *Registry) AutoSave(ctx context.Context, interval time.Duration) {
@@ -182,6 +175,21 @@ func (r *Registry) doSave(code string) error {
 	return nil
 }
 
+func (r *Registry) doLoad(code string) (RegistryItem, error) {
+	buf, err := os.ReadFile(fullPath(r.dir, code))
+	if err != nil {
+		return RegistryItem{}, err
+	}
+
+	var item RegistryItem
+
+	if err := json.Unmarshal(buf, &item); err != nil {
+		return RegistryItem{}, err
+	}
+
+	return item, nil
+}
+
 func findDictionaries(dir string) ([]fs.DirEntry, error) {
 	files, err := os.ReadDir(dir)
 	if err != nil {

internal/spellchecker/registry_test.go

@@ -14,24 +14,9 @@ import (
 func Test_NewRegistry(t *testing.T) {
 	t.Parallel()
 
-	createTestFile := func(t *testing.T, dir string, name string) {
-		t.Helper()
-
-		sc, err := spellchecker.New("abc")
-		require.NoError(t, err)
-
-		item := RegistryItem{
-			Spellchecker: sc,
-		}
-
-		data, err := json.Marshal(&item)
-		require.NoError(t, err)
-
-		err = os.WriteFile(path.Join(dir, fileName(name)), data, 0755)
-		require.NoError(t, err)
-	}
-
 	t.Run("no files", func(t *testing.T) {
+		t.Parallel()
+
 		dir := t.TempDir()
 
 		result, err := NewRegistry(context.Background(), dir)
@@ -41,6 +26,8 @@ func Test_NewRegistry(t *testing.T) {
 	})
 
 	t.Run("one file", func(t *testing.T) {
+		t.Parallel()
+
 		dir := t.TempDir()
 
 		createTestFile(t, dir, "code")
@@ -52,6 +39,8 @@ func Test_NewRegistry(t *testing.T) {
 	})
 
 	t.Run("two files, ok", func(t *testing.T) {
+		t.Parallel()
+
 		dir := t.TempDir()
 
 		createTestFile(t, dir, "code1")
@@ -65,6 +54,8 @@ func Test_NewRegistry(t *testing.T) {
 	})
 
 	t.Run("two files, one has invalid extension", func(t *testing.T) {
+		t.Parallel()
+
 		dir := t.TempDir()
 
 		createTestFile(t, dir, "code1")
@@ -81,6 +72,8 @@ func Test_NewRegistry(t *testing.T) {
 	})
 
 	t.Run("two files, one is corrupted", func(t *testing.T) {
+		t.Parallel()
+
 		dir := t.TempDir()
 
 		createTestFile(t, dir, "code1")
@@ -261,3 +254,20 @@ func Test_Registry_Save(t *testing.T) {
 		require.Contains(t, r2.items, code)
 	})
 }
+
+func createTestFile(t *testing.T, dir string, name string) {
+	t.Helper()
+
+	sc, err := spellchecker.New("abc")
+	require.NoError(t, err)
+
+	item := RegistryItem{
+		Spellchecker: sc,
+	}
+
+	data, err := json.Marshal(&item)
+	require.NoError(t, err)
+
+	err = os.WriteFile(path.Join(dir, fileName(name)), data, 0755)
+	require.NoError(t, err)
+}

server.go

@@ -2,6 +2,7 @@ package web
 
 import (
 	"context"
+	"regexp"
 
 	"github.com/f1monkey/spellchecker-web/internal/routes"
 	"github.com/f1monkey/spellchecker-web/internal/spellchecker"
@@ -10,14 +11,14 @@ import (
 	swgui "github.com/swaggest/swgui/v5emb"
 )
 
-func NewServer(appCtx context.Context, registry *spellchecker.Registry) *web.Service {
+func NewServer(appCtx context.Context, registry *spellchecker.Registry, splitter *regexp.Regexp) *web.Service {
 	s := web.NewService(openapi31.NewReflector())
 
 	s.OpenAPISchema().SetTitle("Spellchecker")
 	s.OpenAPISchema().SetDescription("To fix words")
 	s.OpenAPISchema().SetVersion("v1")
 
-	s.Route("/v1", routes.Routes(registry))
+	s.Route("/v1", routes.Routes(registry, splitter))
 
 	// Swagger UI endpoint at /docs.
 	s.Docs("/docs", swgui.New)