Merge pull request #650 from Kinnouts/feat/spell-check-batch

Feat/spell check batch

Author: UltiRequiem

apps/api/app/routes_v1.go

@@ -75,7 +75,7 @@ func registerV1Routes(ctx context.Context, r chi.Router, pool *pgxpool.Pool, rdb
 
 	if serviceEnabled(cfg, "text") {
 		textRouter := chi.NewRouter()
-		text.RegisterRoutes(textRouter, pool)
+		text.RegisterRoutes(textRouter, pool, cfg)
 		r.Mount("/text", textRouter)
 	}
 

apps/api/go.mod

@@ -58,7 +58,6 @@ require (
 	github.com/nyaruka/phonenumbers v1.7.4
 	github.com/pemistahl/lingua-go v1.4.0
 	github.com/ringsaturn/tzf v1.2.1
-	github.com/sajari/fuzzy v1.0.0
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/spf13/cobra v1.10.2
 	github.com/yuin/goldmark v1.8.2

apps/api/platform/config/config.go

@@ -16,6 +16,7 @@ type Config struct {
 	EnabledServices    string // comma-separated list of services to mount; empty = all
 	Environment        string // "development" | "staging" | "production"
 	SentryDSN          string
+	LanguageToolURL    string // base URL for LanguageTool spell-check service
 }
 
 func Load() Config {
@@ -33,6 +34,7 @@ func Load() Config {
 		EnabledServices:    envOrDefault("ENABLED_SERVICES", ""),
 		Environment:        envOrDefault("ENVIRONMENT", "development"),
 		SentryDSN:          envOrDefault("SENTRY_DSN", "https://df9023edf98442d4887d1040de81b768@issues.bobadilla.tech/1"),
+		LanguageToolURL:    envOrDefault("LANGUAGETOOL_URL", "http://localhost:8010"),
 	}
 }
 

apps/api/services/text/router.go

@@ -4,6 +4,7 @@ import (
 	"github.com/go-chi/chi/v5"
 	"github.com/jackc/pgx/v5/pgxpool"
 
+	"requiems-api/platform/config"
 	"requiems-api/services/text/detectlanguage"
 	"requiems-api/services/text/lorem"
 	"requiems-api/services/text/normalize"
@@ -14,14 +15,14 @@ import (
 	"requiems-api/services/text/words"
 )
 
-func RegisterRoutes(r chi.Router, pool *pgxpool.Pool) {
+func RegisterRoutes(r chi.Router, pool *pgxpool.Pool, cfg config.Config) {
 	wordsSvc := words.NewService(pool)
 	words.RegisterRoutes(r, wordsSvc)
 
 	loremSvc := lorem.NewService()
 	lorem.RegisterRoutes(r, loremSvc)
 
-	spellcheckSvc := spellcheck.NewService()
+	spellcheckSvc := spellcheck.NewService(cfg.LanguageToolURL)
 	spellcheck.RegisterRoutes(r, spellcheckSvc)
 
 	thesaurusSvc := thesaurus.NewService()

apps/api/services/text/spellcheck/service.go

@@ -1,189 +1,170 @@
 package spellcheck
 
 import (
-	"bufio"
-	"bytes"
-	_ "embed"
-	"errors"
+	"encoding/json"
 	"fmt"
-	"regexp"
+	"net/http"
+	"net/url"
 	"strings"
-	"unicode"
-
-	"github.com/sajari/fuzzy"
+	"sync"
+	"time"
 )
 
-//go:embed data/words.txt
-var wordListData []byte
+const defaultLanguageToolURL = "http://localhost:8010"
 
-// wordRe matches sequences of ASCII letters (words).
-var wordRe = regexp.MustCompile(`[a-zA-Z]+`)
+// ltResponse is the subset of the LanguageTool /v2/check response we need.
+type ltResponse struct {
+	Matches []ltMatch `json:"matches"`
+}
 
-// Correction describes a single spelling mistake and its suggested fix.
-type Correction struct {
-	Original  string `json:"original"`
-	Suggested string `json:"suggested"`
-	Position  int    `json:"position"`
+type ltMatch struct {
+	Offset       int             `json:"offset"`
+	Length       int             `json:"length"`
+	Replacements []ltReplacement `json:"replacements"`
 }
 
-// Result is the response payload for the spell check endpoint.
-type Result struct {
-	Corrected   string       `json:"corrected"`
-	Corrections []Correction `json:"corrections"`
+type ltReplacement struct {
+	Value string `json:"value"`
 }
 
-// Service performs spell checking and correction.
+// Service calls LanguageTool for spell checking.
 type Service struct {
-	model   *fuzzy.Model
-	initErr error
+	client  *http.Client
+	baseURL string
 }
 
-// NewService returns a new spellcheck Service. If the embedded word list
-// cannot be loaded the service is still valid but Check will return an error.
-func NewService() *Service {
-	m := fuzzy.NewModel()
-	m.SetThreshold(1)
-
-	var words []string
-	scanner := bufio.NewScanner(bytes.NewReader(wordListData))
-	for scanner.Scan() {
-		if w := strings.TrimSpace(scanner.Text()); w != "" {
-			words = append(words, w)
-		}
+// NewService returns a new spellcheck Service backed by LanguageTool.
+// baseURL defaults to http://localhost:8010 if empty.
+func NewService(baseURL string) *Service {
+	if baseURL == "" {
+		baseURL = defaultLanguageToolURL
 	}
-	if err := scanner.Err(); err != nil {
-		return &Service{initErr: fmt.Errorf("spellcheck: word list unavailable: %w", err)}
+	return &Service{
+		client:  &http.Client{Timeout: 10 * time.Second},
+		baseURL: strings.TrimRight(baseURL, "/"),
 	}
-	m.Train(words)
-	return &Service{model: m}
 }
 
-// errUnavailable is returned by Check when the service failed to initialise.
-var errUnavailable = errors.New("spellcheck: service unavailable")
-
-// Check inspects text for spelling mistakes and returns the corrected text
-// together with a list of individual corrections. It returns an error if the
-// service failed to initialise.
+// Check inspects text for spelling mistakes using LanguageTool and returns
+// the corrected text together with a list of individual corrections.
 func (s *Service) Check(text string) (Result, error) {
-	if s.initErr != nil {
-		return Result{}, errUnavailable
+	form := url.Values{}
+	form.Set("text", text)
+	form.Set("language", "en-US")
+
+	resp, err := s.client.Post(
+		s.baseURL+"/v2/check",
+		"application/x-www-form-urlencoded",
+		strings.NewReader(form.Encode()),
+	)
+	if err != nil {
+		return Result{}, fmt.Errorf("spellcheck: languagetool unreachable: %w", err)
 	}
+	defer resp.Body.Close()
 
-	var corrections []Correction
-
-	// Find all word positions so that we can replace in one pass.
-	locs := wordRe.FindAllStringIndex(text, -1)
-
-	// Build the corrected string by walking through every word location.
-	var builder strings.Builder
-	prev := 0
-	for _, loc := range locs {
-		start, end := loc[0], loc[1]
-		word := text[start:end]
-		lower := strings.ToLower(word)
-
-		suggestion := bestSuggestion(lower, s.model)
-
-		// bestSuggestion returns the input unchanged when the word is already
-		// in the model's vocabulary.
-		if suggestion != lower && suggestion != "" {
-			display := matchCase(word, suggestion)
-
-			corrections = append(corrections, Correction{
-				Original:  word,
-				Suggested: display,
-				Position:  len([]rune(text[:start])),
-			})
-
-			// Copy verbatim text before this word, then the correction.
-			builder.WriteString(text[prev:start])
-			builder.WriteString(display)
-		} else {
-			builder.WriteString(text[prev:end])
-		}
-		prev = end
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return Result{}, fmt.Errorf("spellcheck: languagetool returned status %d", resp.StatusCode)
 	}
-	// Append any trailing non-word characters.
-	builder.WriteString(text[prev:])
 
-	if corrections == nil {
-		corrections = []Correction{}
+	var ltResp ltResponse
+	if err := json.NewDecoder(resp.Body).Decode(&ltResp); err != nil {
+		return Result{}, fmt.Errorf("spellcheck: failed to decode response: %w", err)
 	}
 
-	return Result{
-		Corrected:   builder.String(),
-		Corrections: corrections,
-	}, nil
+	return buildResult(text, ltResp.Matches), nil
 }
 
-// bestSuggestion picks the most likely correction from the model's exhaustive
-// potentials by preferring (in order): lowest Levenshtein distance, then same
-// first letter as the input, then highest corpus score.
-func bestSuggestion(input string, m *fuzzy.Model) string {
-	potentials := m.Potentials(input, true)
-	if len(potentials) == 0 {
-		return input
+// CheckBatch processes multiple texts concurrently and returns spell-check
+// results for each. Up to 10 LanguageTool requests run in parallel to avoid
+// flooding the instance. Results are written to pre-allocated index slots so
+// input order is always preserved. If an individual text fails, its slot
+// receives a zero-value Result and the rest of the batch continues.
+func (s *Service) CheckBatch(texts []string) []Result {
+	const maxWorkers = 10
+
+	results := make([]Result, len(texts))
+	sem := make(chan struct{}, maxWorkers)
+	var wg sync.WaitGroup
+
+	for i, text := range texts {
+		wg.Add(1)
+		sem <- struct{}{}
+		go func(i int, text string) {
+			defer wg.Done()
+			defer func() { <-sem }()
+			defer func() {
+				if rec := recover(); rec != nil {
+					results[i] = Result{}
+				}
+			}()
+
+			r, err := s.Check(text)
+			if err != nil {
+				results[i] = Result{}
+				return
+			}
+			results[i] = r
+		}(i, text)
 	}
 
-	// Return the word unchanged when it is already in the vocabulary.
-	if p, ok := potentials[input]; ok && p.Leven == 0 {
-		return input
-	}
+	wg.Wait()
+	return results
+}
 
-	type candidate struct {
-		term  string
-		lev   int
-		score int
-	}
+// isValidSpan reports whether m describes a non-empty, non-overflowing span
+// within a slice of runeCount runes.
+func isValidSpan(m ltMatch, runeCount int) bool {
+	return len(m.Replacements) > 0 &&
+		m.Offset >= 0 &&
+		m.Length >= 0 &&
+		m.Offset < runeCount &&
+		m.Length <= runeCount-m.Offset
+}
 
-	var best candidate
-	first := true
-	for _, p := range potentials {
-		if p.Leven == 0 {
-			// Exact match – the word is correct, no suggestion needed.
-			return input
-		}
-		bonus := 0
-		if p.Term != "" && input != "" && p.Term[0] == input[0] {
-			bonus = 100
+// buildResult converts LanguageTool matches into our Result type.
+// Replacements are applied in reverse order to keep offsets valid.
+func buildResult(text string, matches []ltMatch) Result {
+	runes := []rune(text)
+	corrections := make([]Correction, 0, len(matches))
+
+	for _, m := range matches {
+		if !isValidSpan(m, len(runes)) {
+			continue
 		}
-		effectiveScore := p.Score + bonus
-		if first || p.Leven < best.lev || (p.Leven == best.lev && effectiveScore > best.score) {
-			best = candidate{term: p.Term, lev: p.Leven, score: effectiveScore}
-			first = false
+		// Collect up to 3 suggestions so callers can present a picker.
+		suggestions := make([]string, 0, 3)
+		for i, r := range m.Replacements {
+			if i >= 3 {
+				break
+			}
+			suggestions = append(suggestions, r.Value)
 		}
+		corrections = append(corrections, Correction{
+			Original:    string(runes[m.Offset : m.Offset+m.Length]),
+			Suggested:   m.Replacements[0].Value,
+			Suggestions: suggestions,
+			Position:    m.Offset,
+		})
 	}
 
-	return best.term
-}
-
-// matchCase applies the capitalisation pattern of src to dst so that, for
-// example, "Ths" → "This" and "THs" → "THIS".
-func matchCase(src, dst string) string {
-	if src == "" || dst == "" {
-		return dst
-	}
-
-	srcRunes := []rune(src)
-	dstRunes := []rune(dst)
-
-	// All-uppercase source → uppercase suggestion.
-	allUpper := true
-	for _, r := range srcRunes {
-		if unicode.IsLetter(r) && !unicode.IsUpper(r) {
-			allUpper = false
-			break
+	// Apply replacements in reverse order to preserve offsets.
+	corrected := make([]rune, len(runes))
+	copy(corrected, runes)
+	for i := len(matches) - 1; i >= 0; i-- {
+		m := matches[i]
+		if !isValidSpan(m, len(corrected)) {
+			continue
 		}
-	}
-	if allUpper {
-		return strings.ToUpper(dst)
+		repl := []rune(m.Replacements[0].Value)
+		corrected = append(corrected[:m.Offset], append(repl, corrected[m.Offset+m.Length:]...)...)
 	}
 
-	// First letter capitalised → capitalise first letter of suggestion.
-	if unicode.IsUpper(srcRunes[0]) {
-		dstRunes[0] = unicode.ToUpper(dstRunes[0])
-		return string(dstRunes)
+	if corrections == nil {
+		corrections = []Correction{}
 	}
 
-	return dst
+	return Result{
+		Corrected:   string(corrected),
+		Corrections: corrections,
+	}
 }