feat: accept multi-word phrases in the vocabulary (#1035)

Signed-off-by: Joseph Kato <joseph@jdkato.io>

Author: jdkato

internal/check/conditional.go

@@ -15,6 +15,7 @@ type Conditional struct {
 	First      string
 	Second     string
 	exceptRe   *regexp2.Regexp
+	phraseRe   *regexp2.Regexp
 	Ignorecase bool
 	Vocab      bool
 }
@@ -39,6 +40,7 @@ func NewConditional(cfg *core.Config, generic baseCheck, path string) (Condition
 		return rule, core.NewE201FromPosition(err.Error(), path, 1)
 	}
 	rule.exceptRe = re
+	rule.phraseRe = buildPhraseRe(rule.Exceptions, cfg.AcceptedTokens, rule.Vocab)
 
 	re, err = regexp2.CompileStd(rule.Second)
 	if err != nil {
@@ -91,7 +93,7 @@ func (c Conditional) Run(blk nlp.Block, f *core.File, cfg *core.Config) ([]core.
 			return alerts, err
 		}
 
-		if !core.StringInSlice(s, f.Sequences) && !isMatch(c.exceptRe, s) {
+		if !core.StringInSlice(s, f.Sequences) && !isMatch(c.exceptRe, s) && !withinPhrase(c.phraseRe, txt, loc) {
 			// If we've found one (e.g., "WHO") and we haven't marked it as
 			// being defined previously, send an Alert.
 			a, erra := makeAlert(c.Definition, loc, txt, cfg)

internal/check/definition.go

@@ -375,6 +375,53 @@ func updateExceptions(previous []string, current []string, vocab bool) (*regexp2
 	return &regexp2.Regexp{}, nil
 }
 
+// buildPhraseRe compiles a regex matching the multi-word entries among the
+// given exception terms (a rule's `exceptions` plus, when `vocab` is set, the
+// project's accepted vocabulary). It lets a rule suppress a finding that falls
+// within an accepted phrase even when the rule matched only one of the phrase's
+// component words -- e.g. `mea` within an accepted `mea culpa`. See #1035.
+//
+// Returns nil when there are no multi-word terms (or one fails to compile, in
+// which case `updateExceptions` surfaces the error).
+func buildPhraseRe(previous, current []string, vocab bool) *regexp2.Regexp {
+	terms := append([]string{}, previous...)
+	if vocab {
+		terms = append(terms, current...)
+	}
+
+	phrases := []string{}
+	for _, term := range terms {
+		if strings.ContainsAny(term, " \t") || strings.Contains(term, `\s`) {
+			phrases = append(phrases, term)
+		}
+	}
+
+	if len(phrases) == 0 {
+		return nil
+	}
+
+	re, err := regexp2.CompileStd(ignoreCase + `\b(?:` + strings.Join(phrases, "|") + `)\b`)
+	if err != nil {
+		return nil
+	}
+	return re
+}
+
+// withinPhrase reports whether the span `loc` falls entirely within a match of
+// `phraseRe` (an accepted multi-word phrase) in `txt`. Both `loc` and the
+// phrase spans are rune offsets, as returned by regexp2's FindAllStringIndex.
+func withinPhrase(phraseRe *regexp2.Regexp, txt string, loc []int) bool {
+	if phraseRe == nil {
+		return false
+	}
+	for _, span := range phraseRe.FindAllStringIndex(txt, -1) {
+		if loc[0] >= span[0] && loc[1] <= span[1] {
+			return true
+		}
+	}
+	return false
+}
+
 func decodeRule(input interface{}, output interface{}) error {
 	config := mapstructure.DecoderConfig{
 		ErrorUnused:      true,

internal/check/existence.go

@@ -18,6 +18,7 @@ type Existence struct {
 	// `exceptions` (`array`): An array of strings to be ignored.
 	Exceptions []string
 	exceptRe   *regexp2.Regexp
+	phraseRe   *regexp2.Regexp
 	pattern    *regexp2.Regexp
 	Append     bool
 	IgnoreCase bool
@@ -50,6 +51,7 @@ func NewExistence(cfg *core.Config, generic baseCheck, path string) (Existence,
 		return rule, core.NewE201FromPosition(err.Error(), path, 1)
 	}
 	rule.exceptRe = re
+	rule.phraseRe = buildPhraseRe(rule.Exceptions, cfg.AcceptedTokens, rule.Vocab && !rule.Nonword)
 
 	regex := makeRegexp(
 		cfg.WordTemplate,
@@ -90,7 +92,7 @@ func (e Existence) Run(blk nlp.Block, _ *core.File, cfg *core.Config) ([]core.Al
 		}
 
 		observed := strings.TrimSpace(converted)
-		if !isMatch(e.exceptRe, observed) {
+		if !isMatch(e.exceptRe, observed) && !withinPhrase(e.phraseRe, blk.Text, loc) {
 			a, erra := makeAlert(e.Definition, loc, blk.Text, cfg)
 			if erra != nil {
 				return alerts, erra

internal/check/repetition.go

@@ -20,6 +20,7 @@ type Repetition struct {
 	Exceptions []string
 
 	exceptRe *regexp2.Regexp
+	phraseRe *regexp2.Regexp
 	pattern  *regexp2.Regexp
 }
 
@@ -42,6 +43,7 @@ func NewRepetition(cfg *core.Config, generic baseCheck, path string) (Repetition
 		return rule, core.NewE201FromPosition(err.Error(), path, 1)
 	}
 	rule.exceptRe = re
+	rule.phraseRe = buildPhraseRe(rule.Exceptions, cfg.AcceptedTokens, rule.Vocab)
 
 	regex := ""
 	if rule.Ignorecase {
@@ -102,9 +104,8 @@ func (o Repetition) Run(blk nlp.Block, _ *core.File, cfg *core.Config) ([]core.A
 				// I almost forgot about that. That is important.
 				//
 				// All plans except a Personal plan can use Redis. Redis ...
-				if !isMatch(o.exceptRe, converted) {
-					floc := []int{ploc[0], loc[1]}
-
+				floc := []int{ploc[0], loc[1]}
+				if !isMatch(o.exceptRe, converted) && !withinPhrase(o.phraseRe, txt, floc) {
 					a, erra := makeAlert(o.Definition, floc, txt, cfg)
 					if erra != nil {
 						return alerts, erra

internal/check/spelling.go

@@ -35,6 +35,7 @@ type Spelling struct {
 	Dicpath      string
 	Threshold    int
 	exceptRe     *regexp2.Regexp
+	phraseRe     *regexp2.Regexp
 	gs           *spell.Checker
 	Custom       bool
 	Append       bool
@@ -85,6 +86,12 @@ func addExceptions(s *Spelling, generic baseCheck, cfg *core.Config) error { //n
 			ignoreCase + strings.Join(s.Exceptions, "|"))
 	}
 
+	// A multi-word term (e.g. `mea culpa`) is accepted only as a phrase; its
+	// component words are still spell-checked on their own. We mask these in
+	// `Run` via `phraseRe`, built from the same vocabulary as every other
+	// Vocab-aware rule. See #1035.
+	s.phraseRe = buildPhraseRe(nil, cfg.AcceptedTokens, true)
+
 	return nil
 }
 
@@ -173,16 +180,30 @@ func (s Spelling) Run(blk nlp.Block, _ *core.File, _ *core.Config) ([]core.Alert
 	// See https://github.com/errata-ai/vale/v2/issues/148.
 	txt = s.gs.Convert(txt)
 
+	// Mask any accepted multi-word phrases (e.g. `mea culpa`) so their
+	// component words aren't spell-checked individually, while the same words
+	// elsewhere still are. We replace each match with an equal-length run of
+	// spaces, which preserves the byte offsets of every other word. See #1035.
+	checkTxt := txt
+	if s.phraseRe != nil {
+		masked, err := s.phraseRe.ReplaceFunc(txt, func(m regexp2.Match) string {
+			return strings.Repeat(" ", len(m.String()))
+		}, -1, -1)
+		if err == nil {
+			checkTxt = masked
+		}
+	}
+
 OUTER:
-	for _, word := range nlp.WordTokenizer.Tokenize(txt) {
+	for _, word := range nlp.WordTokenizer.Tokenize(checkTxt) {
 		for _, filter := range s.Filters {
 			if filter.MatchString(word) {
 				continue OUTER
 			}
 		}
 
 		if !s.gs.Spell(word) && !isMatch(s.exceptRe, word) {
-			offset := strings.Index(txt, word)
+			offset := strings.Index(checkTxt, word)
 			loc := []int{offset, offset + len(word)}
 
 			a := core.Alert{Check: s.Name, Severity: s.Level, Span: loc,

internal/check/substitution.go

@@ -19,6 +19,7 @@ type Substitution struct {
 	repl       []string
 	Swap       map[string]string
 	exceptRe   *regexp2.Regexp
+	phraseRe   *regexp2.Regexp
 	pattern    *regexp2.Regexp
 	Ignorecase bool
 	Nonword    bool
@@ -50,6 +51,7 @@ func NewSubstitution(cfg *core.Config, generic baseCheck, path string) (Substitu
 		return rule, core.NewE201FromPosition(err.Error(), path, 1)
 	}
 	rule.exceptRe = re
+	rule.phraseRe = buildPhraseRe(rule.Exceptions, cfg.AcceptedTokens, rule.Vocab)
 
 	regex := makeRegexp(
 		cfg.WordTemplate,
@@ -136,7 +138,7 @@ func (s Substitution) Run(blk nlp.Block, _ *core.File, cfg *core.Config) ([]core
 				} else {
 					same = matchToken(expected, observed, false)
 				}
-				if !same && !isMatch(s.exceptRe, observed) {
+				if !same && !isMatch(s.exceptRe, observed) && !withinPhrase(s.phraseRe, txt, loc) {
 					action := s.Fields().Action
 					if action.Name == "replace" && len(action.Params) == 0 {
 						action.Params = getOptions(expected)

testdata/features/misc.feature

@@ -36,6 +36,17 @@ Feature: Misc
             test.md:19:5:Vale.Terms:Use 'MyProduct Enterprise' instead of 'MyProduct enterprise'.
             """
 
+    Scenario: Multi-word vocab phrases (#1035)
+        # An accepted phrase is exempt across every Vocab-aware rule (here both
+        # Vale.Spelling and a custom existence rule), while its component words
+        # are still flagged on their own.
+        When I use Vocab "Phrases"
+        Then the output should contain exactly:
+            """
+            test.md:3:8:Test.Place:Avoid 'place'.
+            test.md:5:3:Vale.Spelling:Did you really mean 'flooberg'?
+            """
+
     Scenario: Line Endings
         When I test "misc/line-endings"
         Then the output should contain exactly:

testdata/fixtures/vocab/Phrases/.vale.ini

@@ -0,0 +1,7 @@
+StylesPath = ../styles
+MinAlertLevel = suggestion
+
+Vocab = Phrases
+
+[*.md]
+BasedOnStyles = Vale, Test

testdata/fixtures/vocab/Phrases/test.md

@@ -0,0 +1,5 @@
+A flooberg place appeared today.
+
+Just a place on its own here.
+
+A flooberg by itself there.

testdata/fixtures/vocab/styles/Test/Place.yml

@@ -0,0 +1,5 @@
+extends: existence
+message: "Avoid 'place'."
+level: warning
+tokens:
+  - 'place'