OPENNLP-1883: Route TermAnalyzer factory-based stemming through CachingStemmer

Builder.stem(StemmerFactory) now wraps the factory in a CachingStemmer
(default capacity), with a stem(StemmerFactory, int) overload for
explicit cache sizing, and NormalizationProfile.matchingAnalyzer()
builds through the factory again so profile analyzers get the per-thread
stem cache for free. Analyzer results are unchanged; repeated words
resolve from the cache instead of being re-stemmed.

Author: krickert

opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/NormalizationProfile.java

@@ -80,7 +80,9 @@ public Stemmer newStemmer() {
   /**
    * Returns a matching analyzer for this language: NFC, case folding, the language's
    * {@linkplain #accentFold() diacritic fold} when it has one, then stemming. The returned
-   * analyzer is thread-safe: it stems through a thread-safe {@link SnowballStemmer}.
+   * analyzer is thread-safe, and repeated words resolve from a bounded per-thread stem cache
+   * instead of being re-stemmed (natural text is Zipf-distributed, so most tokens are cache
+   * hits).
    *
    * @return the analyzer.
    */
@@ -89,6 +91,6 @@ public TermAnalyzer matchingAnalyzer() {
     if (accentFold != null) {
       builder.transform(Dimension.ACCENT_FOLD, accentFold);
     }
-    return builder.stem(newStemmer()).build();
+    return builder.stem(stemmerFactory()).build();
   }
 }

opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TermAnalyzer.java

@@ -25,7 +25,7 @@
 import java.util.Set;
 
 import opennlp.tools.lemmatizer.Lemmatizer;
-import opennlp.tools.stemmer.SharingStemmer;
+import opennlp.tools.stemmer.CachingStemmer;
 import opennlp.tools.stemmer.Stemmer;
 import opennlp.tools.stemmer.StemmerFactory;
 import opennlp.tools.tokenize.uax29.WordTokenizer;
@@ -45,10 +45,10 @@
  * <p>An instance is immutable and is thread-safe when its configured transforms are. The built-in
  * character normalizers are stateless and the Snowball stemmers are thread-safe. When
  * {@link Dimension#STEM} is enabled through {@link Builder#stem(StemmerFactory)}, stemming is
- * routed through a {@link SharingStemmer} and the analyzer is safe to share across threads even
- * when the factory mints stateful stemmers. A raw {@link Stemmer} passed to
- * {@link Builder#stem(Stemmer)} is only safe when that stemmer is itself thread-safe or the
- * analyzer is confined to one thread.</p>
+ * routed through a {@link CachingStemmer}: the analyzer is safe to share across threads even when
+ * the factory mints stateful stemmers, and repeated words resolve from a bounded per-thread cache
+ * instead of being re-stemmed. A raw {@link Stemmer} passed to {@link Builder#stem(Stemmer)} is
+ * only safe when that stemmer is itself thread-safe or the analyzer is confined to one thread.</p>
  */
 public final class TermAnalyzer {
 
@@ -398,17 +398,31 @@ public Builder stem(Stemmer value) {
 
     /**
      * Enables {@link Dimension#STEM} through a {@link StemmerFactory}. The analyzer receives a
-     * {@link SharingStemmer} so it can be shared across threads.
+     * {@link CachingStemmer} with the {@linkplain CachingStemmer#DEFAULT_CAPACITY default cache
+     * capacity}: it can be shared across threads, and repeated words resolve from a bounded
+     * per-thread cache instead of being re-stemmed.
      *
      * @param factory The stemmer factory. Must not be {@code null}.
      * @return this builder
      * @throws IllegalArgumentException if {@code factory} is {@code null}.
      */
     public Builder stem(StemmerFactory factory) {
-      if (factory == null) {
-        throw new IllegalArgumentException("factory must not be null");
-      }
-      return stem(new SharingStemmer(factory));
+      return stem(factory, CachingStemmer.DEFAULT_CAPACITY);
+    }
+
+    /**
+     * Enables {@link Dimension#STEM} through a {@link StemmerFactory} with an explicit stem cache
+     * capacity, otherwise identical to {@link #stem(StemmerFactory)}.
+     *
+     * @param factory   The stemmer factory. Must not be {@code null}.
+     * @param cacheSize The maximum number of word-to-stem entries kept per thread; must be
+     *                  positive.
+     * @return this builder
+     * @throws IllegalArgumentException if {@code factory} is {@code null} or {@code cacheSize} is
+     *     not positive.
+     */
+    public Builder stem(StemmerFactory factory, int cacheSize) {
+      return stem(new CachingStemmer(factory, cacheSize));
     }
 
     /**

opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerTest.java

@@ -32,6 +32,8 @@
 import opennlp.tools.stemmer.PorterStemmer;
 import opennlp.tools.stemmer.Stemmer;
 import opennlp.tools.stemmer.StemmerFactory;
+import opennlp.tools.stemmer.snowball.SnowballStemmer;
+import opennlp.tools.stemmer.snowball.SnowballStemmerFactory;
 import opennlp.tools.util.Span;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -312,10 +314,29 @@ void testBuilderRejectsNullArguments() {
         () -> TermAnalyzer.builder().stem((Stemmer) null));
     assertThrows(IllegalArgumentException.class,
         () -> TermAnalyzer.builder().stem((StemmerFactory) null));
+    assertThrows(IllegalArgumentException.class,
+        () -> TermAnalyzer.builder().stem(new SnowballStemmerFactory(
+            SnowballStemmer.ALGORITHM.ENGLISH), 0));
     assertThrows(IllegalArgumentException.class, () -> TermAnalyzer.builder().lemmatize(null));
     assertThrows(IllegalArgumentException.class, () -> TermAnalyzer.builder().tokenizer(null));
   }
 
+  @Test
+  void testStemFactoryCachedResultsMatchDirectStemmer() {
+    final TermAnalyzer cached = TermAnalyzer.builder().caseFold()
+        .stem(new SnowballStemmerFactory(SnowballStemmer.ALGORITHM.ENGLISH)).build();
+    final TermAnalyzer direct = TermAnalyzer.builder().caseFold()
+        .stem(new SnowballStemmer(SnowballStemmer.ALGORITHM.ENGLISH)).build();
+
+    final String text = "Running runners kept running while the RUNNING continued";
+    final List<Term> cachedTerms = cached.analyze(text);
+    final List<Term> directTerms = direct.analyze(text);
+    assertEquals(directTerms.size(), cachedTerms.size());
+    for (int i = 0; i < directTerms.size(); i++) {
+      assertEquals(directTerms.get(i).at(Dimension.STEM), cachedTerms.get(i).at(Dimension.STEM));
+    }
+  }
+
   @Test
   void testMaxTokenLengthRejectsNonPositiveValues() {
     assertThrows(IllegalArgumentException.class, () -> TermAnalyzer.builder().maxTokenLength(0));