diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 20c0e582f184..76e203b00816 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -340,6 +340,9 @@ Optimizations * GITHUB#16370: Sort two-phase iterators by matchCost in DenseConjunctionBulkScorer. (Alan Woodward) +* GITHUB#16356: Cache stem results in SnowballFilter to avoid redundant stemmer invocations on + repeated tokens. 2.3-2.7x throughput improvement. Disableable via new constructor. (Costin Leau) + Bug Fixes --------------------- * GITHUB#16350: Disable bulk-scoring in monitor queries. (Alan Woodward) diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java index ebe32ca6df79..ce9f95daa455 100644 --- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.util.Objects; +import org.apache.lucene.analysis.CharArrayMap; import org.apache.lucene.analysis.LowerCaseFilter; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; @@ -47,14 +48,33 @@ */ public final class SnowballFilter extends TokenFilter { + private static final int DEFAULT_MAX_CACHE_SIZE = 1024; + private static final int MAX_CACHEABLE_LENGTH = 10; + private final SnowballStemmer stemmer; + private final int maxCacheSize; + private final CharArrayMap cache; private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class); public SnowballFilter(TokenStream input, SnowballStemmer stemmer) { + this(input, stemmer, DEFAULT_MAX_CACHE_SIZE); + } + + /** + * Creates a SnowballFilter with the specified cache size. The cache stores stem results keyed by + * the original token text. Only tokens with length ≤ {@value #MAX_CACHEABLE_LENGTH} are + * cached. Set {@code maxCacheSize} to 0 to disable caching. + */ + public SnowballFilter(TokenStream input, SnowballStemmer stemmer, int maxCacheSize) { super(input); this.stemmer = Objects.requireNonNull(stemmer, "stemmer"); + if (maxCacheSize < 0) { + throw new IllegalArgumentException("maxCacheSize must be >= 0, got: " + maxCacheSize); + } + this.maxCacheSize = maxCacheSize; + this.cache = maxCacheSize > 0 ? new CharArrayMap<>(Math.min(maxCacheSize, 256), false) : null; } /** @@ -68,8 +88,16 @@ public SnowballFilter(TokenStream input, SnowballStemmer stemmer) { * @param name the name of a stemmer */ public SnowballFilter(TokenStream in, String name) { + this(in, name, DEFAULT_MAX_CACHE_SIZE); + } + + /** Same as {@link #SnowballFilter(TokenStream, String)} with a configurable cache size. */ + public SnowballFilter(TokenStream in, String name, int maxCacheSize) { super(in); Objects.requireNonNull(name, "name"); + if (maxCacheSize < 0) { + throw new IllegalArgumentException("maxCacheSize must be >= 0, got: " + maxCacheSize); + } // it was called "German2" for eons, but snowball folded it into "German" and deleted "German2" // for now, don't annoy our users... if (name.equals("German2")) { @@ -85,6 +113,8 @@ public SnowballFilter(TokenStream in, String name) { } catch (ReflectiveOperationException e) { throw new IllegalArgumentException("Invalid stemmer class specified: " + name, e); } + this.maxCacheSize = maxCacheSize; + this.cache = maxCacheSize > 0 ? new CharArrayMap<>(Math.min(maxCacheSize, 256), false) : null; } /** Returns the next input Token, after being stemmed */ @@ -94,12 +124,34 @@ public boolean incrementToken() throws IOException { if (!keywordAttr.isKeyword()) { char[] termBuffer = termAtt.buffer(); final int length = termAtt.length(); + final boolean lookupCache = cache != null && length <= MAX_CACHEABLE_LENGTH; + + if (lookupCache) { + char[] cached = cache.get(termBuffer, 0, length); + if (cached != null) { + termAtt.copyBuffer(cached, 0, cached.length); + return true; + } + } + + char[] key = null; + if (lookupCache && cache.size() < maxCacheSize) { + key = new char[length]; + System.arraycopy(termBuffer, 0, key, 0, length); + } + stemmer.setCurrent(termBuffer, length); stemmer.stem(); final char[] finalTerm = stemmer.getCurrentBuffer(); final int newLength = stemmer.getCurrentBufferLength(); if (finalTerm != termBuffer) termAtt.copyBuffer(finalTerm, 0, newLength); else termAtt.setLength(newLength); + + if (key != null) { + char[] value = new char[newLength]; + System.arraycopy(finalTerm, 0, value, 0, newLength); + cache.put(key, value); + } } return true; } else { diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java index dfbdfe1f5bc5..e75815710b73 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java @@ -18,8 +18,10 @@ import java.io.IOException; import java.io.InputStream; +import java.io.StringReader; import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.List; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; @@ -142,4 +144,123 @@ protected TokenStreamComponents createComponents(String fieldName) { checkRandomData(random(), a, 20 * RANDOM_MULTIPLIER); a.close(); } + + public void testCacheProducesSameOutput() throws Exception { + String input = "he abhorred accents running acknowledging internationalization"; + + List uncached = stemAll(input, 0); + List cached = stemAll(input, 1024); + + assertEquals(uncached, cached); + } + + public void testCacheDisabled() throws Exception { + Analyzer a = + new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new MockTokenizer(); + return new TokenStreamComponents( + tokenizer, new SnowballFilter(tokenizer, "English", 0)); + } + }; + + assertAnalyzesTo(a, "he abhorred accents", new String[] {"he", "abhor", "accent"}); + a.close(); + } + + public void testLongTokensBypassCache() throws Exception { + String input = "acknowledging internationalization"; + + List uncached = stemAll(input, 0); + List cached = stemAll(input, 1024); + + assertEquals(uncached, cached); + assertEquals(2, cached.size()); + } + + public void testCacheAccumulatesAcrossReset() throws Exception { + String input = "he abhorred accents"; + String[] expected = new String[] {"he", "abhor", "accent"}; + + Analyzer a = + new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new MockTokenizer(); + return new TokenStreamComponents( + tokenizer, new SnowballFilter(tokenizer, "English", 1024)); + } + }; + + assertAnalyzesTo(a, input, expected); + assertAnalyzesTo(a, input, expected); + a.close(); + } + + public void testCacheActuallyHits() throws Exception { + int[] stemCallCount = {0}; + org.tartarus.snowball.ext.EnglishStemmer countingStemmer = + new org.tartarus.snowball.ext.EnglishStemmer() { + @Override + public boolean stem() { + stemCallCount[0]++; + return super.stem(); + } + }; + + Analyzer a = + new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new MockTokenizer(); + return new TokenStreamComponents( + tokenizer, new SnowballFilter(tokenizer, countingStemmer, 1024)); + } + }; + + try (TokenStream ts = a.tokenStream("field", new StringReader("running jumping"))) { + ts.reset(); + while (ts.incrementToken()) {} + ts.end(); + } + assertEquals(2, stemCallCount[0]); + + try (TokenStream ts = a.tokenStream("field", new StringReader("running jumping"))) { + CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); + ts.reset(); + List stems = new ArrayList<>(); + while (ts.incrementToken()) { + stems.add(termAtt.toString()); + } + ts.end(); + assertEquals(List.of("run", "jump"), stems); + } + assertEquals("Cache should prevent additional stem() calls", 2, stemCallCount[0]); + + a.close(); + } + + private List stemAll(String input, int cacheSize) throws IOException { + List result = new ArrayList<>(); + Analyzer a = + new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new MockTokenizer(); + return new TokenStreamComponents( + tokenizer, new SnowballFilter(tokenizer, "English", cacheSize)); + } + }; + try (TokenStream ts = a.tokenStream("field", new StringReader(input))) { + CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); + ts.reset(); + while (ts.incrementToken()) { + result.add(termAtt.toString()); + } + ts.end(); + } + a.close(); + return result; + } } diff --git a/lucene/benchmark-jmh/build.gradle b/lucene/benchmark-jmh/build.gradle index 6d64bb7a4f72..1e45c279a19a 100644 --- a/lucene/benchmark-jmh/build.gradle +++ b/lucene/benchmark-jmh/build.gradle @@ -19,6 +19,7 @@ description = 'Lucene JMH micro-benchmarking module' dependencies { moduleImplementation project(':lucene:core') + moduleImplementation project(':lucene:analysis:common') moduleImplementation project(':lucene:expressions') moduleImplementation project(':lucene:join') moduleImplementation project(':lucene:sandbox') diff --git a/lucene/benchmark-jmh/src/java/module-info.java b/lucene/benchmark-jmh/src/java/module-info.java index 8090c7554739..23afa840d1bf 100644 --- a/lucene/benchmark-jmh/src/java/module-info.java +++ b/lucene/benchmark-jmh/src/java/module-info.java @@ -23,6 +23,7 @@ requires jmh.core; requires jdk.unsupported; requires org.apache.lucene.core; + requires org.apache.lucene.analysis.common; requires org.apache.lucene.expressions; requires org.apache.lucene.join; requires org.apache.lucene.sandbox; diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/SnowballStemBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/SnowballStemBenchmark.java new file mode 100644 index 000000000000..97709138a7f7 --- /dev/null +++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/SnowballStemBenchmark.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.benchmark.jmh; + +import java.io.IOException; +import java.io.StringReader; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.core.WhitespaceTokenizer; +import org.apache.lucene.analysis.snowball.SnowballFilter; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; + +/** Benchmark for {@link SnowballFilter} comparing stemming throughput with and without caching. */ +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +@State(Scope.Thread) +@Warmup(iterations = 5, time = 1) +@Measurement(iterations = 10, time = 1) +@Fork(3) +public class SnowballStemBenchmark { + + @Param({"0", "1024", "4096", "8192"}) + int cacheSize; + + @Param({"English", "German"}) + String language; + + @Param({"500", "5000", "50000"}) + int vocabSize; + + private String corpus; + private Analyzer analyzer; + + @Setup(Level.Trial) + public void setup() { + corpus = buildZipfianCorpus(vocabSize, 10_000, 42); + analyzer = + new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new WhitespaceTokenizer(); + return new TokenStreamComponents( + tokenizer, new SnowballFilter(tokenizer, language, cacheSize)); + } + }; + } + + @TearDown(Level.Trial) + public void tearDown() { + if (analyzer != null) { + analyzer.close(); + } + } + + @Benchmark + public int stem() throws IOException { + int count = 0; + try (TokenStream ts = analyzer.tokenStream("field", new StringReader(corpus))) { + CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); + ts.reset(); + while (ts.incrementToken()) { + count += termAtt.length(); + } + ts.end(); + } + return count; + } + + private static String buildZipfianCorpus(int uniqueWords, int totalTokens, long seed) { + Random rng = new Random(seed); + String[] vocabulary = generateVocabulary(uniqueWords, rng); + + double[] weights = new double[uniqueWords]; + double totalWeight = 0; + for (int i = 0; i < uniqueWords; i++) { + weights[i] = 1.0 / (i + 1); + totalWeight += weights[i]; + } + + double[] cumulative = new double[uniqueWords]; + cumulative[0] = weights[0] / totalWeight; + for (int i = 1; i < uniqueWords; i++) { + cumulative[i] = cumulative[i - 1] + weights[i] / totalWeight; + } + + StringBuilder sb = new StringBuilder(totalTokens * 8); + for (int t = 0; t < totalTokens; t++) { + if (t > 0) { + sb.append(' '); + } + double r = rng.nextDouble(); + int idx = findBucket(cumulative, r); + sb.append(vocabulary[idx]); + } + return sb.toString(); + } + + private static int findBucket(double[] cumulative, double r) { + int lo = 0; + int hi = cumulative.length - 1; + while (lo < hi) { + int mid = (lo + hi) >>> 1; + if (cumulative[mid] < r) { + lo = mid + 1; + } else { + hi = mid; + } + } + return lo; + } + + private static String[] generateVocabulary(int count, Random rng) { + String[] suffixes = { + "ing", "tion", "ness", "ment", "able", "ible", "ful", "less", "ous", "ive", "ity", "ence", + "ance", "ly", "er", "ed", "es", "al", "ism", "ist" + }; + String[] roots = { + "act", + "run", + "walk", + "play", + "work", + "think", + "build", + "creat", + "develop", + "establish", + "manag", + "process", + "produc", + "communic", + "determin", + "recommend", + "understand", + "perform", + "consider", + "represent", + "organiz", + "recogniz", + "transform", + "implement", + "invest", + "increas", + "reduc", + "improv", + "measur", + "distribut", + "compar", + "contribut", + "demonstrat", + "environ", + "experienc", + "gener", + "govern", + "individu", + "interpret", + "legislat" + }; + + String[] words = new String[count]; + for (int i = 0; i < count; i++) { + String root = roots[rng.nextInt(roots.length)]; + if (rng.nextDouble() < 0.6) { + words[i] = root + suffixes[rng.nextInt(suffixes.length)]; + } else { + words[i] = root; + } + } + return words; + } +}