From afac1d99cf399a54e381a3cf8ddf674fd0e32d4e Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Fri, 3 Jul 2026 19:53:00 +0300 Subject: [PATCH 1/6] Cache stem results in SnowballFilter Avoid redundant Snowball stemmer invocations on repeated tokens via insert-only CharArrayMap cache; 2.3-2.7x faster. Default 1024 entries (~75 KB), disableable via constructor. --- lucene/CHANGES.txt | 3 + .../analysis/snowball/SnowballFilter.java | 52 +++++ .../analysis/snowball/TestSnowball.java | 121 +++++++++++ lucene/benchmark-jmh/build.gradle | 1 + .../benchmark-jmh/src/java/module-info.java | 1 + .../benchmark/jmh/SnowballStemBenchmark.java | 201 ++++++++++++++++++ 6 files changed, 379 insertions(+) create mode 100644 lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/SnowballStemBenchmark.java diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 1d1d5a8154a7..d243a7153b75 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -323,6 +323,9 @@ Optimizations * GITHUB#16283: Use Panama Vector API to SIMD-evaluate fixed-cardinality sorted numeric range queries in rangeIntoBitSet. (Costin Leau) +* GITHUB#XXXX: Cache stem results in SnowballFilter to avoid redundant stemmer invocations on + repeated tokens. 2.3-2.7x throughput improvement. Disableable via new constructor. (Costin Leau) + Bug Fixes --------------------- diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java index ebe32ca6df79..ce9f95daa455 100644 --- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.util.Objects; +import org.apache.lucene.analysis.CharArrayMap; import org.apache.lucene.analysis.LowerCaseFilter; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; @@ -47,14 +48,33 @@ */ public final class SnowballFilter extends TokenFilter { + private static final int DEFAULT_MAX_CACHE_SIZE = 1024; + private static final int MAX_CACHEABLE_LENGTH = 10; + private final SnowballStemmer stemmer; + private final int maxCacheSize; + private final CharArrayMap cache; private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class); public SnowballFilter(TokenStream input, SnowballStemmer stemmer) { + this(input, stemmer, DEFAULT_MAX_CACHE_SIZE); + } + + /** + * Creates a SnowballFilter with the specified cache size. The cache stores stem results keyed by + * the original token text. Only tokens with length ≤ {@value #MAX_CACHEABLE_LENGTH} are + * cached. Set {@code maxCacheSize} to 0 to disable caching. + */ + public SnowballFilter(TokenStream input, SnowballStemmer stemmer, int maxCacheSize) { super(input); this.stemmer = Objects.requireNonNull(stemmer, "stemmer"); + if (maxCacheSize < 0) { + throw new IllegalArgumentException("maxCacheSize must be >= 0, got: " + maxCacheSize); + } + this.maxCacheSize = maxCacheSize; + this.cache = maxCacheSize > 0 ? new CharArrayMap<>(Math.min(maxCacheSize, 256), false) : null; } /** @@ -68,8 +88,16 @@ public SnowballFilter(TokenStream input, SnowballStemmer stemmer) { * @param name the name of a stemmer */ public SnowballFilter(TokenStream in, String name) { + this(in, name, DEFAULT_MAX_CACHE_SIZE); + } + + /** Same as {@link #SnowballFilter(TokenStream, String)} with a configurable cache size. */ + public SnowballFilter(TokenStream in, String name, int maxCacheSize) { super(in); Objects.requireNonNull(name, "name"); + if (maxCacheSize < 0) { + throw new IllegalArgumentException("maxCacheSize must be >= 0, got: " + maxCacheSize); + } // it was called "German2" for eons, but snowball folded it into "German" and deleted "German2" // for now, don't annoy our users... if (name.equals("German2")) { @@ -85,6 +113,8 @@ public SnowballFilter(TokenStream in, String name) { } catch (ReflectiveOperationException e) { throw new IllegalArgumentException("Invalid stemmer class specified: " + name, e); } + this.maxCacheSize = maxCacheSize; + this.cache = maxCacheSize > 0 ? new CharArrayMap<>(Math.min(maxCacheSize, 256), false) : null; } /** Returns the next input Token, after being stemmed */ @@ -94,12 +124,34 @@ public boolean incrementToken() throws IOException { if (!keywordAttr.isKeyword()) { char[] termBuffer = termAtt.buffer(); final int length = termAtt.length(); + final boolean lookupCache = cache != null && length <= MAX_CACHEABLE_LENGTH; + + if (lookupCache) { + char[] cached = cache.get(termBuffer, 0, length); + if (cached != null) { + termAtt.copyBuffer(cached, 0, cached.length); + return true; + } + } + + char[] key = null; + if (lookupCache && cache.size() < maxCacheSize) { + key = new char[length]; + System.arraycopy(termBuffer, 0, key, 0, length); + } + stemmer.setCurrent(termBuffer, length); stemmer.stem(); final char[] finalTerm = stemmer.getCurrentBuffer(); final int newLength = stemmer.getCurrentBufferLength(); if (finalTerm != termBuffer) termAtt.copyBuffer(finalTerm, 0, newLength); else termAtt.setLength(newLength); + + if (key != null) { + char[] value = new char[newLength]; + System.arraycopy(finalTerm, 0, value, 0, newLength); + cache.put(key, value); + } } return true; } else { diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java index dfbdfe1f5bc5..e75815710b73 100644 --- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java +++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java @@ -18,8 +18,10 @@ import java.io.IOException; import java.io.InputStream; +import java.io.StringReader; import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.List; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; @@ -142,4 +144,123 @@ protected TokenStreamComponents createComponents(String fieldName) { checkRandomData(random(), a, 20 * RANDOM_MULTIPLIER); a.close(); } + + public void testCacheProducesSameOutput() throws Exception { + String input = "he abhorred accents running acknowledging internationalization"; + + List uncached = stemAll(input, 0); + List cached = stemAll(input, 1024); + + assertEquals(uncached, cached); + } + + public void testCacheDisabled() throws Exception { + Analyzer a = + new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new MockTokenizer(); + return new TokenStreamComponents( + tokenizer, new SnowballFilter(tokenizer, "English", 0)); + } + }; + + assertAnalyzesTo(a, "he abhorred accents", new String[] {"he", "abhor", "accent"}); + a.close(); + } + + public void testLongTokensBypassCache() throws Exception { + String input = "acknowledging internationalization"; + + List uncached = stemAll(input, 0); + List cached = stemAll(input, 1024); + + assertEquals(uncached, cached); + assertEquals(2, cached.size()); + } + + public void testCacheAccumulatesAcrossReset() throws Exception { + String input = "he abhorred accents"; + String[] expected = new String[] {"he", "abhor", "accent"}; + + Analyzer a = + new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new MockTokenizer(); + return new TokenStreamComponents( + tokenizer, new SnowballFilter(tokenizer, "English", 1024)); + } + }; + + assertAnalyzesTo(a, input, expected); + assertAnalyzesTo(a, input, expected); + a.close(); + } + + public void testCacheActuallyHits() throws Exception { + int[] stemCallCount = {0}; + org.tartarus.snowball.ext.EnglishStemmer countingStemmer = + new org.tartarus.snowball.ext.EnglishStemmer() { + @Override + public boolean stem() { + stemCallCount[0]++; + return super.stem(); + } + }; + + Analyzer a = + new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new MockTokenizer(); + return new TokenStreamComponents( + tokenizer, new SnowballFilter(tokenizer, countingStemmer, 1024)); + } + }; + + try (TokenStream ts = a.tokenStream("field", new StringReader("running jumping"))) { + ts.reset(); + while (ts.incrementToken()) {} + ts.end(); + } + assertEquals(2, stemCallCount[0]); + + try (TokenStream ts = a.tokenStream("field", new StringReader("running jumping"))) { + CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); + ts.reset(); + List stems = new ArrayList<>(); + while (ts.incrementToken()) { + stems.add(termAtt.toString()); + } + ts.end(); + assertEquals(List.of("run", "jump"), stems); + } + assertEquals("Cache should prevent additional stem() calls", 2, stemCallCount[0]); + + a.close(); + } + + private List stemAll(String input, int cacheSize) throws IOException { + List result = new ArrayList<>(); + Analyzer a = + new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new MockTokenizer(); + return new TokenStreamComponents( + tokenizer, new SnowballFilter(tokenizer, "English", cacheSize)); + } + }; + try (TokenStream ts = a.tokenStream("field", new StringReader(input))) { + CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); + ts.reset(); + while (ts.incrementToken()) { + result.add(termAtt.toString()); + } + ts.end(); + } + a.close(); + return result; + } } diff --git a/lucene/benchmark-jmh/build.gradle b/lucene/benchmark-jmh/build.gradle index 6d64bb7a4f72..1e45c279a19a 100644 --- a/lucene/benchmark-jmh/build.gradle +++ b/lucene/benchmark-jmh/build.gradle @@ -19,6 +19,7 @@ description = 'Lucene JMH micro-benchmarking module' dependencies { moduleImplementation project(':lucene:core') + moduleImplementation project(':lucene:analysis:common') moduleImplementation project(':lucene:expressions') moduleImplementation project(':lucene:join') moduleImplementation project(':lucene:sandbox') diff --git a/lucene/benchmark-jmh/src/java/module-info.java b/lucene/benchmark-jmh/src/java/module-info.java index 8090c7554739..23afa840d1bf 100644 --- a/lucene/benchmark-jmh/src/java/module-info.java +++ b/lucene/benchmark-jmh/src/java/module-info.java @@ -23,6 +23,7 @@ requires jmh.core; requires jdk.unsupported; requires org.apache.lucene.core; + requires org.apache.lucene.analysis.common; requires org.apache.lucene.expressions; requires org.apache.lucene.join; requires org.apache.lucene.sandbox; diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/SnowballStemBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/SnowballStemBenchmark.java new file mode 100644 index 000000000000..97709138a7f7 --- /dev/null +++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/SnowballStemBenchmark.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.benchmark.jmh; + +import java.io.IOException; +import java.io.StringReader; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.core.WhitespaceTokenizer; +import org.apache.lucene.analysis.snowball.SnowballFilter; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; + +/** Benchmark for {@link SnowballFilter} comparing stemming throughput with and without caching. */ +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +@State(Scope.Thread) +@Warmup(iterations = 5, time = 1) +@Measurement(iterations = 10, time = 1) +@Fork(3) +public class SnowballStemBenchmark { + + @Param({"0", "1024", "4096", "8192"}) + int cacheSize; + + @Param({"English", "German"}) + String language; + + @Param({"500", "5000", "50000"}) + int vocabSize; + + private String corpus; + private Analyzer analyzer; + + @Setup(Level.Trial) + public void setup() { + corpus = buildZipfianCorpus(vocabSize, 10_000, 42); + analyzer = + new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new WhitespaceTokenizer(); + return new TokenStreamComponents( + tokenizer, new SnowballFilter(tokenizer, language, cacheSize)); + } + }; + } + + @TearDown(Level.Trial) + public void tearDown() { + if (analyzer != null) { + analyzer.close(); + } + } + + @Benchmark + public int stem() throws IOException { + int count = 0; + try (TokenStream ts = analyzer.tokenStream("field", new StringReader(corpus))) { + CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); + ts.reset(); + while (ts.incrementToken()) { + count += termAtt.length(); + } + ts.end(); + } + return count; + } + + private static String buildZipfianCorpus(int uniqueWords, int totalTokens, long seed) { + Random rng = new Random(seed); + String[] vocabulary = generateVocabulary(uniqueWords, rng); + + double[] weights = new double[uniqueWords]; + double totalWeight = 0; + for (int i = 0; i < uniqueWords; i++) { + weights[i] = 1.0 / (i + 1); + totalWeight += weights[i]; + } + + double[] cumulative = new double[uniqueWords]; + cumulative[0] = weights[0] / totalWeight; + for (int i = 1; i < uniqueWords; i++) { + cumulative[i] = cumulative[i - 1] + weights[i] / totalWeight; + } + + StringBuilder sb = new StringBuilder(totalTokens * 8); + for (int t = 0; t < totalTokens; t++) { + if (t > 0) { + sb.append(' '); + } + double r = rng.nextDouble(); + int idx = findBucket(cumulative, r); + sb.append(vocabulary[idx]); + } + return sb.toString(); + } + + private static int findBucket(double[] cumulative, double r) { + int lo = 0; + int hi = cumulative.length - 1; + while (lo < hi) { + int mid = (lo + hi) >>> 1; + if (cumulative[mid] < r) { + lo = mid + 1; + } else { + hi = mid; + } + } + return lo; + } + + private static String[] generateVocabulary(int count, Random rng) { + String[] suffixes = { + "ing", "tion", "ness", "ment", "able", "ible", "ful", "less", "ous", "ive", "ity", "ence", + "ance", "ly", "er", "ed", "es", "al", "ism", "ist" + }; + String[] roots = { + "act", + "run", + "walk", + "play", + "work", + "think", + "build", + "creat", + "develop", + "establish", + "manag", + "process", + "produc", + "communic", + "determin", + "recommend", + "understand", + "perform", + "consider", + "represent", + "organiz", + "recogniz", + "transform", + "implement", + "invest", + "increas", + "reduc", + "improv", + "measur", + "distribut", + "compar", + "contribut", + "demonstrat", + "environ", + "experienc", + "gener", + "govern", + "individu", + "interpret", + "legislat" + }; + + String[] words = new String[count]; + for (int i = 0; i < count; i++) { + String root = roots[rng.nextInt(roots.length)]; + if (rng.nextDouble() < 0.6) { + words[i] = root + suffixes[rng.nextInt(suffixes.length)]; + } else { + words[i] = root; + } + } + return words; + } +} From 904aa4a625c744b0e0ffee46377f48e5033705e9 Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Sat, 4 Jul 2026 04:29:11 -0700 Subject: [PATCH 2/6] Update CHANGES.txt --- lucene/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index c6f07b2778db..aed7b8b0f99e 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -325,7 +325,7 @@ Optimizations * GITHUB#16352: Better cost() estimation for SkipBlockRangeIterator. (Alan Woodward) -* GITHUB#XXXX: Cache stem results in SnowballFilter to avoid redundant stemmer invocations on +* GITHUB#16356: Cache stem results in SnowballFilter to avoid redundant stemmer invocations on repeated tokens. 2.3-2.7x throughput improvement. Disableable via new constructor. (Costin Leau) Bug Fixes From c93ffbdc62d3db214b6efb449882a0757dd4efbe Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Tue, 28 Jul 2026 18:14:38 +0300 Subject: [PATCH 3/6] Add stopwords filter to benchmark --- .../benchmark/jmh/SnowballStemBenchmark.java | 43 ++++++++++++++++--- 1 file changed, 36 insertions(+), 7 deletions(-) diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/SnowballStemBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/SnowballStemBenchmark.java index 97709138a7f7..f8822e9ae39f 100644 --- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/SnowballStemBenchmark.java +++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/SnowballStemBenchmark.java @@ -21,9 +21,13 @@ import java.util.Random; import java.util.concurrent.TimeUnit; import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.CharArraySet; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.core.StopFilter; import org.apache.lucene.analysis.core.WhitespaceTokenizer; +import org.apache.lucene.analysis.de.GermanAnalyzer; +import org.apache.lucene.analysis.en.EnglishAnalyzer; import org.apache.lucene.analysis.snowball.SnowballFilter; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.openjdk.jmh.annotations.Benchmark; @@ -58,19 +62,33 @@ public class SnowballStemBenchmark { @Param({"500", "5000", "50000"}) int vocabSize; + @Param({"true", "false"}) + String stopFilter; + private String corpus; private Analyzer analyzer; @Setup(Level.Trial) public void setup() { - corpus = buildZipfianCorpus(vocabSize, 10_000, 42); + boolean useStopFilter = Boolean.parseBoolean(stopFilter); + CharArraySet stopWords = + switch (language) { + case "English" -> EnglishAnalyzer.getDefaultStopSet(); + case "German" -> GermanAnalyzer.getDefaultStopSet(); + default -> CharArraySet.EMPTY_SET; + }; + corpus = buildZipfianCorpus(vocabSize, 10_000, 42, useStopFilter ? stopWords : null); analyzer = new Analyzer() { @Override protected TokenStreamComponents createComponents(String fieldName) { Tokenizer tokenizer = new WhitespaceTokenizer(); - return new TokenStreamComponents( - tokenizer, new SnowballFilter(tokenizer, language, cacheSize)); + TokenStream stream = tokenizer; + if (useStopFilter) { + stream = new StopFilter(stream, stopWords); + } + stream = new SnowballFilter(stream, language, cacheSize); + return new TokenStreamComponents(tokenizer, stream); } }; } @@ -96,7 +114,14 @@ public int stem() throws IOException { return count; } - private static String buildZipfianCorpus(int uniqueWords, int totalTokens, long seed) { + private static final String[] COMMON_STOP_WORDS = { + "the", "a", "an", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into", + "is", "it", "no", "not", "of", "on", "or", "such", "that", "the", "their", "then", "there", + "these", "they", "this", "to", "was", "will", "with" + }; + + private static String buildZipfianCorpus( + int uniqueWords, int totalTokens, long seed, CharArraySet stopWords) { Random rng = new Random(seed); String[] vocabulary = generateVocabulary(uniqueWords, rng); @@ -118,9 +143,13 @@ private static String buildZipfianCorpus(int uniqueWords, int totalTokens, long if (t > 0) { sb.append(' '); } - double r = rng.nextDouble(); - int idx = findBucket(cumulative, r); - sb.append(vocabulary[idx]); + if (stopWords != null && rng.nextDouble() < 0.45) { + sb.append(COMMON_STOP_WORDS[rng.nextInt(COMMON_STOP_WORDS.length)]); + } else { + double r = rng.nextDouble(); + int idx = findBucket(cumulative, r); + sb.append(vocabulary[idx]); + } } return sb.toString(); } From 8af237a9f362955d9af581c3bb7c1db6adc8bba7 Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Tue, 28 Jul 2026 21:46:50 +0300 Subject: [PATCH 4/6] Benchmark Romanian/Turkish stemmers with cache size sweep Use languages where Snowball is the only stemmer available. Add numAnalyzers param to simulate multi-index memory pressure. Sweep cache sizes 64-512 to find the right default. --- .../benchmark/jmh/SnowballStemBenchmark.java | 99 ++++++++++++------- 1 file changed, 61 insertions(+), 38 deletions(-) diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/SnowballStemBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/SnowballStemBenchmark.java index f8822e9ae39f..58a6674b0188 100644 --- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/SnowballStemBenchmark.java +++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/SnowballStemBenchmark.java @@ -18,6 +18,8 @@ import java.io.IOException; import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; import java.util.Random; import java.util.concurrent.TimeUnit; import org.apache.lucene.analysis.Analyzer; @@ -26,10 +28,10 @@ import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.StopFilter; import org.apache.lucene.analysis.core.WhitespaceTokenizer; -import org.apache.lucene.analysis.de.GermanAnalyzer; -import org.apache.lucene.analysis.en.EnglishAnalyzer; +import org.apache.lucene.analysis.ro.RomanianAnalyzer; import org.apache.lucene.analysis.snowball.SnowballFilter; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tr.TurkishAnalyzer; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -53,75 +55,96 @@ @Fork(3) public class SnowballStemBenchmark { - @Param({"0", "1024", "4096", "8192"}) + @Param({"0", "64", "128", "256", "512"}) int cacheSize; - @Param({"English", "German"}) + @Param({"Romanian", "Turkish"}) String language; - @Param({"500", "5000", "50000"}) + @Param({"5000", "50000"}) int vocabSize; @Param({"true", "false"}) String stopFilter; + @Param({"1", "4"}) + int numAnalyzers; + private String corpus; - private Analyzer analyzer; + private Analyzer[] analyzers; @Setup(Level.Trial) public void setup() { boolean useStopFilter = Boolean.parseBoolean(stopFilter); CharArraySet stopWords = switch (language) { - case "English" -> EnglishAnalyzer.getDefaultStopSet(); - case "German" -> GermanAnalyzer.getDefaultStopSet(); + case "Romanian" -> RomanianAnalyzer.getDefaultStopSet(); + case "Turkish" -> TurkishAnalyzer.getDefaultStopSet(); default -> CharArraySet.EMPTY_SET; }; - corpus = buildZipfianCorpus(vocabSize, 10_000, 42, useStopFilter ? stopWords : null); - analyzer = - new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName) { - Tokenizer tokenizer = new WhitespaceTokenizer(); - TokenStream stream = tokenizer; - if (useStopFilter) { - stream = new StopFilter(stream, stopWords); - } - stream = new SnowballFilter(stream, language, cacheSize); - return new TokenStreamComponents(tokenizer, stream); - } - }; + + List stopWordList = extractStopWords(stopWords); + corpus = buildZipfianCorpus(vocabSize, 10_000, 42, useStopFilter ? stopWordList : null); + + analyzers = new Analyzer[numAnalyzers]; + for (int i = 0; i < numAnalyzers; i++) { + analyzers[i] = createAnalyzer(language, cacheSize, useStopFilter, stopWords); + } + } + + private static Analyzer createAnalyzer( + String language, int cacheSize, boolean useStopFilter, CharArraySet stopWords) { + return new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName) { + Tokenizer tokenizer = new WhitespaceTokenizer(); + TokenStream stream = tokenizer; + if (useStopFilter) { + stream = new StopFilter(stream, stopWords); + } + stream = new SnowballFilter(stream, language, cacheSize); + return new TokenStreamComponents(tokenizer, stream); + } + }; } @TearDown(Level.Trial) public void tearDown() { - if (analyzer != null) { - analyzer.close(); + for (Analyzer a : analyzers) { + if (a != null) { + a.close(); + } } } @Benchmark public int stem() throws IOException { int count = 0; - try (TokenStream ts = analyzer.tokenStream("field", new StringReader(corpus))) { - CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); - ts.reset(); - while (ts.incrementToken()) { - count += termAtt.length(); + for (Analyzer analyzer : analyzers) { + try (TokenStream ts = analyzer.tokenStream("field", new StringReader(corpus))) { + CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); + ts.reset(); + while (ts.incrementToken()) { + count += termAtt.length(); + } + ts.end(); } - ts.end(); } return count; } - private static final String[] COMMON_STOP_WORDS = { - "the", "a", "an", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into", - "is", "it", "no", "not", "of", "on", "or", "such", "that", "the", "their", "then", "there", - "these", "they", "this", "to", "was", "will", "with" - }; + private static List extractStopWords(CharArraySet stopWords) { + List words = new ArrayList<>(); + for (Object obj : stopWords) { + if (obj instanceof char[] chars) { + words.add(new String(chars)); + } + } + return words; + } private static String buildZipfianCorpus( - int uniqueWords, int totalTokens, long seed, CharArraySet stopWords) { + int uniqueWords, int totalTokens, long seed, List stopWords) { Random rng = new Random(seed); String[] vocabulary = generateVocabulary(uniqueWords, rng); @@ -143,8 +166,8 @@ private static String buildZipfianCorpus( if (t > 0) { sb.append(' '); } - if (stopWords != null && rng.nextDouble() < 0.45) { - sb.append(COMMON_STOP_WORDS[rng.nextInt(COMMON_STOP_WORDS.length)]); + if (stopWords != null && !stopWords.isEmpty() && rng.nextDouble() < 0.45) { + sb.append(stopWords.get(rng.nextInt(stopWords.size()))); } else { double r = rng.nextDouble(); int idx = findBucket(cumulative, r); From e53a73b7ba8becf3e2eb1768cf89c88ed98d576c Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Tue, 28 Jul 2026 23:03:33 +0300 Subject: [PATCH 5/6] Reduce default stem cache from 1024 to 128 entries 128 entries (~9KB) gives ~2x throughput while keeping memory bounded. Diminishing returns past 256 entries. --- .../org/apache/lucene/analysis/snowball/SnowballFilter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java index ce9f95daa455..ee8ee1b59876 100644 --- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java @@ -48,7 +48,7 @@ */ public final class SnowballFilter extends TokenFilter { - private static final int DEFAULT_MAX_CACHE_SIZE = 1024; + private static final int DEFAULT_MAX_CACHE_SIZE = 128; private static final int MAX_CACHEABLE_LENGTH = 10; private final SnowballStemmer stemmer; From 94009a8f73e6d3981faec000fb2b3053c4f09d39 Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Wed, 29 Jul 2026 18:34:07 +0300 Subject: [PATCH 6/6] Lazy cache allocation, clear-on-full for adaptability Cache is allocated lazily on first cacheable token. When full, clears and refills from subsequent tokens to adapt to distribution changes since CharArrayMap does not support removal/LRU. Updated benchmark to process corpus as multiple documents matching real indexing. --- .../analysis/snowball/SnowballFilter.java | 27 ++++++++-------- .../benchmark/jmh/SnowballStemBenchmark.java | 31 ++++++++++++++----- 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java index ee8ee1b59876..b3a86e825af0 100644 --- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java +++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/snowball/SnowballFilter.java @@ -53,7 +53,7 @@ public final class SnowballFilter extends TokenFilter { private final SnowballStemmer stemmer; private final int maxCacheSize; - private final CharArrayMap cache; + private CharArrayMap cache; private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class); @@ -65,7 +65,8 @@ public SnowballFilter(TokenStream input, SnowballStemmer stemmer) { /** * Creates a SnowballFilter with the specified cache size. The cache stores stem results keyed by * the original token text. Only tokens with length ≤ {@value #MAX_CACHEABLE_LENGTH} are - * cached. Set {@code maxCacheSize} to 0 to disable caching. + * cached. When the cache fills up, it is cleared and repopulated from subsequent tokens, adapting + * to the current token distribution. Set {@code maxCacheSize} to 0 to disable caching. */ public SnowballFilter(TokenStream input, SnowballStemmer stemmer, int maxCacheSize) { super(input); @@ -74,7 +75,6 @@ public SnowballFilter(TokenStream input, SnowballStemmer stemmer, int maxCacheSi throw new IllegalArgumentException("maxCacheSize must be >= 0, got: " + maxCacheSize); } this.maxCacheSize = maxCacheSize; - this.cache = maxCacheSize > 0 ? new CharArrayMap<>(Math.min(maxCacheSize, 256), false) : null; } /** @@ -114,7 +114,6 @@ public SnowballFilter(TokenStream in, String name, int maxCacheSize) { throw new IllegalArgumentException("Invalid stemmer class specified: " + name, e); } this.maxCacheSize = maxCacheSize; - this.cache = maxCacheSize > 0 ? new CharArrayMap<>(Math.min(maxCacheSize, 256), false) : null; } /** Returns the next input Token, after being stemmed */ @@ -124,9 +123,9 @@ public boolean incrementToken() throws IOException { if (!keywordAttr.isKeyword()) { char[] termBuffer = termAtt.buffer(); final int length = termAtt.length(); - final boolean lookupCache = cache != null && length <= MAX_CACHEABLE_LENGTH; + final boolean cacheable = maxCacheSize > 0 && length <= MAX_CACHEABLE_LENGTH; - if (lookupCache) { + if (cacheable && cache != null) { char[] cached = cache.get(termBuffer, 0, length); if (cached != null) { termAtt.copyBuffer(cached, 0, cached.length); @@ -134,12 +133,6 @@ public boolean incrementToken() throws IOException { } } - char[] key = null; - if (lookupCache && cache.size() < maxCacheSize) { - key = new char[length]; - System.arraycopy(termBuffer, 0, key, 0, length); - } - stemmer.setCurrent(termBuffer, length); stemmer.stem(); final char[] finalTerm = stemmer.getCurrentBuffer(); @@ -147,7 +140,15 @@ public boolean incrementToken() throws IOException { if (finalTerm != termBuffer) termAtt.copyBuffer(finalTerm, 0, newLength); else termAtt.setLength(newLength); - if (key != null) { + if (cacheable) { + if (cache == null) { + cache = new CharArrayMap<>(Math.min(maxCacheSize, 128), false); + } + if (cache.size() >= maxCacheSize) { + cache.clear(); + } + char[] key = new char[length]; + System.arraycopy(termBuffer, 0, key, 0, length); char[] value = new char[newLength]; System.arraycopy(finalTerm, 0, value, 0, newLength); cache.put(key, value); diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/SnowballStemBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/SnowballStemBenchmark.java index 58a6674b0188..d8adebb0dc4b 100644 --- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/SnowballStemBenchmark.java +++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/SnowballStemBenchmark.java @@ -70,7 +70,7 @@ public class SnowballStemBenchmark { @Param({"1", "4"}) int numAnalyzers; - private String corpus; + private String[] documents; private Analyzer[] analyzers; @Setup(Level.Trial) @@ -84,7 +84,8 @@ public void setup() { }; List stopWordList = extractStopWords(stopWords); - corpus = buildZipfianCorpus(vocabSize, 10_000, 42, useStopFilter ? stopWordList : null); + String corpus = buildZipfianCorpus(vocabSize, 10_000, 42, useStopFilter ? stopWordList : null); + documents = splitIntoDocuments(corpus, 500); analyzers = new Analyzer[numAnalyzers]; for (int i = 0; i < numAnalyzers; i++) { @@ -121,18 +122,32 @@ public void tearDown() { public int stem() throws IOException { int count = 0; for (Analyzer analyzer : analyzers) { - try (TokenStream ts = analyzer.tokenStream("field", new StringReader(corpus))) { - CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); - ts.reset(); - while (ts.incrementToken()) { - count += termAtt.length(); + for (String doc : documents) { + try (TokenStream ts = analyzer.tokenStream("field", new StringReader(doc))) { + CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); + ts.reset(); + while (ts.incrementToken()) { + count += termAtt.length(); + } + ts.end(); } - ts.end(); } } return count; } + private static String[] splitIntoDocuments(String corpus, int tokensPerDoc) { + String[] allTokens = corpus.split(" "); + int numDocs = (allTokens.length + tokensPerDoc - 1) / tokensPerDoc; + String[] docs = new String[numDocs]; + for (int i = 0; i < numDocs; i++) { + int start = i * tokensPerDoc; + int end = Math.min(start + tokensPerDoc, allTokens.length); + docs[i] = String.join(" ", java.util.Arrays.copyOfRange(allTokens, start, end)); + } + return docs; + } + private static List extractStopWords(CharArraySet stopWords) { List words = new ArrayList<>(); for (Object obj : stopWords) {