Skip to content

Commit e283a77

Browse files
krickertkrickert
committed
Add UniNE light and minimal stemmer tiers with vocabulary parity fixtures
Sixteen stemmers in the new opennlp.tools.stemmer.light package: light and minimal tiers for German, French, Spanish, Norwegian (Bokmaal and Nynorsk varieties), Swedish, plus light stemmers for Finnish, Hungarian, Italian, Portuguese, Russian and the English minimal stemmer. Adapted from Apache Lucene's analysis-common module; the UniNE algorithms carry Jacques Savoy's BSD notice in each source file and in the distribution LICENSE. These fill the tier between no stemming and the aggressive Snowball algorithms. Every stemmer is stateless and thread-safe, implements Stemmer, and is its own StemmerFactory. Input is expected lowercase, matching the algorithms' original contract; null input fails loudly. Parity is asserted against the original implementations: bundled fixtures sample the algorithms' vocabulary test data (up to 2000 word/stem pairs per stemmer, all pairs for the small Norwegian lists) regenerated by running the Lucene classes, so any behavioral drift fails the test. The Galician and Portuguese minimal stemmers are excluded because they build on the RSLP rule engine, which is its own effort.
1 parent b76eef4 commit e283a77

41 files changed

Lines changed: 31447 additions & 0 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/light/EnglishMinimalStemmer.java‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package opennlp.tools.stemmer.light;
19+
20+
import opennlp.tools.commons.ThreadSafe;
21+
import opennlp.tools.stemmer.Stemmer;
22+
import opennlp.tools.stemmer.StemmerFactory;
23+
/**
24+
* Minimal plural stemmer for English.
25+
*
26+
* <p>This stemmer implements the "S-Stemmer" from <i>How Effective Is Suffixing?</i> Donna Harman.
27+
*
28+
* <p>Adapted from the identically named algorithm in Apache Lucene's analysis-common module.
29+
* Instances are stateless and safe for concurrent use by multiple threads; each instance is also
30+
* its own {@link StemmerFactory}. Input is expected to be lowercase, as produced by a
31+
* case-folding normalization step; the stemmer does not fold case itself.</p>
32+
*/
33+
@ThreadSafe
34+
public final class EnglishMinimalStemmer implements Stemmer, StemmerFactory {
35+
@Override
36+
public CharSequence stem(CharSequence word) {
37+
if (word == null) {
38+
throw new IllegalArgumentException("The word must not be null.");
39+
}
40+
final int length = word.length();
41+
final char[] buffer = new char[length];
42+
for (int i = 0; i < length; i++) {
43+
buffer[i] = word.charAt(i);
44+
}
45+
final int stemmedLength = stem(buffer, length);
46+
return new String(buffer, 0, stemmedLength);
47+
}
48+
49+
@Override
50+
public Stemmer newStemmer() {
51+
return this;
52+
}
53+
54+
@SuppressWarnings("fallthrough")
55+
private int stem(char[] s, int len) {
56+
if (len < 3 || s[len - 1] != 's') return len;
57+
58+
switch (s[len - 2]) {
59+
case 'u':
60+
case 's':
61+
return len;
62+
case 'e':
63+
if (len > 3 && s[len - 3] == 'i' && s[len - 4] != 'a' && s[len - 4] != 'e') {
64+
s[len - 3] = 'y';
65+
return len - 2;
66+
}
67+
if (s[len - 3] == 'i' || s[len - 3] == 'a' || s[len - 3] == 'o' || s[len - 3] == 'e')
68+
return len; /* intentional fallthrough */
69+
default:
70+
return len - 1;
71+
}
72+
}
73+
}

‎opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/stemmer/light/FinnishLightStemmer.java‎

Lines changed: 262 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,262 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
/*
19+
* This algorithm is updated based on code located at:
20+
* http://members.unine.ch/jacques.savoy/clef/
21+
*
22+
* Full copyright for that code follows:
23+
*/
24+
/*
25+
* Copyright (c) 2005, Jacques Savoy
26+
* All rights reserved.
27+
*
28+
* Redistribution and use in source and binary forms, with or without
29+
* modification, are permitted provided that the following conditions are met:
30+
*
31+
* Redistributions of source code must retain the above copyright notice, this
32+
* list of conditions and the following disclaimer. Redistributions in binary
33+
* form must reproduce the above copyright notice, this list of conditions and
34+
* the following disclaimer in the documentation and/or other materials
35+
* provided with the distribution. Neither the name of the author nor the names
36+
* of its contributors may be used to endorse or promote products derived from
37+
* this software without specific prior written permission.
38+
*
39+
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
40+
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
41+
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
42+
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
43+
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
44+
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
45+
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
46+
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
47+
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
48+
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
49+
* POSSIBILITY OF SUCH DAMAGE.
50+
*/
51+
52+
package opennlp.tools.stemmer.light;
53+
54+
import opennlp.tools.commons.ThreadSafe;
55+
import opennlp.tools.stemmer.Stemmer;
56+
import opennlp.tools.stemmer.StemmerFactory;
57+
58+
import static opennlp.tools.stemmer.light.StemmerUtil.delete;
59+
import static opennlp.tools.stemmer.light.StemmerUtil.endsWith;
60+
/**
61+
* Light Stemmer for Finnish.
62+
*
63+
* <p>This stemmer implements the algorithm described in: <i>Report on CLEF-2003 Monolingual
64+
* Tracks</i> Jacques Savoy
65+
*
66+
* <p>Adapted from the identically named algorithm in Apache Lucene's analysis-common module.
67+
* Instances are stateless and safe for concurrent use by multiple threads; each instance is also
68+
* its own {@link StemmerFactory}. Input is expected to be lowercase, as produced by a
69+
* case-folding normalization step; the stemmer does not fold case itself.</p>
70+
*/
71+
@ThreadSafe
72+
public final class FinnishLightStemmer implements Stemmer, StemmerFactory {
73+
@Override
74+
public CharSequence stem(CharSequence word) {
75+
if (word == null) {
76+
throw new IllegalArgumentException("The word must not be null.");
77+
}
78+
final int length = word.length();
79+
final char[] buffer = new char[length];
80+
for (int i = 0; i < length; i++) {
81+
buffer[i] = word.charAt(i);
82+
}
83+
final int stemmedLength = stem(buffer, length);
84+
return new String(buffer, 0, stemmedLength);
85+
}
86+
87+
@Override
88+
public Stemmer newStemmer() {
89+
return this;
90+
}
91+
92+
private int stem(char[] s, int len) {
93+
if (len < 4) return len;
94+
95+
for (int i = 0; i < len; i++)
96+
switch (s[i]) {
97+
case 'ä':
98+
case 'å':
99+
s[i] = 'a';
100+
break;
101+
case 'ö':
102+
s[i] = 'o';
103+
break;
104+
}
105+
106+
len = step1(s, len);
107+
len = step2(s, len);
108+
len = step3(s, len);
109+
len = norm1(s, len);
110+
len = norm2(s, len);
111+
return len;
112+
}
113+
114+
private int step1(char[] s, int len) {
115+
if (len > 8) {
116+
if (endsWith(s, len, "kin")) return step1(s, len - 3);
117+
if (endsWith(s, len, "ko")) return step1(s, len - 2);
118+
}
119+
120+
if (len > 11) {
121+
if (endsWith(s, len, "dellinen")) return len - 8;
122+
if (endsWith(s, len, "dellisuus")) return len - 9;
123+
}
124+
return len;
125+
}
126+
127+
private int step2(char[] s, int len) {
128+
if (len > 5) {
129+
if (endsWith(s, len, "lla") || endsWith(s, len, "tse") || endsWith(s, len, "sti"))
130+
return len - 3;
131+
132+
if (endsWith(s, len, "ni")) return len - 2;
133+
134+
if (endsWith(s, len, "aa")) return len - 1; // aa -> a
135+
}
136+
137+
return len;
138+
}
139+
140+
private int step3(char[] s, int len) {
141+
if (len > 8) {
142+
if (endsWith(s, len, "nnen")) {
143+
s[len - 4] = 's';
144+
return len - 3;
145+
}
146+
147+
if (endsWith(s, len, "ntena")) {
148+
s[len - 5] = 's';
149+
return len - 4;
150+
}
151+
152+
if (endsWith(s, len, "tten")) return len - 4;
153+
154+
if (endsWith(s, len, "eiden")) return len - 5;
155+
}
156+
157+
if (len > 6) {
158+
if (endsWith(s, len, "neen")
159+
|| endsWith(s, len, "niin")
160+
|| endsWith(s, len, "seen")
161+
|| endsWith(s, len, "teen")
162+
|| endsWith(s, len, "inen")) return len - 4;
163+
164+
if (s[len - 3] == 'h' && isVowel(s[len - 2]) && s[len - 1] == 'n') return len - 3;
165+
166+
if (endsWith(s, len, "den")) {
167+
s[len - 3] = 's';
168+
return len - 2;
169+
}
170+
171+
if (endsWith(s, len, "ksen")) {
172+
s[len - 4] = 's';
173+
return len - 3;
174+
}
175+
176+
if (endsWith(s, len, "ssa")
177+
|| endsWith(s, len, "sta")
178+
|| endsWith(s, len, "lla")
179+
|| endsWith(s, len, "lta")
180+
|| endsWith(s, len, "tta")
181+
|| endsWith(s, len, "ksi")
182+
|| endsWith(s, len, "lle")) return len - 3;
183+
}
184+
185+
if (len > 5) {
186+
if (endsWith(s, len, "na") || endsWith(s, len, "ne")) return len - 2;
187+
188+
if (endsWith(s, len, "nei")) return len - 3;
189+
}
190+
191+
if (len > 4) {
192+
if (endsWith(s, len, "ja") || endsWith(s, len, "ta")) return len - 2;
193+
194+
if (s[len - 1] == 'a') return len - 1;
195+
196+
if (s[len - 1] == 'n' && isVowel(s[len - 2])) return len - 2;
197+
198+
if (s[len - 1] == 'n') return len - 1;
199+
}
200+
201+
return len;
202+
}
203+
204+
private int norm1(char[] s, int len) {
205+
if (len > 5 && endsWith(s, len, "hde")) {
206+
s[len - 3] = 'k';
207+
s[len - 2] = 's';
208+
s[len - 1] = 'i';
209+
}
210+
211+
if (len > 4) {
212+
if (endsWith(s, len, "ei") || endsWith(s, len, "at")) return len - 2;
213+
}
214+
215+
if (len > 3)
216+
switch (s[len - 1]) {
217+
case 't':
218+
case 's':
219+
case 'j':
220+
case 'e':
221+
case 'a':
222+
case 'i':
223+
return len - 1;
224+
}
225+
226+
return len;
227+
}
228+
229+
private int norm2(char[] s, int len) {
230+
if (len > 8) {
231+
if (s[len - 1] == 'e' || s[len - 1] == 'o' || s[len - 1] == 'u') len--;
232+
}
233+
234+
if (len > 4) {
235+
if (s[len - 1] == 'i') len--;
236+
237+
if (len > 4) {
238+
char ch = s[0];
239+
for (int i = 1; i < len; i++) {
240+
if (s[i] == ch && (ch == 'k' || ch == 'p' || ch == 't')) len = delete(s, i--, len);
241+
else ch = s[i];
242+
}
243+
}
244+
}
245+
246+
return len;
247+
}
248+
249+
private boolean isVowel(char ch) {
250+
switch (ch) {
251+
case 'a':
252+
case 'e':
253+
case 'i':
254+
case 'o':
255+
case 'u':
256+
case 'y':
257+
return true;
258+
default:
259+
return false;
260+
}
261+
}
262+
}

0 commit comments

Comments
 (0)