Add the Morphy lemmatizer with exception-list support

Author: krickert

opennlp-extensions/opennlp-wordnet/pom.xml

@@ -48,6 +48,12 @@
       <artifactId>junit-jupiter-engine</artifactId>
       <scope>test</scope>
     </dependency>
+
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-params</artifactId>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
 </project>

opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyExceptions.java

@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.wordnet;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+
+import opennlp.tools.commons.ThreadSafe;
+import opennlp.tools.wordnet.WordNetPos;
+
+/**
+ * The Morphy exception lists: the per-part-of-speech tables of irregular inflected forms
+ * ({@code mice} to {@code mouse}, {@code went} to {@code go}) that the Morphy algorithm
+ * consults before its detachment rules.
+ *
+ * <p>{@link #load(Path)} reads the four {@code *.exc} files ({@code noun.exc},
+ * {@code verb.exc}, {@code adj.exc}, {@code adv.exc}) in the documented WNDB format: one entry
+ * per line, the inflected form followed by one or more base forms, space separated, with
+ * underscores standing for spaces in multiword entries. All four files must be present; note
+ * that Princeton's database-only download ({@code WNdb}) does not include them, while the full
+ * WordNet package does. No exception data is bundled with this module; see
+ * {@link MorphyLemmatizer} for the tiering rationale.</p>
+ *
+ * <p>Lookups fold the queried word the same way the lexicon seam folds lemmas: lowercase with
+ * the root locale, underscore as space.</p>
+ *
+ * <p>Instances are immutable after loading and safe for concurrent lookups.</p>
+ */
+@ThreadSafe
+public final class MorphyExceptions {
+
+  private final Map<WordNetPos, Map<String, List<String>>> byPos;
+
+  private MorphyExceptions(Map<WordNetPos, Map<String, List<String>>> byPos) {
+    this.byPos = byPos;
+  }
+
+  /**
+   * Loads the four exception lists from a directory.
+   *
+   * @param directory The directory containing {@code noun.exc}, {@code verb.exc},
+   *                  {@code adj.exc}, and {@code adv.exc}. Must not be {@code null} and must
+   *                  exist.
+   * @return The loaded exception lists.
+   * @throws IllegalArgumentException Thrown if {@code directory} is {@code null} or not a
+   *     directory, one of the four files is missing, or a line is malformed; the message names
+   *     the file and line.
+   * @throws UncheckedIOException Thrown if reading a file fails.
+   */
+  public static MorphyExceptions load(Path directory) {
+    if (directory == null) {
+      throw new IllegalArgumentException("Directory must not be null");
+    }
+    if (!Files.isDirectory(directory)) {
+      throw new IllegalArgumentException(
+          "Directory does not exist or is not a directory: " + directory);
+    }
+    final Map<WordNetPos, Map<String, List<String>>> byPos = new EnumMap<>(WordNetPos.class);
+    byPos.put(WordNetPos.NOUN, loadFile(directory, "noun.exc"));
+    byPos.put(WordNetPos.VERB, loadFile(directory, "verb.exc"));
+    byPos.put(WordNetPos.ADJECTIVE, loadFile(directory, "adj.exc"));
+    byPos.put(WordNetPos.ADVERB, loadFile(directory, "adv.exc"));
+    return new MorphyExceptions(byPos);
+  }
+
+  /**
+   * Finds the base forms of an irregular inflected form.
+   *
+   * @param word The inflected form; folded before lookup. Must not be {@code null}.
+   * @param pos  The part of speech. Must not be {@code null}.
+   * @return The base forms in file order, never {@code null}; empty when the word has no entry.
+   * @throws IllegalArgumentException Thrown if {@code word} or {@code pos} is {@code null}.
+   */
+  public List<String> lookup(String word, WordNetPos pos) {
+    if (word == null) {
+      throw new IllegalArgumentException("Word must not be null");
+    }
+    if (pos == null) {
+      throw new IllegalArgumentException("Pos must not be null");
+    }
+    final List<String> lemmas = byPos.get(pos).get(fold(word));
+    return lemmas == null ? List.of() : lemmas;
+  }
+
+  static String fold(String word) {
+    return word.replace('_', ' ').toLowerCase(Locale.ROOT);
+  }
+
+  private static Map<String, List<String>> loadFile(Path directory, String fileName) {
+    final Path file = directory.resolve(fileName);
+    if (!Files.isRegularFile(file)) {
+      throw new IllegalArgumentException("Missing exception list file: " + file);
+    }
+    final List<String> lines;
+    try {
+      lines = Files.readAllLines(file, StandardCharsets.ISO_8859_1);
+    } catch (IOException e) {
+      throw new UncheckedIOException("Unable to read exception list file " + file, e);
+    }
+    final Map<String, List<String>> entries = new HashMap<>(lines.size() * 2);
+    for (int i = 0; i < lines.size(); i++) {
+      final String line = lines.get(i);
+      if (line.isEmpty()) {
+        continue;
+      }
+      final List<String> fields = splitOnSpaces(line);
+      if (fields.size() < 2) {
+        throw new IllegalArgumentException("Malformed exception list " + fileName + " at line "
+            + (i + 1) + ": expected an inflected form and at least one base form, got: " + line);
+      }
+      final List<String> lemmas = new ArrayList<>(fields.size() - 1);
+      for (final String lemma : fields.subList(1, fields.size())) {
+        lemmas.add(fold(lemma));
+      }
+      // A form listed twice keeps its first entry, matching first-match lookup semantics.
+      entries.putIfAbsent(fold(fields.get(0)), List.copyOf(lemmas));
+    }
+    return Map.copyOf(entries);
+  }
+
+  private static List<String> splitOnSpaces(String line) {
+    final List<String> fields = new ArrayList<>(3);
+    int start = 0;
+    while (start < line.length()) {
+      final int space = line.indexOf(' ', start);
+      if (space < 0) {
+        fields.add(line.substring(start));
+        break;
+      }
+      if (space > start) {
+        fields.add(line.substring(start, space));
+      }
+      start = space + 1;
+    }
+    return fields;
+  }
+}

opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java

@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.wordnet;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+import opennlp.tools.commons.ThreadSafe;
+import opennlp.tools.lemmatizer.Lemmatizer;
+import opennlp.tools.wordnet.WordNetLexicon;
+import opennlp.tools.wordnet.WordNetPos;
+
+/**
+ * A clean-room implementation of the documented Morphy algorithm as a {@link Lemmatizer}:
+ * exception-list lookup first, then the per-part-of-speech iterative detachment rules, with
+ * every rule-derived candidate validated against a {@link WordNetLexicon} before it is
+ * returned.
+ *
+ * <p><b>Algorithm.</b> For each token the part-of-speech tag is mapped to a
+ * {@link WordNetPos}; the token is folded (lowercase with the root locale, underscore as
+ * space); the {@link MorphyExceptions exception lists} are consulted and a hit is returned
+ * directly, without lexicon validation, because the lists themselves are authoritative for
+ * irregular forms; otherwise the folded token itself, when the lexicon contains it, and every
+ * detachment-rule result the lexicon confirms become the candidate lemmas, in rule order. The
+ * rules are the documented Morphy suffix substitutions: for nouns {@code -s}, {@code -ses},
+ * {@code -xes}, {@code -zes}, {@code -ches}, {@code -shes}, {@code -men}, {@code -ies}; for
+ * verbs {@code -s}, {@code -ies}, {@code -es}, {@code -ed}, {@code -ing} (with their {@code e}
+ * restorations); for adjectives {@code -er} and {@code -est} (plain and {@code e}-restoring);
+ * and none for adverbs, which rely on the exception list alone. Returned lemmas are in the
+ * lexicon's folded canonical form (lowercase, spaces in multiword lemmas).</p>
+ *
+ * <p><b>Tag mapping.</b> Tags map to WordNet parts of speech by their conventional Penn
+ * Treebank prefixes: {@code N} to noun, {@code V} to verb, {@code J} to adjective, and
+ * {@code R} to adverb, case-insensitively. The names {@code ADJ} and {@code ADV} and the
+ * WordNet letters {@code n}, {@code v}, {@code a}, {@code r}, and {@code s} (satellite,
+ * treated as adjective) are also accepted. Anything else, including Penn tags for closed
+ * classes such as {@code DT} or {@code MD}, maps to no part of speech and yields the
+ * unknown-word result.</p>
+ *
+ * <p><b>Unknown words.</b> Following the convention of
+ * {@code opennlp.tools.lemmatizer.DictionaryLemmatizer}, a token with no lemma yields the
+ * string {@code "O"} from {@link #lemmatize(String[], String[])} and a singleton
+ * {@code ["O"]} from {@link #lemmatize(List, List)}.</p>
+ *
+ * <p>No lexicon or exception data is bundled: point {@link WndbReader} and
+ * {@link MorphyExceptions#load(java.nio.file.Path) MorphyExceptions} at a WordNet database
+ * directory you
+ * downloaded, or combine {@link WnLmfReader} with a downloaded exception-list directory. Both
+ * inputs are required because rule-derived candidates are meaningless without a lexicon to
+ * validate them against; an exception-only mode would be a misleading half-feature, so it does
+ * not exist. Bundling the permissively licensed data is a recorded follow-up.</p>
+ *
+ * <p>Instances are immutable and safe for concurrent use once constructed with a loaded
+ * lexicon and exception lists.</p>
+ */
+@ThreadSafe
+public final class MorphyLemmatizer implements Lemmatizer {
+
+  /** The unknown-word output, matching the DictionaryLemmatizer convention. */
+  private static final String UNKNOWN = "O";
+
+  private static final String[][] NOUN_RULES = {
+      {"s", ""}, {"ses", "s"}, {"xes", "x"}, {"zes", "z"},
+      {"ches", "ch"}, {"shes", "sh"}, {"men", "man"}, {"ies", "y"},
+  };
+
+  private static final String[][] VERB_RULES = {
+      {"s", ""}, {"ies", "y"}, {"es", "e"}, {"es", ""},
+      {"ed", "e"}, {"ed", ""}, {"ing", "e"}, {"ing", ""},
+  };
+
+  private static final String[][] ADJECTIVE_RULES = {
+      {"er", ""}, {"est", ""}, {"er", "e"}, {"est", "e"},
+  };
+
+  private static final String[][] NO_RULES = {};
+
+  private final WordNetLexicon lexicon;
+  private final MorphyExceptions exceptions;
+
+  /**
+   * Creates a Morphy lemmatizer over a loaded lexicon and exception lists.
+   *
+   * @param lexicon    The lexicon rule candidates are validated against. Must not be
+   *                   {@code null}.
+   * @param exceptions The irregular-form exception lists. Must not be {@code null}.
+   * @throws IllegalArgumentException Thrown if {@code lexicon} or {@code exceptions} is
+   *     {@code null}.
+   */
+  public MorphyLemmatizer(WordNetLexicon lexicon, MorphyExceptions exceptions) {
+    if (lexicon == null) {
+      throw new IllegalArgumentException("Lexicon must not be null");
+    }
+    if (exceptions == null) {
+      throw new IllegalArgumentException("Exceptions must not be null");
+    }
+    this.lexicon = lexicon;
+    this.exceptions = exceptions;
+  }
+
+  @Override
+  public String[] lemmatize(String[] toks, String[] tags) {
+    if (toks == null || tags == null) {
+      throw new IllegalArgumentException("Toks and tags must not be null");
+    }
+    if (toks.length != tags.length) {
+      throw new IllegalArgumentException("Toks and tags must have the same length, got "
+          + toks.length + " and " + tags.length);
+    }
+    final String[] lemmas = new String[toks.length];
+    for (int i = 0; i < toks.length; i++) {
+      final List<String> candidates = lemmasOf(toks[i], tags[i]);
+      lemmas[i] = candidates.isEmpty() ? UNKNOWN : candidates.get(0);
+    }
+    return lemmas;
+  }
+
+  @Override
+  public List<List<String>> lemmatize(List<String> toks, List<String> tags) {
+    if (toks == null || tags == null) {
+      throw new IllegalArgumentException("Toks and tags must not be null");
+    }
+    if (toks.size() != tags.size()) {
+      throw new IllegalArgumentException("Toks and tags must have the same size, got "
+          + toks.size() + " and " + tags.size());
+    }
+    final List<List<String>> lemmas = new ArrayList<>(toks.size());
+    for (int i = 0; i < toks.size(); i++) {
+      final List<String> candidates = lemmasOf(toks.get(i), tags.get(i));
+      lemmas.add(candidates.isEmpty() ? List.of(UNKNOWN) : candidates);
+    }
+    return lemmas;
+  }
+
+  // All lemmas of one token, most preferred first; empty when the word is unknown.
+  private List<String> lemmasOf(String token, String tag) {
+    if (token == null || tag == null) {
+      throw new IllegalArgumentException("Tokens and tags must not contain null elements");
+    }
+    final WordNetPos pos = posFromTag(tag);
+    if (pos == null) {
+      return List.of();
+    }
+    final String folded = MorphyExceptions.fold(token);
+    final List<String> irregular = exceptions.lookup(folded, pos);
+    if (!irregular.isEmpty()) {
+      return irregular;
+    }
+    final List<String> candidates = new ArrayList<>(2);
+    if (lexicon.contains(folded, pos)) {
+      candidates.add(folded);
+    }
+    for (final String[] rule : rulesFor(pos)) {
+      final String suffix = rule[0];
+      if (folded.length() > suffix.length() && folded.endsWith(suffix)) {
+        final String candidate =
+            folded.substring(0, folded.length() - suffix.length()) + rule[1];
+        if (!candidates.contains(candidate) && lexicon.contains(candidate, pos)) {
+          candidates.add(candidate);
+        }
+      }
+    }
+    return candidates;
+  }
+
+  private static String[][] rulesFor(WordNetPos pos) {
+    return switch (pos) {
+      case NOUN -> NOUN_RULES;
+      case VERB -> VERB_RULES;
+      case ADJECTIVE -> ADJECTIVE_RULES;
+      case ADVERB -> NO_RULES;
+    };
+  }
+
+  // The documented tag mapping; package-private so tests can pin it directly.
+  static WordNetPos posFromTag(String tag) {
+    if (tag.isEmpty()) {
+      return null;
+    }
+    final String upper = tag.toUpperCase(Locale.ROOT);
+    if (upper.startsWith("ADJ")) {
+      return WordNetPos.ADJECTIVE;
+    }
+    if (upper.startsWith("ADV")) {
+      return WordNetPos.ADVERB;
+    }
+    return switch (upper.charAt(0)) {
+      case 'N' -> WordNetPos.NOUN;
+      case 'V' -> WordNetPos.VERB;
+      case 'J', 'A', 'S' -> WordNetPos.ADJECTIVE;
+      case 'R' -> WordNetPos.ADVERB;
+      default -> null;
+    };
+  }
+}