lemmas = new ArrayList<>(fields.size() - 1);
+ for (final String lemma : fields.subList(1, fields.size())) {
+ lemmas.add(LemmaFolding.fold(lemma));
+ }
+ // A form listed twice keeps its first entry, matching first-match lookup semantics.
+ entries.putIfAbsent(LemmaFolding.fold(fields.get(0)), List.copyOf(lemmas));
+ }
+ return Map.copyOf(entries);
+ }
+}
diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java
new file mode 100644
index 0000000000..a004643f82
--- /dev/null
+++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.wordnet;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+import opennlp.tools.commons.ThreadSafe;
+import opennlp.tools.lemmatizer.Lemmatizer;
+import opennlp.tools.wordnet.LexicalKnowledgeBase;
+import opennlp.tools.wordnet.WordNetPOS;
+
+/**
+ * A {@link Lemmatizer} implementing the Morphy algorithm: exception-list lookup first, then the
+ * per-part-of-speech iterative detachment rules, with every rule-derived candidate validated
+ * against a {@link LexicalKnowledgeBase} before it is returned. A token is folded (lowercase with
+ * the root locale, underscore as space) before lookup, and returned lemmas are in that folded
+ * form.
+ *
+ * Part-of-speech tags map to a {@link WordNetPOS} by their conventional Penn Treebank
+ * prefixes ({@code N}, {@code V}, {@code J}, {@code R}), the names {@code ADJ} and {@code ADV},
+ * and the one-letter WordNet codes {@code n}, {@code v}, {@code a}, {@code r}, and {@code s}
+ * (satellite, treated as adjective), case-insensitively. A tag that maps to no part of speech
+ * yields the unknown-word result.
+ *
+ * Following {@code opennlp.tools.lemmatizer.DictionaryLemmatizer}, a token with no lemma
+ * yields {@code "O"} from {@link #lemmatize(String[], String[])} and {@code ["O"]} from
+ * {@link #lemmatize(List, List)}. Both a lexicon and exception lists are required. Instances are
+ * immutable and safe for concurrent use.
+ */
+@ThreadSafe
+public final class MorphyLemmatizer implements Lemmatizer {
+
+ /** The unknown-word output, matching the DictionaryLemmatizer convention. */
+ private static final String UNKNOWN = "O";
+
+ private static final String[][] NOUN_RULES = {
+ {"s", ""}, {"ses", "s"}, {"xes", "x"}, {"zes", "z"},
+ {"ches", "ch"}, {"shes", "sh"}, {"men", "man"}, {"ies", "y"},
+ };
+
+ private static final String[][] VERB_RULES = {
+ {"s", ""}, {"ies", "y"}, {"es", "e"}, {"es", ""},
+ {"ed", "e"}, {"ed", ""}, {"ing", "e"}, {"ing", ""},
+ };
+
+ private static final String[][] ADJECTIVE_RULES = {
+ {"er", ""}, {"est", ""}, {"er", "e"}, {"est", "e"},
+ };
+
+ private static final String[][] NO_RULES = {};
+
+ private final LexicalKnowledgeBase lexicon;
+ private final MorphyExceptions exceptions;
+
+ /**
+ * Creates a Morphy lemmatizer over a loaded lexicon and exception lists.
+ *
+ * @param lexicon The lexicon rule candidates are validated against. Must not be
+ * {@code null}.
+ * @param exceptions The irregular-form exception lists. Must not be {@code null}.
+ * @throws IllegalArgumentException Thrown if {@code lexicon} or {@code exceptions} is
+ * {@code null}.
+ */
+ public MorphyLemmatizer(LexicalKnowledgeBase lexicon, MorphyExceptions exceptions) {
+ if (lexicon == null) {
+ throw new IllegalArgumentException("Lexicon must not be null");
+ }
+ if (exceptions == null) {
+ throw new IllegalArgumentException("Exceptions must not be null");
+ }
+ this.lexicon = lexicon;
+ this.exceptions = exceptions;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public String[] lemmatize(String[] toks, String[] tags) {
+ if (toks == null || tags == null) {
+ throw new IllegalArgumentException("Toks and tags must not be null");
+ }
+ if (toks.length != tags.length) {
+ throw new IllegalArgumentException("Toks and tags must have the same length, got "
+ + toks.length + " and " + tags.length);
+ }
+ final String[] lemmas = new String[toks.length];
+ for (int i = 0; i < toks.length; i++) {
+ final List candidates = lemmasOf(toks[i], tags[i]);
+ lemmas[i] = candidates.isEmpty() ? UNKNOWN : candidates.get(0);
+ }
+ return lemmas;
+ }
+
+ /** {@inheritDoc} */
+ @Override
+ public List> lemmatize(List toks, List tags) {
+ if (toks == null || tags == null) {
+ throw new IllegalArgumentException("Toks and tags must not be null");
+ }
+ if (toks.size() != tags.size()) {
+ throw new IllegalArgumentException("Toks and tags must have the same size, got "
+ + toks.size() + " and " + tags.size());
+ }
+ final List> lemmas = new ArrayList<>(toks.size());
+ for (int i = 0; i < toks.size(); i++) {
+ final List candidates = lemmasOf(toks.get(i), tags.get(i));
+ lemmas.add(candidates.isEmpty() ? List.of(UNKNOWN) : candidates);
+ }
+ return lemmas;
+ }
+
+ /**
+ * Finds all lemmas of one token, most preferred first.
+ *
+ * @param token The token to lemmatize.
+ * @param tag The part-of-speech tag.
+ * @return The candidate lemmas, empty when the word is unknown or the tag maps to no part of
+ * speech.
+ */
+ private List lemmasOf(String token, String tag) {
+ if (token == null || tag == null) {
+ throw new IllegalArgumentException("Tokens and tags must not contain null elements");
+ }
+ final WordNetPOS pos = posFromTag(tag);
+ if (pos == null) {
+ return List.of();
+ }
+ final String folded = LemmaFolding.fold(token);
+ final List irregular = exceptions.lookup(folded, pos);
+ if (!irregular.isEmpty()) {
+ return irregular;
+ }
+ final List candidates = new ArrayList<>(2);
+ if (lexicon.contains(folded, pos)) {
+ candidates.add(folded);
+ }
+ for (final String[] rule : rulesFor(pos)) {
+ final String suffix = rule[0];
+ if (folded.length() > suffix.length() && folded.endsWith(suffix)) {
+ final String candidate =
+ folded.substring(0, folded.length() - suffix.length()) + rule[1];
+ if (!candidates.contains(candidate) && lexicon.contains(candidate, pos)) {
+ candidates.add(candidate);
+ }
+ }
+ }
+ return candidates;
+ }
+
+ /**
+ * Selects the detachment-rule table for a part of speech.
+ *
+ * @param pos The part of speech.
+ * @return The suffix-substitution rules, empty for adverbs.
+ */
+ private static String[][] rulesFor(WordNetPOS pos) {
+ return switch (pos) {
+ case NOUN -> NOUN_RULES;
+ case VERB -> VERB_RULES;
+ case ADJECTIVE -> ADJECTIVE_RULES;
+ case ADVERB -> NO_RULES;
+ };
+ }
+
+ /**
+ * Maps a part-of-speech tag to a {@link WordNetPOS}. Package-private so tests can pin the
+ * mapping directly.
+ *
+ * @param tag The tag to map. Must not be {@code null}.
+ * @return The part of speech, or {@code null} when the tag names none.
+ * @throws IllegalArgumentException Thrown if {@code tag} is {@code null}.
+ */
+ static WordNetPOS posFromTag(String tag) {
+ if (tag == null) {
+ throw new IllegalArgumentException("Tag must not be null");
+ }
+ if (tag.isEmpty()) {
+ return null;
+ }
+ final String upper = tag.toUpperCase(Locale.ROOT);
+ if (upper.startsWith("ADJ")) {
+ return WordNetPOS.ADJECTIVE;
+ }
+ if (upper.startsWith("ADV")) {
+ return WordNetPOS.ADVERB;
+ }
+ return switch (upper.charAt(0)) {
+ case 'N' -> WordNetPOS.NOUN;
+ case 'V' -> WordNetPOS.VERB;
+ case 'J' -> WordNetPOS.ADJECTIVE;
+ // Codes a and s mean adjective only as one-letter tags; AUX, ADP and the like do not.
+ case 'A', 'S' -> tag.length() == 1 ? WordNetPOS.ADJECTIVE : null;
+ case 'R' -> WordNetPOS.ADVERB;
+ default -> null;
+ };
+ }
+}
diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java
new file mode 100644
index 0000000000..bcb7bea0ce
--- /dev/null
+++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java
@@ -0,0 +1,590 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.wordnet;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import javax.xml.XMLConstants;
+import javax.xml.stream.Location;
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamConstants;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
+
+import opennlp.tools.util.InvalidFormatException;
+import opennlp.tools.wordnet.LexicalKnowledgeBase;
+import opennlp.tools.wordnet.Synset;
+import opennlp.tools.wordnet.WordNetPOS;
+import opennlp.tools.wordnet.WordNetRelation;
+
+/**
+ * Reads a WN-LMF XML document (the Global WordNet Association interchange format, used by Open
+ * English WordNet and many other language wordnets) into a {@link LexicalKnowledgeBase} using the
+ * JDK StAX parser.
+ *
+ * It reads the subset of the format the contract serves: lexical entries, synsets with their
+ * definitions and typed relations, and sense relations, which are lifted to the synset level as
+ * documented on {@link WordNetRelation}. Elements outside that subset are skipped, as are
+ * relations of type {@code other} (the format's untyped escape hatch); any other unknown
+ * relation type fails loud.
+ *
+ * The parser is hardened against XXE: DTD processing and external entities are disabled, so a
+ * DOCTYPE is skipped but nothing it names is fetched or resolved.
+ *
+ * Malformed structure fails loud with an {@link InvalidFormatException} naming the resource
+ * and, where the parser provides one, the line; I/O failures propagate as {@link IOException}.
+ * Part-of-speech code {@code s} normalizes to {@link WordNetPOS#ADJECTIVE}, and a {@code similar}
+ * relation on a verb synset maps to {@link WordNetRelation#VERB_GROUP} rather than
+ * {@link WordNetRelation#SIMILAR_TO}. The returned lexicon is immutable and safe for concurrent
+ * lookups.
+ */
+public final class WnLmfReader {
+
+ private static final Map RELATION_NAMES = relationNames();
+
+ /** The format's escape-hatch relation type; carries no type the contract can express. */
+ private static final String OTHER_RELATION = "other";
+
+ /** Not instantiable. */
+ private WnLmfReader() {
+ }
+
+ /**
+ * Reads a WN-LMF XML file.
+ *
+ * @param file The XML file. Must not be {@code null} and must exist.
+ * @return The loaded lexicon.
+ * @throws IllegalArgumentException Thrown if {@code file} is {@code null} or missing.
+ * @throws InvalidFormatException Thrown if the document is malformed; the message names the
+ * file and, where available, the line.
+ * @throws IOException Thrown if reading the file fails.
+ */
+ public static LexicalKnowledgeBase read(Path file) throws IOException {
+ if (file == null) {
+ throw new IllegalArgumentException("File must not be null");
+ }
+ if (!Files.isRegularFile(file)) {
+ throw new IllegalArgumentException("File does not exist or is not a regular file: " + file);
+ }
+ try (InputStream in = new BufferedInputStream(Files.newInputStream(file))) {
+ return read(in, file.toString());
+ }
+ }
+
+ /**
+ * Reads a WN-LMF XML document from a stream. The stream is not closed.
+ *
+ * @param in The document stream. Must not be {@code null}.
+ * @param resourceName The name used in error messages. Must not be {@code null}.
+ * @return The loaded lexicon.
+ * @throws IllegalArgumentException Thrown if an argument is {@code null}.
+ * @throws InvalidFormatException Thrown if the document is malformed; the message names the
+ * resource and, where available, the line.
+ * @throws IOException Thrown if reading the stream fails.
+ */
+ public static LexicalKnowledgeBase read(InputStream in, String resourceName) throws IOException {
+ if (in == null) {
+ throw new IllegalArgumentException("In must not be null");
+ }
+ if (resourceName == null) {
+ throw new IllegalArgumentException("ResourceName must not be null");
+ }
+ final Parser parser = new Parser(resourceName);
+ try {
+ final XMLStreamReader reader = hardenedFactory().createXMLStreamReader(in);
+ try {
+ parser.parse(reader);
+ } finally {
+ reader.close();
+ }
+ } catch (XMLStreamException e) {
+ // StAX wraps a failing stream read in an XMLStreamException; surface it as the I/O failure.
+ final Throwable nested = e.getNestedException() == null ? e.getCause()
+ : e.getNestedException();
+ if (nested instanceof IOException io) {
+ throw io;
+ }
+ throw parser.malformed(e.getLocation(), "XML error: " + e.getMessage(), e);
+ }
+ return parser.build();
+ }
+
+ /**
+ * Builds an XXE-hardened StAX factory: the DTD internal subset is not processed and external
+ * entities and the external DTD subset are denied, so a DOCTYPE is skipped but never resolved.
+ *
+ * @return The hardened factory.
+ */
+ private static XMLInputFactory hardenedFactory() {
+ final XMLInputFactory factory = XMLInputFactory.newFactory();
+ factory.setProperty(XMLInputFactory.SUPPORT_DTD, Boolean.FALSE);
+ factory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, Boolean.FALSE);
+ factory.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, "");
+ factory.setProperty(XMLInputFactory.IS_COALESCING, Boolean.TRUE);
+ factory.setXMLResolver((publicId, systemId, baseUri, namespace) -> {
+ throw new XMLStreamException("External entity resolution is disabled, refusing " + systemId);
+ });
+ return factory;
+ }
+
+ /** Holds the streaming parse state and performs post-parse resolution. */
+ private static final class Parser {
+
+ private final String resourceName;
+
+ // Entry state.
+ private final Set entryIds = new HashSet<>();
+ private final Map lemmaByEntryId = new HashMap<>();
+ private final Map posByEntryId = new HashMap<>();
+ private final Map synsetBySenseId = new HashMap<>();
+ private final Map> senseOrder =
+ new LinkedHashMap<>();
+ private final List senseRelations = new ArrayList<>();
+ private final Map rawSynsets = new LinkedHashMap<>();
+ // Fallback membership (entry ids per synset in document order) when members is absent.
+ private final Map> entryIdsBySynset = new HashMap<>();
+
+ // Cursor state.
+ private String currentEntryId;
+ private String currentEntryLemma;
+ private WordNetPOS currentEntryPos;
+ private String currentSenseId;
+ private RawSynset currentSynset;
+
+ /**
+ * Creates a parser.
+ *
+ * @param resourceName The name used in error messages.
+ */
+ Parser(String resourceName) {
+ this.resourceName = resourceName;
+ }
+
+ /**
+ * Streams the document, dispatching start and end elements.
+ *
+ * @param reader The StAX reader.
+ * @throws XMLStreamException Thrown if the stream read fails.
+ * @throws InvalidFormatException Thrown if the document is malformed.
+ */
+ void parse(XMLStreamReader reader) throws XMLStreamException, InvalidFormatException {
+ while (reader.hasNext()) {
+ final int event = reader.next();
+ // A DTD event carries nothing that can affect parsing once the factory is hardened.
+ if (event == XMLStreamConstants.START_ELEMENT) {
+ startElement(reader);
+ } else if (event == XMLStreamConstants.END_ELEMENT) {
+ endElement(reader.getLocalName());
+ }
+ }
+ }
+
+ /**
+ * Handles one start element, updating cursor state and collecting raw entries, senses, and
+ * synsets.
+ *
+ * @param reader The StAX reader positioned on the start element.
+ * @throws XMLStreamException Thrown if reading element text fails.
+ * @throws InvalidFormatException Thrown if the element violates the format.
+ */
+ private void startElement(XMLStreamReader reader)
+ throws XMLStreamException, InvalidFormatException {
+ final String name = reader.getLocalName();
+ switch (name) {
+ case "LexicalEntry" -> {
+ currentEntryId = requireAttribute(reader, "id");
+ if (!entryIds.add(currentEntryId)) {
+ throw malformed(reader.getLocation(),
+ "Duplicate lexical entry id " + currentEntryId, null);
+ }
+ currentEntryLemma = null;
+ currentEntryPos = null;
+ }
+ case "Lemma" -> {
+ if (currentEntryId == null) {
+ throw malformed(reader.getLocation(), "Lemma outside a LexicalEntry", null);
+ }
+ currentEntryLemma = requireAttribute(reader, "writtenForm");
+ currentEntryPos = parsePos(requireAttribute(reader, "partOfSpeech"),
+ reader.getLocation());
+ lemmaByEntryId.put(currentEntryId, currentEntryLemma);
+ posByEntryId.put(currentEntryId, currentEntryPos);
+ }
+ case "Sense" -> {
+ if (currentEntryLemma == null) {
+ throw malformed(reader.getLocation(),
+ "Sense before its entry's Lemma in LexicalEntry " + currentEntryId, null);
+ }
+ currentSenseId = requireAttribute(reader, "id");
+ final String synsetId = requireAttribute(reader, "synset");
+ if (synsetBySenseId.putIfAbsent(currentSenseId, synsetId) != null) {
+ throw malformed(reader.getLocation(), "Duplicate sense id " + currentSenseId, null);
+ }
+ entryIdsBySynset.computeIfAbsent(synsetId, unused -> new ArrayList<>(2))
+ .add(currentEntryId);
+ final List order = senseOrder.computeIfAbsent(
+ InMemoryWordNetLexicon.LemmaKey.of(currentEntryLemma, currentEntryPos),
+ unused -> new ArrayList<>(2));
+ if (!order.contains(synsetId)) {
+ order.add(synsetId);
+ }
+ }
+ case "SenseRelation" -> {
+ if (currentSenseId == null) {
+ throw malformed(reader.getLocation(), "SenseRelation outside a Sense", null);
+ }
+ senseRelations.add(new RawSenseRelation(currentSenseId,
+ requireAttribute(reader, "relType"), requireAttribute(reader, "target"),
+ line(reader.getLocation())));
+ }
+ case "Synset" -> {
+ final String id = requireAttribute(reader, "id");
+ final WordNetPOS pos = parsePos(requireAttribute(reader, "partOfSpeech"),
+ reader.getLocation());
+ currentSynset = new RawSynset(id, pos, reader.getAttributeValue(null, "members"),
+ line(reader.getLocation()));
+ if (rawSynsets.putIfAbsent(id, currentSynset) != null) {
+ throw malformed(reader.getLocation(), "Duplicate synset id " + id, null);
+ }
+ }
+ case "Definition" -> {
+ if (currentSynset != null && currentSynset.gloss == null) {
+ currentSynset.gloss = reader.getElementText();
+ }
+ }
+ case "SynsetRelation" -> {
+ if (currentSynset == null) {
+ throw malformed(reader.getLocation(), "SynsetRelation outside a Synset", null);
+ }
+ final String relType = requireAttribute(reader, "relType");
+ final String target = requireAttribute(reader, "target");
+ // The escape-hatch type is a documented skip, not a rejection.
+ if (!OTHER_RELATION.equals(relType)) {
+ currentSynset.relations.add(
+ new RawRelation(relType, target, line(reader.getLocation())));
+ }
+ }
+ default -> {
+ // Pronunciation, Form, Example, SyntacticBehaviour, ILIDefinition, and other
+ // elements outside the contract subset are skipped.
+ }
+ }
+ }
+
+ /**
+ * Clears cursor state when a tracked element closes.
+ *
+ * @param name The local name of the closing element.
+ */
+ private void endElement(String name) {
+ switch (name) {
+ case "LexicalEntry" -> {
+ currentEntryId = null;
+ currentEntryLemma = null;
+ currentEntryPos = null;
+ }
+ case "Sense" -> currentSenseId = null;
+ case "Synset" -> currentSynset = null;
+ default -> {
+ // Nothing to close for skipped elements.
+ }
+ }
+ }
+
+ /**
+ * Resolves the collected raw state into an immutable lexicon: validates sense targets, lifts
+ * sense relations to the synset level, and materializes the contract synsets.
+ *
+ * @return The loaded lexicon.
+ * @throws InvalidFormatException Thrown if a sense or relation references an undeclared
+ * target, or a synset has no members.
+ */
+ LexicalKnowledgeBase build() throws InvalidFormatException {
+ // Every sense must point to a declared synset, with a consistent part of speech.
+ for (final Map.Entry sense : synsetBySenseId.entrySet()) {
+ final RawSynset target = rawSynsets.get(sense.getValue());
+ if (target == null) {
+ throw malformed(null,
+ "Sense " + sense.getKey() + " references undeclared synset " + sense.getValue(),
+ null);
+ }
+ }
+ // Lift sense relations to the synset level.
+ for (final RawSenseRelation relation : senseRelations) {
+ if (OTHER_RELATION.equals(relation.relType)) {
+ continue;
+ }
+ final String sourceSynsetId = synsetBySenseId.get(relation.sourceSenseId);
+ final String targetSynsetId = synsetBySenseId.get(relation.targetSenseId);
+ if (targetSynsetId == null) {
+ throw malformed(null, "SenseRelation at line " + relation.line + " from sense "
+ + relation.sourceSenseId + " references undeclared sense " + relation.targetSenseId,
+ null);
+ }
+ final RawSynset source = rawSynsets.get(sourceSynsetId);
+ source.relations.add(new RawRelation(relation.relType, targetSynsetId, relation.line));
+ }
+ // Resolve raw synsets into contract synsets.
+ final Map synsetsById = new LinkedHashMap<>(rawSynsets.size() * 2);
+ for (final RawSynset raw : rawSynsets.values()) {
+ final Map> relations = resolveRelations(raw);
+ synsetsById.put(raw.id,
+ new Synset(raw.id, raw.pos, memberLemmas(raw), raw.gloss == null ? "" : raw.gloss,
+ relations));
+ }
+ return new InMemoryWordNetLexicon(synsetsById, senseOrder);
+ }
+
+ /**
+ * Resolves a raw synset's relations into typed target-id lists, deduplicated in source order.
+ *
+ * @param raw The raw synset.
+ * @return The typed relations for the contract synset.
+ * @throws InvalidFormatException Thrown if a relation type is unknown or its target is
+ * undeclared.
+ */
+ private Map> resolveRelations(RawSynset raw)
+ throws InvalidFormatException {
+ final Map> typed = new LinkedHashMap<>();
+ for (final RawRelation relation : raw.relations) {
+ final WordNetRelation type = parseRelation(relation.relType, raw.pos, relation.line);
+ final RawSynset target = rawSynsets.get(relation.target);
+ if (target == null) {
+ throw malformed(null, "Relation " + relation.relType + " at line " + relation.line
+ + " on synset " + raw.id + " references undeclared synset " + relation.target, null);
+ }
+ // Share the synset table's id instance so only one copy of each id is retained.
+ typed.computeIfAbsent(type, unused -> new LinkedHashSet<>()).add(target.id);
+ }
+ final Map> relations = new LinkedHashMap<>(typed.size() * 2);
+ for (final Map.Entry> entry : typed.entrySet()) {
+ relations.put(entry.getKey(), List.copyOf(entry.getValue()));
+ }
+ return relations;
+ }
+
+ /**
+ * Resolves a synset's member entry ids to their lemmas, from the {@code members} attribute
+ * when present and otherwise from the senses that pointed at the synset.
+ *
+ * @param raw The raw synset.
+ * @return The member lemmas in source order, deduplicated.
+ * @throws InvalidFormatException Thrown if the synset has no members, names an undeclared
+ * entry, or a member's part of speech disagrees with the synset's.
+ */
+ private List memberLemmas(RawSynset raw) throws InvalidFormatException {
+ final List entryIds;
+ if (raw.members != null && !raw.members.isEmpty()) {
+ entryIds = LemmaFolding.splitOnSpaces(raw.members);
+ } else {
+ final List fromSenses = entryIdsBySynset.get(raw.id);
+ entryIds = fromSenses == null ? List.of() : fromSenses;
+ }
+ if (entryIds.isEmpty()) {
+ throw malformed(null, "Synset " + raw.id + " at line " + raw.line
+ + " has no member entries", null);
+ }
+ final List lemmas = new ArrayList<>(entryIds.size());
+ for (final String entryId : entryIds) {
+ final String lemma = lemmaByEntryId.get(entryId);
+ if (lemma == null) {
+ throw malformed(null, "Synset " + raw.id + " at line " + raw.line
+ + " lists undeclared member entry " + entryId, null);
+ }
+ if (raw.pos != posByEntryId.get(entryId)) {
+ throw malformed(null, "Synset " + raw.id + " at line " + raw.line
+ + " has part of speech " + raw.pos + " but member entry " + entryId
+ + " has " + posByEntryId.get(entryId), null);
+ }
+ if (!lemmas.contains(lemma)) {
+ lemmas.add(lemma);
+ }
+ }
+ return lemmas;
+ }
+
+ /**
+ * Maps a WN-LMF part-of-speech code to a {@link WordNetPOS}; code {@code s} normalizes to
+ * {@link WordNetPOS#ADJECTIVE}.
+ *
+ * @param code The part-of-speech code.
+ * @param location The parser location, for error reporting.
+ * @return The part of speech.
+ * @throws InvalidFormatException Thrown if the code is unknown.
+ */
+ private WordNetPOS parsePos(String code, Location location) throws InvalidFormatException {
+ return switch (code) {
+ case "n" -> WordNetPOS.NOUN;
+ case "v" -> WordNetPOS.VERB;
+ case "a", "s" -> WordNetPOS.ADJECTIVE;
+ case "r" -> WordNetPOS.ADVERB;
+ default -> throw malformed(location, "Unknown part-of-speech code: " + code, null);
+ };
+ }
+
+ /**
+ * Maps a WN-LMF relation name to a {@link WordNetRelation}. A {@code similar} relation on a
+ * verb synset maps to {@link WordNetRelation#VERB_GROUP}, otherwise to
+ * {@link WordNetRelation#SIMILAR_TO}.
+ *
+ * @param relType The relation name.
+ * @param sourcePos The part of speech of the source synset.
+ * @param line The document line, for error reporting.
+ * @return The mapped relation.
+ * @throws InvalidFormatException Thrown if the relation name is unknown.
+ */
+ private WordNetRelation parseRelation(String relType, WordNetPOS sourcePos, int line)
+ throws InvalidFormatException {
+ if ("similar".equals(relType)) {
+ return sourcePos == WordNetPOS.VERB ? WordNetRelation.VERB_GROUP
+ : WordNetRelation.SIMILAR_TO;
+ }
+ final WordNetRelation relation = RELATION_NAMES.get(relType);
+ if (relation == null) {
+ throw malformed(null, "Unknown relation type " + relType + " at line " + line, null);
+ }
+ return relation;
+ }
+
+ /**
+ * Reads a required attribute from the current element.
+ *
+ * @param reader The StAX reader.
+ * @param attribute The attribute name.
+ * @return The non-empty attribute value.
+ * @throws InvalidFormatException Thrown if the attribute is absent or empty.
+ */
+ private String requireAttribute(XMLStreamReader reader, String attribute)
+ throws InvalidFormatException {
+ final String value = reader.getAttributeValue(null, attribute);
+ if (value == null || value.isEmpty()) {
+ throw malformed(reader.getLocation(), "Element " + reader.getLocalName()
+ + " is missing required attribute " + attribute, null);
+ }
+ return value;
+ }
+
+ /**
+ * Builds a malformed-document exception naming the resource and, when known, the line.
+ *
+ * @param location The parser location, or {@code null} when unavailable.
+ * @param message The failure detail.
+ * @param cause The underlying cause, or {@code null}.
+ * @return The exception to throw.
+ */
+ InvalidFormatException malformed(Location location, String message, Throwable cause) {
+ final int line = line(location);
+ final String prefix = line < 0 ? "Malformed WN-LMF document " + resourceName + ": "
+ : "Malformed WN-LMF document " + resourceName + " at line " + line + ": ";
+ return cause == null ? new InvalidFormatException(prefix + message)
+ : new InvalidFormatException(prefix + message, cause);
+ }
+
+ /**
+ * Extracts a line number from a parser location.
+ *
+ * @param location The location, or {@code null}.
+ * @return The line number, or {@code -1} when unknown.
+ */
+ private static int line(Location location) {
+ return location == null ? -1 : location.getLineNumber();
+ }
+ }
+
+ private static final class RawSynset {
+ private final String id;
+ private final WordNetPOS pos;
+ private final String members;
+ private final int line;
+ private final List relations = new ArrayList<>(4);
+ private String gloss;
+
+ /**
+ * Creates a raw synset gathered during parsing.
+ *
+ * @param id The synset id.
+ * @param pos The part of speech.
+ * @param members The {@code members} attribute value, or {@code null} when absent.
+ * @param line The document line.
+ */
+ RawSynset(String id, WordNetPOS pos, String members, int line) {
+ this.id = id;
+ this.pos = pos;
+ this.members = members;
+ this.line = line;
+ }
+ }
+
+ /** A parsed synset relation, kept until the target synset is known. */
+ private record RawRelation(String relType, String target, int line) {
+ }
+
+ /** A parsed sense relation, kept until both sense ids are known. */
+ private record RawSenseRelation(String sourceSenseId, String relType, String targetSenseId,
+ int line) {
+ }
+
+ /**
+ * Builds the WN-LMF relation-name to {@link WordNetRelation} table.
+ *
+ * @return The immutable name table.
+ */
+ private static Map relationNames() {
+ final Map names = new HashMap<>();
+ names.put("antonym", WordNetRelation.ANTONYM);
+ names.put("hypernym", WordNetRelation.HYPERNYM);
+ names.put("instance_hypernym", WordNetRelation.INSTANCE_HYPERNYM);
+ names.put("hyponym", WordNetRelation.HYPONYM);
+ names.put("instance_hyponym", WordNetRelation.INSTANCE_HYPONYM);
+ names.put("holo_member", WordNetRelation.MEMBER_HOLONYM);
+ names.put("holo_substance", WordNetRelation.SUBSTANCE_HOLONYM);
+ names.put("holo_part", WordNetRelation.PART_HOLONYM);
+ names.put("mero_member", WordNetRelation.MEMBER_MERONYM);
+ names.put("mero_substance", WordNetRelation.SUBSTANCE_MERONYM);
+ names.put("mero_part", WordNetRelation.PART_MERONYM);
+ names.put("attribute", WordNetRelation.ATTRIBUTE);
+ names.put("derivation", WordNetRelation.DERIVATIONALLY_RELATED);
+ names.put("entails", WordNetRelation.ENTAILMENT);
+ names.put("is_entailed_by", WordNetRelation.ENTAILED_BY);
+ names.put("causes", WordNetRelation.CAUSE);
+ names.put("is_caused_by", WordNetRelation.CAUSED_BY);
+ names.put("also", WordNetRelation.ALSO_SEE);
+ names.put("participle", WordNetRelation.PARTICIPLE);
+ names.put("pertainym", WordNetRelation.PERTAINYM);
+ names.put("domain_topic", WordNetRelation.DOMAIN_TOPIC);
+ names.put("has_domain_topic", WordNetRelation.MEMBER_OF_DOMAIN_TOPIC);
+ names.put("domain_region", WordNetRelation.DOMAIN_REGION);
+ names.put("has_domain_region", WordNetRelation.MEMBER_OF_DOMAIN_REGION);
+ // The usage domain carries both its current WN-LMF name and the legacy alias.
+ names.put("exemplifies", WordNetRelation.DOMAIN_USAGE);
+ names.put("domain_usage", WordNetRelation.DOMAIN_USAGE);
+ names.put("is_exemplified_by", WordNetRelation.MEMBER_OF_DOMAIN_USAGE);
+ names.put("has_domain_usage", WordNetRelation.MEMBER_OF_DOMAIN_USAGE);
+ return Map.copyOf(names);
+ }
+}
diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java
new file mode 100644
index 0000000000..67e280e3a9
--- /dev/null
+++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java
@@ -0,0 +1,610 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.wordnet;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+
+import opennlp.tools.util.InvalidFormatException;
+import opennlp.tools.wordnet.LexicalKnowledgeBase;
+import opennlp.tools.wordnet.Synset;
+import opennlp.tools.wordnet.WordNetPOS;
+import opennlp.tools.wordnet.WordNetRelation;
+
+/**
+ * Reads a legacy Princeton WNDB database directory ({@code index.noun}, {@code data.noun}, and
+ * the corresponding pairs for verbs, adjectives, and adverbs) into a {@link LexicalKnowledgeBase}.
+ *
+ * All eight index and data files must be present. License preamble lines (which begin with a
+ * space in the released files) are skipped. {@code index.sense} is not read, and the
+ * {@code *.exc} exception lists are the {@link MorphyLemmatizer} companion input, read
+ * separately.
+ *
+ * Synset ids are minted as {@code wndb-}offset{@code -}pos from the data file's
+ * 8-digit byte offset and part-of-speech letter, for example {@code wndb-00001740-n}; the id is
+ * opaque to consumers. Adjective satellite lines normalize to {@link WordNetPOS#ADJECTIVE}, the
+ * syntactic markers the adjective files append ({@code (p)}, {@code (a)}, {@code (ip)}) are
+ * stripped, and underscores in lemmas become spaces. Sense order per lemma follows the index
+ * file's offset order.
+ *
+ * Malformed content fails loud with an {@link InvalidFormatException} naming the file and
+ * line; I/O failures propagate as {@link IOException}. The returned lexicon is immutable and safe
+ * for concurrent lookups.
+ */
+public final class WndbReader {
+
+ private static final Map POINTER_SYMBOLS = pointerSymbols();
+
+ /** Not instantiable. */
+ private WndbReader() {
+ }
+
+ /**
+ * Reads a WNDB database directory.
+ *
+ * @param directory The directory containing the eight index and data files. Must not be
+ * {@code null} and must exist.
+ * @return The loaded lexicon.
+ * @throws IllegalArgumentException Thrown if {@code directory} is {@code null} or not a
+ * directory.
+ * @throws InvalidFormatException Thrown if a database file is missing or any file is
+ * malformed; the message names the file and line.
+ * @throws IOException Thrown if reading a file fails.
+ */
+ public static LexicalKnowledgeBase read(Path directory) throws IOException {
+ if (directory == null) {
+ throw new IllegalArgumentException("Directory must not be null");
+ }
+ if (!Files.isDirectory(directory)) {
+ throw new IllegalArgumentException(
+ "Directory does not exist or is not a directory: " + directory);
+ }
+ final Map rawSynsets = new LinkedHashMap<>();
+ for (final FilePos filePos : FilePos.values()) {
+ parseDataFile(directory, filePos, rawSynsets);
+ }
+ final Map synsetsById = resolve(rawSynsets);
+ final Map> senseOrder = new LinkedHashMap<>();
+ for (final FilePos filePos : FilePos.values()) {
+ parseIndexFile(directory, filePos, rawSynsets, senseOrder);
+ }
+ return new InMemoryWordNetLexicon(synsetsById, senseOrder);
+ }
+
+ /** The four part-of-speech file pairs of a WNDB directory. */
+ private enum FilePos {
+ NOUN("noun", 'n', WordNetPOS.NOUN),
+ VERB("verb", 'v', WordNetPOS.VERB),
+ ADJECTIVE("adj", 'a', WordNetPOS.ADJECTIVE),
+ ADVERB("adv", 'r', WordNetPOS.ADVERB);
+
+ private final String suffix;
+ private final char posChar;
+ private final WordNetPOS pos;
+
+ /**
+ * Binds a part of speech to its file suffix and WNDB letter.
+ *
+ * @param suffix The file suffix, for example {@code noun}.
+ * @param posChar The WNDB part-of-speech letter.
+ * @param pos The mapped part of speech.
+ */
+ FilePos(String suffix, char posChar, WordNetPOS pos) {
+ this.suffix = suffix;
+ this.posChar = posChar;
+ this.pos = pos;
+ }
+ }
+
+ /**
+ * Parses one {@code data.*} file, collecting its synsets keyed by minted id.
+ *
+ * @param directory The database directory.
+ * @param filePos The part-of-speech file pair.
+ * @param rawSynsets The accumulating synset table.
+ * @throws IOException Thrown if the file is missing, malformed, or unreadable.
+ */
+ private static void parseDataFile(Path directory, FilePos filePos,
+ Map rawSynsets) throws IOException {
+ final String fileName = "data." + filePos.suffix;
+ final byte[] bytes = readAll(directory.resolve(fileName), fileName);
+ int lineStart = 0;
+ int lineNumber = 0;
+ while (lineStart < bytes.length) {
+ lineNumber++;
+ int lineEnd = lineStart;
+ while (lineEnd < bytes.length && bytes[lineEnd] != '\n') {
+ lineEnd++;
+ }
+ // ISO-8859-1 decodes bytes one-to-one, keeping offsets exact for any released file.
+ final String line =
+ new String(bytes, lineStart, lineEnd - lineStart, StandardCharsets.ISO_8859_1);
+ if (!line.isEmpty() && line.charAt(0) != ' ') {
+ parseDataLine(line, lineStart, fileName, lineNumber, filePos, rawSynsets);
+ }
+ lineStart = lineEnd + 1;
+ }
+ }
+
+ /**
+ * Parses one data-file synset line into a raw synset.
+ *
+ * @param line The decoded line, without its trailing newline.
+ * @param byteOffset The line's byte offset, matched against the line's own offset field.
+ * @param fileName The data file name, for error reporting.
+ * @param lineNumber The 1-based line number.
+ * @param filePos The part-of-speech file pair.
+ * @param rawSynsets The accumulating synset table.
+ * @throws InvalidFormatException Thrown if the line is malformed or its offset field disagrees
+ * with its byte position.
+ */
+ private static void parseDataLine(String line, int byteOffset, String fileName, int lineNumber,
+ FilePos filePos, Map rawSynsets)
+ throws InvalidFormatException {
+ final Tokenizer tokens = new Tokenizer(line, fileName, lineNumber);
+ final String offsetField = tokens.next("synset_offset");
+ if (parseOffset(offsetField, tokens) != byteOffset) {
+ throw malformed(fileName, lineNumber, "Synset offset field " + offsetField
+ + " disagrees with the actual byte position " + byteOffset);
+ }
+ tokens.next("lex_filenum");
+ final String ssType = tokens.next("ss_type");
+ final boolean validType = switch (filePos) {
+ case ADJECTIVE -> "a".equals(ssType) || "s".equals(ssType);
+ default -> ssType.length() == 1 && ssType.charAt(0) == filePos.posChar;
+ };
+ if (!validType) {
+ throw malformed(fileName, lineNumber,
+ "Synset type " + ssType + " does not belong in " + fileName);
+ }
+ final int wordCount = tokens.nextInt("w_cnt", 16);
+ if (wordCount < 1) {
+ throw malformed(fileName, lineNumber, "Word count must be at least 1, got: " + wordCount);
+ }
+ final List lemmas = new ArrayList<>(wordCount);
+ for (int i = 0; i < wordCount; i++) {
+ final String lemma = cleanLemma(tokens.next("word"), fileName, lineNumber);
+ tokens.nextInt("lex_id", 16);
+ if (!lemmas.contains(lemma)) {
+ lemmas.add(lemma);
+ }
+ }
+ final int pointerCount = tokens.nextInt("p_cnt", 10);
+ final List pointers = new ArrayList<>(pointerCount);
+ for (int i = 0; i < pointerCount; i++) {
+ final String symbol = tokens.next("pointer_symbol");
+ final WordNetRelation relation = POINTER_SYMBOLS.get(symbol);
+ if (relation == null) {
+ throw malformed(fileName, lineNumber, "Undeclared pointer symbol: " + symbol);
+ }
+ final String targetOffset = tokens.next("pointer synset_offset");
+ parseOffset(targetOffset, tokens);
+ final char targetPos = posChar(tokens.next("pointer pos"), tokens);
+ tokens.next("pointer source/target");
+ pointers.add(new RawPointer(relation, "wndb-" + targetOffset + "-" + targetPos, lineNumber));
+ }
+ if (filePos == FilePos.VERB) {
+ final int frameCount = tokens.nextInt("f_cnt", 10);
+ for (int i = 0; i < frameCount; i++) {
+ tokens.next("frame marker");
+ tokens.next("f_num");
+ tokens.next("w_num");
+ }
+ }
+ final String gloss = tokens.gloss();
+ final String id = "wndb-" + offsetField + "-" + filePos.posChar;
+ rawSynsets.put(id, new RawSynset(id, filePos.pos, lemmas, gloss, pointers,
+ fileName, lineNumber));
+ }
+
+ /**
+ * Resolves raw synsets into contract synsets, validating every pointer target.
+ *
+ * @param rawSynsets The parsed synsets keyed by id.
+ * @return The contract synsets keyed by id.
+ * @throws InvalidFormatException Thrown if a pointer targets a nonexistent synset.
+ */
+ private static Map resolve(Map rawSynsets)
+ throws InvalidFormatException {
+ final Map synsetsById = new LinkedHashMap<>(rawSynsets.size() * 2);
+ for (final RawSynset raw : rawSynsets.values()) {
+ final Map> typed = new LinkedHashMap<>();
+ for (final RawPointer pointer : raw.pointers) {
+ final RawSynset target = rawSynsets.get(pointer.targetId);
+ if (target == null) {
+ throw malformed(raw.fileName, pointer.lineNumber, "Synset " + raw.id + " has a "
+ + pointer.relation + " pointer to nonexistent synset " + pointer.targetId);
+ }
+ // Share the synset table's id instance so only one copy of each id is retained.
+ typed.computeIfAbsent(pointer.relation, unused -> new LinkedHashSet<>())
+ .add(target.id);
+ }
+ final Map> relations = new LinkedHashMap<>(typed.size() * 2);
+ for (final Map.Entry> entry : typed.entrySet()) {
+ relations.put(entry.getKey(), List.copyOf(entry.getValue()));
+ }
+ synsetsById.put(raw.id, new Synset(raw.id, raw.pos, raw.lemmas, raw.gloss, relations));
+ }
+ return synsetsById;
+ }
+
+ /**
+ * Parses one {@code index.*} file, building the sense order per folded lemma key.
+ *
+ * @param directory The database directory.
+ * @param filePos The part-of-speech file pair.
+ * @param rawSynsets The resolved synset table, for offset validation.
+ * @param senses The accumulating sense-order map.
+ * @throws IOException Thrown if the file is missing, malformed, or unreadable.
+ */
+ private static void parseIndexFile(Path directory, FilePos filePos,
+ Map rawSynsets,
+ Map> senses)
+ throws IOException {
+ final String fileName = "index." + filePos.suffix;
+ final byte[] bytes = readAll(directory.resolve(fileName), fileName);
+ final String content = new String(bytes, StandardCharsets.ISO_8859_1);
+ int lineNumber = 0;
+ int lineStart = 0;
+ while (lineStart < content.length()) {
+ lineNumber++;
+ int lineEnd = content.indexOf('\n', lineStart);
+ if (lineEnd < 0) {
+ lineEnd = content.length();
+ }
+ final String line = content.substring(lineStart, lineEnd);
+ if (!line.isEmpty() && line.charAt(0) != ' ') {
+ parseIndexLine(line, fileName, lineNumber, filePos, rawSynsets, senses);
+ }
+ lineStart = lineEnd + 1;
+ }
+ }
+
+ /**
+ * Parses one index-file line into a lemma's sense order.
+ *
+ * @param line The line to parse.
+ * @param fileName The index file name, for error reporting.
+ * @param lineNumber The 1-based line number.
+ * @param filePos The part-of-speech file pair.
+ * @param rawSynsets The resolved synset table, for offset validation.
+ * @param senses The accumulating sense-order map.
+ * @throws InvalidFormatException Thrown if the line is malformed or references an unknown
+ * offset.
+ */
+ private static void parseIndexLine(String line, String fileName, int lineNumber,
+ FilePos filePos, Map rawSynsets,
+ Map> senses)
+ throws InvalidFormatException {
+ final Tokenizer tokens = new Tokenizer(line, fileName, lineNumber);
+ final String lemma = tokens.next("lemma");
+ final String pos = tokens.next("pos");
+ if (pos.length() != 1 || pos.charAt(0) != filePos.posChar) {
+ throw malformed(fileName, lineNumber, "Index pos " + pos + " does not belong in "
+ + fileName);
+ }
+ final int synsetCount = tokens.nextInt("synset_cnt", 10);
+ if (synsetCount < 1) {
+ throw malformed(fileName, lineNumber,
+ "Synset count must be at least 1, got: " + synsetCount);
+ }
+ final int pointerTypeCount = tokens.nextInt("p_cnt", 10);
+ for (int i = 0; i < pointerTypeCount; i++) {
+ // The summary symbols are informational; the data file's pointers are authoritative.
+ tokens.next("ptr_symbol");
+ }
+ tokens.next("sense_cnt");
+ tokens.next("tagsense_cnt");
+ final List order = new ArrayList<>(synsetCount);
+ for (int i = 0; i < synsetCount; i++) {
+ final String offset = tokens.next("synset_offset");
+ parseOffset(offset, tokens);
+ final String synsetId = "wndb-" + offset + "-" + filePos.posChar;
+ if (!rawSynsets.containsKey(synsetId)) {
+ throw malformed(fileName, lineNumber, "Lemma " + lemma + " references offset " + offset
+ + " with no data." + filePos.suffix + " line");
+ }
+ if (!order.contains(synsetId)) {
+ order.add(synsetId);
+ }
+ }
+ final InMemoryWordNetLexicon.LemmaKey key =
+ InMemoryWordNetLexicon.LemmaKey.of(lemma, filePos.pos);
+ final List existing = senses.get(key);
+ if (existing == null) {
+ senses.put(key, order);
+ } else {
+ // Two index lemmas can fold to one key; keep first-listed order and append the rest.
+ for (final String synsetId : order) {
+ if (!existing.contains(synsetId)) {
+ existing.add(synsetId);
+ }
+ }
+ }
+ }
+
+ /**
+ * Strips the adjective syntactic markers ({@code (p)}, {@code (a)}, {@code (ip)}) and turns
+ * underscores into spaces.
+ *
+ * @param word The raw word field.
+ * @param fileName The data file name, for error reporting.
+ * @param lineNumber The 1-based line number.
+ * @return The cleaned lemma.
+ * @throws InvalidFormatException Thrown if the word carries an unknown marker or is empty.
+ */
+ private static String cleanLemma(String word, String fileName, int lineNumber)
+ throws InvalidFormatException {
+ String cleaned = word;
+ if (cleaned.endsWith(")")) {
+ final int open = cleaned.lastIndexOf('(');
+ final String marker = open < 0 ? "" : cleaned.substring(open);
+ if (!"(p)".equals(marker) && !"(a)".equals(marker) && !"(ip)".equals(marker)) {
+ throw malformed(fileName, lineNumber, "Unknown syntactic marker on word: " + word);
+ }
+ cleaned = cleaned.substring(0, open);
+ }
+ if (cleaned.isEmpty()) {
+ throw malformed(fileName, lineNumber, "Empty word field");
+ }
+ return cleaned.replace('_', ' ');
+ }
+
+ /**
+ * Parses an 8-digit synset offset.
+ *
+ * @param offset The offset field.
+ * @param tokens The tokenizer, for error reporting.
+ * @return The offset as an integer.
+ * @throws InvalidFormatException Thrown if the field is not 8 digits.
+ */
+ private static int parseOffset(String offset, Tokenizer tokens) throws InvalidFormatException {
+ if (offset.length() != 8) {
+ throw tokens.malformedToken("Synset offset must be 8 digits, got: " + offset);
+ }
+ int value = 0;
+ for (int i = 0; i < 8; i++) {
+ final char c = offset.charAt(i);
+ if (c < '0' || c > '9') {
+ throw tokens.malformedToken("Synset offset must be 8 digits, got: " + offset);
+ }
+ value = value * 10 + (c - '0');
+ }
+ return value;
+ }
+
+ /**
+ * Parses a pointer's one-letter part-of-speech code.
+ *
+ * @param pos The code field.
+ * @param tokens The tokenizer, for error reporting.
+ * @return One of {@code n}, {@code v}, {@code a}, {@code r}.
+ * @throws InvalidFormatException Thrown if the code is not one of those letters.
+ */
+ private static char posChar(String pos, Tokenizer tokens) throws InvalidFormatException {
+ if (pos.length() == 1) {
+ final char c = pos.charAt(0);
+ if (c == 'n' || c == 'v' || c == 'a' || c == 'r') {
+ return c;
+ }
+ }
+ throw tokens.malformedToken("Pointer pos must be one of n, v, a, r, got: " + pos);
+ }
+
+ /**
+ * Reads a required database file in full.
+ *
+ * @param file The file path.
+ * @param fileName The file name, for error reporting.
+ * @return The file bytes.
+ * @throws InvalidFormatException Thrown if the file is missing.
+ * @throws IOException Thrown if reading fails.
+ */
+ private static byte[] readAll(Path file, String fileName) throws IOException {
+ if (!Files.isRegularFile(file)) {
+ throw new InvalidFormatException("Missing WNDB database file: " + file);
+ }
+ return Files.readAllBytes(file);
+ }
+
+ /**
+ * Builds a malformed-file exception naming the file and line.
+ *
+ * @param fileName The file name.
+ * @param lineNumber The 1-based line number.
+ * @param message The failure detail.
+ * @return The exception to throw.
+ */
+ private static InvalidFormatException malformed(String fileName, int lineNumber,
+ String message) {
+ return new InvalidFormatException(
+ "Malformed WNDB file " + fileName + " at line " + lineNumber + ": " + message);
+ }
+
+ /** A cursor over one line's space-separated fields. */
+ private static final class Tokenizer {
+
+ private final String line;
+ private final String fileName;
+ private final int lineNumber;
+ private int position;
+
+ /**
+ * Creates a tokenizer over one line.
+ *
+ * @param line The line to tokenize.
+ * @param fileName The file name, for error reporting.
+ * @param lineNumber The 1-based line number.
+ */
+ Tokenizer(String line, String fileName, int lineNumber) {
+ this.line = line;
+ this.fileName = fileName;
+ this.lineNumber = lineNumber;
+ }
+
+ /**
+ * Reads the next space-separated field.
+ *
+ * @param field The field name, for error reporting.
+ * @return The field value.
+ * @throws InvalidFormatException Thrown if the line is truncated before the field.
+ */
+ String next(String field) throws InvalidFormatException {
+ while (position < line.length() && line.charAt(position) == ' ') {
+ position++;
+ }
+ if (position >= line.length()) {
+ throw malformed(fileName, lineNumber, "Truncated line, missing field: " + field);
+ }
+ final int start = position;
+ while (position < line.length() && line.charAt(position) != ' ') {
+ position++;
+ }
+ return line.substring(start, position);
+ }
+
+ /**
+ * Reads the next field as an integer in the given radix.
+ *
+ * @param field The field name, for error reporting.
+ * @param radix The numeric radix.
+ * @return The parsed value.
+ * @throws InvalidFormatException Thrown if the field is missing or not a valid integer.
+ */
+ int nextInt(String field, int radix) throws InvalidFormatException {
+ final String token = next(field);
+ try {
+ return Integer.parseInt(token, radix);
+ } catch (NumberFormatException e) {
+ throw new InvalidFormatException(malformed(fileName, lineNumber,
+ "Field " + field + " is not a base-" + radix + " integer: " + token).getMessage(), e);
+ }
+ }
+
+ /**
+ * Reads the gloss: the remainder after the pipe separator, trimmed of surrounding spaces.
+ *
+ * @return The gloss text.
+ * @throws InvalidFormatException Thrown if the pipe separator is missing.
+ */
+ String gloss() throws InvalidFormatException {
+ final String separator = next("gloss separator");
+ if (!"|".equals(separator)) {
+ throw malformed(fileName, lineNumber, "Expected the | gloss separator, got: " + separator);
+ }
+ int start = position;
+ while (start < line.length() && line.charAt(start) == ' ') {
+ start++;
+ }
+ int end = line.length();
+ while (end > start && line.charAt(end - 1) == ' ') {
+ end--;
+ }
+ return line.substring(start, end);
+ }
+
+ /**
+ * Builds a malformed-file exception at this tokenizer's line.
+ *
+ * @param message The failure detail.
+ * @return The exception to throw.
+ */
+ InvalidFormatException malformedToken(String message) {
+ return malformed(fileName, lineNumber, message);
+ }
+ }
+
+ /** A parsed pointer line, kept until the target synset is known. */
+ private record RawPointer(WordNetRelation relation, String targetId, int lineNumber) {
+ }
+
+ private static final class RawSynset {
+ private final String id;
+ private final WordNetPOS pos;
+ private final List lemmas;
+ private final String gloss;
+ private final List pointers;
+ private final String fileName;
+ private final int lineNumber;
+
+ /**
+ * Creates a raw synset gathered while parsing a data file.
+ *
+ * @param id The minted synset id.
+ * @param pos The part of speech.
+ * @param lemmas The member lemmas.
+ * @param gloss The gloss text.
+ * @param pointers The raw pointers to resolve.
+ * @param fileName The source file name.
+ * @param lineNumber The source line number.
+ */
+ RawSynset(String id, WordNetPOS pos, List lemmas, String gloss,
+ List pointers, String fileName, int lineNumber) {
+ this.id = id;
+ this.pos = pos;
+ this.lemmas = lemmas;
+ this.gloss = gloss;
+ this.pointers = pointers;
+ this.fileName = fileName;
+ this.lineNumber = lineNumber;
+ }
+ }
+
+ /**
+ * Builds the WNDB pointer-symbol to {@link WordNetRelation} table.
+ *
+ * @return The immutable symbol table.
+ */
+ private static Map pointerSymbols() {
+ final Map symbols = new HashMap<>();
+ symbols.put("!", WordNetRelation.ANTONYM);
+ symbols.put("@", WordNetRelation.HYPERNYM);
+ symbols.put("@i", WordNetRelation.INSTANCE_HYPERNYM);
+ symbols.put("~", WordNetRelation.HYPONYM);
+ symbols.put("~i", WordNetRelation.INSTANCE_HYPONYM);
+ symbols.put("#m", WordNetRelation.MEMBER_HOLONYM);
+ symbols.put("#s", WordNetRelation.SUBSTANCE_HOLONYM);
+ symbols.put("#p", WordNetRelation.PART_HOLONYM);
+ symbols.put("%m", WordNetRelation.MEMBER_MERONYM);
+ symbols.put("%s", WordNetRelation.SUBSTANCE_MERONYM);
+ symbols.put("%p", WordNetRelation.PART_MERONYM);
+ symbols.put("=", WordNetRelation.ATTRIBUTE);
+ symbols.put("+", WordNetRelation.DERIVATIONALLY_RELATED);
+ symbols.put("*", WordNetRelation.ENTAILMENT);
+ symbols.put(">", WordNetRelation.CAUSE);
+ symbols.put("^", WordNetRelation.ALSO_SEE);
+ symbols.put("$", WordNetRelation.VERB_GROUP);
+ symbols.put("&", WordNetRelation.SIMILAR_TO);
+ symbols.put("<", WordNetRelation.PARTICIPLE);
+ symbols.put("\\", WordNetRelation.PERTAINYM);
+ symbols.put(";c", WordNetRelation.DOMAIN_TOPIC);
+ symbols.put("-c", WordNetRelation.MEMBER_OF_DOMAIN_TOPIC);
+ symbols.put(";r", WordNetRelation.DOMAIN_REGION);
+ symbols.put("-r", WordNetRelation.MEMBER_OF_DOMAIN_REGION);
+ symbols.put(";u", WordNetRelation.DOMAIN_USAGE);
+ symbols.put("-u", WordNetRelation.MEMBER_OF_DOMAIN_USAGE);
+ return Map.copyOf(symbols);
+ }
+}
diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/InMemoryWordNetLexiconTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/InMemoryWordNetLexiconTest.java
new file mode 100644
index 0000000000..8a238832fb
--- /dev/null
+++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/InMemoryWordNetLexiconTest.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.wordnet;
+
+import java.util.List;
+import java.util.Map;
+
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.wordnet.Synset;
+import opennlp.tools.wordnet.WordNetPOS;
+import opennlp.tools.wordnet.WordNetRelation;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Exercises the constructor's referential-integrity validation directly, with deliberately
+ * inconsistent maps a reader would never produce: any future reader relies on these checks,
+ * so they are pinned independently of both existing readers.
+ */
+public class InMemoryWordNetLexiconTest {
+
+ private static Synset synset(String id, Map> relations) {
+ return new Synset(id, WordNetPOS.NOUN, List.of("lemma"), "a gloss", relations);
+ }
+
+ @Test
+ void testAcceptsConsistentMaps() {
+ final Synset a = synset("a", Map.of(WordNetRelation.HYPERNYM, List.of("b")));
+ final Synset b = synset("b", Map.of());
+ final InMemoryWordNetLexicon lexicon = new InMemoryWordNetLexicon(
+ Map.of("a", a, "b", b),
+ Map.of(InMemoryWordNetLexicon.LemmaKey.of("lemma", WordNetPOS.NOUN), List.of("a", "b")));
+ assertEquals(2, lexicon.size());
+ assertEquals(List.of(a, b), lexicon.lookup("lemma", WordNetPOS.NOUN));
+ }
+
+ @Test
+ void testRejectsKeyThatDoesNotMatchSynsetId() {
+ final Map table = Map.of("wrong-key", synset("real-id", Map.of()));
+ final IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+ () -> new InMemoryWordNetLexicon(table, Map.of()));
+ assertTrue(e.getMessage().contains("wrong-key"));
+ }
+
+ @Test
+ void testRejectsDanglingRelationTarget() {
+ final Map table =
+ Map.of("a", synset("a", Map.of(WordNetRelation.HYPERNYM, List.of("nope"))));
+ final IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+ () -> new InMemoryWordNetLexicon(table, Map.of()));
+ assertTrue(e.getMessage().contains("nope"));
+ assertTrue(e.getMessage().contains("HYPERNYM"));
+ }
+
+ @Test
+ void testRejectsSenseOrderEntryWithUnknownSynset() {
+ final Map table = Map.of("a", synset("a", Map.of()));
+ final Map> senseOrder =
+ Map.of(InMemoryWordNetLexicon.LemmaKey.of("lemma", WordNetPOS.NOUN), List.of("missing"));
+ final IllegalArgumentException e = assertThrows(IllegalArgumentException.class,
+ () -> new InMemoryWordNetLexicon(table, senseOrder));
+ assertTrue(e.getMessage().contains("missing"));
+ assertTrue(e.getMessage().contains("lemma"));
+ }
+
+ @Test
+ void testRejectsNullMaps() {
+ assertThrows(IllegalArgumentException.class, () -> new InMemoryWordNetLexicon(null, Map.of()));
+ assertThrows(IllegalArgumentException.class,
+ () -> new InMemoryWordNetLexicon(Map.of(), null));
+ }
+}
diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LemmaFoldingTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LemmaFoldingTest.java
new file mode 100644
index 0000000000..3d32eb4431
--- /dev/null
+++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LemmaFoldingTest.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.wordnet;
+
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.wordnet.WordNetPOS;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+/**
+ * Pins the shared fold and split behavior every user of {@link LemmaFolding} depends on:
+ * the exception lists, the sense index keys, and the WN-LMF members parsing all fold and
+ * split through this one implementation.
+ */
+public class LemmaFoldingTest {
+
+ @Test
+ void testFoldLowercasesWithRootLocaleAndTreatsUnderscoreAsSpace() {
+ assertEquals("mice", LemmaFolding.fold("MICE"));
+ assertEquals("domestic dog", LemmaFolding.fold("Domestic_Dog"));
+ assertEquals("attorney general", LemmaFolding.fold("attorney_general"));
+ assertEquals("dog", LemmaFolding.fold("dog"));
+ assertEquals("", LemmaFolding.fold(""));
+ }
+
+ @Test
+ void testSplitOnSpacesCollapsesRunsAndIgnoresEdges() {
+ assertEquals(List.of("a", "b", "c"), LemmaFolding.splitOnSpaces("a b c"));
+ assertEquals(List.of("a", "b"), LemmaFolding.splitOnSpaces("a b"));
+ assertEquals(List.of("a"), LemmaFolding.splitOnSpaces("a"));
+ assertEquals(List.of("a"), LemmaFolding.splitOnSpaces(" a "));
+ assertEquals(List.of(), LemmaFolding.splitOnSpaces(""));
+ assertEquals(List.of(), LemmaFolding.splitOnSpaces(" "));
+ }
+
+ @Test
+ void testLemmaKeyAndExceptionLookupAgreeOnTheFold() {
+ // The agreement that makes Morphy correct: a key built from a stored written form and a
+ // query folded at lookup time land on the same canonical shape.
+ assertEquals(InMemoryWordNetLexicon.LemmaKey.of("Domestic_Dog", WordNetPOS.NOUN),
+ InMemoryWordNetLexicon.LemmaKey.of(LemmaFolding.fold("DOMESTIC_DOG"), WordNetPOS.NOUN));
+ }
+
+ @Test
+ void testFoldRejectsNull() {
+ assertThrows(IllegalArgumentException.class, () -> LemmaFolding.fold(null));
+ }
+}
diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexiconConcurrencyTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexiconConcurrencyTest.java
new file mode 100644
index 0000000000..0292f7ffe2
--- /dev/null
+++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexiconConcurrencyTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.wordnet;
+
+import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.wordnet.LexicalKnowledgeBase;
+import opennlp.tools.wordnet.WordNetPOS;
+import opennlp.tools.wordnet.WordNetRelation;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Exercises the immutable-after-load contract: one loaded lexicon serves many threads issuing
+ * concurrent lookups, and every thread observes exactly the single-threaded results.
+ */
+public class LexiconConcurrencyTest {
+
+ private static final int THREADS = 8;
+ private static final int ITERATIONS = 500;
+
+ @Test
+ void testConcurrentLookupsSeeConsistentResults() throws InterruptedException {
+ final LexicalKnowledgeBase lexicon = WndbReaderTest.fixture();
+ final CountDownLatch start = new CountDownLatch(1);
+ final CountDownLatch done = new CountDownLatch(THREADS);
+ final Queue problems = new ConcurrentLinkedQueue<>();
+ for (int t = 0; t < THREADS; t++) {
+ final Thread thread = new Thread(() -> {
+ try {
+ start.await();
+ for (int i = 0; i < ITERATIONS; i++) {
+ verifyOnce(lexicon, problems);
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ problems.add("Interrupted: " + e);
+ } catch (RuntimeException e) {
+ problems.add("Unexpected exception: " + e);
+ } finally {
+ done.countDown();
+ }
+ });
+ thread.setDaemon(true);
+ thread.start();
+ }
+ start.countDown();
+ assertTrue(done.await(60, TimeUnit.SECONDS), "Worker threads must finish in time");
+ assertEquals(List.of(), List.copyOf(problems));
+ }
+
+ private static void verifyOnce(LexicalKnowledgeBase lexicon, Queue problems) {
+ if (!"wndb-00001075-n".equals(lexicon.lookup("dog", WordNetPOS.NOUN).get(0).id())) {
+ problems.add("Wrong dog lookup");
+ }
+ if (lexicon.lookup("run", WordNetPOS.NOUN).size() != 2) {
+ problems.add("Wrong run sense count");
+ }
+ if (!List.of("wndb-00001160-n")
+ .equals(lexicon.related("wndb-00001075-n", WordNetRelation.HYPERNYM))) {
+ problems.add("Wrong dog hypernym");
+ }
+ if (lexicon.contains("zebra", WordNetPOS.NOUN)) {
+ problems.add("Phantom zebra");
+ }
+ if (!lexicon.contains("walk", WordNetPOS.VERB)) {
+ problems.add("Missing walk verb");
+ }
+ }
+}
diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyExceptionsTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyExceptionsTest.java
new file mode 100644
index 0000000000..b8ed76181d
--- /dev/null
+++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyExceptionsTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.wordnet;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import opennlp.tools.util.InvalidFormatException;
+import opennlp.tools.wordnet.WordNetPOS;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class MorphyExceptionsTest {
+
+ static MorphyExceptions fixture() {
+ try {
+ return MorphyExceptions.load(WndbReaderTest.fixtureDirectory());
+ } catch (IOException e) {
+ throw new IllegalStateException("Unexpected IOException reading the fixture lists", e);
+ }
+ }
+
+ @Test
+ void testLookupPerPartOfSpeech() {
+ final MorphyExceptions exceptions = fixture();
+ assertEquals(List.of("mouse"), exceptions.lookup("mice", WordNetPOS.NOUN));
+ assertEquals(List.of("go"), exceptions.lookup("went", WordNetPOS.VERB));
+ assertEquals(List.of("good"), exceptions.lookup("better", WordNetPOS.ADJECTIVE));
+ assertEquals(List.of("well"), exceptions.lookup("best", WordNetPOS.ADVERB));
+ // Entries are part-of-speech scoped: went is only a verb exception.
+ assertTrue(exceptions.lookup("went", WordNetPOS.NOUN).isEmpty());
+ assertTrue(exceptions.lookup("dog", WordNetPOS.NOUN).isEmpty());
+ }
+
+ @Test
+ void testLookupFoldsCase() {
+ assertEquals(List.of("mouse"), fixture().lookup("Mice", WordNetPOS.NOUN));
+ assertEquals(List.of("mouse"), fixture().lookup("MICE", WordNetPOS.NOUN));
+ }
+
+ @Test
+ void testLookupRejectsNulls() {
+ final MorphyExceptions exceptions = fixture();
+ assertThrows(IllegalArgumentException.class,
+ () -> exceptions.lookup(null, WordNetPOS.NOUN));
+ assertThrows(IllegalArgumentException.class, () -> exceptions.lookup("mice", null));
+ }
+
+ @Test
+ void testLoadRejectsNullAndMissingDirectory(@TempDir Path tempDir) {
+ assertThrows(IllegalArgumentException.class, () -> MorphyExceptions.load(null));
+ assertThrows(IllegalArgumentException.class,
+ () -> MorphyExceptions.load(tempDir.resolve("absent")));
+ }
+
+ @Test
+ void testLoadRejectsMissingFile(@TempDir Path tempDir) throws IOException {
+ Files.writeString(tempDir.resolve("noun.exc"), "mice mouse\n");
+ Files.writeString(tempDir.resolve("verb.exc"), "went go\n");
+ Files.writeString(tempDir.resolve("adj.exc"), "better good\n");
+ final InvalidFormatException e = assertThrows(InvalidFormatException.class,
+ () -> MorphyExceptions.load(tempDir));
+ assertTrue(e.getMessage().contains("adv.exc"));
+ }
+
+ @Test
+ void testLoadRejectsMalformedLine(@TempDir Path tempDir) throws IOException {
+ Files.writeString(tempDir.resolve("noun.exc"), "mice mouse\nlonely\n");
+ Files.writeString(tempDir.resolve("verb.exc"), "went go\n");
+ Files.writeString(tempDir.resolve("adj.exc"), "better good\n");
+ Files.writeString(tempDir.resolve("adv.exc"), "best well\n");
+ final InvalidFormatException e = assertThrows(InvalidFormatException.class,
+ () -> MorphyExceptions.load(tempDir));
+ assertTrue(e.getMessage().contains("noun.exc"));
+ assertTrue(e.getMessage().contains("line 2"));
+ }
+
+ @Test
+ void testMultipleBaseFormsKeepFileOrder(@TempDir Path tempDir) throws IOException {
+ Files.writeString(tempDir.resolve("noun.exc"), "axes axis ax\n");
+ Files.writeString(tempDir.resolve("verb.exc"), "went go\n");
+ Files.writeString(tempDir.resolve("adj.exc"), "better good\n");
+ Files.writeString(tempDir.resolve("adv.exc"), "best well\n");
+ assertEquals(List.of("axis", "ax"),
+ MorphyExceptions.load(tempDir).lookup("axes", WordNetPOS.NOUN));
+ }
+}
diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java
new file mode 100644
index 0000000000..0dd3ca6ff0
--- /dev/null
+++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.wordnet;
+
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+
+import opennlp.tools.wordnet.WordNetPOS;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class MorphyLemmatizerTest {
+
+ private static MorphyLemmatizer morphy() {
+ return new MorphyLemmatizer(WndbReaderTest.fixture(), MorphyExceptionsTest.fixture());
+ }
+
+ private static String one(String token, String tag) {
+ return morphy().lemmatize(new String[] {token}, new String[] {tag})[0];
+ }
+
+ @ParameterizedTest
+ @CsvSource({
+ // Irregular forms resolve through the exception lists.
+ "mice, NN, mouse",
+ "Mice, NNS, mouse",
+ "men, NNS, man",
+ "ran, VBD, run",
+ "running, VBG, run",
+ "went, VBD, go",
+ "gone, VBN, go",
+ "best, RBS, well",
+ // Regular detachments, validated against the lexicon.
+ "dogs, NNS, dog",
+ "boxes, NNS, box",
+ "berries, NNS, berry",
+ "runs, NNS, run",
+ "runs, VBZ, run",
+ "walked, VBD, walk",
+ "walking, VBG, walk",
+ "walks, VBZ, walk",
+ "moved, VBD, move",
+ "taller, JJR, tall",
+ "tallest, JJS, tall",
+ "larger, JJR, large",
+ // A word that is already a lemma comes back as itself.
+ "dog, NN, dog",
+ "quickly, RB, quickly",
+ // WordNet letter tags are accepted alongside Penn tags.
+ "dogs, n, dog",
+ "walked, v, walk",
+ "taller, a, tall",
+ "best, r, well",
+ })
+ void testLemmatizesToken(String token, String tag, String lemma) {
+ assertEquals(lemma, one(token, tag));
+ }
+
+ @ParameterizedTest
+ @CsvSource({
+ // Rule candidates not in the lexicon are rejected, not returned.
+ "dogged, VBD",
+ "boxes, VBZ",
+ "glarbs, NNS",
+ // A known word under the wrong part of speech is unknown.
+ "walk, NN",
+ // Tags outside the mapping yield the unknown marker.
+ "dog, DT",
+ "dog, XYZ",
+ "dogs, ''",
+ // Multi-letter closed-class tags that merely begin with a WordNet letter code are not
+ // adjective lookups: AUX was must be unknown, and AUX taller must not detach to tall.
+ "was, AUX",
+ "taller, AUX",
+ })
+ void testUnknownYieldsMarker(String token, String tag) {
+ assertEquals("O", one(token, tag));
+ }
+
+ @Test
+ void testExceptionHitsAreReturnedWithoutLexiconValidation() {
+ // oxen maps to ox, which the miniature lexicon does not contain; the exception list is
+ // authoritative for irregulars, so the lemma is returned anyway.
+ assertEquals("ox", one("oxen", "NNS"));
+ // better maps to good, also absent from the miniature lexicon.
+ assertEquals("good", one("better", "JJR"));
+ }
+
+ @Test
+ void testArrayFormKeepsPositions() {
+ final String[] lemmas = morphy().lemmatize(
+ new String[] {"The", "mice", "ran", "quickly"},
+ new String[] {"DT", "NNS", "VBD", "RB"});
+ assertArrayEquals(new String[] {"O", "mouse", "run", "quickly"}, lemmas);
+ }
+
+ @Test
+ void testListFormReturnsAllCandidates() {
+ final List> lemmas = morphy().lemmatize(
+ List.of("glarbs", "berries"), List.of("NNS", "NNS"));
+ assertEquals(List.of("O"), lemmas.get(0));
+ assertEquals(List.of("berry"), lemmas.get(1));
+ }
+
+ @Test
+ void testWorksIdenticallyOverTheWnLmfLexicon() {
+ final MorphyLemmatizer lmfMorphy =
+ new MorphyLemmatizer(WnLmfReaderTest.fixture(), MorphyExceptionsTest.fixture());
+ assertArrayEquals(new String[] {"mouse", "box", "walk", "large", "O"},
+ lmfMorphy.lemmatize(
+ new String[] {"mice", "boxes", "walking", "larger", "dogged"},
+ new String[] {"NNS", "NNS", "VBG", "JJR", "VBD"}));
+ }
+
+ @Test
+ void testPosFromTagMapping() {
+ assertEquals(WordNetPOS.NOUN, MorphyLemmatizer.posFromTag("NNP"));
+ assertEquals(WordNetPOS.VERB, MorphyLemmatizer.posFromTag("VBZ"));
+ assertEquals(WordNetPOS.ADJECTIVE, MorphyLemmatizer.posFromTag("JJ"));
+ assertEquals(WordNetPOS.ADVERB, MorphyLemmatizer.posFromTag("RBR"));
+ assertEquals(WordNetPOS.ADJECTIVE, MorphyLemmatizer.posFromTag("a"));
+ assertEquals(WordNetPOS.ADJECTIVE, MorphyLemmatizer.posFromTag("s"));
+ assertEquals(WordNetPOS.ADJECTIVE, MorphyLemmatizer.posFromTag("ADJ"));
+ assertEquals(WordNetPOS.ADVERB, MorphyLemmatizer.posFromTag("ADV"));
+ assertEquals(WordNetPOS.ADVERB, MorphyLemmatizer.posFromTag("r"));
+ assertNull(MorphyLemmatizer.posFromTag("DT"));
+ assertNull(MorphyLemmatizer.posFromTag(""));
+ // The letter codes a and s match only as one-letter tags: multi-letter tags beginning
+ // with those letters are closed-class or symbol tags, never adjectives.
+ assertNull(MorphyLemmatizer.posFromTag("AUX"));
+ assertNull(MorphyLemmatizer.posFromTag("ADP"));
+ assertNull(MorphyLemmatizer.posFromTag("SCONJ"));
+ assertNull(MorphyLemmatizer.posFromTag("SYM"));
+ assertThrows(IllegalArgumentException.class, () -> MorphyLemmatizer.posFromTag(null));
+ }
+
+ @Test
+ void testConstructorFailsLoudWithoutInputs() {
+ final MorphyExceptions exceptions = MorphyExceptionsTest.fixture();
+ assertThrows(IllegalArgumentException.class,
+ () -> new MorphyLemmatizer(null, exceptions));
+ assertThrows(IllegalArgumentException.class,
+ () -> new MorphyLemmatizer(WndbReaderTest.fixture(), null));
+ }
+
+ @Test
+ void testRejectsNullOrMismatchedSequences() {
+ final MorphyLemmatizer morphy = morphy();
+ assertThrows(IllegalArgumentException.class,
+ () -> morphy.lemmatize((String[]) null, new String[0]));
+ assertThrows(IllegalArgumentException.class,
+ () -> morphy.lemmatize(new String[0], (String[]) null));
+ assertThrows(IllegalArgumentException.class,
+ () -> morphy.lemmatize(new String[] {"a", "b"}, new String[] {"NN"}));
+ assertThrows(IllegalArgumentException.class,
+ () -> morphy.lemmatize(List.of("a"), List.of("NN", "NN")));
+ assertThrows(IllegalArgumentException.class,
+ () -> morphy.lemmatize(new String[] {null}, new String[] {"NN"}));
+ assertThrows(IllegalArgumentException.class,
+ () -> morphy.lemmatize(new String[] {"dog"}, new String[] {null}));
+ }
+}
diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/ReaderEquivalenceTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/ReaderEquivalenceTest.java
new file mode 100644
index 0000000000..b9f3523cbc
--- /dev/null
+++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/ReaderEquivalenceTest.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.wordnet;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.junit.jupiter.api.Test;
+
+import opennlp.tools.wordnet.LexicalKnowledgeBase;
+import opennlp.tools.wordnet.Synset;
+import opennlp.tools.wordnet.WordNetPOS;
+import opennlp.tools.wordnet.WordNetRelation;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+/**
+ * Asserts that the WN-LMF fixture and the WNDB fixture, which encode the same miniature
+ * wordnet, load into equivalent lexicon views. Synset ids are reader-minted and intentionally
+ * differ, so the comparison is structural, joining synsets on their glosses (unique within the
+ * fixtures) and comparing everything else through that join.
+ */
+public class ReaderEquivalenceTest {
+
+ @Test
+ void testBothReadersProduceEquivalentViews() {
+ final InMemoryWordNetLexicon lmf = (InMemoryWordNetLexicon) WnLmfReaderTest.fixture();
+ final InMemoryWordNetLexicon wndb = (InMemoryWordNetLexicon) WndbReaderTest.fixture();
+ assertEquals(lmf.size(), wndb.size(), "Both fixtures encode the same synsets");
+
+ final Map wndbByGloss = byGloss(wndb);
+ assertEquals(byGloss(lmf).keySet(), wndbByGloss.keySet(), "Same glosses on both sides");
+
+ for (final Synset expected : lmf.synsets()) {
+ final Synset actual = wndbByGloss.get(expected.gloss());
+ assertNotNull(actual, "WNDB view has a synset for gloss: " + expected.gloss());
+ assertEquals(expected.pos(), actual.pos(), "Part of speech for: " + expected.gloss());
+ assertEquals(expected.lemmas(), actual.lemmas(), "Lemmas for: " + expected.gloss());
+ assertEquals(relationsByGloss(expected, lmf), relationsByGloss(actual, wndb),
+ "Relations for: " + expected.gloss());
+ }
+ }
+
+ @Test
+ void testLookupAgreesForEveryLemmaAndPos() {
+ final LexicalKnowledgeBase lmf = WnLmfReaderTest.fixture();
+ final InMemoryWordNetLexicon wndb = (InMemoryWordNetLexicon) WndbReaderTest.fixture();
+ final Set checked = new HashSet<>();
+ for (final Synset synset : wndb.synsets()) {
+ for (final String lemma : synset.lemmas()) {
+ if (!checked.add(lemma + "/" + synset.pos())) {
+ continue;
+ }
+ assertEquals(
+ glosses(lmf.lookup(lemma, synset.pos())),
+ glosses(wndb.lookup(lemma, synset.pos())),
+ "Sense sequence for " + lemma + " as " + synset.pos());
+ }
+ }
+ for (final WordNetPOS pos : WordNetPOS.values()) {
+ assertEquals(lmf.contains("dog", pos), wndb.contains("dog", pos));
+ }
+ }
+
+ @Test
+ void testSenseOrderAgreesForMultiSenseLemma() {
+ final LexicalKnowledgeBase lmf = WnLmfReaderTest.fixture();
+ final LexicalKnowledgeBase wndb = WndbReaderTest.fixture();
+ final List lmfOrder = glosses(lmf.lookup("run", WordNetPOS.NOUN));
+ final List wndbOrder = glosses(wndb.lookup("run", WordNetPOS.NOUN));
+ assertEquals(2, lmfOrder.size());
+ assertEquals(lmfOrder, wndbOrder);
+ }
+
+ private static Map byGloss(InMemoryWordNetLexicon lexicon) {
+ final Map byGloss = new HashMap<>();
+ for (final Synset synset : lexicon.synsets()) {
+ final Synset previous = byGloss.put(synset.gloss(), synset);
+ assertEquals(null, previous, "Fixture glosses must be unique, duplicated: "
+ + synset.gloss());
+ }
+ return byGloss;
+ }
+
+ // A synset's relations with targets replaced by their glosses, id-scheme independent.
+ private static Map> relationsByGloss(Synset synset,
+ LexicalKnowledgeBase lexicon) {
+ final Map> result = new HashMap<>();
+ for (final Map.Entry> relation :
+ synset.relations().entrySet()) {
+ final Set targetGlosses = new HashSet<>();
+ for (final String targetId : relation.getValue()) {
+ targetGlosses.add(lexicon.synset(targetId).orElseThrow().gloss());
+ }
+ result.put(relation.getKey(), targetGlosses);
+ }
+ return result;
+ }
+
+ private static List glosses(List synsets) {
+ return synsets.stream().map(Synset::gloss).toList();
+ }
+}
diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java
new file mode 100644
index 0000000000..5d48d17969
--- /dev/null
+++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.wordnet;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import opennlp.tools.util.InvalidFormatException;
+import opennlp.tools.wordnet.LexicalKnowledgeBase;
+import opennlp.tools.wordnet.Synset;
+import opennlp.tools.wordnet.WordNetPOS;
+import opennlp.tools.wordnet.WordNetRelation;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class WnLmfReaderTest {
+
+ static LexicalKnowledgeBase fixture() {
+ try (InputStream in = WnLmfReaderTest.class.getResourceAsStream("mini-wn-lmf.xml")) {
+ assertNotNull(in, "Fixture mini-wn-lmf.xml must be on the test classpath");
+ return WnLmfReader.read(in, "mini-wn-lmf.xml");
+ } catch (IOException e) {
+ throw new IllegalStateException("Unexpected IOException from a classpath stream", e);
+ }
+ }
+
+ private static LexicalKnowledgeBase parse(String document) throws IOException {
+ return WnLmfReader.read(
+ new ByteArrayInputStream(document.getBytes(StandardCharsets.UTF_8)), "inline.xml");
+ }
+
+ private static String wrap(String body) {
+ return "\n\n"
+ + "\n"
+ + body + "\n\n\n";
+ }
+
+ @Test
+ void testLookupReturnsSynsetWithAllComponents() {
+ final List senses = fixture().lookup("dog", WordNetPOS.NOUN);
+ assertEquals(1, senses.size());
+ final Synset dog = senses.get(0);
+ assertEquals("mini-n1", dog.id());
+ assertEquals(WordNetPOS.NOUN, dog.pos());
+ assertEquals(List.of("dog", "domestic dog"), dog.lemmas());
+ assertEquals("a domesticated canid", dog.gloss());
+ assertEquals(List.of("mini-n2"), dog.related(WordNetRelation.HYPERNYM));
+ }
+
+ @Test
+ void testLookupFoldsCaseAndUnderscore() {
+ final LexicalKnowledgeBase lexicon = fixture();
+ assertEquals("mini-n1", lexicon.lookup("Domestic_Dog", WordNetPOS.NOUN).get(0).id());
+ assertEquals("mini-n1", lexicon.lookup("DOG", WordNetPOS.NOUN).get(0).id());
+ }
+
+ @Test
+ void testLookupKeepsSenseOrder() {
+ final List runSenses = fixture().lookup("run", WordNetPOS.NOUN);
+ assertEquals(List.of("mini-n5", "mini-n9"),
+ runSenses.stream().map(Synset::id).toList());
+ }
+
+ @Test
+ void testLookupIsPosScoped() {
+ final LexicalKnowledgeBase lexicon = fixture();
+ assertEquals(1, lexicon.lookup("run", WordNetPOS.VERB).size());
+ assertTrue(lexicon.lookup("dog", WordNetPOS.VERB).isEmpty());
+ assertFalse(lexicon.contains("walk", WordNetPOS.NOUN));
+ assertTrue(lexicon.contains("walk", WordNetPOS.VERB));
+ }
+
+ @Test
+ void testRelationNavigation() {
+ final LexicalKnowledgeBase lexicon = fixture();
+ assertEquals(List.of("mini-n1"), lexicon.related("mini-n2", WordNetRelation.HYPONYM));
+ assertEquals(List.of("mini-v1", "mini-v2"),
+ lexicon.related("mini-v4", WordNetRelation.HYPONYM));
+ assertEquals(List.of("mini-v4"), lexicon.related("mini-v1", WordNetRelation.HYPERNYM));
+ }
+
+ @Test
+ void testRelationTargetSharesCanonicalIdInstance() {
+ final LexicalKnowledgeBase lexicon = fixture();
+ final String target = lexicon.synset("mini-n1").orElseThrow()
+ .related(WordNetRelation.HYPERNYM).get(0);
+ // Not just equal: the identical instance from the synset table, so a loaded lexicon keeps
+ // one copy of each id no matter how many relations point at it.
+ assertSame(lexicon.synset("mini-n2").orElseThrow().id(), target);
+ }
+
+ @Test
+ void testSenseRelationsAreLiftedToSynsetLevel() {
+ final LexicalKnowledgeBase lexicon = fixture();
+ assertEquals(List.of("mini-a2"), lexicon.related("mini-a1", WordNetRelation.ANTONYM));
+ assertEquals(List.of("mini-a1"), lexicon.related("mini-a2", WordNetRelation.ANTONYM));
+ assertEquals(List.of("mini-v1"),
+ lexicon.related("mini-n5", WordNetRelation.DERIVATIONALLY_RELATED));
+ assertEquals(List.of("mini-n5"),
+ lexicon.related("mini-v1", WordNetRelation.DERIVATIONALLY_RELATED));
+ }
+
+ @Test
+ void testSatelliteNormalizesToAdjective() {
+ final List senses = fixture().lookup("large", WordNetPOS.ADJECTIVE);
+ assertEquals(1, senses.size());
+ assertEquals(WordNetPOS.ADJECTIVE, senses.get(0).pos());
+ assertEquals(List.of("mini-a4"), fixture().related("mini-a3", WordNetRelation.SIMILAR_TO));
+ assertEquals(List.of("mini-a3"), fixture().related("mini-a4", WordNetRelation.SIMILAR_TO));
+ }
+
+ @Test
+ void testSimilarOnVerbSynsetMapsToVerbGroup() throws IOException {
+ // Documents derived from Princeton data express verb groups as similar on verb synsets;
+ // the fixture only carries similar on adjectives, so this pins the verb branch directly.
+ final LexicalKnowledgeBase lexicon = parse(wrap(
+ ""
+ + ""
+ + ""
+ + ""
+ + ""
+ + "produce musical tones"
+ + ""
+ + ""
+ + "sing monotonously"));
+ assertEquals(List.of("t-v2"), lexicon.related("t-v1", WordNetRelation.VERB_GROUP));
+ assertTrue(lexicon.related("t-v1", WordNetRelation.SIMILAR_TO).isEmpty());
+ }
+
+ @Test
+ void testUnknownLemmaOrSynsetIsEmpty() {
+ final LexicalKnowledgeBase lexicon = fixture();
+ assertTrue(lexicon.lookup("zebra", WordNetPOS.NOUN).isEmpty());
+ assertTrue(lexicon.synset("mini-n99").isEmpty());
+ }
+
+ @Test
+ void testReadPath(@TempDir Path tempDir) throws IOException {
+ final Path file = tempDir.resolve("tiny.xml");
+ Files.writeString(file, wrap(
+ ""
+ + ""
+ + "a feline"));
+ final LexicalKnowledgeBase lexicon = WnLmfReader.read(file);
+ assertEquals("a feline", lexicon.lookup("cat", WordNetPOS.NOUN).get(0).gloss());
+ }
+
+ @Test
+ void testReadPathRejectsNullAndMissing(@TempDir Path tempDir) {
+ assertThrows(IllegalArgumentException.class, () -> WnLmfReader.read((Path) null));
+ assertThrows(IllegalArgumentException.class,
+ () -> WnLmfReader.read(tempDir.resolve("absent.xml")));
+ }
+
+ @Test
+ void testReadStreamRejectsNulls() {
+ assertThrows(IllegalArgumentException.class, () -> WnLmfReader.read(null, "x"));
+ assertThrows(IllegalArgumentException.class,
+ () -> WnLmfReader.read(new ByteArrayInputStream(new byte[0]), null));
+ }
+
+ @Test
+ void testStreamReadFailurePropagatesAsIOException() {
+ final InputStream failing = new InputStream() {
+ @Override
+ public int read() throws IOException {
+ throw new IOException("Simulated stream failure");
+ }
+ };
+ final IOException e =
+ assertThrows(IOException.class, () -> WnLmfReader.read(failing, "failing.xml"));
+ // The I/O failure must surface as itself, not be misreported as a malformed document.
+ assertFalse(e instanceof InvalidFormatException);
+ }
+
+ @Test
+ void testSkipsDoctypeDeclaration() throws IOException {
+ // Real Open English WordNet releases ship exactly this shape: a DOCTYPE naming the schema
+ // DTD by an unreachable SYSTEM identifier (example.invalid is the RFC 2606 reserved domain
+ // that must never resolve). The reader must parse past it without attempting to fetch it.
+ final String document = "\n"
+ + "\n"
+ + ""
+ + ""
+ + ""
+ + "a feline"
+ + "";
+ final LexicalKnowledgeBase lexicon = parse(document);
+ assertEquals("a feline", lexicon.lookup("cat", WordNetPOS.NOUN).get(0).gloss());
+ }
+
+ @Test
+ void testInternalSubsetEntityIsNeverExpanded(@TempDir Path tempDir) throws IOException {
+ // A DOCTYPE-declared internal-subset entity is the classic XXE payload: if the parser ever
+ // honored it, the entity reference below would be replaced by the target file's content.
+ // With SUPPORT_DTD disabled the declaration itself is never registered, so the reference is
+ // undefined and parsing must fail loud rather than silently expand it.
+ final Path secret = tempDir.resolve("secret.txt");
+ Files.writeString(secret, "xxe-marker-should-never-appear");
+ final String document = "\n"
+ + "]>\n"
+ + ""
+ + ""
+ + ""
+ + "a feline"
+ + "";
+ final InvalidFormatException e =
+ assertThrows(InvalidFormatException.class, () -> parse(document));
+ assertFalse(e.getMessage().contains("xxe-marker-should-never-appear"));
+ }
+
+ @Test
+ void testRejectsTruncatedDocument() {
+ final InvalidFormatException e = assertThrows(InvalidFormatException.class,
+ () -> parse("\n parse(
+ wrap(""
+ + "")));
+ assertTrue(e.getMessage().contains("synset"));
+ }
+
+ @Test
+ void testRejectsSenseToUndeclaredSynset() {
+ final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse(
+ wrap(""
+ + "")));
+ assertTrue(e.getMessage().contains("t-9"));
+ }
+
+ @Test
+ void testRejectsRelationToUndeclaredSynset() {
+ final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse(
+ wrap(""
+ + ""
+ + "a feline"
+ + "")));
+ assertTrue(e.getMessage().contains("t-9"));
+ }
+
+ @Test
+ void testRejectsUnknownRelationType() {
+ final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse(
+ wrap(""
+ + ""
+ + "a feline"
+ + "")));
+ assertTrue(e.getMessage().contains("quasi_synonym"));
+ }
+
+ @Test
+ void testSkipsOtherRelationTypeOnSenseRelation() throws IOException {
+ final LexicalKnowledgeBase lexicon = parse(
+ wrap(""
+ + ""
+ + ""
+ + "a feline"));
+ assertTrue(lexicon.synset("t-1").orElseThrow().relations().isEmpty());
+ }
+
+ @Test
+ void testSkipsOtherRelationTypeOnSynsetRelation() throws IOException {
+ // The DTD permits relType="other" on SynsetRelation too, and several OMW-family wordnets
+ // emit it; it is skipped exactly like the SenseRelation case, not rejected.
+ final LexicalKnowledgeBase lexicon = parse(
+ wrap(""
+ + ""
+ + "a feline"
+ + ""));
+ assertTrue(lexicon.synset("t-1").orElseThrow().relations().isEmpty());
+ }
+
+ @Test
+ void testRejectsUnknownPartOfSpeech() {
+ final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse(
+ wrap(""
+ + ""
+ + "a feline")));
+ assertTrue(e.getMessage().contains("x"));
+ }
+
+ @Test
+ void testRejectsSynsetWithoutMembers() {
+ final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse(
+ wrap("orphan")));
+ assertTrue(e.getMessage().contains("t-1"));
+ }
+
+ @Test
+ void testRejectsDuplicateSynsetId() {
+ final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse(
+ wrap(""
+ + ""
+ + "a feline"
+ + "a repeat")));
+ assertTrue(e.getMessage().contains("Duplicate synset id t-1"));
+ }
+
+ @Test
+ void testRejectsDuplicateLexicalEntryId() {
+ final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse(
+ wrap(""
+ + ""
+ + ""
+ + ""
+ + "a feline")));
+ assertTrue(e.getMessage().contains("Duplicate lexical entry id t-cat-n"));
+ }
+
+ @Test
+ void testRejectsDuplicateSenseId() {
+ final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse(
+ wrap(""
+ + ""
+ + ""
+ + "a feline"
+ + "a second")));
+ assertTrue(e.getMessage().contains("Duplicate sense id t-cat-n-1"));
+ }
+
+ @Test
+ void testRejectsSynsetMemberPosMismatch() {
+ final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse(
+ wrap(""
+ + ""
+ + "a feline")));
+ assertTrue(e.getMessage().contains("t-cat-n"));
+ assertTrue(e.getMessage().contains("VERB"));
+ assertTrue(e.getMessage().contains("NOUN"));
+ }
+
+ @Test
+ void testRejectsSenseRelationToUndeclaredSense() {
+ final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse(
+ wrap(""
+ + ""
+ + ""
+ + "a feline")));
+ assertTrue(e.getMessage().contains("t-ghost-1"));
+ }
+
+ @Test
+ void testRejectsLemmaOutsideLexicalEntry() {
+ final InvalidFormatException e = assertThrows(InvalidFormatException.class,
+ () -> parse(wrap("")));
+ assertTrue(e.getMessage().contains("Lemma outside a LexicalEntry"));
+ }
+
+ @Test
+ void testRejectsSenseBeforeLemma() {
+ final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse(
+ wrap(""
+ + ""
+ + "a feline")));
+ assertTrue(e.getMessage().contains("Sense before its entry's Lemma"));
+ }
+
+ @Test
+ void testRejectsSenseRelationOutsideSense() {
+ final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse(
+ wrap(""
+ + ""
+ + ""
+ + "a feline")));
+ assertTrue(e.getMessage().contains("SenseRelation outside a Sense"));
+ }
+
+ @Test
+ void testRejectsSynsetRelationOutsideSynset() {
+ final InvalidFormatException e = assertThrows(InvalidFormatException.class,
+ () -> parse(wrap("")));
+ assertTrue(e.getMessage().contains("SynsetRelation outside a Synset"));
+ }
+}
diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WndbReaderTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WndbReaderTest.java
new file mode 100644
index 0000000000..5931957e3b
--- /dev/null
+++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WndbReaderTest.java
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.wordnet;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Locale;
+import java.util.function.UnaryOperator;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import opennlp.tools.util.InvalidFormatException;
+import opennlp.tools.wordnet.LexicalKnowledgeBase;
+import opennlp.tools.wordnet.Synset;
+import opennlp.tools.wordnet.WordNetPOS;
+import opennlp.tools.wordnet.WordNetRelation;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class WndbReaderTest {
+
+ private static final String DOG_ID = "wndb-00001075-n";
+ private static final String CANID_ID = "wndb-00001160-n";
+
+ static Path fixtureDirectory() {
+ final URL url = WndbReaderTest.class.getResource("mini-wndb");
+ assertNotNull(url, "Fixture directory mini-wndb must be on the test classpath");
+ try {
+ return Path.of(url.toURI());
+ } catch (URISyntaxException e) {
+ throw new IllegalStateException("Unexpected fixture URI: " + url, e);
+ }
+ }
+
+ static LexicalKnowledgeBase fixture() {
+ try {
+ return WndbReader.read(fixtureDirectory());
+ } catch (IOException e) {
+ throw new IllegalStateException("Unexpected IOException reading the WNDB fixture", e);
+ }
+ }
+
+ @Test
+ void testLookupReturnsSynsetWithAllComponents() {
+ final List senses = fixture().lookup("dog", WordNetPOS.NOUN);
+ assertEquals(1, senses.size());
+ final Synset dog = senses.get(0);
+ assertEquals(DOG_ID, dog.id());
+ assertEquals(WordNetPOS.NOUN, dog.pos());
+ assertEquals(List.of("dog", "domestic dog"), dog.lemmas());
+ assertEquals("a domesticated canid", dog.gloss());
+ assertEquals(List.of(CANID_ID), dog.related(WordNetRelation.HYPERNYM));
+ }
+
+ @Test
+ void testLookupFoldsCaseAndUnderscore() {
+ final LexicalKnowledgeBase lexicon = fixture();
+ assertEquals(DOG_ID, lexicon.lookup("Domestic_Dog", WordNetPOS.NOUN).get(0).id());
+ assertEquals(DOG_ID, lexicon.lookup("DOG", WordNetPOS.NOUN).get(0).id());
+ }
+
+ @Test
+ void testLookupKeepsIndexSenseOrder() {
+ assertEquals(List.of("wndb-00001427-n", "wndb-00001669-n"),
+ fixture().lookup("run", WordNetPOS.NOUN).stream().map(Synset::id).toList());
+ }
+
+ @Test
+ void testRelationNavigation() {
+ final LexicalKnowledgeBase lexicon = fixture();
+ assertEquals(List.of(DOG_ID), lexicon.related(CANID_ID, WordNetRelation.HYPONYM));
+ assertEquals(List.of("wndb-00001075-v", "wndb-00001171-v"),
+ lexicon.related("wndb-00001324-v", WordNetRelation.HYPONYM));
+ assertEquals(List.of("wndb-00001075-v"),
+ lexicon.related("wndb-00001427-n", WordNetRelation.DERIVATIONALLY_RELATED));
+ }
+
+ @Test
+ void testRelationTargetSharesCanonicalIdInstance() {
+ final LexicalKnowledgeBase lexicon = fixture();
+ final String target = lexicon.synset(DOG_ID).orElseThrow()
+ .related(WordNetRelation.HYPERNYM).get(0);
+ // Not just equal: the identical instance from the synset table, so a loaded lexicon keeps
+ // one copy of each id no matter how many pointers reference it.
+ assertSame(lexicon.synset(CANID_ID).orElseThrow().id(), target);
+ }
+
+ @Test
+ void testLexicalPointersSurfaceAtSynsetLevel() {
+ final LexicalKnowledgeBase lexicon = fixture();
+ assertEquals(List.of("wndb-00001141-a"),
+ lexicon.related("wndb-00001075-a", WordNetRelation.ANTONYM));
+ assertEquals(List.of("wndb-00001075-a"),
+ lexicon.related("wndb-00001141-a", WordNetRelation.ANTONYM));
+ }
+
+ @Test
+ void testSatelliteNormalizesToAdjectiveAndMarkerIsStripped() {
+ final LexicalKnowledgeBase lexicon = fixture();
+ final Synset large = lexicon.lookup("large", WordNetPOS.ADJECTIVE).get(0);
+ assertEquals(WordNetPOS.ADJECTIVE, large.pos());
+ assertEquals(List.of("wndb-00001211-a"), large.related(WordNetRelation.SIMILAR_TO));
+ // short is stored as short(p); the syntactic marker is not part of the lemma.
+ assertEquals(List.of("short"),
+ lexicon.lookup("short", WordNetPOS.ADJECTIVE).get(0).lemmas());
+ }
+
+ @Test
+ void testVerbGroupPointerMapsToVerbGroup(@TempDir Path tempDir) throws IOException {
+ // The fixture has no $ pointer, so the VERB_GROUP mapping is pinned against a minimal
+ // constructed database whose byte offsets are computed, not hard-coded: every offset field
+ // is exactly eight digits, so the second line's position is independent of the digit values.
+ writeEmptyDb(tempDir, "noun", "adj", "adv");
+ final String template =
+ "00000000 29 v 01 sing 0 001 $ XXXXXXXX v 0000 00 | produce musical tones";
+ final String off2 = String.format(Locale.ROOT, "%08d", template.length() + 1);
+ final String line1 = template.replace("XXXXXXXX", off2);
+ final String line2 = off2 + " 29 v 01 chant 0 001 $ 00000000 v 0000 00 | sing monotonously";
+ Files.writeString(tempDir.resolve("data.verb"), line1 + "\n" + line2 + "\n",
+ StandardCharsets.ISO_8859_1);
+ Files.writeString(tempDir.resolve("index.verb"),
+ "chant v 1 1 $ 1 0 " + off2 + "\nsing v 1 1 $ 1 0 00000000\n",
+ StandardCharsets.ISO_8859_1);
+ final LexicalKnowledgeBase lexicon = WndbReader.read(tempDir);
+ assertEquals(List.of("wndb-" + off2 + "-v"),
+ lexicon.related("wndb-00000000-v", WordNetRelation.VERB_GROUP));
+ assertEquals(List.of("wndb-00000000-v"),
+ lexicon.related("wndb-" + off2 + "-v", WordNetRelation.VERB_GROUP));
+ }
+
+ @Test
+ void testUnknownLemmaOrSynsetIsEmpty() {
+ final LexicalKnowledgeBase lexicon = fixture();
+ assertTrue(lexicon.lookup("zebra", WordNetPOS.NOUN).isEmpty());
+ assertTrue(lexicon.synset("wndb-99999999-n").isEmpty());
+ }
+
+ @Test
+ void testRejectsNullAndMissingDirectory(@TempDir Path tempDir) {
+ assertThrows(IllegalArgumentException.class, () -> WndbReader.read(null));
+ assertThrows(IllegalArgumentException.class,
+ () -> WndbReader.read(tempDir.resolve("absent")));
+ }
+
+ @Test
+ void testRejectsMissingDatabaseFile(@TempDir Path tempDir) throws IOException {
+ copyFixture(tempDir);
+ Files.delete(tempDir.resolve("data.verb"));
+ final InvalidFormatException e =
+ assertThrows(InvalidFormatException.class, () -> WndbReader.read(tempDir));
+ assertTrue(e.getMessage().contains("data.verb"));
+ }
+
+ @Test
+ void testRejectsIndexOffsetWithoutDataLine(@TempDir Path tempDir) throws IOException {
+ copyFixture(tempDir);
+ mutate(tempDir, "index.noun", line -> line.startsWith("berry ")
+ ? line.replace("00001564", "00001565") : line);
+ final InvalidFormatException e =
+ assertThrows(InvalidFormatException.class, () -> WndbReader.read(tempDir));
+ assertTrue(e.getMessage().contains("berry"));
+ assertTrue(e.getMessage().contains("00001565"));
+ }
+
+ @Test
+ void testRejectsDataOffsetFieldMismatch(@TempDir Path tempDir) throws IOException {
+ copyFixture(tempDir);
+ mutate(tempDir, "data.noun",
+ line -> line.replace("00001503 03 n 01 box", "00001504 03 n 01 box"));
+ final InvalidFormatException e =
+ assertThrows(InvalidFormatException.class, () -> WndbReader.read(tempDir));
+ assertTrue(e.getMessage().contains("disagrees"));
+ }
+
+ @Test
+ void testRejectsTruncatedDataLine(@TempDir Path tempDir) throws IOException {
+ copyFixture(tempDir);
+ mutate(tempDir, "data.noun", line -> line.startsWith("00001564")
+ ? line.substring(0, line.indexOf(" 000 |")) : line);
+ final InvalidFormatException e =
+ assertThrows(InvalidFormatException.class, () -> WndbReader.read(tempDir));
+ assertTrue(e.getMessage().contains("data.noun"));
+ assertTrue(e.getMessage().contains("Truncated"));
+ }
+
+ @Test
+ void testRejectsUndeclaredPointerSymbol(@TempDir Path tempDir) throws IOException {
+ copyFixture(tempDir);
+ mutate(tempDir, "data.noun", line -> line.replace("001 @ 00001160 n 0000",
+ "001 ? 00001160 n 0000"));
+ final InvalidFormatException e =
+ assertThrows(InvalidFormatException.class, () -> WndbReader.read(tempDir));
+ assertTrue(e.getMessage().contains("Undeclared pointer symbol: ?"));
+ }
+
+ @Test
+ void testRejectsPointerToNonexistentSynset(@TempDir Path tempDir) throws IOException {
+ copyFixture(tempDir);
+ mutate(tempDir, "data.noun", line -> line.replace("001 @ 00001160 n 0000",
+ "001 @ 00009999 n 0000"));
+ final InvalidFormatException e =
+ assertThrows(InvalidFormatException.class, () -> WndbReader.read(tempDir));
+ assertTrue(e.getMessage().contains("wndb-00009999-n"));
+ }
+
+ @Test
+ void testDanglingPointerErrorNamesPointerLine(@TempDir Path tempDir) throws IOException {
+ // A constructed database with no preamble, so the dangling pointer sits on a known line
+ // and the error message can be pinned to name it.
+ writeEmptyDb(tempDir, "noun", "adj", "adv");
+ Files.writeString(tempDir.resolve("data.verb"),
+ "00000000 29 v 01 sing 0 001 $ 00009999 v 0000 00 | produce musical tones\n",
+ StandardCharsets.ISO_8859_1);
+ Files.writeString(tempDir.resolve("index.verb"), "sing v 1 1 $ 1 0 00000000\n",
+ StandardCharsets.ISO_8859_1);
+ final InvalidFormatException e =
+ assertThrows(InvalidFormatException.class, () -> WndbReader.read(tempDir));
+ assertTrue(e.getMessage().contains("wndb-00009999-v"));
+ assertTrue(e.getMessage().contains("line 1"));
+ }
+
+ private static void writeEmptyDb(Path directory, String... suffixes) throws IOException {
+ for (final String suffix : suffixes) {
+ Files.writeString(directory.resolve("data." + suffix), "");
+ Files.writeString(directory.resolve("index." + suffix), "");
+ }
+ }
+
+ private static void copyFixture(Path target) throws IOException {
+ try (var files = Files.list(fixtureDirectory())) {
+ for (final Path file : files.toList()) {
+ Files.copy(file, target.resolve(file.getFileName().toString()));
+ }
+ }
+ }
+
+ // Applies a line transformation to one fixture file. The mutations only ever keep or shrink
+ // line lengths of the affected line's own fields, so surrounding offsets stay valid.
+ private static void mutate(Path directory, String fileName, UnaryOperator edit)
+ throws IOException {
+ final Path file = directory.resolve(fileName);
+ final List lines = Files.readAllLines(file, StandardCharsets.ISO_8859_1);
+ final StringBuilder out = new StringBuilder();
+ for (final String line : lines) {
+ out.append(edit.apply(line)).append('\n');
+ }
+ Files.writeString(file, out.toString(), StandardCharsets.ISO_8859_1);
+ }
+}
diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wn-lmf.xml b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wn-lmf.xml
new file mode 100644
index 0000000000..10e039214b
--- /dev/null
+++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wn-lmf.xml
@@ -0,0 +1,183 @@
+
+
+
+
+
+
+ dog
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ a domesticated canid
+
+ the dog barked
+
+
+ a carnivorous mammal with nonretractile claws
+
+
+
+ a small rodent with a long tail
+
+
+
+ a gnawing mammal with chisel teeth
+
+
+
+ an act of running at speed
+
+
+ a rigid rectangular container
+
+
+ a small juicy fruit
+
+
+ an adult male person
+
+
+ a score made in baseball
+
+
+ move fast on foot
+
+
+
+ move at a regular pace
+
+
+
+ change location or position
+
+
+ change position in space
+
+
+
+
+ of great height
+
+
+ of small height
+
+
+ of great size
+
+
+
+ above average in size
+
+
+
+ with speed
+
+
+ in a good or proper manner
+
+
+
diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/.gitattributes b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/.gitattributes
new file mode 100644
index 0000000000..3868b4d103
--- /dev/null
+++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/.gitattributes
@@ -0,0 +1,7 @@
+# WNDB is a byte-offset format: each data line embeds its own byte position in the
+# file, and WndbReader validates that offset against the actual position. Line-ending
+# normalization on checkout (the repo root's `* text=auto`) would insert a CR before
+# every LF on Windows, shifting every offset after the first line and breaking every
+# fixture that has more than one line. Disable it here so checkout is byte-identical
+# on every platform.
+* -text
diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/adj.exc b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/adj.exc
new file mode 100644
index 0000000000..404a2e4ddb
--- /dev/null
+++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/adj.exc
@@ -0,0 +1 @@
+better good
diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/adv.exc b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/adv.exc
new file mode 100644
index 0000000000..c43a2cd529
--- /dev/null
+++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/adv.exc
@@ -0,0 +1 @@
+best well
diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.adj b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.adj
new file mode 100644
index 0000000000..1340d16a45
--- /dev/null
+++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.adj
@@ -0,0 +1,22 @@
+ 1 Licensed to the Apache Software Foundation (ASF) under one or more
+ 2 contributor license agreements. See the NOTICE file distributed with
+ 3 this work for additional information regarding copyright ownership.
+ 4 The ASF licenses this file to You under the Apache License, Version 2.0
+ 5 (the "License"); you may not use this file except in compliance with
+ 6 the License. You may obtain a copy of the License at
+ 7
+ 8 http://www.apache.org/licenses/LICENSE-2.0
+ 9
+ 10 Unless required by applicable law or agreed to in writing, software
+ 11 distributed under the License is distributed on an "AS IS" BASIS,
+ 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ 13 See the License for the specific language governing permissions and
+ 14 limitations under the License.
+ 15
+ 16 Project-authored miniature WNDB fixture mirroring mini-wn-lmf.xml.
+ 17 License preamble lines begin with two spaces, as in released WNDB files,
+ 18 so readers skip them; data line offsets include this preamble.
+00001075 00 a 01 tall 0 001 ! 00001141 a 0101 | of great height
+00001141 00 a 01 short(p) 0 001 ! 00001075 a 0101 | of small height
+00001211 00 a 01 big 0 001 & 00001274 a 0000 | of great size
+00001274 00 s 01 large 0 001 & 00001211 a 0000 | above average in size
diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.adv b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.adv
new file mode 100644
index 0000000000..732c21dfd5
--- /dev/null
+++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.adv
@@ -0,0 +1,20 @@
+ 1 Licensed to the Apache Software Foundation (ASF) under one or more
+ 2 contributor license agreements. See the NOTICE file distributed with
+ 3 this work for additional information regarding copyright ownership.
+ 4 The ASF licenses this file to You under the Apache License, Version 2.0
+ 5 (the "License"); you may not use this file except in compliance with
+ 6 the License. You may obtain a copy of the License at
+ 7
+ 8 http://www.apache.org/licenses/LICENSE-2.0
+ 9
+ 10 Unless required by applicable law or agreed to in writing, software
+ 11 distributed under the License is distributed on an "AS IS" BASIS,
+ 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ 13 See the License for the specific language governing permissions and
+ 14 limitations under the License.
+ 15
+ 16 Project-authored miniature WNDB fixture mirroring mini-wn-lmf.xml.
+ 17 License preamble lines begin with two spaces, as in released WNDB files,
+ 18 so readers skip them; data line offsets include this preamble.
+00001075 02 r 01 quickly 0 000 | with speed
+00001121 02 r 01 well 0 000 | in a good or proper manner
diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.noun b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.noun
new file mode 100644
index 0000000000..0598111bf1
--- /dev/null
+++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.noun
@@ -0,0 +1,27 @@
+ 1 Licensed to the Apache Software Foundation (ASF) under one or more
+ 2 contributor license agreements. See the NOTICE file distributed with
+ 3 this work for additional information regarding copyright ownership.
+ 4 The ASF licenses this file to You under the Apache License, Version 2.0
+ 5 (the "License"); you may not use this file except in compliance with
+ 6 the License. You may obtain a copy of the License at
+ 7
+ 8 http://www.apache.org/licenses/LICENSE-2.0
+ 9
+ 10 Unless required by applicable law or agreed to in writing, software
+ 11 distributed under the License is distributed on an "AS IS" BASIS,
+ 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ 13 See the License for the specific language governing permissions and
+ 14 limitations under the License.
+ 15
+ 16 Project-authored miniature WNDB fixture mirroring mini-wn-lmf.xml.
+ 17 License preamble lines begin with two spaces, as in released WNDB files,
+ 18 so readers skip them; data line offsets include this preamble.
+00001075 03 n 02 dog 0 domestic_dog 0 001 @ 00001160 n 0000 | a domesticated canid
+00001160 03 n 01 canid 0 001 ~ 00001075 n 0000 | a carnivorous mammal with nonretractile claws
+00001257 03 n 01 mouse 0 001 @ 00001340 n 0000 | a small rodent with a long tail
+00001340 03 n 01 rodent 0 001 ~ 00001257 n 0000 | a gnawing mammal with chisel teeth
+00001427 03 n 01 run 0 001 + 00001075 v 0101 | an act of running at speed
+00001503 03 n 01 box 0 000 | a rigid rectangular container
+00001564 03 n 01 berry 0 000 | a small juicy fruit
+00001617 03 n 01 man 0 000 | an adult male person
+00001669 03 n 01 run 0 000 | a score made in baseball
diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.verb b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.verb
new file mode 100644
index 0000000000..048546ed71
--- /dev/null
+++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.verb
@@ -0,0 +1,22 @@
+ 1 Licensed to the Apache Software Foundation (ASF) under one or more
+ 2 contributor license agreements. See the NOTICE file distributed with
+ 3 this work for additional information regarding copyright ownership.
+ 4 The ASF licenses this file to You under the Apache License, Version 2.0
+ 5 (the "License"); you may not use this file except in compliance with
+ 6 the License. You may obtain a copy of the License at
+ 7
+ 8 http://www.apache.org/licenses/LICENSE-2.0
+ 9
+ 10 Unless required by applicable law or agreed to in writing, software
+ 11 distributed under the License is distributed on an "AS IS" BASIS,
+ 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ 13 See the License for the specific language governing permissions and
+ 14 limitations under the License.
+ 15
+ 16 Project-authored miniature WNDB fixture mirroring mini-wn-lmf.xml.
+ 17 License preamble lines begin with two spaces, as in released WNDB files,
+ 18 so readers skip them; data line offsets include this preamble.
+00001075 29 v 01 run 0 002 @ 00001324 v 0000 + 00001427 n 0101 01 + 02 00 | move fast on foot
+00001171 29 v 01 walk 0 001 @ 00001324 v 0000 01 + 02 00 | move at a regular pace
+00001255 29 v 01 go 0 000 01 + 02 00 | change location or position
+00001324 29 v 01 move 0 002 ~ 00001075 v 0000 ~ 00001171 v 0000 01 + 02 00 | change position in space
diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.adj b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.adj
new file mode 100644
index 0000000000..827a988a7d
--- /dev/null
+++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.adj
@@ -0,0 +1,22 @@
+ 1 Licensed to the Apache Software Foundation (ASF) under one or more
+ 2 contributor license agreements. See the NOTICE file distributed with
+ 3 this work for additional information regarding copyright ownership.
+ 4 The ASF licenses this file to You under the Apache License, Version 2.0
+ 5 (the "License"); you may not use this file except in compliance with
+ 6 the License. You may obtain a copy of the License at
+ 7
+ 8 http://www.apache.org/licenses/LICENSE-2.0
+ 9
+ 10 Unless required by applicable law or agreed to in writing, software
+ 11 distributed under the License is distributed on an "AS IS" BASIS,
+ 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ 13 See the License for the specific language governing permissions and
+ 14 limitations under the License.
+ 15
+ 16 Project-authored miniature WNDB fixture mirroring mini-wn-lmf.xml.
+ 17 License preamble lines begin with two spaces, as in released WNDB files,
+ 18 so readers skip them; data line offsets include this preamble.
+big a 1 1 & 1 0 00001211
+large a 1 1 & 1 0 00001274
+short a 1 1 ! 1 0 00001141
+tall a 1 1 ! 1 0 00001075
diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.adv b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.adv
new file mode 100644
index 0000000000..da20fe1193
--- /dev/null
+++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.adv
@@ -0,0 +1,20 @@
+ 1 Licensed to the Apache Software Foundation (ASF) under one or more
+ 2 contributor license agreements. See the NOTICE file distributed with
+ 3 this work for additional information regarding copyright ownership.
+ 4 The ASF licenses this file to You under the Apache License, Version 2.0
+ 5 (the "License"); you may not use this file except in compliance with
+ 6 the License. You may obtain a copy of the License at
+ 7
+ 8 http://www.apache.org/licenses/LICENSE-2.0
+ 9
+ 10 Unless required by applicable law or agreed to in writing, software
+ 11 distributed under the License is distributed on an "AS IS" BASIS,
+ 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ 13 See the License for the specific language governing permissions and
+ 14 limitations under the License.
+ 15
+ 16 Project-authored miniature WNDB fixture mirroring mini-wn-lmf.xml.
+ 17 License preamble lines begin with two spaces, as in released WNDB files,
+ 18 so readers skip them; data line offsets include this preamble.
+quickly r 1 0 1 0 00001075
+well r 1 0 1 0 00001121
diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.noun b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.noun
new file mode 100644
index 0000000000..41a8a4317b
--- /dev/null
+++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.noun
@@ -0,0 +1,27 @@
+ 1 Licensed to the Apache Software Foundation (ASF) under one or more
+ 2 contributor license agreements. See the NOTICE file distributed with
+ 3 this work for additional information regarding copyright ownership.
+ 4 The ASF licenses this file to You under the Apache License, Version 2.0
+ 5 (the "License"); you may not use this file except in compliance with
+ 6 the License. You may obtain a copy of the License at
+ 7
+ 8 http://www.apache.org/licenses/LICENSE-2.0
+ 9
+ 10 Unless required by applicable law or agreed to in writing, software
+ 11 distributed under the License is distributed on an "AS IS" BASIS,
+ 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ 13 See the License for the specific language governing permissions and
+ 14 limitations under the License.
+ 15
+ 16 Project-authored miniature WNDB fixture mirroring mini-wn-lmf.xml.
+ 17 License preamble lines begin with two spaces, as in released WNDB files,
+ 18 so readers skip them; data line offsets include this preamble.
+berry n 1 0 1 0 00001564
+box n 1 0 1 0 00001503
+canid n 1 1 ~ 1 0 00001160
+dog n 1 1 @ 1 0 00001075
+domestic_dog n 1 1 @ 1 0 00001075
+man n 1 0 1 0 00001617
+mouse n 1 1 @ 1 0 00001257
+rodent n 1 1 ~ 1 0 00001340
+run n 2 1 + 2 1 00001427 00001669
diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.verb b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.verb
new file mode 100644
index 0000000000..2b380478de
--- /dev/null
+++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.verb
@@ -0,0 +1,22 @@
+ 1 Licensed to the Apache Software Foundation (ASF) under one or more
+ 2 contributor license agreements. See the NOTICE file distributed with
+ 3 this work for additional information regarding copyright ownership.
+ 4 The ASF licenses this file to You under the Apache License, Version 2.0
+ 5 (the "License"); you may not use this file except in compliance with
+ 6 the License. You may obtain a copy of the License at
+ 7
+ 8 http://www.apache.org/licenses/LICENSE-2.0
+ 9
+ 10 Unless required by applicable law or agreed to in writing, software
+ 11 distributed under the License is distributed on an "AS IS" BASIS,
+ 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ 13 See the License for the specific language governing permissions and
+ 14 limitations under the License.
+ 15
+ 16 Project-authored miniature WNDB fixture mirroring mini-wn-lmf.xml.
+ 17 License preamble lines begin with two spaces, as in released WNDB files,
+ 18 so readers skip them; data line offsets include this preamble.
+go v 1 0 1 0 00001255
+move v 1 1 ~ 1 0 00001324
+run v 1 2 @ + 1 1 00001075
+walk v 1 1 @ 1 0 00001171
diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/noun.exc b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/noun.exc
new file mode 100644
index 0000000000..e5b3080a88
--- /dev/null
+++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/noun.exc
@@ -0,0 +1,3 @@
+men man
+mice mouse
+oxen ox
diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/verb.exc b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/verb.exc
new file mode 100644
index 0000000000..486d0c7851
--- /dev/null
+++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/verb.exc
@@ -0,0 +1,4 @@
+gone go
+ran run
+running run
+went go
diff --git a/opennlp-extensions/pom.xml b/opennlp-extensions/pom.xml
index 9afcd3fe3c..8c0e432a58 100644
--- a/opennlp-extensions/pom.xml
+++ b/opennlp-extensions/pom.xml
@@ -41,6 +41,7 @@
opennlp-morfologik
opennlp-spellcheck
opennlp-uima
+ opennlp-wordnet
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index c74e2dc021..bf8a790d5f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -216,6 +216,12 @@
${project.version}