From 61a361bab99aa99c4e8da18c7e31b182d7117644 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 7 Jul 2026 00:23:34 -0400 Subject: [PATCH 01/12] Add wordnet lexicon contracts in opennlp.tools.wordnet --- .../java/opennlp/tools/wordnet/Synset.java | 130 +++++++++++++++ .../opennlp/tools/wordnet/WordNetLexicon.java | 100 ++++++++++++ .../opennlp/tools/wordnet/WordNetPos.java | 46 ++++++ .../tools/wordnet/WordNetRelation.java | 129 +++++++++++++++ .../opennlp/tools/wordnet/SynsetTest.java | 150 ++++++++++++++++++ .../tools/wordnet/WordNetLexiconTest.java | 105 ++++++++++++ 6 files changed, 660 insertions(+) create mode 100644 opennlp-api/src/main/java/opennlp/tools/wordnet/Synset.java create mode 100644 opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetLexicon.java create mode 100644 opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetPos.java create mode 100644 opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetRelation.java create mode 100644 opennlp-api/src/test/java/opennlp/tools/wordnet/SynsetTest.java create mode 100644 opennlp-api/src/test/java/opennlp/tools/wordnet/WordNetLexiconTest.java diff --git a/opennlp-api/src/main/java/opennlp/tools/wordnet/Synset.java b/opennlp-api/src/main/java/opennlp/tools/wordnet/Synset.java new file mode 100644 index 0000000000..c550efe8b0 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/wordnet/Synset.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.wordnet; + +import java.util.Collections; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; + +import opennlp.tools.commons.ThreadSafe; + +/** + * One synonym set: a single lexicalized concept with its member lemmas, gloss, and typed + * relations to other synsets. + * + *

The shape is deliberately format-agnostic. The {@link #id() id} is an opaque, + * source-qualified string minted by whichever reader produced the synset (a WN-LMF document's + * own synset id, or an id a WNDB reader derives from the file position); consumers must not + * parse it, only pass it back to {@link WordNetLexicon#synset(String)} and compare it for + * equality. Keeping identity opaque is what lets a bundled lexicon and a user-downloaded + * lexicon sit behind the same seam, and lets later alignment layers join sources without a + * contract change.

+ * + *

Relations map each {@link WordNetRelation} present on this synset to the target synset + * ids in source order. Relations some formats draw between individual senses (antonymy, + * derivation) surface here at the synset level; see {@link WordNetRelation}.

+ * + *

Instances are immutable and thread-safe: the list and map components are defensively + * copied to immutable views at construction.

+ * + * @param id The opaque, source-qualified synset identifier. Must not be {@code null} or + * empty. + * @param pos The part of speech. Must not be {@code null}. + * @param lemmas The member lemmas in source order, human-readable (multiword lemmas use + * spaces, not the underscores some formats store). Must not be {@code null} or + * empty, and must not contain {@code null} or empty elements. + * @param gloss The definition text, possibly empty when the source has none. Must not be + * {@code null}. + * @param relations The typed relations, each mapping to the target synset ids in source order. + * Must not be {@code null}; keys must not be {@code null}; each value must be + * a non-empty list of non-{@code null}, non-empty target ids. + */ +@ThreadSafe +public record Synset( + String id, + WordNetPos pos, + List lemmas, + String gloss, + Map> relations) { + + /** + * Creates a synset. + * + * @throws IllegalArgumentException Thrown if any component violates its documented constraint. + */ + public Synset { + if (id == null || id.isEmpty()) { + throw new IllegalArgumentException("Id must not be null or empty"); + } + if (pos == null) { + throw new IllegalArgumentException("Pos must not be null"); + } + if (lemmas == null || lemmas.isEmpty()) { + throw new IllegalArgumentException("Lemmas must not be null or empty for synset " + id); + } + for (final String lemma : lemmas) { + if (lemma == null || lemma.isEmpty()) { + throw new IllegalArgumentException( + "Lemmas must not contain a null or empty element for synset " + id); + } + } + if (gloss == null) { + throw new IllegalArgumentException("Gloss must not be null for synset " + id); + } + if (relations == null) { + throw new IllegalArgumentException("Relations must not be null for synset " + id); + } + final Map> copiedRelations = + new EnumMap<>(WordNetRelation.class); + for (final Map.Entry> relation : relations.entrySet()) { + if (relation.getKey() == null) { + throw new IllegalArgumentException("Relations must not contain a null key for synset " + id); + } + final List targets = relation.getValue(); + if (targets == null || targets.isEmpty()) { + throw new IllegalArgumentException("Relation " + relation.getKey() + + " must map to a non-empty target list for synset " + id); + } + for (final String target : targets) { + if (target == null || target.isEmpty()) { + throw new IllegalArgumentException("Relation " + relation.getKey() + + " must not contain a null or empty target id for synset " + id); + } + } + copiedRelations.put(relation.getKey(), List.copyOf(targets)); + } + lemmas = List.copyOf(lemmas); + relations = Collections.unmodifiableMap(copiedRelations); + } + + /** + * Finds the target synset ids of one relation type. + * + * @param relation The relation type. Must not be {@code null}. + * @return The target synset ids in source order, never {@code null}; empty when this synset + * has no relation of that type. + * @throws IllegalArgumentException Thrown if {@code relation} is {@code null}. + */ + public List related(WordNetRelation relation) { + if (relation == null) { + throw new IllegalArgumentException("Relation must not be null"); + } + final List targets = relations.get(relation); + return targets == null ? List.of() : targets; + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetLexicon.java b/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetLexicon.java new file mode 100644 index 0000000000..a1d719808a --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetLexicon.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.wordnet; + +import java.util.List; +import java.util.Optional; + +/** + * The wordnet lexicon seam: lemma and synset lookup over a loaded wordnet-style resource. + * + *

This interface is the contract every wordnet-shaped dataset sits behind. A legacy Princeton + * WNDB directory, a WN-LMF XML document (the Global WordNet Association interchange format used + * by Open English WordNet and many other language wordnets), a future bundled + * permissively-licensed lexicon, and a future user-downloaded CC-BY lexicon are all + * implementations of this one seam, so a consumer written against {@code WordNetLexicon} never + * changes when the data tier does. Nothing in the contract names a particular resource; synset + * identity is opaque and source-qualified (see {@link Synset#id()}).

+ * + *

The surface is intentionally minimal but sufficient for the layered features above it: + * lemma lookup and membership for morphological analysis (Morphy-style lemmatization), and + * synset retrieval with typed relation navigation for query expansion and, later, similarity + * measures. Feature layers stack on these four operations without a contract change.

+ * + *

Lemma matching semantics are the implementation's concern. The reference implementations + * match case-insensitively (case folding with the root locale) and treat the underscore some + * formats store in multiword lemmas as a space, which is the recommended behavior; an + * implementation with different semantics must document them. Returned {@link Synset#lemmas() + * lemmas} preserve the source's written forms, with spaces in multiword lemmas.

+ * + *

Implementations must be immutable and thread-safe after loading: one instance is meant to + * be shared across an application's threads for concurrent lookups.

+ */ +public interface WordNetLexicon { + + /** + * Finds the synsets containing a lemma with a part of speech, in the source's sense order + * (the most salient sense first when the source ranks senses). + * + * @param lemma The lemma to look up. Must not be {@code null}. + * @param pos The part of speech to look it up as. Must not be {@code null}. + * @return The matching synsets, never {@code null}; empty when the lexicon does not contain + * the lemma with that part of speech. + * @throws IllegalArgumentException Thrown if {@code lemma} or {@code pos} is {@code null}. + */ + List lookup(String lemma, WordNetPos pos); + + /** + * Finds a synset by its opaque identifier. + * + * @param synsetId The synset identifier, as minted by this lexicon. Must not be {@code null}. + * @return The synset, or empty when this lexicon has no synset with that identifier. + * @throws IllegalArgumentException Thrown if {@code synsetId} is {@code null}. + */ + Optional synset(String synsetId); + + /** + * Navigates one typed relation from a synset. + * + * @param synsetId The source synset identifier. Must not be {@code null}. + * @param relation The relation type to follow. Must not be {@code null}. + * @return The target synset ids in source order, never {@code null}; empty when the synset is + * unknown or has no relation of that type. + * @throws IllegalArgumentException Thrown if {@code synsetId} or {@code relation} is + * {@code null}. + */ + default List related(String synsetId, WordNetRelation relation) { + if (relation == null) { + throw new IllegalArgumentException("Relation must not be null"); + } + return synset(synsetId).map(s -> s.related(relation)).orElse(List.of()); + } + + /** + * Tests whether the lexicon contains a lemma with a part of speech. This is the membership + * check morphological rules validate their candidates against; implementations may override + * it with a cheaper check than {@link #lookup(String, WordNetPos)}. + * + * @param lemma The lemma to test. Must not be {@code null}. + * @param pos The part of speech to test it as. Must not be {@code null}. + * @return {@code true} if the lexicon contains the lemma with that part of speech. + * @throws IllegalArgumentException Thrown if {@code lemma} or {@code pos} is {@code null}. + */ + default boolean contains(String lemma, WordNetPos pos) { + return !lookup(lemma, pos).isEmpty(); + } +} diff --git a/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetPos.java b/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetPos.java new file mode 100644 index 0000000000..0e24085fb2 --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetPos.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.wordnet; + +/** + * The four parts of speech a wordnet-style lexicon distinguishes. + * + *

This enum is deliberately format-free: it carries none of the single-letter codes the + * on-disk formats use, so the contract does not leak a storage detail. Readers own the mapping + * from their format's codes to these values.

+ * + *

Adjective satellites (the {@code s} code some formats use for adjectives clustered around + * a head adjective) normalize to {@link #ADJECTIVE}: the head/satellite split is a storage + * layout, not a part of speech, and the cluster structure is preserved through the + * {@link WordNetRelation#SIMILAR_TO} relation instead.

+ * + *

Enum constants are immutable and safe to share across threads.

+ */ +public enum WordNetPos { + + /** Nouns. */ + NOUN, + + /** Verbs. */ + VERB, + + /** Adjectives, including adjective satellites. */ + ADJECTIVE, + + /** Adverbs. */ + ADVERB +} diff --git a/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetRelation.java b/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetRelation.java new file mode 100644 index 0000000000..0794a8608d --- /dev/null +++ b/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetRelation.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.wordnet; + +/** + * The typed relations a wordnet-style lexicon draws between {@link Synset synsets}. + * + *

The value set covers the pointer types documented for the legacy Princeton WNDB format, + * which is also the common core the WN-LMF interchange format expresses; both readers map their + * format's names onto these values, so consumers navigate one relation vocabulary regardless of + * the data tier behind the {@link WordNetLexicon} seam. {@link #PARTICIPLE} is part of that + * documented set (the adjective "participle of verb" pointer) even though it is easy + * to overlook; without it, real Princeton data would be unreadable. Two values go beyond the + * WNDB pointer set: {@link #ENTAILED_BY} and {@link #CAUSED_BY} cover the inverse relations + * WN-LMF documents such as Open English WordNet materialize, so reading such a document loses + * no typed relation; a WNDB-backed lexicon never produces them because the legacy format stores + * only the forward direction.

+ * + *

Some of these relations are, in the source formats, drawn between individual word senses + * rather than whole synsets (antonymy and derivation are the common examples). In this v1 + * contract they surface at the synset level: the synset containing the source sense carries the + * relation to the synset containing the target sense. A sense-level view is a later, additive + * layer.

+ * + *

Enum constants are immutable and safe to share across threads.

+ */ +public enum WordNetRelation { + + /** Opposition in meaning, for example between the adjectives for tall and short. */ + ANTONYM, + + /** The more general concept: a dog is a kind of canid. */ + HYPERNYM, + + /** The class a named instance belongs to: a specific river is an instance of river. */ + INSTANCE_HYPERNYM, + + /** The more specific concept: canid has the hyponym dog. */ + HYPONYM, + + /** A named instance of this class. */ + INSTANCE_HYPONYM, + + /** The group this synset is a member of. */ + MEMBER_HOLONYM, + + /** The whole this synset is a substance of. */ + SUBSTANCE_HOLONYM, + + /** The whole this synset is a part of. */ + PART_HOLONYM, + + /** A member of this group. */ + MEMBER_MERONYM, + + /** A substance this synset is made of. */ + SUBSTANCE_MERONYM, + + /** A part of this synset. */ + PART_MERONYM, + + /** The attribute a value expresses, or a value of this attribute. */ + ATTRIBUTE, + + /** A derivationally related form, typically across parts of speech. */ + DERIVATIONALLY_RELATED, + + /** An action entailed by this verb: snoring entails sleeping. */ + ENTAILMENT, + + /** The verb that entails this one; the inverse of {@link #ENTAILMENT}. */ + ENTAILED_BY, + + /** An effect this verb causes. */ + CAUSE, + + /** The cause of this verb; the inverse of {@link #CAUSE}. */ + CAUSED_BY, + + /** A related synset worth consulting. */ + ALSO_SEE, + + /** A verb sense grouped with this one. */ + VERB_GROUP, + + /** A satellite or head adjective in the same similarity cluster. */ + SIMILAR_TO, + + /** The verb an adjective is the participle of. */ + PARTICIPLE, + + /** + * The noun an adjective pertains to, or the adjective an adverb derives from. The source + * formats use one pointer for both directions of derivation, so this value does too. + */ + PERTAINYM, + + /** The topical domain this synset belongs to. */ + DOMAIN_TOPIC, + + /** A synset belonging to this topical domain. */ + MEMBER_OF_DOMAIN_TOPIC, + + /** The regional domain this synset belongs to. */ + DOMAIN_REGION, + + /** A synset belonging to this regional domain. */ + MEMBER_OF_DOMAIN_REGION, + + /** The usage domain this synset belongs to, for example slang or archaism. */ + DOMAIN_USAGE, + + /** A synset belonging to this usage domain. */ + MEMBER_OF_DOMAIN_USAGE +} diff --git a/opennlp-api/src/test/java/opennlp/tools/wordnet/SynsetTest.java b/opennlp-api/src/test/java/opennlp/tools/wordnet/SynsetTest.java new file mode 100644 index 0000000000..abcb45117b --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/wordnet/SynsetTest.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.wordnet; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class SynsetTest { + + private static Synset dog() { + return new Synset("test-1-n", WordNetPos.NOUN, List.of("dog", "domestic dog"), + "a domesticated canid", + Map.of(WordNetRelation.HYPERNYM, List.of("test-2-n"))); + } + + @Test + void testComponents() { + final Synset synset = dog(); + assertEquals("test-1-n", synset.id()); + assertEquals(WordNetPos.NOUN, synset.pos()); + assertEquals(List.of("dog", "domestic dog"), synset.lemmas()); + assertEquals("a domesticated canid", synset.gloss()); + assertEquals(Map.of(WordNetRelation.HYPERNYM, List.of("test-2-n")), synset.relations()); + } + + @Test + void testRelatedReturnsTargetsInOrder() { + final Synset synset = new Synset("test-1-n", WordNetPos.NOUN, List.of("dog"), "", + Map.of(WordNetRelation.HYPONYM, List.of("test-3-n", "test-2-n"))); + assertEquals(List.of("test-3-n", "test-2-n"), synset.related(WordNetRelation.HYPONYM)); + } + + @Test + void testRelatedIsEmptyForAbsentRelation() { + assertTrue(dog().related(WordNetRelation.ANTONYM).isEmpty()); + } + + @Test + void testRelatedRejectsNull() { + assertThrows(IllegalArgumentException.class, () -> dog().related(null)); + } + + @Test + void testEmptyGlossAndNoRelationsAreValid() { + final Synset synset = new Synset("test-9-r", WordNetPos.ADVERB, List.of("well"), "", Map.of()); + assertEquals("", synset.gloss()); + assertTrue(synset.relations().isEmpty()); + } + + @Test + void testDefensiveCopies() { + final List lemmas = new ArrayList<>(List.of("dog")); + final List targets = new ArrayList<>(List.of("test-2-n")); + final Map> relations = new HashMap<>(); + relations.put(WordNetRelation.HYPERNYM, targets); + final Synset synset = new Synset("test-1-n", WordNetPos.NOUN, lemmas, "gloss", relations); + lemmas.add("mutated"); + targets.add("mutated"); + relations.put(WordNetRelation.ANTONYM, List.of("test-3-n")); + assertEquals(List.of("dog"), synset.lemmas()); + assertEquals(List.of("test-2-n"), synset.related(WordNetRelation.HYPERNYM)); + assertEquals(1, synset.relations().size()); + } + + @Test + void testReturnedCollectionsAreImmutable() { + final Synset synset = dog(); + assertThrows(UnsupportedOperationException.class, () -> synset.lemmas().add("x")); + assertThrows(UnsupportedOperationException.class, + () -> synset.relations().put(WordNetRelation.ANTONYM, List.of("x"))); + assertThrows(UnsupportedOperationException.class, + () -> synset.related(WordNetRelation.HYPERNYM).add("x")); + } + + @Test + void testRejectsNullOrEmptyId() { + assertThrows(IllegalArgumentException.class, + () -> new Synset(null, WordNetPos.NOUN, List.of("dog"), "", Map.of())); + assertThrows(IllegalArgumentException.class, + () -> new Synset("", WordNetPos.NOUN, List.of("dog"), "", Map.of())); + } + + @Test + void testRejectsNullPos() { + assertThrows(IllegalArgumentException.class, + () -> new Synset("test-1-n", null, List.of("dog"), "", Map.of())); + } + + @Test + void testRejectsNullOrEmptyLemmas() { + assertThrows(IllegalArgumentException.class, + () -> new Synset("test-1-n", WordNetPos.NOUN, null, "", Map.of())); + assertThrows(IllegalArgumentException.class, + () -> new Synset("test-1-n", WordNetPos.NOUN, List.of(), "", Map.of())); + final List withNull = new ArrayList<>(); + withNull.add(null); + assertThrows(IllegalArgumentException.class, + () -> new Synset("test-1-n", WordNetPos.NOUN, withNull, "", Map.of())); + assertThrows(IllegalArgumentException.class, + () -> new Synset("test-1-n", WordNetPos.NOUN, List.of(""), "", Map.of())); + } + + @Test + void testRejectsNullGloss() { + assertThrows(IllegalArgumentException.class, + () -> new Synset("test-1-n", WordNetPos.NOUN, List.of("dog"), null, Map.of())); + } + + @Test + void testRejectsInvalidRelations() { + assertThrows(IllegalArgumentException.class, + () -> new Synset("test-1-n", WordNetPos.NOUN, List.of("dog"), "", null)); + final Map> nullKey = new HashMap<>(); + nullKey.put(null, List.of("test-2-n")); + assertThrows(IllegalArgumentException.class, + () -> new Synset("test-1-n", WordNetPos.NOUN, List.of("dog"), "", nullKey)); + final Map> nullTargets = new HashMap<>(); + nullTargets.put(WordNetRelation.HYPERNYM, null); + assertThrows(IllegalArgumentException.class, + () -> new Synset("test-1-n", WordNetPos.NOUN, List.of("dog"), "", nullTargets)); + assertThrows(IllegalArgumentException.class, + () -> new Synset("test-1-n", WordNetPos.NOUN, List.of("dog"), "", + Map.of(WordNetRelation.HYPERNYM, List.of()))); + assertThrows(IllegalArgumentException.class, + () -> new Synset("test-1-n", WordNetPos.NOUN, List.of("dog"), "", + Map.of(WordNetRelation.HYPERNYM, List.of("")))); + } +} diff --git a/opennlp-api/src/test/java/opennlp/tools/wordnet/WordNetLexiconTest.java b/opennlp-api/src/test/java/opennlp/tools/wordnet/WordNetLexiconTest.java new file mode 100644 index 0000000000..0823c6236d --- /dev/null +++ b/opennlp-api/src/test/java/opennlp/tools/wordnet/WordNetLexiconTest.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.wordnet; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Exercises the {@link WordNetLexicon} default methods against a minimal in-memory + * implementation, so the defaults are validated independently of any reader. + */ +public class WordNetLexiconTest { + + private static final Synset DOG = new Synset("test-1-n", WordNetPos.NOUN, List.of("dog"), + "a domesticated canid", Map.of(WordNetRelation.HYPERNYM, List.of("test-2-n"))); + + private static final Synset CANID = new Synset("test-2-n", WordNetPos.NOUN, List.of("canid"), + "a carnivorous mammal", Map.of(WordNetRelation.HYPONYM, List.of("test-1-n"))); + + // A deliberately tiny implementation of only the two abstract methods. + private static final WordNetLexicon LEXICON = new WordNetLexicon() { + + @Override + public List lookup(String lemma, WordNetPos pos) { + if (lemma == null) { + throw new IllegalArgumentException("Lemma must not be null"); + } + if (pos == null) { + throw new IllegalArgumentException("Pos must not be null"); + } + if (pos == WordNetPos.NOUN && "dog".equals(lemma)) { + return List.of(DOG); + } + return List.of(); + } + + @Override + public Optional synset(String synsetId) { + if (synsetId == null) { + throw new IllegalArgumentException("SynsetId must not be null"); + } + if (DOG.id().equals(synsetId)) { + return Optional.of(DOG); + } + if (CANID.id().equals(synsetId)) { + return Optional.of(CANID); + } + return Optional.empty(); + } + }; + + @Test + void testRelatedNavigatesThroughSynset() { + assertEquals(List.of("test-2-n"), LEXICON.related("test-1-n", WordNetRelation.HYPERNYM)); + assertEquals(List.of("test-1-n"), LEXICON.related("test-2-n", WordNetRelation.HYPONYM)); + } + + @Test + void testRelatedIsEmptyForAbsentRelationOrUnknownSynset() { + assertTrue(LEXICON.related("test-1-n", WordNetRelation.ANTONYM).isEmpty()); + assertTrue(LEXICON.related("test-99-n", WordNetRelation.HYPERNYM).isEmpty()); + } + + @Test + void testRelatedRejectsNulls() { + assertThrows(IllegalArgumentException.class, + () -> LEXICON.related(null, WordNetRelation.HYPERNYM)); + assertThrows(IllegalArgumentException.class, () -> LEXICON.related("test-1-n", null)); + } + + @Test + void testContainsFollowsLookup() { + assertTrue(LEXICON.contains("dog", WordNetPos.NOUN)); + assertFalse(LEXICON.contains("dog", WordNetPos.VERB)); + assertFalse(LEXICON.contains("cat", WordNetPos.NOUN)); + } + + @Test + void testContainsRejectsNulls() { + assertThrows(IllegalArgumentException.class, () -> LEXICON.contains(null, WordNetPos.NOUN)); + assertThrows(IllegalArgumentException.class, () -> LEXICON.contains("dog", null)); + } +} From 98d235b8fd7e3101000145a60307cf832f5dc9e9 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 7 Jul 2026 00:36:25 -0400 Subject: [PATCH 02/12] Add opennlp-wordnet module with the WN-LMF reader --- opennlp-extensions/opennlp-wordnet/pom.xml | 53 ++ .../wordnet/InMemoryWordNetLexicon.java | 151 ++++++ .../java/opennlp/wordnet/WnLmfReader.java | 484 ++++++++++++++++++ .../java/opennlp/wordnet/WnLmfReaderTest.java | 238 +++++++++ .../resources/opennlp/wordnet/mini-wn-lmf.xml | 183 +++++++ opennlp-extensions/pom.xml | 1 + 6 files changed, 1110 insertions(+) create mode 100644 opennlp-extensions/opennlp-wordnet/pom.xml create mode 100644 opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wn-lmf.xml diff --git a/opennlp-extensions/opennlp-wordnet/pom.xml b/opennlp-extensions/opennlp-wordnet/pom.xml new file mode 100644 index 0000000000..854315eaf2 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/pom.xml @@ -0,0 +1,53 @@ + + + + + + 4.0.0 + + org.apache.opennlp + opennlp-extensions + 3.0.0-SNAPSHOT + + + opennlp-wordnet + jar + Apache OpenNLP :: Ext :: WordNet + + + + org.apache.opennlp + opennlp-api + + + + org.junit.jupiter + junit-jupiter-api + test + + + + org.junit.jupiter + junit-jupiter-engine + test + + + + diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java new file mode 100644 index 0000000000..e81fe4b768 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; + +import opennlp.tools.commons.ThreadSafe; +import opennlp.tools.wordnet.Synset; +import opennlp.tools.wordnet.WordNetLexicon; +import opennlp.tools.wordnet.WordNetPos; +import opennlp.tools.wordnet.WordNetRelation; + +/** + * The immutable in-memory {@link WordNetLexicon} both readers produce: a synset table plus a + * folded (lemma, part of speech) index. Package-private because it is a reader product, not a + * public entry point; consumers hold it as {@link WordNetLexicon}. + * + *

Lemma matching follows the recommended seam semantics: keys are folded by lowercasing with + * the root locale and treating the underscore some formats store in multiword lemmas as a + * space. Queries are folded identically at lookup time.

+ * + *

Construction verifies referential integrity: every relation target of every synset must + * resolve to a synset in the table, so a lexicon can never hand out a dangling identifier. + * After construction all state is immutable, making instances safe for concurrent lookups.

+ */ +@ThreadSafe +final class InMemoryWordNetLexicon implements WordNetLexicon { + + private final Map synsetsById; + private final Map> senseIndex; + + /** + * Indexes the given synsets. + * + * @param synsetsById The synset table keyed by synset id; every key must equal its synset's + * {@link Synset#id() id}. Must not be {@code null}. + * @param senseOrder The sense order per folded (lemma, part of speech) key: for each key the + * ids of the synsets containing the lemma, most salient sense first, each + * id resolvable in {@code synsetsById} and free of duplicates per key. + * Must not be {@code null}. + * @throws IllegalArgumentException Thrown if a relation target or sense entry does not + * resolve, or a key disagrees with its synset id. + */ + InMemoryWordNetLexicon(Map synsetsById, Map> senseOrder) { + if (synsetsById == null) { + throw new IllegalArgumentException("SynsetsById must not be null"); + } + if (senseOrder == null) { + throw new IllegalArgumentException("SenseOrder must not be null"); + } + final Map byId = new HashMap<>(synsetsById.size() * 2); + for (final Map.Entry entry : synsetsById.entrySet()) { + if (entry.getValue() == null || !entry.getValue().id().equals(entry.getKey())) { + throw new IllegalArgumentException( + "Synset table key " + entry.getKey() + " does not match its synset"); + } + byId.put(entry.getKey(), entry.getValue()); + } + for (final Synset synset : byId.values()) { + for (final Map.Entry> relation : + synset.relations().entrySet()) { + for (final String target : relation.getValue()) { + if (!byId.containsKey(target)) { + throw new IllegalArgumentException("Synset " + synset.id() + " has a " + + relation.getKey() + " relation to unknown synset " + target); + } + } + } + } + final Map> index = new HashMap<>(senseOrder.size() * 2); + for (final Map.Entry> entry : senseOrder.entrySet()) { + final List senses = new ArrayList<>(entry.getValue().size()); + for (final String synsetId : entry.getValue()) { + final Synset synset = byId.get(synsetId); + if (synset == null) { + throw new IllegalArgumentException("Sense index entry " + entry.getKey().lemma() + + " (" + entry.getKey().pos() + ") references unknown synset " + synsetId); + } + senses.add(synset); + } + index.put(entry.getKey(), List.copyOf(senses)); + } + this.synsetsById = byId; + this.senseIndex = index; + } + + @Override + public List lookup(String lemma, WordNetPos pos) { + if (lemma == null) { + throw new IllegalArgumentException("Lemma must not be null"); + } + if (pos == null) { + throw new IllegalArgumentException("Pos must not be null"); + } + final List senses = senseIndex.get(LemmaKey.of(lemma, pos)); + return senses == null ? List.of() : senses; + } + + @Override + public Optional synset(String synsetId) { + if (synsetId == null) { + throw new IllegalArgumentException("SynsetId must not be null"); + } + return Optional.ofNullable(synsetsById.get(synsetId)); + } + + /** {@return the number of synsets in this lexicon} */ + int size() { + return synsetsById.size(); + } + + /** + * A folded sense-index key. Build with {@link #of(String, WordNetPos)} so every key passes + * through the same fold as every query. + * + * @param lemma The folded lemma. + * @param pos The part of speech. + */ + record LemmaKey(String lemma, WordNetPos pos) { + + /** + * Folds a written form into a key: lowercase with the root locale, underscore as space. + * + * @param writtenForm The lemma as written in the source or query. Must not be {@code null}. + * @param pos The part of speech. Must not be {@code null}. + * @return The folded key. + */ + static LemmaKey of(String writtenForm, WordNetPos pos) { + return new LemmaKey(writtenForm.replace('_', ' ').toLowerCase(Locale.ROOT), pos); + } + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java new file mode 100644 index 0000000000..d5f930233a --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java @@ -0,0 +1,484 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import javax.xml.stream.Location; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamConstants; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; + +import opennlp.tools.wordnet.Synset; +import opennlp.tools.wordnet.WordNetLexicon; +import opennlp.tools.wordnet.WordNetPos; +import opennlp.tools.wordnet.WordNetRelation; + +/** + * Reads a WN-LMF XML document (the Global WordNet Association interchange format, used by Open + * English WordNet and many other language wordnets) into a {@link WordNetLexicon}. + * + *

The reader is clean-room, built from the published format documentation, and uses only the + * JDK's StAX parser. It reads the subset of the format the {@link WordNetLexicon} contract + * serves: lexical entries (lemma, part of speech, senses), synsets (definition and typed + * relations), and sense relations, which are lifted to the synset level as documented on + * {@link WordNetRelation}. Elements outside that subset ({@code Pronunciation}, {@code Form}, + * {@code Example}, {@code SyntacticBehaviour}, {@code ILIDefinition}, and similar) are skipped. + * Relations of type {@code other}, the format's escape hatch for relations typed only by + * metadata, are also skipped: the contract has no untyped relation slot. Every other unknown + * relation type fails loud.

+ * + *

Security. The parser is hardened against XXE: DTD processing and external entity + * resolution are disabled, nothing is ever fetched from the network, and a document containing + * a DOCTYPE declaration is rejected outright. Note that Open English WordNet releases ship with + * a DOCTYPE line referencing the schema DTD; to read such a file with this v1 reader, delete + * that one line first. Rejecting DTDs wholesale rather than trusting parser-specific partial + * DTD support is a deliberate v1 posture.

+ * + *

Errors. Malformed structure fails loud with an {@link IllegalArgumentException} + * naming the resource and, where the parser provides one, the line: missing required + * attributes, a sense pointing to an undeclared synset, a relation to an undeclared target, an + * unknown part-of-speech code, an unknown relation type, a synset with no member entries.

+ * + *

Part-of-speech code {@code s} (adjective satellite) normalizes to + * {@link WordNetPos#ADJECTIVE}, and a {@code similar} relation whose source is a verb synset + * maps to {@link WordNetRelation#VERB_GROUP} (that is how WN-LMF documents derived from + * Princeton data express verb groups); on any other part of speech it maps to + * {@link WordNetRelation#SIMILAR_TO}.

+ * + *

The returned lexicon is immutable and safe for concurrent lookups.

+ */ +public final class WnLmfReader { + + private static final Map RELATION_NAMES = relationNames(); + + /** The format's escape-hatch relation type; carries no type the contract can express. */ + private static final String OTHER_RELATION = "other"; + + private WnLmfReader() { + } + + /** + * Reads a WN-LMF XML file. + * + * @param file The XML file. Must not be {@code null} and must exist. + * @return The loaded lexicon. + * @throws IllegalArgumentException Thrown if {@code file} is {@code null} or missing, or the + * document is malformed; the message names the file and, where available, the line. + * @throws UncheckedIOException Thrown if reading the file fails. + */ + public static WordNetLexicon read(Path file) { + if (file == null) { + throw new IllegalArgumentException("File must not be null"); + } + if (!Files.isRegularFile(file)) { + throw new IllegalArgumentException("File does not exist or is not a regular file: " + file); + } + try (InputStream in = new BufferedInputStream(Files.newInputStream(file))) { + return read(in, file.toString()); + } catch (IOException e) { + throw new UncheckedIOException("Unable to read WN-LMF document " + file, e); + } + } + + /** + * Reads a WN-LMF XML document from a stream. The stream is not closed. + * + * @param in The document stream. Must not be {@code null}. + * @param resourceName The name used in error messages. Must not be {@code null}. + * @return The loaded lexicon. + * @throws IllegalArgumentException Thrown if an argument is {@code null} or the document is + * malformed; the message names the resource and, where available, the line. + */ + public static WordNetLexicon read(InputStream in, String resourceName) { + if (in == null) { + throw new IllegalArgumentException("In must not be null"); + } + if (resourceName == null) { + throw new IllegalArgumentException("ResourceName must not be null"); + } + final Parser parser = new Parser(resourceName); + try { + final XMLStreamReader reader = hardenedFactory().createXMLStreamReader(in); + try { + parser.parse(reader); + } finally { + reader.close(); + } + } catch (XMLStreamException e) { + throw parser.malformed(e.getLocation(), "XML error: " + e.getMessage(), e); + } + return parser.build(); + } + + private static XMLInputFactory hardenedFactory() { + final XMLInputFactory factory = XMLInputFactory.newFactory(); + // XXE hardening: no DTD processing, no external entities, nothing fetched from anywhere. + factory.setProperty(XMLInputFactory.SUPPORT_DTD, Boolean.FALSE); + factory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, Boolean.FALSE); + factory.setProperty(XMLInputFactory.IS_COALESCING, Boolean.TRUE); + factory.setXMLResolver((publicId, systemId, baseUri, namespace) -> { + throw new XMLStreamException("External entity resolution is disabled, refusing " + systemId); + }); + return factory; + } + + // The streaming parse state and post-parse resolution. + private static final class Parser { + + private final String resourceName; + + // Entry state. + private final Map lemmaByEntryId = new HashMap<>(); + private final Map posByEntryId = new HashMap<>(); + private final Map synsetBySenseId = new HashMap<>(); + private final Map> senseOrder = + new LinkedHashMap<>(); + private final List senseRelations = new ArrayList<>(); + private final Map rawSynsets = new LinkedHashMap<>(); + // Fallback membership (entry ids per synset in document order) when members is absent. + private final Map> entryIdsBySynset = new HashMap<>(); + + // Cursor state. + private String currentEntryId; + private String currentEntryLemma; + private WordNetPos currentEntryPos; + private String currentSenseId; + private RawSynset currentSynset; + + Parser(String resourceName) { + this.resourceName = resourceName; + } + + void parse(XMLStreamReader reader) throws XMLStreamException { + while (reader.hasNext()) { + final int event = reader.next(); + if (event == XMLStreamConstants.DTD) { + throw malformed(reader.getLocation(), + "Document contains a DOCTYPE declaration; the hardened reader rejects DTDs", null); + } + if (event == XMLStreamConstants.START_ELEMENT) { + startElement(reader); + } else if (event == XMLStreamConstants.END_ELEMENT) { + endElement(reader.getLocalName()); + } + } + } + + private void startElement(XMLStreamReader reader) throws XMLStreamException { + final String name = reader.getLocalName(); + switch (name) { + case "LexicalEntry" -> { + currentEntryId = requireAttribute(reader, "id"); + currentEntryLemma = null; + currentEntryPos = null; + } + case "Lemma" -> { + if (currentEntryId == null) { + throw malformed(reader.getLocation(), "Lemma outside a LexicalEntry", null); + } + currentEntryLemma = requireAttribute(reader, "writtenForm"); + currentEntryPos = parsePos(requireAttribute(reader, "partOfSpeech"), + reader.getLocation()); + lemmaByEntryId.put(currentEntryId, currentEntryLemma); + posByEntryId.put(currentEntryId, currentEntryPos); + } + case "Sense" -> { + if (currentEntryLemma == null) { + throw malformed(reader.getLocation(), + "Sense before its entry's Lemma in LexicalEntry " + currentEntryId, null); + } + currentSenseId = requireAttribute(reader, "id"); + final String synsetId = requireAttribute(reader, "synset"); + synsetBySenseId.put(currentSenseId, synsetId); + entryIdsBySynset.computeIfAbsent(synsetId, unused -> new ArrayList<>(2)) + .add(currentEntryId); + final List order = senseOrder.computeIfAbsent( + InMemoryWordNetLexicon.LemmaKey.of(currentEntryLemma, currentEntryPos), + unused -> new ArrayList<>(2)); + if (!order.contains(synsetId)) { + order.add(synsetId); + } + } + case "SenseRelation" -> { + if (currentSenseId == null) { + throw malformed(reader.getLocation(), "SenseRelation outside a Sense", null); + } + senseRelations.add(new RawSenseRelation(currentSenseId, + requireAttribute(reader, "relType"), requireAttribute(reader, "target"), + line(reader.getLocation()))); + } + case "Synset" -> { + final String id = requireAttribute(reader, "id"); + final WordNetPos pos = parsePos(requireAttribute(reader, "partOfSpeech"), + reader.getLocation()); + currentSynset = new RawSynset(id, pos, reader.getAttributeValue(null, "members"), + line(reader.getLocation())); + if (rawSynsets.putIfAbsent(id, currentSynset) != null) { + throw malformed(reader.getLocation(), "Duplicate synset id " + id, null); + } + } + case "Definition" -> { + if (currentSynset != null && currentSynset.gloss == null) { + currentSynset.gloss = reader.getElementText(); + } + } + case "SynsetRelation" -> { + if (currentSynset == null) { + throw malformed(reader.getLocation(), "SynsetRelation outside a Synset", null); + } + currentSynset.relations.add(new RawRelation(requireAttribute(reader, "relType"), + requireAttribute(reader, "target"), line(reader.getLocation()))); + } + default -> { + // Pronunciation, Form, Example, SyntacticBehaviour, ILIDefinition, and other + // elements outside the contract subset are skipped. + } + } + } + + private void endElement(String name) { + switch (name) { + case "LexicalEntry" -> { + currentEntryId = null; + currentEntryLemma = null; + currentEntryPos = null; + } + case "Sense" -> currentSenseId = null; + case "Synset" -> currentSynset = null; + default -> { + // Nothing to close for skipped elements. + } + } + } + + WordNetLexicon build() { + // Every sense must point to a declared synset, with a consistent part of speech. + for (final Map.Entry sense : synsetBySenseId.entrySet()) { + final RawSynset target = rawSynsets.get(sense.getValue()); + if (target == null) { + throw malformed(null, + "Sense " + sense.getKey() + " references undeclared synset " + sense.getValue(), + null); + } + } + // Lift sense relations to the synset level. + for (final RawSenseRelation relation : senseRelations) { + if (OTHER_RELATION.equals(relation.relType)) { + continue; + } + final String sourceSynsetId = synsetBySenseId.get(relation.sourceSenseId); + final String targetSynsetId = synsetBySenseId.get(relation.targetSenseId); + if (targetSynsetId == null) { + throw malformed(null, "SenseRelation at line " + relation.line + " from sense " + + relation.sourceSenseId + " references undeclared sense " + relation.targetSenseId, + null); + } + final RawSynset source = rawSynsets.get(sourceSynsetId); + source.relations.add(new RawRelation(relation.relType, targetSynsetId, relation.line)); + } + // Resolve raw synsets into contract synsets. + final Map synsetsById = new LinkedHashMap<>(rawSynsets.size() * 2); + for (final RawSynset raw : rawSynsets.values()) { + final Map> relations = resolveRelations(raw); + synsetsById.put(raw.id, + new Synset(raw.id, raw.pos, memberLemmas(raw), raw.gloss == null ? "" : raw.gloss, + relations)); + } + return new InMemoryWordNetLexicon(synsetsById, senseOrder); + } + + private Map> resolveRelations(RawSynset raw) { + final Map> typed = new LinkedHashMap<>(); + for (final RawRelation relation : raw.relations) { + final WordNetRelation type = parseRelation(relation.relType, raw.pos, relation.line); + if (!rawSynsets.containsKey(relation.target)) { + throw malformed(null, "Relation " + relation.relType + " at line " + relation.line + + " on synset " + raw.id + " references undeclared synset " + relation.target, null); + } + typed.computeIfAbsent(type, unused -> new LinkedHashSet<>()).add(relation.target); + } + final Map> relations = new LinkedHashMap<>(typed.size() * 2); + for (final Map.Entry> entry : typed.entrySet()) { + relations.put(entry.getKey(), List.copyOf(entry.getValue())); + } + return relations; + } + + private List memberLemmas(RawSynset raw) { + final List entryIds; + if (raw.members != null && !raw.members.isEmpty()) { + entryIds = splitOnSpaces(raw.members); + } else { + final List fromSenses = entryIdsBySynset.get(raw.id); + entryIds = fromSenses == null ? List.of() : fromSenses; + } + if (entryIds.isEmpty()) { + throw malformed(null, "Synset " + raw.id + " at line " + raw.line + + " has no member entries", null); + } + final List lemmas = new ArrayList<>(entryIds.size()); + for (final String entryId : entryIds) { + final String lemma = lemmaByEntryId.get(entryId); + if (lemma == null) { + throw malformed(null, "Synset " + raw.id + " at line " + raw.line + + " lists undeclared member entry " + entryId, null); + } + if (raw.pos != posByEntryId.get(entryId)) { + throw malformed(null, "Synset " + raw.id + " at line " + raw.line + + " has part of speech " + raw.pos + " but member entry " + entryId + + " has " + posByEntryId.get(entryId), null); + } + if (!lemmas.contains(lemma)) { + lemmas.add(lemma); + } + } + return lemmas; + } + + private WordNetPos parsePos(String code, Location location) { + // The adjective satellite code normalizes to ADJECTIVE; see WordNetPos. + return switch (code) { + case "n" -> WordNetPos.NOUN; + case "v" -> WordNetPos.VERB; + case "a", "s" -> WordNetPos.ADJECTIVE; + case "r" -> WordNetPos.ADVERB; + default -> throw malformed(location, "Unknown part-of-speech code: " + code, null); + }; + } + + private WordNetRelation parseRelation(String relType, WordNetPos sourcePos, int line) { + // Documents derived from Princeton data express verb groups as similar on verb synsets. + if ("similar".equals(relType)) { + return sourcePos == WordNetPos.VERB ? WordNetRelation.VERB_GROUP + : WordNetRelation.SIMILAR_TO; + } + final WordNetRelation relation = RELATION_NAMES.get(relType); + if (relation == null) { + throw malformed(null, "Unknown relation type " + relType + " at line " + line, null); + } + return relation; + } + + private String requireAttribute(XMLStreamReader reader, String attribute) + throws XMLStreamException { + final String value = reader.getAttributeValue(null, attribute); + if (value == null || value.isEmpty()) { + throw malformed(reader.getLocation(), "Element " + reader.getLocalName() + + " is missing required attribute " + attribute, null); + } + return value; + } + + IllegalArgumentException malformed(Location location, String message, Throwable cause) { + final int line = line(location); + final String prefix = line < 0 ? "Malformed WN-LMF document " + resourceName + ": " + : "Malformed WN-LMF document " + resourceName + " at line " + line + ": "; + return cause == null ? new IllegalArgumentException(prefix + message) + : new IllegalArgumentException(prefix + message, cause); + } + + private static int line(Location location) { + return location == null ? -1 : location.getLineNumber(); + } + + private static List splitOnSpaces(String value) { + final List parts = new ArrayList<>(4); + int start = 0; + while (start < value.length()) { + final int space = value.indexOf(' ', start); + if (space < 0) { + parts.add(value.substring(start)); + break; + } + if (space > start) { + parts.add(value.substring(start, space)); + } + start = space + 1; + } + return parts; + } + } + + private static final class RawSynset { + private final String id; + private final WordNetPos pos; + private final String members; + private final int line; + private final List relations = new ArrayList<>(4); + private String gloss; + + RawSynset(String id, WordNetPos pos, String members, int line) { + this.id = id; + this.pos = pos; + this.members = members; + this.line = line; + } + } + + private record RawRelation(String relType, String target, int line) { + } + + private record RawSenseRelation(String sourceSenseId, String relType, String targetSenseId, + int line) { + } + + private static Map relationNames() { + final Map names = new HashMap<>(); + names.put("antonym", WordNetRelation.ANTONYM); + names.put("hypernym", WordNetRelation.HYPERNYM); + names.put("instance_hypernym", WordNetRelation.INSTANCE_HYPERNYM); + names.put("hyponym", WordNetRelation.HYPONYM); + names.put("instance_hyponym", WordNetRelation.INSTANCE_HYPONYM); + names.put("holo_member", WordNetRelation.MEMBER_HOLONYM); + names.put("holo_substance", WordNetRelation.SUBSTANCE_HOLONYM); + names.put("holo_part", WordNetRelation.PART_HOLONYM); + names.put("mero_member", WordNetRelation.MEMBER_MERONYM); + names.put("mero_substance", WordNetRelation.SUBSTANCE_MERONYM); + names.put("mero_part", WordNetRelation.PART_MERONYM); + names.put("attribute", WordNetRelation.ATTRIBUTE); + names.put("derivation", WordNetRelation.DERIVATIONALLY_RELATED); + names.put("entails", WordNetRelation.ENTAILMENT); + names.put("is_entailed_by", WordNetRelation.ENTAILED_BY); + names.put("causes", WordNetRelation.CAUSE); + names.put("is_caused_by", WordNetRelation.CAUSED_BY); + names.put("also", WordNetRelation.ALSO_SEE); + names.put("participle", WordNetRelation.PARTICIPLE); + names.put("pertainym", WordNetRelation.PERTAINYM); + names.put("domain_topic", WordNetRelation.DOMAIN_TOPIC); + names.put("has_domain_topic", WordNetRelation.MEMBER_OF_DOMAIN_TOPIC); + names.put("domain_region", WordNetRelation.DOMAIN_REGION); + names.put("has_domain_region", WordNetRelation.MEMBER_OF_DOMAIN_REGION); + // The usage domain carries both its current WN-LMF name and the legacy alias. + names.put("exemplifies", WordNetRelation.DOMAIN_USAGE); + names.put("domain_usage", WordNetRelation.DOMAIN_USAGE); + names.put("is_exemplified_by", WordNetRelation.MEMBER_OF_DOMAIN_USAGE); + names.put("has_domain_usage", WordNetRelation.MEMBER_OF_DOMAIN_USAGE); + return Map.copyOf(names); + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java new file mode 100644 index 0000000000..b978925ebc --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.wordnet.Synset; +import opennlp.tools.wordnet.WordNetLexicon; +import opennlp.tools.wordnet.WordNetPos; +import opennlp.tools.wordnet.WordNetRelation; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class WnLmfReaderTest { + + static WordNetLexicon fixture() { + try (InputStream in = WnLmfReaderTest.class.getResourceAsStream("mini-wn-lmf.xml")) { + assertNotNull(in, "Fixture mini-wn-lmf.xml must be on the test classpath"); + return WnLmfReader.read(in, "mini-wn-lmf.xml"); + } catch (IOException e) { + throw new IllegalStateException("Unexpected IOException from a classpath stream", e); + } + } + + private static WordNetLexicon parse(String document) { + return WnLmfReader.read( + new ByteArrayInputStream(document.getBytes(StandardCharsets.UTF_8)), "inline.xml"); + } + + private static String wrap(String body) { + return "\n\n" + + "\n" + + body + "\n\n\n"; + } + + @Test + void testLookupReturnsSynsetWithAllComponents() { + final List senses = fixture().lookup("dog", WordNetPos.NOUN); + assertEquals(1, senses.size()); + final Synset dog = senses.get(0); + assertEquals("mini-n1", dog.id()); + assertEquals(WordNetPos.NOUN, dog.pos()); + assertEquals(List.of("dog", "domestic dog"), dog.lemmas()); + assertEquals("a domesticated canid", dog.gloss()); + assertEquals(List.of("mini-n2"), dog.related(WordNetRelation.HYPERNYM)); + } + + @Test + void testLookupFoldsCaseAndUnderscore() { + final WordNetLexicon lexicon = fixture(); + assertEquals("mini-n1", lexicon.lookup("Domestic_Dog", WordNetPos.NOUN).get(0).id()); + assertEquals("mini-n1", lexicon.lookup("DOG", WordNetPos.NOUN).get(0).id()); + } + + @Test + void testLookupKeepsSenseOrder() { + final List runSenses = fixture().lookup("run", WordNetPos.NOUN); + assertEquals(List.of("mini-n5", "mini-n9"), + runSenses.stream().map(Synset::id).toList()); + } + + @Test + void testLookupIsPosScoped() { + final WordNetLexicon lexicon = fixture(); + assertEquals(1, lexicon.lookup("run", WordNetPos.VERB).size()); + assertTrue(lexicon.lookup("dog", WordNetPos.VERB).isEmpty()); + assertFalse(lexicon.contains("walk", WordNetPos.NOUN)); + assertTrue(lexicon.contains("walk", WordNetPos.VERB)); + } + + @Test + void testRelationNavigation() { + final WordNetLexicon lexicon = fixture(); + assertEquals(List.of("mini-n1"), lexicon.related("mini-n2", WordNetRelation.HYPONYM)); + assertEquals(List.of("mini-v1", "mini-v2"), + lexicon.related("mini-v4", WordNetRelation.HYPONYM)); + assertEquals(List.of("mini-v4"), lexicon.related("mini-v1", WordNetRelation.HYPERNYM)); + } + + @Test + void testSenseRelationsAreLiftedToSynsetLevel() { + final WordNetLexicon lexicon = fixture(); + assertEquals(List.of("mini-a2"), lexicon.related("mini-a1", WordNetRelation.ANTONYM)); + assertEquals(List.of("mini-a1"), lexicon.related("mini-a2", WordNetRelation.ANTONYM)); + assertEquals(List.of("mini-v1"), + lexicon.related("mini-n5", WordNetRelation.DERIVATIONALLY_RELATED)); + assertEquals(List.of("mini-n5"), + lexicon.related("mini-v1", WordNetRelation.DERIVATIONALLY_RELATED)); + } + + @Test + void testSatelliteNormalizesToAdjective() { + final List senses = fixture().lookup("large", WordNetPos.ADJECTIVE); + assertEquals(1, senses.size()); + assertEquals(WordNetPos.ADJECTIVE, senses.get(0).pos()); + assertEquals(List.of("mini-a4"), fixture().related("mini-a3", WordNetRelation.SIMILAR_TO)); + assertEquals(List.of("mini-a3"), fixture().related("mini-a4", WordNetRelation.SIMILAR_TO)); + } + + @Test + void testUnknownLemmaOrSynsetIsEmpty() { + final WordNetLexicon lexicon = fixture(); + assertTrue(lexicon.lookup("zebra", WordNetPos.NOUN).isEmpty()); + assertTrue(lexicon.synset("mini-n99").isEmpty()); + } + + @Test + void testReadPath(@TempDir Path tempDir) throws IOException { + final Path file = tempDir.resolve("tiny.xml"); + Files.writeString(file, wrap( + "" + + "" + + "a feline")); + final WordNetLexicon lexicon = WnLmfReader.read(file); + assertEquals("a feline", lexicon.lookup("cat", WordNetPos.NOUN).get(0).gloss()); + } + + @Test + void testReadPathRejectsNullAndMissing(@TempDir Path tempDir) { + assertThrows(IllegalArgumentException.class, () -> WnLmfReader.read((Path) null)); + assertThrows(IllegalArgumentException.class, + () -> WnLmfReader.read(tempDir.resolve("absent.xml"))); + } + + @Test + void testReadStreamRejectsNulls() { + assertThrows(IllegalArgumentException.class, () -> WnLmfReader.read(null, "x")); + assertThrows(IllegalArgumentException.class, + () -> WnLmfReader.read(new ByteArrayInputStream(new byte[0]), null)); + } + + @Test + void testRejectsDoctype() { + final String document = "\n" + + "\n" + + "" + + ""; + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> parse(document)); + assertTrue(e.getMessage().contains("inline.xml")); + } + + @Test + void testRejectsTruncatedDocument() { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> parse("\n parse( + wrap("" + + ""))); + assertTrue(e.getMessage().contains("synset")); + } + + @Test + void testRejectsSenseToUndeclaredSynset() { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> parse( + wrap("" + + ""))); + assertTrue(e.getMessage().contains("t-9")); + } + + @Test + void testRejectsRelationToUndeclaredSynset() { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> parse( + wrap("" + + "" + + "a feline" + + ""))); + assertTrue(e.getMessage().contains("t-9")); + } + + @Test + void testRejectsUnknownRelationType() { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> parse( + wrap("" + + "" + + "a feline" + + ""))); + assertTrue(e.getMessage().contains("quasi_synonym")); + } + + @Test + void testSkipsOtherRelationType() { + final WordNetLexicon lexicon = parse( + wrap("" + + "" + + "" + + "a feline")); + assertTrue(lexicon.synset("t-1").orElseThrow().relations().isEmpty()); + } + + @Test + void testRejectsUnknownPartOfSpeech() { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> parse( + wrap("" + + "" + + "a feline"))); + assertTrue(e.getMessage().contains("x")); + } + + @Test + void testRejectsSynsetWithoutMembers() { + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> parse( + wrap("orphan"))); + assertTrue(e.getMessage().contains("t-1")); + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wn-lmf.xml b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wn-lmf.xml new file mode 100644 index 0000000000..10e039214b --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wn-lmf.xml @@ -0,0 +1,183 @@ + + + + + + + dog + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + a domesticated canid + + the dog barked + + + a carnivorous mammal with nonretractile claws + + + + a small rodent with a long tail + + + + a gnawing mammal with chisel teeth + + + + an act of running at speed + + + a rigid rectangular container + + + a small juicy fruit + + + an adult male person + + + a score made in baseball + + + move fast on foot + + + + move at a regular pace + + + + change location or position + + + change position in space + + + + + of great height + + + of small height + + + of great size + + + + above average in size + + + + with speed + + + in a good or proper manner + + + diff --git a/opennlp-extensions/pom.xml b/opennlp-extensions/pom.xml index 9afcd3fe3c..8c0e432a58 100644 --- a/opennlp-extensions/pom.xml +++ b/opennlp-extensions/pom.xml @@ -41,6 +41,7 @@ opennlp-morfologik opennlp-spellcheck opennlp-uima + opennlp-wordnet \ No newline at end of file From fa2ceed115f7582a52cc87895a88b232aa573cae Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 7 Jul 2026 00:50:19 -0400 Subject: [PATCH 03/12] Add the legacy WNDB reader with reader-equivalence coverage --- .../wordnet/InMemoryWordNetLexicon.java | 7 + .../main/java/opennlp/wordnet/WndbReader.java | 468 ++++++++++++++++++ .../wordnet/LexiconConcurrencyTest.java | 91 ++++ .../wordnet/ReaderEquivalenceTest.java | 121 +++++ .../java/opennlp/wordnet/WndbReaderTest.java | 210 ++++++++ .../opennlp/wordnet/mini-wndb/data.adj | 22 + .../opennlp/wordnet/mini-wndb/data.adv | 20 + .../opennlp/wordnet/mini-wndb/data.noun | 27 + .../opennlp/wordnet/mini-wndb/data.verb | 22 + .../opennlp/wordnet/mini-wndb/index.adj | 22 + .../opennlp/wordnet/mini-wndb/index.adv | 20 + .../opennlp/wordnet/mini-wndb/index.noun | 27 + .../opennlp/wordnet/mini-wndb/index.verb | 22 + 13 files changed, 1079 insertions(+) create mode 100644 opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexiconConcurrencyTest.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/ReaderEquivalenceTest.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WndbReaderTest.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.adj create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.adv create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.noun create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.verb create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.adj create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.adv create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.noun create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.verb diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java index e81fe4b768..b22f5aa526 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java @@ -17,6 +17,8 @@ package opennlp.wordnet; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -128,6 +130,11 @@ int size() { return synsetsById.size(); } + /** {@return all synsets, for equivalence checks and diagnostics within this package} */ + Collection synsets() { + return Collections.unmodifiableCollection(synsetsById.values()); + } + /** * A folded sense-index key. Build with {@link #of(String, WordNetPos)} so every key passes * through the same fold as every query. diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java new file mode 100644 index 0000000000..5725c04717 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java @@ -0,0 +1,468 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; + +import opennlp.tools.wordnet.Synset; +import opennlp.tools.wordnet.WordNetLexicon; +import opennlp.tools.wordnet.WordNetPos; +import opennlp.tools.wordnet.WordNetRelation; + +/** + * Reads a legacy Princeton WNDB database directory ({@code index.noun}, {@code data.noun}, and + * the corresponding pairs for verbs, adjectives, and adverbs) into a {@link WordNetLexicon}. + * + *

The reader is clean-room, built from the published {@code wndb(5WN)} and + * {@code wninput(5WN)} format documentation, with no third-party WordNet library involved. All + * eight files must be present in the directory. License preamble lines (which begin with a + * space in the released files) are skipped in index and data files. {@code index.sense} is not + * read: the v1 contract has no sense-key surface, so sense keys belong to the later sense + * inventory layer. The {@code *.exc} morphological exception lists in the same directory are + * the {@link MorphyLemmatizer} companion input and are read separately.

+ * + *

Synset ids are minted as {@code wndb-}offset{@code -}pos from the data + * file's 8-digit byte offset and part-of-speech letter, for example {@code wndb-00001740-n}. + * The id is opaque to consumers; the format mirrors WN-LMF-style ids only for readability. + * Adjective satellite lines ({@code ss_type} {@code s}) normalize to + * {@link WordNetPos#ADJECTIVE}, and the syntactic markers the adjective files append to some + * words ({@code (p)}, {@code (a)}, {@code (ip)}) are stripped from lemmas. Underscores in + * stored lemmas become spaces. Sense order per lemma follows the index file's offset order, + * which the format documents as most frequent first.

+ * + *

Errors. Malformed content fails loud with an {@link IllegalArgumentException} + * naming the file and line: a missing file, a truncated or overlong line, a data line whose + * offset field disagrees with its actual byte position (the format's documented seek + * contract), an index entry referencing an offset with no data line, an undeclared pointer + * symbol, or a pointer to a nonexistent target.

+ * + *

The returned lexicon is immutable and safe for concurrent lookups.

+ */ +public final class WndbReader { + + private static final Map POINTER_SYMBOLS = pointerSymbols(); + + private WndbReader() { + } + + /** + * Reads a WNDB database directory. + * + * @param directory The directory containing the eight index and data files. Must not be + * {@code null} and must exist. + * @return The loaded lexicon. + * @throws IllegalArgumentException Thrown if {@code directory} is {@code null} or not a + * directory, a database file is missing, or any file is malformed; the message names the + * file and line. + * @throws UncheckedIOException Thrown if reading a file fails. + */ + public static WordNetLexicon read(Path directory) { + if (directory == null) { + throw new IllegalArgumentException("Directory must not be null"); + } + if (!Files.isDirectory(directory)) { + throw new IllegalArgumentException( + "Directory does not exist or is not a directory: " + directory); + } + final Map rawSynsets = new LinkedHashMap<>(); + for (final FilePos filePos : FilePos.values()) { + parseDataFile(directory, filePos, rawSynsets); + } + final Map synsetsById = resolve(rawSynsets); + final Map> senseOrder = new LinkedHashMap<>(); + for (final FilePos filePos : FilePos.values()) { + parseIndexFile(directory, filePos, rawSynsets, senseOrder); + } + return new InMemoryWordNetLexicon(synsetsById, senseOrder); + } + + // The four part-of-speech file pairs of a WNDB directory. + private enum FilePos { + NOUN("noun", 'n', WordNetPos.NOUN), + VERB("verb", 'v', WordNetPos.VERB), + ADJECTIVE("adj", 'a', WordNetPos.ADJECTIVE), + ADVERB("adv", 'r', WordNetPos.ADVERB); + + private final String suffix; + private final char posChar; + private final WordNetPos pos; + + FilePos(String suffix, char posChar, WordNetPos pos) { + this.suffix = suffix; + this.posChar = posChar; + this.pos = pos; + } + } + + private static void parseDataFile(Path directory, FilePos filePos, + Map rawSynsets) { + final String fileName = "data." + filePos.suffix; + final byte[] bytes = readAll(directory.resolve(fileName), fileName); + int lineStart = 0; + int lineNumber = 0; + while (lineStart < bytes.length) { + lineNumber++; + int lineEnd = lineStart; + while (lineEnd < bytes.length && bytes[lineEnd] != '\n') { + lineEnd++; + } + // ISO-8859-1 decodes bytes one-to-one, keeping offsets exact for any released file. + final String line = + new String(bytes, lineStart, lineEnd - lineStart, StandardCharsets.ISO_8859_1); + if (!line.isEmpty() && line.charAt(0) != ' ') { + parseDataLine(line, lineStart, fileName, lineNumber, filePos, rawSynsets); + } + lineStart = lineEnd + 1; + } + } + + private static void parseDataLine(String line, int byteOffset, String fileName, int lineNumber, + FilePos filePos, Map rawSynsets) { + final Tokenizer tokens = new Tokenizer(line, fileName, lineNumber); + final String offsetField = tokens.next("synset_offset"); + if (parseOffset(offsetField, tokens) != byteOffset) { + throw malformed(fileName, lineNumber, "Synset offset field " + offsetField + + " disagrees with the actual byte position " + byteOffset); + } + tokens.next("lex_filenum"); + final String ssType = tokens.next("ss_type"); + final boolean validType = switch (filePos) { + case ADJECTIVE -> "a".equals(ssType) || "s".equals(ssType); + default -> ssType.length() == 1 && ssType.charAt(0) == filePos.posChar; + }; + if (!validType) { + throw malformed(fileName, lineNumber, + "Synset type " + ssType + " does not belong in " + fileName); + } + final int wordCount = tokens.nextInt("w_cnt", 16); + if (wordCount < 1) { + throw malformed(fileName, lineNumber, "Word count must be at least 1, got: " + wordCount); + } + final List lemmas = new ArrayList<>(wordCount); + for (int i = 0; i < wordCount; i++) { + final String lemma = cleanLemma(tokens.next("word"), fileName, lineNumber); + tokens.nextInt("lex_id", 16); + if (!lemmas.contains(lemma)) { + lemmas.add(lemma); + } + } + final int pointerCount = tokens.nextInt("p_cnt", 10); + final List pointers = new ArrayList<>(pointerCount); + for (int i = 0; i < pointerCount; i++) { + final String symbol = tokens.next("pointer_symbol"); + final WordNetRelation relation = POINTER_SYMBOLS.get(symbol); + if (relation == null) { + throw malformed(fileName, lineNumber, "Undeclared pointer symbol: " + symbol); + } + final String targetOffset = tokens.next("pointer synset_offset"); + parseOffset(targetOffset, tokens); + final char targetPos = posChar(tokens.next("pointer pos"), tokens); + tokens.next("pointer source/target"); + pointers.add(new RawPointer(relation, "wndb-" + targetOffset + "-" + targetPos, lineNumber)); + } + if (filePos == FilePos.VERB) { + final int frameCount = tokens.nextInt("f_cnt", 10); + for (int i = 0; i < frameCount; i++) { + tokens.next("frame marker"); + tokens.next("f_num"); + tokens.next("w_num"); + } + } + final String gloss = tokens.gloss(); + final String id = "wndb-" + offsetField + "-" + filePos.posChar; + rawSynsets.put(id, new RawSynset(id, filePos.pos, lemmas, gloss, pointers, + fileName, lineNumber)); + } + + private static Map resolve(Map rawSynsets) { + final Map synsetsById = new LinkedHashMap<>(rawSynsets.size() * 2); + for (final RawSynset raw : rawSynsets.values()) { + final Map> typed = new LinkedHashMap<>(); + for (final RawPointer pointer : raw.pointers) { + if (!rawSynsets.containsKey(pointer.targetId)) { + throw malformed(raw.fileName, raw.lineNumber, "Synset " + raw.id + " has a " + + pointer.relation + " pointer to nonexistent synset " + pointer.targetId); + } + typed.computeIfAbsent(pointer.relation, unused -> new LinkedHashSet<>()) + .add(pointer.targetId); + } + final Map> relations = new LinkedHashMap<>(typed.size() * 2); + for (final Map.Entry> entry : typed.entrySet()) { + relations.put(entry.getKey(), List.copyOf(entry.getValue())); + } + synsetsById.put(raw.id, new Synset(raw.id, raw.pos, raw.lemmas, raw.gloss, relations)); + } + return synsetsById; + } + + private static void parseIndexFile(Path directory, FilePos filePos, + Map rawSynsets, + Map> senses) { + final String fileName = "index." + filePos.suffix; + final byte[] bytes = readAll(directory.resolve(fileName), fileName); + final String content = new String(bytes, StandardCharsets.ISO_8859_1); + int lineNumber = 0; + int lineStart = 0; + while (lineStart < content.length()) { + lineNumber++; + int lineEnd = content.indexOf('\n', lineStart); + if (lineEnd < 0) { + lineEnd = content.length(); + } + final String line = content.substring(lineStart, lineEnd); + if (!line.isEmpty() && line.charAt(0) != ' ') { + parseIndexLine(line, fileName, lineNumber, filePos, rawSynsets, senses); + } + lineStart = lineEnd + 1; + } + } + + private static void parseIndexLine(String line, String fileName, int lineNumber, + FilePos filePos, Map rawSynsets, + Map> senses) { + final Tokenizer tokens = new Tokenizer(line, fileName, lineNumber); + final String lemma = tokens.next("lemma"); + final String pos = tokens.next("pos"); + if (pos.length() != 1 || pos.charAt(0) != filePos.posChar) { + throw malformed(fileName, lineNumber, "Index pos " + pos + " does not belong in " + + fileName); + } + final int synsetCount = tokens.nextInt("synset_cnt", 10); + if (synsetCount < 1) { + throw malformed(fileName, lineNumber, + "Synset count must be at least 1, got: " + synsetCount); + } + final int pointerTypeCount = tokens.nextInt("p_cnt", 10); + for (int i = 0; i < pointerTypeCount; i++) { + // The index file's pointer-type summary is informational; the typed pointers in the + // data file are authoritative, so the summary symbols are not validated here. + tokens.next("ptr_symbol"); + } + tokens.next("sense_cnt"); + tokens.next("tagsense_cnt"); + final List order = new ArrayList<>(synsetCount); + for (int i = 0; i < synsetCount; i++) { + final String offset = tokens.next("synset_offset"); + parseOffset(offset, tokens); + final String synsetId = "wndb-" + offset + "-" + filePos.posChar; + if (!rawSynsets.containsKey(synsetId)) { + throw malformed(fileName, lineNumber, "Lemma " + lemma + " references offset " + offset + + " with no data." + filePos.suffix + " line"); + } + if (!order.contains(synsetId)) { + order.add(synsetId); + } + } + final InMemoryWordNetLexicon.LemmaKey key = + InMemoryWordNetLexicon.LemmaKey.of(lemma, filePos.pos); + final List existing = senses.get(key); + if (existing == null) { + senses.put(key, order); + } else { + // Two index lemmas can fold to one key; keep first-listed order and append the rest. + for (final String synsetId : order) { + if (!existing.contains(synsetId)) { + existing.add(synsetId); + } + } + } + } + + // Strips the documented adjective syntactic markers and turns underscores into spaces. + private static String cleanLemma(String word, String fileName, int lineNumber) { + String cleaned = word; + if (cleaned.endsWith(")")) { + final int open = cleaned.lastIndexOf('('); + final String marker = open < 0 ? "" : cleaned.substring(open); + if (!"(p)".equals(marker) && !"(a)".equals(marker) && !"(ip)".equals(marker)) { + throw malformed(fileName, lineNumber, "Unknown syntactic marker on word: " + word); + } + cleaned = cleaned.substring(0, open); + } + if (cleaned.isEmpty()) { + throw malformed(fileName, lineNumber, "Empty word field"); + } + return cleaned.replace('_', ' '); + } + + private static int parseOffset(String offset, Tokenizer tokens) { + if (offset.length() != 8) { + throw tokens.malformedToken("Synset offset must be 8 digits, got: " + offset); + } + int value = 0; + for (int i = 0; i < 8; i++) { + final char c = offset.charAt(i); + if (c < '0' || c > '9') { + throw tokens.malformedToken("Synset offset must be 8 digits, got: " + offset); + } + value = value * 10 + (c - '0'); + } + return value; + } + + private static char posChar(String pos, Tokenizer tokens) { + if (pos.length() == 1) { + final char c = pos.charAt(0); + if (c == 'n' || c == 'v' || c == 'a' || c == 'r') { + return c; + } + } + throw tokens.malformedToken("Pointer pos must be one of n, v, a, r, got: " + pos); + } + + private static byte[] readAll(Path file, String fileName) { + if (!Files.isRegularFile(file)) { + throw new IllegalArgumentException("Missing WNDB database file: " + file); + } + try { + return Files.readAllBytes(file); + } catch (IOException e) { + throw new UncheckedIOException("Unable to read WNDB database file " + file, e); + } + } + + private static IllegalArgumentException malformed(String fileName, int lineNumber, + String message) { + return new IllegalArgumentException( + "Malformed WNDB file " + fileName + " at line " + lineNumber + ": " + message); + } + + // A cursor over one line's space-separated fields; no regular expressions involved. + private static final class Tokenizer { + + private final String line; + private final String fileName; + private final int lineNumber; + private int position; + + Tokenizer(String line, String fileName, int lineNumber) { + this.line = line; + this.fileName = fileName; + this.lineNumber = lineNumber; + } + + String next(String field) { + while (position < line.length() && line.charAt(position) == ' ') { + position++; + } + if (position >= line.length()) { + throw malformed(fileName, lineNumber, "Truncated line, missing field: " + field); + } + final int start = position; + while (position < line.length() && line.charAt(position) != ' ') { + position++; + } + return line.substring(start, position); + } + + int nextInt(String field, int radix) { + final String token = next(field); + try { + return Integer.parseInt(token, radix); + } catch (NumberFormatException e) { + throw new IllegalArgumentException(malformed(fileName, lineNumber, + "Field " + field + " is not a base-" + radix + " integer: " + token).getMessage(), e); + } + } + + // The remainder after the pipe separator, trimmed of the surrounding spaces. + String gloss() { + final String separator = next("gloss separator"); + if (!"|".equals(separator)) { + throw malformed(fileName, lineNumber, "Expected the | gloss separator, got: " + separator); + } + int start = position; + while (start < line.length() && line.charAt(start) == ' ') { + start++; + } + int end = line.length(); + while (end > start && line.charAt(end - 1) == ' ') { + end--; + } + return line.substring(start, end); + } + + IllegalArgumentException malformedToken(String message) { + return malformed(fileName, lineNumber, message); + } + } + + private record RawPointer(WordNetRelation relation, String targetId, int lineNumber) { + } + + private static final class RawSynset { + private final String id; + private final WordNetPos pos; + private final List lemmas; + private final String gloss; + private final List pointers; + private final String fileName; + private final int lineNumber; + + RawSynset(String id, WordNetPos pos, List lemmas, String gloss, + List pointers, String fileName, int lineNumber) { + this.id = id; + this.pos = pos; + this.lemmas = lemmas; + this.gloss = gloss; + this.pointers = pointers; + this.fileName = fileName; + this.lineNumber = lineNumber; + } + } + + private static Map pointerSymbols() { + final Map symbols = new HashMap<>(); + symbols.put("!", WordNetRelation.ANTONYM); + symbols.put("@", WordNetRelation.HYPERNYM); + symbols.put("@i", WordNetRelation.INSTANCE_HYPERNYM); + symbols.put("~", WordNetRelation.HYPONYM); + symbols.put("~i", WordNetRelation.INSTANCE_HYPONYM); + symbols.put("#m", WordNetRelation.MEMBER_HOLONYM); + symbols.put("#s", WordNetRelation.SUBSTANCE_HOLONYM); + symbols.put("#p", WordNetRelation.PART_HOLONYM); + symbols.put("%m", WordNetRelation.MEMBER_MERONYM); + symbols.put("%s", WordNetRelation.SUBSTANCE_MERONYM); + symbols.put("%p", WordNetRelation.PART_MERONYM); + symbols.put("=", WordNetRelation.ATTRIBUTE); + symbols.put("+", WordNetRelation.DERIVATIONALLY_RELATED); + symbols.put("*", WordNetRelation.ENTAILMENT); + symbols.put(">", WordNetRelation.CAUSE); + symbols.put("^", WordNetRelation.ALSO_SEE); + symbols.put("$", WordNetRelation.VERB_GROUP); + symbols.put("&", WordNetRelation.SIMILAR_TO); + symbols.put("<", WordNetRelation.PARTICIPLE); + symbols.put("\\", WordNetRelation.PERTAINYM); + symbols.put(";c", WordNetRelation.DOMAIN_TOPIC); + symbols.put("-c", WordNetRelation.MEMBER_OF_DOMAIN_TOPIC); + symbols.put(";r", WordNetRelation.DOMAIN_REGION); + symbols.put("-r", WordNetRelation.MEMBER_OF_DOMAIN_REGION); + symbols.put(";u", WordNetRelation.DOMAIN_USAGE); + symbols.put("-u", WordNetRelation.MEMBER_OF_DOMAIN_USAGE); + return Map.copyOf(symbols); + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexiconConcurrencyTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexiconConcurrencyTest.java new file mode 100644 index 0000000000..a94709145d --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexiconConcurrencyTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.wordnet.WordNetLexicon; +import opennlp.tools.wordnet.WordNetPos; +import opennlp.tools.wordnet.WordNetRelation; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Exercises the immutable-after-load contract: one loaded lexicon serves many threads issuing + * concurrent lookups, and every thread observes exactly the single-threaded results. + */ +public class LexiconConcurrencyTest { + + private static final int THREADS = 8; + private static final int ITERATIONS = 500; + + @Test + void testConcurrentLookupsSeeConsistentResults() throws InterruptedException { + final WordNetLexicon lexicon = WndbReaderTest.fixture(); + final CountDownLatch start = new CountDownLatch(1); + final CountDownLatch done = new CountDownLatch(THREADS); + final Queue problems = new ConcurrentLinkedQueue<>(); + for (int t = 0; t < THREADS; t++) { + final Thread thread = new Thread(() -> { + try { + start.await(); + for (int i = 0; i < ITERATIONS; i++) { + verifyOnce(lexicon, problems); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + problems.add("Interrupted: " + e); + } catch (RuntimeException e) { + problems.add("Unexpected exception: " + e); + } finally { + done.countDown(); + } + }); + thread.setDaemon(true); + thread.start(); + } + start.countDown(); + assertTrue(done.await(60, TimeUnit.SECONDS), "Worker threads must finish in time"); + assertEquals(List.of(), List.copyOf(problems)); + } + + private static void verifyOnce(WordNetLexicon lexicon, Queue problems) { + if (!"wndb-00001075-n".equals(lexicon.lookup("dog", WordNetPos.NOUN).get(0).id())) { + problems.add("Wrong dog lookup"); + } + if (lexicon.lookup("run", WordNetPos.NOUN).size() != 2) { + problems.add("Wrong run sense count"); + } + if (!List.of("wndb-00001160-n") + .equals(lexicon.related("wndb-00001075-n", WordNetRelation.HYPERNYM))) { + problems.add("Wrong dog hypernym"); + } + if (lexicon.contains("zebra", WordNetPos.NOUN)) { + problems.add("Phantom zebra"); + } + if (!lexicon.contains("walk", WordNetPos.VERB)) { + problems.add("Missing walk verb"); + } + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/ReaderEquivalenceTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/ReaderEquivalenceTest.java new file mode 100644 index 0000000000..c212738a03 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/ReaderEquivalenceTest.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.wordnet.Synset; +import opennlp.tools.wordnet.WordNetLexicon; +import opennlp.tools.wordnet.WordNetPos; +import opennlp.tools.wordnet.WordNetRelation; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * Asserts that the WN-LMF fixture and the WNDB fixture, which encode the same miniature + * wordnet, load into equivalent lexicon views. Synset ids are reader-minted and intentionally + * differ, so the comparison is structural, joining synsets on their glosses (unique within the + * fixtures) and comparing everything else through that join. + */ +public class ReaderEquivalenceTest { + + @Test + void testBothReadersProduceEquivalentViews() { + final InMemoryWordNetLexicon lmf = (InMemoryWordNetLexicon) WnLmfReaderTest.fixture(); + final InMemoryWordNetLexicon wndb = (InMemoryWordNetLexicon) WndbReaderTest.fixture(); + assertEquals(lmf.size(), wndb.size(), "Both fixtures encode the same synsets"); + + final Map wndbByGloss = byGloss(wndb); + assertEquals(byGloss(lmf).keySet(), wndbByGloss.keySet(), "Same glosses on both sides"); + + for (final Synset expected : lmf.synsets()) { + final Synset actual = wndbByGloss.get(expected.gloss()); + assertNotNull(actual, "WNDB view has a synset for gloss: " + expected.gloss()); + assertEquals(expected.pos(), actual.pos(), "Part of speech for: " + expected.gloss()); + assertEquals(expected.lemmas(), actual.lemmas(), "Lemmas for: " + expected.gloss()); + assertEquals(relationsByGloss(expected, lmf), relationsByGloss(actual, wndb), + "Relations for: " + expected.gloss()); + } + } + + @Test + void testLookupAgreesForEveryLemmaAndPos() { + final WordNetLexicon lmf = WnLmfReaderTest.fixture(); + final InMemoryWordNetLexicon wndb = (InMemoryWordNetLexicon) WndbReaderTest.fixture(); + final Set checked = new HashSet<>(); + for (final Synset synset : wndb.synsets()) { + for (final String lemma : synset.lemmas()) { + if (!checked.add(lemma + "/" + synset.pos())) { + continue; + } + assertEquals( + glosses(lmf.lookup(lemma, synset.pos())), + glosses(wndb.lookup(lemma, synset.pos())), + "Sense sequence for " + lemma + " as " + synset.pos()); + } + } + for (final WordNetPos pos : WordNetPos.values()) { + assertEquals(lmf.contains("dog", pos), wndb.contains("dog", pos)); + } + } + + @Test + void testSenseOrderAgreesForMultiSenseLemma() { + final WordNetLexicon lmf = WnLmfReaderTest.fixture(); + final WordNetLexicon wndb = WndbReaderTest.fixture(); + final List lmfOrder = glosses(lmf.lookup("run", WordNetPos.NOUN)); + final List wndbOrder = glosses(wndb.lookup("run", WordNetPos.NOUN)); + assertEquals(2, lmfOrder.size()); + assertEquals(lmfOrder, wndbOrder); + } + + private static Map byGloss(InMemoryWordNetLexicon lexicon) { + final Map byGloss = new HashMap<>(); + for (final Synset synset : lexicon.synsets()) { + final Synset previous = byGloss.put(synset.gloss(), synset); + assertEquals(null, previous, "Fixture glosses must be unique, duplicated: " + + synset.gloss()); + } + return byGloss; + } + + // A synset's relations with targets replaced by their glosses, id-scheme independent. + private static Map> relationsByGloss(Synset synset, + WordNetLexicon lexicon) { + final Map> result = new HashMap<>(); + for (final Map.Entry> relation : + synset.relations().entrySet()) { + final Set targetGlosses = new HashSet<>(); + for (final String targetId : relation.getValue()) { + targetGlosses.add(lexicon.synset(targetId).orElseThrow().gloss()); + } + result.put(relation.getKey(), targetGlosses); + } + return result; + } + + private static List glosses(List synsets) { + return synsets.stream().map(Synset::gloss).toList(); + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WndbReaderTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WndbReaderTest.java new file mode 100644 index 0000000000..559914820f --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WndbReaderTest.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.function.UnaryOperator; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.wordnet.Synset; +import opennlp.tools.wordnet.WordNetLexicon; +import opennlp.tools.wordnet.WordNetPos; +import opennlp.tools.wordnet.WordNetRelation; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class WndbReaderTest { + + private static final String DOG_ID = "wndb-00001075-n"; + private static final String CANID_ID = "wndb-00001160-n"; + + static Path fixtureDirectory() { + final URL url = WndbReaderTest.class.getResource("mini-wndb"); + assertNotNull(url, "Fixture directory mini-wndb must be on the test classpath"); + try { + return Path.of(url.toURI()); + } catch (URISyntaxException e) { + throw new IllegalStateException("Unexpected fixture URI: " + url, e); + } + } + + static WordNetLexicon fixture() { + return WndbReader.read(fixtureDirectory()); + } + + @Test + void testLookupReturnsSynsetWithAllComponents() { + final List senses = fixture().lookup("dog", WordNetPos.NOUN); + assertEquals(1, senses.size()); + final Synset dog = senses.get(0); + assertEquals(DOG_ID, dog.id()); + assertEquals(WordNetPos.NOUN, dog.pos()); + assertEquals(List.of("dog", "domestic dog"), dog.lemmas()); + assertEquals("a domesticated canid", dog.gloss()); + assertEquals(List.of(CANID_ID), dog.related(WordNetRelation.HYPERNYM)); + } + + @Test + void testLookupFoldsCaseAndUnderscore() { + final WordNetLexicon lexicon = fixture(); + assertEquals(DOG_ID, lexicon.lookup("Domestic_Dog", WordNetPos.NOUN).get(0).id()); + assertEquals(DOG_ID, lexicon.lookup("DOG", WordNetPos.NOUN).get(0).id()); + } + + @Test + void testLookupKeepsIndexSenseOrder() { + assertEquals(List.of("wndb-00001427-n", "wndb-00001669-n"), + fixture().lookup("run", WordNetPos.NOUN).stream().map(Synset::id).toList()); + } + + @Test + void testRelationNavigation() { + final WordNetLexicon lexicon = fixture(); + assertEquals(List.of(DOG_ID), lexicon.related(CANID_ID, WordNetRelation.HYPONYM)); + assertEquals(List.of("wndb-00001075-v", "wndb-00001171-v"), + lexicon.related("wndb-00001324-v", WordNetRelation.HYPONYM)); + assertEquals(List.of("wndb-00001075-v"), + lexicon.related("wndb-00001427-n", WordNetRelation.DERIVATIONALLY_RELATED)); + } + + @Test + void testLexicalPointersSurfaceAtSynsetLevel() { + final WordNetLexicon lexicon = fixture(); + assertEquals(List.of("wndb-00001141-a"), + lexicon.related("wndb-00001075-a", WordNetRelation.ANTONYM)); + assertEquals(List.of("wndb-00001075-a"), + lexicon.related("wndb-00001141-a", WordNetRelation.ANTONYM)); + } + + @Test + void testSatelliteNormalizesToAdjectiveAndMarkerIsStripped() { + final WordNetLexicon lexicon = fixture(); + final Synset large = lexicon.lookup("large", WordNetPos.ADJECTIVE).get(0); + assertEquals(WordNetPos.ADJECTIVE, large.pos()); + assertEquals(List.of("wndb-00001211-a"), large.related(WordNetRelation.SIMILAR_TO)); + // short is stored as short(p); the syntactic marker is not part of the lemma. + assertEquals(List.of("short"), + lexicon.lookup("short", WordNetPos.ADJECTIVE).get(0).lemmas()); + } + + @Test + void testUnknownLemmaOrSynsetIsEmpty() { + final WordNetLexicon lexicon = fixture(); + assertTrue(lexicon.lookup("zebra", WordNetPos.NOUN).isEmpty()); + assertTrue(lexicon.synset("wndb-99999999-n").isEmpty()); + } + + @Test + void testRejectsNullAndMissingDirectory(@TempDir Path tempDir) { + assertThrows(IllegalArgumentException.class, () -> WndbReader.read(null)); + assertThrows(IllegalArgumentException.class, + () -> WndbReader.read(tempDir.resolve("absent"))); + } + + @Test + void testRejectsMissingDatabaseFile(@TempDir Path tempDir) throws IOException { + copyFixture(tempDir); + Files.delete(tempDir.resolve("data.verb")); + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> WndbReader.read(tempDir)); + assertTrue(e.getMessage().contains("data.verb")); + } + + @Test + void testRejectsIndexOffsetWithoutDataLine(@TempDir Path tempDir) throws IOException { + copyFixture(tempDir); + mutate(tempDir, "index.noun", line -> line.startsWith("berry ") + ? line.replace("00001564", "00001565") : line); + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> WndbReader.read(tempDir)); + assertTrue(e.getMessage().contains("berry")); + assertTrue(e.getMessage().contains("00001565")); + } + + @Test + void testRejectsDataOffsetFieldMismatch(@TempDir Path tempDir) throws IOException { + copyFixture(tempDir); + mutate(tempDir, "data.noun", + line -> line.replace("00001503 03 n 01 box", "00001504 03 n 01 box")); + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> WndbReader.read(tempDir)); + assertTrue(e.getMessage().contains("disagrees")); + } + + @Test + void testRejectsTruncatedDataLine(@TempDir Path tempDir) throws IOException { + copyFixture(tempDir); + mutate(tempDir, "data.noun", line -> line.startsWith("00001564") + ? line.substring(0, line.indexOf(" 000 |")) : line); + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> WndbReader.read(tempDir)); + assertTrue(e.getMessage().contains("data.noun")); + assertTrue(e.getMessage().contains("Truncated")); + } + + @Test + void testRejectsUndeclaredPointerSymbol(@TempDir Path tempDir) throws IOException { + copyFixture(tempDir); + mutate(tempDir, "data.noun", line -> line.replace("001 @ 00001160 n 0000", + "001 ? 00001160 n 0000")); + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> WndbReader.read(tempDir)); + assertTrue(e.getMessage().contains("Undeclared pointer symbol: ?")); + } + + @Test + void testRejectsPointerToNonexistentSynset(@TempDir Path tempDir) throws IOException { + copyFixture(tempDir); + mutate(tempDir, "data.noun", line -> line.replace("001 @ 00001160 n 0000", + "001 @ 00009999 n 0000")); + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> WndbReader.read(tempDir)); + assertTrue(e.getMessage().contains("wndb-00009999-n")); + } + + private static void copyFixture(Path target) throws IOException { + try (var files = Files.list(fixtureDirectory())) { + for (final Path file : files.toList()) { + Files.copy(file, target.resolve(file.getFileName().toString())); + } + } + } + + // Applies a line transformation to one fixture file. The mutations only ever keep or shrink + // line lengths of the affected line's own fields, so surrounding offsets stay valid. + private static void mutate(Path directory, String fileName, UnaryOperator edit) + throws IOException { + final Path file = directory.resolve(fileName); + final List lines = Files.readAllLines(file, StandardCharsets.ISO_8859_1); + final StringBuilder out = new StringBuilder(); + for (final String line : lines) { + out.append(edit.apply(line)).append('\n'); + } + Files.writeString(file, out.toString(), StandardCharsets.ISO_8859_1); + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.adj b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.adj new file mode 100644 index 0000000000..1340d16a45 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.adj @@ -0,0 +1,22 @@ + 1 Licensed to the Apache Software Foundation (ASF) under one or more + 2 contributor license agreements. See the NOTICE file distributed with + 3 this work for additional information regarding copyright ownership. + 4 The ASF licenses this file to You under the Apache License, Version 2.0 + 5 (the "License"); you may not use this file except in compliance with + 6 the License. You may obtain a copy of the License at + 7 + 8 http://www.apache.org/licenses/LICENSE-2.0 + 9 + 10 Unless required by applicable law or agreed to in writing, software + 11 distributed under the License is distributed on an "AS IS" BASIS, + 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + 13 See the License for the specific language governing permissions and + 14 limitations under the License. + 15 + 16 Project-authored miniature WNDB fixture mirroring mini-wn-lmf.xml. + 17 License preamble lines begin with two spaces, as in released WNDB files, + 18 so readers skip them; data line offsets include this preamble. +00001075 00 a 01 tall 0 001 ! 00001141 a 0101 | of great height +00001141 00 a 01 short(p) 0 001 ! 00001075 a 0101 | of small height +00001211 00 a 01 big 0 001 & 00001274 a 0000 | of great size +00001274 00 s 01 large 0 001 & 00001211 a 0000 | above average in size diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.adv b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.adv new file mode 100644 index 0000000000..732c21dfd5 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.adv @@ -0,0 +1,20 @@ + 1 Licensed to the Apache Software Foundation (ASF) under one or more + 2 contributor license agreements. See the NOTICE file distributed with + 3 this work for additional information regarding copyright ownership. + 4 The ASF licenses this file to You under the Apache License, Version 2.0 + 5 (the "License"); you may not use this file except in compliance with + 6 the License. You may obtain a copy of the License at + 7 + 8 http://www.apache.org/licenses/LICENSE-2.0 + 9 + 10 Unless required by applicable law or agreed to in writing, software + 11 distributed under the License is distributed on an "AS IS" BASIS, + 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + 13 See the License for the specific language governing permissions and + 14 limitations under the License. + 15 + 16 Project-authored miniature WNDB fixture mirroring mini-wn-lmf.xml. + 17 License preamble lines begin with two spaces, as in released WNDB files, + 18 so readers skip them; data line offsets include this preamble. +00001075 02 r 01 quickly 0 000 | with speed +00001121 02 r 01 well 0 000 | in a good or proper manner diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.noun b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.noun new file mode 100644 index 0000000000..0598111bf1 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.noun @@ -0,0 +1,27 @@ + 1 Licensed to the Apache Software Foundation (ASF) under one or more + 2 contributor license agreements. See the NOTICE file distributed with + 3 this work for additional information regarding copyright ownership. + 4 The ASF licenses this file to You under the Apache License, Version 2.0 + 5 (the "License"); you may not use this file except in compliance with + 6 the License. You may obtain a copy of the License at + 7 + 8 http://www.apache.org/licenses/LICENSE-2.0 + 9 + 10 Unless required by applicable law or agreed to in writing, software + 11 distributed under the License is distributed on an "AS IS" BASIS, + 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + 13 See the License for the specific language governing permissions and + 14 limitations under the License. + 15 + 16 Project-authored miniature WNDB fixture mirroring mini-wn-lmf.xml. + 17 License preamble lines begin with two spaces, as in released WNDB files, + 18 so readers skip them; data line offsets include this preamble. +00001075 03 n 02 dog 0 domestic_dog 0 001 @ 00001160 n 0000 | a domesticated canid +00001160 03 n 01 canid 0 001 ~ 00001075 n 0000 | a carnivorous mammal with nonretractile claws +00001257 03 n 01 mouse 0 001 @ 00001340 n 0000 | a small rodent with a long tail +00001340 03 n 01 rodent 0 001 ~ 00001257 n 0000 | a gnawing mammal with chisel teeth +00001427 03 n 01 run 0 001 + 00001075 v 0101 | an act of running at speed +00001503 03 n 01 box 0 000 | a rigid rectangular container +00001564 03 n 01 berry 0 000 | a small juicy fruit +00001617 03 n 01 man 0 000 | an adult male person +00001669 03 n 01 run 0 000 | a score made in baseball diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.verb b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.verb new file mode 100644 index 0000000000..048546ed71 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/data.verb @@ -0,0 +1,22 @@ + 1 Licensed to the Apache Software Foundation (ASF) under one or more + 2 contributor license agreements. See the NOTICE file distributed with + 3 this work for additional information regarding copyright ownership. + 4 The ASF licenses this file to You under the Apache License, Version 2.0 + 5 (the "License"); you may not use this file except in compliance with + 6 the License. You may obtain a copy of the License at + 7 + 8 http://www.apache.org/licenses/LICENSE-2.0 + 9 + 10 Unless required by applicable law or agreed to in writing, software + 11 distributed under the License is distributed on an "AS IS" BASIS, + 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + 13 See the License for the specific language governing permissions and + 14 limitations under the License. + 15 + 16 Project-authored miniature WNDB fixture mirroring mini-wn-lmf.xml. + 17 License preamble lines begin with two spaces, as in released WNDB files, + 18 so readers skip them; data line offsets include this preamble. +00001075 29 v 01 run 0 002 @ 00001324 v 0000 + 00001427 n 0101 01 + 02 00 | move fast on foot +00001171 29 v 01 walk 0 001 @ 00001324 v 0000 01 + 02 00 | move at a regular pace +00001255 29 v 01 go 0 000 01 + 02 00 | change location or position +00001324 29 v 01 move 0 002 ~ 00001075 v 0000 ~ 00001171 v 0000 01 + 02 00 | change position in space diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.adj b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.adj new file mode 100644 index 0000000000..827a988a7d --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.adj @@ -0,0 +1,22 @@ + 1 Licensed to the Apache Software Foundation (ASF) under one or more + 2 contributor license agreements. See the NOTICE file distributed with + 3 this work for additional information regarding copyright ownership. + 4 The ASF licenses this file to You under the Apache License, Version 2.0 + 5 (the "License"); you may not use this file except in compliance with + 6 the License. You may obtain a copy of the License at + 7 + 8 http://www.apache.org/licenses/LICENSE-2.0 + 9 + 10 Unless required by applicable law or agreed to in writing, software + 11 distributed under the License is distributed on an "AS IS" BASIS, + 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + 13 See the License for the specific language governing permissions and + 14 limitations under the License. + 15 + 16 Project-authored miniature WNDB fixture mirroring mini-wn-lmf.xml. + 17 License preamble lines begin with two spaces, as in released WNDB files, + 18 so readers skip them; data line offsets include this preamble. +big a 1 1 & 1 0 00001211 +large a 1 1 & 1 0 00001274 +short a 1 1 ! 1 0 00001141 +tall a 1 1 ! 1 0 00001075 diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.adv b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.adv new file mode 100644 index 0000000000..da20fe1193 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.adv @@ -0,0 +1,20 @@ + 1 Licensed to the Apache Software Foundation (ASF) under one or more + 2 contributor license agreements. See the NOTICE file distributed with + 3 this work for additional information regarding copyright ownership. + 4 The ASF licenses this file to You under the Apache License, Version 2.0 + 5 (the "License"); you may not use this file except in compliance with + 6 the License. You may obtain a copy of the License at + 7 + 8 http://www.apache.org/licenses/LICENSE-2.0 + 9 + 10 Unless required by applicable law or agreed to in writing, software + 11 distributed under the License is distributed on an "AS IS" BASIS, + 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + 13 See the License for the specific language governing permissions and + 14 limitations under the License. + 15 + 16 Project-authored miniature WNDB fixture mirroring mini-wn-lmf.xml. + 17 License preamble lines begin with two spaces, as in released WNDB files, + 18 so readers skip them; data line offsets include this preamble. +quickly r 1 0 1 0 00001075 +well r 1 0 1 0 00001121 diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.noun b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.noun new file mode 100644 index 0000000000..41a8a4317b --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.noun @@ -0,0 +1,27 @@ + 1 Licensed to the Apache Software Foundation (ASF) under one or more + 2 contributor license agreements. See the NOTICE file distributed with + 3 this work for additional information regarding copyright ownership. + 4 The ASF licenses this file to You under the Apache License, Version 2.0 + 5 (the "License"); you may not use this file except in compliance with + 6 the License. You may obtain a copy of the License at + 7 + 8 http://www.apache.org/licenses/LICENSE-2.0 + 9 + 10 Unless required by applicable law or agreed to in writing, software + 11 distributed under the License is distributed on an "AS IS" BASIS, + 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + 13 See the License for the specific language governing permissions and + 14 limitations under the License. + 15 + 16 Project-authored miniature WNDB fixture mirroring mini-wn-lmf.xml. + 17 License preamble lines begin with two spaces, as in released WNDB files, + 18 so readers skip them; data line offsets include this preamble. +berry n 1 0 1 0 00001564 +box n 1 0 1 0 00001503 +canid n 1 1 ~ 1 0 00001160 +dog n 1 1 @ 1 0 00001075 +domestic_dog n 1 1 @ 1 0 00001075 +man n 1 0 1 0 00001617 +mouse n 1 1 @ 1 0 00001257 +rodent n 1 1 ~ 1 0 00001340 +run n 2 1 + 2 1 00001427 00001669 diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.verb b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.verb new file mode 100644 index 0000000000..2b380478de --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/index.verb @@ -0,0 +1,22 @@ + 1 Licensed to the Apache Software Foundation (ASF) under one or more + 2 contributor license agreements. See the NOTICE file distributed with + 3 this work for additional information regarding copyright ownership. + 4 The ASF licenses this file to You under the Apache License, Version 2.0 + 5 (the "License"); you may not use this file except in compliance with + 6 the License. You may obtain a copy of the License at + 7 + 8 http://www.apache.org/licenses/LICENSE-2.0 + 9 + 10 Unless required by applicable law or agreed to in writing, software + 11 distributed under the License is distributed on an "AS IS" BASIS, + 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + 13 See the License for the specific language governing permissions and + 14 limitations under the License. + 15 + 16 Project-authored miniature WNDB fixture mirroring mini-wn-lmf.xml. + 17 License preamble lines begin with two spaces, as in released WNDB files, + 18 so readers skip them; data line offsets include this preamble. +go v 1 0 1 0 00001255 +move v 1 1 ~ 1 0 00001324 +run v 1 2 @ + 1 1 00001075 +walk v 1 1 @ 1 0 00001171 From 5807fcbb1b5ac4661ce76dda36bef8fa78fbc598 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 7 Jul 2026 00:56:22 -0400 Subject: [PATCH 04/12] Add the Morphy lemmatizer with exception-list support --- opennlp-extensions/opennlp-wordnet/pom.xml | 6 + .../opennlp/wordnet/MorphyExceptions.java | 160 +++++++++++++ .../opennlp/wordnet/MorphyLemmatizer.java | 210 ++++++++++++++++++ .../opennlp/wordnet/MorphyExceptionsTest.java | 103 +++++++++ .../opennlp/wordnet/MorphyLemmatizerTest.java | 171 ++++++++++++++ .../opennlp/wordnet/mini-wndb/adj.exc | 1 + .../opennlp/wordnet/mini-wndb/adv.exc | 1 + .../opennlp/wordnet/mini-wndb/noun.exc | 3 + .../opennlp/wordnet/mini-wndb/verb.exc | 4 + rat-excludes | 9 + 10 files changed, 668 insertions(+) create mode 100644 opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyExceptions.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyExceptionsTest.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/adj.exc create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/adv.exc create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/noun.exc create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/verb.exc diff --git a/opennlp-extensions/opennlp-wordnet/pom.xml b/opennlp-extensions/opennlp-wordnet/pom.xml index 854315eaf2..0721fcfc98 100644 --- a/opennlp-extensions/opennlp-wordnet/pom.xml +++ b/opennlp-extensions/opennlp-wordnet/pom.xml @@ -48,6 +48,12 @@ junit-jupiter-engine test + + + org.junit.jupiter + junit-jupiter-params + test + diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyExceptions.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyExceptions.java new file mode 100644 index 0000000000..10a5cc67d3 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyExceptions.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import opennlp.tools.commons.ThreadSafe; +import opennlp.tools.wordnet.WordNetPos; + +/** + * The Morphy exception lists: the per-part-of-speech tables of irregular inflected forms + * ({@code mice} to {@code mouse}, {@code went} to {@code go}) that the Morphy algorithm + * consults before its detachment rules. + * + *

{@link #load(Path)} reads the four {@code *.exc} files ({@code noun.exc}, + * {@code verb.exc}, {@code adj.exc}, {@code adv.exc}) in the documented WNDB format: one entry + * per line, the inflected form followed by one or more base forms, space separated, with + * underscores standing for spaces in multiword entries. All four files must be present; note + * that Princeton's database-only download ({@code WNdb}) does not include them, while the full + * WordNet package does. No exception data is bundled with this module; see + * {@link MorphyLemmatizer} for the tiering rationale.

+ * + *

Lookups fold the queried word the same way the lexicon seam folds lemmas: lowercase with + * the root locale, underscore as space.

+ * + *

Instances are immutable after loading and safe for concurrent lookups.

+ */ +@ThreadSafe +public final class MorphyExceptions { + + private final Map>> byPos; + + private MorphyExceptions(Map>> byPos) { + this.byPos = byPos; + } + + /** + * Loads the four exception lists from a directory. + * + * @param directory The directory containing {@code noun.exc}, {@code verb.exc}, + * {@code adj.exc}, and {@code adv.exc}. Must not be {@code null} and must + * exist. + * @return The loaded exception lists. + * @throws IllegalArgumentException Thrown if {@code directory} is {@code null} or not a + * directory, one of the four files is missing, or a line is malformed; the message names + * the file and line. + * @throws UncheckedIOException Thrown if reading a file fails. + */ + public static MorphyExceptions load(Path directory) { + if (directory == null) { + throw new IllegalArgumentException("Directory must not be null"); + } + if (!Files.isDirectory(directory)) { + throw new IllegalArgumentException( + "Directory does not exist or is not a directory: " + directory); + } + final Map>> byPos = new EnumMap<>(WordNetPos.class); + byPos.put(WordNetPos.NOUN, loadFile(directory, "noun.exc")); + byPos.put(WordNetPos.VERB, loadFile(directory, "verb.exc")); + byPos.put(WordNetPos.ADJECTIVE, loadFile(directory, "adj.exc")); + byPos.put(WordNetPos.ADVERB, loadFile(directory, "adv.exc")); + return new MorphyExceptions(byPos); + } + + /** + * Finds the base forms of an irregular inflected form. + * + * @param word The inflected form; folded before lookup. Must not be {@code null}. + * @param pos The part of speech. Must not be {@code null}. + * @return The base forms in file order, never {@code null}; empty when the word has no entry. + * @throws IllegalArgumentException Thrown if {@code word} or {@code pos} is {@code null}. + */ + public List lookup(String word, WordNetPos pos) { + if (word == null) { + throw new IllegalArgumentException("Word must not be null"); + } + if (pos == null) { + throw new IllegalArgumentException("Pos must not be null"); + } + final List lemmas = byPos.get(pos).get(fold(word)); + return lemmas == null ? List.of() : lemmas; + } + + static String fold(String word) { + return word.replace('_', ' ').toLowerCase(Locale.ROOT); + } + + private static Map> loadFile(Path directory, String fileName) { + final Path file = directory.resolve(fileName); + if (!Files.isRegularFile(file)) { + throw new IllegalArgumentException("Missing exception list file: " + file); + } + final List lines; + try { + lines = Files.readAllLines(file, StandardCharsets.ISO_8859_1); + } catch (IOException e) { + throw new UncheckedIOException("Unable to read exception list file " + file, e); + } + final Map> entries = new HashMap<>(lines.size() * 2); + for (int i = 0; i < lines.size(); i++) { + final String line = lines.get(i); + if (line.isEmpty()) { + continue; + } + final List fields = splitOnSpaces(line); + if (fields.size() < 2) { + throw new IllegalArgumentException("Malformed exception list " + fileName + " at line " + + (i + 1) + ": expected an inflected form and at least one base form, got: " + line); + } + final List lemmas = new ArrayList<>(fields.size() - 1); + for (final String lemma : fields.subList(1, fields.size())) { + lemmas.add(fold(lemma)); + } + // A form listed twice keeps its first entry, matching first-match lookup semantics. + entries.putIfAbsent(fold(fields.get(0)), List.copyOf(lemmas)); + } + return Map.copyOf(entries); + } + + private static List splitOnSpaces(String line) { + final List fields = new ArrayList<>(3); + int start = 0; + while (start < line.length()) { + final int space = line.indexOf(' ', start); + if (space < 0) { + fields.add(line.substring(start)); + break; + } + if (space > start) { + fields.add(line.substring(start, space)); + } + start = space + 1; + } + return fields; + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java new file mode 100644 index 0000000000..812b7092f8 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +import opennlp.tools.commons.ThreadSafe; +import opennlp.tools.lemmatizer.Lemmatizer; +import opennlp.tools.wordnet.WordNetLexicon; +import opennlp.tools.wordnet.WordNetPos; + +/** + * A clean-room implementation of the documented Morphy algorithm as a {@link Lemmatizer}: + * exception-list lookup first, then the per-part-of-speech iterative detachment rules, with + * every rule-derived candidate validated against a {@link WordNetLexicon} before it is + * returned. + * + *

Algorithm. For each token the part-of-speech tag is mapped to a + * {@link WordNetPos}; the token is folded (lowercase with the root locale, underscore as + * space); the {@link MorphyExceptions exception lists} are consulted and a hit is returned + * directly, without lexicon validation, because the lists themselves are authoritative for + * irregular forms; otherwise the folded token itself, when the lexicon contains it, and every + * detachment-rule result the lexicon confirms become the candidate lemmas, in rule order. The + * rules are the documented Morphy suffix substitutions: for nouns {@code -s}, {@code -ses}, + * {@code -xes}, {@code -zes}, {@code -ches}, {@code -shes}, {@code -men}, {@code -ies}; for + * verbs {@code -s}, {@code -ies}, {@code -es}, {@code -ed}, {@code -ing} (with their {@code e} + * restorations); for adjectives {@code -er} and {@code -est} (plain and {@code e}-restoring); + * and none for adverbs, which rely on the exception list alone. Returned lemmas are in the + * lexicon's folded canonical form (lowercase, spaces in multiword lemmas).

+ * + *

Tag mapping. Tags map to WordNet parts of speech by their conventional Penn + * Treebank prefixes: {@code N} to noun, {@code V} to verb, {@code J} to adjective, and + * {@code R} to adverb, case-insensitively. The names {@code ADJ} and {@code ADV} and the + * WordNet letters {@code n}, {@code v}, {@code a}, {@code r}, and {@code s} (satellite, + * treated as adjective) are also accepted. Anything else, including Penn tags for closed + * classes such as {@code DT} or {@code MD}, maps to no part of speech and yields the + * unknown-word result.

+ * + *

Unknown words. Following the convention of + * {@code opennlp.tools.lemmatizer.DictionaryLemmatizer}, a token with no lemma yields the + * string {@code "O"} from {@link #lemmatize(String[], String[])} and a singleton + * {@code ["O"]} from {@link #lemmatize(List, List)}.

+ * + *

No lexicon or exception data is bundled: point {@link WndbReader} and + * {@link MorphyExceptions#load(java.nio.file.Path) MorphyExceptions} at a WordNet database + * directory you + * downloaded, or combine {@link WnLmfReader} with a downloaded exception-list directory. Both + * inputs are required because rule-derived candidates are meaningless without a lexicon to + * validate them against; an exception-only mode would be a misleading half-feature, so it does + * not exist. Bundling the permissively licensed data is a recorded follow-up.

+ * + *

Instances are immutable and safe for concurrent use once constructed with a loaded + * lexicon and exception lists.

+ */ +@ThreadSafe +public final class MorphyLemmatizer implements Lemmatizer { + + /** The unknown-word output, matching the DictionaryLemmatizer convention. */ + private static final String UNKNOWN = "O"; + + private static final String[][] NOUN_RULES = { + {"s", ""}, {"ses", "s"}, {"xes", "x"}, {"zes", "z"}, + {"ches", "ch"}, {"shes", "sh"}, {"men", "man"}, {"ies", "y"}, + }; + + private static final String[][] VERB_RULES = { + {"s", ""}, {"ies", "y"}, {"es", "e"}, {"es", ""}, + {"ed", "e"}, {"ed", ""}, {"ing", "e"}, {"ing", ""}, + }; + + private static final String[][] ADJECTIVE_RULES = { + {"er", ""}, {"est", ""}, {"er", "e"}, {"est", "e"}, + }; + + private static final String[][] NO_RULES = {}; + + private final WordNetLexicon lexicon; + private final MorphyExceptions exceptions; + + /** + * Creates a Morphy lemmatizer over a loaded lexicon and exception lists. + * + * @param lexicon The lexicon rule candidates are validated against. Must not be + * {@code null}. + * @param exceptions The irregular-form exception lists. Must not be {@code null}. + * @throws IllegalArgumentException Thrown if {@code lexicon} or {@code exceptions} is + * {@code null}. + */ + public MorphyLemmatizer(WordNetLexicon lexicon, MorphyExceptions exceptions) { + if (lexicon == null) { + throw new IllegalArgumentException("Lexicon must not be null"); + } + if (exceptions == null) { + throw new IllegalArgumentException("Exceptions must not be null"); + } + this.lexicon = lexicon; + this.exceptions = exceptions; + } + + @Override + public String[] lemmatize(String[] toks, String[] tags) { + if (toks == null || tags == null) { + throw new IllegalArgumentException("Toks and tags must not be null"); + } + if (toks.length != tags.length) { + throw new IllegalArgumentException("Toks and tags must have the same length, got " + + toks.length + " and " + tags.length); + } + final String[] lemmas = new String[toks.length]; + for (int i = 0; i < toks.length; i++) { + final List candidates = lemmasOf(toks[i], tags[i]); + lemmas[i] = candidates.isEmpty() ? UNKNOWN : candidates.get(0); + } + return lemmas; + } + + @Override + public List> lemmatize(List toks, List tags) { + if (toks == null || tags == null) { + throw new IllegalArgumentException("Toks and tags must not be null"); + } + if (toks.size() != tags.size()) { + throw new IllegalArgumentException("Toks and tags must have the same size, got " + + toks.size() + " and " + tags.size()); + } + final List> lemmas = new ArrayList<>(toks.size()); + for (int i = 0; i < toks.size(); i++) { + final List candidates = lemmasOf(toks.get(i), tags.get(i)); + lemmas.add(candidates.isEmpty() ? List.of(UNKNOWN) : candidates); + } + return lemmas; + } + + // All lemmas of one token, most preferred first; empty when the word is unknown. + private List lemmasOf(String token, String tag) { + if (token == null || tag == null) { + throw new IllegalArgumentException("Tokens and tags must not contain null elements"); + } + final WordNetPos pos = posFromTag(tag); + if (pos == null) { + return List.of(); + } + final String folded = MorphyExceptions.fold(token); + final List irregular = exceptions.lookup(folded, pos); + if (!irregular.isEmpty()) { + return irregular; + } + final List candidates = new ArrayList<>(2); + if (lexicon.contains(folded, pos)) { + candidates.add(folded); + } + for (final String[] rule : rulesFor(pos)) { + final String suffix = rule[0]; + if (folded.length() > suffix.length() && folded.endsWith(suffix)) { + final String candidate = + folded.substring(0, folded.length() - suffix.length()) + rule[1]; + if (!candidates.contains(candidate) && lexicon.contains(candidate, pos)) { + candidates.add(candidate); + } + } + } + return candidates; + } + + private static String[][] rulesFor(WordNetPos pos) { + return switch (pos) { + case NOUN -> NOUN_RULES; + case VERB -> VERB_RULES; + case ADJECTIVE -> ADJECTIVE_RULES; + case ADVERB -> NO_RULES; + }; + } + + // The documented tag mapping; package-private so tests can pin it directly. + static WordNetPos posFromTag(String tag) { + if (tag.isEmpty()) { + return null; + } + final String upper = tag.toUpperCase(Locale.ROOT); + if (upper.startsWith("ADJ")) { + return WordNetPos.ADJECTIVE; + } + if (upper.startsWith("ADV")) { + return WordNetPos.ADVERB; + } + return switch (upper.charAt(0)) { + case 'N' -> WordNetPos.NOUN; + case 'V' -> WordNetPos.VERB; + case 'J', 'A', 'S' -> WordNetPos.ADJECTIVE; + case 'R' -> WordNetPos.ADVERB; + default -> null; + }; + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyExceptionsTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyExceptionsTest.java new file mode 100644 index 0000000000..a9d810173d --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyExceptionsTest.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import opennlp.tools.wordnet.WordNetPos; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class MorphyExceptionsTest { + + static MorphyExceptions fixture() { + return MorphyExceptions.load(WndbReaderTest.fixtureDirectory()); + } + + @Test + void testLookupPerPartOfSpeech() { + final MorphyExceptions exceptions = fixture(); + assertEquals(List.of("mouse"), exceptions.lookup("mice", WordNetPos.NOUN)); + assertEquals(List.of("go"), exceptions.lookup("went", WordNetPos.VERB)); + assertEquals(List.of("good"), exceptions.lookup("better", WordNetPos.ADJECTIVE)); + assertEquals(List.of("well"), exceptions.lookup("best", WordNetPos.ADVERB)); + // Entries are part-of-speech scoped: went is only a verb exception. + assertTrue(exceptions.lookup("went", WordNetPos.NOUN).isEmpty()); + assertTrue(exceptions.lookup("dog", WordNetPos.NOUN).isEmpty()); + } + + @Test + void testLookupFoldsCase() { + assertEquals(List.of("mouse"), fixture().lookup("Mice", WordNetPos.NOUN)); + assertEquals(List.of("mouse"), fixture().lookup("MICE", WordNetPos.NOUN)); + } + + @Test + void testLookupRejectsNulls() { + final MorphyExceptions exceptions = fixture(); + assertThrows(IllegalArgumentException.class, + () -> exceptions.lookup(null, WordNetPos.NOUN)); + assertThrows(IllegalArgumentException.class, () -> exceptions.lookup("mice", null)); + } + + @Test + void testLoadRejectsNullAndMissingDirectory(@TempDir Path tempDir) { + assertThrows(IllegalArgumentException.class, () -> MorphyExceptions.load(null)); + assertThrows(IllegalArgumentException.class, + () -> MorphyExceptions.load(tempDir.resolve("absent"))); + } + + @Test + void testLoadRejectsMissingFile(@TempDir Path tempDir) throws IOException { + Files.writeString(tempDir.resolve("noun.exc"), "mice mouse\n"); + Files.writeString(tempDir.resolve("verb.exc"), "went go\n"); + Files.writeString(tempDir.resolve("adj.exc"), "better good\n"); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> MorphyExceptions.load(tempDir)); + assertTrue(e.getMessage().contains("adv.exc")); + } + + @Test + void testLoadRejectsMalformedLine(@TempDir Path tempDir) throws IOException { + Files.writeString(tempDir.resolve("noun.exc"), "mice mouse\nlonely\n"); + Files.writeString(tempDir.resolve("verb.exc"), "went go\n"); + Files.writeString(tempDir.resolve("adj.exc"), "better good\n"); + Files.writeString(tempDir.resolve("adv.exc"), "best well\n"); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> MorphyExceptions.load(tempDir)); + assertTrue(e.getMessage().contains("noun.exc")); + assertTrue(e.getMessage().contains("line 2")); + } + + @Test + void testMultipleBaseFormsKeepFileOrder(@TempDir Path tempDir) throws IOException { + Files.writeString(tempDir.resolve("noun.exc"), "axes axis ax\n"); + Files.writeString(tempDir.resolve("verb.exc"), "went go\n"); + Files.writeString(tempDir.resolve("adj.exc"), "better good\n"); + Files.writeString(tempDir.resolve("adv.exc"), "best well\n"); + assertEquals(List.of("axis", "ax"), + MorphyExceptions.load(tempDir).lookup("axes", WordNetPos.NOUN)); + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java new file mode 100644 index 0000000000..895d6fab86 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import opennlp.tools.wordnet.WordNetPos; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class MorphyLemmatizerTest { + + private static MorphyLemmatizer morphy() { + return new MorphyLemmatizer(WndbReaderTest.fixture(), MorphyExceptionsTest.fixture()); + } + + private static String one(String token, String tag) { + return morphy().lemmatize(new String[] {token}, new String[] {tag})[0]; + } + + @ParameterizedTest + @CsvSource({ + // Irregular forms resolve through the exception lists. + "mice, NN, mouse", + "Mice, NNS, mouse", + "men, NNS, man", + "ran, VBD, run", + "running, VBG, run", + "went, VBD, go", + "gone, VBN, go", + "best, RBS, well", + // Regular detachments, validated against the lexicon. + "dogs, NNS, dog", + "boxes, NNS, box", + "berries, NNS, berry", + "runs, NNS, run", + "runs, VBZ, run", + "walked, VBD, walk", + "walking, VBG, walk", + "walks, VBZ, walk", + "moved, VBD, move", + "taller, JJR, tall", + "tallest, JJS, tall", + "larger, JJR, large", + // A word that is already a lemma comes back as itself. + "dog, NN, dog", + "quickly, RB, quickly", + // WordNet letter tags are accepted alongside Penn tags. + "dogs, n, dog", + "walked, v, walk", + "taller, a, tall", + "best, r, well", + }) + void testLemmatizesToken(String token, String tag, String lemma) { + assertEquals(lemma, one(token, tag)); + } + + @ParameterizedTest + @CsvSource({ + // Rule candidates not in the lexicon are rejected, not returned. + "dogged, VBD", + "boxes, VBZ", + "glarbs, NNS", + // A known word under the wrong part of speech is unknown. + "walk, NN", + // Tags outside the mapping yield the unknown marker. + "dog, DT", + "dog, XYZ", + "dogs, ''", + }) + void testUnknownYieldsMarker(String token, String tag) { + assertEquals("O", one(token, tag)); + } + + @Test + void testExceptionHitsAreReturnedWithoutLexiconValidation() { + // oxen maps to ox, which the miniature lexicon does not contain; the exception list is + // authoritative for irregulars, so the lemma is returned anyway. + assertEquals("ox", one("oxen", "NNS")); + // better maps to good, also absent from the miniature lexicon. + assertEquals("good", one("better", "JJR")); + } + + @Test + void testArrayFormKeepsPositions() { + final String[] lemmas = morphy().lemmatize( + new String[] {"The", "mice", "ran", "quickly"}, + new String[] {"DT", "NNS", "VBD", "RB"}); + assertArrayEquals(new String[] {"O", "mouse", "run", "quickly"}, lemmas); + } + + @Test + void testListFormReturnsAllCandidates() { + final List> lemmas = morphy().lemmatize( + List.of("glarbs", "berries"), List.of("NNS", "NNS")); + assertEquals(List.of("O"), lemmas.get(0)); + assertEquals(List.of("berry"), lemmas.get(1)); + } + + @Test + void testWorksIdenticallyOverTheWnLmfLexicon() { + final MorphyLemmatizer lmfMorphy = + new MorphyLemmatizer(WnLmfReaderTest.fixture(), MorphyExceptionsTest.fixture()); + assertArrayEquals(new String[] {"mouse", "box", "walk", "large", "O"}, + lmfMorphy.lemmatize( + new String[] {"mice", "boxes", "walking", "larger", "dogged"}, + new String[] {"NNS", "NNS", "VBG", "JJR", "VBD"})); + } + + @Test + void testPosFromTagMapping() { + assertEquals(WordNetPos.NOUN, MorphyLemmatizer.posFromTag("NNP")); + assertEquals(WordNetPos.VERB, MorphyLemmatizer.posFromTag("VBZ")); + assertEquals(WordNetPos.ADJECTIVE, MorphyLemmatizer.posFromTag("JJ")); + assertEquals(WordNetPos.ADVERB, MorphyLemmatizer.posFromTag("RBR")); + assertEquals(WordNetPos.ADJECTIVE, MorphyLemmatizer.posFromTag("a")); + assertEquals(WordNetPos.ADJECTIVE, MorphyLemmatizer.posFromTag("s")); + assertEquals(WordNetPos.ADJECTIVE, MorphyLemmatizer.posFromTag("ADJ")); + assertEquals(WordNetPos.ADVERB, MorphyLemmatizer.posFromTag("ADV")); + assertEquals(WordNetPos.ADVERB, MorphyLemmatizer.posFromTag("r")); + assertNull(MorphyLemmatizer.posFromTag("DT")); + assertNull(MorphyLemmatizer.posFromTag("")); + } + + @Test + void testConstructorFailsLoudWithoutInputs() { + final MorphyExceptions exceptions = MorphyExceptionsTest.fixture(); + assertThrows(IllegalArgumentException.class, + () -> new MorphyLemmatizer(null, exceptions)); + assertThrows(IllegalArgumentException.class, + () -> new MorphyLemmatizer(WndbReaderTest.fixture(), null)); + } + + @Test + void testRejectsNullOrMismatchedSequences() { + final MorphyLemmatizer morphy = morphy(); + assertThrows(IllegalArgumentException.class, + () -> morphy.lemmatize((String[]) null, new String[0])); + assertThrows(IllegalArgumentException.class, + () -> morphy.lemmatize(new String[0], (String[]) null)); + assertThrows(IllegalArgumentException.class, + () -> morphy.lemmatize(new String[] {"a", "b"}, new String[] {"NN"})); + assertThrows(IllegalArgumentException.class, + () -> morphy.lemmatize(List.of("a"), List.of("NN", "NN"))); + assertThrows(IllegalArgumentException.class, + () -> morphy.lemmatize(new String[] {null}, new String[] {"NN"})); + assertThrows(IllegalArgumentException.class, + () -> morphy.lemmatize(new String[] {"dog"}, new String[] {null})); + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/adj.exc b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/adj.exc new file mode 100644 index 0000000000..404a2e4ddb --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/adj.exc @@ -0,0 +1 @@ +better good diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/adv.exc b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/adv.exc new file mode 100644 index 0000000000..c43a2cd529 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/adv.exc @@ -0,0 +1 @@ +best well diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/noun.exc b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/noun.exc new file mode 100644 index 0000000000..e5b3080a88 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/noun.exc @@ -0,0 +1,3 @@ +men man +mice mouse +oxen ox diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/verb.exc b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/verb.exc new file mode 100644 index 0000000000..486d0c7851 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/verb.exc @@ -0,0 +1,4 @@ +gone go +ran run +running run +went go diff --git a/rat-excludes b/rat-excludes index 5a5d86b90c..561869bd46 100644 --- a/rat-excludes +++ b/rat-excludes @@ -70,3 +70,12 @@ src/main/resources/opennlp/tools/tokenize/uax29/WordBreakProperty.txt src/main/resources/opennlp/tools/tokenize/uax29/ExtendedPictographic.txt src/main/resources/opennlp/tools/util/normalizer/confusables.txt src/test/resources/opennlp/tools/tokenize/uax29/WordBreakTest.txt + + +src/test/resources/opennlp/wordnet/mini-wndb/noun.exc +src/test/resources/opennlp/wordnet/mini-wndb/verb.exc +src/test/resources/opennlp/wordnet/mini-wndb/adj.exc +src/test/resources/opennlp/wordnet/mini-wndb/adv.exc From a2dda040a1070a043ae7717f7084f76339373451 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 7 Jul 2026 07:04:19 -0400 Subject: [PATCH 05/12] De-brand the wordnet lexicon contract: WordNetLexicon -> LexicalKnowledgeBase Renames the public seam interface so consumers describe what they are injecting (a lexical knowledge base) rather than naming the WordNet brand in a public API type. Concrete implementations that genuinely are WordNet-format readers (InMemoryWordNetLexicon, WnLmfReader, WndbReader) keep the WordNet name, since they describe the data format they read, not the contract itself. --- ...rdNetLexicon.java => LexicalKnowledgeBase.java} | 11 ++++++----- .../main/java/opennlp/tools/wordnet/Synset.java | 2 +- .../opennlp/tools/wordnet/WordNetRelation.java | 2 +- ...iconTest.java => LexicalKnowledgeBaseTest.java} | 6 +++--- .../opennlp/wordnet/InMemoryWordNetLexicon.java | 8 ++++---- .../java/opennlp/wordnet/MorphyLemmatizer.java | 8 ++++---- .../src/main/java/opennlp/wordnet/WndbReader.java | 6 +++--- .../opennlp/wordnet/LexiconConcurrencyTest.java | 6 +++--- .../opennlp/wordnet/ReaderEquivalenceTest.java | 10 +++++----- .../test/java/opennlp/wordnet/WndbReaderTest.java | 14 +++++++------- 10 files changed, 37 insertions(+), 36 deletions(-) rename opennlp-api/src/main/java/opennlp/tools/wordnet/{WordNetLexicon.java => LexicalKnowledgeBase.java} (92%) rename opennlp-api/src/test/java/opennlp/tools/wordnet/{WordNetLexiconTest.java => LexicalKnowledgeBaseTest.java} (94%) diff --git a/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetLexicon.java b/opennlp-api/src/main/java/opennlp/tools/wordnet/LexicalKnowledgeBase.java similarity index 92% rename from opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetLexicon.java rename to opennlp-api/src/main/java/opennlp/tools/wordnet/LexicalKnowledgeBase.java index a1d719808a..949599869c 100644 --- a/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetLexicon.java +++ b/opennlp-api/src/main/java/opennlp/tools/wordnet/LexicalKnowledgeBase.java @@ -20,15 +20,16 @@ import java.util.Optional; /** - * The wordnet lexicon seam: lemma and synset lookup over a loaded wordnet-style resource. + * The lexical knowledge base seam: lemma and synset lookup over a loaded lexical-semantic + * resource in the WordNet family. * *

This interface is the contract every wordnet-shaped dataset sits behind. A legacy Princeton * WNDB directory, a WN-LMF XML document (the Global WordNet Association interchange format used * by Open English WordNet and many other language wordnets), a future bundled * permissively-licensed lexicon, and a future user-downloaded CC-BY lexicon are all - * implementations of this one seam, so a consumer written against {@code WordNetLexicon} never - * changes when the data tier does. Nothing in the contract names a particular resource; synset - * identity is opaque and source-qualified (see {@link Synset#id()}).

+ * implementations of this one seam, so a consumer written against {@code LexicalKnowledgeBase} + * never changes when the data tier does. Nothing in the contract names a particular resource; + * synset identity is opaque and source-qualified (see {@link Synset#id()}).

* *

The surface is intentionally minimal but sufficient for the layered features above it: * lemma lookup and membership for morphological analysis (Morphy-style lemmatization), and @@ -44,7 +45,7 @@ *

Implementations must be immutable and thread-safe after loading: one instance is meant to * be shared across an application's threads for concurrent lookups.

*/ -public interface WordNetLexicon { +public interface LexicalKnowledgeBase { /** * Finds the synsets containing a lemma with a part of speech, in the source's sense order diff --git a/opennlp-api/src/main/java/opennlp/tools/wordnet/Synset.java b/opennlp-api/src/main/java/opennlp/tools/wordnet/Synset.java index c550efe8b0..d5863eb95b 100644 --- a/opennlp-api/src/main/java/opennlp/tools/wordnet/Synset.java +++ b/opennlp-api/src/main/java/opennlp/tools/wordnet/Synset.java @@ -30,7 +30,7 @@ *

The shape is deliberately format-agnostic. The {@link #id() id} is an opaque, * source-qualified string minted by whichever reader produced the synset (a WN-LMF document's * own synset id, or an id a WNDB reader derives from the file position); consumers must not - * parse it, only pass it back to {@link WordNetLexicon#synset(String)} and compare it for + * parse it, only pass it back to {@link LexicalKnowledgeBase#synset(String)} and compare it for * equality. Keeping identity opaque is what lets a bundled lexicon and a user-downloaded * lexicon sit behind the same seam, and lets later alignment layers join sources without a * contract change.

diff --git a/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetRelation.java b/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetRelation.java index 0794a8608d..f2ffd71641 100644 --- a/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetRelation.java +++ b/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetRelation.java @@ -22,7 +22,7 @@ *

The value set covers the pointer types documented for the legacy Princeton WNDB format, * which is also the common core the WN-LMF interchange format expresses; both readers map their * format's names onto these values, so consumers navigate one relation vocabulary regardless of - * the data tier behind the {@link WordNetLexicon} seam. {@link #PARTICIPLE} is part of that + * the data tier behind the {@link LexicalKnowledgeBase} seam. {@link #PARTICIPLE} is part of that * documented set (the adjective "participle of verb" pointer) even though it is easy * to overlook; without it, real Princeton data would be unreadable. Two values go beyond the * WNDB pointer set: {@link #ENTAILED_BY} and {@link #CAUSED_BY} cover the inverse relations diff --git a/opennlp-api/src/test/java/opennlp/tools/wordnet/WordNetLexiconTest.java b/opennlp-api/src/test/java/opennlp/tools/wordnet/LexicalKnowledgeBaseTest.java similarity index 94% rename from opennlp-api/src/test/java/opennlp/tools/wordnet/WordNetLexiconTest.java rename to opennlp-api/src/test/java/opennlp/tools/wordnet/LexicalKnowledgeBaseTest.java index 0823c6236d..9cf9cb6730 100644 --- a/opennlp-api/src/test/java/opennlp/tools/wordnet/WordNetLexiconTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/wordnet/LexicalKnowledgeBaseTest.java @@ -28,10 +28,10 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** - * Exercises the {@link WordNetLexicon} default methods against a minimal in-memory + * Exercises the {@link LexicalKnowledgeBase} default methods against a minimal in-memory * implementation, so the defaults are validated independently of any reader. */ -public class WordNetLexiconTest { +public class LexicalKnowledgeBaseTest { private static final Synset DOG = new Synset("test-1-n", WordNetPos.NOUN, List.of("dog"), "a domesticated canid", Map.of(WordNetRelation.HYPERNYM, List.of("test-2-n"))); @@ -40,7 +40,7 @@ public class WordNetLexiconTest { "a carnivorous mammal", Map.of(WordNetRelation.HYPONYM, List.of("test-1-n"))); // A deliberately tiny implementation of only the two abstract methods. - private static final WordNetLexicon LEXICON = new WordNetLexicon() { + private static final LexicalKnowledgeBase LEXICON = new LexicalKnowledgeBase() { @Override public List lookup(String lemma, WordNetPos pos) { diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java index b22f5aa526..123d0a6d81 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java @@ -26,15 +26,15 @@ import java.util.Optional; import opennlp.tools.commons.ThreadSafe; +import opennlp.tools.wordnet.LexicalKnowledgeBase; import opennlp.tools.wordnet.Synset; -import opennlp.tools.wordnet.WordNetLexicon; import opennlp.tools.wordnet.WordNetPos; import opennlp.tools.wordnet.WordNetRelation; /** - * The immutable in-memory {@link WordNetLexicon} both readers produce: a synset table plus a + * The immutable in-memory {@link LexicalKnowledgeBase} both readers produce: a synset table plus a * folded (lemma, part of speech) index. Package-private because it is a reader product, not a - * public entry point; consumers hold it as {@link WordNetLexicon}. + * public entry point; consumers hold it as {@link LexicalKnowledgeBase}. * *

Lemma matching follows the recommended seam semantics: keys are folded by lowercasing with * the root locale and treating the underscore some formats store in multiword lemmas as a @@ -45,7 +45,7 @@ * After construction all state is immutable, making instances safe for concurrent lookups.

*/ @ThreadSafe -final class InMemoryWordNetLexicon implements WordNetLexicon { +final class InMemoryWordNetLexicon implements LexicalKnowledgeBase { private final Map synsetsById; private final Map> senseIndex; diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java index 812b7092f8..0874ee97fd 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java @@ -22,13 +22,13 @@ import opennlp.tools.commons.ThreadSafe; import opennlp.tools.lemmatizer.Lemmatizer; -import opennlp.tools.wordnet.WordNetLexicon; +import opennlp.tools.wordnet.LexicalKnowledgeBase; import opennlp.tools.wordnet.WordNetPos; /** * A clean-room implementation of the documented Morphy algorithm as a {@link Lemmatizer}: * exception-list lookup first, then the per-part-of-speech iterative detachment rules, with - * every rule-derived candidate validated against a {@link WordNetLexicon} before it is + * every rule-derived candidate validated against a {@link LexicalKnowledgeBase} before it is * returned. * *

Algorithm. For each token the part-of-speech tag is mapped to a @@ -90,7 +90,7 @@ public final class MorphyLemmatizer implements Lemmatizer { private static final String[][] NO_RULES = {}; - private final WordNetLexicon lexicon; + private final LexicalKnowledgeBase lexicon; private final MorphyExceptions exceptions; /** @@ -102,7 +102,7 @@ public final class MorphyLemmatizer implements Lemmatizer { * @throws IllegalArgumentException Thrown if {@code lexicon} or {@code exceptions} is * {@code null}. */ - public MorphyLemmatizer(WordNetLexicon lexicon, MorphyExceptions exceptions) { + public MorphyLemmatizer(LexicalKnowledgeBase lexicon, MorphyExceptions exceptions) { if (lexicon == null) { throw new IllegalArgumentException("Lexicon must not be null"); } diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java index 5725c04717..3e40e05305 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java @@ -28,14 +28,14 @@ import java.util.List; import java.util.Map; +import opennlp.tools.wordnet.LexicalKnowledgeBase; import opennlp.tools.wordnet.Synset; -import opennlp.tools.wordnet.WordNetLexicon; import opennlp.tools.wordnet.WordNetPos; import opennlp.tools.wordnet.WordNetRelation; /** * Reads a legacy Princeton WNDB database directory ({@code index.noun}, {@code data.noun}, and - * the corresponding pairs for verbs, adjectives, and adverbs) into a {@link WordNetLexicon}. + * the corresponding pairs for verbs, adjectives, and adverbs) into a {@link LexicalKnowledgeBase}. * *

The reader is clean-room, built from the published {@code wndb(5WN)} and * {@code wninput(5WN)} format documentation, with no third-party WordNet library involved. All @@ -80,7 +80,7 @@ private WndbReader() { * file and line. * @throws UncheckedIOException Thrown if reading a file fails. */ - public static WordNetLexicon read(Path directory) { + public static LexicalKnowledgeBase read(Path directory) { if (directory == null) { throw new IllegalArgumentException("Directory must not be null"); } diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexiconConcurrencyTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexiconConcurrencyTest.java index a94709145d..1fdb4c0d7a 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexiconConcurrencyTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexiconConcurrencyTest.java @@ -24,7 +24,7 @@ import org.junit.jupiter.api.Test; -import opennlp.tools.wordnet.WordNetLexicon; +import opennlp.tools.wordnet.LexicalKnowledgeBase; import opennlp.tools.wordnet.WordNetPos; import opennlp.tools.wordnet.WordNetRelation; @@ -42,7 +42,7 @@ public class LexiconConcurrencyTest { @Test void testConcurrentLookupsSeeConsistentResults() throws InterruptedException { - final WordNetLexicon lexicon = WndbReaderTest.fixture(); + final LexicalKnowledgeBase lexicon = WndbReaderTest.fixture(); final CountDownLatch start = new CountDownLatch(1); final CountDownLatch done = new CountDownLatch(THREADS); final Queue problems = new ConcurrentLinkedQueue<>(); @@ -70,7 +70,7 @@ void testConcurrentLookupsSeeConsistentResults() throws InterruptedException { assertEquals(List.of(), List.copyOf(problems)); } - private static void verifyOnce(WordNetLexicon lexicon, Queue problems) { + private static void verifyOnce(LexicalKnowledgeBase lexicon, Queue problems) { if (!"wndb-00001075-n".equals(lexicon.lookup("dog", WordNetPos.NOUN).get(0).id())) { problems.add("Wrong dog lookup"); } diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/ReaderEquivalenceTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/ReaderEquivalenceTest.java index c212738a03..d7119d2f9d 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/ReaderEquivalenceTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/ReaderEquivalenceTest.java @@ -24,8 +24,8 @@ import org.junit.jupiter.api.Test; +import opennlp.tools.wordnet.LexicalKnowledgeBase; import opennlp.tools.wordnet.Synset; -import opennlp.tools.wordnet.WordNetLexicon; import opennlp.tools.wordnet.WordNetPos; import opennlp.tools.wordnet.WordNetRelation; @@ -61,7 +61,7 @@ void testBothReadersProduceEquivalentViews() { @Test void testLookupAgreesForEveryLemmaAndPos() { - final WordNetLexicon lmf = WnLmfReaderTest.fixture(); + final LexicalKnowledgeBase lmf = WnLmfReaderTest.fixture(); final InMemoryWordNetLexicon wndb = (InMemoryWordNetLexicon) WndbReaderTest.fixture(); final Set checked = new HashSet<>(); for (final Synset synset : wndb.synsets()) { @@ -82,8 +82,8 @@ void testLookupAgreesForEveryLemmaAndPos() { @Test void testSenseOrderAgreesForMultiSenseLemma() { - final WordNetLexicon lmf = WnLmfReaderTest.fixture(); - final WordNetLexicon wndb = WndbReaderTest.fixture(); + final LexicalKnowledgeBase lmf = WnLmfReaderTest.fixture(); + final LexicalKnowledgeBase wndb = WndbReaderTest.fixture(); final List lmfOrder = glosses(lmf.lookup("run", WordNetPos.NOUN)); final List wndbOrder = glosses(wndb.lookup("run", WordNetPos.NOUN)); assertEquals(2, lmfOrder.size()); @@ -102,7 +102,7 @@ private static Map byGloss(InMemoryWordNetLexicon lexicon) { // A synset's relations with targets replaced by their glosses, id-scheme independent. private static Map> relationsByGloss(Synset synset, - WordNetLexicon lexicon) { + LexicalKnowledgeBase lexicon) { final Map> result = new HashMap<>(); for (final Map.Entry> relation : synset.relations().entrySet()) { diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WndbReaderTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WndbReaderTest.java index 559914820f..07843d275d 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WndbReaderTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WndbReaderTest.java @@ -28,8 +28,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import opennlp.tools.wordnet.LexicalKnowledgeBase; import opennlp.tools.wordnet.Synset; -import opennlp.tools.wordnet.WordNetLexicon; import opennlp.tools.wordnet.WordNetPos; import opennlp.tools.wordnet.WordNetRelation; @@ -53,7 +53,7 @@ static Path fixtureDirectory() { } } - static WordNetLexicon fixture() { + static LexicalKnowledgeBase fixture() { return WndbReader.read(fixtureDirectory()); } @@ -71,7 +71,7 @@ void testLookupReturnsSynsetWithAllComponents() { @Test void testLookupFoldsCaseAndUnderscore() { - final WordNetLexicon lexicon = fixture(); + final LexicalKnowledgeBase lexicon = fixture(); assertEquals(DOG_ID, lexicon.lookup("Domestic_Dog", WordNetPos.NOUN).get(0).id()); assertEquals(DOG_ID, lexicon.lookup("DOG", WordNetPos.NOUN).get(0).id()); } @@ -84,7 +84,7 @@ void testLookupKeepsIndexSenseOrder() { @Test void testRelationNavigation() { - final WordNetLexicon lexicon = fixture(); + final LexicalKnowledgeBase lexicon = fixture(); assertEquals(List.of(DOG_ID), lexicon.related(CANID_ID, WordNetRelation.HYPONYM)); assertEquals(List.of("wndb-00001075-v", "wndb-00001171-v"), lexicon.related("wndb-00001324-v", WordNetRelation.HYPONYM)); @@ -94,7 +94,7 @@ void testRelationNavigation() { @Test void testLexicalPointersSurfaceAtSynsetLevel() { - final WordNetLexicon lexicon = fixture(); + final LexicalKnowledgeBase lexicon = fixture(); assertEquals(List.of("wndb-00001141-a"), lexicon.related("wndb-00001075-a", WordNetRelation.ANTONYM)); assertEquals(List.of("wndb-00001075-a"), @@ -103,7 +103,7 @@ void testLexicalPointersSurfaceAtSynsetLevel() { @Test void testSatelliteNormalizesToAdjectiveAndMarkerIsStripped() { - final WordNetLexicon lexicon = fixture(); + final LexicalKnowledgeBase lexicon = fixture(); final Synset large = lexicon.lookup("large", WordNetPos.ADJECTIVE).get(0); assertEquals(WordNetPos.ADJECTIVE, large.pos()); assertEquals(List.of("wndb-00001211-a"), large.related(WordNetRelation.SIMILAR_TO)); @@ -114,7 +114,7 @@ void testSatelliteNormalizesToAdjectiveAndMarkerIsStripped() { @Test void testUnknownLemmaOrSynsetIsEmpty() { - final WordNetLexicon lexicon = fixture(); + final LexicalKnowledgeBase lexicon = fixture(); assertTrue(lexicon.lookup("zebra", WordNetPos.NOUN).isEmpty()); assertTrue(lexicon.synset("wndb-99999999-n").isEmpty()); } From d01ddffb1011a285c4b942d45ebe327e56335267 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Tue, 7 Jul 2026 07:04:33 -0400 Subject: [PATCH 06/12] WnLmfReader: adopt LexicalKnowledgeBase and safe-skip WN-LMF DOCTYPE declarations Completes the LexicalKnowledgeBase rename in this file, and switches DOCTYPE handling from outright rejection to the OWASP-documented alternative for formats that require DOCTYPE support: the DOCTYPE event is skipped rather than resolved, while SUPPORT_DTD stays off (so no custom entity, internal or external, is ever declared), IS_SUPPORTING_EXTERNAL_ENTITIES and ACCESS_EXTERNAL_DTD stay off, and the XMLResolver still throws on any resolution attempt. This closes the same attack surface as outright rejection while letting the reader parse Open English WordNet releases unmodified, since real OEWN files ship a DOCTYPE line referencing the schema DTD. Test changes: testRejectsDoctype becomes testSkipsDoctypeDeclaration and now asserts a DOCTYPE-bearing document parses correctly instead of throwing. Added testInternalSubsetEntityIsNeverExpanded, which proves a classic XXE payload (a DOCTYPE-declared internal-subset entity pointing at a local file) still fails loud rather than expanding into the parsed output. --- .../java/opennlp/wordnet/WnLmfReader.java | 41 +++++++++------- .../java/opennlp/wordnet/WnLmfReaderTest.java | 49 ++++++++++++++----- 2 files changed, 61 insertions(+), 29 deletions(-) diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java index d5f930233a..32cb51f010 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java @@ -28,23 +28,24 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import javax.xml.XMLConstants; import javax.xml.stream.Location; import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamConstants; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; +import opennlp.tools.wordnet.LexicalKnowledgeBase; import opennlp.tools.wordnet.Synset; -import opennlp.tools.wordnet.WordNetLexicon; import opennlp.tools.wordnet.WordNetPos; import opennlp.tools.wordnet.WordNetRelation; /** * Reads a WN-LMF XML document (the Global WordNet Association interchange format, used by Open - * English WordNet and many other language wordnets) into a {@link WordNetLexicon}. + * English WordNet and many other language wordnets) into a {@link LexicalKnowledgeBase}. * *

The reader is clean-room, built from the published format documentation, and uses only the - * JDK's StAX parser. It reads the subset of the format the {@link WordNetLexicon} contract + * JDK's StAX parser. It reads the subset of the format the {@link LexicalKnowledgeBase} contract * serves: lexical entries (lemma, part of speech, senses), synsets (definition and typed * relations), and sense relations, which are lifted to the synset level as documented on * {@link WordNetRelation}. Elements outside that subset ({@code Pronunciation}, {@code Form}, @@ -53,12 +54,14 @@ * metadata, are also skipped: the contract has no untyped relation slot. Every other unknown * relation type fails loud.

* - *

Security. The parser is hardened against XXE: DTD processing and external entity - * resolution are disabled, nothing is ever fetched from the network, and a document containing - * a DOCTYPE declaration is rejected outright. Note that Open English WordNet releases ship with - * a DOCTYPE line referencing the schema DTD; to read such a file with this v1 reader, delete - * that one line first. Rejecting DTDs wholesale rather than trusting parser-specific partial - * DTD support is a deliberate v1 posture.

+ *

Security. The parser is hardened against XXE per the OWASP-documented posture for + * formats that carry a DOCTYPE: a DOCTYPE declaration is tokenized and skipped, but nothing it + * names is ever resolved. External entities and the external DTD subset are both disabled + * ({@code IS_SUPPORTING_EXTERNAL_ENTITIES} and {@code ACCESS_EXTERNAL_DTD} are off), and the + * {@link javax.xml.stream.XMLResolver} throws on any resolution attempt regardless, so no + * network or filesystem access can be triggered by a DOCTYPE, internal or external. Open English + * WordNet releases ship with a DOCTYPE line referencing the schema DTD; this reader parses such + * a file unmodified.

* *

Errors. Malformed structure fails loud with an {@link IllegalArgumentException} * naming the resource and, where the parser provides one, the line: missing required @@ -92,7 +95,7 @@ private WnLmfReader() { * document is malformed; the message names the file and, where available, the line. * @throws UncheckedIOException Thrown if reading the file fails. */ - public static WordNetLexicon read(Path file) { + public static LexicalKnowledgeBase read(Path file) { if (file == null) { throw new IllegalArgumentException("File must not be null"); } @@ -115,7 +118,7 @@ public static WordNetLexicon read(Path file) { * @throws IllegalArgumentException Thrown if an argument is {@code null} or the document is * malformed; the message names the resource and, where available, the line. */ - public static WordNetLexicon read(InputStream in, String resourceName) { + public static LexicalKnowledgeBase read(InputStream in, String resourceName) { if (in == null) { throw new IllegalArgumentException("In must not be null"); } @@ -138,9 +141,13 @@ public static WordNetLexicon read(InputStream in, String resourceName) { private static XMLInputFactory hardenedFactory() { final XMLInputFactory factory = XMLInputFactory.newFactory(); - // XXE hardening: no DTD processing, no external entities, nothing fetched from anywhere. + // XXE hardening, the OWASP-documented posture for DOCTYPE-bearing formats: the internal + // subset is not processed (so no custom entity gets declared at all), and both the + // external DTD subset and external entities are denied, so a DOCTYPE is tokenized and + // skipped but nothing it names is ever fetched. factory.setProperty(XMLInputFactory.SUPPORT_DTD, Boolean.FALSE); factory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, Boolean.FALSE); + factory.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, ""); factory.setProperty(XMLInputFactory.IS_COALESCING, Boolean.TRUE); factory.setXMLResolver((publicId, systemId, baseUri, namespace) -> { throw new XMLStreamException("External entity resolution is disabled, refusing " + systemId); @@ -178,10 +185,10 @@ private static final class Parser { void parse(XMLStreamReader reader) throws XMLStreamException { while (reader.hasNext()) { final int event = reader.next(); - if (event == XMLStreamConstants.DTD) { - throw malformed(reader.getLocation(), - "Document contains a DOCTYPE declaration; the hardened reader rejects DTDs", null); - } + // A DTD event (the DOCTYPE declaration) is intentionally not handled here: with + // external entities and the external subset denied in the factory, it carries nothing + // that can affect parsing, so it is skipped exactly like a comment or an ignored + // element. if (event == XMLStreamConstants.START_ELEMENT) { startElement(reader); } else if (event == XMLStreamConstants.END_ELEMENT) { @@ -277,7 +284,7 @@ private void endElement(String name) { } } - WordNetLexicon build() { + LexicalKnowledgeBase build() { // Every sense must point to a declared synset, with a consistent part of speech. for (final Map.Entry sense : synsetBySenseId.entrySet()) { final RawSynset target = rawSynsets.get(sense.getValue()); diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java index b978925ebc..37dd081bbb 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java @@ -27,8 +27,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import opennlp.tools.wordnet.LexicalKnowledgeBase; import opennlp.tools.wordnet.Synset; -import opennlp.tools.wordnet.WordNetLexicon; import opennlp.tools.wordnet.WordNetPos; import opennlp.tools.wordnet.WordNetRelation; @@ -40,7 +40,7 @@ public class WnLmfReaderTest { - static WordNetLexicon fixture() { + static LexicalKnowledgeBase fixture() { try (InputStream in = WnLmfReaderTest.class.getResourceAsStream("mini-wn-lmf.xml")) { assertNotNull(in, "Fixture mini-wn-lmf.xml must be on the test classpath"); return WnLmfReader.read(in, "mini-wn-lmf.xml"); @@ -49,7 +49,7 @@ static WordNetLexicon fixture() { } } - private static WordNetLexicon parse(String document) { + private static LexicalKnowledgeBase parse(String document) { return WnLmfReader.read( new ByteArrayInputStream(document.getBytes(StandardCharsets.UTF_8)), "inline.xml"); } @@ -74,7 +74,7 @@ void testLookupReturnsSynsetWithAllComponents() { @Test void testLookupFoldsCaseAndUnderscore() { - final WordNetLexicon lexicon = fixture(); + final LexicalKnowledgeBase lexicon = fixture(); assertEquals("mini-n1", lexicon.lookup("Domestic_Dog", WordNetPos.NOUN).get(0).id()); assertEquals("mini-n1", lexicon.lookup("DOG", WordNetPos.NOUN).get(0).id()); } @@ -88,7 +88,7 @@ void testLookupKeepsSenseOrder() { @Test void testLookupIsPosScoped() { - final WordNetLexicon lexicon = fixture(); + final LexicalKnowledgeBase lexicon = fixture(); assertEquals(1, lexicon.lookup("run", WordNetPos.VERB).size()); assertTrue(lexicon.lookup("dog", WordNetPos.VERB).isEmpty()); assertFalse(lexicon.contains("walk", WordNetPos.NOUN)); @@ -97,7 +97,7 @@ void testLookupIsPosScoped() { @Test void testRelationNavigation() { - final WordNetLexicon lexicon = fixture(); + final LexicalKnowledgeBase lexicon = fixture(); assertEquals(List.of("mini-n1"), lexicon.related("mini-n2", WordNetRelation.HYPONYM)); assertEquals(List.of("mini-v1", "mini-v2"), lexicon.related("mini-v4", WordNetRelation.HYPONYM)); @@ -106,7 +106,7 @@ void testRelationNavigation() { @Test void testSenseRelationsAreLiftedToSynsetLevel() { - final WordNetLexicon lexicon = fixture(); + final LexicalKnowledgeBase lexicon = fixture(); assertEquals(List.of("mini-a2"), lexicon.related("mini-a1", WordNetRelation.ANTONYM)); assertEquals(List.of("mini-a1"), lexicon.related("mini-a2", WordNetRelation.ANTONYM)); assertEquals(List.of("mini-v1"), @@ -126,7 +126,7 @@ void testSatelliteNormalizesToAdjective() { @Test void testUnknownLemmaOrSynsetIsEmpty() { - final WordNetLexicon lexicon = fixture(); + final LexicalKnowledgeBase lexicon = fixture(); assertTrue(lexicon.lookup("zebra", WordNetPos.NOUN).isEmpty()); assertTrue(lexicon.synset("mini-n99").isEmpty()); } @@ -138,7 +138,7 @@ void testReadPath(@TempDir Path tempDir) throws IOException { "" + "" + "a feline")); - final WordNetLexicon lexicon = WnLmfReader.read(file); + final LexicalKnowledgeBase lexicon = WnLmfReader.read(file); assertEquals("a feline", lexicon.lookup("cat", WordNetPos.NOUN).get(0).gloss()); } @@ -157,14 +157,39 @@ void testReadStreamRejectsNulls() { } @Test - void testRejectsDoctype() { + void testSkipsDoctypeDeclaration() { + // Real Open English WordNet releases ship exactly this shape: a DOCTYPE naming the schema + // DTD by an unreachable SYSTEM identifier (example.invalid is the RFC 2606 reserved domain + // that must never resolve). The reader must parse past it without attempting to fetch it. final String document = "\n" + "\n" + "" + + "" + + "" + + "a feline" + + ""; + final LexicalKnowledgeBase lexicon = parse(document); + assertEquals("a feline", lexicon.lookup("cat", WordNetPos.NOUN).get(0).gloss()); + } + + @Test + void testInternalSubsetEntityIsNeverExpanded(@TempDir Path tempDir) throws IOException { + // A DOCTYPE-declared internal-subset entity is the classic XXE payload: if the parser ever + // honored it, the entity reference below would be replaced by the target file's content. + // With SUPPORT_DTD disabled the declaration itself is never registered, so the reference is + // undefined and parsing must fail loud rather than silently expand it. + final Path secret = tempDir.resolve("secret.txt"); + Files.writeString(secret, "xxe-marker-should-never-appear"); + final String document = "\n" + + "]>\n" + + "" + + "" + + "" + + "a feline" + ""; final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> parse(document)); - assertTrue(e.getMessage().contains("inline.xml")); + assertFalse(e.getMessage().contains("xxe-marker-should-never-appear")); } @Test @@ -212,7 +237,7 @@ void testRejectsUnknownRelationType() { @Test void testSkipsOtherRelationType() { - final WordNetLexicon lexicon = parse( + final LexicalKnowledgeBase lexicon = parse( wrap("" + "" + "" From 0551c1dafddffa015338dce7e8cfadc5b60236b1 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Thu, 9 Jul 2026 09:50:07 -0400 Subject: [PATCH 07/12] Pin WNDB fixture line endings so Windows checkout doesn't corrupt them WNDB data lines embed their own byte offset, which WndbReader validates against the actual position. The repo's default `* text=auto` lets Git normalize these fixtures to CRLF on Windows checkout, inserting a CR before every LF and shifting every offset after the first line; every WndbReaderTest case failed on Windows CI as a result. Scoping -text to the fixture directory keeps the checkout byte-identical everywhere. --- .../resources/opennlp/wordnet/mini-wndb/.gitattributes | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/.gitattributes diff --git a/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/.gitattributes b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/.gitattributes new file mode 100644 index 0000000000..3868b4d103 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/resources/opennlp/wordnet/mini-wndb/.gitattributes @@ -0,0 +1,7 @@ +# WNDB is a byte-offset format: each data line embeds its own byte position in the +# file, and WndbReader validates that offset against the actual position. Line-ending +# normalization on checkout (the repo root's `* text=auto`) would insert a CR before +# every LF on Windows, shifting every offset after the first line and breaking every +# fixture that has more than one line. Disable it here so checkout is byte-identical +# on every platform. +* -text From 69ead6868b4d28ca6f0046fc38779258a203927b Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Fri, 10 Jul 2026 19:06:06 -0400 Subject: [PATCH 08/12] OPENNLP-1880: Address review: checked exceptions, POS naming, distr wiring, fail-loud coverage The three loading entry points (WndbReader.read, WnLmfReader.read, MorphyExceptions.load) now follow the codebase's loader convention: they declare IOException, malformed content raises InvalidFormatException, and null or argument problems stay IllegalArgumentException. Every internal content-error throw site, the javadocs, and all affected test assertions moved with the contract, and a failing-stream test pins that an I/O error surfaces as itself rather than as a malformed-document report. The public enum WordNetPos is renamed to WordNetPOS to match POSTagger and POSModel before the name freezes at release. Several reader gaps close: SynsetRelation elements with relType "other" are now skipped like their SenseRelation counterparts instead of rejected, duplicate LexicalEntry and Sense ids fail loud like duplicate synset ids, and relation resolution in both readers now stores the synset table's canonical id instance per target so a full lexicon keeps one copy of each id regardless of pointer count. Dangling WNDB pointer errors name the pointer's own line through the previously unread RawPointer component, and posFromTag accepts the WordNet letter codes a and s only as one-letter tags, so AUX, ADP, SCONJ, and SYM map to the unknown-word result instead of adjective lookups. The duplicated lemma fold and space-split logic collapse into a package-private LemmaFolding helper shared by the exception lists, the sense index, and the WN-LMF parser. The module is now wired into the root pom's dependencyManagement, the binary distribution's dependencies, and the aggregated apidocs assembly, mirroring the other extension modules. New coverage pins the verb-group mapping in both readers (a WN-LMF document with similar on verb synsets, and a constructed WNDB directory with computed byte offsets exercising the dollar pointer), the InMemoryWordNetLexicon constructor validation, the LemmaFolding behavior, and every documented WN-LMF structural fail-loud path. --- .../tools/wordnet/LexicalKnowledgeBase.java | 6 +- .../java/opennlp/tools/wordnet/Synset.java | 2 +- .../{WordNetPos.java => WordNetPOS.java} | 2 +- .../wordnet/LexicalKnowledgeBaseTest.java | 16 +- .../opennlp/tools/wordnet/SynsetTest.java | 34 +-- opennlp-distr/pom.xml | 4 + opennlp-distr/src/main/assembly/bin.xml | 7 + .../wordnet/InMemoryWordNetLexicon.java | 13 +- .../java/opennlp/wordnet/LemmaFolding.java | 71 +++++++ .../opennlp/wordnet/MorphyExceptions.java | 73 +++---- .../opennlp/wordnet/MorphyLemmatizer.java | 39 ++-- .../java/opennlp/wordnet/WnLmfReader.java | 135 ++++++------ .../main/java/opennlp/wordnet/WndbReader.java | 94 +++++---- .../wordnet/InMemoryWordNetLexiconTest.java | 89 ++++++++ .../opennlp/wordnet/LemmaFoldingTest.java | 60 ++++++ .../wordnet/LexiconConcurrencyTest.java | 10 +- .../opennlp/wordnet/MorphyExceptionsTest.java | 33 +-- .../opennlp/wordnet/MorphyLemmatizerTest.java | 30 ++- .../wordnet/ReaderEquivalenceTest.java | 8 +- .../java/opennlp/wordnet/WnLmfReaderTest.java | 196 +++++++++++++++--- .../java/opennlp/wordnet/WndbReaderTest.java | 109 ++++++++-- pom.xml | 6 + 22 files changed, 748 insertions(+), 289 deletions(-) rename opennlp-api/src/main/java/opennlp/tools/wordnet/{WordNetPos.java => WordNetPOS.java} (98%) create mode 100644 opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/InMemoryWordNetLexiconTest.java create mode 100644 opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LemmaFoldingTest.java diff --git a/opennlp-api/src/main/java/opennlp/tools/wordnet/LexicalKnowledgeBase.java b/opennlp-api/src/main/java/opennlp/tools/wordnet/LexicalKnowledgeBase.java index 949599869c..88518aa761 100644 --- a/opennlp-api/src/main/java/opennlp/tools/wordnet/LexicalKnowledgeBase.java +++ b/opennlp-api/src/main/java/opennlp/tools/wordnet/LexicalKnowledgeBase.java @@ -57,7 +57,7 @@ public interface LexicalKnowledgeBase { * the lemma with that part of speech. * @throws IllegalArgumentException Thrown if {@code lemma} or {@code pos} is {@code null}. */ - List lookup(String lemma, WordNetPos pos); + List lookup(String lemma, WordNetPOS pos); /** * Finds a synset by its opaque identifier. @@ -88,14 +88,14 @@ default List related(String synsetId, WordNetRelation relation) { /** * Tests whether the lexicon contains a lemma with a part of speech. This is the membership * check morphological rules validate their candidates against; implementations may override - * it with a cheaper check than {@link #lookup(String, WordNetPos)}. + * it with a cheaper check than {@link #lookup(String, WordNetPOS)}. * * @param lemma The lemma to test. Must not be {@code null}. * @param pos The part of speech to test it as. Must not be {@code null}. * @return {@code true} if the lexicon contains the lemma with that part of speech. * @throws IllegalArgumentException Thrown if {@code lemma} or {@code pos} is {@code null}. */ - default boolean contains(String lemma, WordNetPos pos) { + default boolean contains(String lemma, WordNetPOS pos) { return !lookup(lemma, pos).isEmpty(); } } diff --git a/opennlp-api/src/main/java/opennlp/tools/wordnet/Synset.java b/opennlp-api/src/main/java/opennlp/tools/wordnet/Synset.java index d5863eb95b..83fac7bdcc 100644 --- a/opennlp-api/src/main/java/opennlp/tools/wordnet/Synset.java +++ b/opennlp-api/src/main/java/opennlp/tools/wordnet/Synset.java @@ -57,7 +57,7 @@ @ThreadSafe public record Synset( String id, - WordNetPos pos, + WordNetPOS pos, List lemmas, String gloss, Map> relations) { diff --git a/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetPos.java b/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetPOS.java similarity index 98% rename from opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetPos.java rename to opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetPOS.java index 0e24085fb2..117d2c650a 100644 --- a/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetPos.java +++ b/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetPOS.java @@ -30,7 +30,7 @@ * *

Enum constants are immutable and safe to share across threads.

*/ -public enum WordNetPos { +public enum WordNetPOS { /** Nouns. */ NOUN, diff --git a/opennlp-api/src/test/java/opennlp/tools/wordnet/LexicalKnowledgeBaseTest.java b/opennlp-api/src/test/java/opennlp/tools/wordnet/LexicalKnowledgeBaseTest.java index 9cf9cb6730..2b698797f1 100644 --- a/opennlp-api/src/test/java/opennlp/tools/wordnet/LexicalKnowledgeBaseTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/wordnet/LexicalKnowledgeBaseTest.java @@ -33,24 +33,24 @@ */ public class LexicalKnowledgeBaseTest { - private static final Synset DOG = new Synset("test-1-n", WordNetPos.NOUN, List.of("dog"), + private static final Synset DOG = new Synset("test-1-n", WordNetPOS.NOUN, List.of("dog"), "a domesticated canid", Map.of(WordNetRelation.HYPERNYM, List.of("test-2-n"))); - private static final Synset CANID = new Synset("test-2-n", WordNetPos.NOUN, List.of("canid"), + private static final Synset CANID = new Synset("test-2-n", WordNetPOS.NOUN, List.of("canid"), "a carnivorous mammal", Map.of(WordNetRelation.HYPONYM, List.of("test-1-n"))); // A deliberately tiny implementation of only the two abstract methods. private static final LexicalKnowledgeBase LEXICON = new LexicalKnowledgeBase() { @Override - public List lookup(String lemma, WordNetPos pos) { + public List lookup(String lemma, WordNetPOS pos) { if (lemma == null) { throw new IllegalArgumentException("Lemma must not be null"); } if (pos == null) { throw new IllegalArgumentException("Pos must not be null"); } - if (pos == WordNetPos.NOUN && "dog".equals(lemma)) { + if (pos == WordNetPOS.NOUN && "dog".equals(lemma)) { return List.of(DOG); } return List.of(); @@ -92,14 +92,14 @@ void testRelatedRejectsNulls() { @Test void testContainsFollowsLookup() { - assertTrue(LEXICON.contains("dog", WordNetPos.NOUN)); - assertFalse(LEXICON.contains("dog", WordNetPos.VERB)); - assertFalse(LEXICON.contains("cat", WordNetPos.NOUN)); + assertTrue(LEXICON.contains("dog", WordNetPOS.NOUN)); + assertFalse(LEXICON.contains("dog", WordNetPOS.VERB)); + assertFalse(LEXICON.contains("cat", WordNetPOS.NOUN)); } @Test void testContainsRejectsNulls() { - assertThrows(IllegalArgumentException.class, () -> LEXICON.contains(null, WordNetPos.NOUN)); + assertThrows(IllegalArgumentException.class, () -> LEXICON.contains(null, WordNetPOS.NOUN)); assertThrows(IllegalArgumentException.class, () -> LEXICON.contains("dog", null)); } } diff --git a/opennlp-api/src/test/java/opennlp/tools/wordnet/SynsetTest.java b/opennlp-api/src/test/java/opennlp/tools/wordnet/SynsetTest.java index abcb45117b..e9cbb4c063 100644 --- a/opennlp-api/src/test/java/opennlp/tools/wordnet/SynsetTest.java +++ b/opennlp-api/src/test/java/opennlp/tools/wordnet/SynsetTest.java @@ -30,7 +30,7 @@ public class SynsetTest { private static Synset dog() { - return new Synset("test-1-n", WordNetPos.NOUN, List.of("dog", "domestic dog"), + return new Synset("test-1-n", WordNetPOS.NOUN, List.of("dog", "domestic dog"), "a domesticated canid", Map.of(WordNetRelation.HYPERNYM, List.of("test-2-n"))); } @@ -39,7 +39,7 @@ private static Synset dog() { void testComponents() { final Synset synset = dog(); assertEquals("test-1-n", synset.id()); - assertEquals(WordNetPos.NOUN, synset.pos()); + assertEquals(WordNetPOS.NOUN, synset.pos()); assertEquals(List.of("dog", "domestic dog"), synset.lemmas()); assertEquals("a domesticated canid", synset.gloss()); assertEquals(Map.of(WordNetRelation.HYPERNYM, List.of("test-2-n")), synset.relations()); @@ -47,7 +47,7 @@ void testComponents() { @Test void testRelatedReturnsTargetsInOrder() { - final Synset synset = new Synset("test-1-n", WordNetPos.NOUN, List.of("dog"), "", + final Synset synset = new Synset("test-1-n", WordNetPOS.NOUN, List.of("dog"), "", Map.of(WordNetRelation.HYPONYM, List.of("test-3-n", "test-2-n"))); assertEquals(List.of("test-3-n", "test-2-n"), synset.related(WordNetRelation.HYPONYM)); } @@ -64,7 +64,7 @@ void testRelatedRejectsNull() { @Test void testEmptyGlossAndNoRelationsAreValid() { - final Synset synset = new Synset("test-9-r", WordNetPos.ADVERB, List.of("well"), "", Map.of()); + final Synset synset = new Synset("test-9-r", WordNetPOS.ADVERB, List.of("well"), "", Map.of()); assertEquals("", synset.gloss()); assertTrue(synset.relations().isEmpty()); } @@ -75,7 +75,7 @@ void testDefensiveCopies() { final List targets = new ArrayList<>(List.of("test-2-n")); final Map> relations = new HashMap<>(); relations.put(WordNetRelation.HYPERNYM, targets); - final Synset synset = new Synset("test-1-n", WordNetPos.NOUN, lemmas, "gloss", relations); + final Synset synset = new Synset("test-1-n", WordNetPOS.NOUN, lemmas, "gloss", relations); lemmas.add("mutated"); targets.add("mutated"); relations.put(WordNetRelation.ANTONYM, List.of("test-3-n")); @@ -97,9 +97,9 @@ void testReturnedCollectionsAreImmutable() { @Test void testRejectsNullOrEmptyId() { assertThrows(IllegalArgumentException.class, - () -> new Synset(null, WordNetPos.NOUN, List.of("dog"), "", Map.of())); + () -> new Synset(null, WordNetPOS.NOUN, List.of("dog"), "", Map.of())); assertThrows(IllegalArgumentException.class, - () -> new Synset("", WordNetPos.NOUN, List.of("dog"), "", Map.of())); + () -> new Synset("", WordNetPOS.NOUN, List.of("dog"), "", Map.of())); } @Test @@ -111,40 +111,40 @@ void testRejectsNullPos() { @Test void testRejectsNullOrEmptyLemmas() { assertThrows(IllegalArgumentException.class, - () -> new Synset("test-1-n", WordNetPos.NOUN, null, "", Map.of())); + () -> new Synset("test-1-n", WordNetPOS.NOUN, null, "", Map.of())); assertThrows(IllegalArgumentException.class, - () -> new Synset("test-1-n", WordNetPos.NOUN, List.of(), "", Map.of())); + () -> new Synset("test-1-n", WordNetPOS.NOUN, List.of(), "", Map.of())); final List withNull = new ArrayList<>(); withNull.add(null); assertThrows(IllegalArgumentException.class, - () -> new Synset("test-1-n", WordNetPos.NOUN, withNull, "", Map.of())); + () -> new Synset("test-1-n", WordNetPOS.NOUN, withNull, "", Map.of())); assertThrows(IllegalArgumentException.class, - () -> new Synset("test-1-n", WordNetPos.NOUN, List.of(""), "", Map.of())); + () -> new Synset("test-1-n", WordNetPOS.NOUN, List.of(""), "", Map.of())); } @Test void testRejectsNullGloss() { assertThrows(IllegalArgumentException.class, - () -> new Synset("test-1-n", WordNetPos.NOUN, List.of("dog"), null, Map.of())); + () -> new Synset("test-1-n", WordNetPOS.NOUN, List.of("dog"), null, Map.of())); } @Test void testRejectsInvalidRelations() { assertThrows(IllegalArgumentException.class, - () -> new Synset("test-1-n", WordNetPos.NOUN, List.of("dog"), "", null)); + () -> new Synset("test-1-n", WordNetPOS.NOUN, List.of("dog"), "", null)); final Map> nullKey = new HashMap<>(); nullKey.put(null, List.of("test-2-n")); assertThrows(IllegalArgumentException.class, - () -> new Synset("test-1-n", WordNetPos.NOUN, List.of("dog"), "", nullKey)); + () -> new Synset("test-1-n", WordNetPOS.NOUN, List.of("dog"), "", nullKey)); final Map> nullTargets = new HashMap<>(); nullTargets.put(WordNetRelation.HYPERNYM, null); assertThrows(IllegalArgumentException.class, - () -> new Synset("test-1-n", WordNetPos.NOUN, List.of("dog"), "", nullTargets)); + () -> new Synset("test-1-n", WordNetPOS.NOUN, List.of("dog"), "", nullTargets)); assertThrows(IllegalArgumentException.class, - () -> new Synset("test-1-n", WordNetPos.NOUN, List.of("dog"), "", + () -> new Synset("test-1-n", WordNetPOS.NOUN, List.of("dog"), "", Map.of(WordNetRelation.HYPERNYM, List.of()))); assertThrows(IllegalArgumentException.class, - () -> new Synset("test-1-n", WordNetPos.NOUN, List.of("dog"), "", + () -> new Synset("test-1-n", WordNetPOS.NOUN, List.of("dog"), "", Map.of(WordNetRelation.HYPERNYM, List.of("")))); } } diff --git a/opennlp-distr/pom.xml b/opennlp-distr/pom.xml index e9092d8821..39269575e8 100644 --- a/opennlp-distr/pom.xml +++ b/opennlp-distr/pom.xml @@ -91,6 +91,10 @@ org.apache.opennlp opennlp-spellcheck + + org.apache.opennlp + opennlp-wordnet + diff --git a/opennlp-distr/src/main/assembly/bin.xml b/opennlp-distr/src/main/assembly/bin.xml index 2db4eafc65..36c556d473 100644 --- a/opennlp-distr/src/main/assembly/bin.xml +++ b/opennlp-distr/src/main/assembly/bin.xml @@ -239,6 +239,13 @@ docs/apidocs/opennlp-spellcheck + + ../opennlp-extensions/opennlp-wordnet/target/reports/apidocs + 644 + 755 + docs/apidocs/opennlp-wordnet + + ../opennlp-extensions/opennlp-uima/target/reports/apidocs 644 diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java index 123d0a6d81..08782a8dd4 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java @@ -21,14 +21,13 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Optional; import opennlp.tools.commons.ThreadSafe; import opennlp.tools.wordnet.LexicalKnowledgeBase; import opennlp.tools.wordnet.Synset; -import opennlp.tools.wordnet.WordNetPos; +import opennlp.tools.wordnet.WordNetPOS; import opennlp.tools.wordnet.WordNetRelation; /** @@ -106,7 +105,7 @@ final class InMemoryWordNetLexicon implements LexicalKnowledgeBase { } @Override - public List lookup(String lemma, WordNetPos pos) { + public List lookup(String lemma, WordNetPOS pos) { if (lemma == null) { throw new IllegalArgumentException("Lemma must not be null"); } @@ -136,13 +135,13 @@ Collection synsets() { } /** - * A folded sense-index key. Build with {@link #of(String, WordNetPos)} so every key passes + * A folded sense-index key. Build with {@link #of(String, WordNetPOS)} so every key passes * through the same fold as every query. * * @param lemma The folded lemma. * @param pos The part of speech. */ - record LemmaKey(String lemma, WordNetPos pos) { + record LemmaKey(String lemma, WordNetPOS pos) { /** * Folds a written form into a key: lowercase with the root locale, underscore as space. @@ -151,8 +150,8 @@ record LemmaKey(String lemma, WordNetPos pos) { * @param pos The part of speech. Must not be {@code null}. * @return The folded key. */ - static LemmaKey of(String writtenForm, WordNetPos pos) { - return new LemmaKey(writtenForm.replace('_', ' ').toLowerCase(Locale.ROOT), pos); + static LemmaKey of(String writtenForm, WordNetPOS pos) { + return new LemmaKey(LemmaFolding.fold(writtenForm), pos); } } } diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java new file mode 100644 index 0000000000..a4730ad97c --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +/** + * The single home of the lemma fold and the space-separated field split this package relies on. + * + *

The fold agreement is load-bearing: {@link MorphyExceptions} keys, the + * {@link InMemoryWordNetLexicon.LemmaKey sense-index keys}, and every query fold identically, + * so a rule-derived candidate validated against the lexicon and an exception-list hit can never + * disagree on canonical form. Keeping one implementation here prevents the copies from + * drifting apart.

+ */ +final class LemmaFolding { + + private LemmaFolding() { + } + + /** + * Folds a written form into its canonical shape: lowercase with the root locale, with the + * underscore some formats store in multiword lemmas treated as a space. + * + * @param writtenForm The form as written in a source file or query. Must not be {@code null}. + * @return The folded form. + */ + static String fold(String writtenForm) { + return writtenForm.replace('_', ' ').toLowerCase(Locale.ROOT); + } + + /** + * Splits a space-separated field list, collapsing runs of spaces; no regular expressions + * involved. + * + * @param value The field list. Must not be {@code null}. + * @return The non-empty fields in order, never {@code null}. + */ + static List splitOnSpaces(String value) { + final List parts = new ArrayList<>(4); + int start = 0; + while (start < value.length()) { + final int space = value.indexOf(' ', start); + if (space < 0) { + parts.add(value.substring(start)); + break; + } + if (space > start) { + parts.add(value.substring(start, space)); + } + start = space + 1; + } + return parts; + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyExceptions.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyExceptions.java index 10a5cc67d3..91b6eac5fd 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyExceptions.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyExceptions.java @@ -17,7 +17,6 @@ package opennlp.wordnet; import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -25,11 +24,11 @@ import java.util.EnumMap; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import opennlp.tools.commons.ThreadSafe; -import opennlp.tools.wordnet.WordNetPos; +import opennlp.tools.util.InvalidFormatException; +import opennlp.tools.wordnet.WordNetPOS; /** * The Morphy exception lists: the per-part-of-speech tables of irregular inflected forms @@ -52,9 +51,9 @@ @ThreadSafe public final class MorphyExceptions { - private final Map>> byPos; + private final Map>> byPos; - private MorphyExceptions(Map>> byPos) { + private MorphyExceptions(Map>> byPos) { this.byPos = byPos; } @@ -66,11 +65,12 @@ private MorphyExceptions(Map>> byPos) { * exist. * @return The loaded exception lists. * @throws IllegalArgumentException Thrown if {@code directory} is {@code null} or not a - * directory, one of the four files is missing, or a line is malformed; the message names - * the file and line. - * @throws UncheckedIOException Thrown if reading a file fails. + * directory. + * @throws InvalidFormatException Thrown if one of the four files is missing or a line is + * malformed; the message names the file and line. + * @throws IOException Thrown if reading a file fails. */ - public static MorphyExceptions load(Path directory) { + public static MorphyExceptions load(Path directory) throws IOException { if (directory == null) { throw new IllegalArgumentException("Directory must not be null"); } @@ -78,11 +78,11 @@ public static MorphyExceptions load(Path directory) { throw new IllegalArgumentException( "Directory does not exist or is not a directory: " + directory); } - final Map>> byPos = new EnumMap<>(WordNetPos.class); - byPos.put(WordNetPos.NOUN, loadFile(directory, "noun.exc")); - byPos.put(WordNetPos.VERB, loadFile(directory, "verb.exc")); - byPos.put(WordNetPos.ADJECTIVE, loadFile(directory, "adj.exc")); - byPos.put(WordNetPos.ADVERB, loadFile(directory, "adv.exc")); + final Map>> byPos = new EnumMap<>(WordNetPOS.class); + byPos.put(WordNetPOS.NOUN, loadFile(directory, "noun.exc")); + byPos.put(WordNetPOS.VERB, loadFile(directory, "verb.exc")); + byPos.put(WordNetPOS.ADJECTIVE, loadFile(directory, "adj.exc")); + byPos.put(WordNetPOS.ADVERB, loadFile(directory, "adv.exc")); return new MorphyExceptions(byPos); } @@ -94,67 +94,42 @@ public static MorphyExceptions load(Path directory) { * @return The base forms in file order, never {@code null}; empty when the word has no entry. * @throws IllegalArgumentException Thrown if {@code word} or {@code pos} is {@code null}. */ - public List lookup(String word, WordNetPos pos) { + public List lookup(String word, WordNetPOS pos) { if (word == null) { throw new IllegalArgumentException("Word must not be null"); } if (pos == null) { throw new IllegalArgumentException("Pos must not be null"); } - final List lemmas = byPos.get(pos).get(fold(word)); + final List lemmas = byPos.get(pos).get(LemmaFolding.fold(word)); return lemmas == null ? List.of() : lemmas; } - static String fold(String word) { - return word.replace('_', ' ').toLowerCase(Locale.ROOT); - } - - private static Map> loadFile(Path directory, String fileName) { + private static Map> loadFile(Path directory, String fileName) + throws IOException { final Path file = directory.resolve(fileName); if (!Files.isRegularFile(file)) { - throw new IllegalArgumentException("Missing exception list file: " + file); - } - final List lines; - try { - lines = Files.readAllLines(file, StandardCharsets.ISO_8859_1); - } catch (IOException e) { - throw new UncheckedIOException("Unable to read exception list file " + file, e); + throw new InvalidFormatException("Missing exception list file: " + file); } + final List lines = Files.readAllLines(file, StandardCharsets.ISO_8859_1); final Map> entries = new HashMap<>(lines.size() * 2); for (int i = 0; i < lines.size(); i++) { final String line = lines.get(i); if (line.isEmpty()) { continue; } - final List fields = splitOnSpaces(line); + final List fields = LemmaFolding.splitOnSpaces(line); if (fields.size() < 2) { - throw new IllegalArgumentException("Malformed exception list " + fileName + " at line " + throw new InvalidFormatException("Malformed exception list " + fileName + " at line " + (i + 1) + ": expected an inflected form and at least one base form, got: " + line); } final List lemmas = new ArrayList<>(fields.size() - 1); for (final String lemma : fields.subList(1, fields.size())) { - lemmas.add(fold(lemma)); + lemmas.add(LemmaFolding.fold(lemma)); } // A form listed twice keeps its first entry, matching first-match lookup semantics. - entries.putIfAbsent(fold(fields.get(0)), List.copyOf(lemmas)); + entries.putIfAbsent(LemmaFolding.fold(fields.get(0)), List.copyOf(lemmas)); } return Map.copyOf(entries); } - - private static List splitOnSpaces(String line) { - final List fields = new ArrayList<>(3); - int start = 0; - while (start < line.length()) { - final int space = line.indexOf(' ', start); - if (space < 0) { - fields.add(line.substring(start)); - break; - } - if (space > start) { - fields.add(line.substring(start, space)); - } - start = space + 1; - } - return fields; - } } diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java index 0874ee97fd..c92e9ab9dd 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java @@ -23,7 +23,7 @@ import opennlp.tools.commons.ThreadSafe; import opennlp.tools.lemmatizer.Lemmatizer; import opennlp.tools.wordnet.LexicalKnowledgeBase; -import opennlp.tools.wordnet.WordNetPos; +import opennlp.tools.wordnet.WordNetPOS; /** * A clean-room implementation of the documented Morphy algorithm as a {@link Lemmatizer}: @@ -32,7 +32,7 @@ * returned. * *

Algorithm. For each token the part-of-speech tag is mapped to a - * {@link WordNetPos}; the token is folded (lowercase with the root locale, underscore as + * {@link WordNetPOS}; the token is folded (lowercase with the root locale, underscore as * space); the {@link MorphyExceptions exception lists} are consulted and a hit is returned * directly, without lexicon validation, because the lists themselves are authoritative for * irregular forms; otherwise the folded token itself, when the lexicon contains it, and every @@ -46,11 +46,12 @@ * *

Tag mapping. Tags map to WordNet parts of speech by their conventional Penn * Treebank prefixes: {@code N} to noun, {@code V} to verb, {@code J} to adjective, and - * {@code R} to adverb, case-insensitively. The names {@code ADJ} and {@code ADV} and the - * WordNet letters {@code n}, {@code v}, {@code a}, {@code r}, and {@code s} (satellite, - * treated as adjective) are also accepted. Anything else, including Penn tags for closed - * classes such as {@code DT} or {@code MD}, maps to no part of speech and yields the - * unknown-word result.

+ * {@code R} to adverb, case-insensitively. The names {@code ADJ} and {@code ADV} and, only as + * one-letter tags, the WordNet letters {@code n}, {@code v}, {@code a}, {@code r}, and + * {@code s} (satellite, treated as adjective) are also accepted. Anything else, including Penn + * tags for closed classes such as {@code DT} or {@code MD} and multi-letter tags that merely + * begin with an accepted letter, such as {@code AUX}, {@code ADP}, {@code SCONJ}, or + * {@code SYM}, maps to no part of speech and yields the unknown-word result.

* *

Unknown words. Following the convention of * {@code opennlp.tools.lemmatizer.DictionaryLemmatizer}, a token with no lemma yields the @@ -152,11 +153,11 @@ private List lemmasOf(String token, String tag) { if (token == null || tag == null) { throw new IllegalArgumentException("Tokens and tags must not contain null elements"); } - final WordNetPos pos = posFromTag(tag); + final WordNetPOS pos = posFromTag(tag); if (pos == null) { return List.of(); } - final String folded = MorphyExceptions.fold(token); + final String folded = LemmaFolding.fold(token); final List irregular = exceptions.lookup(folded, pos); if (!irregular.isEmpty()) { return irregular; @@ -178,7 +179,7 @@ private List lemmasOf(String token, String tag) { return candidates; } - private static String[][] rulesFor(WordNetPos pos) { + private static String[][] rulesFor(WordNetPOS pos) { return switch (pos) { case NOUN -> NOUN_RULES; case VERB -> VERB_RULES; @@ -188,22 +189,26 @@ private static String[][] rulesFor(WordNetPos pos) { } // The documented tag mapping; package-private so tests can pin it directly. - static WordNetPos posFromTag(String tag) { + static WordNetPOS posFromTag(String tag) { if (tag.isEmpty()) { return null; } final String upper = tag.toUpperCase(Locale.ROOT); if (upper.startsWith("ADJ")) { - return WordNetPos.ADJECTIVE; + return WordNetPOS.ADJECTIVE; } if (upper.startsWith("ADV")) { - return WordNetPos.ADVERB; + return WordNetPOS.ADVERB; } return switch (upper.charAt(0)) { - case 'N' -> WordNetPos.NOUN; - case 'V' -> WordNetPos.VERB; - case 'J', 'A', 'S' -> WordNetPos.ADJECTIVE; - case 'R' -> WordNetPos.ADVERB; + case 'N' -> WordNetPOS.NOUN; + case 'V' -> WordNetPOS.VERB; + case 'J' -> WordNetPOS.ADJECTIVE; + // The WordNet letter codes a and s stand for adjective only as one-letter tags: + // multi-letter tags such as AUX, ADP, SCONJ, or SYM name closed classes or symbols, + // not adjectives, so they map to no part of speech. + case 'A', 'S' -> tag.length() == 1 ? WordNetPOS.ADJECTIVE : null; + case 'R' -> WordNetPOS.ADVERB; default -> null; }; } diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java index 32cb51f010..3ca689ecb0 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java @@ -19,15 +19,16 @@ import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; -import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import javax.xml.XMLConstants; import javax.xml.stream.Location; import javax.xml.stream.XMLInputFactory; @@ -35,9 +36,10 @@ import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; +import opennlp.tools.util.InvalidFormatException; import opennlp.tools.wordnet.LexicalKnowledgeBase; import opennlp.tools.wordnet.Synset; -import opennlp.tools.wordnet.WordNetPos; +import opennlp.tools.wordnet.WordNetPOS; import opennlp.tools.wordnet.WordNetRelation; /** @@ -63,13 +65,15 @@ * WordNet releases ship with a DOCTYPE line referencing the schema DTD; this reader parses such * a file unmodified.

* - *

Errors. Malformed structure fails loud with an {@link IllegalArgumentException} + *

Errors. Malformed structure fails loud with an {@link InvalidFormatException} * naming the resource and, where the parser provides one, the line: missing required - * attributes, a sense pointing to an undeclared synset, a relation to an undeclared target, an - * unknown part-of-speech code, an unknown relation type, a synset with no member entries.

+ * attributes, a duplicate lexical entry, sense, or synset id, a sense pointing to an undeclared + * synset, a relation to an undeclared target, an unknown part-of-speech code, an unknown + * relation type, a synset with no member entries. I/O failures propagate as + * {@link IOException}.

* *

Part-of-speech code {@code s} (adjective satellite) normalizes to - * {@link WordNetPos#ADJECTIVE}, and a {@code similar} relation whose source is a verb synset + * {@link WordNetPOS#ADJECTIVE}, and a {@code similar} relation whose source is a verb synset * maps to {@link WordNetRelation#VERB_GROUP} (that is how WN-LMF documents derived from * Princeton data express verb groups); on any other part of speech it maps to * {@link WordNetRelation#SIMILAR_TO}.

@@ -91,11 +95,12 @@ private WnLmfReader() { * * @param file The XML file. Must not be {@code null} and must exist. * @return The loaded lexicon. - * @throws IllegalArgumentException Thrown if {@code file} is {@code null} or missing, or the - * document is malformed; the message names the file and, where available, the line. - * @throws UncheckedIOException Thrown if reading the file fails. + * @throws IllegalArgumentException Thrown if {@code file} is {@code null} or missing. + * @throws InvalidFormatException Thrown if the document is malformed; the message names the + * file and, where available, the line. + * @throws IOException Thrown if reading the file fails. */ - public static LexicalKnowledgeBase read(Path file) { + public static LexicalKnowledgeBase read(Path file) throws IOException { if (file == null) { throw new IllegalArgumentException("File must not be null"); } @@ -104,8 +109,6 @@ public static LexicalKnowledgeBase read(Path file) { } try (InputStream in = new BufferedInputStream(Files.newInputStream(file))) { return read(in, file.toString()); - } catch (IOException e) { - throw new UncheckedIOException("Unable to read WN-LMF document " + file, e); } } @@ -115,10 +118,12 @@ public static LexicalKnowledgeBase read(Path file) { * @param in The document stream. Must not be {@code null}. * @param resourceName The name used in error messages. Must not be {@code null}. * @return The loaded lexicon. - * @throws IllegalArgumentException Thrown if an argument is {@code null} or the document is - * malformed; the message names the resource and, where available, the line. + * @throws IllegalArgumentException Thrown if an argument is {@code null}. + * @throws InvalidFormatException Thrown if the document is malformed; the message names the + * resource and, where available, the line. + * @throws IOException Thrown if reading the stream fails. */ - public static LexicalKnowledgeBase read(InputStream in, String resourceName) { + public static LexicalKnowledgeBase read(InputStream in, String resourceName) throws IOException { if (in == null) { throw new IllegalArgumentException("In must not be null"); } @@ -134,6 +139,13 @@ public static LexicalKnowledgeBase read(InputStream in, String resourceName) { reader.close(); } } catch (XMLStreamException e) { + // StAX wraps a failing stream read in an XMLStreamException; surface it as the I/O + // failure it is instead of misreporting it as a malformed document. + final Throwable nested = e.getNestedException() == null ? e.getCause() + : e.getNestedException(); + if (nested instanceof IOException io) { + throw io; + } throw parser.malformed(e.getLocation(), "XML error: " + e.getMessage(), e); } return parser.build(); @@ -161,8 +173,9 @@ private static final class Parser { private final String resourceName; // Entry state. + private final Set entryIds = new HashSet<>(); private final Map lemmaByEntryId = new HashMap<>(); - private final Map posByEntryId = new HashMap<>(); + private final Map posByEntryId = new HashMap<>(); private final Map synsetBySenseId = new HashMap<>(); private final Map> senseOrder = new LinkedHashMap<>(); @@ -174,7 +187,7 @@ private static final class Parser { // Cursor state. private String currentEntryId; private String currentEntryLemma; - private WordNetPos currentEntryPos; + private WordNetPOS currentEntryPos; private String currentSenseId; private RawSynset currentSynset; @@ -182,7 +195,7 @@ private static final class Parser { this.resourceName = resourceName; } - void parse(XMLStreamReader reader) throws XMLStreamException { + void parse(XMLStreamReader reader) throws XMLStreamException, InvalidFormatException { while (reader.hasNext()) { final int event = reader.next(); // A DTD event (the DOCTYPE declaration) is intentionally not handled here: with @@ -197,11 +210,16 @@ void parse(XMLStreamReader reader) throws XMLStreamException { } } - private void startElement(XMLStreamReader reader) throws XMLStreamException { + private void startElement(XMLStreamReader reader) + throws XMLStreamException, InvalidFormatException { final String name = reader.getLocalName(); switch (name) { case "LexicalEntry" -> { currentEntryId = requireAttribute(reader, "id"); + if (!entryIds.add(currentEntryId)) { + throw malformed(reader.getLocation(), + "Duplicate lexical entry id " + currentEntryId, null); + } currentEntryLemma = null; currentEntryPos = null; } @@ -222,7 +240,9 @@ private void startElement(XMLStreamReader reader) throws XMLStreamException { } currentSenseId = requireAttribute(reader, "id"); final String synsetId = requireAttribute(reader, "synset"); - synsetBySenseId.put(currentSenseId, synsetId); + if (synsetBySenseId.putIfAbsent(currentSenseId, synsetId) != null) { + throw malformed(reader.getLocation(), "Duplicate sense id " + currentSenseId, null); + } entryIdsBySynset.computeIfAbsent(synsetId, unused -> new ArrayList<>(2)) .add(currentEntryId); final List order = senseOrder.computeIfAbsent( @@ -242,7 +262,7 @@ private void startElement(XMLStreamReader reader) throws XMLStreamException { } case "Synset" -> { final String id = requireAttribute(reader, "id"); - final WordNetPos pos = parsePos(requireAttribute(reader, "partOfSpeech"), + final WordNetPOS pos = parsePos(requireAttribute(reader, "partOfSpeech"), reader.getLocation()); currentSynset = new RawSynset(id, pos, reader.getAttributeValue(null, "members"), line(reader.getLocation())); @@ -259,8 +279,14 @@ private void startElement(XMLStreamReader reader) throws XMLStreamException { if (currentSynset == null) { throw malformed(reader.getLocation(), "SynsetRelation outside a Synset", null); } - currentSynset.relations.add(new RawRelation(requireAttribute(reader, "relType"), - requireAttribute(reader, "target"), line(reader.getLocation()))); + final String relType = requireAttribute(reader, "relType"); + final String target = requireAttribute(reader, "target"); + // The escape-hatch type is skipped here exactly as it is for sense relations in + // build(): documented skip, not rejection. + if (!OTHER_RELATION.equals(relType)) { + currentSynset.relations.add( + new RawRelation(relType, target, line(reader.getLocation()))); + } } default -> { // Pronunciation, Form, Example, SyntacticBehaviour, ILIDefinition, and other @@ -284,7 +310,7 @@ private void endElement(String name) { } } - LexicalKnowledgeBase build() { + LexicalKnowledgeBase build() throws InvalidFormatException { // Every sense must point to a declared synset, with a consistent part of speech. for (final Map.Entry sense : synsetBySenseId.entrySet()) { final RawSynset target = rawSynsets.get(sense.getValue()); @@ -320,15 +346,20 @@ LexicalKnowledgeBase build() { return new InMemoryWordNetLexicon(synsetsById, senseOrder); } - private Map> resolveRelations(RawSynset raw) { + private Map> resolveRelations(RawSynset raw) + throws InvalidFormatException { final Map> typed = new LinkedHashMap<>(); for (final RawRelation relation : raw.relations) { final WordNetRelation type = parseRelation(relation.relType, raw.pos, relation.line); - if (!rawSynsets.containsKey(relation.target)) { + final RawSynset target = rawSynsets.get(relation.target); + if (target == null) { throw malformed(null, "Relation " + relation.relType + " at line " + relation.line + " on synset " + raw.id + " references undeclared synset " + relation.target, null); } - typed.computeIfAbsent(type, unused -> new LinkedHashSet<>()).add(relation.target); + // The target's canonical id instance, not the pointer's own string: a full lexicon + // carries hundreds of thousands of pointers, and sharing the synset table's instances + // keeps only one copy of each id in memory. + typed.computeIfAbsent(type, unused -> new LinkedHashSet<>()).add(target.id); } final Map> relations = new LinkedHashMap<>(typed.size() * 2); for (final Map.Entry> entry : typed.entrySet()) { @@ -337,10 +368,10 @@ private Map> resolveRelations(RawSynset raw) { return relations; } - private List memberLemmas(RawSynset raw) { + private List memberLemmas(RawSynset raw) throws InvalidFormatException { final List entryIds; if (raw.members != null && !raw.members.isEmpty()) { - entryIds = splitOnSpaces(raw.members); + entryIds = LemmaFolding.splitOnSpaces(raw.members); } else { final List fromSenses = entryIdsBySynset.get(raw.id); entryIds = fromSenses == null ? List.of() : fromSenses; @@ -368,21 +399,22 @@ private List memberLemmas(RawSynset raw) { return lemmas; } - private WordNetPos parsePos(String code, Location location) { - // The adjective satellite code normalizes to ADJECTIVE; see WordNetPos. + private WordNetPOS parsePos(String code, Location location) throws InvalidFormatException { + // The adjective satellite code normalizes to ADJECTIVE; see WordNetPOS. return switch (code) { - case "n" -> WordNetPos.NOUN; - case "v" -> WordNetPos.VERB; - case "a", "s" -> WordNetPos.ADJECTIVE; - case "r" -> WordNetPos.ADVERB; + case "n" -> WordNetPOS.NOUN; + case "v" -> WordNetPOS.VERB; + case "a", "s" -> WordNetPOS.ADJECTIVE; + case "r" -> WordNetPOS.ADVERB; default -> throw malformed(location, "Unknown part-of-speech code: " + code, null); }; } - private WordNetRelation parseRelation(String relType, WordNetPos sourcePos, int line) { + private WordNetRelation parseRelation(String relType, WordNetPOS sourcePos, int line) + throws InvalidFormatException { // Documents derived from Princeton data express verb groups as similar on verb synsets. if ("similar".equals(relType)) { - return sourcePos == WordNetPos.VERB ? WordNetRelation.VERB_GROUP + return sourcePos == WordNetPOS.VERB ? WordNetRelation.VERB_GROUP : WordNetRelation.SIMILAR_TO; } final WordNetRelation relation = RELATION_NAMES.get(relType); @@ -393,7 +425,7 @@ private WordNetRelation parseRelation(String relType, WordNetPos sourcePos, int } private String requireAttribute(XMLStreamReader reader, String attribute) - throws XMLStreamException { + throws InvalidFormatException { final String value = reader.getAttributeValue(null, attribute); if (value == null || value.isEmpty()) { throw malformed(reader.getLocation(), "Element " + reader.getLocalName() @@ -402,45 +434,28 @@ private String requireAttribute(XMLStreamReader reader, String attribute) return value; } - IllegalArgumentException malformed(Location location, String message, Throwable cause) { + InvalidFormatException malformed(Location location, String message, Throwable cause) { final int line = line(location); final String prefix = line < 0 ? "Malformed WN-LMF document " + resourceName + ": " : "Malformed WN-LMF document " + resourceName + " at line " + line + ": "; - return cause == null ? new IllegalArgumentException(prefix + message) - : new IllegalArgumentException(prefix + message, cause); + return cause == null ? new InvalidFormatException(prefix + message) + : new InvalidFormatException(prefix + message, cause); } private static int line(Location location) { return location == null ? -1 : location.getLineNumber(); } - - private static List splitOnSpaces(String value) { - final List parts = new ArrayList<>(4); - int start = 0; - while (start < value.length()) { - final int space = value.indexOf(' ', start); - if (space < 0) { - parts.add(value.substring(start)); - break; - } - if (space > start) { - parts.add(value.substring(start, space)); - } - start = space + 1; - } - return parts; - } } private static final class RawSynset { private final String id; - private final WordNetPos pos; + private final WordNetPOS pos; private final String members; private final int line; private final List relations = new ArrayList<>(4); private String gloss; - RawSynset(String id, WordNetPos pos, String members, int line) { + RawSynset(String id, WordNetPOS pos, String members, int line) { this.id = id; this.pos = pos; this.members = members; diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java index 3e40e05305..8c572ed3da 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java @@ -17,7 +17,6 @@ package opennlp.wordnet; import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -28,9 +27,10 @@ import java.util.List; import java.util.Map; +import opennlp.tools.util.InvalidFormatException; import opennlp.tools.wordnet.LexicalKnowledgeBase; import opennlp.tools.wordnet.Synset; -import opennlp.tools.wordnet.WordNetPos; +import opennlp.tools.wordnet.WordNetPOS; import opennlp.tools.wordnet.WordNetRelation; /** @@ -49,16 +49,17 @@ * file's 8-digit byte offset and part-of-speech letter, for example {@code wndb-00001740-n}. * The id is opaque to consumers; the format mirrors WN-LMF-style ids only for readability. * Adjective satellite lines ({@code ss_type} {@code s}) normalize to - * {@link WordNetPos#ADJECTIVE}, and the syntactic markers the adjective files append to some + * {@link WordNetPOS#ADJECTIVE}, and the syntactic markers the adjective files append to some * words ({@code (p)}, {@code (a)}, {@code (ip)}) are stripped from lemmas. Underscores in * stored lemmas become spaces. Sense order per lemma follows the index file's offset order, * which the format documents as most frequent first.

* - *

Errors. Malformed content fails loud with an {@link IllegalArgumentException} + *

Errors. Malformed content fails loud with an {@link InvalidFormatException} * naming the file and line: a missing file, a truncated or overlong line, a data line whose * offset field disagrees with its actual byte position (the format's documented seek * contract), an index entry referencing an offset with no data line, an undeclared pointer - * symbol, or a pointer to a nonexistent target.

+ * symbol, or a pointer to a nonexistent target. I/O failures propagate as + * {@link IOException}.

* *

The returned lexicon is immutable and safe for concurrent lookups.

*/ @@ -76,11 +77,12 @@ private WndbReader() { * {@code null} and must exist. * @return The loaded lexicon. * @throws IllegalArgumentException Thrown if {@code directory} is {@code null} or not a - * directory, a database file is missing, or any file is malformed; the message names the - * file and line. - * @throws UncheckedIOException Thrown if reading a file fails. + * directory. + * @throws InvalidFormatException Thrown if a database file is missing or any file is + * malformed; the message names the file and line. + * @throws IOException Thrown if reading a file fails. */ - public static LexicalKnowledgeBase read(Path directory) { + public static LexicalKnowledgeBase read(Path directory) throws IOException { if (directory == null) { throw new IllegalArgumentException("Directory must not be null"); } @@ -102,16 +104,16 @@ public static LexicalKnowledgeBase read(Path directory) { // The four part-of-speech file pairs of a WNDB directory. private enum FilePos { - NOUN("noun", 'n', WordNetPos.NOUN), - VERB("verb", 'v', WordNetPos.VERB), - ADJECTIVE("adj", 'a', WordNetPos.ADJECTIVE), - ADVERB("adv", 'r', WordNetPos.ADVERB); + NOUN("noun", 'n', WordNetPOS.NOUN), + VERB("verb", 'v', WordNetPOS.VERB), + ADJECTIVE("adj", 'a', WordNetPOS.ADJECTIVE), + ADVERB("adv", 'r', WordNetPOS.ADVERB); private final String suffix; private final char posChar; - private final WordNetPos pos; + private final WordNetPOS pos; - FilePos(String suffix, char posChar, WordNetPos pos) { + FilePos(String suffix, char posChar, WordNetPOS pos) { this.suffix = suffix; this.posChar = posChar; this.pos = pos; @@ -119,7 +121,7 @@ private enum FilePos { } private static void parseDataFile(Path directory, FilePos filePos, - Map rawSynsets) { + Map rawSynsets) throws IOException { final String fileName = "data." + filePos.suffix; final byte[] bytes = readAll(directory.resolve(fileName), fileName); int lineStart = 0; @@ -141,7 +143,8 @@ private static void parseDataFile(Path directory, FilePos filePos, } private static void parseDataLine(String line, int byteOffset, String fileName, int lineNumber, - FilePos filePos, Map rawSynsets) { + FilePos filePos, Map rawSynsets) + throws InvalidFormatException { final Tokenizer tokens = new Tokenizer(line, fileName, lineNumber); final String offsetField = tokens.next("synset_offset"); if (parseOffset(offsetField, tokens) != byteOffset) { @@ -198,17 +201,23 @@ private static void parseDataLine(String line, int byteOffset, String fileName, fileName, lineNumber)); } - private static Map resolve(Map rawSynsets) { + private static Map resolve(Map rawSynsets) + throws InvalidFormatException { final Map synsetsById = new LinkedHashMap<>(rawSynsets.size() * 2); for (final RawSynset raw : rawSynsets.values()) { final Map> typed = new LinkedHashMap<>(); for (final RawPointer pointer : raw.pointers) { - if (!rawSynsets.containsKey(pointer.targetId)) { - throw malformed(raw.fileName, raw.lineNumber, "Synset " + raw.id + " has a " + final RawSynset target = rawSynsets.get(pointer.targetId); + if (target == null) { + // The pointer's own line, so the error names exactly where the dangling pointer sits. + throw malformed(raw.fileName, pointer.lineNumber, "Synset " + raw.id + " has a " + pointer.relation + " pointer to nonexistent synset " + pointer.targetId); } + // The target's canonical id instance, not the pointer's freshly minted string: a full + // database carries hundreds of thousands of pointers, and sharing the synset table's + // instances keeps only one copy of each id in memory. typed.computeIfAbsent(pointer.relation, unused -> new LinkedHashSet<>()) - .add(pointer.targetId); + .add(target.id); } final Map> relations = new LinkedHashMap<>(typed.size() * 2); for (final Map.Entry> entry : typed.entrySet()) { @@ -221,7 +230,8 @@ private static Map resolve(Map rawSynsets) { private static void parseIndexFile(Path directory, FilePos filePos, Map rawSynsets, - Map> senses) { + Map> senses) + throws IOException { final String fileName = "index." + filePos.suffix; final byte[] bytes = readAll(directory.resolve(fileName), fileName); final String content = new String(bytes, StandardCharsets.ISO_8859_1); @@ -243,7 +253,8 @@ private static void parseIndexFile(Path directory, FilePos filePos, private static void parseIndexLine(String line, String fileName, int lineNumber, FilePos filePos, Map rawSynsets, - Map> senses) { + Map> senses) + throws InvalidFormatException { final Tokenizer tokens = new Tokenizer(line, fileName, lineNumber); final String lemma = tokens.next("lemma"); final String pos = tokens.next("pos"); @@ -293,7 +304,8 @@ private static void parseIndexLine(String line, String fileName, int lineNumber, } // Strips the documented adjective syntactic markers and turns underscores into spaces. - private static String cleanLemma(String word, String fileName, int lineNumber) { + private static String cleanLemma(String word, String fileName, int lineNumber) + throws InvalidFormatException { String cleaned = word; if (cleaned.endsWith(")")) { final int open = cleaned.lastIndexOf('('); @@ -309,7 +321,7 @@ private static String cleanLemma(String word, String fileName, int lineNumber) { return cleaned.replace('_', ' '); } - private static int parseOffset(String offset, Tokenizer tokens) { + private static int parseOffset(String offset, Tokenizer tokens) throws InvalidFormatException { if (offset.length() != 8) { throw tokens.malformedToken("Synset offset must be 8 digits, got: " + offset); } @@ -324,7 +336,7 @@ private static int parseOffset(String offset, Tokenizer tokens) { return value; } - private static char posChar(String pos, Tokenizer tokens) { + private static char posChar(String pos, Tokenizer tokens) throws InvalidFormatException { if (pos.length() == 1) { final char c = pos.charAt(0); if (c == 'n' || c == 'v' || c == 'a' || c == 'r') { @@ -334,20 +346,16 @@ private static char posChar(String pos, Tokenizer tokens) { throw tokens.malformedToken("Pointer pos must be one of n, v, a, r, got: " + pos); } - private static byte[] readAll(Path file, String fileName) { + private static byte[] readAll(Path file, String fileName) throws IOException { if (!Files.isRegularFile(file)) { - throw new IllegalArgumentException("Missing WNDB database file: " + file); - } - try { - return Files.readAllBytes(file); - } catch (IOException e) { - throw new UncheckedIOException("Unable to read WNDB database file " + file, e); + throw new InvalidFormatException("Missing WNDB database file: " + file); } + return Files.readAllBytes(file); } - private static IllegalArgumentException malformed(String fileName, int lineNumber, - String message) { - return new IllegalArgumentException( + private static InvalidFormatException malformed(String fileName, int lineNumber, + String message) { + return new InvalidFormatException( "Malformed WNDB file " + fileName + " at line " + lineNumber + ": " + message); } @@ -365,7 +373,7 @@ private static final class Tokenizer { this.lineNumber = lineNumber; } - String next(String field) { + String next(String field) throws InvalidFormatException { while (position < line.length() && line.charAt(position) == ' ') { position++; } @@ -379,18 +387,18 @@ String next(String field) { return line.substring(start, position); } - int nextInt(String field, int radix) { + int nextInt(String field, int radix) throws InvalidFormatException { final String token = next(field); try { return Integer.parseInt(token, radix); } catch (NumberFormatException e) { - throw new IllegalArgumentException(malformed(fileName, lineNumber, + throw new InvalidFormatException(malformed(fileName, lineNumber, "Field " + field + " is not a base-" + radix + " integer: " + token).getMessage(), e); } } // The remainder after the pipe separator, trimmed of the surrounding spaces. - String gloss() { + String gloss() throws InvalidFormatException { final String separator = next("gloss separator"); if (!"|".equals(separator)) { throw malformed(fileName, lineNumber, "Expected the | gloss separator, got: " + separator); @@ -406,7 +414,7 @@ String gloss() { return line.substring(start, end); } - IllegalArgumentException malformedToken(String message) { + InvalidFormatException malformedToken(String message) { return malformed(fileName, lineNumber, message); } } @@ -416,14 +424,14 @@ private record RawPointer(WordNetRelation relation, String targetId, int lineNum private static final class RawSynset { private final String id; - private final WordNetPos pos; + private final WordNetPOS pos; private final List lemmas; private final String gloss; private final List pointers; private final String fileName; private final int lineNumber; - RawSynset(String id, WordNetPos pos, List lemmas, String gloss, + RawSynset(String id, WordNetPOS pos, List lemmas, String gloss, List pointers, String fileName, int lineNumber) { this.id = id; this.pos = pos; diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/InMemoryWordNetLexiconTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/InMemoryWordNetLexiconTest.java new file mode 100644 index 0000000000..8a238832fb --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/InMemoryWordNetLexiconTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.wordnet.Synset; +import opennlp.tools.wordnet.WordNetPOS; +import opennlp.tools.wordnet.WordNetRelation; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Exercises the constructor's referential-integrity validation directly, with deliberately + * inconsistent maps a reader would never produce: any future reader relies on these checks, + * so they are pinned independently of both existing readers. + */ +public class InMemoryWordNetLexiconTest { + + private static Synset synset(String id, Map> relations) { + return new Synset(id, WordNetPOS.NOUN, List.of("lemma"), "a gloss", relations); + } + + @Test + void testAcceptsConsistentMaps() { + final Synset a = synset("a", Map.of(WordNetRelation.HYPERNYM, List.of("b"))); + final Synset b = synset("b", Map.of()); + final InMemoryWordNetLexicon lexicon = new InMemoryWordNetLexicon( + Map.of("a", a, "b", b), + Map.of(InMemoryWordNetLexicon.LemmaKey.of("lemma", WordNetPOS.NOUN), List.of("a", "b"))); + assertEquals(2, lexicon.size()); + assertEquals(List.of(a, b), lexicon.lookup("lemma", WordNetPOS.NOUN)); + } + + @Test + void testRejectsKeyThatDoesNotMatchSynsetId() { + final Map table = Map.of("wrong-key", synset("real-id", Map.of())); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new InMemoryWordNetLexicon(table, Map.of())); + assertTrue(e.getMessage().contains("wrong-key")); + } + + @Test + void testRejectsDanglingRelationTarget() { + final Map table = + Map.of("a", synset("a", Map.of(WordNetRelation.HYPERNYM, List.of("nope")))); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new InMemoryWordNetLexicon(table, Map.of())); + assertTrue(e.getMessage().contains("nope")); + assertTrue(e.getMessage().contains("HYPERNYM")); + } + + @Test + void testRejectsSenseOrderEntryWithUnknownSynset() { + final Map table = Map.of("a", synset("a", Map.of())); + final Map> senseOrder = + Map.of(InMemoryWordNetLexicon.LemmaKey.of("lemma", WordNetPOS.NOUN), List.of("missing")); + final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + () -> new InMemoryWordNetLexicon(table, senseOrder)); + assertTrue(e.getMessage().contains("missing")); + assertTrue(e.getMessage().contains("lemma")); + } + + @Test + void testRejectsNullMaps() { + assertThrows(IllegalArgumentException.class, () -> new InMemoryWordNetLexicon(null, Map.of())); + assertThrows(IllegalArgumentException.class, + () -> new InMemoryWordNetLexicon(Map.of(), null)); + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LemmaFoldingTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LemmaFoldingTest.java new file mode 100644 index 0000000000..f2f566d791 --- /dev/null +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LemmaFoldingTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.wordnet; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.wordnet.WordNetPOS; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Pins the shared fold and split behavior every user of {@link LemmaFolding} depends on: + * the exception lists, the sense index keys, and the WN-LMF members parsing all fold and + * split through this one implementation. + */ +public class LemmaFoldingTest { + + @Test + void testFoldLowercasesWithRootLocaleAndTreatsUnderscoreAsSpace() { + assertEquals("mice", LemmaFolding.fold("MICE")); + assertEquals("domestic dog", LemmaFolding.fold("Domestic_Dog")); + assertEquals("attorney general", LemmaFolding.fold("attorney_general")); + assertEquals("dog", LemmaFolding.fold("dog")); + assertEquals("", LemmaFolding.fold("")); + } + + @Test + void testSplitOnSpacesCollapsesRunsAndIgnoresEdges() { + assertEquals(List.of("a", "b", "c"), LemmaFolding.splitOnSpaces("a b c")); + assertEquals(List.of("a", "b"), LemmaFolding.splitOnSpaces("a b")); + assertEquals(List.of("a"), LemmaFolding.splitOnSpaces("a")); + assertEquals(List.of("a"), LemmaFolding.splitOnSpaces(" a ")); + assertEquals(List.of(), LemmaFolding.splitOnSpaces("")); + assertEquals(List.of(), LemmaFolding.splitOnSpaces(" ")); + } + + @Test + void testLemmaKeyAndExceptionLookupAgreeOnTheFold() { + // The agreement that makes Morphy correct: a key built from a stored written form and a + // query folded at lookup time land on the same canonical shape. + assertEquals(InMemoryWordNetLexicon.LemmaKey.of("Domestic_Dog", WordNetPOS.NOUN), + InMemoryWordNetLexicon.LemmaKey.of(LemmaFolding.fold("DOMESTIC_DOG"), WordNetPOS.NOUN)); + } +} diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexiconConcurrencyTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexiconConcurrencyTest.java index 1fdb4c0d7a..0292f7ffe2 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexiconConcurrencyTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LexiconConcurrencyTest.java @@ -25,7 +25,7 @@ import org.junit.jupiter.api.Test; import opennlp.tools.wordnet.LexicalKnowledgeBase; -import opennlp.tools.wordnet.WordNetPos; +import opennlp.tools.wordnet.WordNetPOS; import opennlp.tools.wordnet.WordNetRelation; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -71,20 +71,20 @@ void testConcurrentLookupsSeeConsistentResults() throws InterruptedException { } private static void verifyOnce(LexicalKnowledgeBase lexicon, Queue problems) { - if (!"wndb-00001075-n".equals(lexicon.lookup("dog", WordNetPos.NOUN).get(0).id())) { + if (!"wndb-00001075-n".equals(lexicon.lookup("dog", WordNetPOS.NOUN).get(0).id())) { problems.add("Wrong dog lookup"); } - if (lexicon.lookup("run", WordNetPos.NOUN).size() != 2) { + if (lexicon.lookup("run", WordNetPOS.NOUN).size() != 2) { problems.add("Wrong run sense count"); } if (!List.of("wndb-00001160-n") .equals(lexicon.related("wndb-00001075-n", WordNetRelation.HYPERNYM))) { problems.add("Wrong dog hypernym"); } - if (lexicon.contains("zebra", WordNetPos.NOUN)) { + if (lexicon.contains("zebra", WordNetPOS.NOUN)) { problems.add("Phantom zebra"); } - if (!lexicon.contains("walk", WordNetPos.VERB)) { + if (!lexicon.contains("walk", WordNetPOS.VERB)) { problems.add("Missing walk verb"); } } diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyExceptionsTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyExceptionsTest.java index a9d810173d..b8ed76181d 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyExceptionsTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyExceptionsTest.java @@ -24,7 +24,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import opennlp.tools.wordnet.WordNetPos; +import opennlp.tools.util.InvalidFormatException; +import opennlp.tools.wordnet.WordNetPOS; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -33,32 +34,36 @@ public class MorphyExceptionsTest { static MorphyExceptions fixture() { - return MorphyExceptions.load(WndbReaderTest.fixtureDirectory()); + try { + return MorphyExceptions.load(WndbReaderTest.fixtureDirectory()); + } catch (IOException e) { + throw new IllegalStateException("Unexpected IOException reading the fixture lists", e); + } } @Test void testLookupPerPartOfSpeech() { final MorphyExceptions exceptions = fixture(); - assertEquals(List.of("mouse"), exceptions.lookup("mice", WordNetPos.NOUN)); - assertEquals(List.of("go"), exceptions.lookup("went", WordNetPos.VERB)); - assertEquals(List.of("good"), exceptions.lookup("better", WordNetPos.ADJECTIVE)); - assertEquals(List.of("well"), exceptions.lookup("best", WordNetPos.ADVERB)); + assertEquals(List.of("mouse"), exceptions.lookup("mice", WordNetPOS.NOUN)); + assertEquals(List.of("go"), exceptions.lookup("went", WordNetPOS.VERB)); + assertEquals(List.of("good"), exceptions.lookup("better", WordNetPOS.ADJECTIVE)); + assertEquals(List.of("well"), exceptions.lookup("best", WordNetPOS.ADVERB)); // Entries are part-of-speech scoped: went is only a verb exception. - assertTrue(exceptions.lookup("went", WordNetPos.NOUN).isEmpty()); - assertTrue(exceptions.lookup("dog", WordNetPos.NOUN).isEmpty()); + assertTrue(exceptions.lookup("went", WordNetPOS.NOUN).isEmpty()); + assertTrue(exceptions.lookup("dog", WordNetPOS.NOUN).isEmpty()); } @Test void testLookupFoldsCase() { - assertEquals(List.of("mouse"), fixture().lookup("Mice", WordNetPos.NOUN)); - assertEquals(List.of("mouse"), fixture().lookup("MICE", WordNetPos.NOUN)); + assertEquals(List.of("mouse"), fixture().lookup("Mice", WordNetPOS.NOUN)); + assertEquals(List.of("mouse"), fixture().lookup("MICE", WordNetPOS.NOUN)); } @Test void testLookupRejectsNulls() { final MorphyExceptions exceptions = fixture(); assertThrows(IllegalArgumentException.class, - () -> exceptions.lookup(null, WordNetPos.NOUN)); + () -> exceptions.lookup(null, WordNetPOS.NOUN)); assertThrows(IllegalArgumentException.class, () -> exceptions.lookup("mice", null)); } @@ -74,7 +79,7 @@ void testLoadRejectsMissingFile(@TempDir Path tempDir) throws IOException { Files.writeString(tempDir.resolve("noun.exc"), "mice mouse\n"); Files.writeString(tempDir.resolve("verb.exc"), "went go\n"); Files.writeString(tempDir.resolve("adj.exc"), "better good\n"); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> MorphyExceptions.load(tempDir)); assertTrue(e.getMessage().contains("adv.exc")); } @@ -85,7 +90,7 @@ void testLoadRejectsMalformedLine(@TempDir Path tempDir) throws IOException { Files.writeString(tempDir.resolve("verb.exc"), "went go\n"); Files.writeString(tempDir.resolve("adj.exc"), "better good\n"); Files.writeString(tempDir.resolve("adv.exc"), "best well\n"); - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> MorphyExceptions.load(tempDir)); assertTrue(e.getMessage().contains("noun.exc")); assertTrue(e.getMessage().contains("line 2")); @@ -98,6 +103,6 @@ void testMultipleBaseFormsKeepFileOrder(@TempDir Path tempDir) throws IOExceptio Files.writeString(tempDir.resolve("adj.exc"), "better good\n"); Files.writeString(tempDir.resolve("adv.exc"), "best well\n"); assertEquals(List.of("axis", "ax"), - MorphyExceptions.load(tempDir).lookup("axes", WordNetPos.NOUN)); + MorphyExceptions.load(tempDir).lookup("axes", WordNetPOS.NOUN)); } } diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java index 895d6fab86..ecbbfda9eb 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java @@ -22,7 +22,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; -import opennlp.tools.wordnet.WordNetPos; +import opennlp.tools.wordnet.WordNetPOS; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -88,6 +88,10 @@ void testLemmatizesToken(String token, String tag, String lemma) { "dog, DT", "dog, XYZ", "dogs, ''", + // Multi-letter closed-class tags that merely begin with a WordNet letter code are not + // adjective lookups: AUX was must be unknown, and AUX taller must not detach to tall. + "was, AUX", + "taller, AUX", }) void testUnknownYieldsMarker(String token, String tag) { assertEquals("O", one(token, tag)); @@ -130,17 +134,23 @@ void testWorksIdenticallyOverTheWnLmfLexicon() { @Test void testPosFromTagMapping() { - assertEquals(WordNetPos.NOUN, MorphyLemmatizer.posFromTag("NNP")); - assertEquals(WordNetPos.VERB, MorphyLemmatizer.posFromTag("VBZ")); - assertEquals(WordNetPos.ADJECTIVE, MorphyLemmatizer.posFromTag("JJ")); - assertEquals(WordNetPos.ADVERB, MorphyLemmatizer.posFromTag("RBR")); - assertEquals(WordNetPos.ADJECTIVE, MorphyLemmatizer.posFromTag("a")); - assertEquals(WordNetPos.ADJECTIVE, MorphyLemmatizer.posFromTag("s")); - assertEquals(WordNetPos.ADJECTIVE, MorphyLemmatizer.posFromTag("ADJ")); - assertEquals(WordNetPos.ADVERB, MorphyLemmatizer.posFromTag("ADV")); - assertEquals(WordNetPos.ADVERB, MorphyLemmatizer.posFromTag("r")); + assertEquals(WordNetPOS.NOUN, MorphyLemmatizer.posFromTag("NNP")); + assertEquals(WordNetPOS.VERB, MorphyLemmatizer.posFromTag("VBZ")); + assertEquals(WordNetPOS.ADJECTIVE, MorphyLemmatizer.posFromTag("JJ")); + assertEquals(WordNetPOS.ADVERB, MorphyLemmatizer.posFromTag("RBR")); + assertEquals(WordNetPOS.ADJECTIVE, MorphyLemmatizer.posFromTag("a")); + assertEquals(WordNetPOS.ADJECTIVE, MorphyLemmatizer.posFromTag("s")); + assertEquals(WordNetPOS.ADJECTIVE, MorphyLemmatizer.posFromTag("ADJ")); + assertEquals(WordNetPOS.ADVERB, MorphyLemmatizer.posFromTag("ADV")); + assertEquals(WordNetPOS.ADVERB, MorphyLemmatizer.posFromTag("r")); assertNull(MorphyLemmatizer.posFromTag("DT")); assertNull(MorphyLemmatizer.posFromTag("")); + // The letter codes a and s match only as one-letter tags: multi-letter tags beginning + // with those letters are closed-class or symbol tags, never adjectives. + assertNull(MorphyLemmatizer.posFromTag("AUX")); + assertNull(MorphyLemmatizer.posFromTag("ADP")); + assertNull(MorphyLemmatizer.posFromTag("SCONJ")); + assertNull(MorphyLemmatizer.posFromTag("SYM")); } @Test diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/ReaderEquivalenceTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/ReaderEquivalenceTest.java index d7119d2f9d..b9f3523cbc 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/ReaderEquivalenceTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/ReaderEquivalenceTest.java @@ -26,7 +26,7 @@ import opennlp.tools.wordnet.LexicalKnowledgeBase; import opennlp.tools.wordnet.Synset; -import opennlp.tools.wordnet.WordNetPos; +import opennlp.tools.wordnet.WordNetPOS; import opennlp.tools.wordnet.WordNetRelation; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -75,7 +75,7 @@ void testLookupAgreesForEveryLemmaAndPos() { "Sense sequence for " + lemma + " as " + synset.pos()); } } - for (final WordNetPos pos : WordNetPos.values()) { + for (final WordNetPOS pos : WordNetPOS.values()) { assertEquals(lmf.contains("dog", pos), wndb.contains("dog", pos)); } } @@ -84,8 +84,8 @@ void testLookupAgreesForEveryLemmaAndPos() { void testSenseOrderAgreesForMultiSenseLemma() { final LexicalKnowledgeBase lmf = WnLmfReaderTest.fixture(); final LexicalKnowledgeBase wndb = WndbReaderTest.fixture(); - final List lmfOrder = glosses(lmf.lookup("run", WordNetPos.NOUN)); - final List wndbOrder = glosses(wndb.lookup("run", WordNetPos.NOUN)); + final List lmfOrder = glosses(lmf.lookup("run", WordNetPOS.NOUN)); + final List wndbOrder = glosses(wndb.lookup("run", WordNetPOS.NOUN)); assertEquals(2, lmfOrder.size()); assertEquals(lmfOrder, wndbOrder); } diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java index 37dd081bbb..5d48d17969 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WnLmfReaderTest.java @@ -27,14 +27,16 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import opennlp.tools.util.InvalidFormatException; import opennlp.tools.wordnet.LexicalKnowledgeBase; import opennlp.tools.wordnet.Synset; -import opennlp.tools.wordnet.WordNetPos; +import opennlp.tools.wordnet.WordNetPOS; import opennlp.tools.wordnet.WordNetRelation; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -49,7 +51,7 @@ static LexicalKnowledgeBase fixture() { } } - private static LexicalKnowledgeBase parse(String document) { + private static LexicalKnowledgeBase parse(String document) throws IOException { return WnLmfReader.read( new ByteArrayInputStream(document.getBytes(StandardCharsets.UTF_8)), "inline.xml"); } @@ -62,11 +64,11 @@ private static String wrap(String body) { @Test void testLookupReturnsSynsetWithAllComponents() { - final List senses = fixture().lookup("dog", WordNetPos.NOUN); + final List senses = fixture().lookup("dog", WordNetPOS.NOUN); assertEquals(1, senses.size()); final Synset dog = senses.get(0); assertEquals("mini-n1", dog.id()); - assertEquals(WordNetPos.NOUN, dog.pos()); + assertEquals(WordNetPOS.NOUN, dog.pos()); assertEquals(List.of("dog", "domestic dog"), dog.lemmas()); assertEquals("a domesticated canid", dog.gloss()); assertEquals(List.of("mini-n2"), dog.related(WordNetRelation.HYPERNYM)); @@ -75,13 +77,13 @@ void testLookupReturnsSynsetWithAllComponents() { @Test void testLookupFoldsCaseAndUnderscore() { final LexicalKnowledgeBase lexicon = fixture(); - assertEquals("mini-n1", lexicon.lookup("Domestic_Dog", WordNetPos.NOUN).get(0).id()); - assertEquals("mini-n1", lexicon.lookup("DOG", WordNetPos.NOUN).get(0).id()); + assertEquals("mini-n1", lexicon.lookup("Domestic_Dog", WordNetPOS.NOUN).get(0).id()); + assertEquals("mini-n1", lexicon.lookup("DOG", WordNetPOS.NOUN).get(0).id()); } @Test void testLookupKeepsSenseOrder() { - final List runSenses = fixture().lookup("run", WordNetPos.NOUN); + final List runSenses = fixture().lookup("run", WordNetPOS.NOUN); assertEquals(List.of("mini-n5", "mini-n9"), runSenses.stream().map(Synset::id).toList()); } @@ -89,10 +91,10 @@ void testLookupKeepsSenseOrder() { @Test void testLookupIsPosScoped() { final LexicalKnowledgeBase lexicon = fixture(); - assertEquals(1, lexicon.lookup("run", WordNetPos.VERB).size()); - assertTrue(lexicon.lookup("dog", WordNetPos.VERB).isEmpty()); - assertFalse(lexicon.contains("walk", WordNetPos.NOUN)); - assertTrue(lexicon.contains("walk", WordNetPos.VERB)); + assertEquals(1, lexicon.lookup("run", WordNetPOS.VERB).size()); + assertTrue(lexicon.lookup("dog", WordNetPOS.VERB).isEmpty()); + assertFalse(lexicon.contains("walk", WordNetPOS.NOUN)); + assertTrue(lexicon.contains("walk", WordNetPOS.VERB)); } @Test @@ -104,6 +106,16 @@ void testRelationNavigation() { assertEquals(List.of("mini-v4"), lexicon.related("mini-v1", WordNetRelation.HYPERNYM)); } + @Test + void testRelationTargetSharesCanonicalIdInstance() { + final LexicalKnowledgeBase lexicon = fixture(); + final String target = lexicon.synset("mini-n1").orElseThrow() + .related(WordNetRelation.HYPERNYM).get(0); + // Not just equal: the identical instance from the synset table, so a loaded lexicon keeps + // one copy of each id no matter how many relations point at it. + assertSame(lexicon.synset("mini-n2").orElseThrow().id(), target); + } + @Test void testSenseRelationsAreLiftedToSynsetLevel() { final LexicalKnowledgeBase lexicon = fixture(); @@ -117,17 +129,35 @@ void testSenseRelationsAreLiftedToSynsetLevel() { @Test void testSatelliteNormalizesToAdjective() { - final List senses = fixture().lookup("large", WordNetPos.ADJECTIVE); + final List senses = fixture().lookup("large", WordNetPOS.ADJECTIVE); assertEquals(1, senses.size()); - assertEquals(WordNetPos.ADJECTIVE, senses.get(0).pos()); + assertEquals(WordNetPOS.ADJECTIVE, senses.get(0).pos()); assertEquals(List.of("mini-a4"), fixture().related("mini-a3", WordNetRelation.SIMILAR_TO)); assertEquals(List.of("mini-a3"), fixture().related("mini-a4", WordNetRelation.SIMILAR_TO)); } + @Test + void testSimilarOnVerbSynsetMapsToVerbGroup() throws IOException { + // Documents derived from Princeton data express verb groups as similar on verb synsets; + // the fixture only carries similar on adjectives, so this pins the verb branch directly. + final LexicalKnowledgeBase lexicon = parse(wrap( + "" + + "" + + "" + + "" + + "" + + "produce musical tones" + + "" + + "" + + "sing monotonously")); + assertEquals(List.of("t-v2"), lexicon.related("t-v1", WordNetRelation.VERB_GROUP)); + assertTrue(lexicon.related("t-v1", WordNetRelation.SIMILAR_TO).isEmpty()); + } + @Test void testUnknownLemmaOrSynsetIsEmpty() { final LexicalKnowledgeBase lexicon = fixture(); - assertTrue(lexicon.lookup("zebra", WordNetPos.NOUN).isEmpty()); + assertTrue(lexicon.lookup("zebra", WordNetPOS.NOUN).isEmpty()); assertTrue(lexicon.synset("mini-n99").isEmpty()); } @@ -139,7 +169,7 @@ void testReadPath(@TempDir Path tempDir) throws IOException { + "" + "a feline")); final LexicalKnowledgeBase lexicon = WnLmfReader.read(file); - assertEquals("a feline", lexicon.lookup("cat", WordNetPos.NOUN).get(0).gloss()); + assertEquals("a feline", lexicon.lookup("cat", WordNetPOS.NOUN).get(0).gloss()); } @Test @@ -157,7 +187,21 @@ void testReadStreamRejectsNulls() { } @Test - void testSkipsDoctypeDeclaration() { + void testStreamReadFailurePropagatesAsIOException() { + final InputStream failing = new InputStream() { + @Override + public int read() throws IOException { + throw new IOException("Simulated stream failure"); + } + }; + final IOException e = + assertThrows(IOException.class, () -> WnLmfReader.read(failing, "failing.xml")); + // The I/O failure must surface as itself, not be misreported as a malformed document. + assertFalse(e instanceof InvalidFormatException); + } + + @Test + void testSkipsDoctypeDeclaration() throws IOException { // Real Open English WordNet releases ship exactly this shape: a DOCTYPE naming the schema // DTD by an unreachable SYSTEM identifier (example.invalid is the RFC 2606 reserved domain // that must never resolve). The reader must parse past it without attempting to fetch it. @@ -169,7 +213,7 @@ void testSkipsDoctypeDeclaration() { + "a feline" + ""; final LexicalKnowledgeBase lexicon = parse(document); - assertEquals("a feline", lexicon.lookup("cat", WordNetPos.NOUN).get(0).gloss()); + assertEquals("a feline", lexicon.lookup("cat", WordNetPOS.NOUN).get(0).gloss()); } @Test @@ -187,21 +231,21 @@ void testInternalSubsetEntityIsNeverExpanded(@TempDir Path tempDir) throws IOExc + "" + "a feline" + ""; - final IllegalArgumentException e = - assertThrows(IllegalArgumentException.class, () -> parse(document)); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> parse(document)); assertFalse(e.getMessage().contains("xxe-marker-should-never-appear")); } @Test void testRejectsTruncatedDocument() { - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse("\n parse( + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( wrap("" + ""))); assertTrue(e.getMessage().contains("synset")); @@ -209,7 +253,7 @@ void testRejectsSenseWithoutSynsetAttribute() { @Test void testRejectsSenseToUndeclaredSynset() { - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> parse( + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( wrap("" + ""))); assertTrue(e.getMessage().contains("t-9")); @@ -217,7 +261,7 @@ void testRejectsSenseToUndeclaredSynset() { @Test void testRejectsRelationToUndeclaredSynset() { - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> parse( + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( wrap("" + "" + "a feline" @@ -227,7 +271,7 @@ void testRejectsRelationToUndeclaredSynset() { @Test void testRejectsUnknownRelationType() { - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> parse( + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( wrap("" + "" + "a feline" @@ -236,7 +280,7 @@ void testRejectsUnknownRelationType() { } @Test - void testSkipsOtherRelationType() { + void testSkipsOtherRelationTypeOnSenseRelation() throws IOException { final LexicalKnowledgeBase lexicon = parse( wrap("" + "" @@ -245,9 +289,21 @@ void testSkipsOtherRelationType() { assertTrue(lexicon.synset("t-1").orElseThrow().relations().isEmpty()); } + @Test + void testSkipsOtherRelationTypeOnSynsetRelation() throws IOException { + // The DTD permits relType="other" on SynsetRelation too, and several OMW-family wordnets + // emit it; it is skipped exactly like the SenseRelation case, not rejected. + final LexicalKnowledgeBase lexicon = parse( + wrap("" + + "" + + "a feline" + + "")); + assertTrue(lexicon.synset("t-1").orElseThrow().relations().isEmpty()); + } + @Test void testRejectsUnknownPartOfSpeech() { - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> parse( + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( wrap("" + "" + "a feline"))); @@ -256,8 +312,94 @@ void testRejectsUnknownPartOfSpeech() { @Test void testRejectsSynsetWithoutMembers() { - final IllegalArgumentException e = assertThrows(IllegalArgumentException.class, () -> parse( + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( wrap("orphan"))); assertTrue(e.getMessage().contains("t-1")); } + + @Test + void testRejectsDuplicateSynsetId() { + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( + wrap("" + + "" + + "a feline" + + "a repeat"))); + assertTrue(e.getMessage().contains("Duplicate synset id t-1")); + } + + @Test + void testRejectsDuplicateLexicalEntryId() { + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( + wrap("" + + "" + + "" + + "" + + "a feline"))); + assertTrue(e.getMessage().contains("Duplicate lexical entry id t-cat-n")); + } + + @Test + void testRejectsDuplicateSenseId() { + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( + wrap("" + + "" + + "" + + "a feline" + + "a second"))); + assertTrue(e.getMessage().contains("Duplicate sense id t-cat-n-1")); + } + + @Test + void testRejectsSynsetMemberPosMismatch() { + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( + wrap("" + + "" + + "a feline"))); + assertTrue(e.getMessage().contains("t-cat-n")); + assertTrue(e.getMessage().contains("VERB")); + assertTrue(e.getMessage().contains("NOUN")); + } + + @Test + void testRejectsSenseRelationToUndeclaredSense() { + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( + wrap("" + + "" + + "" + + "a feline"))); + assertTrue(e.getMessage().contains("t-ghost-1")); + } + + @Test + void testRejectsLemmaOutsideLexicalEntry() { + final InvalidFormatException e = assertThrows(InvalidFormatException.class, + () -> parse(wrap(""))); + assertTrue(e.getMessage().contains("Lemma outside a LexicalEntry")); + } + + @Test + void testRejectsSenseBeforeLemma() { + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( + wrap("" + + "" + + "a feline"))); + assertTrue(e.getMessage().contains("Sense before its entry's Lemma")); + } + + @Test + void testRejectsSenseRelationOutsideSense() { + final InvalidFormatException e = assertThrows(InvalidFormatException.class, () -> parse( + wrap("" + + "" + + "" + + "a feline"))); + assertTrue(e.getMessage().contains("SenseRelation outside a Sense")); + } + + @Test + void testRejectsSynsetRelationOutsideSynset() { + final InvalidFormatException e = assertThrows(InvalidFormatException.class, + () -> parse(wrap(""))); + assertTrue(e.getMessage().contains("SynsetRelation outside a Synset")); + } } diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WndbReaderTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WndbReaderTest.java index 07843d275d..5931957e3b 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WndbReaderTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/WndbReaderTest.java @@ -23,18 +23,21 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import java.util.Locale; import java.util.function.UnaryOperator; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import opennlp.tools.util.InvalidFormatException; import opennlp.tools.wordnet.LexicalKnowledgeBase; import opennlp.tools.wordnet.Synset; -import opennlp.tools.wordnet.WordNetPos; +import opennlp.tools.wordnet.WordNetPOS; import opennlp.tools.wordnet.WordNetRelation; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -54,16 +57,20 @@ static Path fixtureDirectory() { } static LexicalKnowledgeBase fixture() { - return WndbReader.read(fixtureDirectory()); + try { + return WndbReader.read(fixtureDirectory()); + } catch (IOException e) { + throw new IllegalStateException("Unexpected IOException reading the WNDB fixture", e); + } } @Test void testLookupReturnsSynsetWithAllComponents() { - final List senses = fixture().lookup("dog", WordNetPos.NOUN); + final List senses = fixture().lookup("dog", WordNetPOS.NOUN); assertEquals(1, senses.size()); final Synset dog = senses.get(0); assertEquals(DOG_ID, dog.id()); - assertEquals(WordNetPos.NOUN, dog.pos()); + assertEquals(WordNetPOS.NOUN, dog.pos()); assertEquals(List.of("dog", "domestic dog"), dog.lemmas()); assertEquals("a domesticated canid", dog.gloss()); assertEquals(List.of(CANID_ID), dog.related(WordNetRelation.HYPERNYM)); @@ -72,14 +79,14 @@ void testLookupReturnsSynsetWithAllComponents() { @Test void testLookupFoldsCaseAndUnderscore() { final LexicalKnowledgeBase lexicon = fixture(); - assertEquals(DOG_ID, lexicon.lookup("Domestic_Dog", WordNetPos.NOUN).get(0).id()); - assertEquals(DOG_ID, lexicon.lookup("DOG", WordNetPos.NOUN).get(0).id()); + assertEquals(DOG_ID, lexicon.lookup("Domestic_Dog", WordNetPOS.NOUN).get(0).id()); + assertEquals(DOG_ID, lexicon.lookup("DOG", WordNetPOS.NOUN).get(0).id()); } @Test void testLookupKeepsIndexSenseOrder() { assertEquals(List.of("wndb-00001427-n", "wndb-00001669-n"), - fixture().lookup("run", WordNetPos.NOUN).stream().map(Synset::id).toList()); + fixture().lookup("run", WordNetPOS.NOUN).stream().map(Synset::id).toList()); } @Test @@ -92,6 +99,16 @@ void testRelationNavigation() { lexicon.related("wndb-00001427-n", WordNetRelation.DERIVATIONALLY_RELATED)); } + @Test + void testRelationTargetSharesCanonicalIdInstance() { + final LexicalKnowledgeBase lexicon = fixture(); + final String target = lexicon.synset(DOG_ID).orElseThrow() + .related(WordNetRelation.HYPERNYM).get(0); + // Not just equal: the identical instance from the synset table, so a loaded lexicon keeps + // one copy of each id no matter how many pointers reference it. + assertSame(lexicon.synset(CANID_ID).orElseThrow().id(), target); + } + @Test void testLexicalPointersSurfaceAtSynsetLevel() { final LexicalKnowledgeBase lexicon = fixture(); @@ -104,18 +121,41 @@ void testLexicalPointersSurfaceAtSynsetLevel() { @Test void testSatelliteNormalizesToAdjectiveAndMarkerIsStripped() { final LexicalKnowledgeBase lexicon = fixture(); - final Synset large = lexicon.lookup("large", WordNetPos.ADJECTIVE).get(0); - assertEquals(WordNetPos.ADJECTIVE, large.pos()); + final Synset large = lexicon.lookup("large", WordNetPOS.ADJECTIVE).get(0); + assertEquals(WordNetPOS.ADJECTIVE, large.pos()); assertEquals(List.of("wndb-00001211-a"), large.related(WordNetRelation.SIMILAR_TO)); // short is stored as short(p); the syntactic marker is not part of the lemma. assertEquals(List.of("short"), - lexicon.lookup("short", WordNetPos.ADJECTIVE).get(0).lemmas()); + lexicon.lookup("short", WordNetPOS.ADJECTIVE).get(0).lemmas()); + } + + @Test + void testVerbGroupPointerMapsToVerbGroup(@TempDir Path tempDir) throws IOException { + // The fixture has no $ pointer, so the VERB_GROUP mapping is pinned against a minimal + // constructed database whose byte offsets are computed, not hard-coded: every offset field + // is exactly eight digits, so the second line's position is independent of the digit values. + writeEmptyDb(tempDir, "noun", "adj", "adv"); + final String template = + "00000000 29 v 01 sing 0 001 $ XXXXXXXX v 0000 00 | produce musical tones"; + final String off2 = String.format(Locale.ROOT, "%08d", template.length() + 1); + final String line1 = template.replace("XXXXXXXX", off2); + final String line2 = off2 + " 29 v 01 chant 0 001 $ 00000000 v 0000 00 | sing monotonously"; + Files.writeString(tempDir.resolve("data.verb"), line1 + "\n" + line2 + "\n", + StandardCharsets.ISO_8859_1); + Files.writeString(tempDir.resolve("index.verb"), + "chant v 1 1 $ 1 0 " + off2 + "\nsing v 1 1 $ 1 0 00000000\n", + StandardCharsets.ISO_8859_1); + final LexicalKnowledgeBase lexicon = WndbReader.read(tempDir); + assertEquals(List.of("wndb-" + off2 + "-v"), + lexicon.related("wndb-00000000-v", WordNetRelation.VERB_GROUP)); + assertEquals(List.of("wndb-00000000-v"), + lexicon.related("wndb-" + off2 + "-v", WordNetRelation.VERB_GROUP)); } @Test void testUnknownLemmaOrSynsetIsEmpty() { final LexicalKnowledgeBase lexicon = fixture(); - assertTrue(lexicon.lookup("zebra", WordNetPos.NOUN).isEmpty()); + assertTrue(lexicon.lookup("zebra", WordNetPOS.NOUN).isEmpty()); assertTrue(lexicon.synset("wndb-99999999-n").isEmpty()); } @@ -130,8 +170,8 @@ void testRejectsNullAndMissingDirectory(@TempDir Path tempDir) { void testRejectsMissingDatabaseFile(@TempDir Path tempDir) throws IOException { copyFixture(tempDir); Files.delete(tempDir.resolve("data.verb")); - final IllegalArgumentException e = - assertThrows(IllegalArgumentException.class, () -> WndbReader.read(tempDir)); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> WndbReader.read(tempDir)); assertTrue(e.getMessage().contains("data.verb")); } @@ -140,8 +180,8 @@ void testRejectsIndexOffsetWithoutDataLine(@TempDir Path tempDir) throws IOExcep copyFixture(tempDir); mutate(tempDir, "index.noun", line -> line.startsWith("berry ") ? line.replace("00001564", "00001565") : line); - final IllegalArgumentException e = - assertThrows(IllegalArgumentException.class, () -> WndbReader.read(tempDir)); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> WndbReader.read(tempDir)); assertTrue(e.getMessage().contains("berry")); assertTrue(e.getMessage().contains("00001565")); } @@ -151,8 +191,8 @@ void testRejectsDataOffsetFieldMismatch(@TempDir Path tempDir) throws IOExceptio copyFixture(tempDir); mutate(tempDir, "data.noun", line -> line.replace("00001503 03 n 01 box", "00001504 03 n 01 box")); - final IllegalArgumentException e = - assertThrows(IllegalArgumentException.class, () -> WndbReader.read(tempDir)); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> WndbReader.read(tempDir)); assertTrue(e.getMessage().contains("disagrees")); } @@ -161,8 +201,8 @@ void testRejectsTruncatedDataLine(@TempDir Path tempDir) throws IOException { copyFixture(tempDir); mutate(tempDir, "data.noun", line -> line.startsWith("00001564") ? line.substring(0, line.indexOf(" 000 |")) : line); - final IllegalArgumentException e = - assertThrows(IllegalArgumentException.class, () -> WndbReader.read(tempDir)); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> WndbReader.read(tempDir)); assertTrue(e.getMessage().contains("data.noun")); assertTrue(e.getMessage().contains("Truncated")); } @@ -172,8 +212,8 @@ void testRejectsUndeclaredPointerSymbol(@TempDir Path tempDir) throws IOExceptio copyFixture(tempDir); mutate(tempDir, "data.noun", line -> line.replace("001 @ 00001160 n 0000", "001 ? 00001160 n 0000")); - final IllegalArgumentException e = - assertThrows(IllegalArgumentException.class, () -> WndbReader.read(tempDir)); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> WndbReader.read(tempDir)); assertTrue(e.getMessage().contains("Undeclared pointer symbol: ?")); } @@ -182,11 +222,34 @@ void testRejectsPointerToNonexistentSynset(@TempDir Path tempDir) throws IOExcep copyFixture(tempDir); mutate(tempDir, "data.noun", line -> line.replace("001 @ 00001160 n 0000", "001 @ 00009999 n 0000")); - final IllegalArgumentException e = - assertThrows(IllegalArgumentException.class, () -> WndbReader.read(tempDir)); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> WndbReader.read(tempDir)); assertTrue(e.getMessage().contains("wndb-00009999-n")); } + @Test + void testDanglingPointerErrorNamesPointerLine(@TempDir Path tempDir) throws IOException { + // A constructed database with no preamble, so the dangling pointer sits on a known line + // and the error message can be pinned to name it. + writeEmptyDb(tempDir, "noun", "adj", "adv"); + Files.writeString(tempDir.resolve("data.verb"), + "00000000 29 v 01 sing 0 001 $ 00009999 v 0000 00 | produce musical tones\n", + StandardCharsets.ISO_8859_1); + Files.writeString(tempDir.resolve("index.verb"), "sing v 1 1 $ 1 0 00000000\n", + StandardCharsets.ISO_8859_1); + final InvalidFormatException e = + assertThrows(InvalidFormatException.class, () -> WndbReader.read(tempDir)); + assertTrue(e.getMessage().contains("wndb-00009999-v")); + assertTrue(e.getMessage().contains("line 1")); + } + + private static void writeEmptyDb(Path directory, String... suffixes) throws IOException { + for (final String suffix : suffixes) { + Files.writeString(directory.resolve("data." + suffix), ""); + Files.writeString(directory.resolve("index." + suffix), ""); + } + } + private static void copyFixture(Path target) throws IOException { try (var files = Files.list(fixtureDirectory())) { for (final Path file : files.toList()) { diff --git a/pom.xml b/pom.xml index c74e2dc021..bf8a790d5f 100644 --- a/pom.xml +++ b/pom.xml @@ -216,6 +216,12 @@ ${project.version} + + opennlp-wordnet + ${project.groupId} + ${project.version} + + opennlp-uima ${project.groupId} From 36c8ad400a8070bd97dc4149db3e1bb2cc301548 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 12 Jul 2026 15:32:06 -0400 Subject: [PATCH 09/12] OPENNLP-1880: Trim commentary and tighten javadoc per review conventions --- .../tools/wordnet/LexicalKnowledgeBase.java | 24 +-- .../java/opennlp/tools/wordnet/Synset.java | 15 +- .../opennlp/tools/wordnet/WordNetPOS.java | 14 +- .../tools/wordnet/WordNetRelation.java | 25 +-- .../wordnet/InMemoryWordNetLexicon.java | 13 +- .../java/opennlp/wordnet/LemmaFolding.java | 12 +- .../opennlp/wordnet/MorphyExceptions.java | 30 ++- .../opennlp/wordnet/MorphyLemmatizer.java | 83 ++++---- .../java/opennlp/wordnet/WnLmfReader.java | 178 +++++++++++----- .../main/java/opennlp/wordnet/WndbReader.java | 199 +++++++++++++++--- 10 files changed, 385 insertions(+), 208 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/wordnet/LexicalKnowledgeBase.java b/opennlp-api/src/main/java/opennlp/tools/wordnet/LexicalKnowledgeBase.java index 88518aa761..a2f11c7f6a 100644 --- a/opennlp-api/src/main/java/opennlp/tools/wordnet/LexicalKnowledgeBase.java +++ b/opennlp-api/src/main/java/opennlp/tools/wordnet/LexicalKnowledgeBase.java @@ -20,27 +20,15 @@ import java.util.Optional; /** - * The lexical knowledge base seam: lemma and synset lookup over a loaded lexical-semantic - * resource in the WordNet family. - * - *

This interface is the contract every wordnet-shaped dataset sits behind. A legacy Princeton - * WNDB directory, a WN-LMF XML document (the Global WordNet Association interchange format used - * by Open English WordNet and many other language wordnets), a future bundled - * permissively-licensed lexicon, and a future user-downloaded CC-BY lexicon are all - * implementations of this one seam, so a consumer written against {@code LexicalKnowledgeBase} - * never changes when the data tier does. Nothing in the contract names a particular resource; - * synset identity is opaque and source-qualified (see {@link Synset#id()}).

- * - *

The surface is intentionally minimal but sufficient for the layered features above it: - * lemma lookup and membership for morphological analysis (Morphy-style lemmatization), and - * synset retrieval with typed relation navigation for query expansion and, later, similarity - * measures. Feature layers stack on these four operations without a contract change.

+ * Lemma and synset lookup over a loaded lexical-semantic resource in the WordNet family. A + * consumer written against this seam is independent of the data tier behind it; synset identity + * is opaque and source-qualified (see {@link Synset#id()}). * *

Lemma matching semantics are the implementation's concern. The reference implementations * match case-insensitively (case folding with the root locale) and treat the underscore some - * formats store in multiword lemmas as a space, which is the recommended behavior; an - * implementation with different semantics must document them. Returned {@link Synset#lemmas() - * lemmas} preserve the source's written forms, with spaces in multiword lemmas.

+ * formats store in multiword lemmas as a space; an implementation with different semantics must + * document them. Returned {@link Synset#lemmas() lemmas} preserve the source's written forms, + * with spaces in multiword lemmas.

* *

Implementations must be immutable and thread-safe after loading: one instance is meant to * be shared across an application's threads for concurrent lookups.

diff --git a/opennlp-api/src/main/java/opennlp/tools/wordnet/Synset.java b/opennlp-api/src/main/java/opennlp/tools/wordnet/Synset.java index 83fac7bdcc..477fa38d90 100644 --- a/opennlp-api/src/main/java/opennlp/tools/wordnet/Synset.java +++ b/opennlp-api/src/main/java/opennlp/tools/wordnet/Synset.java @@ -27,17 +27,10 @@ * One synonym set: a single lexicalized concept with its member lemmas, gloss, and typed * relations to other synsets. * - *

The shape is deliberately format-agnostic. The {@link #id() id} is an opaque, - * source-qualified string minted by whichever reader produced the synset (a WN-LMF document's - * own synset id, or an id a WNDB reader derives from the file position); consumers must not - * parse it, only pass it back to {@link LexicalKnowledgeBase#synset(String)} and compare it for - * equality. Keeping identity opaque is what lets a bundled lexicon and a user-downloaded - * lexicon sit behind the same seam, and lets later alignment layers join sources without a - * contract change.

- * - *

Relations map each {@link WordNetRelation} present on this synset to the target synset - * ids in source order. Relations some formats draw between individual senses (antonymy, - * derivation) surface here at the synset level; see {@link WordNetRelation}.

+ *

The {@link #id() id} is an opaque, source-qualified string minted by the reader that + * produced the synset; consumers must not parse it, only pass it back to + * {@link LexicalKnowledgeBase#synset(String)} and compare it for equality. Relations map each + * {@link WordNetRelation} present on this synset to the target synset ids in source order.

* *

Instances are immutable and thread-safe: the list and map components are defensively * copied to immutable views at construction.

diff --git a/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetPOS.java b/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetPOS.java index 117d2c650a..65f6e691ec 100644 --- a/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetPOS.java +++ b/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetPOS.java @@ -19,16 +19,10 @@ /** * The four parts of speech a wordnet-style lexicon distinguishes. * - *

This enum is deliberately format-free: it carries none of the single-letter codes the - * on-disk formats use, so the contract does not leak a storage detail. Readers own the mapping - * from their format's codes to these values.

- * - *

Adjective satellites (the {@code s} code some formats use for adjectives clustered around - * a head adjective) normalize to {@link #ADJECTIVE}: the head/satellite split is a storage - * layout, not a part of speech, and the cluster structure is preserved through the - * {@link WordNetRelation#SIMILAR_TO} relation instead.

- * - *

Enum constants are immutable and safe to share across threads.

+ *

The enum carries none of the single-letter codes the on-disk formats use; readers own the + * mapping from their format's codes to these values. Adjective satellites normalize to + * {@link #ADJECTIVE}, with the cluster structure preserved through + * {@link WordNetRelation#SIMILAR_TO}.

*/ public enum WordNetPOS { diff --git a/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetRelation.java b/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetRelation.java index f2ffd71641..5597a93062 100644 --- a/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetRelation.java +++ b/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetRelation.java @@ -17,26 +17,13 @@ package opennlp.tools.wordnet; /** - * The typed relations a wordnet-style lexicon draws between {@link Synset synsets}. + * The typed relations a wordnet-style lexicon draws between {@link Synset synsets}. Both readers + * map their format's relation names onto these values, so consumers navigate one vocabulary + * regardless of the data tier behind the {@link LexicalKnowledgeBase} seam. * - *

The value set covers the pointer types documented for the legacy Princeton WNDB format, - * which is also the common core the WN-LMF interchange format expresses; both readers map their - * format's names onto these values, so consumers navigate one relation vocabulary regardless of - * the data tier behind the {@link LexicalKnowledgeBase} seam. {@link #PARTICIPLE} is part of that - * documented set (the adjective "participle of verb" pointer) even though it is easy - * to overlook; without it, real Princeton data would be unreadable. Two values go beyond the - * WNDB pointer set: {@link #ENTAILED_BY} and {@link #CAUSED_BY} cover the inverse relations - * WN-LMF documents such as Open English WordNet materialize, so reading such a document loses - * no typed relation; a WNDB-backed lexicon never produces them because the legacy format stores - * only the forward direction.

- * - *

Some of these relations are, in the source formats, drawn between individual word senses - * rather than whole synsets (antonymy and derivation are the common examples). In this v1 - * contract they surface at the synset level: the synset containing the source sense carries the - * relation to the synset containing the target sense. A sense-level view is a later, additive - * layer.

- * - *

Enum constants are immutable and safe to share across threads.

+ *

Relations that a source format draws between individual word senses (antonymy and + * derivation, for example) surface here at the synset level: the synset containing the source + * sense carries the relation to the synset containing the target sense.

*/ public enum WordNetRelation { diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java index 08782a8dd4..2401bfab6a 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/InMemoryWordNetLexicon.java @@ -35,13 +35,10 @@ * folded (lemma, part of speech) index. Package-private because it is a reader product, not a * public entry point; consumers hold it as {@link LexicalKnowledgeBase}. * - *

Lemma matching follows the recommended seam semantics: keys are folded by lowercasing with - * the root locale and treating the underscore some formats store in multiword lemmas as a - * space. Queries are folded identically at lookup time.

- * - *

Construction verifies referential integrity: every relation target of every synset must - * resolve to a synset in the table, so a lexicon can never hand out a dangling identifier. - * After construction all state is immutable, making instances safe for concurrent lookups.

+ *

Keys and queries are folded identically (see {@link LemmaFolding}). Construction verifies + * referential integrity: every relation target of every synset must resolve to a synset in the + * table, so a lexicon can never hand out a dangling identifier. After construction all state is + * immutable, making instances safe for concurrent lookups.

*/ @ThreadSafe final class InMemoryWordNetLexicon implements LexicalKnowledgeBase { @@ -104,6 +101,7 @@ final class InMemoryWordNetLexicon implements LexicalKnowledgeBase { this.senseIndex = index; } + /** {@inheritDoc} */ @Override public List lookup(String lemma, WordNetPOS pos) { if (lemma == null) { @@ -116,6 +114,7 @@ public List lookup(String lemma, WordNetPOS pos) { return senses == null ? List.of() : senses; } + /** {@inheritDoc} */ @Override public Optional synset(String synsetId) { if (synsetId == null) { diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java index a4730ad97c..242237cc09 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java @@ -22,15 +22,12 @@ /** * The single home of the lemma fold and the space-separated field split this package relies on. - * - *

The fold agreement is load-bearing: {@link MorphyExceptions} keys, the - * {@link InMemoryWordNetLexicon.LemmaKey sense-index keys}, and every query fold identically, - * so a rule-derived candidate validated against the lexicon and an exception-list hit can never - * disagree on canonical form. Keeping one implementation here prevents the copies from - * drifting apart.

+ * {@link MorphyExceptions} keys, the {@link InMemoryWordNetLexicon.LemmaKey sense-index keys}, + * and every query must fold through {@link #fold(String)} so their canonical forms agree. */ final class LemmaFolding { + /** Not instantiable. */ private LemmaFolding() { } @@ -46,8 +43,7 @@ static String fold(String writtenForm) { } /** - * Splits a space-separated field list, collapsing runs of spaces; no regular expressions - * involved. + * Splits a space-separated field list, collapsing runs of spaces. * * @param value The field list. Must not be {@code null}. * @return The non-empty fields in order, never {@code null}. diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyExceptions.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyExceptions.java index 91b6eac5fd..4f6c0b2934 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyExceptions.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyExceptions.java @@ -36,23 +36,24 @@ * consults before its detachment rules. * *

{@link #load(Path)} reads the four {@code *.exc} files ({@code noun.exc}, - * {@code verb.exc}, {@code adj.exc}, {@code adv.exc}) in the documented WNDB format: one entry - * per line, the inflected form followed by one or more base forms, space separated, with - * underscores standing for spaces in multiword entries. All four files must be present; note - * that Princeton's database-only download ({@code WNdb}) does not include them, while the full - * WordNet package does. No exception data is bundled with this module; see - * {@link MorphyLemmatizer} for the tiering rationale.

+ * {@code verb.exc}, {@code adj.exc}, {@code adv.exc}), which must all be present, in the WNDB + * format: one entry per line, the inflected form followed by one or more base forms, space + * separated, with underscores standing for spaces in multiword entries. No exception data is + * bundled; the caller supplies a directory.

* - *

Lookups fold the queried word the same way the lexicon seam folds lemmas: lowercase with - * the root locale, underscore as space.

- * - *

Instances are immutable after loading and safe for concurrent lookups.

+ *

Lookups fold the queried word the same way the lexicon seam folds lemmas. Instances are + * immutable after loading and safe for concurrent lookups.

*/ @ThreadSafe public final class MorphyExceptions { private final Map>> byPos; + /** + * Wraps the per-part-of-speech exception tables. + * + * @param byPos The loaded tables, one per part of speech. + */ private MorphyExceptions(Map>> byPos) { this.byPos = byPos; } @@ -105,6 +106,15 @@ public List lookup(String word, WordNetPOS pos) { return lemmas == null ? List.of() : lemmas; } + /** + * Loads one {@code *.exc} file into a folded inflected-form to base-forms map. + * + * @param directory The directory holding the file. + * @param fileName The exception file name, for example {@code noun.exc}. + * @return The folded exception entries. + * @throws InvalidFormatException Thrown if the file is missing or a line is malformed. + * @throws IOException Thrown if reading the file fails. + */ private static Map> loadFile(Path directory, String fileName) throws IOException { final Path file = directory.resolve(fileName); diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java index c92e9ab9dd..c472fa2fbc 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java @@ -26,48 +26,22 @@ import opennlp.tools.wordnet.WordNetPOS; /** - * A clean-room implementation of the documented Morphy algorithm as a {@link Lemmatizer}: - * exception-list lookup first, then the per-part-of-speech iterative detachment rules, with - * every rule-derived candidate validated against a {@link LexicalKnowledgeBase} before it is - * returned. + * A {@link Lemmatizer} implementing the Morphy algorithm: exception-list lookup first, then the + * per-part-of-speech iterative detachment rules, with every rule-derived candidate validated + * against a {@link LexicalKnowledgeBase} before it is returned. A token is folded (lowercase with + * the root locale, underscore as space) before lookup, and returned lemmas are in that folded + * form. * - *

Algorithm. For each token the part-of-speech tag is mapped to a - * {@link WordNetPOS}; the token is folded (lowercase with the root locale, underscore as - * space); the {@link MorphyExceptions exception lists} are consulted and a hit is returned - * directly, without lexicon validation, because the lists themselves are authoritative for - * irregular forms; otherwise the folded token itself, when the lexicon contains it, and every - * detachment-rule result the lexicon confirms become the candidate lemmas, in rule order. The - * rules are the documented Morphy suffix substitutions: for nouns {@code -s}, {@code -ses}, - * {@code -xes}, {@code -zes}, {@code -ches}, {@code -shes}, {@code -men}, {@code -ies}; for - * verbs {@code -s}, {@code -ies}, {@code -es}, {@code -ed}, {@code -ing} (with their {@code e} - * restorations); for adjectives {@code -er} and {@code -est} (plain and {@code e}-restoring); - * and none for adverbs, which rely on the exception list alone. Returned lemmas are in the - * lexicon's folded canonical form (lowercase, spaces in multiword lemmas).

+ *

Part-of-speech tags map to a {@link WordNetPOS} by their conventional Penn Treebank + * prefixes ({@code N}, {@code V}, {@code J}, {@code R}), the names {@code ADJ} and {@code ADV}, + * and the one-letter WordNet codes {@code n}, {@code v}, {@code a}, {@code r}, and {@code s} + * (satellite, treated as adjective), case-insensitively. A tag that maps to no part of speech + * yields the unknown-word result.

* - *

Tag mapping. Tags map to WordNet parts of speech by their conventional Penn - * Treebank prefixes: {@code N} to noun, {@code V} to verb, {@code J} to adjective, and - * {@code R} to adverb, case-insensitively. The names {@code ADJ} and {@code ADV} and, only as - * one-letter tags, the WordNet letters {@code n}, {@code v}, {@code a}, {@code r}, and - * {@code s} (satellite, treated as adjective) are also accepted. Anything else, including Penn - * tags for closed classes such as {@code DT} or {@code MD} and multi-letter tags that merely - * begin with an accepted letter, such as {@code AUX}, {@code ADP}, {@code SCONJ}, or - * {@code SYM}, maps to no part of speech and yields the unknown-word result.

- * - *

Unknown words. Following the convention of - * {@code opennlp.tools.lemmatizer.DictionaryLemmatizer}, a token with no lemma yields the - * string {@code "O"} from {@link #lemmatize(String[], String[])} and a singleton - * {@code ["O"]} from {@link #lemmatize(List, List)}.

- * - *

No lexicon or exception data is bundled: point {@link WndbReader} and - * {@link MorphyExceptions#load(java.nio.file.Path) MorphyExceptions} at a WordNet database - * directory you - * downloaded, or combine {@link WnLmfReader} with a downloaded exception-list directory. Both - * inputs are required because rule-derived candidates are meaningless without a lexicon to - * validate them against; an exception-only mode would be a misleading half-feature, so it does - * not exist. Bundling the permissively licensed data is a recorded follow-up.

- * - *

Instances are immutable and safe for concurrent use once constructed with a loaded - * lexicon and exception lists.

+ *

Following {@code opennlp.tools.lemmatizer.DictionaryLemmatizer}, a token with no lemma + * yields {@code "O"} from {@link #lemmatize(String[], String[])} and {@code ["O"]} from + * {@link #lemmatize(List, List)}. Both a lexicon and exception lists are required. Instances are + * immutable and safe for concurrent use.

*/ @ThreadSafe public final class MorphyLemmatizer implements Lemmatizer { @@ -114,6 +88,7 @@ public MorphyLemmatizer(LexicalKnowledgeBase lexicon, MorphyExceptions exception this.exceptions = exceptions; } + /** {@inheritDoc} */ @Override public String[] lemmatize(String[] toks, String[] tags) { if (toks == null || tags == null) { @@ -131,6 +106,7 @@ public String[] lemmatize(String[] toks, String[] tags) { return lemmas; } + /** {@inheritDoc} */ @Override public List> lemmatize(List toks, List tags) { if (toks == null || tags == null) { @@ -148,7 +124,14 @@ public List> lemmatize(List toks, List tags) { return lemmas; } - // All lemmas of one token, most preferred first; empty when the word is unknown. + /** + * Finds all lemmas of one token, most preferred first. + * + * @param token The token to lemmatize. + * @param tag The part-of-speech tag. + * @return The candidate lemmas, empty when the word is unknown or the tag maps to no part of + * speech. + */ private List lemmasOf(String token, String tag) { if (token == null || tag == null) { throw new IllegalArgumentException("Tokens and tags must not contain null elements"); @@ -179,6 +162,12 @@ private List lemmasOf(String token, String tag) { return candidates; } + /** + * Selects the detachment-rule table for a part of speech. + * + * @param pos The part of speech. + * @return The suffix-substitution rules, empty for adverbs. + */ private static String[][] rulesFor(WordNetPOS pos) { return switch (pos) { case NOUN -> NOUN_RULES; @@ -188,7 +177,13 @@ private static String[][] rulesFor(WordNetPOS pos) { }; } - // The documented tag mapping; package-private so tests can pin it directly. + /** + * Maps a part-of-speech tag to a {@link WordNetPOS}. Package-private so tests can pin the + * mapping directly. + * + * @param tag The tag to map. + * @return The part of speech, or {@code null} when the tag names none. + */ static WordNetPOS posFromTag(String tag) { if (tag.isEmpty()) { return null; @@ -204,9 +199,7 @@ static WordNetPOS posFromTag(String tag) { case 'N' -> WordNetPOS.NOUN; case 'V' -> WordNetPOS.VERB; case 'J' -> WordNetPOS.ADJECTIVE; - // The WordNet letter codes a and s stand for adjective only as one-letter tags: - // multi-letter tags such as AUX, ADP, SCONJ, or SYM name closed classes or symbols, - // not adjectives, so they map to no part of speech. + // Codes a and s mean adjective only as one-letter tags; AUX, ADP and the like do not. case 'A', 'S' -> tag.length() == 1 ? WordNetPOS.ADJECTIVE : null; case 'R' -> WordNetPOS.ADVERB; default -> null; diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java index 3ca689ecb0..a7be4fd8e4 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java @@ -44,41 +44,25 @@ /** * Reads a WN-LMF XML document (the Global WordNet Association interchange format, used by Open - * English WordNet and many other language wordnets) into a {@link LexicalKnowledgeBase}. + * English WordNet and many other language wordnets) into a {@link LexicalKnowledgeBase} using the + * JDK StAX parser. * - *

The reader is clean-room, built from the published format documentation, and uses only the - * JDK's StAX parser. It reads the subset of the format the {@link LexicalKnowledgeBase} contract - * serves: lexical entries (lemma, part of speech, senses), synsets (definition and typed - * relations), and sense relations, which are lifted to the synset level as documented on - * {@link WordNetRelation}. Elements outside that subset ({@code Pronunciation}, {@code Form}, - * {@code Example}, {@code SyntacticBehaviour}, {@code ILIDefinition}, and similar) are skipped. - * Relations of type {@code other}, the format's escape hatch for relations typed only by - * metadata, are also skipped: the contract has no untyped relation slot. Every other unknown + *

It reads the subset of the format the contract serves: lexical entries, synsets with their + * definitions and typed relations, and sense relations, which are lifted to the synset level as + * documented on {@link WordNetRelation}. Elements outside that subset are skipped, as are + * relations of type {@code other} (the format's untyped escape hatch); any other unknown * relation type fails loud.

* - *

Security. The parser is hardened against XXE per the OWASP-documented posture for - * formats that carry a DOCTYPE: a DOCTYPE declaration is tokenized and skipped, but nothing it - * names is ever resolved. External entities and the external DTD subset are both disabled - * ({@code IS_SUPPORTING_EXTERNAL_ENTITIES} and {@code ACCESS_EXTERNAL_DTD} are off), and the - * {@link javax.xml.stream.XMLResolver} throws on any resolution attempt regardless, so no - * network or filesystem access can be triggered by a DOCTYPE, internal or external. Open English - * WordNet releases ship with a DOCTYPE line referencing the schema DTD; this reader parses such - * a file unmodified.

+ *

The parser is hardened against XXE: the DTD internal subset is not processed, and external + * entities and the external DTD subset are disabled, so a DOCTYPE is tokenized and skipped but + * nothing it names is ever resolved. A release carrying a DOCTYPE line parses unmodified.

* - *

Errors. Malformed structure fails loud with an {@link InvalidFormatException} - * naming the resource and, where the parser provides one, the line: missing required - * attributes, a duplicate lexical entry, sense, or synset id, a sense pointing to an undeclared - * synset, a relation to an undeclared target, an unknown part-of-speech code, an unknown - * relation type, a synset with no member entries. I/O failures propagate as - * {@link IOException}.

- * - *

Part-of-speech code {@code s} (adjective satellite) normalizes to - * {@link WordNetPOS#ADJECTIVE}, and a {@code similar} relation whose source is a verb synset - * maps to {@link WordNetRelation#VERB_GROUP} (that is how WN-LMF documents derived from - * Princeton data express verb groups); on any other part of speech it maps to - * {@link WordNetRelation#SIMILAR_TO}.

- * - *

The returned lexicon is immutable and safe for concurrent lookups.

+ *

Malformed structure fails loud with an {@link InvalidFormatException} naming the resource + * and, where the parser provides one, the line; I/O failures propagate as {@link IOException}. + * Part-of-speech code {@code s} normalizes to {@link WordNetPOS#ADJECTIVE}, and a {@code similar} + * relation on a verb synset maps to {@link WordNetRelation#VERB_GROUP} rather than + * {@link WordNetRelation#SIMILAR_TO}. The returned lexicon is immutable and safe for concurrent + * lookups.

*/ public final class WnLmfReader { @@ -87,6 +71,7 @@ public final class WnLmfReader { /** The format's escape-hatch relation type; carries no type the contract can express. */ private static final String OTHER_RELATION = "other"; + /** Not instantiable. */ private WnLmfReader() { } @@ -151,12 +136,14 @@ public static LexicalKnowledgeBase read(InputStream in, String resourceName) thr return parser.build(); } + /** + * Builds an XXE-hardened StAX factory: the DTD internal subset is not processed and external + * entities and the external DTD subset are denied, so a DOCTYPE is skipped but never resolved. + * + * @return The hardened factory. + */ private static XMLInputFactory hardenedFactory() { final XMLInputFactory factory = XMLInputFactory.newFactory(); - // XXE hardening, the OWASP-documented posture for DOCTYPE-bearing formats: the internal - // subset is not processed (so no custom entity gets declared at all), and both the - // external DTD subset and external entities are denied, so a DOCTYPE is tokenized and - // skipped but nothing it names is ever fetched. factory.setProperty(XMLInputFactory.SUPPORT_DTD, Boolean.FALSE); factory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, Boolean.FALSE); factory.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, ""); @@ -167,7 +154,7 @@ private static XMLInputFactory hardenedFactory() { return factory; } - // The streaming parse state and post-parse resolution. + /** Holds the streaming parse state and performs post-parse resolution. */ private static final class Parser { private final String resourceName; @@ -191,17 +178,26 @@ private static final class Parser { private String currentSenseId; private RawSynset currentSynset; + /** + * Creates a parser. + * + * @param resourceName The name used in error messages. + */ Parser(String resourceName) { this.resourceName = resourceName; } + /** + * Streams the document, dispatching start and end elements. + * + * @param reader The StAX reader. + * @throws XMLStreamException Thrown if the stream read fails. + * @throws InvalidFormatException Thrown if the document is malformed. + */ void parse(XMLStreamReader reader) throws XMLStreamException, InvalidFormatException { while (reader.hasNext()) { final int event = reader.next(); - // A DTD event (the DOCTYPE declaration) is intentionally not handled here: with - // external entities and the external subset denied in the factory, it carries nothing - // that can affect parsing, so it is skipped exactly like a comment or an ignored - // element. + // A DTD event carries nothing that can affect parsing once the factory is hardened. if (event == XMLStreamConstants.START_ELEMENT) { startElement(reader); } else if (event == XMLStreamConstants.END_ELEMENT) { @@ -210,6 +206,14 @@ void parse(XMLStreamReader reader) throws XMLStreamException, InvalidFormatExcep } } + /** + * Handles one start element, updating cursor state and collecting raw entries, senses, and + * synsets. + * + * @param reader The StAX reader positioned on the start element. + * @throws XMLStreamException Thrown if reading element text fails. + * @throws InvalidFormatException Thrown if the element violates the format. + */ private void startElement(XMLStreamReader reader) throws XMLStreamException, InvalidFormatException { final String name = reader.getLocalName(); @@ -281,8 +285,7 @@ private void startElement(XMLStreamReader reader) } final String relType = requireAttribute(reader, "relType"); final String target = requireAttribute(reader, "target"); - // The escape-hatch type is skipped here exactly as it is for sense relations in - // build(): documented skip, not rejection. + // The escape-hatch type is a documented skip, not a rejection. if (!OTHER_RELATION.equals(relType)) { currentSynset.relations.add( new RawRelation(relType, target, line(reader.getLocation()))); @@ -295,6 +298,11 @@ private void startElement(XMLStreamReader reader) } } + /** + * Clears cursor state when a tracked element closes. + * + * @param name The local name of the closing element. + */ private void endElement(String name) { switch (name) { case "LexicalEntry" -> { @@ -310,6 +318,14 @@ private void endElement(String name) { } } + /** + * Resolves the collected raw state into an immutable lexicon: validates sense targets, lifts + * sense relations to the synset level, and materializes the contract synsets. + * + * @return The loaded lexicon. + * @throws InvalidFormatException Thrown if a sense or relation references an undeclared + * target, or a synset has no members. + */ LexicalKnowledgeBase build() throws InvalidFormatException { // Every sense must point to a declared synset, with a consistent part of speech. for (final Map.Entry sense : synsetBySenseId.entrySet()) { @@ -346,6 +362,14 @@ LexicalKnowledgeBase build() throws InvalidFormatException { return new InMemoryWordNetLexicon(synsetsById, senseOrder); } + /** + * Resolves a raw synset's relations into typed target-id lists, deduplicated in source order. + * + * @param raw The raw synset. + * @return The typed relations for the contract synset. + * @throws InvalidFormatException Thrown if a relation type is unknown or its target is + * undeclared. + */ private Map> resolveRelations(RawSynset raw) throws InvalidFormatException { final Map> typed = new LinkedHashMap<>(); @@ -356,9 +380,7 @@ private Map> resolveRelations(RawSynset raw) throw malformed(null, "Relation " + relation.relType + " at line " + relation.line + " on synset " + raw.id + " references undeclared synset " + relation.target, null); } - // The target's canonical id instance, not the pointer's own string: a full lexicon - // carries hundreds of thousands of pointers, and sharing the synset table's instances - // keeps only one copy of each id in memory. + // Share the synset table's id instance so only one copy of each id is retained. typed.computeIfAbsent(type, unused -> new LinkedHashSet<>()).add(target.id); } final Map> relations = new LinkedHashMap<>(typed.size() * 2); @@ -368,6 +390,15 @@ private Map> resolveRelations(RawSynset raw) return relations; } + /** + * Resolves a synset's member entry ids to their lemmas, from the {@code members} attribute + * when present and otherwise from the senses that pointed at the synset. + * + * @param raw The raw synset. + * @return The member lemmas in source order, deduplicated. + * @throws InvalidFormatException Thrown if the synset has no members, names an undeclared + * entry, or a member's part of speech disagrees with the synset's. + */ private List memberLemmas(RawSynset raw) throws InvalidFormatException { final List entryIds; if (raw.members != null && !raw.members.isEmpty()) { @@ -399,8 +430,16 @@ private List memberLemmas(RawSynset raw) throws InvalidFormatException { return lemmas; } + /** + * Maps a WN-LMF part-of-speech code to a {@link WordNetPOS}; code {@code s} normalizes to + * {@link WordNetPOS#ADJECTIVE}. + * + * @param code The part-of-speech code. + * @param location The parser location, for error reporting. + * @return The part of speech. + * @throws InvalidFormatException Thrown if the code is unknown. + */ private WordNetPOS parsePos(String code, Location location) throws InvalidFormatException { - // The adjective satellite code normalizes to ADJECTIVE; see WordNetPOS. return switch (code) { case "n" -> WordNetPOS.NOUN; case "v" -> WordNetPOS.VERB; @@ -410,9 +449,19 @@ private WordNetPOS parsePos(String code, Location location) throws InvalidFormat }; } + /** + * Maps a WN-LMF relation name to a {@link WordNetRelation}. A {@code similar} relation on a + * verb synset maps to {@link WordNetRelation#VERB_GROUP}, otherwise to + * {@link WordNetRelation#SIMILAR_TO}. + * + * @param relType The relation name. + * @param sourcePos The part of speech of the source synset. + * @param line The document line, for error reporting. + * @return The mapped relation. + * @throws InvalidFormatException Thrown if the relation name is unknown. + */ private WordNetRelation parseRelation(String relType, WordNetPOS sourcePos, int line) throws InvalidFormatException { - // Documents derived from Princeton data express verb groups as similar on verb synsets. if ("similar".equals(relType)) { return sourcePos == WordNetPOS.VERB ? WordNetRelation.VERB_GROUP : WordNetRelation.SIMILAR_TO; @@ -424,6 +473,14 @@ private WordNetRelation parseRelation(String relType, WordNetPOS sourcePos, int return relation; } + /** + * Reads a required attribute from the current element. + * + * @param reader The StAX reader. + * @param attribute The attribute name. + * @return The non-empty attribute value. + * @throws InvalidFormatException Thrown if the attribute is absent or empty. + */ private String requireAttribute(XMLStreamReader reader, String attribute) throws InvalidFormatException { final String value = reader.getAttributeValue(null, attribute); @@ -434,6 +491,14 @@ private String requireAttribute(XMLStreamReader reader, String attribute) return value; } + /** + * Builds a malformed-document exception naming the resource and, when known, the line. + * + * @param location The parser location, or {@code null} when unavailable. + * @param message The failure detail. + * @param cause The underlying cause, or {@code null}. + * @return The exception to throw. + */ InvalidFormatException malformed(Location location, String message, Throwable cause) { final int line = line(location); final String prefix = line < 0 ? "Malformed WN-LMF document " + resourceName + ": " @@ -442,6 +507,12 @@ InvalidFormatException malformed(Location location, String message, Throwable ca : new InvalidFormatException(prefix + message, cause); } + /** + * Extracts a line number from a parser location. + * + * @param location The location, or {@code null}. + * @return The line number, or {@code -1} when unknown. + */ private static int line(Location location) { return location == null ? -1 : location.getLineNumber(); } @@ -455,6 +526,14 @@ private static final class RawSynset { private final List relations = new ArrayList<>(4); private String gloss; + /** + * Creates a raw synset gathered during parsing. + * + * @param id The synset id. + * @param pos The part of speech. + * @param members The {@code members} attribute value, or {@code null} when absent. + * @param line The document line. + */ RawSynset(String id, WordNetPOS pos, String members, int line) { this.id = id; this.pos = pos; @@ -470,6 +549,11 @@ private record RawSenseRelation(String sourceSenseId, String relType, String tar int line) { } + /** + * Builds the WN-LMF relation-name to {@link WordNetRelation} table. + * + * @return The immutable name table. + */ private static Map relationNames() { final Map names = new HashMap<>(); names.put("antonym", WordNetRelation.ANTONYM); diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java index 8c572ed3da..719d545b22 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java @@ -37,36 +37,27 @@ * Reads a legacy Princeton WNDB database directory ({@code index.noun}, {@code data.noun}, and * the corresponding pairs for verbs, adjectives, and adverbs) into a {@link LexicalKnowledgeBase}. * - *

The reader is clean-room, built from the published {@code wndb(5WN)} and - * {@code wninput(5WN)} format documentation, with no third-party WordNet library involved. All - * eight files must be present in the directory. License preamble lines (which begin with a - * space in the released files) are skipped in index and data files. {@code index.sense} is not - * read: the v1 contract has no sense-key surface, so sense keys belong to the later sense - * inventory layer. The {@code *.exc} morphological exception lists in the same directory are - * the {@link MorphyLemmatizer} companion input and are read separately.

+ *

All eight index and data files must be present. License preamble lines (which begin with a + * space in the released files) are skipped. {@code index.sense} is not read, and the + * {@code *.exc} exception lists are the {@link MorphyLemmatizer} companion input, read + * separately.

* - *

Synset ids are minted as {@code wndb-}offset{@code -}pos from the data - * file's 8-digit byte offset and part-of-speech letter, for example {@code wndb-00001740-n}. - * The id is opaque to consumers; the format mirrors WN-LMF-style ids only for readability. - * Adjective satellite lines ({@code ss_type} {@code s}) normalize to - * {@link WordNetPOS#ADJECTIVE}, and the syntactic markers the adjective files append to some - * words ({@code (p)}, {@code (a)}, {@code (ip)}) are stripped from lemmas. Underscores in - * stored lemmas become spaces. Sense order per lemma follows the index file's offset order, - * which the format documents as most frequent first.

+ *

Synset ids are minted as {@code wndb-}offset{@code -}pos from the data file's + * 8-digit byte offset and part-of-speech letter, for example {@code wndb-00001740-n}; the id is + * opaque to consumers. Adjective satellite lines normalize to {@link WordNetPOS#ADJECTIVE}, the + * syntactic markers the adjective files append ({@code (p)}, {@code (a)}, {@code (ip)}) are + * stripped, and underscores in lemmas become spaces. Sense order per lemma follows the index + * file's offset order.

* - *

Errors. Malformed content fails loud with an {@link InvalidFormatException} - * naming the file and line: a missing file, a truncated or overlong line, a data line whose - * offset field disagrees with its actual byte position (the format's documented seek - * contract), an index entry referencing an offset with no data line, an undeclared pointer - * symbol, or a pointer to a nonexistent target. I/O failures propagate as - * {@link IOException}.

- * - *

The returned lexicon is immutable and safe for concurrent lookups.

+ *

Malformed content fails loud with an {@link InvalidFormatException} naming the file and + * line; I/O failures propagate as {@link IOException}. The returned lexicon is immutable and safe + * for concurrent lookups.

*/ public final class WndbReader { private static final Map POINTER_SYMBOLS = pointerSymbols(); + /** Not instantiable. */ private WndbReader() { } @@ -102,7 +93,7 @@ public static LexicalKnowledgeBase read(Path directory) throws IOException { return new InMemoryWordNetLexicon(synsetsById, senseOrder); } - // The four part-of-speech file pairs of a WNDB directory. + /** The four part-of-speech file pairs of a WNDB directory. */ private enum FilePos { NOUN("noun", 'n', WordNetPOS.NOUN), VERB("verb", 'v', WordNetPOS.VERB), @@ -113,6 +104,13 @@ private enum FilePos { private final char posChar; private final WordNetPOS pos; + /** + * Binds a part of speech to its file suffix and WNDB letter. + * + * @param suffix The file suffix, for example {@code noun}. + * @param posChar The WNDB part-of-speech letter. + * @param pos The mapped part of speech. + */ FilePos(String suffix, char posChar, WordNetPOS pos) { this.suffix = suffix; this.posChar = posChar; @@ -120,6 +118,14 @@ private enum FilePos { } } + /** + * Parses one {@code data.*} file, collecting its synsets keyed by minted id. + * + * @param directory The database directory. + * @param filePos The part-of-speech file pair. + * @param rawSynsets The accumulating synset table. + * @throws IOException Thrown if the file is missing, malformed, or unreadable. + */ private static void parseDataFile(Path directory, FilePos filePos, Map rawSynsets) throws IOException { final String fileName = "data." + filePos.suffix; @@ -142,6 +148,18 @@ private static void parseDataFile(Path directory, FilePos filePos, } } + /** + * Parses one data-file synset line into a raw synset. + * + * @param line The decoded line, without its trailing newline. + * @param byteOffset The line's byte offset, matched against the line's own offset field. + * @param fileName The data file name, for error reporting. + * @param lineNumber The 1-based line number. + * @param filePos The part-of-speech file pair. + * @param rawSynsets The accumulating synset table. + * @throws InvalidFormatException Thrown if the line is malformed or its offset field disagrees + * with its byte position. + */ private static void parseDataLine(String line, int byteOffset, String fileName, int lineNumber, FilePos filePos, Map rawSynsets) throws InvalidFormatException { @@ -201,6 +219,13 @@ private static void parseDataLine(String line, int byteOffset, String fileName, fileName, lineNumber)); } + /** + * Resolves raw synsets into contract synsets, validating every pointer target. + * + * @param rawSynsets The parsed synsets keyed by id. + * @return The contract synsets keyed by id. + * @throws InvalidFormatException Thrown if a pointer targets a nonexistent synset. + */ private static Map resolve(Map rawSynsets) throws InvalidFormatException { final Map synsetsById = new LinkedHashMap<>(rawSynsets.size() * 2); @@ -209,13 +234,10 @@ private static Map resolve(Map rawSynsets) for (final RawPointer pointer : raw.pointers) { final RawSynset target = rawSynsets.get(pointer.targetId); if (target == null) { - // The pointer's own line, so the error names exactly where the dangling pointer sits. throw malformed(raw.fileName, pointer.lineNumber, "Synset " + raw.id + " has a " + pointer.relation + " pointer to nonexistent synset " + pointer.targetId); } - // The target's canonical id instance, not the pointer's freshly minted string: a full - // database carries hundreds of thousands of pointers, and sharing the synset table's - // instances keeps only one copy of each id in memory. + // Share the synset table's id instance so only one copy of each id is retained. typed.computeIfAbsent(pointer.relation, unused -> new LinkedHashSet<>()) .add(target.id); } @@ -228,6 +250,15 @@ private static Map resolve(Map rawSynsets) return synsetsById; } + /** + * Parses one {@code index.*} file, building the sense order per folded lemma key. + * + * @param directory The database directory. + * @param filePos The part-of-speech file pair. + * @param rawSynsets The resolved synset table, for offset validation. + * @param senses The accumulating sense-order map. + * @throws IOException Thrown if the file is missing, malformed, or unreadable. + */ private static void parseIndexFile(Path directory, FilePos filePos, Map rawSynsets, Map> senses) @@ -251,6 +282,18 @@ private static void parseIndexFile(Path directory, FilePos filePos, } } + /** + * Parses one index-file line into a lemma's sense order. + * + * @param line The line to parse. + * @param fileName The index file name, for error reporting. + * @param lineNumber The 1-based line number. + * @param filePos The part-of-speech file pair. + * @param rawSynsets The resolved synset table, for offset validation. + * @param senses The accumulating sense-order map. + * @throws InvalidFormatException Thrown if the line is malformed or references an unknown + * offset. + */ private static void parseIndexLine(String line, String fileName, int lineNumber, FilePos filePos, Map rawSynsets, Map> senses) @@ -269,8 +312,7 @@ private static void parseIndexLine(String line, String fileName, int lineNumber, } final int pointerTypeCount = tokens.nextInt("p_cnt", 10); for (int i = 0; i < pointerTypeCount; i++) { - // The index file's pointer-type summary is informational; the typed pointers in the - // data file are authoritative, so the summary symbols are not validated here. + // The summary symbols are informational; the data file's pointers are authoritative. tokens.next("ptr_symbol"); } tokens.next("sense_cnt"); @@ -303,7 +345,16 @@ private static void parseIndexLine(String line, String fileName, int lineNumber, } } - // Strips the documented adjective syntactic markers and turns underscores into spaces. + /** + * Strips the adjective syntactic markers ({@code (p)}, {@code (a)}, {@code (ip)}) and turns + * underscores into spaces. + * + * @param word The raw word field. + * @param fileName The data file name, for error reporting. + * @param lineNumber The 1-based line number. + * @return The cleaned lemma. + * @throws InvalidFormatException Thrown if the word carries an unknown marker or is empty. + */ private static String cleanLemma(String word, String fileName, int lineNumber) throws InvalidFormatException { String cleaned = word; @@ -321,6 +372,14 @@ private static String cleanLemma(String word, String fileName, int lineNumber) return cleaned.replace('_', ' '); } + /** + * Parses an 8-digit synset offset. + * + * @param offset The offset field. + * @param tokens The tokenizer, for error reporting. + * @return The offset as an integer. + * @throws InvalidFormatException Thrown if the field is not 8 digits. + */ private static int parseOffset(String offset, Tokenizer tokens) throws InvalidFormatException { if (offset.length() != 8) { throw tokens.malformedToken("Synset offset must be 8 digits, got: " + offset); @@ -336,6 +395,14 @@ private static int parseOffset(String offset, Tokenizer tokens) throws InvalidFo return value; } + /** + * Parses a pointer's one-letter part-of-speech code. + * + * @param pos The code field. + * @param tokens The tokenizer, for error reporting. + * @return One of {@code n}, {@code v}, {@code a}, {@code r}. + * @throws InvalidFormatException Thrown if the code is not one of those letters. + */ private static char posChar(String pos, Tokenizer tokens) throws InvalidFormatException { if (pos.length() == 1) { final char c = pos.charAt(0); @@ -346,6 +413,15 @@ private static char posChar(String pos, Tokenizer tokens) throws InvalidFormatEx throw tokens.malformedToken("Pointer pos must be one of n, v, a, r, got: " + pos); } + /** + * Reads a required database file in full. + * + * @param file The file path. + * @param fileName The file name, for error reporting. + * @return The file bytes. + * @throws InvalidFormatException Thrown if the file is missing. + * @throws IOException Thrown if reading fails. + */ private static byte[] readAll(Path file, String fileName) throws IOException { if (!Files.isRegularFile(file)) { throw new InvalidFormatException("Missing WNDB database file: " + file); @@ -353,13 +429,21 @@ private static byte[] readAll(Path file, String fileName) throws IOException { return Files.readAllBytes(file); } + /** + * Builds a malformed-file exception naming the file and line. + * + * @param fileName The file name. + * @param lineNumber The 1-based line number. + * @param message The failure detail. + * @return The exception to throw. + */ private static InvalidFormatException malformed(String fileName, int lineNumber, String message) { return new InvalidFormatException( "Malformed WNDB file " + fileName + " at line " + lineNumber + ": " + message); } - // A cursor over one line's space-separated fields; no regular expressions involved. + /** A cursor over one line's space-separated fields. */ private static final class Tokenizer { private final String line; @@ -367,12 +451,26 @@ private static final class Tokenizer { private final int lineNumber; private int position; + /** + * Creates a tokenizer over one line. + * + * @param line The line to tokenize. + * @param fileName The file name, for error reporting. + * @param lineNumber The 1-based line number. + */ Tokenizer(String line, String fileName, int lineNumber) { this.line = line; this.fileName = fileName; this.lineNumber = lineNumber; } + /** + * Reads the next space-separated field. + * + * @param field The field name, for error reporting. + * @return The field value. + * @throws InvalidFormatException Thrown if the line is truncated before the field. + */ String next(String field) throws InvalidFormatException { while (position < line.length() && line.charAt(position) == ' ') { position++; @@ -387,6 +485,14 @@ String next(String field) throws InvalidFormatException { return line.substring(start, position); } + /** + * Reads the next field as an integer in the given radix. + * + * @param field The field name, for error reporting. + * @param radix The numeric radix. + * @return The parsed value. + * @throws InvalidFormatException Thrown if the field is missing or not a valid integer. + */ int nextInt(String field, int radix) throws InvalidFormatException { final String token = next(field); try { @@ -397,7 +503,12 @@ int nextInt(String field, int radix) throws InvalidFormatException { } } - // The remainder after the pipe separator, trimmed of the surrounding spaces. + /** + * Reads the gloss: the remainder after the pipe separator, trimmed of surrounding spaces. + * + * @return The gloss text. + * @throws InvalidFormatException Thrown if the pipe separator is missing. + */ String gloss() throws InvalidFormatException { final String separator = next("gloss separator"); if (!"|".equals(separator)) { @@ -414,6 +525,12 @@ String gloss() throws InvalidFormatException { return line.substring(start, end); } + /** + * Builds a malformed-file exception at this tokenizer's line. + * + * @param message The failure detail. + * @return The exception to throw. + */ InvalidFormatException malformedToken(String message) { return malformed(fileName, lineNumber, message); } @@ -431,6 +548,17 @@ private static final class RawSynset { private final String fileName; private final int lineNumber; + /** + * Creates a raw synset gathered while parsing a data file. + * + * @param id The minted synset id. + * @param pos The part of speech. + * @param lemmas The member lemmas. + * @param gloss The gloss text. + * @param pointers The raw pointers to resolve. + * @param fileName The source file name. + * @param lineNumber The source line number. + */ RawSynset(String id, WordNetPOS pos, List lemmas, String gloss, List pointers, String fileName, int lineNumber) { this.id = id; @@ -443,6 +571,11 @@ private static final class RawSynset { } } + /** + * Builds the WNDB pointer-symbol to {@link WordNetRelation} table. + * + * @return The immutable symbol table. + */ private static Map pointerSymbols() { final Map symbols = new HashMap<>(); symbols.put("!", WordNetRelation.ANTONYM); From eb53eadf11617ae5382b606126f162ef6ddf5723 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Sun, 12 Jul 2026 20:42:42 -0400 Subject: [PATCH 10/12] OPENNLP-1880: Tighten javadoc to contracts and document nested types Applies the review conventions from the OPENNLP-1869 review: philosophy prose shrinks to interface contracts, nested types carry javadoc instead of comments, inline rationale becomes single contract sentences, and the internal folding helpers guard their documented null contracts. --- .../java/opennlp/tools/wordnet/LexicalKnowledgeBase.java | 6 +++--- .../main/java/opennlp/tools/wordnet/WordNetRelation.java | 5 ++--- .../src/main/java/opennlp/wordnet/LemmaFolding.java | 3 +++ .../src/main/java/opennlp/wordnet/MorphyLemmatizer.java | 6 +++++- .../src/main/java/opennlp/wordnet/WnLmfReader.java | 8 +++----- 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/opennlp-api/src/main/java/opennlp/tools/wordnet/LexicalKnowledgeBase.java b/opennlp-api/src/main/java/opennlp/tools/wordnet/LexicalKnowledgeBase.java index a2f11c7f6a..214ef74db1 100644 --- a/opennlp-api/src/main/java/opennlp/tools/wordnet/LexicalKnowledgeBase.java +++ b/opennlp-api/src/main/java/opennlp/tools/wordnet/LexicalKnowledgeBase.java @@ -20,9 +20,9 @@ import java.util.Optional; /** - * Lemma and synset lookup over a loaded lexical-semantic resource in the WordNet family. A - * consumer written against this seam is independent of the data tier behind it; synset identity - * is opaque and source-qualified (see {@link Synset#id()}). + * Lemma and synset lookup over a loaded lexical-semantic resource in the WordNet family. Synset + * identity is opaque and source-qualified (see {@link Synset#id()}). Lookups return their matches + * in the source's sense order and never return {@code null}. * *

Lemma matching semantics are the implementation's concern. The reference implementations * match case-insensitively (case folding with the root locale) and treat the underscore some diff --git a/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetRelation.java b/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetRelation.java index 5597a93062..10da3cea34 100644 --- a/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetRelation.java +++ b/opennlp-api/src/main/java/opennlp/tools/wordnet/WordNetRelation.java @@ -17,9 +17,8 @@ package opennlp.tools.wordnet; /** - * The typed relations a wordnet-style lexicon draws between {@link Synset synsets}. Both readers - * map their format's relation names onto these values, so consumers navigate one vocabulary - * regardless of the data tier behind the {@link LexicalKnowledgeBase} seam. + * The typed relations a wordnet-style lexicon draws between {@link Synset synsets}. Readers map + * their source format's relation names onto these values. * *

Relations that a source format draws between individual word senses (antonymy and * derivation, for example) surface here at the synset level: the synset containing the source diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java index 242237cc09..0d3b153c04 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java @@ -19,6 +19,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; +import java.util.Objects; /** * The single home of the lemma fold and the space-separated field split this package relies on. @@ -37,8 +38,10 @@ private LemmaFolding() { * * @param writtenForm The form as written in a source file or query. Must not be {@code null}. * @return The folded form. + * @throws NullPointerException Thrown if {@code writtenForm} is {@code null}. */ static String fold(String writtenForm) { + Objects.requireNonNull(writtenForm, "writtenForm must not be null"); return writtenForm.replace('_', ' ').toLowerCase(Locale.ROOT); } diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java index c472fa2fbc..a004643f82 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/MorphyLemmatizer.java @@ -181,10 +181,14 @@ private static String[][] rulesFor(WordNetPOS pos) { * Maps a part-of-speech tag to a {@link WordNetPOS}. Package-private so tests can pin the * mapping directly. * - * @param tag The tag to map. + * @param tag The tag to map. Must not be {@code null}. * @return The part of speech, or {@code null} when the tag names none. + * @throws IllegalArgumentException Thrown if {@code tag} is {@code null}. */ static WordNetPOS posFromTag(String tag) { + if (tag == null) { + throw new IllegalArgumentException("Tag must not be null"); + } if (tag.isEmpty()) { return null; } diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java index a7be4fd8e4..9bf2bd6e3b 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java @@ -53,9 +53,8 @@ * relations of type {@code other} (the format's untyped escape hatch); any other unknown * relation type fails loud.

* - *

The parser is hardened against XXE: the DTD internal subset is not processed, and external - * entities and the external DTD subset are disabled, so a DOCTYPE is tokenized and skipped but - * nothing it names is ever resolved. A release carrying a DOCTYPE line parses unmodified.

+ *

The parser is hardened against XXE: DTD processing and external entities are disabled, so a + * DOCTYPE is skipped but nothing it names is fetched or resolved.

* *

Malformed structure fails loud with an {@link InvalidFormatException} naming the resource * and, where the parser provides one, the line; I/O failures propagate as {@link IOException}. @@ -124,8 +123,7 @@ public static LexicalKnowledgeBase read(InputStream in, String resourceName) thr reader.close(); } } catch (XMLStreamException e) { - // StAX wraps a failing stream read in an XMLStreamException; surface it as the I/O - // failure it is instead of misreporting it as a malformed document. + // StAX wraps a failing stream read in an XMLStreamException; surface it as the I/O failure. final Throwable nested = e.getNestedException() == null ? e.getCause() : e.getNestedException(); if (nested instanceof IOException io) { From a17dd2481637b6baa708e6116629f546b7dc629f Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 13 Jul 2026 03:26:56 -0400 Subject: [PATCH 11/12] OPENNLP-1880: Use IllegalArgumentException for null arguments and trim commentary Aligns null-argument validation with the project convention and removes remaining rationale commentary, keeping javadoc terse. --- .../src/main/java/opennlp/wordnet/LemmaFolding.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java index 0d3b153c04..4da713d520 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/LemmaFolding.java @@ -19,7 +19,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; -import java.util.Objects; /** * The single home of the lemma fold and the space-separated field split this package relies on. @@ -38,10 +37,12 @@ private LemmaFolding() { * * @param writtenForm The form as written in a source file or query. Must not be {@code null}. * @return The folded form. - * @throws NullPointerException Thrown if {@code writtenForm} is {@code null}. + * @throws IllegalArgumentException Thrown if {@code writtenForm} is {@code null}. */ static String fold(String writtenForm) { - Objects.requireNonNull(writtenForm, "writtenForm must not be null"); + if (writtenForm == null) { + throw new IllegalArgumentException("writtenForm must not be null"); + } return writtenForm.replace('_', ' ').toLowerCase(Locale.ROOT); } From f2bd38503d059a68e26b336c54fa78e17d759f17 Mon Sep 17 00:00:00 2001 From: Kristian Rickert Date: Mon, 13 Jul 2026 04:08:01 -0400 Subject: [PATCH 12/12] OPENNLP-1880: Document the raw parse records and test the null argument guards --- .../src/main/java/opennlp/wordnet/WnLmfReader.java | 2 ++ .../src/main/java/opennlp/wordnet/WndbReader.java | 1 + .../src/test/java/opennlp/wordnet/LemmaFoldingTest.java | 6 ++++++ .../src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java | 1 + 4 files changed, 10 insertions(+) diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java index 9bf2bd6e3b..bcb7bea0ce 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WnLmfReader.java @@ -540,9 +540,11 @@ private static final class RawSynset { } } + /** A parsed synset relation, kept until the target synset is known. */ private record RawRelation(String relType, String target, int line) { } + /** A parsed sense relation, kept until both sense ids are known. */ private record RawSenseRelation(String sourceSenseId, String relType, String targetSenseId, int line) { } diff --git a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java index 719d545b22..67e280e3a9 100644 --- a/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java +++ b/opennlp-extensions/opennlp-wordnet/src/main/java/opennlp/wordnet/WndbReader.java @@ -536,6 +536,7 @@ InvalidFormatException malformedToken(String message) { } } + /** A parsed pointer line, kept until the target synset is known. */ private record RawPointer(WordNetRelation relation, String targetId, int lineNumber) { } diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LemmaFoldingTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LemmaFoldingTest.java index f2f566d791..3d32eb4431 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LemmaFoldingTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/LemmaFoldingTest.java @@ -23,6 +23,7 @@ import opennlp.tools.wordnet.WordNetPOS; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; /** * Pins the shared fold and split behavior every user of {@link LemmaFolding} depends on: @@ -57,4 +58,9 @@ void testLemmaKeyAndExceptionLookupAgreeOnTheFold() { assertEquals(InMemoryWordNetLexicon.LemmaKey.of("Domestic_Dog", WordNetPOS.NOUN), InMemoryWordNetLexicon.LemmaKey.of(LemmaFolding.fold("DOMESTIC_DOG"), WordNetPOS.NOUN)); } + + @Test + void testFoldRejectsNull() { + assertThrows(IllegalArgumentException.class, () -> LemmaFolding.fold(null)); + } } diff --git a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java index ecbbfda9eb..0dd3ca6ff0 100644 --- a/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java +++ b/opennlp-extensions/opennlp-wordnet/src/test/java/opennlp/wordnet/MorphyLemmatizerTest.java @@ -151,6 +151,7 @@ void testPosFromTagMapping() { assertNull(MorphyLemmatizer.posFromTag("ADP")); assertNull(MorphyLemmatizer.posFromTag("SCONJ")); assertNull(MorphyLemmatizer.posFromTag("SYM")); + assertThrows(IllegalArgumentException.class, () -> MorphyLemmatizer.posFromTag(null)); } @Test