diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/langdetect/LanguageDetectorFactory.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/langdetect/LanguageDetectorFactory.java index 2893b634d5..3796f2beaf 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/langdetect/LanguageDetectorFactory.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/langdetect/LanguageDetectorFactory.java @@ -45,8 +45,10 @@ public class LanguageDetectorFactory extends BaseToolFactory { /** - * @return Retrieves a {@link LanguageDetectorContextGenerator}. + * @return Retrieves a {@link LanguageDetectorContextGenerator}. The deprecated emoji + * normalizer stays in this chain because existing models were trained with it. */ + @SuppressWarnings("deprecation") public LanguageDetectorContextGenerator getContextGenerator() { return new DefaultLanguageDetectorContextGenerator(1, 3, EmojiCharSequenceNormalizer.getInstance(), diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/Dimension.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/Dimension.java index 5aaab15122..915aa47f24 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/Dimension.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/Dimension.java @@ -60,6 +60,9 @@ public enum Dimension { /** Diacritic and accent folding; lossy, script gated, and language-wrong for some languages. */ ACCENT_FOLD(AccentFoldCharSequenceNormalizer::getInstance), + /** Emoji folded to ASCII emoticons where a mapping exists; lossy, for matching only. */ + EMOJI_FOLD(EmojiToEmoticonCharSequenceNormalizer::getInstance), + /** Confusable (homoglyph) skeleton folding per UTS #39; lossy, for matching only. */ CONFUSABLE_FOLD(ConfusableSkeletonCharSequenceNormalizer::getInstance), diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizer.java index 4925aae79c..baaccd6edc 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiCharSequenceNormalizer.java @@ -21,7 +21,11 @@ /** * A {@link EmojiCharSequenceNormalizer} implementation that normalizes text * in terms of emojis. Every encounter will be replaced by a whitespace. + * + * @deprecated Replaces every supplementary-plane code point with a space, not only emoji. Use + * {@link EmojiToEmoticonCharSequenceNormalizer} instead. */ +@Deprecated(since = "3.0.0") public class EmojiCharSequenceNormalizer implements CharSequenceNormalizer { private static final long serialVersionUID = 4553401197981667914L; diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiEmoticons.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiEmoticons.java new file mode 100644 index 0000000000..1675586247 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiEmoticons.java @@ -0,0 +1,427 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.util.normalizer; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * The bundled emoji/emoticon fold tables ({@code emoji-emoticons.txt}) and the sequence + * substitution shared by {@link EmojiToEmoticonCharSequenceNormalizer} and + * {@link EmoticonToEmojiCharSequenceNormalizer}. Both directions substitute code point + * sequences, longest match first at each position. In the emoticon direction a source + * matches only when delimited by the text boundary or whitespace on both sides; in the emoji + * direction a mapped pictograph inside a ZWJ sequence or before U+FE0E is left untouched, and a + * trailing U+FE0F is absorbed into the fold. + */ +final class EmojiEmoticons { + + private static final String RESOURCE = "emoji-emoticons.txt"; + + // Sequence-context code points of the pictographic direction. + private static final int ZERO_WIDTH_JOINER = 0x200D; + private static final int VARIATION_SELECTOR_TEXT = 0xFE0E; + private static final int VARIATION_SELECTOR_EMOJI = 0xFE0F; + + private static final EmojiEmoticons INSTANCE = new EmojiEmoticons(loadBundled()); + + private final Direction emojiToEmoticon; + private final Direction emoticonToEmoji; + + private EmojiEmoticons(Tables tables) { + this.emojiToEmoticon = tables.emojiToEmoticon(); + this.emoticonToEmoji = tables.emoticonToEmoji(); + } + + /** + * {@return the shared instance over the bundled {@code emoji-emoticons.txt} data} The tables are + * loaded once when this class initializes. + */ + static EmojiEmoticons getInstance() { + return INSTANCE; + } + + /** + * One fold row: a source code point sequence and its replacement. + * + * @param source the code point sequence to replace. + * @param target the replacement sequence. + */ + record Mapping(String source, String target) { + } + + /** + * One direction table, keyed by the first code point of the source sequence, candidates ordered + * longest source first so a scan is longest-match. The first-code-point range bounds let the + * scan skip the map lookup for the common code point that no source starts with. + * + * @param table the candidate mappings keyed by first source code point. + * @param minFirst the smallest first code point of any source. + * @param maxFirst the largest first code point of any source. + */ + record Direction(Map> table, int minFirst, int maxFirst) { + + /** + * {@return the candidate mappings whose source starts with {@code codePoint}, longest source + * first, or {@code null} if no source starts with it} + * + * @param codePoint the code point at the current scan position. + */ + List candidates(int codePoint) { + if (codePoint < minFirst || codePoint > maxFirst) { + return null; + } + return table.get(codePoint); + } + } + + /** + * The two direction tables produced by {@link #parse(InputStream)}. + * + * @param emojiToEmoticon the pictograph-to-emoticon direction. + * @param emoticonToEmoji the emoticon-to-pictograph direction. + */ + record Tables(Direction emojiToEmoticon, Direction emoticonToEmoji) { + } + + /** + * Folds mapped pictographs in {@code text} to their ASCII emoticons. + * + * @param text the text to fold. Must not be {@code null}. + * @return the folded text. + */ + String emojiToEmoticon(CharSequence text) { + return substitute(text, emojiToEmoticon, false); + } + + /** + * Folds mapped pictographs in {@code text} to their ASCII emoticons, producing the + * {@link Alignment} back to the original text. Each replaced source sequence, including an + * absorbed trailing U+FE0F, maps to its replacement as one block. + * + * @param text the text to fold. Must not be {@code null}. + * @return the folded text with its alignment. + */ + AlignedText emojiToEmoticonAligned(CharSequence text) { + return substituteAligned(text, emojiToEmoticon, false); + } + + /** + * Folds whitespace-delimited ASCII emoticons in {@code text} to their pictographs. + * + * @param text the text to fold. Must not be {@code null}. + * @return the folded text. + */ + String emoticonToEmoji(CharSequence text) { + return substitute(text, emoticonToEmoji, true); + } + + /** + * Folds whitespace-delimited ASCII emoticons in {@code text} to their pictographs, producing + * the {@link Alignment} back to the original text. + * + * @param text the text to fold. Must not be {@code null}. + * @return the folded text with its alignment. + */ + AlignedText emoticonToEmojiAligned(CharSequence text) { + return substituteAligned(text, emoticonToEmoji, true); + } + + /** + * Applies the direction table in a single longest-match-first cursor pass. + * + * @param text the text to fold. + * @param direction the direction table to apply. + * @param delimited if {@code true}, a source matches only when delimited by the text boundary + * or Unicode {@code White_Space} on both sides (the emoticon direction). If + * {@code false}, the pictographic sequence rules apply instead: no fold inside + * a ZWJ sequence or before U+FE0E, and a trailing U+FE0F is absorbed. + * @return the folded text. + */ + private static String substitute(CharSequence text, Direction direction, boolean delimited) { + final StringBuilder out = new StringBuilder(text.length()); + final int length = text.length(); + int i = 0; + while (i < length) { + final int codePoint = Character.codePointAt(text, i); + final long match = matchAt(text, i, direction.candidates(codePoint), delimited); + if (match >= 0) { + out.append(direction.candidates(codePoint).get((int) (match >>> 32)).target()); + i += (int) match; + } else { + out.appendCodePoint(codePoint); + i += Character.charCount(codePoint); + } + } + return out.toString(); + } + + /** + * Like {@link #substitute(CharSequence, Direction, boolean)} but also produces the + * {@link Alignment} back to the original text. Each replaced source sequence, including an + * absorbed trailing U+FE0F, maps to its replacement as one block. + * + * @param text the text to fold. + * @param direction the direction table to apply. + * @param delimited see {@link #substitute(CharSequence, Direction, boolean)}. + * @return the folded text with its alignment. + */ + private static AlignedText substituteAligned(CharSequence text, Direction direction, + boolean delimited) { + final StringBuilder out = new StringBuilder(text.length()); + final Alignment.Builder alignment = new Alignment.Builder(); + final int length = text.length(); + int i = 0; + while (i < length) { + final int codePoint = Character.codePointAt(text, i); + final long match = matchAt(text, i, direction.candidates(codePoint), delimited); + if (match >= 0) { + final String target = direction.candidates(codePoint).get((int) (match >>> 32)).target(); + final int consumed = (int) match; + out.append(target); + alignment.replace(consumed, target.length()); + i += consumed; + } else { + out.appendCodePoint(codePoint); + final int charCount = Character.charCount(codePoint); + alignment.equal(charCount); + i += charCount; + } + } + return new AlignedText(text, out.toString(), alignment.build(length)); + } + + /** + * Finds the winning candidate at position {@code i}. Candidates are pre-sorted longest source + * first, so the first acceptable region match wins. In the pictographic direction + * ({@code delimited == false}) a match is rejected when the source adjoins a ZWJ sequence or is + * followed by U+FE0E, and a trailing U+FE0F joins the consumed region. + * + * @param text the text being scanned. + * @param i the scan position. + * @param candidates the candidates whose source starts with the code point at {@code i}, or + * {@code null} if there are none. + * @param delimited whether the whitespace-delimited boundary rule applies. + * @return the winning candidate encoded as {@code (candidateIndex << 32) | consumedChars}, or + * {@code -1} when nothing folds here. + */ + private static long matchAt(CharSequence text, int i, List candidates, + boolean delimited) { + if (candidates == null || (delimited && !boundaryBefore(text, i))) { + return -1; + } + if (!delimited && i > 0 && Character.codePointBefore(text, i) == ZERO_WIDTH_JOINER) { + return -1; + } + for (int index = 0; index < candidates.size(); index++) { + final Mapping candidate = candidates.get(index); + int end = i + candidate.source().length(); + if (end > text.length() || !regionMatches(text, i, candidate.source())) { + continue; + } + if (delimited) { + if (boundaryAfter(text, end)) { + return ((long) index << 32) | (end - i); + } + continue; + } + // Absorb one trailing emoji-presentation selector into the fold. + if (end < text.length() && Character.codePointAt(text, end) == VARIATION_SELECTOR_EMOJI) { + end++; + } + if (end < text.length()) { + final int following = Character.codePointAt(text, end); + if (following == ZERO_WIDTH_JOINER || following == VARIATION_SELECTOR_TEXT) { + continue; + } + } + return ((long) index << 32) | (end - i); + } + return -1; + } + + /** + * {@return whether position {@code i} is preceded by the text boundary or whitespace} + * + * @param text the text being scanned. + * @param i the scan position. + */ + private static boolean boundaryBefore(CharSequence text, int i) { + return i == 0 || CharClass.whitespace().contains(Character.codePointBefore(text, i)); + } + + /** + * {@return whether position {@code end} is the text boundary or followed by whitespace} + * + * @param text the text being scanned. + * @param end the position just past a candidate match. + */ + private static boolean boundaryAfter(CharSequence text, int end) { + return end == text.length() || CharClass.whitespace().contains(Character.codePointAt(text, end)); + } + + /** + * {@return whether {@code text} contains exactly {@code source} starting at {@code start}} + * + * @param text the text being scanned. + * @param start the position to compare from. The caller guarantees + * {@code start + source.length() <= text.length()}. + * @param source the sequence to compare against. + */ + private static boolean regionMatches(CharSequence text, int start, String source) { + for (int k = 0; k < source.length(); k++) { + if (text.charAt(start + k) != source.charAt(k)) { + return false; + } + } + return true; + } + + /** + * {@return the tables parsed from the bundled {@code emoji-emoticons.txt} resource} + * + * @throws IllegalStateException if the resource is missing. + * @throws UncheckedIOException if the resource cannot be read. + */ + private static Tables loadBundled() { + try (InputStream in = EmojiEmoticons.class.getResourceAsStream(RESOURCE)) { + if (in == null) { + throw new IllegalStateException("Missing emoji/emoticon fold data resource: " + RESOURCE); + } + return parse(in); + } catch (IOException e) { + throw new UncheckedIOException("Unable to read emoji/emoticon fold data resource " + + RESOURCE, e); + } + } + + /** + * Parses rows of {@code source ; target ; fold_type ; standard ; unicode_version ; notes} with + * space-separated hexadecimal code points; {@code '#'} starts a comment line. {@code EMOJI} rows + * load into the emoji-to-emoticon table, {@code EMOTICON} rows into the emoticon-to-emoji + * table. Package-private so the malformed-data handling can be exercised without the bundled + * resource. + * + * @param in the stream to parse. Must not be {@code null}. + * @return the two direction tables. + * @throws IOException if the stream cannot be read. + * @throws IllegalArgumentException if the data is malformed. + */ + static Tables parse(InputStream in) throws IOException { + Objects.requireNonNull(in, "in must not be null"); + final Map> emojiToEmoticon = new HashMap<>(); + final Map> emoticonToEmoji = new HashMap<>(); + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) { + String line; + int lineNumber = 0; + while ((line = reader.readLine()) != null) { + lineNumber++; + final String content = line.strip(); + if (content.isEmpty() || content.startsWith("#")) { + continue; + } + // Bounded split: the notes column is free text that may itself contain ';' (for example + // the winking emoticon), so only the first five separators are structural. + final String[] fields = content.split(";", 6); + if (fields.length != 6) { + throw new IllegalArgumentException("Malformed emoji/emoticon fold data in " + RESOURCE + + " at line " + lineNumber + ": expected 6 fields, got " + fields.length + + " in: " + content); + } + final String source = decode(fields[0], lineNumber, content); + final String target = decode(fields[1], lineNumber, content); + final String foldType = fields[2].strip(); + final Map> table = switch (foldType) { + case "EMOJI" -> emojiToEmoticon; + case "EMOTICON" -> emoticonToEmoji; + default -> throw new IllegalArgumentException("Malformed emoji/emoticon fold data in " + + RESOURCE + " at line " + lineNumber + ": unrecognized fold_type '" + foldType + + "' in: " + content); + }; + final int firstCodePoint = source.codePointAt(0); + final List candidates = + table.computeIfAbsent(firstCodePoint, k -> new ArrayList<>()); + for (final Mapping existing : candidates) { + if (existing.source().equals(source)) { + throw new IllegalArgumentException("Malformed emoji/emoticon fold data in " + RESOURCE + + " at line " + lineNumber + ": duplicate " + foldType + " source in: " + content); + } + } + candidates.add(new Mapping(source, target)); + } + } + return new Tables(direction(emojiToEmoticon), direction(emoticonToEmoji)); + } + + /** + * Builds a {@link Direction} from a parsed table: sorts each candidate list longest source + * first, so the scan in {@link #matchAt} is longest-match by construction, and computes the + * first-code-point bounds that feed the no-match short circuit. + * + * @param table the mutable table produced by {@link #parse(InputStream)}. + * @return the immutable direction. + */ + private static Direction direction(Map> table) { + int min = Integer.MAX_VALUE; + int max = Integer.MIN_VALUE; + for (final Map.Entry> entry : table.entrySet()) { + min = Math.min(min, entry.getKey()); + max = Math.max(max, entry.getKey()); + entry.getValue().sort( + Comparator.comparingInt((Mapping m) -> m.source().length()).reversed()); + } + return new Direction(Map.copyOf(table), min, max); + } + + /** + * Decodes a space-separated hexadecimal code point sequence. + * + * @param hexCodePoints the field to decode. + * @param lineNumber the line number, for the error message. + * @param content the full line, for the error message. + * @return the decoded sequence. + * @throws IllegalArgumentException if the field is empty or not valid hexadecimal. + */ + private static String decode(String hexCodePoints, int lineNumber, String content) { + final String stripped = hexCodePoints.strip(); + if (stripped.isEmpty()) { + throw new IllegalArgumentException("Malformed emoji/emoticon fold data in " + RESOURCE + + " at line " + lineNumber + ": empty code point sequence in: " + content); + } + try { + final StringBuilder decoded = new StringBuilder(); + for (final String hex : stripped.split(" ")) { + decoded.appendCodePoint(Integer.parseInt(hex, 16)); + } + return decoded.toString(); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Malformed emoji/emoticon fold data in " + RESOURCE + + " at line " + lineNumber + ": " + content, e); + } + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiToEmoticonCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiToEmoticonCharSequenceNormalizer.java new file mode 100644 index 0000000000..5c8736eb27 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmojiToEmoticonCharSequenceNormalizer.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.util.normalizer; + +import java.util.Objects; + +/** + * A {@link CharSequenceNormalizer} that folds emoji to ASCII emoticons, using the bundled + * {@code emoji-emoticons.txt} mapping (for example U+1F642 SLIGHTLY SMILING FACE to {@code :)}). + * + *

A mapped pictograph inside a larger ZWJ sequence or followed by U+FE0E VARIATION SELECTOR-15 + * is left untouched; a trailing U+FE0F VARIATION SELECTOR-16 after any mapped pictograph is + * absorbed into the fold, so no dangling variation selector is left behind. This is an expanding, + * offset-changing transform: {@link #normalizeAligned(CharSequence)} reports the {@link Alignment} + * from the folded text back to the input. The reverse direction is + * {@link EmoticonToEmojiCharSequenceNormalizer}.

+ */ +public final class EmojiToEmoticonCharSequenceNormalizer implements OffsetAwareNormalizer { + + private static final long serialVersionUID = 7118395871093868025L; + + private static final EmojiToEmoticonCharSequenceNormalizer INSTANCE = + new EmojiToEmoticonCharSequenceNormalizer(); + + /** Instantiated once for {@link #getInstance()}. */ + private EmojiToEmoticonCharSequenceNormalizer() { + } + + /** {@return the shared, stateless instance} */ + public static EmojiToEmoticonCharSequenceNormalizer getInstance() { + return INSTANCE; + } + + /** + * {@inheritDoc} + * + * @throws NullPointerException if {@code text} is {@code null}. + */ + @Override + public CharSequence normalize(CharSequence text) { + Objects.requireNonNull(text, "text must not be null"); + return EmojiEmoticons.getInstance().emojiToEmoticon(text); + } + + /** + * {@inheritDoc} + * + * @throws NullPointerException if {@code text} is {@code null}. + */ + @Override + public AlignedText normalizeAligned(CharSequence text) { + Objects.requireNonNull(text, "text must not be null"); + return EmojiEmoticons.getInstance().emojiToEmoticonAligned(text); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmoticonToEmojiCharSequenceNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmoticonToEmojiCharSequenceNormalizer.java new file mode 100644 index 0000000000..7539ad827c --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/EmoticonToEmojiCharSequenceNormalizer.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.util.normalizer; + +import java.util.Objects; + +/** + * A {@link CharSequenceNormalizer} that folds ASCII emoticons to emoji, using the bundled + * {@code emoji-emoticons.txt} mapping (for example {@code :-)} to U+1F642 SLIGHTLY SMILING FACE). + * + *

An emoticon folds only when it stands alone as a whitespace-delimited unit (the text boundary + * or Unicode {@code White_Space} on both sides), so sequences inside ordinary text such as the + * {@code :/} in {@code https://} are never touched. Matching is longest first at each position. + * Apply this normalizer before tokenization if emoticons should survive as single tokens. This is + * an offset-changing transform: {@link #normalizeAligned(CharSequence)} reports the + * {@link Alignment} from the folded text back to the input. The reverse direction is + * {@link EmojiToEmoticonCharSequenceNormalizer}.

+ */ +public final class EmoticonToEmojiCharSequenceNormalizer implements OffsetAwareNormalizer { + + private static final long serialVersionUID = 3520486210777693086L; + + private static final EmoticonToEmojiCharSequenceNormalizer INSTANCE = + new EmoticonToEmojiCharSequenceNormalizer(); + + /** Instantiated once for {@link #getInstance()}. */ + private EmoticonToEmojiCharSequenceNormalizer() { + } + + /** {@return the shared, stateless instance} */ + public static EmoticonToEmojiCharSequenceNormalizer getInstance() { + return INSTANCE; + } + + /** + * {@inheritDoc} + * + * @throws NullPointerException if {@code text} is {@code null}. + */ + @Override + public CharSequence normalize(CharSequence text) { + Objects.requireNonNull(text, "text must not be null"); + return EmojiEmoticons.getInstance().emoticonToEmoji(text); + } + + /** + * {@inheritDoc} + * + * @throws NullPointerException if {@code text} is {@code null}. + */ + @Override + public AlignedText normalizeAligned(CharSequence text) { + Objects.requireNonNull(text, "text must not be null"); + return EmojiEmoticons.getInstance().emoticonToEmojiAligned(text); + } +} diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TermAnalyzer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TermAnalyzer.java index e604c30c04..4859ee2dfe 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TermAnalyzer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TermAnalyzer.java @@ -368,6 +368,19 @@ public Builder accentFold(Set foldScripts, boolean fold new AccentFoldCharSequenceNormalizer(foldScripts, foldStrokeLetters)); } + /** + * Enables {@link Dimension#EMOJI_FOLD}, folding emoji to ASCII emoticons. Only this direction + * exists as a per-token layer: the reverse (emoticon to emoji) is a pre-tokenization transform, + * because a tokenizer splits an emoticon such as {@code :-)} into punctuation, so it can never + * arrive as a single token; see {@link EmoticonToEmojiCharSequenceNormalizer}. + * + * @return this builder + */ + public Builder emojiFold() { + chain.add(Dimension.EMOJI_FOLD); + return this; + } + /** * Enables {@link Dimension#CONFUSABLE_FOLD}. * diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TextNormalizer.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TextNormalizer.java index ee45ce8f91..0690e2fd3c 100644 --- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TextNormalizer.java +++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/normalizer/TextNormalizer.java @@ -157,6 +157,29 @@ public Builder accentFold() { return add(Dimension.ACCENT_FOLD.defaultNormalizer()); } + /** + * {@return this builder with emoji-to-emoticon folding appended} + * + *

Folds pictographs with a bundled ASCII emoticon mapping (for example U+1F642 to + * {@code :)}) and copies everything else through. It is offset-aware, so it composes into + * {@link #buildAligned()}.

+ */ + public Builder emojiToEmoticon() { + return add(Dimension.EMOJI_FOLD.defaultNormalizer()); + } + + /** + * {@return this builder with emoticon-to-emoji folding appended} + * + *

Folds a whitespace-delimited ASCII emoticon to its pictograph (for example {@code :-)} to + * U+1F642); an emoticon sequence embedded in other text, such as the {@code :/} in + * {@code https://}, is left alone. Apply before tokenization so emoticons survive as single + * emoji tokens. It is offset-aware, so it composes into {@link #buildAligned()}.

+ */ + public Builder emoticonToEmoji() { + return add(EmoticonToEmojiCharSequenceNormalizer.getInstance()); + } + /** * Appends a custom normalizer. * diff --git a/opennlp-core/opennlp-runtime/src/main/resources/opennlp/tools/util/normalizer/emoji-emoticons.txt b/opennlp-core/opennlp-runtime/src/main/resources/opennlp/tools/util/normalizer/emoji-emoticons.txt new file mode 100644 index 0000000000..5879fb0b52 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/main/resources/opennlp/tools/util/normalizer/emoji-emoticons.txt @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Bidirectional emoji/emoticon fold data, authored by this project. One row per mapping: +# +# source ; target ; fold_type ; standard ; unicode_version ; notes +# +# source and target are space-separated hexadecimal Unicode code points. fold_type EMOJI maps a +# pictograph to its ASCII emoticon; EMOTICON maps an emoticon to its canonical pictograph. The +# standard column is UNSPECIFIED (project-authored mapping). Every fold target is a source in the +# opposite direction, asserted by EmojiEmoticonsTest. A line starting with '#' is a comment; the +# notes column is the sixth and final field, so it may contain ';' but not '#'. +# +# fold_type EMOJI: pictograph to canonical ASCII emoticon (many to one). +1F642 ; 003A 0029 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; SLIGHTLY SMILING FACE -> :) +1F60A ; 003A 0029 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; SMILING FACE WITH SMILING EYES -> :) +263A FE0F ; 003A 0029 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; WHITE SMILING FACE, emoji presentation -> :) +263A ; 003A 0029 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; WHITE SMILING FACE -> :) +1F600 ; 003A 0044 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; GRINNING FACE -> :D +1F601 ; 003A 0044 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; GRINNING FACE WITH SMILING EYES -> :D +1F603 ; 003A 0044 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; SMILING FACE WITH OPEN MOUTH -> :D +1F604 ; 003A 0044 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; SMILING FACE WITH OPEN MOUTH AND SMILING EYES -> :D +1F641 ; 003A 0028 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; SLIGHTLY FROWNING FACE -> :( +2639 FE0F ; 003A 0028 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; WHITE FROWNING FACE, emoji presentation -> :( +2639 ; 003A 0028 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; WHITE FROWNING FACE -> :( +1F609 ; 003B 0029 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; WINKING FACE -> ;) +1F61B ; 003A 0050 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; FACE WITH STUCK-OUT TONGUE -> :P +1F61C ; 003A 0050 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; FACE WITH STUCK-OUT TONGUE AND WINKING EYE -> :P +1F61D ; 003A 0050 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; FACE WITH STUCK-OUT TONGUE AND TIGHTLY-CLOSED EYES -> :P +1F62E ; 003A 004F ; EMOJI ; UNSPECIFIED ; 17.0.0 ; FACE WITH OPEN MOUTH -> :O +1F632 ; 003A 004F ; EMOJI ; UNSPECIFIED ; 17.0.0 ; ASTONISHED FACE -> :O +1F622 ; 003A 0027 0028 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; CRYING FACE -> :'( +1F62D ; 003A 0027 0028 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; LOUDLY CRYING FACE -> :'( +1F610 ; 003A 007C ; EMOJI ; UNSPECIFIED ; 17.0.0 ; NEUTRAL FACE -> :| +1F611 ; 003A 007C ; EMOJI ; UNSPECIFIED ; 17.0.0 ; EXPRESSIONLESS FACE -> :| +1F615 ; 003A 002F ; EMOJI ; UNSPECIFIED ; 17.0.0 ; CONFUSED FACE -> :/ +1F617 ; 003A 002A ; EMOJI ; UNSPECIFIED ; 17.0.0 ; KISSING FACE -> :* +1F618 ; 003A 002A ; EMOJI ; UNSPECIFIED ; 17.0.0 ; FACE THROWING A KISS -> :* +2764 FE0F ; 003C 0033 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; HEAVY BLACK HEART, emoji presentation -> <3 +2764 ; 003C 0033 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; HEAVY BLACK HEART -> <3 +1F499 ; 003C 0033 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; BLUE HEART -> <3 +1F49A ; 003C 0033 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; GREEN HEART -> <3 +1F49B ; 003C 0033 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; YELLOW HEART -> <3 +1F49C ; 003C 0033 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; PURPLE HEART -> <3 +1F5A4 ; 003C 0033 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; BLACK HEART -> <3 +1F90D ; 003C 0033 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; WHITE HEART -> <3 +1F90E ; 003C 0033 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; BROWN HEART -> <3 +1F9E1 ; 003C 0033 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; ORANGE HEART -> <3 +1F494 ; 003C 002F 0033 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; BROKEN HEART -> SLIGHTLY SMILING FACE +003A 002D 0029 ; 1F642 ; EMOTICON ; UNSPECIFIED ; 17.0.0 ; :-) -> SLIGHTLY SMILING FACE +003D 0029 ; 1F642 ; EMOTICON ; UNSPECIFIED ; 17.0.0 ; =) -> SLIGHTLY SMILING FACE +003A 0044 ; 1F600 ; EMOTICON ; UNSPECIFIED ; 17.0.0 ; :D -> GRINNING FACE +003A 002D 0044 ; 1F600 ; EMOTICON ; UNSPECIFIED ; 17.0.0 ; :-D -> GRINNING FACE +003A 0028 ; 1F641 ; EMOTICON ; UNSPECIFIED ; 17.0.0 ; :( -> SLIGHTLY FROWNING FACE +003A 002D 0028 ; 1F641 ; EMOTICON ; UNSPECIFIED ; 17.0.0 ; :-( -> SLIGHTLY FROWNING FACE +003B 0029 ; 1F609 ; EMOTICON ; UNSPECIFIED ; 17.0.0 ; ;) -> WINKING FACE +003B 002D 0029 ; 1F609 ; EMOTICON ; UNSPECIFIED ; 17.0.0 ; ;-) -> WINKING FACE +003A 0050 ; 1F61B ; EMOTICON ; UNSPECIFIED ; 17.0.0 ; :P -> FACE WITH STUCK-OUT TONGUE +003A 002D 0050 ; 1F61B ; EMOTICON ; UNSPECIFIED ; 17.0.0 ; :-P -> FACE WITH STUCK-OUT TONGUE +003A 0070 ; 1F61B ; EMOTICON ; UNSPECIFIED ; 17.0.0 ; :p -> FACE WITH STUCK-OUT TONGUE +003A 002D 0070 ; 1F61B ; EMOTICON ; UNSPECIFIED ; 17.0.0 ; :-p -> FACE WITH STUCK-OUT TONGUE +003A 004F ; 1F62E ; EMOTICON ; UNSPECIFIED ; 17.0.0 ; :O -> FACE WITH OPEN MOUTH +003A 002D 004F ; 1F62E ; EMOTICON ; UNSPECIFIED ; 17.0.0 ; :-O -> FACE WITH OPEN MOUTH +003A 006F ; 1F62E ; EMOTICON ; UNSPECIFIED ; 17.0.0 ; :o -> FACE WITH OPEN MOUTH +003A 002D 006F ; 1F62E ; EMOTICON ; UNSPECIFIED ; 17.0.0 ; :-o -> FACE WITH OPEN MOUTH +003A 0027 0028 ; 1F622 ; EMOTICON ; UNSPECIFIED ; 17.0.0 ; :'( -> CRYING FACE +003A 007C ; 1F610 ; EMOTICON ; UNSPECIFIED ; 17.0.0 ; :| -> NEUTRAL FACE +003A 002D 007C ; 1F610 ; EMOTICON ; UNSPECIFIED ; 17.0.0 ; :-| -> NEUTRAL FACE +003A 002F ; 1F615 ; EMOTICON ; UNSPECIFIED ; 17.0.0 ; :/ -> CONFUSED FACE +003A 002D 002F ; 1F615 ; EMOTICON ; UNSPECIFIED ; 17.0.0 ; :-/ -> CONFUSED FACE +003A 002A ; 1F617 ; EMOTICON ; UNSPECIFIED ; 17.0.0 ; :* -> KISSING FACE +003A 002D 002A ; 1F617 ; EMOTICON ; UNSPECIFIED ; 17.0.0 ; :-* -> KISSING FACE +003C 0033 ; 2764 FE0F ; EMOTICON ; UNSPECIFIED ; 17.0.0 ; <3 -> HEAVY BLACK HEART, emoji presentation +003C 002F 0033 ; 1F494 ; EMOTICON ; UNSPECIFIED ; 17.0.0 ; BROKEN HEART diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiEmoticonsTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiEmoticonsTest.java new file mode 100644 index 0000000000..6243e14840 --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiEmoticonsTest.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.util.normalizer; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class EmojiEmoticonsTest { + + private static EmojiEmoticons.Tables bundled() throws IOException { + try (InputStream in = EmojiEmoticons.class.getResourceAsStream("emoji-emoticons.txt")) { + return EmojiEmoticons.parse(in); + } + } + + private static Set sources(Map> table) { + final Set sources = new HashSet<>(); + for (final List candidates : table.values()) { + for (final EmojiEmoticons.Mapping mapping : candidates) { + sources.add(mapping.source()); + } + } + return sources; + } + + @Test + void directionsCloseOverEachOther() throws IOException { + // Every EMOJI target must be an EMOTICON source and every EMOTICON target an EMOJI source, so + // folding one direction and then the other converges on a canonical form instead of producing + // text the reverse table does not know. + final EmojiEmoticons.Tables tables = bundled(); + final Set emoticonSources = sources(tables.emoticonToEmoji().table()); + final Set emojiSources = sources(tables.emojiToEmoticon().table()); + for (final List candidates : tables.emojiToEmoticon().table().values()) { + for (final EmojiEmoticons.Mapping mapping : candidates) { + assertTrue(emoticonSources.contains(mapping.target()), + "EMOJI target has no EMOTICON row: " + mapping); + } + } + for (final List candidates : tables.emoticonToEmoji().table().values()) { + for (final EmojiEmoticons.Mapping mapping : candidates) { + assertTrue(emojiSources.contains(mapping.target()), + "EMOTICON target has no EMOJI row: " + mapping); + } + } + } + + @Test + void emoticonSourcesAreAsciiAndEmojiSourcesAreNot() throws IOException { + // The whitespace-delimited boundary guard exists because emoticon sources are ordinary ASCII + // that also occurs inside text; the emoji direction runs unguarded because its sources are + // pictographs. + final EmojiEmoticons.Tables tables = bundled(); + for (final String source : sources(tables.emoticonToEmoji().table())) { + source.chars().forEach(c -> + assertTrue(c > 0x20 && c < 0x7F, "Non-ASCII-printable emoticon source: " + source)); + } + for (final String source : sources(tables.emojiToEmoticon().table())) { + assertTrue(source.codePointAt(0) > 0x7F, "ASCII-leading emoji source: " + source); + } + } + + @Test + void bundledRowCountsAreAudited() throws IOException { + // Locks the bundled row counts so a data edit trips this test for a conscious bump. + final EmojiEmoticons.Tables tables = bundled(); + assertEquals(35, sources(tables.emojiToEmoticon().table()).size()); + assertEquals(26, sources(tables.emoticonToEmoji().table()).size()); + } + + @Test + void candidatesAreSortedLongestFirst() throws IOException { + // The scan takes the first region match, so longest-match correctness rests on this ordering + // in BOTH directions; the emoji table relies on it to try "2764 FE0F" before bare "2764". + final EmojiEmoticons.Tables tables = bundled(); + for (final EmojiEmoticons.Direction direction + : List.of(tables.emoticonToEmoji(), tables.emojiToEmoticon())) { + for (final List candidates : direction.table().values()) { + for (int i = 1; i < candidates.size(); i++) { + assertTrue( + candidates.get(i - 1).source().length() >= candidates.get(i).source().length(), + "Candidates not longest-first: " + candidates); + } + } + } + } + + @Test + void parseRejectsANullStream() { + assertThrows(NullPointerException.class, () -> EmojiEmoticons.parse(null)); + } + + @Test + void parseFailsLoudOnWrongFieldCount() { + final String data = "1F642 ; 003A 0029 ; EMOJI ; UNSPECIFIED ; 17.0.0\n"; + assertThrows(IllegalArgumentException.class, () -> EmojiEmoticons.parse( + new ByteArrayInputStream(data.getBytes(StandardCharsets.UTF_8)))); + } + + @Test + void parseFailsLoudOnUnrecognizedFoldType() { + final String data = "1F642 ; 003A 0029 ; SMILEY ; UNSPECIFIED ; 17.0.0 ; bad type\n"; + assertThrows(IllegalArgumentException.class, () -> EmojiEmoticons.parse( + new ByteArrayInputStream(data.getBytes(StandardCharsets.UTF_8)))); + } + + @Test + void parseFailsLoudOnMalformedHex() { + final String data = "1F64X ; 003A 0029 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; bad hex\n"; + assertThrows(IllegalArgumentException.class, () -> EmojiEmoticons.parse( + new ByteArrayInputStream(data.getBytes(StandardCharsets.UTF_8)))); + } + + @Test + void parseFailsLoudOnEmptySourceOrTarget() { + final String empty = " ; 003A 0029 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; empty source\n"; + assertThrows(IllegalArgumentException.class, () -> EmojiEmoticons.parse( + new ByteArrayInputStream(empty.getBytes(StandardCharsets.UTF_8)))); + final String emptyTarget = "1F642 ; ; EMOJI ; UNSPECIFIED ; 17.0.0 ; empty target\n"; + assertThrows(IllegalArgumentException.class, () -> EmojiEmoticons.parse( + new ByteArrayInputStream(emptyTarget.getBytes(StandardCharsets.UTF_8)))); + } + + @Test + void parseFailsLoudOnDuplicateSourceWithinADirection() { + final String data = "1F642 ; 003A 0029 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; first\n" + + "1F642 ; 003A 0044 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; duplicate source\n"; + assertThrows(IllegalArgumentException.class, () -> EmojiEmoticons.parse( + new ByteArrayInputStream(data.getBytes(StandardCharsets.UTF_8)))); + } + + @Test + void sameSourceInBothDirectionsIsNotADuplicate() throws IOException { + // A sequence may legitimately be a source in one direction and a target in the other; only a + // duplicate within one direction is ambiguous. + final String data = "263A ; 003A 0029 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; emoji row\n" + + "003A 0029 ; 263A ; EMOTICON ; UNSPECIFIED ; 17.0.0 ; emoticon row\n"; + final EmojiEmoticons.Tables tables = EmojiEmoticons.parse( + new ByteArrayInputStream(data.getBytes(StandardCharsets.UTF_8))); + assertEquals(1, sources(tables.emojiToEmoticon().table()).size()); + assertEquals(1, sources(tables.emoticonToEmoji().table()).size()); + } + + @Test + void commentAndBlankLinesAreSkipped() throws IOException { + final String data = "# a comment line\n\n \n" + + "263A ; 003A 0029 ; EMOJI ; UNSPECIFIED ; 17.0.0 ; the only data row\n"; + final EmojiEmoticons.Tables tables = EmojiEmoticons.parse( + new ByteArrayInputStream(data.getBytes(StandardCharsets.UTF_8))); + assertEquals(1, sources(tables.emojiToEmoticon().table()).size()); + assertTrue(sources(tables.emoticonToEmoji().table()).isEmpty()); + assertFalse(sources(tables.emojiToEmoticon().table()).contains("#")); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiToEmoticonCharSequenceNormalizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiToEmoticonCharSequenceNormalizerTest.java new file mode 100644 index 0000000000..0f3001e4bd --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmojiToEmoticonCharSequenceNormalizerTest.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.util.normalizer; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class EmojiToEmoticonCharSequenceNormalizerTest { + + private static EmojiToEmoticonCharSequenceNormalizer norm() { + return EmojiToEmoticonCharSequenceNormalizer.getInstance(); + } + + private static String cp(int codePoint) { + return new String(Character.toChars(codePoint)); + } + + @Test + void nullInputIsRejectedAtThePublicBoundary() { + assertThrows(NullPointerException.class, () -> norm().normalize(null)); + assertThrows(NullPointerException.class, () -> norm().normalizeAligned(null)); + } + + @Test + void foldsSupplementaryEmojiToEmoticon() { + assertEquals(":)", norm().normalize(cp(0x1F642)).toString()); // SLIGHTLY SMILING FACE + assertEquals(":D", norm().normalize(cp(0x1F600)).toString()); // GRINNING FACE + assertEquals(":'(", norm().normalize(cp(0x1F622)).toString()); // CRYING FACE + } + + @Test + void foldsBmpPictographToEmoticon() { + // U+263A is a one-character BMP pictograph expanding to a two-character emoticon, the case the + // deprecated surrogate-block normalizer never matched at all. + assertEquals(":)", norm().normalize(cp(0x263A)).toString()); + assertEquals("<3", norm().normalize(cp(0x2764)).toString()); + } + + @Test + void emojiPresentationSequenceFoldsAsOneUnit() { + // The base pictograph plus U+FE0F VARIATION SELECTOR-16 must fold together; a bare-pictograph + // match that left the selector behind would emit a dangling invisible character. + assertEquals("<3", norm().normalize(cp(0x2764) + cp(0xFE0F)).toString()); + assertEquals(":)", norm().normalize(cp(0x263A) + cp(0xFE0F)).toString()); + } + + @Test + void manyToOneVariantsConvergeOnTheCanonicalEmoticon() { + assertEquals(":D:D", norm().normalize(cp(0x1F603) + cp(0x1F604)).toString()); + assertEquals("<3<3", norm().normalize(cp(0x1F499) + cp(0x1F9E1)).toString()); + } + + @Test + void unmappedContentPassesThrough() { + final String rocket = cp(0x1F680); // no emoticon mapping + assertEquals("go " + rocket + " now", norm().normalize("go " + rocket + " now").toString()); + assertEquals("plain text", norm().normalize("plain text").toString()); + assertEquals("", norm().normalize("").toString()); + } + + @Test + void foldsInsideRunningTextWithoutBoundaries() { + // Unlike the emoticon direction, the emoji direction needs no delimiter guard: a pictograph + // glued to a word is still unambiguous. + assertEquals("great:D news", norm().normalize("great" + cp(0x1F600) + " news").toString()); + } + + @Test + void alignedNormalizedMatchesNormalize() { + final String in = "a " + cp(0x1F622) + " b " + cp(0x2764) + cp(0xFE0F); + assertEquals(norm().normalize(in).toString(), norm().normalizeAligned(in).normalizedString()); + } + + @Test + void alignmentMapsExpansionBackToThePictograph() { + // "x :'( y" from "x y": the three-character emoticon (output 2..5) maps back to + // the two-unit surrogate pair at input 2..4. + final AlignedText at = norm().normalizeAligned("x " + cp(0x1F622) + " y"); + assertEquals("x :'( y", at.normalizedString()); + assertEquals(new Span(2, 4), at.toOriginalSpan(2, 5)); + } + + @Test + void adjacentFoldsEachMapToTheirOwnSource() { + // Two pictographs back to back must not blur into one block; each emoticon maps to its own. + final AlignedText at = norm().normalizeAligned(cp(0x1F642) + cp(0x1F600)); + assertEquals(":):D", at.normalizedString()); + assertEquals(new Span(0, 2), at.toOriginalSpan(0, 2)); + assertEquals(new Span(2, 4), at.toOriginalSpan(2, 4)); + } + + @Test + void composesIntoTheOffsetAwarePipeline() { + // Collapse the double space (contracting), then fold the pictograph (expanding): a hit in the + // output maps through both stages back to original document coordinates. + final OffsetAwareNormalizer pipeline = + TextNormalizer.builder().whitespace().emojiToEmoticon().buildAligned(); + final AlignedText at = pipeline.normalizeAligned("hi " + cp(0x1F642)); + assertEquals("hi :)", at.normalizedString()); + assertEquals(new Span(4, 6), at.toOriginalSpan(3, 5)); // ":)" maps back to the pictograph + } + + @Test + void zwjSequenceIsNotCorrupted() { + // HEART ON FIRE is HEAVY BLACK HEART + FE0F + ZWJ + FIRE: the embedded heart must not fold, + // which would emit "<3" and leave a dangling joiner in front of the flame. + final String heartOnFire = cp(0x2764) + cp(0xFE0F) + cp(0x200D) + cp(0x1F525); + assertEquals(heartOnFire, norm().normalize(heartOnFire).toString()); + + // COUPLE WITH HEART: the heart sits between two joiners; neither side may fold. + final String couple = cp(0x1F469) + cp(0x200D) + cp(0x2764) + cp(0xFE0F) + cp(0x200D) + + cp(0x1F48B) + cp(0x200D) + cp(0x1F468); + assertEquals(couple, norm().normalize(couple).toString()); + } + + @Test + void zwjSequenceNextToAFoldableEmojiOnlyFoldsTheStandaloneOne() { + final String heartOnFire = cp(0x2764) + cp(0xFE0F) + cp(0x200D) + cp(0x1F525); + final String input = cp(0x1F642) + " " + heartOnFire; + assertEquals(":) " + heartOnFire, norm().normalize(input).toString()); + } + + @Test + void textPresentationSelectorSuppressesTheFold() { + // U+FE0E explicitly requests text presentation; folding would both misread the author's + // intent and leave the selector dangling. + final String textSmiley = cp(0x263A) + cp(0xFE0E); + assertEquals(textSmiley, norm().normalize(textSmiley).toString()); + } + + @Test + void trailingEmojiPresentationSelectorIsAbsorbedForEveryMappedPictograph() { + // 1F642 has no explicit FE0F row; the selector must still fold with it, not dangle. + assertEquals(":)", norm().normalize(cp(0x1F642) + cp(0xFE0F)).toString()); + } + + @Test + void absorbedSelectorMapsBackToTheWholeSequence() { + final AlignedText at = norm().normalizeAligned(cp(0x1F642) + cp(0xFE0F) + "!"); + assertEquals(":)!", at.normalizedString()); + // ":)" covers the pictograph plus the absorbed selector: three UTF-16 units. + assertEquals(new Span(0, 3), at.toOriginalSpan(0, 2)); + assertEquals(at.normalizedString(), + norm().normalize(cp(0x1F642) + cp(0xFE0F) + "!").toString()); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmoticonToEmojiCharSequenceNormalizerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmoticonToEmojiCharSequenceNormalizerTest.java new file mode 100644 index 0000000000..0bd09c44df --- /dev/null +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/EmoticonToEmojiCharSequenceNormalizerTest.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package opennlp.tools.util.normalizer; + +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import opennlp.tools.tokenize.uax29.WordTokenizer; +import opennlp.tools.tokenize.uax29.WordType; +import opennlp.tools.util.Span; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class EmoticonToEmojiCharSequenceNormalizerTest { + + private static EmoticonToEmojiCharSequenceNormalizer norm() { + return EmoticonToEmojiCharSequenceNormalizer.getInstance(); + } + + private static String cp(int codePoint) { + return new String(Character.toChars(codePoint)); + } + + @Test + void nullInputIsRejectedAtThePublicBoundary() { + assertThrows(NullPointerException.class, () -> norm().normalize(null)); + assertThrows(NullPointerException.class, () -> norm().normalizeAligned(null)); + } + + @Test + void foldsDelimitedEmoticons() { + assertEquals(cp(0x1F642), norm().normalize(":)").toString()); + assertEquals("a " + cp(0x1F641) + " b", norm().normalize("a :( b").toString()); + assertEquals("love <3".replace("<3", cp(0x2764) + cp(0xFE0F)), + norm().normalize("love <3").toString()); + } + + @Test + void longestMatchWinsAtAPosition() { + // ":-)" must fold as one unit; a shortest-first scan would never reach it, and a prefix match + // of a shorter source would leave stray characters. + assertEquals("hi " + cp(0x1F642), norm().normalize("hi :-)").toString()); + assertEquals(cp(0x1F494), norm().normalize(" b" from "a :-) b": the surrogate pair (output 2..4) maps back to + // the three-character emoticon at input 2..5. + final AlignedText at = norm().normalizeAligned("a :-) b"); + assertEquals("a " + cp(0x1F642) + " b", at.normalizedString()); + assertEquals(new Span(2, 5), at.toOriginalSpan(2, 4)); + } + + @Test + void composesIntoTheOffsetAwarePipeline() { + // Collapse the whitespace run (contracting), then fold the emoticon (also contracting): a hit + // on the emoji in the output maps through both stages back to the original coordinates. + final OffsetAwareNormalizer pipeline = + TextNormalizer.builder().whitespace().emoticonToEmoji().buildAligned(); + final AlignedText at = pipeline.normalizeAligned("ok :-)"); + assertEquals("ok " + cp(0x1F642), at.normalizedString()); + assertEquals(new Span(5, 8), at.toOriginalSpan(3, 5)); + } + + @Test + void roundTripThroughBothDirectionsConvergesOnCanonicalForms() { + // :-) folds to the pictograph; folding back yields the canonical short emoticon :), not the + // original long variant. Convergence, not restoration, is the documented contract. + final CharSequence folded = norm().normalize("well :-) then"); + assertEquals("well :) then", + EmojiToEmoticonCharSequenceNormalizer.getInstance().normalize(folded).toString()); + } + + @Test + void foldingBeforeTokenizationMakesEmoticonsAndEmojiOneClass() { + // The UAX #29 word tokenizer drops an unfolded emoticon as punctuation but keeps a pictograph + // as an EMOJI token, so folding before tokenization is what lets the signal survive. + final WordTokenizer tokenizer = new WordTokenizer(); + assertArrayEquals(new String[] {"great"}, tokenizer.tokenize("great :)")); + final String folded = norm().normalize("great :)").toString(); + assertArrayEquals(new String[] {"great", cp(0x1F642)}, tokenizer.tokenize(folded)); + final List types = new ArrayList<>(); + tokenizer.tokenize(folded, (start, end, type) -> types.add(type)); + assertEquals(List.of(WordType.ALPHANUMERIC, WordType.EMOJI), types); + } +} diff --git a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerTest.java b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerTest.java index 8f10121fd1..db9018e996 100644 --- a/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerTest.java +++ b/opennlp-core/opennlp-runtime/src/test/java/opennlp/tools/util/normalizer/TermAnalyzerTest.java @@ -448,4 +448,26 @@ void testFullCaseFoldExpandsThroughTheTermModel() { assertEquals("masse", term.normalized()); assertEquals("masse", term.at(Dimension.FULL_CASE_FOLD)); } + + @Test + void testEmojiFoldLayersTheEmoticonOnTheToken() { + // A one-code-point pictograph token gains an EMOJI_FOLD layer of a different length. + final TermAnalyzer analyzer = TermAnalyzer.builder().emojiFold().build(); + final Term term = analyzer.analyze(cp(0x1F600)).get(0); // GRINNING FACE + assertEquals(":D", term.at(Dimension.EMOJI_FOLD)); + assertEquals(cp(0x1F600), term.at(Dimension.ORIGINAL)); + } + + @Test + void testEmojiFoldLeavesPlainTokensUntouched() { + final TermAnalyzer analyzer = TermAnalyzer.builder().emojiFold().build(); + final Term term = analyzer.analyze("hello").get(0); + assertEquals("hello", term.at(Dimension.EMOJI_FOLD)); + } + + @Test + void testEmojiFoldDefaultNormalizerWiring() { + assertEquals(EmojiToEmoticonCharSequenceNormalizer.getInstance(), + Dimension.EMOJI_FOLD.defaultNormalizer()); + } } diff --git a/opennlp-docs/src/docbkx/normalizer.xml b/opennlp-docs/src/docbkx/normalizer.xml index bbeda9d39b..f008a84059 100644 --- a/opennlp-docs/src/docbkx/normalizer.xml +++ b/opennlp-docs/src/docbkx/normalizer.xml @@ -164,6 +164,19 @@ Reduces lookalike characters to a confusable skeleton for matching (UTS #39); see below. + + EmojiToEmoticonCharSequenceNormalizer + Folds emoji to ASCII emoticons from a bundled project-authored mapping + (U+1F642 to :)); an emoji presentation sequence (a pictograph + followed by U+FE0F) folds as one unit. Unmapped content passes through. + + + EmoticonToEmojiCharSequenceNormalizer + Folds an ASCII emoticon that stands alone between whitespace to its + pictograph (:-) to U+1F642). An embedded sequence, such as the + :/ in https://, is left alone. Applied before + tokenization, it lets emoticons survive as single emoji tokens. + @@ -245,9 +258,11 @@ Span hit = aligned.toOriginalSpan(5, 14); // "the-match" in the normalized tex for it with a plain instanceof, the same pattern the name finder uses for OffsetMappingNameFinder. Every per-code-point fold implements it: whitespace, the line-break-preserving whitespace rung, dashes, invisible-control stripping, quotes, digits, - ellipsis, bullets, the German umlaut transliteration, and Unicode full case folding + ellipsis, bullets, the German umlaut transliteration, Unicode full case folding (fullCaseFold(), whose expansions come from a bundled table with known lengths, so - it reports its edits). The folds that route through + it reports its edits), and the emoji/emoticon folds (emojiToEmoticon() and + emoticonToEmoji(), backed by the same kind of known-length bundled table). The + folds that route through java.text.Normalizer (NFC, NFKC, accent folding, and confusable folding) or through JDK case mapping (the locale-based caseFold()) cannot report their per-character edits, so they do not implement the interface, and a chain that contains one is rejected by @@ -454,7 +469,7 @@ CharClass wsPlus = CharClass.whitespace().withAdditional(extra);]]> highlighting, even when normalization changes a token's length. A Term is one token projected through an ordered chain of Dimensions: original, NFC, NFKC, whitespace, dash, case fold, full case fold, accent fold, - confusable fold, stem, and lemma. The order is fixed because the transforms do not commute + emoji fold, confusable fold, stem, and lemma. The order is fixed because the transforms do not commute (case folding then accent folding differs from the reverse). The original is always kept, so aggressive folding stays safe and a match on any layer maps back to the source through the token's Span.