OPENNLP-1875: Address review: comment sweep, StringUtil.isUnicodeWhitespace facade

Author: krickert

opennlp-api/src/main/java/opennlp/tools/namefind/NameSample.java

@@ -262,8 +262,6 @@ private static String errorTokenWithContext(String[] sentence, int index) {
     return errorString.toString();
   }
 
-  // Annotation-markup parsing: the <START:type> tag grammar itself excludes ASCII
-  // whitespace in type names; this is the markup format, not user-text classification.
   private static final Pattern START_TAG_PATTERN = Pattern.compile("<START(:([^:>\\s]*))?>");
 
 
@@ -294,10 +292,6 @@ public static NameSample parse(String taggedTokens, boolean clearAdaptiveData) t
   public static NameSample parse(String taggedTokens, String defaultType, boolean clearAdaptiveData)
           throws IOException {
 
-    // Sample parsing deliberately shares the runtime WhitespaceTokenizer, so training samples
-    // split exactly like inference input (Unicode White_Space since 3.0): annotation lines
-    // containing U+0085 or U+001C..U+001F yield correspondingly shifted tokens and name
-    // indices, pinned in NameSampleTest.
     String[] parts = WhitespaceTokenizer.INSTANCE.tokenize(taggedTokens);
 
     List<String> tokenList = new ArrayList<>(parts.length);

opennlp-api/src/main/java/opennlp/tools/parser/Parse.java

@@ -119,7 +119,6 @@ public class Parse implements Cloneable, Comparable<Parse> {
 
   /**
    * The patter used to identify tokens in Penn Treebank labeled constituents.
-   * Format parsing: the bracketed PTB syntax is ASCII-whitespace delimited by definition.
    */
   private static final Pattern tokenPattern = Pattern.compile("^[^ ()]+ ([^ ()]+)\\s*\\)");
 

opennlp-api/src/main/java/opennlp/tools/postag/POSSample.java

@@ -162,9 +162,6 @@ public String toString() {
    */
   public static POSSample parse(String sentenceString) throws InvalidFormatException {
 
-    // Sample parsing deliberately shares the runtime WhitespaceTokenizer, so training samples
-    // split exactly like inference input (Unicode White_Space since 3.0): lines containing
-    // U+0085 or U+001C..U+001F yield correspondingly shifted word_tag pairs.
     String[] tokenTags = WhitespaceTokenizer.INSTANCE.tokenize(sentenceString);
 
     String[] sentence = new String[tokenTags.length];

opennlp-api/src/main/java/opennlp/tools/tokenize/BertNormalization.java

@@ -63,11 +63,9 @@ static boolean isControl(int codePoint) {
    * A whitespace character in the BERT sense: space, tab, newline, carriage
    * return, or Unicode space separators ({@code Zs}).
    *
-   * <p>Intentional exception to the Unicode {@code White_Space} migration: this definition
-   * is fixed by the reference BERT implementation, and pre-trained BERT vocabularies were
-   * built with exactly this tokenization. It therefore excludes the {@code White_Space}
-   * members {@code U+000B}, {@code U+000C}, {@code U+0085}, {@code U+2028} and
-   * {@code U+2029} on purpose. Pinned by {@code BertNormalizationWhitespaceTest}.</p>
+   * <p>Deliberately not the Unicode {@code White_Space} set: this definition is fixed by
+   * the reference BERT implementation, and pre-trained vocabularies depend on it. Pinned by
+   * {@code BertNormalizationWhitespaceTest}.</p>
    */
   static boolean isWhitespace(int codePoint) {
     if (codePoint == ' ' || codePoint == '\t' || codePoint == '\n' || codePoint == '\r') {

opennlp-api/src/main/java/opennlp/tools/tokenize/TokenSample.java

@@ -179,9 +179,6 @@ public static TokenSample parse(String sampleString, String separatorChars) {
     Objects.requireNonNull(sampleString, "sampleString must not be null");
     Objects.requireNonNull(separatorChars, "separatorChars must not be null");
 
-    // Sample parsing deliberately shares the runtime WhitespaceTokenizer, so training samples
-    // split exactly like inference input (Unicode White_Space since 3.0): sample lines
-    // containing U+0085 or U+001C..U+001F yield correspondingly shifted token spans.
     Span[] whitespaceTokenSpans = WhitespaceTokenizer.INSTANCE.tokenizePos(sampleString);
 
     // Pre-allocate 20% for newly created tokens

opennlp-api/src/main/java/opennlp/tools/tokenize/WhitespaceTokenizer.java

@@ -21,23 +21,16 @@
 import java.util.List;
 
 import opennlp.tools.util.Span;
-import opennlp.tools.util.normalizer.UnicodeWhitespace;
+import opennlp.tools.util.StringUtil;
 
 /**
  * A basic {@link Tokenizer} implementation which performs tokenization
  * using white spaces.
  * <p>
- * Whitespace is the Unicode {@code White_Space} set
- * ({@link UnicodeWhitespace#isWhitespace(int)}). Since 3.0 the next line control
- * {@code U+0085} separates tokens and the {@code U+001C}..{@code U+001F} information
- * separators no longer do, matching the standard instead of the JVM predicates. Token
- * boundaries produced for text containing those code points differ from earlier releases,
- * which can also shift the candidate spans {@code TokenizerME} scores for such text.
- * <p>
- * With {@link #setKeepNewLines(boolean)} enabled, the line separator code points
- * {@code \n}, {@code \r} and, since 3.0, the next line control {@code U+0085}, the line
- * separator {@code U+2028} and the paragraph separator {@code U+2029} are returned as
- * tokens of their own; all other whitespace is dropped.
+ * Since 3.0, whitespace is the Unicode {@code White_Space} set
+ * ({@link StringUtil#isUnicodeWhitespace(int)}); see OPENNLP-1875 for the delta to earlier
+ * releases. With {@link #setKeepNewLines(boolean)} enabled, line separator code points are
+ * returned as tokens of their own.
  * <p>
  * To obtain an instance of this tokenizer use the static final
  * {@link #INSTANCE} field.
@@ -65,7 +58,7 @@ public Span[] tokenizePos(String d) {
     // gather potential tokens
     int end = d.length();
     for (int i = 0; i < end; i++) {
-      if (UnicodeWhitespace.isWhitespace(d.charAt(i))) {
+      if (StringUtil.isUnicodeWhitespace(d.charAt(i))) {
         if (inTok) {
           tokens.add(new Span(tokStart, i));
           inTok = false;
@@ -91,10 +84,6 @@ public Span[] tokenizePos(String d) {
     return tokens.toArray(new Span[0]);
   }
 
-  // The line separator code points emitted as tokens under keepNewLines: the ASCII pair,
-  // plus NEL (U+0085) and the Unicode line and paragraph separators, which count as
-  // whitespace since 3.0. (The previous version compared against the Character category
-  // constants LINE_SEPARATOR and LETTER_NUMBER, whose byte values happen to be 13 and 10.)
   private boolean isLineSeparator(char character) {
     return character == '\n' || character == '\r'
         || character == '\u0085' || character == '\u2028' || character == '\u2029';

opennlp-api/src/main/java/opennlp/tools/util/Span.java

@@ -21,8 +21,6 @@
 import java.io.Serializable;
 import java.util.Objects;
 
-import opennlp.tools.util.normalizer.UnicodeWhitespace;
-
 /**
  * Class for storing start and end integer offsets.
  *
@@ -259,10 +257,8 @@ public CharSequence getCoveredText(CharSequence text) {
   }
 
   /**
-   * Whitespace here is the Unicode {@code White_Space} set
-   * ({@link UnicodeWhitespace#isWhitespace(int)}). Since 3.0 this trims the next line
-   * control {@code U+0085} and no longer trims the {@code U+001C}..{@code U+001F}
-   * information separators, matching the standard instead of the JVM predicates.
+   * Since 3.0, whitespace here is the Unicode {@code White_Space} set
+   * ({@link StringUtil#isUnicodeWhitespace(int)}); see OPENNLP-1875.
    *
    * @param text The {@link CharSequence text} to analyze.
    *
@@ -273,12 +269,12 @@ public Span trim(CharSequence text) {
 
     int newStartOffset = getStart();
 
-    for (int i = getStart(); i < getEnd() && UnicodeWhitespace.isWhitespace(text.charAt(i)); i++) {
+    for (int i = getStart(); i < getEnd() && StringUtil.isUnicodeWhitespace(text.charAt(i)); i++) {
       newStartOffset++;
     }
 
     int newEndOffset = getEnd();
-    for (int i = getEnd(); i > getStart() && UnicodeWhitespace.isWhitespace(text.charAt(i - 1)); i--) {
+    for (int i = getEnd(); i > getStart() && StringUtil.isUnicodeWhitespace(text.charAt(i - 1)); i--) {
       newEndOffset--;
     }
 

opennlp-api/src/main/java/opennlp/tools/util/StringUtil.java

@@ -22,6 +22,8 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import opennlp.tools.util.normalizer.UnicodeWhitespace;
+
 public class StringUtil {
 
   private static final Logger logger = LoggerFactory.getLogger(StringUtil.class);
@@ -32,15 +34,10 @@ public class StringUtil {
    * category ({@link Character#SPACE_SEPARATOR}), which adds the no-break spaces that
    * {@code Character.isWhitespace} excludes.
    *
-   * <p>This predicate is deliberately <em>not</em> the Unicode {@code White_Space} property:
-   * it additionally treats the information separators {@code U+001C}..{@code U+001F} as
-   * whitespace and does not cover the next line control {@code U+0085}. It is kept frozen
-   * because the feature generation of trained models is built on it
-   * ({@code DefaultSDContextGenerator}, {@code DefaultTokenContextGenerator}); changing its
-   * semantics would silently change the features generated for existing models. Code that
-   * classifies whitespace in user text should use the standards-sourced
-   * {@link opennlp.tools.util.normalizer.UnicodeWhitespace#isWhitespace(int)} or
-   * {@link opennlp.tools.util.normalizer.CharClass#whitespace()} instead.</p>
+   * <p>This predicate is deliberately not the Unicode {@code White_Space} property and is
+   * kept frozen because trained-model feature generation depends on it. Use
+   * {@link #isUnicodeWhitespace(char)} for whitespace classification in user text
+   * (OPENNLP-1875).</p>
    *
    * @param charCode The character to check.
    *
@@ -57,15 +54,10 @@ public static boolean isWhitespace(char charCode) {
    * category ({@link Character#SPACE_SEPARATOR}), which adds the no-break spaces that
    * {@code Character.isWhitespace} excludes.
    *
-   * <p>This predicate is deliberately <em>not</em> the Unicode {@code White_Space} property:
-   * it additionally treats the information separators {@code U+001C}..{@code U+001F} as
-   * whitespace and does not cover the next line control {@code U+0085}. It is kept frozen
-   * because the feature generation of trained models is built on it
-   * ({@code DefaultSDContextGenerator}, {@code DefaultTokenContextGenerator}); changing its
-   * semantics would silently change the features generated for existing models. Code that
-   * classifies whitespace in user text should use the standards-sourced
-   * {@link opennlp.tools.util.normalizer.UnicodeWhitespace#isWhitespace(int)} or
-   * {@link opennlp.tools.util.normalizer.CharClass#whitespace()} instead.</p>
+   * <p>This predicate is deliberately not the Unicode {@code White_Space} property and is
+   * kept frozen because trained-model feature generation depends on it. Use
+   * {@link #isUnicodeWhitespace(int)} for whitespace classification in user text
+   * (OPENNLP-1875).</p>
    *
    * @param charCode An int representation of a character to check.
    *
@@ -76,6 +68,34 @@ public static boolean isWhitespace(int charCode) {
         Character.getType(charCode) == Character.SPACE_SEPARATOR;
   }
 
+  /**
+   * Determines if the specified {@link Character} is a whitespace under the Unicode
+   * {@code White_Space} property; delegates to
+   * {@link opennlp.tools.util.normalizer.UnicodeWhitespace#isWhitespace(int)}.
+   *
+   * @param charCode The character to check.
+   *
+   * @return {@code true} if {@code charCode} has the {@code White_Space} property,
+   *     {@code false} otherwise.
+   */
+  public static boolean isUnicodeWhitespace(char charCode) {
+    return UnicodeWhitespace.isWhitespace(charCode);
+  }
+
+  /**
+   * Determines if the specified code point is a whitespace under the Unicode
+   * {@code White_Space} property; delegates to
+   * {@link opennlp.tools.util.normalizer.UnicodeWhitespace#isWhitespace(int)}.
+   *
+   * @param charCode An int representation of a character to check.
+   *
+   * @return {@code true} if {@code charCode} has the {@code White_Space} property,
+   *     {@code false} otherwise.
+   */
+  public static boolean isUnicodeWhitespace(int charCode) {
+    return UnicodeWhitespace.isWhitespace(charCode);
+  }
+
 
   /**
    * Converts a {@link CharSequence} to lower case, independent of the current

opennlp-core/opennlp-cli/src/main/java/opennlp/tools/cmdline/stopword/StopwordFilterTool.java

@@ -90,8 +90,6 @@ public void run(final String[] args) {
           writer.println();
           continue;
         }
-        // User text: tokenize on the Unicode White_Space set (WhitespaceTokenizer)
-        // instead of the ASCII \s regex, so NBSP-style separators split tokens too.
         final String[] tokens = WhitespaceTokenizer.INSTANCE.tokenize(line);
         final String[] kept = filter.filter(tokens);
         writer.println(String.join(" ", kept));

opennlp-core/opennlp-formats/src/main/java/opennlp/tools/formats/ad/ADNameSampleStream.java

@@ -70,7 +70,6 @@ public class ADNameSampleStream implements ObjectStream<NameSample> {
    * Pattern of a NER tag in Arvores Deitadas
    */
   private static final Pattern TAG_PATTERN = Pattern.compile("<(NER:)?(.*?)>");
-  // Corpus-format parsing: AD tag fields are ASCII-whitespace delimited by the format.
   private static final Pattern WHITESPACE_PATTERN = Pattern.compile("\\s+");
   private static final Pattern UNDERLINE_PATTERN = Pattern.compile("[_]+");
   private static final Pattern HYPHEN_PATTERN =
@@ -402,7 +401,6 @@ private static String getNER(String tags) {
     if (tags.contains("<NER2>")) {
       return null;
     }
-    // Corpus-format parsing: the AD tag field is ASCII-whitespace delimited by the format.
     String[] tag = tags.split("\\s+");
     for (String t : tag) {
       Matcher matcher = TAG_PATTERN.matcher(t);