OPENNLP-1885: Pure-Java SentencePiece inference with exact original-text spans (opennlp-subword)

opennlp-api/src/main/java/opennlp/tools/tokenize/BertNormalization.java

@@ -19,7 +19,7 @@
 
 /**
  * Character classifications and text transforms of the reference BERT
- * {@code BasicTokenizer}, shared by {@link BertTokenizer} and
+ * {@code BasicTokenizer}, shared by {@link WordpieceEncoder} and
  * {@link WordpieceTokenizer}.
  */
 final class BertNormalization {

opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordPiece.java

@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.tokenize;
+
+import opennlp.tools.util.Span;
+
+/**
+ * One subword unit produced by a {@link SubwordTokenizer}, carrying both the vocabulary view
+ * (the piece string and its id) and the exact place in the caller's text it came from.
+ *
+ * <p>The piece string is in the tokenizer's normalized form, so it is generally not a substring of
+ * the input. {@code start} and {@code end} are UTF-16 offsets into the original text, so the
+ * surface that produced this piece is {@code text.subSequence(start, end)}. Pieces that carry no
+ * surface of their own, such as control symbols or the fill bytes of a byte-fallback expansion,
+ * report an empty span with {@code start == end}.</p>
+ *
+ * @param piece The piece in the vocabulary's normalized form; never null or empty.
+ * @param id    The vocabulary id of the piece.
+ * @param start The inclusive start offset in the original text.
+ * @param end   The exclusive end offset in the original text; not less than {@code start}.
+ */
+public record SubwordPiece(String piece, int id, int start, int end) {
+
+  /**
+   * Instantiates a {@link SubwordPiece}.
+   *
+   * @throws IllegalArgumentException Thrown if {@code piece} is null or empty, or the span is
+   *     negative or inverted.
+   */
+  public SubwordPiece {
+    if (piece == null || piece.isEmpty()) {
+      throw new IllegalArgumentException("The piece must not be null or empty.");
+    }
+    if (start < 0 || end < start) {
+      throw new IllegalArgumentException(
+          "The span [" + start + ", " + end + ") must not be negative or inverted.");
+    }
+  }
+
+  /** {@return the original-text span of this piece as a {@link Span}} */
+  public Span span() {
+    return new Span(start, end);
+  }
+}

opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordTokenizer.java

@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.tokenize;
+
+import java.util.List;
+
+/**
+ * Splits text into subword units against a fixed vocabulary, reporting for every unit its
+ * vocabulary id and the exact span of the original text it covers.
+ *
+ * <p>The segmentation is vocabulary-driven rather than linguistic, and each piece is in the
+ * model's normalized form, so a piece is generally not a substring of the input. The offsets
+ * carried by each {@link SubwordPiece} always refer to the caller's original text.</p>
+ *
+ * <p>Implementations are expected to be safe for concurrent use by multiple threads; any
+ * implementation that is not must document it.</p>
+ */
+public interface SubwordTokenizer {
+
+  /**
+   * Encodes text into subword pieces.
+   *
+   * @param text The text to encode; must not be null.
+   * @return The pieces in text order; empty when the text contains nothing encodable.
+   * @throws IllegalArgumentException Thrown if {@code text} is null.
+   */
+  List<SubwordPiece> encode(CharSequence text);
+
+  /**
+   * Encodes text into vocabulary ids.
+   *
+   * @param text The text to encode; must not be null.
+   * @return The ids in text order; empty when the text contains nothing encodable.
+   * @throws IllegalArgumentException Thrown if {@code text} is null.
+   */
+  default int[] encodeToIds(CharSequence text) {
+    final List<SubwordPiece> pieces = encode(text);
+    final int[] ids = new int[pieces.size()];
+    for (int i = 0; i < ids.length; i++) {
+      ids[i] = pieces.get(i).id();
+    }
+    return ids;
+  }
+
+  /**
+   * Encodes text into piece strings in the vocabulary's normalized form.
+   *
+   * @param text The text to encode; must not be null.
+   * @return The pieces in text order; empty when the text contains nothing encodable.
+   * @throws IllegalArgumentException Thrown if {@code text} is null.
+   */
+  default String[] encodeToPieces(CharSequence text) {
+    final List<SubwordPiece> pieces = encode(text);
+    final String[] out = new String[pieces.size()];
+    for (int i = 0; i < out.length; i++) {
+      out[i] = pieces.get(i).piece();
+    }
+    return out;
+  }
+}