OPENNLP-1885: Move the subword contract into opennlp-api

SubwordTokenizer and SubwordPiece move to opennlp.tools.tokenize, next to
Tokenizer and WordpieceTokenizer, matching where every other seam of this
round lives. The opennlp-subword module keeps only the SentencePiece
implementation.

Author: krickert

…n/java/opennlp/subword/SubwordPiece.java …opennlp/tools/tokenize/SubwordPiece.javaopennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/SubwordPiece.java renamed to opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordPiece.java opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/SubwordPiece.java renamed to opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordPiece.java

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package opennlp.subword;
+package opennlp.tools.tokenize;
 
 import opennlp.tools.util.Span;
 

…va/opennlp/subword/SubwordTokenizer.java …nlp/tools/tokenize/SubwordTokenizer.javaopennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/SubwordTokenizer.java renamed to opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordTokenizer.java opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/SubwordTokenizer.java renamed to opennlp-api/src/main/java/opennlp/tools/tokenize/SubwordTokenizer.java

@@ -14,7 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package opennlp.subword;
+package opennlp.tools.tokenize;
 
 import java.util.List;
 
@@ -24,7 +24,7 @@
  *
  * <p>Subword tokenization is the input layer of modern sequence models: text is decomposed into
  * pieces from a trained vocabulary so that any input, including words never seen in training, maps
- * to a bounded id space. Unlike a linguistic {@code Tokenizer}, the segmentation is
+ * to a bounded id space. Unlike a linguistic {@link Tokenizer}, the segmentation is
  * vocabulary-driven, and the pieces are in the model's normalized form rather than substrings of
  * the input. The offsets carried by each {@link SubwordPiece} are what tie the two worlds
  * together: they always refer to the caller's original text.</p>

opennlp-extensions/opennlp-subword/src/main/java/opennlp/subword/sentencepiece/SentencePieceTokenizer.java

@@ -26,8 +26,8 @@
 import java.util.List;
 import java.util.Map;
 
-import opennlp.subword.SubwordPiece;
-import opennlp.subword.SubwordTokenizer;
+import opennlp.tools.tokenize.SubwordPiece;
+import opennlp.tools.tokenize.SubwordTokenizer;
 import opennlp.tools.util.normalizer.AlignedText;
 import opennlp.tools.util.normalizer.Alignment;
 import opennlp.tools.util.normalizer.OffsetAwareNormalizer;

opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceAlignmentTest.java

@@ -22,7 +22,7 @@
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 
-import opennlp.subword.SubwordPiece;
+import opennlp.tools.tokenize.SubwordPiece;
 import opennlp.tools.util.Span;
 import opennlp.tools.util.normalizer.AlignedText;
 

opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceModelValidationTest.java

@@ -30,7 +30,7 @@
 
 import org.junit.jupiter.api.Test;
 
-import opennlp.subword.SubwordPiece;
+import opennlp.tools.tokenize.SubwordPiece;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;

opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceParityTest.java

@@ -30,7 +30,7 @@
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 
-import opennlp.subword.SubwordPiece;
+import opennlp.tools.tokenize.SubwordPiece;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;

opennlp-extensions/opennlp-subword/src/test/java/opennlp/subword/sentencepiece/SentencePieceRealModelEvalTest.java

@@ -26,7 +26,7 @@
 
 import org.junit.jupiter.api.Test;
 
-import opennlp.subword.SubwordPiece;
+import opennlp.tools.tokenize.SubwordPiece;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;