|
37 | 37 | from tensorflow.python.platform import test |
38 | 38 | from tensorflow.python.saved_model import load |
39 | 39 | from tensorflow.python.saved_model import save |
| 40 | +from tensorflow.python.framework import load_library |
| 41 | +from tensorflow.python.platform import resource_loader |
| 42 | +gen_sentencepiece_tokenizer = load_library.load_op_library(resource_loader.get_path_to_datafile('_sentencepiece_tokenizer.so')) |
40 | 43 | from tensorflow_text.python.ops.sentencepiece_tokenizer import SentencepieceTokenizer |
41 | 44 |
|
42 | 45 |
|
@@ -451,6 +454,17 @@ def testEmptyInputDetokenize(self): |
451 | 454 | detokenized = sp.detokenize(constant_op.constant([], dtypes.int32)) |
452 | 455 | self.assertAllEqual('', detokenized) |
453 | 456 |
|
| 457 | + def testEmptyInputSplitsDetokenizeOpZeroShape(self): |
| 458 | + sp = SentencepieceTokenizer(self.model) |
| 459 | + # Providing an empty tensor (length 0) as input_splits directly to the op |
| 460 | + # should safely raise InvalidArgumentError instead of crashing. |
| 461 | + with self.assertRaises((errors.InvalidArgumentError, ValueError)): |
| 462 | + _ = gen_sentencepiece_tokenizer.sentencepiece_detokenize_op( |
| 463 | + sp._model_resource.resource_handle, |
| 464 | + constant_op.constant([], dtypes.int32), |
| 465 | + constant_op.constant([], dtypes.int64), |
| 466 | + add_bos=False, add_eos=False, reverse=False) |
| 467 | + |
454 | 468 | def testReturnNbestAndDetokenize(self): |
455 | 469 | sp = SentencepieceTokenizer( |
456 | 470 | self.model, nbest_size=2, out_type=dtypes.int32, return_nbest=True) |
|
0 commit comments