3030_RE_WORD_CHAR = re .compile (r"\w" )
3131
3232
33-
3433def word_detokenize (
3534 segments : Union [list [list [str ]], list [str ]], output : str = "str"
3635) -> Union [list [str ], str ]:
@@ -483,16 +482,13 @@ def sent_tokenize(
483482 if not text or not isinstance (text , (str , list )):
484483 return []
485484
486- is_list_input = isinstance (text , list )
487-
488- if is_list_input :
485+ if isinstance (text , list ):
489486 try :
490487 original_text = "" .join (text )
491488 except ValueError :
492489 return []
493-
494490 else :
495- original_text = text
491+ original_text = str ( text )
496492
497493 segments = []
498494
@@ -501,13 +497,13 @@ def sent_tokenize(
501497
502498 segments = segment (original_text )
503499
504- if is_list_input :
500+ if isinstance ( text , list ) :
505501 word_indices = indices_words (text )
506502 result = map_indices_to_words (word_indices , [original_text ])
507503 return result
508504 elif engine == "whitespace" :
509505 segments = re .split (r" +" , original_text , flags = re .U )
510- if is_list_input :
506+ if isinstance ( text , list ) :
511507 result = []
512508 _temp : list [str ] = []
513509 for i , w in enumerate (text ):
@@ -523,7 +519,7 @@ def sent_tokenize(
523519 return result
524520 elif engine == "whitespace+newline" :
525521 segments = original_text .split ()
526- if is_list_input :
522+ if isinstance ( text , list ) :
527523 result = []
528524 _temp = []
529525 for i , w in enumerate (text ):
@@ -538,24 +534,22 @@ def sent_tokenize(
538534 result .append (_temp )
539535 return result
540536 elif engine == "tltk" :
541- from pythainlp .tokenize .tltk import sent_tokenize as segment
537+ from pythainlp .tokenize .tltk import sent_tokenize as tltk_sent_tokenize
542538
543- segments = segment (original_text )
539+ segments = tltk_sent_tokenize (original_text )
544540 elif engine == "thaisum" :
545- from pythainlp .tokenize .thaisumcut import (
546- ThaiSentenceSegmentor as segmentor ,
547- )
541+ from pythainlp .tokenize .thaisumcut import ThaiSentenceSegmentor
548542
549- segment = segmentor ()
550- segments = segment .split_into_sentences (original_text )
543+ segmentor = ThaiSentenceSegmentor ()
544+ segments = segmentor .split_into_sentences (original_text )
551545 elif engine .startswith ("wtp" ):
552546 if "-" not in engine :
553547 _size = "mini"
554548 else :
555549 _size = engine .split ("-" )[- 1 ]
556- from pythainlp .tokenize .wtsplit import tokenize as segment
550+ from pythainlp .tokenize .wtsplit import tokenize
557551
558- segments = segment ( original_text , size = _size , tokenize = "sentence" )
552+ segments = tokenize ( text = original_text , size = _size , tokenize = "sentence" )
559553 else :
560554 raise ValueError (
561555 f"""Tokenizer \" { engine } \" not found.
@@ -565,7 +559,7 @@ def sent_tokenize(
565559 if not keep_whitespace :
566560 segments = strip_whitespace (segments )
567561
568- if is_list_input and engine not in ["crfcut" ]:
562+ if isinstance ( text , list ) and engine not in ["crfcut" ]:
569563 word_indices = indices_words (text )
570564 result = map_indices_to_words (word_indices , segments )
571565 return result
@@ -914,7 +908,7 @@ class Tokenizer:
914908
915909 def __init__ (
916910 self ,
917- custom_dict : Union [Trie , Iterable [str ], str ] = [] ,
911+ custom_dict : Union [Trie , Iterable [str ], str , None ] = None ,
918912 engine : str = "newmm" ,
919913 keep_whitespace : bool = True ,
920914 join_broken_num : bool = True ,
@@ -937,10 +931,8 @@ def __init__(
937931 self .__engine = engine
938932 if self .__engine not in ["newmm" , "mm" , "longest" , "deepcut" ]:
939933 raise NotImplementedError (
940- """
941- The Tokenizer class is not support %s for custom tokenizer
942- """
943- % self .__engine
934+ "The Tokenizer class does not support "
935+ f"{ self .__engine } for custom tokenizer."
944936 )
945937 self .__keep_whitespace = keep_whitespace
946938 self .__join_broken_num = join_broken_num
0 commit comments