Skip to content

Commit 028f201

Browse files
committed
fix process strings
1 parent 9fe79bd commit 028f201

File tree

1 file changed

+3
-17
lines changed

1 file changed

+3
-17
lines changed

text_preprocessing/preprocessor.py

Lines changed: 3 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -279,23 +279,9 @@ def process_texts(
279279

280280
def process_string(self, text: str, keep_all: bool = True) -> Tokens:
281281
"""Take a string and return a list of preprocessed tokens"""
282-
with mp.Pool(1) as pool:
283-
for tokens in pool.apply(
284-
process_batch_texts,
285-
(
286-
self.text_fetcher_args,
287-
[text],
288-
self.language_model,
289-
self.normalize_options,
290-
self.do_nlp,
291-
keep_all,
292-
self.ngram_config,
293-
self.post_func,
294-
),
295-
):
296-
output_tokens = Tokens(tokens, keep_all=keep_all)
297-
break
298-
return output_tokens
282+
progress_info = {"count": 0, "doc_count": 0, "progress": False, "progress_prefix": ""}
283+
result = self.__process_batch([text], keep_all, progress_info)
284+
return next(result)
299285

300286

301287
class TextFetcher:

0 commit comments

Comments
 (0)