We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent 2c741aa commit 3be22a9Copy full SHA for 3be22a9
text_preprocessing/preprocessor.py
@@ -68,7 +68,7 @@ def process_batch_texts(
68
text_fetcher = TextFetcher(nlp, **text_fetcher_args) # Initialize text_fetcher with required params
69
previous_philo_id = None
70
for tokens, _ in text_fetcher(batch_texts, do_nlp=do_nlp, keep_all=keep_all, progress=False):
71
- if isinstance(tokens, PreparedDoc) and using_gpu is True:
+ if isinstance(tokens, PreparedDoc):
72
spacy_doc = make_spacy_doc(nlp, tokens)
73
if spacy_doc._.char_num > 10000 and using_gpu is True:
74
split_doc = split_spacy_docs(nlp, spacy_doc)
0 commit comments