bug fix

Author: clovis

text_preprocessing/preprocessor.py

@@ -143,7 +143,7 @@ def process_texts(
         fetched_texts = self.text_fetcher(
             texts, do_nlp=self.do_nlp, keep_all=keep_all, progress=progress, post_func=self.post_func
         )
-        if self.text_fetcher.text_object_type in ("para", "sent"):
+        if self.text_fetcher.text_object_type in ("para", "sent") and self.do_nlp is True:
             fetched_texts = self.nlp.pipe(
                 ((make_spacy_doc(self.nlp, tokens), c) for tokens, c in fetched_texts),
                 as_tuples=True,

text_preprocessing/spacy_helpers.py

@@ -1,7 +1,6 @@
 """Helper functions for Spacy"""
 
 import os
-import pickle
 import re
 import sys
 import unicodedata
@@ -10,14 +9,14 @@
 from typing import Any, Dict, Iterable, List, Optional, Union
 from xml.sax.saxutils import unescape as unescape_xml
 
+import pickle
 import spacy
 from spacy.language import Language
 from spacy.tokens import Doc, Token
 from Stemmer import Stemmer
 from thinc.api import prefer_gpu, set_gpu_allocator
 from unidecode import unidecode
 
-
 # Updated as of 8/23/2022
 SPACY_LANGUAGE_MODEL_MAP: Dict[str, List[str]] = {
     "catalan": ["ca_core_news_sm", "ca_core_news_md", "ca_core_news_lg", "ca_core_news_trf"],
@@ -71,8 +70,6 @@ class PreprocessorToken(str):
 
     """
 
-    ext: dict[str, Any]
-
     def __new__(cls, value, pos_="", ent_type_="", ext={}):
         return str.__new__(cls, value)
 
@@ -83,6 +80,7 @@ def __init__(
         ent_type_: str = "",
         ext: dict[str, Any] | None = None,
     ):
+        super().__init__()
         self.text = text or ""
         self.ext = ext or {}
         if self.ext is not None:
@@ -149,8 +147,8 @@ def __get_tokens(self, doc: Doc):
                 yield PreprocessorToken(token.text, token.pos_, token.ent_type_, token._.ext)
             elif self.keep_all is True:
                 yield PreprocessorToken("", token.pos_, token.ent_type_, token._.ext)
-                if token.whitespace_ and index < max_index:  # remove trailing whitespace
-                    yield PreprocessorToken(token.whitespace_, "", "", {**token._.ext, "token": token.whitespace_})
+            if token.whitespace_ and index < max_index:  # remove trailing whitespace
+                yield PreprocessorToken(token.whitespace_, "", "", {**token._.ext, "token": token.whitespace_})
 
     def __iter__(self) -> Iterable[PreprocessorToken]:
         for token in self.tokens: