fix typo

clovis · clovis · commit fc91d7a16f59 · 2023-08-29T14:00:54.000-05:00
diff --git a/text_preprocessing/spacy_helpers.py b/text_preprocessing/spacy_helpers.py
@@ -457,7 +457,7 @@ def normalize_from_tokens(self, tokens: Tokens) -> Tokens:
             if not normalized_text:
                 normalized_text = "#DEL#"
             token.text = normalized_text
-            new_tokens.append(tokens)
+            new_tokens.append(token)
         return Tokens(new_tokens, tokens.metadata)
 
     def __filter_token(self, token: Token | PreprocessorToken) -> bool:
