fix: truncation=True is needed

meilame-tayebjee · meilame-tayebjee · commit 7fdb4e3a4d7c · 2025-11-12T12:56:10.000Z
diff --git a/torchTextClassifiers/tokenizers/base.py b/torchTextClassifiers/tokenizers/base.py
@@ -129,6 +129,7 @@ def tokenize(
             text,
             padding=padding,
             return_tensors="pt",
+            truncation=True,
             max_length=self.output_dim,
             return_offsets_mapping=return_offsets_mapping,
         )  # method from PreTrainedTokenizerFast
