fix: update vocab_size after training

Author: meilame-tayebjee

torchTextClassifiers/tokenizers/base.py

@@ -113,7 +113,7 @@ def __init__(
         trained: bool = False,
     ):
         super().__init__(
-            vocab_size, output_vectorized=False, output_dim=output_dim
+            vocab_size, output_vectorized=False, output_dim=output_dim, padding_idx=padding_idx
         )  # it outputs token ids and not vectors
 
         self.trained = trained

torchTextClassifiers/tokenizers/ngram.py

@@ -282,6 +282,7 @@ def __init__(
             self.subword_cache = None
 
         self.vocab_size = 3 + self.nwords + self.num_tokens
+        print("brrrrr ", self.vocab_size)
         super().__init__(
             vocab_size=self.vocab_size, padding_idx=self.pad_token_id, output_dim=output_dim
         )
@@ -301,6 +302,7 @@ def train(self, training_text: List[str]):
                 idx += 1
 
         self.nwords = len(self.word_to_id)
+        self.vocab_size = 3 + self.nwords + self.num_tokens
 
         # Create reverse mapping
         self.id_to_word = {v: k for k, v in self.word_to_id.items()}