chore: force all tokenizers to have a load_from_s3 method

Author: meilame-tayebjee

torchTextClassifiers/tokenizers/base.py

@@ -103,6 +103,11 @@ def __repr__(self):
     def __call__(self, text: Union[str, List[str]], **kwargs) -> list:
         return self.tokenize(text, **kwargs)
 
+    @classmethod
+    @abstractmethod
+    def load_from_s3(cls, s3_path: str, filesystem):
+        pass
+
 
 class HuggingFaceTokenizer(BaseTokenizer):
     def __init__(

torchTextClassifiers/tokenizers/ngram.py

@@ -432,11 +432,24 @@ def save_pretrained(self, save_directory: str):
         print(f"✓ Tokenizer saved to {save_directory}")
 
     @classmethod
-    def from_pretrained(cls, directory: str):
+    def load_from_s3(cls, s3_path: str, filesystem):
         """Load tokenizer from saved configuration."""
-        with open(f"{directory}/tokenizer.json", "r") as f:
+
+        config = json.load(filesystem.open(s3_path, "r"))
+        tokenizer = cls.build_from_config(config)
+        return tokenizer
+
+    @classmethod
+    def load(cls, path: str):
+        """Load tokenizer from saved configuration."""
+
+        with open(path, "r") as f:
             config = json.load(f)
+        tokenizer = cls.build_from_config(config)
+        return tokenizer
 
+    @classmethod
+    def build_from_config(cls, config):
         tokenizer = cls(
             min_count=config["min_count"],
             min_n=config["min_n"],
@@ -468,5 +481,4 @@ def from_pretrained(cls, directory: str):
         )
         print("✓ Subword cache built")
 
-        print(f"✓ Tokenizer loaded from {directory}")
         return tokenizer