Inject SST

Author: mariosasko

docs/source/advanced.rst

@@ -417,8 +417,8 @@ For this reason, usage of :class:`podium.datasets.BucketIterator` is recommended
   >>> label = LabelField(name='label')
   >>> fields = {'text': text, 'label': label}
   >>>
-  >>> sst_train, sst_valid, sst_test = SST.get_dataset_splits(fields=fields)
-  >>> sst_train.finalize_fields()
+  >>> train, valid, test = SST.get_dataset_splits(fields=fields)
+  >>> train.finalize_fields()
   >>>
   >>> # Define the iterators and our sort key
   >>> from podium import Iterator, BucketIterator

docs/source/notebooks/advanced.ipynb

@@ -901,8 +901,8 @@
     "label = LabelField(name='label')\n",
     "fields = {'text': text, 'label': label}\n",
     "\n",
-    "sst_train, sst_valid, sst_test = SST.get_dataset_splits(fields=fields)\n",
-    "sst_train.finalize_fields()\n",
+    "train, valid, test = SST.get_dataset_splits(fields=fields)\n",
+    "train.finalize_fields()\n",
     "\n",
     "# Define the iterators and our sort key\n",
     "from podium import Iterator, BucketIterator\n",

docs/source/scripts/check_notebooks.py

@@ -1,7 +1,10 @@
 import argparse
 import copy
 import multiprocess
+import os
+import shutil
 import string
+import subprocess
 import textwrap
 from functools import partial
 from pathlib import Path
@@ -13,9 +16,30 @@
 NOTEBOOKS_PATH = "../notebooks"
 INSTALL_RELEASE_VERSION_COMMAND = "! pip install podium-nlp"
 INSTALL_SOURCE_VERSION_COMMAND = "# ! pip install git+https://github.com/TakeLab/podium.git"
+INSTALL_SST_COMMAND = "python -c \"from podium.datasets import SST; SST.get_dataset_splits()\""
 TRANS_TABLE = str.maketrans(dict.fromkeys(string.whitespace))
 
 
+def inject_sst():
+    delim = "&" if os.name == "nt" else ";"
+    subprocess.call(
+        delim.join([INSTALL_SOURCE_VERSION_COMMAND[4:], INSTALL_SST_COMMAND]),
+        shell=True,
+        cwd=Path(NOTEBOOKS_PATH).absolute(),
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.STDOUT,
+    )
+
+
+def cleanup(snap_before_exec, snap_after_exec):
+    created_paths = set(snap_after_exec) - set(snap_before_exec)
+    for path in created_paths:
+        if path.is_dir():
+            shutil.rmtree(path)
+        else:
+            path.unlink()
+
+
 def print_notebook_name_with_error(func):
     def wrapper(*args, **kwargs):
         if args:
@@ -101,6 +125,7 @@ def check_notebook_output(notebook_path, env="python3", ignore_whitespace=False)
     parser.add_argument("--env", default="python3", help="kernel that executes the notebook")
     parser.add_argument("--num_proc", help="number of processes for parallel execution")
     parser.add_argument("--ignore_whitespace", action="store_true", help="ignore whitespace when comparing cell outputs")
+    parser.add_argument("--keep_artifacts", action="store_true", help="save files/directories created during execution")
     args = parser.parse_args()
 
     if args.num_proc is None:
@@ -116,16 +141,24 @@ def check_notebook_output(notebook_path, env="python3", ignore_whitespace=False)
         if not notebook_path.name.endswith("-checkpoint.ipynb")
     ]
 
+    snap_before_exec = list(Path(NOTEBOOKS_PATH).iterdir())
+
     num_proc = min(min(num_proc, multiprocess.cpu_count()), len(notebook_paths))
     if num_proc == 1:
         reports = []
         for notebook_path in notebook_paths:
             report = check_notebook_output(notebook_path, env=args.env, ignore_whitespace=args.ignore_whitespace)
             reports.append(report)
     else:
+        # inject the SST dataset to prevent parallel download
+        inject_sst()
         with multiprocess.Pool(num_proc) as pool:
             reports = pool.map(partial(check_notebook_output, env=args.env, ignore_whitespace=args.ignore_whitespace), notebook_paths)
 
+    if args.keep_artefacts is False:
+        snap_after_exec = list(Path(NOTEBOOKS_PATH).iterdir())
+        cleanup(snap_before_exec, snap_after_exec)
+
     if any(report for _, report in reports):
         reports_str = "\n\n".join([
             f"In notebook {notebook}:\n" + textwrap.indent(

docs/source/scripts/requirements.txt

@@ -1,4 +1,5 @@
 ipykernel
+ipywidgets
 nbformat
 nbconvert
 multiprocess

podium/datasets/arrow.py

@@ -299,7 +299,7 @@ def from_tabular_file(
         format = format.lower()
         csv_reader_params = {} if csv_reader_params is None else csv_reader_params
 
-        with open(os.path.expanduser(path), encoding="utf8") as f:
+        with open(os.path.expanduser(path), encoding="utf-8") as f:
             if format in {"csv", "tsv"}:
                 delimiter = "," if format == "csv" else "\t"
                 reader = csv.reader(f, delimiter=delimiter, **csv_reader_params)
@@ -542,7 +542,7 @@ def load_cache(cache_path) -> "DiskBackedDataset":
         """
         # load fields
         fields_file_path = os.path.join(cache_path, CACHE_FIELDS_FILENAME)
-        with open(fields_file_path, "rb") as fields_cache_file:
+        with open(os.path.expanduser(fields_file_path), "rb") as fields_cache_file:
             fields = pickle.load(fields_cache_file)
 
         # load dataset as memory mapped arrow table
@@ -587,7 +587,7 @@ def dump_cache(self, cache_path: Optional[str] = None) -> str:
 
         # pickle fields
         cache_fields_path = os.path.join(cache_path, CACHE_FIELDS_FILENAME)
-        with open(cache_fields_path, "wb") as fields_cache_file:
+        with open(os.path.expanduser(cache_fields_path), "wb") as fields_cache_file:
             pickle.dump(self.fields, fields_cache_file)
 
         # dump table

podium/datasets/impl/conllu_dataset.py

@@ -2,6 +2,7 @@
 Module contains the CoNLL-U dataset.
 """
 import collections
+import os
 
 from podium.datasets import Dataset
 from podium.datasets.example_factory import ExampleFactory
@@ -87,7 +88,7 @@ def safe_conllu_parse(in_file):
         example_factory = ExampleFactory(fields)
 
         examples = []
-        with open(file_path, encoding="utf-8") as in_file:
+        with open(os.path.expanduser(file_path), encoding="utf-8") as in_file:
             for tokenlist in safe_conllu_parse(in_file):
                 example_dict = collections.defaultdict(lambda: [])
                 for token in tokenlist:

podium/datasets/impl/imdb.py

@@ -153,7 +153,9 @@ def _create_labeled_examples(dir_path, label, fields):
         ]
         examples = []
         for file_path in files_list:
-            with open(file=os.path.join(dir_path, file_path), encoding="utf8") as fpr:
+            with open(
+                os.path.expanduser(os.path.join(dir_path, file_path)), encoding="utf-8"
+            ) as fpr:
                 data = {IMDB.TEXT_FIELD_NAME: fpr.read(), IMDB.LABEL_FIELD_NAME: label}
                 examples.append(example_factory.from_dict(data))
         return examples

podium/datasets/impl/snli.py

@@ -99,7 +99,7 @@ def _create_examples(file_path, fields):
         example_factory = ExampleFactory(fields)
         examples = []
 
-        with open(file=file_path, encoding="utf8") as in_file:
+        with open(os.path.expanduser(file_path), encoding="utf-8") as in_file:
             for line in in_file:
                 examples.append(example_factory.from_json(line))
         return examples

podium/datasets/impl/sst.py

@@ -123,7 +123,7 @@ def label_trf(label):
             return label_to_string_map[label]
 
         examples = []
-        with open(file=file_path, encoding="utf8") as fpr:
+        with open(os.path.expanduser(file_path), encoding="utf-8") as fpr:
             for line in fpr:
 
                 example = example_factory.from_fields_tree(

podium/vectorizers/vectorizer.py

@@ -298,7 +298,7 @@ def _cache_vectors(self):
         """
         Method for caching loaded vectors to cache_dir.
         """
-        with open(self._cache_path, "wb") as cache_file:
+        with open(os.path.expanduser(self._cache_path), "wb") as cache_file:
             for word in self._vectors:
                 vector_values_string = " ".join(map(str, self._vectors[word]))
                 cache_file.write(f"{word} {vector_values_string}\n".encode("utf-8"))
@@ -362,7 +362,9 @@ def _load_vectors(self, vocab=None):
             vocab = set(vocab)
 
         open_mode, split_delimiter = ("rb", b" ") if self._binary else ("r", " ")
-        with open(curr_path, open_mode, encoding=self._encoding) as vector_file:
+        with open(
+            os.path.expanduser(curr_path), open_mode, encoding=self._encoding
+        ) as vector_file:
 
             vectors_loaded = 0
             header_lines = 0