v0.5.1, add support for reading xlsx files… (#180)

* chore: update version to 0.5.1, add support for reading xlsx files, and include Motorola datasets

* strip \r from outputs; add wmt24pp-rm dataset

- Strip \r from parser outputs (read_plain, read_tsv) and echo command. Closes #173
- Add ZurichNLP/wmt24pp-rm: 6 Romansh varieties (deu-roh) test sets. Closes #178

* add --strict flag for langpair ordering in mtdata list. Closes #157

* add scb-mt-en-th-2020 (12 subsets) and TALPCo (8 lang pairs). Closes #151. Closes #152

- VistecAI scb-mt-en-th-2020: 12 CSV subsets with ~1M eng-tha sentence pairs
- TALPCo: jpn paired with eng, kor, mya, zsm, ind, jav, tha, vie
- Fix parser: TSV with 2 files now zips (pairs) instead of flattening
- Fix parser: single-col TSV yields scalar string for proper pairing

* fix CSV parser and SMOL HF datasets

- Add read_csv() using Python csv module for proper quoted-field handling. Fixes VistecAI scb-mt column misalignment
- Support data_files in HF loader for datasets with unregistered configs (SMOL smolsent/smoldoc en_lij)
- Restore Google-smol_sent and Google-smol_doc entries for eng-lij_Latn

Author: thammegowda

mtdata/__init__.py

@@ -4,7 +4,7 @@
 # Created: 4/4/20
 
 
-__version__ = '0.5.0'
+__version__ = '0.5.1'
 __description__ = 'mtdata is a tool to download datasets for machine translation'
 __author__ = 'Thamme Gowda'
 

mtdata/cache.py

@@ -45,7 +45,7 @@ def get_entry(self, entry: Entry, fix_missing=True) -> Union[Path, List[Path]]:
             else:
                 assert isinstance(entry.url, str)
                 local = self.get_local_path(entry.url, filename=entry.filename, fix_missing=fix_missing, entry=entry)
-                if isinstance(local, Path) and (zipfile.is_zipfile(local) or tarfile.is_tarfile(local)):
+                if isinstance(local, Path) and entry.is_archive and (zipfile.is_zipfile(local) or tarfile.is_tarfile(local)):
                     # look inside the archives and get the desired files
                     local = self.get_local_in_paths(path=local, entry=entry)
             return local
@@ -174,6 +174,10 @@ def get_hf_dataset(self, url: str, entry=None):
             streaming=False,
             trust_remote_code=False,
         )
+        data_files = entry.meta.get("data_files", None)
+        if data_files:
+            args['data_files'] = data_files
+            args.pop('name', None)  # data_files and name are mutually exclusive
         log.debug(f"Loading dataset {hf_id} with args: {args}")
         ds = load_dataset(hf_id, **args)
         if split is None and hasattr(ds, 'keys'):
@@ -261,7 +265,13 @@ def download(self, url: str, save_at: Path, timeout=(5, 10), entry=None):
             if valid_flag.exists() and save_at.exists():
                 return save_at
             log.debug(f"GET {url} → {save_at}")
-            resp = requests.get(url=url, allow_redirects=True, headers=headers, stream=True, timeout=timeout)
+            try:
+                resp = requests.get(url=url, allow_redirects=True, headers=headers, stream=True,
+                                    timeout=timeout)
+            except requests.exceptions.SSLError as e:
+                log.warning(f"SSL verification failed for {url}: {e}; retrying without verification")
+                resp = requests.get(url=url, allow_redirects=True, headers=headers, stream=True,
+                                    timeout=timeout, verify=False)
             assert resp.status_code == 200, resp.status_code
             buf_size = 2 ** 14
             tot_bytes = int(resp.headers.get('Content-Length', '0'))

mtdata/index/__init__.py

@@ -202,17 +202,21 @@ def is_compatible(lang1: Union[str, BCP47Tag], lang2: Union[str, BCP47Tag]):
     return lang1.is_compatible(lang2)
 
 
-def bitext_lang_match(pair1, pair2, fuzzy_match=False) -> bool:
-    x1, y1 = sorted(pair1)
-    x2, y2 = sorted(pair2)
+def bitext_lang_match(pair1, pair2, fuzzy_match=False, strict=False) -> bool:
+    if strict:
+        x1, y1 = pair1
+        x2, y2 = pair2
+    else:
+        x1, y1 = sorted(pair1)
+        x2, y2 = sorted(pair2)
     if fuzzy_match:
         return is_compatible(x1, x2) and is_compatible(y1, y2)
     else:
         return x1 == x2 and y1 == y2
 
 
 def get_entries(langs=None, names=None, not_names=None, fuzzy_match=False, 
-                groups=None, not_groups=None) -> List[Entry]:
+                groups=None, not_groups=None, strict=False) -> List[Entry]:
     """
     :param langs: language pairs  to select eg ('en', 'de')
     :param names:  names to select
@@ -236,7 +240,7 @@ def get_entries(langs=None, names=None, not_names=None, fuzzy_match=False,
     if langs:
         if len(langs) == 2:
             select = [e for e in select if 2 == len(e.did.langs)\
-                and bitext_lang_match(langs, e.did.langs, fuzzy_match=fuzzy_match)]
+                and bitext_lang_match(langs, e.did.langs, fuzzy_match=fuzzy_match, strict=strict)]
         else: # monolingual
             assert len(langs) == 1
             lang: BCP47Tag = langs[0]

mtdata/index/huggingface.py

@@ -98,6 +98,9 @@ def _load_hf_dataset(index, data):
             else:
                 meta = dict(orig_id=hf_id, config=config_name, split=split, fields=fields)
 
+            if 'data_files' in config:
+                meta['data_files'] = config['data_files']
+
             data_id = DatasetId(group=group, name=name, version='1', langs=langs)
             if data_id in index:
                 log.warning(f"Duplicate dataset id: {data_id}")

mtdata/index/other.py

@@ -164,3 +164,60 @@ def load_all(index: Index):
             ("2", "*/mono-bho-corpus/monolingual-v0.2.bho")]:
         index += Entry(did=DatasetId(group='BHLTR', name=f'mono', version=version, langs=('bho',)),
                     url=url, filename=filename, ext='zip', in_ext='txt', in_paths=[f1], cite=cite)
+
+    ### Motorola Endangered Indigenous Languages ###
+    _moto_base = "https://wpa_supplier.motorola.com/"
+    _moto_datasets = [
+        # (filename, src_lang, tgt_lang, version-date)
+        ("MotorolaMobility_All-en_US-lld_IT-2024-06-04.xlsx", "eng", "lld", "20240604"),
+        ("MotorolaMobility_All-en_US-chr-2022-03-08.xlsx", "eng", "chr", "20220308"),
+        ("MotorolaMobility_All-en_US-xnr_IN-2023-02-10.xlsx", "eng", "xnr", "20230210"),
+        ("MotorolaMobility_All-en_US-mi_NZ-2023-08-08.xlsx", "eng", "mri", "20230808"),
+        ("MotorolaMobility-en_US-yrl-2022-12-08.xlsx", "eng", "yrl", "20221208"),
+        ("MotorolaMobility-en_US-kgp-2022-12-08.xlsx", "eng", "kgp", "20221208"),
+        ("MotorolaMobility_All-pt_BR-yrl_BR-2022-07-20.xlsx", "por", "yrl", "20220720"),
+        ("MotorolaMobility_All-pt_BR-kgp_BR-2022-07-20.xlsx", "por", "kgp", "20220720"),
+    ]
+    for fname, src, tgt, version in _moto_datasets:
+        url = _moto_base + fname
+        index += Entry(did=DatasetId(group='Motorola', name='lang_revitalization', version=version,
+                                     langs=(src, tgt)),
+                       url=url, filename=fname, ext='xlsx', in_ext='xlsx')
+
+    ### scb-mt-en-th-2020: Thai-English parallel corpus ###
+    _scb_url = "https://github.com/vistec-AI/dataset-releases/releases/download/scb-mt-en-th-2020_v1.0/scb-mt-en-th-2020.zip"
+    _scb_filename = "scb-mt-en-th-2020.zip"
+    _scb_cite = ("lowphansirikul2021scb",)
+    _scb_subsets = [
+        "task_master_1", "generated_reviews_translator", "nus_sms",
+        "msr_paraphrase", "mozilla_common_voice", "generated_reviews_crowd",
+        "generated_reviews_yn", "assorted_government", "thai_websites",
+        "paracrawl", "wikipedia", "apdf",
+    ]
+    for subset in _scb_subsets:
+        in_path = f"scb-mt-en-th-2020/{subset}.csv"
+        index += Entry(did=DatasetId(group='VistecAI', name=f'scb_mt_{subset}', version='1',
+                                     langs=('eng', 'tha')),
+                       url=_scb_url, filename=_scb_filename, ext='zip', in_ext='csvwithheader',
+                       in_paths=[in_path], cite=_scb_cite)
+
+    ### TALPCo: TUFS Asian Language Parallel Corpus ###
+    _talpco_base = "https://raw.githubusercontent.com/matbahasa/TALPCo/master"
+    _talpco_cite = ("nomoto2018talpco",)
+    _talpco_langs = [
+        ("eng", "eng"),
+        ("kor", "kor"),
+        ("mya", "myn"),  # Burmese; file uses myn
+        ("zsm", "zsm"),  # Malay
+        ("ind", "ind"),  # Indonesian
+        ("jav", "jav"),  # Javanese
+        ("tha", "tha"),
+        ("vie", "vie"),
+    ]
+    for iso, file_code in _talpco_langs:
+        src_url = f"{_talpco_base}/jpn/data_jpn.txt"
+        tgt_url = f"{_talpco_base}/{file_code}/data_{file_code}.txt"
+        index += Entry(did=DatasetId(group='TALPCo', name='talpco', version='1',
+                                     langs=('jpn', iso)),
+                       url=[src_url, tgt_url], ext='txt', in_ext='tsv', cols=(1,),
+                       cite=_talpco_cite)

mtdata/main.py

@@ -20,9 +20,9 @@
 DEF_N_JOBS = 1
 
 
-def list_data(langs, names, not_names=None, full=False, groups=None, not_groups=None, id_only=False):
+def list_data(langs, names, not_names=None, full=False, groups=None, not_groups=None, id_only=False, strict=False):
     from mtdata.index import get_entries
-    entries = get_entries(langs, names, not_names, groups=groups, not_groups=not_groups, fuzzy_match=True)
+    entries = get_entries(langs, names, not_names, groups=groups, not_groups=not_groups, fuzzy_match=True, strict=strict)
     for i, ent in enumerate(entries):
         if id_only:
             print(ent.did)
@@ -82,7 +82,7 @@ def echo_data(did:DatasetId, delim='\t'):
             for col in row:
                 if isinstance(col, Mapping):
                     col = json.dumps(col, indent=None, ensure_ascii=False)
-                col = col.replace(delim, ' ').replace('\n', ' ')
+                col = col.replace(delim, ' ').replace('\n', ' ').replace('\r', ' ')
                 rec.append(col)
             rec = delim.join(rec)
         else:
@@ -281,6 +281,8 @@ def parse_args():
     list_p.add_argument('-nn', '--not-names', metavar='NAME', nargs='*', help='Exclude these names')
     list_p.add_argument('-g', '--groups', metavar='GROUP', nargs='*', help='Only select datasets from these groups')
     list_p.add_argument('-ng', '--not-groups', metavar='GROUP', nargs='*', help='Exclude these groups')
+    list_p.add_argument('-s', '--strict', action='store_true', default=False,
+                        help='Strict langpair ordering: eng-deu and deu-eng are treated differently')
     list_p.add_argument('-f', '--full', action='store_true', help='Show Full Citation')
     list_p.add_argument('-o', '--out', type=Path, help='This arg is ignored. Only used in "get" subcommand,'
                                                        ' but added here for convenience of switching b/w get and list')
@@ -386,7 +388,8 @@ def main():
             index_datasets()
         elif args.task == 'list':
             list_data(args.langs, args.names, not_names=args.not_names, full=args.full,
-                    groups=args.groups, not_groups=args.not_groups, id_only=args.id)
+                    groups=args.groups, not_groups=args.not_groups, id_only=args.id,
+                    strict=args.strict)
         elif args.task == 'get':
             get_data(**vars(args))
         elif args.task == 'echo':

mtdata/parser.py

@@ -3,6 +3,7 @@
 # Author: Thamme Gowda [tg (at) isi (dot) edu] 
 # Created: 4/4/20
 
+import csv
 from typing import Optional, Union, Tuple, List
 from dataclasses import dataclass
 from pathlib import Path
@@ -77,7 +78,7 @@ def read_segs(self, show_pbar=True):
                         cols = self.ent.cols
                     readers.append(self.read_tsv(p, cols=cols, meta_fields=meta_fields))
                 elif 'csvwithheader' in self.ext:
-                    readers.append(self.read_tsv(p, delim=',', skipheader=True, meta_fields=meta_fields))
+                    readers.append(self.read_csv(p, meta_fields=meta_fields))
                 elif 'raw' in self.ext or 'txt' in self.ext:
                     readers.append(self.read_plain(p))
                 elif 'tmx' in self.ext:
@@ -91,13 +92,15 @@ def read_segs(self, show_pbar=True):
                     readers.append(read_wmt21_xml(p))
                 elif HF_EXT in self.ext:
                     readers.append(self.read_hfds(p))
+                elif 'xlsx' in self.ext:
+                    readers.append(self.read_xlsx(p))
                 else:
                     raise Exception(f'Not supported {self.ext} : {p}')
 
         if len(readers) == 1:
             data = readers[0]
-        elif self.ext == 'tmx' or self.ext == 'tsv':
-            data = (rec for reader in readers for rec in reader)  # flatten all readers
+        elif self.ext == 'tmx' or (self.ext == 'tsv' and len(self.paths) == 1):
+            data = (rec for reader in readers for rec in reader)  # flatten readers from single source
         elif len(readers) == 2:
             def _zip_n_check():
                 for row in zip_longest(*readers):
@@ -120,7 +123,7 @@ def read_plain(self, path):
         try:
             with IO.reader(path) as stream:
                 for line in stream:
-                    yield line.strip()
+                    yield line.rstrip('\r\n')
         except:
             log.warning(f'Error reading file {path}')
             raise
@@ -138,10 +141,12 @@ def read_tsv(self, path, delim='\t', cols=None, skipheader=False, meta_fields=No
             if skipheader:
                 line = stream.readline()
             for line in stream:
-                row = [x.strip() for x in line.rstrip('\n').split(delim)]
+                row = [x.strip() for x in line.rstrip('\r\n').split(delim)]
                 out_row = row
                 if cols:
                     out_row = [row[idx] for idx in cols]
+                    if len(cols) == 1:
+                        out_row = out_row[0]  # unwrap single-column to scalar
                 if meta_fields:
                     metadata = {}
                     for key, idx in meta_fields.items():
@@ -152,6 +157,51 @@ def read_tsv(self, path, delim='\t', cols=None, skipheader=False, meta_fields=No
                         out_row.append(metadata)
                 yield out_row
 
+    def read_csv(self, path, cols=None, meta_fields=None):
+        """Read data from a CSV file with header using Python's csv module.
+        Handles quoted fields with embedded commas/newlines correctly.
+        :param path: path to CSV file
+        :param cols: column indices to extract; default is (0, 1)
+        """
+        if cols is None:
+            cols = self.ent.cols if (self.ent and self.ent.cols) else (0, 1)
+        with IO.reader(path) as stream:
+            reader = csv.reader(stream)
+            header = next(reader, None)  # skip header
+            for row in reader:
+                if not row or len(row) <= max(cols):
+                    continue
+                out_row = [row[c].strip() for c in cols]
+                if meta_fields:
+                    metadata = {}
+                    for key, idx in meta_fields.items():
+                        if key in ("source", "target") or idx >= len(row) or row[idx] in ("", None):
+                            continue
+                        metadata[key] = row[idx]
+                    if metadata:
+                        out_row.append(metadata)
+                yield out_row
+
+    def read_xlsx(self, path, cols=None):
+        """Read data from an Excel .xlsx file.
+        :param path: path to .xlsx file
+        :param cols: column indices to extract; default uses ent.cols or (0, 1)
+        """
+        try:
+            from openpyxl import load_workbook
+        except ImportError as e:
+            raise ImportError("openpyxl is required to read .xlsx files. Run: pip install openpyxl") from e
+        if cols is None:
+            cols = self.ent.cols if (self.ent and self.ent.cols) else (0, 1)
+        wb = load_workbook(path, read_only=True, data_only=True)
+        ws = wb.active
+        for row in ws.iter_rows(min_row=2, values_only=True):  # skip header
+            out = [str(row[c]).strip() if row[c] is not None else '' for c in cols]
+            if all(v == '' for v in out):
+                continue
+            yield out
+        wb.close()
+
     @staticmethod
     def _nested_get(row, field):
         """Get a value from a dict using dot-separated path for nested access.
@@ -176,11 +226,24 @@ def read_hfds(self, ds):
         # in the current version, I am going to retain all fields to see what all fields exist,
         # and map the subset of fields as per the dict; so, created rev_map.get(orig,orig)
         for row in ds:
-            out_row = [self._nested_get(row, src_field)]
-            if tgt_field is not None:
-                out_row.append(self._nested_get(row, tgt_field))
-            # remap meta fields if necessary
+            src_val = self._nested_get(row, src_field)
+            tgt_val = self._nested_get(row, tgt_field) if tgt_field else None
             top_keys = {f.split('.')[0] for f in [src_field] + ([tgt_field] if tgt_field else [])}
             metadata = {rev_map.get(k, k): v for k, v in row.items() if k not in top_keys}
-            out_row.append(metadata)
-            yield out_row
+
+            src_is_list = isinstance(src_val, list)
+            tgt_is_list = isinstance(tgt_val, list)
+            if src_is_list and tgt_is_list:
+                # Both lists (e.g. SmolDoc srcs/trgs): zip and yield each pair
+                for s, t in zip(src_val, tgt_val):
+                    yield [s, t, metadata]
+            elif not src_is_list and tgt_is_list:
+                # Source is scalar, target is list (e.g. GATITOS src/trgs): expand
+                for t in tgt_val:
+                    yield [src_val, t, metadata]
+            else:
+                out_row = [src_val]
+                if tgt_val is not None:
+                    out_row.append(tgt_val)
+                out_row.append(metadata)
+                yield out_row

mtdata/resource/huggingface-datasets.jsonl

@@ -12,3 +12,7 @@
 {"id": "sfrontull/autonomia-lld_valbadia-ita", "configs": [{"name": null, "langs": ["ita", "lld"], "fields": {"source": "translation.ita", "target": "translation.lld_valbadia"}}]}
 {"id": "sfrontull/stiftungsparkasse-lld_valbadia-ita", "configs": [{"name": null, "langs": ["ita", "lld"], "fields": {"source": "translation.ita", "target": "translation.lld_valbadia"}}]}
 {"id": "sfrontull/pinocchio-lld_valbadia-ita", "configs": [{"name": null, "langs": ["ita", "lld"], "fields": {"source": "translation.ita", "target": "translation.lld_valbadia"}}]}
+# === SMOL: Professional translations for low-resource languages (google/smol) ===
+{"id": "google/smol", "configs": [{"name": "smolsent__en_lij", "ds_name": "smol_sent", "langs": ["eng", "lij_Latn"], "fields": {"source": "src", "target": "trg"}, "data_files": "smolsent/en_lij.jsonl"}, {"name": "smoldoc__en_lij", "ds_name": "smol_doc", "langs": ["eng", "lij_Latn"], "fields": {"source": "srcs", "target": "trgs"}, "data_files": "smoldoc/en_lij.jsonl"}, {"name": "gatitos__en_lij", "ds_name": "smol_gatitos", "langs": ["eng", "lij_Latn"], "fields": {"source": "src", "target": "trgs"}}]}
+# === WMT24++ Romansh (ZurichNLP/wmt24pp-rm) ===
+{"id": "ZurichNLP/wmt24pp-rm", "fields": {"source": "source", "target": "target", "doc_id": "document_id", "domain": "domain", "seg_id": "segment_id"}, "configs": [{"name": "de_DE-rm-rumgr", "split": "test", "langs": ["deu", "roh"], "ds_name": "wmt24pp_rm_rumgr"}, {"name": "de_DE-rm-sursilv", "split": "test", "langs": ["deu", "roh"], "ds_name": "wmt24pp_rm_sursilv"}, {"name": "de_DE-rm-sutsilv", "split": "test", "langs": ["deu", "roh"], "ds_name": "wmt24pp_rm_sutsilv"}, {"name": "de_DE-rm-surmiran", "split": "test", "langs": ["deu", "roh"], "ds_name": "wmt24pp_rm_surmiran"}, {"name": "de_DE-rm-puter", "split": "test", "langs": ["deu", "roh"], "ds_name": "wmt24pp_rm_puter"}, {"name": "de_DE-rm-vallader", "split": "test", "langs": ["deu", "roh"], "ds_name": "wmt24pp_rm_vallader"}]}

mtdata/resource/refs.bib

@@ -726,4 +726,27 @@ @article{ojha2019english
   author={Ojha, Atul Kr},
   journal={arXiv preprint arXiv:1905.02239},
   year={2019}
+}
+
+@inproceedings{vamvas2025expanding,
+  title={Expanding the {WMT}24++ Benchmark with {R}umantsch {G}rischun, {S}ursilvan, {S}utsilvan, {S}urmiran, {P}uter, and {V}allader},
+  author={Vamvas, Jannis and P{\'e}rez Prat, Ignacio and Soliva, Not and Baltermia-Guetg, Sandra and Beeli, Andrina and Beeli, Simona and Capeder, Madlaina and Decurtins, Laura and Gregori, Gian Peder and Hobi, Flavia and Holderegger, Gabriela and Lazzarini, Arina and Lazzarini, Viviana and Rosselli, Walter and Vital, Bettina and Rutkiewicz, Anna and Sennrich, Rico},
+  booktitle={Proceedings of the Tenth Conference on Machine Translation},
+  year={2025},
+  url={https://aclanthology.org/2025.wmt-1.79/}
+}
+
+@article{lowphansirikul2021scb,
+  title={scb-mt-en-th-2020: A Large English-Thai Parallel Corpus},
+  author={Lowphansirikul, Lalita and Polpanumas, Charin and Jantrakulchai, Nawat and Nutanong, Sarana},
+  journal={arXiv preprint arXiv:2007.03541},
+  year={2020}
+}
+
+@inproceedings{nomoto2018talpco,
+  title={{TUFS} {A}sian {L}anguage {P}arallel {C}orpus ({TALPCo})},
+  author={Nomoto, Hiroki and Okano, Kenji and Moeljadi, David and Sawada, Hideo},
+  booktitle={Proceedings of the Twenty-Fourth Annual Meeting of the Association for Natural Language Processing},
+  pages={436--439},
+  year={2018}
 }