fix: support-matrix on fasttext-predict + fun_langid SyntaxWarnings (#1)

Three bugs surfaced when smoke-testing the published 0.1.0.dev1 wheel:

1. `commonlid generate-support-matrix` skipped GlotLID, OpenLID-v2 and
   fasttext with `'_FastText' object has no attribute 'get_labels'`.
   Root cause: the runtime dep `fasttext-predict` is inference-only and
   exposes only `predict`/`multilinePredict`, but
   `_fasttext_base.discover_supported_languages` calls `get_labels()`.
   Add a binary parser for fasttext's `model.bin` dictionary block and
   fall back to it when `get_labels` is missing. Stash the model path on
   the instance at `load()` time.

2. `commonlid.vendor.fun_langid` raised two `SyntaxWarning: invalid
   escape sequence '\/'` on every CLI invocation under Python 3.12+.
   Make the offending strings raw — keeps the data identical to what
   pre-3.12 Python produced (literal backslash + slash).

3. The dev1 wheel itself shipped the *placeholder* `cld3.py`
   (`import cld3`) even though the source had been switched to the
   `gcld3`-based `cld3-py` bindings. The model fix already lived in the
   working tree; including this diff in the same PR ensures it lands
   in the next release.

Add unit tests for the new binary parser (round-trip + magic check) and
for the `fasttext-predict` (no-`get_labels`) discovery path.

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: malteos

src/commonlid/models/_fasttext_base.py

@@ -2,13 +2,59 @@
 
 from __future__ import annotations
 
+import struct
 from collections.abc import Sequence
+from pathlib import Path
 
 import fasttext
 from huggingface_hub import hf_hub_download
 
 from commonlid.core.lid_model import LIDModel
 
+_FASTTEXT_FILEFORMAT_MAGIC_INT32 = 793712314
+
+
+def _read_labels_from_bin(path: str | Path) -> list[str]:
+    """Parse the labels block out of a fasttext ``model.bin`` file.
+
+    The runtime dep ``fasttext-predict`` is inference-only and exposes only
+    ``predict`` / ``multilinePredict`` — no ``get_labels()``. Read the file
+    directly so :meth:`FastTextHubModel.discover_supported_languages` works
+    under either binding.
+
+    File layout (little-endian; mirrors FastText's ``saveModel`` +
+    ``Dictionary::save`` in the upstream C++ source):
+      - 8 bytes:  magic ``int32`` + version ``int32``
+      - 56 bytes: 12 ``int32`` + 1 ``double`` (Args block)
+      - 12 bytes: ``size_``, ``nwords_``, ``nlabels_`` (3 ``int32``)
+      - 16 bytes: ``ntokens_``, ``pruneidx_size_`` (2 ``int64``)
+      - then ``size_`` entries: NUL-terminated UTF-8 word, ``int64`` count,
+        ``int8`` entry type (``0`` = word, ``1`` = label).
+    """
+    with Path(path).open("rb") as f:
+        magic, _version = struct.unpack("<ii", f.read(8))
+        if magic != _FASTTEXT_FILEFORMAT_MAGIC_INT32:
+            msg = f"Not a fasttext model.bin (magic={magic:#x}): {path}"
+            raise ValueError(msg)
+        f.read(56)  # Args: 12 int32 + 1 double
+        size_, _nwords, nlabels_ = struct.unpack("<3i", f.read(12))
+        f.read(16)  # ntokens_ + pruneidx_size_
+        labels: list[str] = []
+        for _ in range(size_):
+            word_bytes = bytearray()
+            while True:
+                b = f.read(1)
+                if not b or b == b"\0":
+                    break
+                word_bytes.extend(b)
+            f.read(8)  # count int64
+            entry_type = f.read(1)
+            if entry_type == b"\x01":
+                labels.append(word_bytes.decode("utf-8"))
+                if len(labels) == nlabels_:
+                    break
+    return labels
+
 
 class FastTextHubModel(LIDModel):
     """Base class for HF-hosted fasttext LID models.
@@ -23,11 +69,13 @@ class FastTextHubModel(LIDModel):
     def __init__(self) -> None:
         super().__init__()
         self._ft: fasttext.FastText._FastText | None = None
+        self._model_path: str | None = None
 
     def load(self) -> None:
         if self._loaded:
             return
         path = hf_hub_download(repo_id=self.hf_repo_id, filename=self.hf_filename)
+        self._model_path = path
         self._ft = fasttext.load_model(path)
         super().load()
 
@@ -72,12 +120,24 @@ def _predict_labels(self, texts: list[str]) -> list[list[str]]:
         return list(predicted)
 
     def discover_supported_languages(self) -> frozenset[str]:
-        """Enumerate every ``__label__{code}`` exposed by the loaded fasttext model."""
+        """Enumerate every ``__label__{code}`` exposed by the loaded fasttext model.
+
+        ``fasttext-wheel`` exposes ``get_labels()`` on the loaded model;
+        ``fasttext-predict`` (the lighter inference-only fork we depend on at
+        runtime) does not. Fall back to parsing the dictionary block out of
+        ``model.bin`` directly when ``get_labels`` is missing.
+        """
         if self._ft is None:
             self.load()
         assert self._ft is not None
+        get_labels = getattr(self._ft, "get_labels", None)
+        if get_labels is not None:
+            raw_labels = list(get_labels())
+        else:
+            assert self._model_path is not None  # set by load()
+            raw_labels = _read_labels_from_bin(self._model_path)
         codes: set[str] = set()
-        for label in self._ft.get_labels():
+        for label in raw_labels:
             if not label.startswith("__label__"):
                 continue
             raw = label.split("__")[2].split("_")[0]

src/commonlid/vendor/fun_langid.py

@@ -841,7 +841,7 @@ def _init_lexicon(self):
             ),
             (
                 "zxx-x-arabocr",
-                " /j/^/i/jl/j j/./l/jj/^ ^/jjj/jjl/*/^ j/jli/jji/jl j/jlj/i j/j ^/ji j/ji/a/)/t/^jl/-/1/u/•/j^j/^j/^jj/j jj/4/i ^/l j/j i/jij/ijl/j 4/jl ^/o/l ^/: j/y/j jl/'/jj j/j^/4 j/j*/ijj/^1/j t/ji ^/aj/:/ij/l jj/>/i jj/c/^ji/jui/aji/l jl/1 ^/juj/ill/ajl/j ^j/ju/lj/1 j/v/f/uji/iji/^ 1/j l/i jl/j 1/jj^/lil/jju/jll/</jjjj/\/ujl/ajj/^ i/(/j^i/l^j/jaj/r/s/ja/ll j/^ij/,/t j/ljl/jjjl/li/ljj/jp/u j/jlp/* j/jl^/* ^/il j/lj j/j^l/jj ^/lji/lij/t ^/jt/l ^j/jls/■/uj/j j^/ii/0/jii/m/a j/^^/»/j u/i i/^ l/p/j c/j ij/ij j/3/j a/4 ^/^j j/jil/jj-/^ u/^lj/jla ".strip().split(
+                r" /j/^/i/jl/j j/./l/jj/^ ^/jjj/jjl/*/^ j/jli/jji/jl j/jlj/i j/j ^/ji j/ji/a/)/t/^jl/-/1/u/•/j^j/^j/^jj/j jj/4/i ^/l j/j i/jij/ijl/j 4/jl ^/o/l ^/: j/y/j jl/'/jj j/j^/4 j/j*/ijj/^1/j t/ji ^/aj/:/ij/l jj/>/i jj/c/^ji/jui/aji/l jl/1 ^/juj/ill/ajl/j ^j/ju/lj/1 j/v/f/uji/iji/^ 1/j l/i jl/j 1/jj^/lil/jju/jll/</jjjj/\/ujl/ajj/^ i/(/j^i/l^j/jaj/r/s/ja/ll j/^ij/,/t j/ljl/jjjl/li/ljj/jp/u j/jlp/* j/jl^/* ^/il j/lj j/j^l/jj ^/lji/lij/t ^/jt/l ^j/jls/■/uj/j j^/ii/0/jii/m/a j/^^/»/j u/i i/^ l/p/j c/j ij/ij j/3/j a/4 ^/^j j/jil/jj-/^ u/^lj/jla ".strip().split(
                     "/"
                 ),
             ),
@@ -2431,7 +2431,7 @@ def _init_lexicon(self):
             ),
             (
                 "zxx-x-latnunk",
-                " jjjj/klzz/lzzw/zzwx/zwxh/wxh:/klz/kl/tk-/, tk/xh:1/xh:2/tk-3/tk-2/esa/0, t/gsa/kkkk/..../ksa/.tk/dks/hhhh/vga/ds/k/xh:5/.../t.../kt../tk2/0sw/se/sw/g, t/tk l/k-31/sec/http/0se/ttp:/tp:/p:/0ne/0nw./sa d/xh:4/lin/link/k li/ink_/nk_5/k_50/ne/2  k/hkh/----/cm/7  k/vk/9  k/4  k/xh:3/jjj/k-21/h:16/gks/k-37/a kl/h:11/mem/ga k/h:15/h:18/1  k/6  k/8  k/xh:6/2, t/tk/dh/dk/0  k/0 tk/3  k/5  k/h:14/h:17/\/k gs/hkk/h:13/k-27/vks/h:20/ksj/tk3/fy,/vksj/gggg/60,/tk-8/201/1/kjh/h:19/nw/h:12/-310/www/tk-7/8, t/www./cm,/100/a dk/0/h:24/k-32/ls/.. t/...1/00 0/. tk/h:10/0g,/k tk/00,/m kl/h:21/-320/0 00/ne 1/-210/60/= 3/p kl/k-22/,oa/tk24/1, t/02,/004/c kl/writ/wri/rite/rk g/ds f/r kl ".strip().split(
+                r" jjjj/klzz/lzzw/zzwx/zwxh/wxh:/klz/kl/tk-/, tk/xh:1/xh:2/tk-3/tk-2/esa/0, t/gsa/kkkk/..../ksa/.tk/dks/hhhh/vga/ds/k/xh:5/.../t.../kt../tk2/0sw/se/sw/g, t/tk l/k-31/sec/http/0se/ttp:/tp:/p:/0ne/0nw./sa d/xh:4/lin/link/k li/ink_/nk_5/k_50/ne/2  k/hkh/----/cm/7  k/vk/9  k/4  k/xh:3/jjj/k-21/h:16/gks/k-37/a kl/h:11/mem/ga k/h:15/h:18/1  k/6  k/8  k/xh:6/2, t/tk/dh/dk/0  k/0 tk/3  k/5  k/h:14/h:17/\/k gs/hkk/h:13/k-27/vks/h:20/ksj/tk3/fy,/vksj/gggg/60,/tk-8/201/1/kjh/h:19/nw/h:12/-310/www/tk-7/8, t/www./cm,/100/a dk/0/h:24/k-32/ls/.. t/...1/00 0/. tk/h:10/0g,/k tk/00,/m kl/h:21/-320/0 00/ne 1/-210/60/= 3/p kl/k-22/,oa/tk24/1, t/02,/004/c kl/writ/wri/rite/rk g/ds f/r kl ".strip().split(
                     "/"
                 ),
             ),

tests/models/test_discover_supported_languages.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 from collections.abc import Sequence
+from pathlib import Path
 from typing import Any
 
 import pytest
@@ -59,6 +60,80 @@ class _Model(FastTextHubModel):
     assert "xyz" not in langs
 
 
+def test_read_labels_from_bin_parses_minimal_blob(tmp_path: Path) -> None:
+    """Round-trip a minimal fasttext binary blob through the file parser."""
+    import struct
+
+    from commonlid.models._fasttext_base import _read_labels_from_bin
+
+    entries = [
+        ("hello", 17, 0),  # word
+        ("world", 5, 0),  # word
+        ("__label__eng_Latn", 100, 1),  # label
+        ("__label__jw_Latn", 7, 1),  # label
+    ]
+    nwords = sum(1 for _, _, t in entries if t == 0)
+    nlabels = sum(1 for _, _, t in entries if t == 1)
+
+    blob = bytearray()
+    blob += struct.pack("<ii", 793712314, 12)  # magic + version
+    blob += struct.pack("<12id", *([1] * 12), 0.0001)  # Args block (12 int32 + 1 double)
+    blob += struct.pack("<3i", len(entries), nwords, nlabels)  # size_, nwords_, nlabels_
+    blob += struct.pack("<2q", 0, -1)  # ntokens_, pruneidx_size_
+    for word, count, entry_type in entries:
+        blob += word.encode("utf-8") + b"\0"
+        blob += struct.pack("<q", count)
+        blob += struct.pack("<b", entry_type)
+
+    path = tmp_path / "fake_model.bin"
+    path.write_bytes(bytes(blob))
+    assert _read_labels_from_bin(path) == ["__label__eng_Latn", "__label__jw_Latn"]
+
+
+def test_read_labels_from_bin_rejects_bad_magic(tmp_path: Path) -> None:
+    from commonlid.models._fasttext_base import _read_labels_from_bin
+
+    path = tmp_path / "garbage.bin"
+    path.write_bytes(b"NOTAFASTTEXTMODEL" + b"\0" * 200)
+    with pytest.raises(ValueError, match=r"Not a fasttext model\.bin"):
+        _read_labels_from_bin(path)
+
+
+def test_fasttext_base_falls_back_to_file_when_get_labels_missing(
+    tmp_path: Path,
+) -> None:
+    """`fasttext-predict` exposes no get_labels(); we must read the file directly."""
+    import struct
+
+    from commonlid.models._fasttext_base import FastTextHubModel
+
+    blob = bytearray()
+    blob += struct.pack("<ii", 793712314, 12)
+    blob += struct.pack("<12id", *([1] * 12), 0.0001)
+    blob += struct.pack("<3i", 2, 0, 2)
+    blob += struct.pack("<2q", 0, -1)
+    for word in (b"__label__eng_Latn", b"__label__jw_Latn"):
+        blob += word + b"\0" + struct.pack("<q", 1) + struct.pack("<b", 1)
+    path = tmp_path / "stub.bin"
+    path.write_bytes(bytes(blob))
+
+    class _PredictOnlyFT:
+        # Mirrors fasttext-predict: no get_labels.
+        def predict(self, texts: list[str]) -> list[list[str]]:
+            return [["__label__eng_Latn"] for _ in texts]
+
+    class _Model(FastTextHubModel):
+        model_id = "_fasttext_predict_stub"
+        hf_repo_id = "stub/stub"
+
+    m = _Model()
+    m._ft = _PredictOnlyFT()  # type: ignore[assignment]
+    m._model_path = str(path)
+    m._loaded = True
+    langs = m.discover_supported_languages()
+    assert langs == frozenset({"eng", "jav"})  # 'jw' conformed to 'jav'
+
+
 def test_base_default_returns_supported_languages_attr() -> None:
     from commonlid.core.lid_model import LIDModel