fix: relax pynini version constraint to >=2.1.6 (#345)

* fix: relax pynini version constraint to >=2.1.6 (#317)

Remove upper bound on pynini version to allow installation with
pynini 2.1.7+, which is easier to install on macOS.

* ci: add macOS and Windows to test matrix

* ci: trigger on toml and yml changes

* ci: fix macOS/Windows pynini installation in CI

* ci: fix Windows deps and disable fail-fast

* ci: test Windows with pip only (no conda)

* ci: use conda for Windows, pip+openfst for macOS, pip for Linux

* fix: specify utf-8 encoding when reading test files for Windows

* ci: cache brew/conda/pip to speed up macOS and Windows CI

Author: pengzhendong

.github/workflows/unittest.yml

@@ -6,31 +6,78 @@ on:
     paths:
       - '**.py'
       - '**.tsv'
+      - '**.toml'
+      - '**.yml'
 
 jobs:
   unit-test:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
-        python-version: [3.9]
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ['3.9']
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - name: Set up Python ${{ matrix.python-version }}
+      - name: Set up Python ${{ matrix.python-version }} (non-Windows)
+        if: runner.os != 'Windows'
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
+      - name: Cache Homebrew (macOS)
+        if: runner.os == 'macOS'
+        uses: actions/cache@v4
+        with:
+          path: |
+            /opt/homebrew/Cellar/openfst
+            /opt/homebrew/include/fst
+            /opt/homebrew/lib/libfst*
+          key: brew-openfst-${{ runner.os }}
+      - name: Install openfst (macOS)
+        if: runner.os == 'macOS'
+        run: |
+          brew install openfst || true
+          echo "CPLUS_INCLUDE_PATH=/opt/homebrew/include:$CPLUS_INCLUDE_PATH" >> $GITHUB_ENV
+          echo "LIBRARY_PATH=/opt/homebrew/lib:$LIBRARY_PATH" >> $GITHUB_ENV
+      - name: Cache pip (non-Windows)
+        if: runner.os != 'Windows'
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: pip-${{ runner.os }}-${{ hashFiles('pyproject.toml') }}
+      - name: Install dependencies (non-Windows)
+        if: runner.os != 'Windows'
         run: |
           python -m pip install --upgrade pip
           pip install .[test]
+      - name: Set up Conda (Windows)
+        if: runner.os == 'Windows'
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+          activate-environment: test
+          use-only-tar-bz2: true
+      - name: Cache Conda (Windows)
+        if: runner.os == 'Windows'
+        uses: actions/cache@v4
+        with:
+          path: C:\Miniconda\envs\test
+          key: conda-${{ runner.os }}-${{ hashFiles('pyproject.toml') }}
+      - name: Install dependencies (Windows)
+        if: runner.os == 'Windows'
+        shell: bash -el {0}
+        run: |
+          conda install -c conda-forge pynini
+          python -m pip install --upgrade pip
+          pip install .[test] --no-deps
+          pip install flake8 pytest importlib_resources
       - name: Lint with flake8
+        shell: bash -el {0}
         run: |
-          # stop the build if there are Python syntax errors or undefined names
           flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
       - name: Test with pytest
-        run: |
-          pytest
+        shell: bash -el {0}
+        run: pytest

pyproject.toml

@@ -19,7 +19,7 @@ classifiers = [
 dynamic = ["version"]
 dependencies = [
     # Core dependencies for text processing functionality
-    "pynini>=2.1.6,<2.1.7",
+    "pynini>=2.1.6",
     "importlib_resources"
 ]
 

tn/chinese/test/utils.py

@@ -19,7 +19,7 @@ def parse_test_case(file_name):
     file = os.path.dirname(os.path.abspath(__file__)) + os.path.sep + file_name
 
     delimiter = "=>"
-    with open(file) as fin:
+    with open(file, encoding="utf-8") as fin:
         for line in fin:
             assert delimiter in line
             arr = line.strip().split(delimiter)