fix: call ripgrep with explicit utf-8 encoding. (#1199)

Author: Sola-ris

.github/workflows/pytest.yaml

@@ -43,6 +43,7 @@ jobs:
             libxcb-xinerama0 \
             libxkbcommon-x11-0 \
             libyaml-dev \
+            ripgrep \
             x11-utils
 
       - name: Execute pytest

src/tagstudio/core/library/refresh.py

@@ -105,8 +105,8 @@ def __get_dir_list(self, library_dir: Path, ignore_patterns: list[str]) -> list[
                 ),
                 cwd=library_dir,
                 capture_output=True,
-                text=True,
                 shell=True,
+                encoding="UTF-8",
             )
             compiled_ignore_path.unlink()
 

tests/macros/test_refresh_dir.py

@@ -2,6 +2,7 @@
 # Licensed under the GPL-3.0 License.
 # Created for TagStudio: https://github.com/CyanVoxel/TagStudio
 
+import shutil
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
@@ -29,3 +30,25 @@ def test_refresh_new_files(library: Library, exclude_mode: bool):
     # Test if the single file was added
     list(registry.refresh_dir(library_dir, force_internal_tools=True))
     assert registry.files_not_in_library == [Path("FOO.MD")]
+
+
+@pytest.mark.parametrize("library", [TemporaryDirectory()], indirect=True)
+def test_refresh_multi_byte_filenames_ripgrep(library: Library):
+    assert shutil.which("rg") is not None
+
+    library_dir = unwrap(library.library_dir)
+    # Given
+    registry = RefreshTracker(library=library)
+    library.included_files.clear()
+    (library_dir / ".TagStudio").mkdir()
+    (library_dir / "こんにちは.txt").touch()
+    (library_dir / "em–dash.txt").touch()
+    (library_dir / "apostrophe’.txt").touch()
+    (library_dir / "umlaute äöü.txt").touch()
+
+    # Test if all files were added with their correct names and without exceptions
+    list(registry.refresh_dir(library_dir))
+    assert Path("こんにちは.txt") in registry.files_not_in_library
+    assert Path("em–dash.txt") in registry.files_not_in_library
+    assert Path("apostrophe’.txt") in registry.files_not_in_library
+    assert Path("umlaute äöü.txt") in registry.files_not_in_library