fix: apparently MIME magic typing doesnt work on Windows OS

Author: luabida

pysus/api/client.py

@@ -6,7 +6,7 @@
 
 import enum
 from collections.abc import Callable
-from datetime import datetime
+from datetime import UTC, datetime
 from pathlib import Path
 from typing import TYPE_CHECKING, Literal
 
@@ -16,6 +16,7 @@
 from pysus import CACHEPATH
 from sqlalchemy import DateTime, Enum, Integer, String, create_engine
 from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, sessionmaker
+from sqlalchemy.pool import NullPool
 
 from .dadosgov import DadosGovClient
 from .ducklake.client import DuckLake
@@ -61,7 +62,7 @@ class LocalFileState(Base):
     sha256: Mapped[str | None] = mapped_column(String, nullable=True)
     last_synced: Mapped[datetime] = mapped_column(
         DateTime,
-        default=datetime.utcnow,
+        default=lambda: datetime.now(UTC).replace(tzinfo=None),
     )
 
 
@@ -85,7 +86,10 @@ def __init__(self, db_path: Path = CACHEPATH / "config.db"):
         db_path.parent.mkdir(parents=True, exist_ok=True)
 
         self.cachepath = db_path.parent
-        self.engine = create_engine(f"duckdb:///{db_path}")
+        self.engine = create_engine(
+            f"duckdb:///{db_path.resolve().as_posix()}",
+            poolclass=NullPool,
+        )
         Base.metadata.create_all(self.engine)
         self.Session = sessionmaker(bind=self.engine)
 
@@ -239,7 +243,7 @@ async def _update_state(
                 session.add(record)
 
             record.status = status
-            record.last_synced = datetime.utcnow()
+            record.last_synced = datetime.now(UTC).replace(tzinfo=None)
             session.commit()
 
     async def download(

pysus/api/extensions.py

@@ -4,6 +4,7 @@
 import csv
 import gzip
 import shutil
+import sys
 import tarfile
 import zipfile
 from collections.abc import AsyncGenerator, Callable
@@ -868,8 +869,7 @@ class ExtensionFactory:
         ".json": JSON,
     }
 
-    _magic_available: bool = True
-    _magic_lock: asyncio.Lock = asyncio.Lock()
+    _magic_available: bool = sys.platform != "win32"
 
     @classmethod
     async def _identify(cls, path: Path) -> type[BaseLocalFile] | None:
@@ -881,16 +881,15 @@ async def _identify(cls, path: Path) -> type[BaseLocalFile] | None:
         except (ImportError, OSError):
             cls._magic_available = False
             return None
-        async with cls._magic_lock:
-            try:
-                mime = await to_thread.run_sync(
-                    magic.from_file,
-                    str(path),
-                    True,
-                )
-                return cls._mime.get(mime)
-            except (magic.MagicException, OSError):
-                return None
+        try:
+            mime = await to_thread.run_sync(
+                magic.from_file,
+                str(path),
+                True,
+            )
+            return cls._mime.get(mime)
+        except (magic.MagicException, OSError):
+            return None
 
     @classmethod
     async def get_file_class(cls, path: Path) -> type[BaseLocalFile]:

pysus/tests/api/ftp/test_models.py

@@ -128,15 +128,15 @@ def formatter(self, f):
 
 
 @pytest.mark.asyncio
-async def test_file_download_calls_client(mock_client, mock_dataset):
+async def test_file_download_calls_client(mock_client, mock_dataset, tmp_path):
     file = File(
         path="/root/test.dbc",
         _info={"path": "/root/test.dbc", "name": "test.dbc"},
         type="file",
         dataset=mock_dataset,
     )
 
-    dest = Path("/tmp/test.dbc")
+    dest = Path(tmp_path / "test.dbc")
     await file._download(output=dest)
 
     mock_client._download_file.assert_called_once_with(file, dest, None)

pysus/tests/api/test_client.py

@@ -77,11 +77,13 @@ def test_download_status_values(self):
 
 class TestLocalFileState:
     @pytest.mark.asyncio
-    async def test_update_state_creates_record(self, test_db_path):
+    async def test_update_state_creates_record(self, test_db_path, tmp_path):
         client = PySUS(db_path=test_db_path)
 
+        local = pathlib.Path(tmp_path / "test.dbc")
+
         await client._update_state(
-            local_path=pathlib.Path("/tmp/test.dbc"),
+            local_path=local,
             remote_path="/remote/test.dbc",
             client_name="ftp",
             status=DownloadStatus.COMPLETED,
@@ -93,9 +95,7 @@ async def test_update_state_creates_record(self, test_db_path):
 
         with client.Session() as session:
             record = (
-                session.query(LocalFileState)
-                .filter_by(path="/tmp/test.dbc")
-                .first()
+                session.query(LocalFileState).filter_by(path=str(local)).first()
             )
             assert record is not None
             assert record.remote_path == "/remote/test.dbc"
@@ -109,34 +109,36 @@ async def test_update_state_creates_record(self, test_db_path):
         await client.__aexit__(None, None, None)
 
     @pytest.mark.asyncio
-    async def test_delete_record_removes_entry(self, test_db_path):
+    async def test_delete_record_removes_entry(self, test_db_path, tmp_path):
         client = PySUS(db_path=test_db_path)
 
+        local = pathlib.Path(tmp_path / "test.dbc")
+
         await client._update_state(
-            local_path=pathlib.Path("/tmp/test.dbc"),
+            local_path=local,
             remote_path="/remote/test.dbc",
             client_name="ftp",
             status=DownloadStatus.COMPLETED,
         )
 
-        await client._delete_record("/tmp/test.dbc")
+        await client._delete_record(str(local))
 
         with client.Session() as session:
             record = (
-                session.query(LocalFileState)
-                .filter_by(path="/tmp/test.dbc")
-                .first()
+                session.query(LocalFileState).filter_by(path=str(local)).first()
             )
             assert record is None
 
         await client.__aexit__(None, None, None)
 
     @pytest.mark.asyncio
-    async def test_get_local_file_finds_existing(self, test_db_path):
+    async def test_get_local_file_finds_existing(self, test_db_path, tmp_path):
         client = PySUS(db_path=test_db_path)
 
+        local = pathlib.Path(tmp_path / "test.dbc")
+
         await client._update_state(
-            local_path=pathlib.Path("/tmp/test.dbc"),
+            local_path=local,
             remote_path="/remote/test.dbc",
             client_name="ftp",
             status=DownloadStatus.COMPLETED,
@@ -158,17 +160,20 @@ async def test_get_local_file_finds_existing(self, test_db_path):
 
 class TestGetCompletedRemotePaths:
     @pytest.mark.asyncio
-    async def test_get_completed_remote_paths(self, test_db_path):
+    async def test_get_completed_remote_paths(self, test_db_path, tmp_path):
         client = PySUS(db_path=test_db_path)
 
+        local1 = pathlib.Path(tmp_path / "test1.dbc")
+        local2 = pathlib.Path(tmp_path / "test2.dbc")
+
         await client._update_state(
-            local_path=pathlib.Path("/tmp/test1.dbc"),
+            local_path=local1,
             remote_path="/remote/test1.dbc",
             client_name="ftp",
             status=DownloadStatus.COMPLETED,
         )
         await client._update_state(
-            local_path=pathlib.Path("/tmp/test2.dbc"),
+            local_path=local2,
             remote_path="/remote/test2.dbc",
             client_name="ftp",
             status=DownloadStatus.PENDING,

pysus/tests/api/test_databases.py

@@ -303,14 +303,27 @@ def test_fetch_data_no_files(self):
 
 
 class TestListFiles:
+    def _mock_asyncio_run(self, return_value):
+        import asyncio
+
+        def _run(coro):
+            if asyncio.iscoroutine(coro):
+                coro.close()
+            return return_value
+
+        return _run
+
     def test_list_files_returns_dataframe(self):
         import pandas as pd
 
-        with patch("pysus.api._impl.databases.asyncio.run") as mock_run:
-            mock_run.return_value = pd.DataFrame(
-                {"name": ["test.parquet"], "path": ["/test.parquet"]}
-            )
+        ret = pd.DataFrame(
+            {"name": ["test.parquet"], "path": ["/test.parquet"]}
+        )
 
+        with patch(
+            "pysus.api._impl.databases.asyncio.run",
+            side_effect=self._mock_asyncio_run(ret),
+        ):
             from pysus.api._impl.databases import list_files
 
             result = list_files(dataset="SINAN")
@@ -321,19 +334,22 @@ def test_list_files_returns_dataframe(self):
     def test_list_files_with_filters(self):
         import pandas as pd
 
-        with patch("pysus.api._impl.databases.asyncio.run") as mock_run:
-            mock_run.return_value = pd.DataFrame(
-                {
-                    "name": ["test1.parquet", "test2.parquet"],
-                    "path": ["/test1.parquet", "/test2.parquet"],
-                    "dataset": ["sinan", "sinan"],
-                    "year": [2024, 2023],
-                    "month": [1, 2],
-                    "state": ["SP", "RJ"],
-                    "modify": ["2024-01-01", "2024-01-02"],
-                }
-            )
-
+        ret = pd.DataFrame(
+            {
+                "name": ["test1.parquet", "test2.parquet"],
+                "path": ["/test1.parquet", "/test2.parquet"],
+                "dataset": ["sinan", "sinan"],
+                "year": [2024, 2023],
+                "month": [1, 2],
+                "state": ["SP", "RJ"],
+                "modify": ["2024-01-01", "2024-01-02"],
+            }
+        )
+
+        with patch(
+            "pysus.api._impl.databases.asyncio.run",
+            side_effect=self._mock_asyncio_run(ret),
+        ):
             from pysus.api._impl.databases import list_files
 
             result = list_files(
@@ -357,9 +373,12 @@ def test_list_files_with_filters(self):
     def test_list_files_empty_result(self):
         import pandas as pd
 
-        with patch("pysus.api._impl.databases.asyncio.run") as mock_run:
-            mock_run.return_value = pd.DataFrame()
+        ret = pd.DataFrame()
 
+        with patch(
+            "pysus.api._impl.databases.asyncio.run",
+            side_effect=self._mock_asyncio_run(ret),
+        ):
             from pysus.api._impl.databases import list_files
 
             result = list_files(dataset="SINAN")