feat: Add Python 3.13t and 3.14t builds / wheels (#619)

* feat: Add Python 3.13t and 3.14t builds / wheels

* Try to fix minio

* Try a different approach with pytest-freethreaded

* Bump the setup-uv version

* Back to 1

* modify testss

* cleaner delete

* Enable `generate-import-lib` pyo3 feature on windows

* fix: Use pytest-run-parallel

---------

Co-authored-by: Kyle Barron <kyle@developmentseed.org>
Co-authored-by: DisturbedOcean <DisturbedOcean@users.noreply.github.com>

Author: DisturbedOcean

.github/workflows/test-python.yml

@@ -64,11 +64,17 @@ jobs:
 
       - name: Build rust submodules
         run: |
-          uv run --python ${{ matrix.python-version }} maturin develop --uv -m obstore/Cargo.toml
+          uv run --python ${{ matrix.python-version }} maturin develop -m obstore/Cargo.toml
 
       - name: Run python tests
+        if: "!endsWith(matrix.python-version, 't')"
         run: |
-          uv run --python ${{ matrix.python-version }} pytest
+          uv run --python ${{ matrix.python-version }} pytest tests
+
+      - name: Run python tests with --parallel-threads=2
+        if: "endsWith(matrix.python-version, 't')"
+        run: |
+          uv run --python ${{ matrix.python-version }} pytest tests --parallel-threads=2
 
       # Ensure docs build without warnings
       - name: Check docs

.github/workflows/wheels.yml

@@ -52,7 +52,7 @@ jobs:
           version: "0.10.x"
 
       - name: Install Python versions
-        run: uv python install 3.10 3.11 pypy3.11
+        run: uv python install 3.10 3.11 3.13t 3.14t pypy3.11
 
       - name: Build abi3-py311 wheels
         uses: PyO3/maturin-action@v1
@@ -66,7 +66,7 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.platform.target }}
-          args: --release --out dist -i 3.10 -i pypy3.11 --manifest-path obstore/Cargo.toml
+          args: --release --out dist -i 3.10 -i 3.13t -i 3.14t -i pypy3.11 --manifest-path obstore/Cargo.toml
           sccache: "true"
           manylinux: ${{ matrix.platform.manylinux }}
 
@@ -99,7 +99,7 @@ jobs:
           version: "0.10.x"
 
       - name: Install Python versions
-        run: uv python install 3.10 3.11 pypy3.11
+        run: uv python install 3.10 3.11 3.13t 3.14t pypy3.11
 
       - name: Build abi3-py311 wheels
         uses: PyO3/maturin-action@v1
@@ -113,7 +113,7 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.platform.target }}
-          args: --release --out dist -i 3.10 -i pypy3.11 --manifest-path obstore/Cargo.toml
+          args: --release --out dist -i 3.10 -i 3.13t -i 3.14t -i pypy3.11 --manifest-path obstore/Cargo.toml
           sccache: "true"
           manylinux: musllinux_1_2
 
@@ -145,14 +145,14 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.platform.target }}
-          args: --release --out dist --features abi3-py311 -i 3.11 --manifest-path obstore/Cargo.toml
+          args: --release --out dist --features abi3-py311 --features generate-import-lib -i 3.11 --manifest-path obstore/Cargo.toml
           sccache: "true"
 
       - name: Build version-specific wheels
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.platform.target }}
-          args: --release --out dist -i 3.10 --manifest-path obstore/Cargo.toml
+          args: --release --out dist --features generate-import-lib -i 3.10 -i 3.13t -i 3.14t --manifest-path obstore/Cargo.toml
           sccache: "true"
       - name: Upload wheels
         uses: actions/upload-artifact@v4
@@ -179,7 +179,7 @@ jobs:
           version: "0.10.x"
 
       - name: Install Python versions
-        run: uv python install 3.10 3.11 pypy3.11
+        run: uv python install 3.10 3.11 3.13t 3.14t pypy3.11
 
       - name: Build abi3-py311 wheels
         uses: PyO3/maturin-action@v1
@@ -192,7 +192,7 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.platform.target }}
-          args: --release --out dist -i 3.10 -i pypy3.11 --manifest-path obstore/Cargo.toml
+          args: --release --out dist -i 3.10 -i 3.13t -i 3.14t -i pypy3.11 --manifest-path obstore/Cargo.toml
           sccache: "true"
       - name: Upload wheels
         uses: actions/upload-artifact@v4

Cargo.lock

@@ -19,6 +19,8 @@ crate-type = ["cdylib"]
 
 [features]
 abi3-py311 = ["pyo3/abi3-py311"]
+# https://www.maturin.rs/distribution.html#cross-compile-to-windows
+generate-import-lib = ["pyo3/generate-import-lib"]
 
 [dependencies]
 arrow = "57"

obstore/Cargo.toml

@@ -43,7 +43,7 @@ fn check_debug_build(_py: Python) -> PyResult<()> {
 }
 
 /// A Python module implemented in Rust.
-#[pymodule]
+#[pymodule(gil_used = false)]
 fn _obstore(py: Python, m: &Bound<PyModule>) -> PyResult<()> {
     check_debug_build(py)?;
 

obstore/src/lib.rs

@@ -27,7 +27,7 @@ examples = ["fastapi>=0.115.12", "tqdm>=4.67.1"]
 dev = [
     "aiohttp-retry>=2.9.1",
     "aiohttp>=3.11.13",
-    "arro3-core>=0.4.2",
+    "arro3-core>=0.6.5",
     "azure-identity>=1.21.0",
     "boto3>=1.38.21",
     "docker>=7.1.0",
@@ -42,10 +42,12 @@ dev = [
     "pyarrow>=17.0.0",
     "pystac-client>=0.8.3",
     "pystac>=1.10.1",
+    "pytest>=8.3.3",
     "pytest-asyncio>=0.24.0",
     "pytest-mypy-plugins>=3.2.0",
-    "pytest>=8.3.3",
+    "pytest-run-parallel>=0.3.0;python_version>='3.13'",
     "python-dotenv>=1.0.1",
+    "pyzmq>=27.1.0",
     "ruff>=0.15.0",
     "types-boto3[s3,sts]>=1.36.23",
     "types-requests>=2.31.0.6",
@@ -108,7 +110,14 @@ executionEnvironments = [
 addopts = "-v --mypy-only-local-stub"
 asyncio_default_fixture_loop_scope = "function"
 testpaths = ["tests"]
-markers = ["network: mark the test as requiring a network connection"]
+markers = [
+    "network: mark the test as requiring a network connection",
+    "parallel_threads: mark test for concurrent thread execution",
+]
+thread_unsafe_fixtures = [
+    "minio_bucket",
+    "minio_store",
+]
 
 [tool.mypy]
 files = ["obstore/python"]

pyproject.toml

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
+import inspect
 import socket
+import sysconfig
 import time
 import warnings
 from typing import TYPE_CHECKING, Any
@@ -13,6 +15,37 @@
 
 from obstore.store import S3Store
 
+
+def pytest_configure(config: pytest.Config) -> None:
+    """On free-threaded Python, enable parallel test execution by default."""
+    if (
+        sysconfig.get_config_var("Py_GIL_DISABLED")
+        and hasattr(
+            config.option,
+            "parallel_threads",
+        )
+        and config.option.parallel_threads == 1
+    ):
+        config.option.parallel_threads = "auto"
+
+
+def pytest_collection_modifyitems(items: list[pytest.Item]) -> None:
+    """Mark async tests as thread-unsafe.
+
+    pytest-asyncio and pytest-run-parallel are incompatible: run-parallel wraps
+    the test function before asyncio can schedule it, producing an unawaited
+    coroutine. Marking every async test as thread_unsafe keeps them on the main
+    thread where pytest-asyncio expects to run them.
+
+    See: https://github.com/Quansight-Labs/pytest-run-parallel/issues/14
+    """
+    for item in items:
+        if isinstance(item, pytest.Function) and inspect.iscoroutinefunction(
+            item.function,
+        ):
+            item.add_marker(pytest.mark.thread_unsafe)
+
+
 if TYPE_CHECKING:
     from collections.abc import Generator
 
@@ -116,13 +149,17 @@ def minio_config() -> Generator[tuple[S3Config, ClientConfig], Any, None]:
 def minio_bucket(
     minio_config: tuple[S3Config, ClientConfig],
 ) -> Generator[tuple[S3Config, ClientConfig], Any, None]:
+    # Clean bucket before each test so tests always start with empty state,
+    # regardless of whether a previous test's teardown failed or was incomplete.
+    store = S3Store(config=minio_config[0], client_options=minio_config[1])
+    objects = store.list().collect()
+    store.delete([obj["path"] for obj in objects])
+
     yield minio_config
 
-    # Remove all files from bucket
-    store = S3Store(config=minio_config[0], client_options=minio_config[1])
+    # Best-effort cleanup after the test as well.
     objects = store.list().collect()
-    paths = [obj["path"] for obj in objects]
-    store.delete(paths)
+    store.delete([obj["path"] for obj in objects])
 
 
 @pytest.fixture

tests/conftest.py

@@ -1,5 +1,6 @@
 import pickle
 from pathlib import Path
+from tempfile import TemporaryDirectory
 
 import pytest
 
@@ -44,30 +45,36 @@ def test_local_from_url():
         store = LocalStore.from_url(url)
 
 
-def test_create_prefix(tmp_path: Path):
-    tmpdir = tmp_path / "abc"
-    assert not tmpdir.exists()
-    LocalStore(tmpdir, mkdir=True)
-    assert tmpdir.exists()
+def test_create_prefix():
+    with TemporaryDirectory() as tmp:
+        tmp_path = Path(tmp)
+        tmpdir = tmp_path / "abc"
+        assert not tmpdir.exists()
+        LocalStore(tmpdir, mkdir=True)
+        assert tmpdir.exists()
 
-    # Assert that mkdir=True works even when the dir already exists
-    LocalStore(tmpdir, mkdir=True)
-    assert tmpdir.exists()
+        # Assert that mkdir=True works even when the dir already exists
+        LocalStore(tmpdir, mkdir=True)
+        assert tmpdir.exists()
 
 
-def test_prefix_property(tmp_path: Path):
-    store = LocalStore(tmp_path)
-    assert store.prefix == tmp_path
-    assert isinstance(store.prefix, Path)
-    # Can pass it back to the store init
-    LocalStore(store.prefix)
+def test_prefix_property():
+    with TemporaryDirectory() as tmp:
+        tmp_path = Path(tmp)
+        store = LocalStore(tmp_path)
+        assert store.prefix == tmp_path
+        assert isinstance(store.prefix, Path)
+        # Can pass it back to the store init
+        LocalStore(store.prefix)
 
 
-def test_pickle(tmp_path: Path):
-    store = LocalStore(tmp_path)
-    store.put("path.txt", b"foo")
-    new_store: LocalStore = pickle.loads(pickle.dumps(store))
-    assert new_store.get("path.txt").bytes() == b"foo"
+def test_pickle():
+    with TemporaryDirectory() as tmp:
+        tmp_path = Path(tmp)
+        store = LocalStore(tmp_path)
+        store.put("path.txt", b"foo")
+        new_store: LocalStore = pickle.loads(pickle.dumps(store))
+        assert new_store.get("path.txt").bytes() == b"foo"
 
 
 def test_eq():
@@ -79,18 +86,20 @@ def test_eq():
     assert store != store3
 
 
-def test_local_store_percent_encoded(tmp_path: Path):
-    fname1 = "hello%20world.txt"
-    content1 = b"Hello, World!"
-    with (tmp_path / fname1).open("wb") as f:
-        f.write(content1)
+def test_local_store_percent_encoded():
+    with TemporaryDirectory() as tmp:
+        tmp_path = Path(tmp)
+        fname1 = "hello%20world.txt"
+        content1 = b"Hello, World!"
+        with (tmp_path / fname1).open("wb") as f:
+            f.write(content1)
 
-    store = LocalStore(tmp_path)
-    assert store.get(fname1).bytes() == content1
+        store = LocalStore(tmp_path)
+        assert store.get(fname1).bytes() == content1
 
-    fname2 = "hello world.txt"
-    content2 = b"Hello, World! (with spaces)"
-    with (tmp_path / fname2).open("wb") as f:
-        f.write(content2)
+        fname2 = "hello world.txt"
+        content2 = b"Hello, World! (with spaces)"
+        with (tmp_path / fname2).open("wb") as f:
+            f.write(content2)
 
-    assert store.get(fname2).bytes() == content2
+        assert store.get(fname2).bytes() == content2