Merge remote-tracking branch 'origin/master' into run-resource-sensitive-tests-alone-on-mac

Author: Pijukatel

pyproject.toml

@@ -112,7 +112,7 @@ dev = [
     "pytest-rerunfailures<17.0.0",
     "pytest-timeout<3.0.0",
     "pytest-xdist<4.0.0",
-    "pytest<9.0.0",
+    "pytest<10.0.0",
     "ruff~=0.15.0",
     "setuptools", # setuptools are used by pytest, but not explicitly required
     "ty~=0.0.0",

tests/unit/request_loaders/test_sitemap_request_loader.py

@@ -8,7 +8,6 @@
 from crawlee.http_clients._base import HttpClient
 from crawlee.request_loaders._sitemap_request_loader import SitemapRequestLoader
 from crawlee.storages import KeyValueStore
-from tests.unit.utils import run_alone_on_mac
 
 BASIC_SITEMAP = """
 <?xml version="1.0" encoding="UTF-8"?>
@@ -51,24 +50,22 @@ def encode_base64(data: bytes) -> str:
     return base64.b64encode(data).decode('utf-8')
 
 
-@run_alone_on_mac
 async def test_sitemap_traversal(server_url: URL, http_client: HttpClient) -> None:
     sitemap_url = (server_url / 'sitemap.xml').with_query(base64=encode_base64(BASIC_SITEMAP.encode()))
     sitemap_loader = SitemapRequestLoader([str(sitemap_url)], http_client=http_client)
 
     while not await sitemap_loader.is_finished():
         item = await sitemap_loader.fetch_next_request()
-        assert item is not None
 
-        await sitemap_loader.mark_request_as_handled(item)
+        if item:
+            await sitemap_loader.mark_request_as_handled(item)
 
     assert await sitemap_loader.is_empty()
     assert await sitemap_loader.is_finished()
     assert await sitemap_loader.get_total_count() == 5
     assert await sitemap_loader.get_handled_count() == 5
 
 
-@run_alone_on_mac
 async def test_is_empty_does_not_depend_on_fetch_next_request(server_url: URL, http_client: HttpClient) -> None:
     sitemap_url = (server_url / 'sitemap.xml').with_query(base64=encode_base64(BASIC_SITEMAP.encode()))
     sitemap_loader = SitemapRequestLoader([str(sitemap_url)], http_client=http_client)
@@ -88,6 +85,9 @@ async def test_is_empty_does_not_depend_on_fetch_next_request(server_url: URL, h
         await sitemap_loader.mark_request_as_handled(item)
 
     assert await sitemap_loader.is_empty()
+
+    await asyncio.sleep(0.1)
+
     assert await sitemap_loader.is_finished()
 
 
@@ -195,12 +195,13 @@ def transform_request(request_options: RequestOptions) -> RequestOptions | Reque
 
     while not await sitemap_loader.is_finished():
         request = await sitemap_loader.fetch_next_request()
-        assert request is not None
-        assert request.user_data.get('transformed') is True
 
-        extracted_urls.add(request.url)
+        if request:
+            assert request.user_data.get('transformed') is True
+
+            extracted_urls.add(request.url)
 
-        await sitemap_loader.mark_request_as_handled(request)
+            await sitemap_loader.mark_request_as_handled(request)
 
     assert len(extracted_urls) == 5
     assert extracted_urls == {