Merge branch 'master' into fix-global-event-manager

Author: Mantisus

CHANGELOG.md

@@ -2,6 +2,15 @@
 
 All notable changes to this project will be documented in this file.
 
+<!-- git-cliff-unreleased-start -->
+## 1.6.1 - **not yet released**
+
+### 🐛 Bug Fixes
+
+- Handle invalid URLs in `RequestList` ([#1803](https://github.com/apify/crawlee-python/pull/1803)) ([0b2e3fc](https://github.com/apify/crawlee-python/commit/0b2e3fc5cbca371131b54085e052a6cda6361b0f)) by [@Mantisus](https://github.com/Mantisus), closes [#1802](https://github.com/apify/crawlee-python/issues/1802)
+
+
+<!-- git-cliff-unreleased-end -->
 ## [1.6.0](https://github.com/apify/crawlee-python/releases/tag/v1.6.0) (2026-03-20)
 
 ### 🚀 Features

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "crawlee"
-version = "1.6.0"
+version = "1.6.1"
 description = "Crawlee for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = { file = "LICENSE" }

src/crawlee/otel/crawler_instrumentor.py

@@ -69,10 +69,10 @@ def _init_wrapper(wrapped: Any, _: Any, args: Any, kwargs: Any) -> None:
 
             async def middleware_wrapper(wrapped: Any, instance: _Middleware, args: Any, kwargs: Any) -> Any:
                 with self._tracer.start_as_current_span(
-                    name=f'{instance.generator.__name__}, {wrapped.__name__}',  # type:ignore[attr-defined]  # valid in our context
+                    name=f'{instance.generator.__name__}, {wrapped.__name__}',  # ty:ignore[unresolved-attribute]  # valid in our context
                     attributes={
                         URL_FULL: instance.input_context.request.url,
-                        CODE_FUNCTION_NAME: instance.generator.__qualname__,  # type:ignore[attr-defined]  # valid in our context
+                        CODE_FUNCTION_NAME: instance.generator.__qualname__,  # ty:ignore[unresolved-attribute]  # valid in our context
                     },
                 ):
                     return await wrapped(*args, **kwargs)

src/crawlee/storage_clients/_base/_dataset_client.py

@@ -88,7 +88,6 @@ async def iterate_items(
         The backend method for the `Dataset.iterate_items` call.
         """
         # This syntax is to make type checker properly work with abstract AsyncIterator.
-        # https://mypy.readthedocs.io/en/stable/more_types.html#asynchronous-iterators
         raise NotImplementedError
         if False:
-            yield 0
+            yield {}

src/crawlee/storage_clients/_base/_key_value_store_client.py

@@ -3,10 +3,12 @@
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING, Any
 
+from crawlee.storage_clients.models import KeyValueStoreRecordMetadata
+
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
 
-    from crawlee.storage_clients.models import KeyValueStoreMetadata, KeyValueStoreRecord, KeyValueStoreRecordMetadata
+    from crawlee.storage_clients.models import KeyValueStoreMetadata, KeyValueStoreRecord
 
 
 class KeyValueStoreClient(ABC):
@@ -73,10 +75,9 @@ async def iterate_keys(
         The backend method for the `KeyValueStore.iterate_keys` call.
         """
         # This syntax is to make type checker properly work with abstract AsyncIterator.
-        # https://mypy.readthedocs.io/en/stable/more_types.html#asynchronous-iterators
         raise NotImplementedError
         if False:
-            yield 0
+            yield KeyValueStoreRecordMetadata()
 
     @abstractmethod
     async def get_public_url(self, *, key: str) -> str:

src/crawlee/storages/_storage_instance_manager.py

@@ -167,7 +167,7 @@ async def open_storage_instance(
 
                 metadata = await client.get_metadata()
 
-                instance = cls(client, metadata.id, metadata.name)  # type: ignore[call-arg]
+                instance = cls(client, metadata.id, metadata.name)  # ty: ignore[too-many-positional-arguments]
                 instance_name = getattr(instance, 'name', None)
 
                 # Cache the instance.

tests/unit/conftest.py

@@ -74,8 +74,8 @@ def _prepare_test_env() -> None:
 
         # Reset global class variables to ensure test isolation.
         KeyValueStore._autosaved_values = {}
-        Statistics._Statistics__next_id = 0  # type:ignore[attr-defined] # Mangled attribute
-        BasicCrawler._BasicCrawler__next_id = 0  # type:ignore[attr-defined] # Mangled attribute
+        Statistics._Statistics__next_id = 0  # ty:ignore[unresolved-attribute] # Mangled attribute
+        BasicCrawler._BasicCrawler__next_id = 0  # ty:ignore[unresolved-attribute] # Mangled attribute
 
     return _prepare_test_env
 

tests/unit/crawlers/_basic/test_basic_crawler.py

@@ -1167,7 +1167,7 @@ async def test_crawler_multiple_stops_in_parallel() -> None:
     # Set concurrency to 2 to ensure two urls are being visited in parallel.
     crawler = BasicCrawler(concurrency_settings=ConcurrencySettings(desired_concurrency=2, max_concurrency=2))
 
-    both_handlers_started = asyncio.Barrier(2)  # type:ignore[attr-defined]  # Test is skipped in older Python versions.
+    both_handlers_started = asyncio.Barrier(2)  # ty:ignore[unresolved-attribute]  # Test is skipped in older Python versions.
     only_one_handler_at_a_time = asyncio.Semaphore(1)
 
     @crawler.router.default_handler
@@ -1351,7 +1351,7 @@ async def test_context_use_state_race_condition_in_handlers(key_value_store: Key
     Result should be incremented by 2.
     Method `use_state` must be implemented in a way that prevents race conditions in such scenario."""
     # Test is skipped in older Python versions.
-    from asyncio import Barrier  # type:ignore[attr-defined] # noqa: PLC0415
+    from asyncio import Barrier  # ty:ignore[unresolved-import] # noqa: PLC0415
 
     crawler = BasicCrawler()
     store = await crawler.get_key_value_store()
@@ -1392,7 +1392,7 @@ async def test_timeout_in_handler(sleep_type: str) -> None:
     Crawler should attempt to retry it.
     This test creates situation where the request handler times out twice, on third retry it does not time out."""
     # Test is skipped in older Python versions.
-    from asyncio import timeout  # type:ignore[attr-defined] # noqa: PLC0415
+    from asyncio import timeout  # ty:ignore[unresolved-import] # noqa: PLC0415
 
     non_realtime_system_coefficient = 10
     handler_timeout = timedelta(seconds=1)

tests/unit/storages/test_storage_instance_manager.py

@@ -136,7 +136,7 @@ async def test_preexisting_unnamed_storage_open_by_id(storage_type: type[Storage
 @pytest.mark.skipif(sys.version_info[:3] < (3, 11), reason='asyncio.Barrier was introduced in Python 3.11.')
 async def test_concurrent_open_datasets() -> None:
     """Test that concurrent open datasets with the same name return the same instance."""
-    from asyncio import Barrier  # type:ignore[attr-defined] # noqa: PLC0415
+    from asyncio import Barrier  # ty:ignore[unresolved-import] # noqa: PLC0415
 
     barrier = Barrier(2)
 
@@ -161,7 +161,7 @@ async def push_data(data: dict) -> None:
 @pytest.mark.skipif(sys.version_info[:3] < (3, 11), reason='asyncio.Barrier was introduced in Python 3.11.')
 async def test_concurrent_open_datasets_with_same_name_and_alias() -> None:
     """Test that concurrent open requests for the same storage return the same instance."""
-    from asyncio import Barrier  # type:ignore[attr-defined] # noqa: PLC0415
+    from asyncio import Barrier  # ty:ignore[unresolved-import] # noqa: PLC0415
 
     valid_kwargs: dict[str, str | None] = {}