fix: Return None from add_request when storage client fails to enqueue request (#1775)

### Description

- This PR updates the signature for `RequestManager.add_request` to
return `None` when the storage client fails to process the request and
returns it in `AddRequestsResponse.unprocessed_requests`. Previously,
accessing `processed_requests[0]` on an empty list caused an
`IndexError`.

### Testing

- Tests updated in accordance with the new signature

Author: Mantisus

src/crawlee/request_loaders/_request_manager.py

@@ -28,7 +28,7 @@ async def add_request(
         request: str | Request,
         *,
         forefront: bool = False,
-    ) -> ProcessedRequest:
+    ) -> ProcessedRequest | None:
         """Add a single request to the manager and store it in underlying resource client.
 
         Args:
@@ -37,7 +37,7 @@ async def add_request(
                 of the manager.
 
         Returns:
-            Information about the request addition to the manager.
+            Information about the request addition to the manager or None if the request was not added.
         """
 
     async def add_requests(
@@ -64,7 +64,8 @@ async def add_requests(
         processed_requests = list[ProcessedRequest]()
         for request in requests:
             processed_request = await self.add_request(request, forefront=forefront)
-            processed_requests.append(processed_request)
+            if processed_request:
+                processed_requests.append(processed_request)
 
     @abstractmethod
     async def reclaim_request(self, request: Request, *, forefront: bool = False) -> ProcessedRequest | None:

src/crawlee/request_loaders/_request_manager_tandem.py

@@ -49,7 +49,7 @@ async def is_finished(self) -> bool:
         return (await self._read_only_loader.is_finished()) and (await self._read_write_manager.is_finished())
 
     @override
-    async def add_request(self, request: str | Request, *, forefront: bool = False) -> ProcessedRequest:
+    async def add_request(self, request: str | Request, *, forefront: bool = False) -> ProcessedRequest | None:
         return await self._read_write_manager.add_request(request, forefront=forefront)
 
     @override

src/crawlee/storages/_request_queue.py

@@ -158,10 +158,23 @@ async def add_request(
         request: str | Request,
         *,
         forefront: bool = False,
-    ) -> ProcessedRequest:
+    ) -> ProcessedRequest | None:
         request = self._transform_request(request)
         response = await self._client.add_batch_of_requests([request], forefront=forefront)
-        return response.processed_requests[0]
+
+        if response.processed_requests:
+            return response.processed_requests[0]
+
+        if response.unprocessed_requests:
+            logger.warning(
+                f'Request {request.url} was not processed by storage client "{self._client.__class__.__name__}".'
+            )
+        else:
+            logger.warning(
+                f'Request {request.url} was not processed by storage client "{self._client.__class__.__name__}" '
+                'received empty response.'
+            )
+        return None
 
     @override
     async def add_requests(

tests/unit/storages/test_request_queue.py

@@ -125,6 +125,7 @@ async def test_add_request_string_url(rq: RequestQueue) -> None:
     result = await rq.add_request(url)
 
     # Verify request was added
+    assert result is not None
     assert result.unique_key is not None
     assert result.was_already_present is False
     assert result.was_already_handled is False
@@ -142,6 +143,7 @@ async def test_add_request_object(rq: RequestQueue) -> None:
     result = await rq.add_request(request)
 
     # Verify request was added
+    assert result is not None
     assert result.unique_key is not None
     assert result.was_already_present is False
     assert result.was_already_handled is False
@@ -158,10 +160,13 @@ async def test_add_duplicate_request(rq: RequestQueue) -> None:
     url = 'https://example.com'
     first_result = await rq.add_request(url)
 
+    assert first_result is not None
+
     # Add the same request again
     second_result = await rq.add_request(url)
 
     # Verify the second request was detected as duplicate
+    assert second_result is not None
     assert second_result.was_already_present is True
     assert second_result.unique_key == first_result.unique_key
 
@@ -336,14 +341,17 @@ async def test_fetch_next_request_and_mark_handled(rq: RequestQueue) -> None:
     assert metadata.pending_request_count == 0
 
     # Verify queue is empty
-    empty_request = await rq.fetch_next_request()
+    empty_request: Request | None = await rq.fetch_next_request()
     assert empty_request is None
 
 
 async def test_get_request_by_id(rq: RequestQueue) -> None:
     """Test retrieving a request by its ID."""
     # Add a request
     added_result = await rq.add_request('https://example.com')
+
+    assert added_result is not None
+
     unique_key = added_result.unique_key
 
     # Retrieve the request by ID
@@ -791,6 +799,7 @@ async def test_alias_request_operations(
 
     for url in urls:
         result = await rq.add_request(url)
+        assert result is not None
         assert result.was_already_present is False
 
     # Test queue metadata