Skip to content

Commit 5d551c2

Browse files
committed
style: trim comments in RequestQueue.add_request retry change
1 parent 164e427 commit 5d551c2

1 file changed

Lines changed: 6 additions & 9 deletions

File tree

src/crawlee/storages/_request_queue.py

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -182,16 +182,14 @@ async def add_request(
182182
forefront: bool = False,
183183
) -> ProcessedRequest | None:
184184
request = self._transform_request(request)
185-
# Route the single request through the same retry mechanism as `add_requests`, so that adding one
186-
# request is just as durable as a batched add against best-effort backends that may return a request
187-
# as unprocessed.
185+
# Route through `_process_batch` so a single add retries unprocessed requests just like a batched one.
188186
response = await self._process_batch([request], base_retry_wait=timedelta(seconds=1), forefront=forefront)
189187

190188
if response.processed_requests:
191189
return response.processed_requests[0]
192190

193-
# `_process_batch` already warns when requests remain unprocessed after the retries are exhausted, so
194-
# only the empty-response case (neither processed nor unprocessed) needs a warning here.
191+
# `_process_batch` already warns about requests left unprocessed after retries; only an empty response
192+
# (neither processed nor unprocessed) is unexpected here.
195193
if not response.unprocessed_requests:
196194
logger.warning(
197195
f'Request {request.url} was not processed by storage client '
@@ -358,8 +356,8 @@ async def _process_batch(
358356
"""Process a batch of requests with automatic retry mechanism.
359357
360358
Returns:
361-
A response aggregating the requests processed across all attempts together with any that remained
362-
unprocessed after the retries were exhausted.
359+
A response aggregating all requests processed across attempts plus any still unprocessed once the
360+
retries are exhausted.
363361
"""
364362
max_attempts = 5
365363
response = await self._client.add_batch_of_requests(batch, forefront=forefront)
@@ -392,8 +390,7 @@ async def _process_batch(
392390
forefront=forefront,
393391
)
394392

395-
# Fold the retry outcome back in: requests processed on retry join this attempt's processed requests,
396-
# and only those still unprocessed after the final attempt remain.
393+
# Merge the retry outcome: processed requests accumulate, unprocessed is whatever the last attempt left.
397394
return AddRequestsResponse(
398395
processed_requests=[*response.processed_requests, *retry_response.processed_requests],
399396
unprocessed_requests=retry_response.unprocessed_requests,

0 commit comments

Comments
 (0)