Skip to content

Commit ee09260

Browse files
authored
fix: Reset all counter in metadata with purge for RequestQueue (#1686)
### Description - Reset all counter in metadata with `purge` for `RequestQueue`. This will be more consistent with JS. Also, saving the `total_request_count` and `handled_request_count` counters causes confusion among users, especially when this happens with the `default` queue. ### Issues - Closes: #1682
1 parent c9c87a6 commit ee09260

File tree

5 files changed

+13
-5
lines changed

5 files changed

+13
-5
lines changed

src/crawlee/storage_clients/_file_system/_request_queue_client.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -312,6 +312,8 @@ async def purge(self) -> None:
312312
update_modified_at=True,
313313
update_accessed_at=True,
314314
new_pending_request_count=0,
315+
new_handled_request_count=0,
316+
new_total_request_count=0,
315317
)
316318

317319
# Invalidate is_empty cache.

src/crawlee/storage_clients/_memory/_request_queue_client.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,8 @@ async def purge(self) -> None:
133133
update_modified_at=True,
134134
update_accessed_at=True,
135135
new_pending_request_count=0,
136+
new_handled_request_count=0,
137+
new_total_request_count=0,
136138
)
137139

138140
@override

src/crawlee/storage_clients/_redis/_request_queue_client.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,8 @@ async def purge(self) -> None:
237237
update_accessed_at=True,
238238
update_modified_at=True,
239239
new_pending_request_count=0,
240+
new_handled_request_count=0,
241+
new_total_request_count=0,
240242
),
241243
)
242244

src/crawlee/storage_clients/_sql/_request_queue_client.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,8 @@ async def purge(self) -> None:
179179
update_accessed_at=True,
180180
update_modified_at=True,
181181
new_pending_request_count=0,
182+
new_handled_request_count=0,
183+
new_total_request_count=0,
182184
force=True,
183185
)
184186
)

tests/unit/storages/test_request_queue.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -629,7 +629,7 @@ async def test_purge(
629629

630630
# Queue should be empty now
631631
metadata = await rq.get_metadata()
632-
assert metadata.total_request_count == 3
632+
assert metadata.total_request_count == 0
633633
assert metadata.pending_request_count == 0
634634
assert metadata.handled_request_count == 0
635635
assert await rq.is_empty() is True
@@ -1144,12 +1144,12 @@ async def test_purge_on_start_enabled(storage_client: StorageClient) -> None:
11441144
assert alias_metadata_after.pending_request_count == 0
11451145
assert named_metadata_after.pending_request_count == 2
11461146

1147-
assert default_metadata_after.handled_request_count == 1
1148-
assert alias_metadata_after.handled_request_count == 1
1147+
assert default_metadata_after.handled_request_count == 0
1148+
assert alias_metadata_after.handled_request_count == 0
11491149
assert named_metadata_after.handled_request_count == 1
11501150

1151-
assert default_metadata_after.total_request_count == 3
1152-
assert alias_metadata_after.total_request_count == 3
1151+
assert default_metadata_after.total_request_count == 0
1152+
assert alias_metadata_after.total_request_count == 0
11531153
assert named_metadata_after.total_request_count == 3
11541154

11551155
# Clean up

0 commit comments

Comments
 (0)