Skip to content

Commit e4c924d

Browse files
vdusekclaude
andcommitted
test: remove flaky integration tests for RQ finished state and producer-consumer
- test_finished_state_accuracy: redundant with test_is_finished_marks_request_as_handled, flaky due to eventual consistency in shared mode - test_operations_performance_pattern: concurrent producer-consumer triggers IndexError in Crawlee's add_batch_of_requests Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 4b51fc1 commit e4c924d

File tree

1 file changed

+0
-99
lines changed

1 file changed

+0
-99
lines changed

tests/integration/test_request_queue.py

Lines changed: 0 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -778,105 +778,6 @@ async def test_request_ordering_with_mixed_operations(request_queue_apify: Reque
778778
Actor.log.info('Request ordering verified successfully')
779779

780780

781-
async def test_finished_state_accuracy(request_queue_apify: RequestQueue) -> None:
782-
"""Test accuracy of is_finished() method in various scenarios."""
783-
784-
rq = request_queue_apify
785-
Actor.log.info('Request queue opened')
786-
787-
# Initially should be finished
788-
initial_finished = await rq.is_finished()
789-
Actor.log.info(f'Initial finished state: {initial_finished}')
790-
assert initial_finished is True, f'initial_finished={initial_finished}'
791-
792-
# Add requests - should not be finished
793-
await rq.add_request('https://example.com/test1')
794-
await rq.add_request('https://example.com/test2')
795-
after_add_finished = await rq.is_finished()
796-
Actor.log.info(f'Finished state after adding requests: {after_add_finished}')
797-
assert after_add_finished is False, f'after_add_finished={after_add_finished}'
798-
799-
# Fetch but don't handle - should not be finished
800-
request1 = await rq.fetch_next_request()
801-
assert request1 is not None, f'request1={request1}'
802-
after_fetch_finished = await rq.is_finished()
803-
Actor.log.info(f'Finished state after fetch (not handled): {after_fetch_finished}')
804-
assert after_fetch_finished is False, f'after_fetch_finished={after_fetch_finished}'
805-
806-
# Reclaim request - should still not be finished
807-
await rq.reclaim_request(request1)
808-
after_reclaim_finished = await rq.is_finished()
809-
Actor.log.info(f'Finished state after reclaim: {after_reclaim_finished}')
810-
assert after_reclaim_finished is False, f'after_reclaim_finished={after_reclaim_finished}'
811-
812-
# Handle all requests - should be finished
813-
processed_count = 0
814-
while next_request := await rq.fetch_next_request():
815-
processed_count += 1
816-
await rq.mark_request_as_handled(next_request)
817-
818-
Actor.log.info(f'Processed {processed_count} requests')
819-
final_finished = await rq.is_finished()
820-
assert final_finished is True, f'final_finished={final_finished}'
821-
822-
823-
async def test_operations_performance_pattern(request_queue_apify: RequestQueue) -> None:
824-
"""Test a common performance pattern: producer-consumer."""
825-
Actor.log.info('Request queue opened')
826-
rq = request_queue_apify
827-
828-
# Producer: Add requests in background
829-
async def producer() -> None:
830-
for i in range(20):
831-
await rq.add_request(f'https://example.com/item/{i}')
832-
if i % 5 == 0: # Add some delay to simulate real production
833-
await asyncio.sleep(0.01)
834-
Actor.log.info('Producer finished adding all 20 requests')
835-
836-
# Consumer: Process requests as they become available
837-
async def consumer() -> int:
838-
processed = 0
839-
consecutive_empty = 0
840-
max_empty_attempts = 5
841-
842-
while consecutive_empty < max_empty_attempts:
843-
request = await rq.fetch_next_request()
844-
if request is None:
845-
consecutive_empty += 1
846-
await asyncio.sleep(0.01) # Brief wait for more requests
847-
continue
848-
849-
consecutive_empty = 0
850-
await rq.mark_request_as_handled(request)
851-
processed += 1
852-
853-
Actor.log.info(f'Consumer finished initial processing, processed {processed} requests')
854-
return processed
855-
856-
# Run producer and consumer concurrently
857-
producer_task = asyncio.create_task(producer())
858-
consumer_task = asyncio.create_task(consumer())
859-
860-
# Wait for both to complete
861-
await producer_task
862-
processed_count = await consumer_task
863-
Actor.log.info(f'Concurrent phase completed, processed {processed_count} requests')
864-
865-
# Process any remaining requests
866-
remaining_count = 0
867-
while next_request := await rq.fetch_next_request():
868-
await rq.mark_request_as_handled(next_request)
869-
processed_count += 1
870-
remaining_count += 1
871-
872-
Actor.log.info(f'Processed {remaining_count} remaining requests')
873-
Actor.log.info(f'Total processed: {processed_count} requests')
874-
assert processed_count == 20, f'processed_count={processed_count}'
875-
876-
final_finished = await rq.is_finished()
877-
assert final_finished is True, f'final_finished={final_finished}'
878-
879-
880781
async def test_request_queue_enhanced_metadata(
881782
request_queue_apify: RequestQueue,
882783
apify_client_async: ApifyClientAsync,

0 commit comments

Comments
 (0)