Add debug to autoscaled pool. Is the problem there?

Author: Pijukatel

src/crawlee/_autoscaling/autoscaled_pool.py

@@ -124,7 +124,9 @@ async def run(self) -> None:
 
         try:
             await run.result
+            logger.info('Finished naturally')
         except AbortError:
+            logger.info('AbortError')
             orchestrator.cancel()
             for task in run.worker_tasks:
                 if not task.done():
@@ -245,16 +247,16 @@ async def _worker_task_orchestrator(self, run: _AutoscaledPoolRun) -> None:
                     await asyncio.wait_for(run.worker_tasks_updated.wait(), timeout=0.5)
         finally:
             if finished:
-                logger.debug('`is_finished_function` reports that we are finished')
+                logger.info('`is_finished_function` reports that we are finished')
             elif run.result.done() and run.result.exception() is not None:
-                logger.debug('Unhandled exception in `run_task_function`')
+                logger.info('Unhandled exception in `run_task_function`')
 
             if run.worker_tasks:
-                logger.debug('Terminating - waiting for tasks to complete')
+                logger.info('Terminating - waiting for tasks to complete')
                 await asyncio.wait(run.worker_tasks, return_when=asyncio.ALL_COMPLETED)
-                logger.debug('Worker tasks finished')
+                logger.info('Worker tasks finished')
             else:
-                logger.debug('Terminating - no running tasks to wait for')
+                logger.info('Terminating - no running tasks to wait for')
 
             if not run.result.done():
                 run.result.set_result(object())

src/crawlee/crawlers/_basic/_basic_crawler.py

@@ -1347,6 +1347,7 @@ async def __is_finished_function(self) -> bool:
 
         if self._abort_on_error and self._failed:
             self._failed = False
+            self.log.info('_abort_on_error')
             return True
 
         if self._keep_alive:

tests/unit/crawlers/_basic/test_basic_crawler.py

@@ -1301,6 +1301,8 @@ async def test_timeout_in_handler(sleep_type: str) -> None:
     )
     crawler.log.setLevel(logging.DEBUG)
     logging.getLogger('crawlee.storage_clients._file_system._request_queue_client').setLevel(logging.DEBUG)
+    logging.getLogger('crawlee._autoscaling.autoscaled_pool').setLevel(logging.INFO)
+
 
     mocked_handler_before_sleep = Mock()
     mocked_handler_after_sleep = Mock()