fix: Apply SharedTimeout to post-navigation hooks (#1839)

## Summary

- Post-navigation hooks in `AbstractHttpCrawler` and `PlaywrightCrawler`
had no timeout enforcement, allowing a misbehaving hook to hang the
crawler indefinitely.
- Applied the existing shared navigation timeout budget to
post-navigation hooks, so pre-nav hooks + navigation + post-nav hooks
all share the same timeout.
- Re-keyed `_shared_navigation_timeouts` by `id(context.request)`
(stable across pipeline steps) instead of `id(context)` (changes type
between steps).

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: vdusek

src/crawlee/crawlers/_abstract_http/_abstract_http_crawler.py

@@ -119,6 +119,7 @@ def _create_static_content_crawler_pipeline(self) -> ContextPipeline[ParsedHttpC
         """Create static content crawler context pipeline with expected pipeline steps."""
         return (
             ContextPipeline()
+            .compose(self._manage_shared_navigation_timeout)
             .compose(self._execute_pre_navigation_hooks)
             .compose(self._make_http_request)
             .compose(self._execute_post_navigation_hooks)
@@ -127,26 +128,37 @@ def _create_static_content_crawler_pipeline(self) -> ContextPipeline[ParsedHttpC
             .compose(self._handle_blocked_request_by_content)
         )
 
-    async def _execute_pre_navigation_hooks(
+    async def _manage_shared_navigation_timeout(
         self, context: BasicCrawlingContext
     ) -> AsyncGenerator[BasicCrawlingContext, None]:
-        context_id = id(context)
-        self._shared_navigation_timeouts[context_id] = SharedTimeout(self._navigation_timeout)
+        """Initialize and clean up the shared navigation timeout for the current request."""
+        request_id = id(context.request)
+        self._shared_navigation_timeouts[request_id] = SharedTimeout(self._navigation_timeout)
 
         try:
-            for hook in self._pre_navigation_hooks:
-                async with self._shared_navigation_timeouts[context_id]:
-                    await hook(context)
-
             yield context
         finally:
-            self._shared_navigation_timeouts.pop(context_id, None)
+            self._shared_navigation_timeouts.pop(request_id, None)
+
+    async def _execute_pre_navigation_hooks(
+        self, context: BasicCrawlingContext
+    ) -> AsyncGenerator[BasicCrawlingContext, None]:
+        request_id = id(context.request)
+
+        for hook in self._pre_navigation_hooks:
+            async with self._shared_navigation_timeouts[request_id]:
+                await hook(context)
+
+        yield context
 
     async def _execute_post_navigation_hooks(
         self, context: HttpCrawlingContext
     ) -> AsyncGenerator[HttpCrawlingContext, None]:
+        request_id = id(context.request)
+
         for hook in self._post_navigation_hooks:
-            await hook(context)
+            async with self._shared_navigation_timeouts[request_id]:
+                await hook(context)
 
         yield context
 
@@ -262,7 +274,7 @@ async def _make_http_request(self, context: BasicCrawlingContext) -> AsyncGenera
         Yields:
             The original crawling context enhanced by HTTP response.
         """
-        async with self._shared_navigation_timeouts[id(context)] as remaining_timeout:
+        async with self._shared_navigation_timeouts[id(context.request)] as remaining_timeout:
             result = await self._http_client.crawl(
                 request=context.request,
                 session=context.session,

src/crawlee/crawlers/_playwright/_playwright_crawler.py

@@ -193,6 +193,7 @@ def __init__(
         # Compose the context pipeline with the Playwright-specific context enhancer.
         kwargs['_context_pipeline'] = (
             ContextPipeline()
+            .compose(self._manage_shared_navigation_timeout)
             .compose(self._open_page)
             .compose(self._navigate)
             .compose(self._execute_post_navigation_hooks)
@@ -216,6 +217,18 @@ def __init__(
 
         super().__init__(**kwargs)
 
+    async def _manage_shared_navigation_timeout(
+        self, context: BasicCrawlingContext
+    ) -> AsyncGenerator[BasicCrawlingContext, None]:
+        """Initialize and clean up the shared navigation timeout for the current request."""
+        request_id = id(context.request)
+        self._shared_navigation_timeouts[request_id] = SharedTimeout(self._navigation_timeout)
+
+        try:
+            yield context
+        finally:
+            self._shared_navigation_timeouts.pop(request_id, None)
+
     async def _open_page(
         self,
         context: BasicCrawlingContext,
@@ -242,21 +255,17 @@ async def _open_page(
             goto_options=GotoOptions(**self._goto_options),
         )
 
-        context_id = id(pre_navigation_context)
-        self._shared_navigation_timeouts[context_id] = SharedTimeout(self._navigation_timeout)
+        request_id = id(pre_navigation_context.request)
 
-        try:
-            # Only use the page context manager here — it sets the current page in a context variable,
-            # making it accessible to PlaywrightHttpClient in subsequent pipeline steps.
-            async with browser_page_context(crawlee_page.page):
-                for hook in self._pre_navigation_hooks:
-                    async with self._shared_navigation_timeouts[context_id]:
-                        await hook(pre_navigation_context)
-
-                # Yield should be inside the browser_page_context.
-                yield pre_navigation_context
-        finally:
-            self._shared_navigation_timeouts.pop(context_id, None)
+        # Only use the page context manager here — it sets the current page in a context variable,
+        # making it accessible to PlaywrightHttpClient in subsequent pipeline steps.
+        async with browser_page_context(crawlee_page.page):
+            for hook in self._pre_navigation_hooks:
+                async with self._shared_navigation_timeouts[request_id]:
+                    await hook(pre_navigation_context)
+
+            # Yield should be inside the browser_page_context.
+            yield pre_navigation_context
 
     def _prepare_request_interceptor(
         self,
@@ -329,7 +338,7 @@ async def _navigate(
             await context.page.route(context.request.url, route_handler)
 
         try:
-            async with self._shared_navigation_timeouts[id(context)] as remaining_timeout:
+            async with self._shared_navigation_timeouts[id(context.request)] as remaining_timeout:
                 response = await context.page.goto(
                     context.request.url, timeout=remaining_timeout.total_seconds() * 1000, **context.goto_options
                 )
@@ -496,8 +505,12 @@ async def _handle_blocked_request_by_content(
     async def _execute_post_navigation_hooks(
         self, context: PlaywrightPostNavCrawlingContext
     ) -> AsyncGenerator[PlaywrightPostNavCrawlingContext, None]:
+        request_id = id(context.request)
+
         for hook in self._post_navigation_hooks:
-            await hook(context)
+            async with self._shared_navigation_timeouts[request_id]:
+                await hook(context)
+
         yield context
 
     async def _create_crawling_context(

tests/unit/otel/test_crawler_instrumentor.py

@@ -82,11 +82,18 @@ async def test_crawler_instrumentor_capability(server_url: URL) -> None:
     assert telemetry_data[2]['context']['trace_id'] != telemetry_data[1]['context']['trace_id']
 
     # Crawler telemetry - all crawler spans will be in one trace as there is only one request in this test.
-    assert telemetry_data[3]['name'] == '_execute_pre_navigation_hooks, action'
-    assert telemetry_data[3]['attributes']['code.function.name'] == 'AbstractHttpCrawler._execute_pre_navigation_hooks'
+    assert telemetry_data[3]['name'] == '_manage_shared_navigation_timeout, action'
+    assert (
+        telemetry_data[3]['attributes']['code.function.name'] == 'AbstractHttpCrawler._manage_shared_navigation_timeout'
+    )
     assert telemetry_data[3]['attributes']['url.full'] == str(server_url)
     assert telemetry_data[3]['resource']['attributes'] == dict(resource.attributes)
 
+    assert telemetry_data[4]['name'] == '_execute_pre_navigation_hooks, action'
+    assert telemetry_data[4]['attributes']['code.function.name'] == 'AbstractHttpCrawler._execute_pre_navigation_hooks'
+    assert telemetry_data[4]['attributes']['url.full'] == str(server_url)
+    assert telemetry_data[4]['resource']['attributes'] == dict(resource.attributes)
+
     assert telemetry_data[-1]['name'] == '__run_task_function'
     assert telemetry_data[-1]['attributes']['code.function.name'] == 'BasicCrawler.__run_task_function'
     assert telemetry_data[-1]['resource']['attributes'] == dict(resource.attributes)