Resolve some type ignores

Author: vdusek

.gitignore

@@ -1,8 +1,8 @@
 # Cache
 __pycache__
-.uv_cache
 .pytest_cache
 .ruff_cache
+.ty_cache
 .uv-cache
 
 # Virtual envs

src/crawlee/_utils/recurring_task.py

@@ -25,7 +25,11 @@ class RecurringTask:
     """
 
     def __init__(self, func: Callable, delay: timedelta) -> None:
-        logger.debug(f'Calling RecurringTask.__init__(func={func.__name__}, delay={delay})...')  # ty: ignore[unresolved-attribute]
+        logger.debug(
+            'Calling RecurringTask.__init__(func={%s}, delay={%s})...',
+            func.__name__ if hasattr(func, '__name__') else func.__class__.__name__,
+            delay,
+        )
         self.func = func
         self.delay = delay
         self.task: asyncio.Task | None = None
@@ -55,7 +59,11 @@ async def _wrapper(self) -> None:
 
     def start(self) -> None:
         """Start the recurring task execution."""
-        self.task = asyncio.create_task(self._wrapper(), name=f'Task-recurring-{self.func.__name__}')  # ty: ignore[possibly-missing-attribute]
+        name = self.func.__name__ if hasattr(self.func, '__name__') else self.func.__class__.__name__
+        self.task = asyncio.create_task(
+            self._wrapper(),
+            name=f'Task-recurring-{name}',
+        )
 
     async def stop(self) -> None:
         """Stop the recurring task execution."""

src/crawlee/_utils/sitemap.py

@@ -430,10 +430,17 @@ async def parse_sitemap(
     up to the specified maximum depth.
     """
     # Set default options
-    options = options or {}  # ty: ignore[invalid-assignment]
-    emit_nested_sitemaps = options.get('emit_nested_sitemaps', False)  # ty: ignore[possibly-missing-attribute]
-    max_depth = options.get('max_depth', float('inf'))  # ty: ignore[possibly-missing-attribute]
-    sitemap_retries = options.get('sitemap_retries', 3)  # ty: ignore[possibly-missing-attribute]
+    default_timeout = timedelta(seconds=30)
+    if options:
+        emit_nested_sitemaps = options['emit_nested_sitemaps']
+        max_depth = options['max_depth']
+        sitemap_retries = options['sitemap_retries']
+        timeout = options.get('timeout', default_timeout)
+    else:
+        emit_nested_sitemaps = False
+        max_depth = float('inf')
+        sitemap_retries = 3
+        timeout = default_timeout
 
     # Setup working state
     sources = list(initial_sources)
@@ -472,7 +479,7 @@ async def parse_sitemap(
                 sitemap_retries,
                 emit_nested_sitemaps=emit_nested_sitemaps,
                 proxy_info=proxy_info,
-                timeout=options.get('timeout', timedelta(seconds=30)),  # ty: ignore[possibly-missing-attribute]
+                timeout=timeout,
             ):
                 yield result
         else:

src/crawlee/browsers/_browser_pool.py

@@ -138,11 +138,11 @@ def with_default_plugin(
             kwargs: Additional arguments for default constructor.
         """
         plugin_options: dict = defaultdict(dict)
-        plugin_options['browser_launch_options'] = browser_launch_options or {}
+        plugin_options['browser_launch_options'] = dict(browser_launch_options) if browser_launch_options else {}
         plugin_options['browser_new_context_options'] = browser_new_context_options or {}
 
         if headless is not None:
-            plugin_options['browser_launch_options']['headless'] = headless  # ty: ignore[invalid-assignment]
+            plugin_options['browser_launch_options']['headless'] = headless
 
         if use_incognito_pages is not None:
             plugin_options['use_incognito_pages'] = use_incognito_pages

src/crawlee/browsers/_playwright_browser.py

@@ -78,7 +78,8 @@ async def new_context(self, **context_options: Any) -> BrowserContext:
 
     async def _delete_temp_dir(self, _: BrowserContext | None) -> None:
         if self._temp_dir and self._temp_dir.exists():
-            await asyncio.to_thread(lambda: shutil.rmtree(self._temp_dir, ignore_errors=True))  # ty: ignore[invalid-argument-type]
+            temp_dir = self._temp_dir
+            await asyncio.to_thread(lambda: shutil.rmtree(temp_dir, ignore_errors=True))
 
     @override
     async def close(self, **kwargs: Any) -> None:

src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py

@@ -27,31 +27,23 @@
 )
 from crawlee.crawlers._beautifulsoup._beautifulsoup_parser import BeautifulSoupParser
 from crawlee.crawlers._parsel._parsel_parser import ParselParser
+from crawlee.crawlers._playwright._playwright_crawler import _PlaywrightCrawlerAdditionalOptions
 from crawlee.statistics import Statistics, StatisticsState
 
-from ._adaptive_playwright_crawler_statistics import (
-    AdaptivePlaywrightCrawlerStatisticState,
-)
+from ._adaptive_playwright_crawler_statistics import AdaptivePlaywrightCrawlerStatisticState
 from ._adaptive_playwright_crawling_context import (
     AdaptivePlaywrightCrawlingContext,
     AdaptivePlaywrightPreNavCrawlingContext,
 )
-from ._rendering_type_predictor import (
-    DefaultRenderingTypePredictor,
-    RenderingType,
-    RenderingTypePredictor,
-)
-from ._result_comparator import (
-    create_default_comparator,
-)
+from ._rendering_type_predictor import DefaultRenderingTypePredictor, RenderingType, RenderingTypePredictor
+from ._result_comparator import create_default_comparator
 
 if TYPE_CHECKING:
     from types import TracebackType
 
     from typing_extensions import Unpack
 
     from crawlee.crawlers._basic._basic_crawler import _BasicCrawlerOptions
-    from crawlee.crawlers._playwright._playwright_crawler import _PlaywrightCrawlerAdditionalOptions
 
 
 TStaticParseResult = TypeVar('TStaticParseResult')
@@ -162,7 +154,7 @@ def __init__(
         super().__init__(statistics=adaptive_statistics, **kwargs)
 
         # Sub crawlers related.
-        playwright_crawler_specific_kwargs = playwright_crawler_specific_kwargs or {}  # ty: ignore[invalid-assignment]
+        playwright_crawler_specific_kwargs = playwright_crawler_specific_kwargs or _PlaywrightCrawlerAdditionalOptions()
 
         # Each sub crawler will use custom logger .
         static_logger = getLogger('Subcrawler_static')
@@ -183,7 +175,7 @@ def __init__(
         )
         playwright_crawler = PlaywrightCrawler(
             statistics=_NonPersistentStatistics(),
-            **playwright_crawler_specific_kwargs,  # ty: ignore[invalid-argument-type]
+            **playwright_crawler_specific_kwargs,
             **basic_crawler_kwargs_for_pw_crawler,
         )
 

src/crawlee/crawlers/_playwright/_playwright_crawler.py

@@ -14,10 +14,7 @@
 
 from crawlee import service_locator
 from crawlee._request import Request, RequestOptions, RequestState
-from crawlee._types import (
-    BasicCrawlingContext,
-    ConcurrencySettings,
-)
+from crawlee._types import BasicCrawlingContext, ConcurrencySettings
 from crawlee._utils.blocked import RETRY_CSS_SELECTORS
 from crawlee._utils.docs import docs_group
 from crawlee._utils.robots import RobotsTxtFile
@@ -177,13 +174,12 @@ def __init__(
         # If browser_pool is not provided, create a new instance of BrowserPool with specified arguments.
         else:
             if fingerprint_generator == 'default':
-                if not browser_type:
-                    generator_browser_type = None
-                else:
-                    generator_browser_type = [fingerprint_browser_type_from_playwright_browser_type(browser_type)]
+                generator_browser_type: list[Literal['chrome', 'firefox', 'safari', 'edge']] | None = (
+                    [fingerprint_browser_type_from_playwright_browser_type(browser_type)] if browser_type else None
+                )
 
                 fingerprint_generator = DefaultFingerprintGenerator(
-                    header_options=HeaderGeneratorOptions(browsers=generator_browser_type)  # ty: ignore[invalid-argument-type]
+                    header_options=HeaderGeneratorOptions(browsers=generator_browser_type)
                 )
 
             browser_pool = BrowserPool.with_default_plugin(
@@ -516,6 +512,7 @@ async def _get_cookies(self, page: Page) -> list[PlaywrightCookieParam]:
 
     async def _update_cookies(self, page: Page, cookies: list[PlaywrightCookieParam]) -> None:
         """Update the cookies in the page context."""
+        # False positive ty error, see https://github.com/astral-sh/ty/issues/1493.
         await page.context.add_cookies([{**cookie} for cookie in cookies])  # ty: ignore[invalid-argument-type]
 
     async def _find_txt_file_for_url(self, url: str) -> RobotsTxtFile:

src/crawlee/events/_event_manager.py

@@ -178,7 +178,8 @@ async def listener_wrapper(event_data: EventData) -> None:
                 else asyncio.to_thread(cast('Callable[..., None]', listener), *bound_args.args, **bound_args.kwargs)
             )
 
-            listener_task = asyncio.create_task(coro, name=f'Task-{event.value}-{listener.__name__}')  # ty: ignore[invalid-argument-type, unresolved-attribute]
+            listener_name = listener.__name__ if hasattr(listener, '__name__') else listener.__class__.__name__
+            listener_task = asyncio.create_task(coro, name=f'Task-{event.value}-{listener_name}')
             self._listener_tasks.add(listener_task)
 
             try:
@@ -189,7 +190,12 @@ async def listener_wrapper(event_data: EventData) -> None:
                 # We need to swallow the exception and just log it here, otherwise it could break the event emitter
                 logger.exception(
                     'Exception in the event listener',
-                    extra={'event_name': event.value, 'listener_name': listener.__name__},  # ty: ignore[unresolved-attribute]
+                    extra={
+                        'event_name': event.value,
+                        'listener_name': listener.__name__
+                        if hasattr(listener, '__name__')
+                        else listener.__class__.__name__,
+                    },
                 )
             finally:
                 logger.debug('EventManager.on.listener_wrapper(): Removing listener task from the set...')

src/crawlee/http_clients/_curl_impersonate.py

@@ -2,7 +2,8 @@
 
 import asyncio
 from contextlib import asynccontextmanager
-from typing import TYPE_CHECKING, Any
+from http.cookiejar import Cookie
+from typing import TYPE_CHECKING, Any, cast
 
 from curl_cffi import CurlInfo
 from curl_cffi.const import CurlHttpVersion
@@ -15,7 +16,7 @@
 from curl_cffi.requests.impersonate import DEFAULT_CHROME as CURL_DEFAULT_CHROME
 from typing_extensions import override
 
-from crawlee._types import HttpHeaders, HttpPayload
+from crawlee._types import HttpHeaders, HttpMethod, HttpPayload
 from crawlee._utils.blocked import ROTATE_PROXY_ERRORS
 from crawlee._utils.docs import docs_group
 from crawlee.errors import ProxyError
@@ -24,11 +25,11 @@
 if TYPE_CHECKING:
     from collections.abc import AsyncGenerator
     from datetime import timedelta
-    from http.cookiejar import Cookie
 
     from curl_cffi import Curl
     from curl_cffi.requests import Request as CurlRequest
     from curl_cffi.requests import Response
+    from curl_cffi.requests.session import HttpMethod as CurlHttpMethod
 
     from crawlee import Request
     from crawlee._types import HttpMethod
@@ -90,13 +91,15 @@ def headers(self) -> HttpHeaders:
     async def read(self) -> bytes:
         if self._response.astream_task:
             raise RuntimeError('Use `read_stream` to read the body of the Response received from the `stream` method')
+
         return self._response.content
 
     async def read_stream(self) -> AsyncGenerator[bytes, None]:
-        if not self._response.astream_task or self._response.astream_task.done():  # ty: ignore[possibly-missing-attribute]
-            raise RuntimeError(
-                'Cannot read stream: either already consumed or Response not obtained from `stream` method'
-            )
+        if not self._response.astream_task:
+            raise RuntimeError('Cannot read stream, Response not obtained from `stream` method.')
+
+        if isinstance(self._response.astream_task, asyncio.Future) and self._response.astream_task.done():
+            raise RuntimeError('Cannot read stream, it was already consumed.')
 
         async for chunk in self._response.aiter_content():
             yield chunk
@@ -156,7 +159,7 @@ async def crawl(
         try:
             response = await client.request(
                 url=request.url,
-                method=request.method.upper(),  # ty: ignore[invalid-argument-type]
+                method=self._convert_method(request.method),
                 headers=request.headers,
                 data=request.payload,
                 cookies=session.cookies.jar if session else None,
@@ -203,7 +206,7 @@ async def send_request(
         try:
             response = await client.request(
                 url=url,
-                method=method.upper(),  # ty: ignore[invalid-argument-type]
+                method=self._convert_method(method),
                 headers=dict(headers) if headers else None,
                 data=payload,
                 cookies=session.cookies.jar if session else None,
@@ -244,7 +247,7 @@ async def stream(
         try:
             response = await client.request(
                 url=url,
-                method=method.upper(),  # ty: ignore[invalid-argument-type]
+                method=self._convert_method(method),
                 headers=dict(headers) if headers else None,
                 data=payload,
                 cookies=session.cookies.jar if session else None,
@@ -291,6 +294,40 @@ def _get_client(self, proxy_url: str | None) -> AsyncSession:
 
         return self._client_by_proxy_url[proxy_url]
 
+    def _convert_method(self, method: HttpMethod) -> CurlHttpMethod:
+        """Convert from Crawlee HTTP method to curl-cffi HTTP method.
+
+        Args:
+            method: Crawlee HTTP method.
+
+        Returns:
+            Corresponding curl-cffi HTTP method.
+
+        Raises:
+            ValueError: If the provided HTTP method is not supported.
+        """
+        method_upper = method.upper()  # curl-cffi requires uppercase methods
+
+        match method_upper:
+            case 'GET':
+                return 'GET'
+            case 'POST':
+                return 'POST'
+            case 'PUT':
+                return 'PUT'
+            case 'DELETE':
+                return 'DELETE'
+            case 'OPTIONS':
+                return 'OPTIONS'
+            case 'HEAD':
+                return 'HEAD'
+            case 'TRACE':
+                return 'TRACE'
+            case 'PATCH':
+                return 'PATCH'
+            case _:
+                raise ValueError(f'HTTP method {method} is not supported in {self.__class__.__name__}.')
+
     @staticmethod
     def _is_proxy_error(error: CurlRequestError) -> bool:
         """Determine whether the given error is related to a proxy issue.
@@ -308,11 +345,16 @@ def _is_proxy_error(error: CurlRequestError) -> bool:
 
     @staticmethod
     def _get_cookies(curl: Curl) -> list[Cookie]:
-        cookies: list[Cookie] = []
-        for curl_cookie in curl.getinfo(CurlInfo.COOKIELIST):  # ty: ignore[not-iterable]
-            curl_morsel = CurlMorsel.from_curl_format(curl_cookie)  # ty: ignore[invalid-argument-type]
+        cookies = list[Cookie]()
+
+        # Implementation of getinfo always returns list[bytes] for CurlInfo.COOKIELIST.
+        cookie_list = cast('list[bytes]', curl.getinfo(CurlInfo.COOKIELIST))
+
+        for curl_cookie in cookie_list:
+            curl_morsel = CurlMorsel.from_curl_format(curl_cookie)
             cookie = curl_morsel.to_cookiejar_cookie()
             cookies.append(cookie)
+
         return cookies
 
     async def cleanup(self) -> None:

src/crawlee/storage_clients/_redis/_dataset_client.py

@@ -179,12 +179,14 @@ async def get_data(
             case (True, int(), None):
                 json_path += f'[:-{offset}]'
             case (True, int(), int()):
+                # ty lacks support for advanced pattern matching, see https://github.com/astral-sh/ty/issues/887.
                 json_path += f'[-{offset + limit}:-{offset}]'  # ty: ignore[unsupported-operator]
             case (False, 0, int()):
                 json_path += f'[:{limit}]'
             case (False, int(), None):
                 json_path += f'[{offset}:]'
             case (False, int(), int()):
+                # ty lacks support for advanced pattern matching, see https://github.com/astral-sh/ty/issues/887.
                 json_path += f'[{offset}:{offset + limit}]'  # ty: ignore[unsupported-operator]
 
         if json_path == '$':