New client commit

Author: vdusek

pyproject.toml

@@ -43,7 +43,7 @@ dependencies = [
     "impit>=0.8.0",
     "lazy-object-proxy>=1.11.0",
     "more_itertools>=10.2.0",
-    "pydantic>=2.11.0",
+    "pydantic[email]>=2.11.0",
     "typing-extensions>=4.1.0",
     "websockets>=14.0",
     "yarl>=1.18.0",

src/apify/_actor.py

@@ -912,8 +912,7 @@ async def start(
         if run is None:
             raise RuntimeError(f'Failed to start Actor with ID "{actor_id}".')
 
-        run_dict = run.model_dump(by_alias=True)
-        return ActorRun.model_validate(run_dict)
+        return ActorRun.from_client_actor_run(run)
 
     async def abort(
         self,
@@ -951,8 +950,7 @@ async def abort(
         if run is None:
             raise RuntimeError(f'Failed to abort Actor run with ID "{run_id}".')
 
-        run_dict = run.model_dump(by_alias=True)
-        return ActorRun.model_validate(run_dict)
+        return ActorRun.from_client_actor_run(run)
 
     async def call(
         self,
@@ -1040,8 +1038,7 @@ async def call(
         if run is None:
             raise RuntimeError(f'Failed to call Actor with ID "{actor_id}".')
 
-        run_dict = run.model_dump(by_alias=True)
-        return ActorRun.model_validate(run_dict)
+        return ActorRun.from_client_actor_run(run)
 
     async def call_task(
         self,
@@ -1116,8 +1113,7 @@ async def call_task(
         if run is None:
             raise RuntimeError(f'Failed to call Task with ID "{task_id}".')
 
-        run_dict = run.model_dump(by_alias=True)
-        return ActorRun.model_validate(run_dict)
+        return ActorRun.from_client_actor_run(run)
 
     async def metamorph(
         self,
@@ -1305,8 +1301,7 @@ async def set_status_message(
                 f'Failed to set status message for Actor run with ID "{self.configuration.actor_run_id}".'
             )
 
-        run_dict = run.model_dump(by_alias=True)
-        return ActorRun.model_validate(run_dict)
+        return ActorRun.from_client_actor_run(run)
 
     async def create_proxy_configuration(
         self,

src/apify/_charging.py

@@ -357,8 +357,7 @@ async def _fetch_pricing_info(self) -> _FetchedPricingInfoDict:
             if run is None:
                 raise RuntimeError('Actor run not found')
 
-            run_dict = run.model_dump(by_alias=True)
-            actor_run = run_validator.validate_python(run_dict)
+            actor_run = ActorRun.from_client_actor_run(run)
 
             if actor_run is None:
                 raise RuntimeError('Actor run not found')

src/apify/_models.py

@@ -5,7 +5,7 @@
 
 from pydantic import BaseModel, BeforeValidator, ConfigDict, Field
 
-from apify_client._models import ActorJobStatus
+from apify_client._models import ActorJobStatus, Run
 from crawlee._utils.models import timedelta_ms
 from crawlee._utils.urls import validate_http_url
 
@@ -48,6 +48,8 @@ class WebhookStats(BaseModel):
 
     total_dispatches: Annotated[int, Field(alias='totalDispatches')]
 
+    from apify_client._models import Run
+
 
 @docs_group('Actor')
 class Webhook(BaseModel):
@@ -322,3 +324,19 @@ class ActorRun(BaseModel):
 
     metamorphs: Annotated[list[Metamorph] | None, Field(alias='metamorphs')] = None
     """List of metamorph events that occurred during the run."""
+
+    @classmethod
+    def from_client_actor_run(cls, client_actor_run: Run) -> ActorRun:
+        """Create an `ActorRun` from an Apify API client's `Run` model.
+
+        Args:
+            client_actor_run: `Run` instance from Apify API client.
+
+        Returns:
+            `ActorRun` instance with properly converted types.
+        """
+        # Dump to dict first with mode='json' to serialize special types
+        client_actor_run_dict = client_actor_run.model_dump(by_alias=True, mode='json')
+
+        # Validate and construct ActorRun from the serialized dict
+        return cls.model_validate(client_actor_run_dict)

src/apify/storage_clients/_apify/_models.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from datetime import datetime, timedelta
-from typing import Annotated
+from typing import TYPE_CHECKING, Annotated
 
 from pydantic import BaseModel, ConfigDict, Field
 
@@ -10,6 +10,9 @@
 from apify import Request
 from apify._utils import docs_group
 
+if TYPE_CHECKING:
+    from apify_client._models import LockedRequestQueueHead
+
 
 @docs_group('Storage data')
 class ApifyKeyValueStoreMetadata(KeyValueStoreMetadata):
@@ -59,6 +62,22 @@ class RequestQueueHead(BaseModel):
     items: Annotated[list[Request], Field(alias='items', default_factory=list[Request])]
     """The list of request objects retrieved from the beginning of the queue."""
 
+    @classmethod
+    def from_client_locked_head(cls, client_locked_head: LockedRequestQueueHead) -> RequestQueueHead:
+        """Create a `RequestQueueHead` from an Apify API client's `LockedRequestQueueHead` model.
+
+        Args:
+            client_locked_head: `LockedRequestQueueHead` instance from Apify API client.
+
+        Returns:
+            `RequestQueueHead` instance with properly converted types.
+        """
+        # Dump to dict with mode='json' to serialize special types like AnyUrl
+        head_dict = client_locked_head.model_dump(by_alias=True, mode='json')
+
+        # Validate and construct RequestQueueHead from the serialized dict
+        return cls.model_validate(head_dict)
+
 
 class KeyValueStoreKeyInfo(BaseModel):
     """Model for a key-value store key info.

src/apify/storage_clients/_apify/_request_queue_client.py

@@ -16,10 +16,9 @@
     from collections.abc import Sequence
 
     from apify_client._resource_clients import RequestQueueClientAsync
-    from crawlee import Request
     from crawlee.storage_clients.models import AddRequestsResponse, ProcessedRequest, RequestQueueMetadata
 
-    from apify import Configuration
+    from apify import Configuration, Request
 
 logger = getLogger(__name__)
 

src/apify/storage_clients/_apify/_request_queue_shared_client.py

@@ -11,14 +11,15 @@
 from crawlee.storage_clients.models import AddRequestsResponse, ProcessedRequest, RequestQueueMetadata
 
 from ._models import ApifyRequestQueueMetadata, CachedRequest, RequestQueueHead
-from ._utils import unique_key_to_request_id
-from apify import Request
+from ._utils import to_crawlee_request, unique_key_to_request_id
 
 if TYPE_CHECKING:
     from collections.abc import Callable, Coroutine, Sequence
 
     from apify_client._resource_clients import RequestQueueClientAsync
 
+    from apify import Request
+
 logger = getLogger(__name__)
 
 
@@ -311,8 +312,7 @@ async def _get_request_by_id(self, request_id: str) -> Request | None:
         if response is None:
             return None
 
-        response_dict = response.model_dump(by_alias=True)
-        return Request.model_validate(response_dict)
+        return to_crawlee_request(response)
 
     async def _ensure_head_is_non_empty(self) -> None:
         """Ensure that the queue head has requests if they are available in the queue."""
@@ -442,7 +442,7 @@ async def _list_head(
         self.metadata.had_multiple_clients = locked_queue_head.had_multiple_clients
 
         for request_data in locked_queue_head.items:
-            request = Request.model_validate(request_data.model_dump(by_alias=True))
+            request = to_crawlee_request(request_data)
             request_id = request_data.id
 
             # Skip requests without ID or unique key
@@ -473,8 +473,7 @@ async def _list_head(
             # After adding new requests to the forefront, any existing leftover locked request is kept in the end.
             self._queue_head.append(leftover_id)
 
-        list_and_lost_dict = locked_queue_head.model_dump(by_alias=True)
-        return RequestQueueHead.model_validate(list_and_lost_dict)
+        return RequestQueueHead.from_client_locked_head(locked_queue_head)
 
     def _cache_request(
         self,

src/apify/storage_clients/_apify/_request_queue_single_client.py

@@ -9,14 +9,15 @@
 
 from crawlee.storage_clients.models import AddRequestsResponse, ProcessedRequest, RequestQueueMetadata
 
-from ._utils import unique_key_to_request_id
-from apify import Request
+from ._utils import to_crawlee_request, unique_key_to_request_id
 
 if TYPE_CHECKING:
     from collections.abc import Sequence
 
     from apify_client._resource_clients import RequestQueueClientAsync
 
+    from apify import Request
+
 logger = getLogger(__name__)
 
 
@@ -294,7 +295,7 @@ async def _list_head(self) -> None:
 
         # Update the cached data
         for request_data in response.items:
-            request = Request.model_validate(request_data.model_dump(by_alias=True))
+            request = to_crawlee_request(request_data)
             request_id = request_data.id
 
             if request_id in self._requests_in_progress:
@@ -331,8 +332,7 @@ async def _get_request_by_id(self, id: str) -> Request | None:
         if response is None:
             return None
 
-        response_dict = response.model_dump(by_alias=True)
-        request = Request.model_validate(response_dict)
+        request = to_crawlee_request(response)
 
         # Updated local caches
         if id in self._requests_in_progress:
@@ -383,7 +383,7 @@ async def _init_caches(self) -> None:
         """
         response = await self._api_client.list_requests(limit=10_000)
         for request_data in response.items:
-            request = Request.model_validate(request_data.model_dump(by_alias=True))
+            request = to_crawlee_request(request_data)
             request_id = request_data.id
 
             if request.was_already_handled:

src/apify/storage_clients/_apify/_utils.py

@@ -7,7 +7,12 @@
 
 from crawlee._utils.crypto import compute_short_hash
 
+from apify import Request
+
 if TYPE_CHECKING:
+    from apify_client._models import HeadRequest, LockedHeadRequest
+    from apify_client._models import Request as ClientRequest
+
     from apify import Configuration
 
 
@@ -39,3 +44,19 @@ def hash_api_base_url_and_token(configuration: Configuration) -> str:
     if configuration.api_public_base_url is None or configuration.token is None:
         raise ValueError("'Configuration.api_public_base_url' and 'Configuration.token' must be set.")
     return compute_short_hash(f'{configuration.api_public_base_url}{configuration.token}'.encode())
+
+
+def to_crawlee_request(client_request: ClientRequest | HeadRequest | LockedHeadRequest) -> Request:
+    """Convert an Apify API client's `Request` model to a Crawlee's `Request` model.
+
+    Args:
+        client_request: Request instances from Apify API client.
+
+    Returns:
+        `Request` instance from Crawlee with properly converted types.
+    """
+    # Dump to dict with mode='json' to serialize special types like AnyUrl
+    request_dict = client_request.model_dump(by_alias=True, mode='json')
+
+    # Validate and construct Crawlee Request from the serialized dict
+    return Request.model_validate(request_dict)

tests/e2e/conftest.py

@@ -391,11 +391,10 @@ async def _run_actor(
         assert call_result is not None, 'Failed to start Actor run: missing run ID in the response.'
 
         run_client = apify_client_async.run(call_result.id)
-        actor_run = await run_client.wait_for_finish(wait_secs=600)
+        client_actor_run = await run_client.wait_for_finish(wait_secs=600)
 
-        assert actor_run is not None, 'Actor run did not finish successfully within the expected time.'
+        assert client_actor_run is not None, 'Actor run did not finish successfully within the expected time.'
 
-        actor_run_dict = actor_run.model_dump(by_alias=True)
-        return ActorRun.model_validate(actor_run_dict)
+        return ActorRun.from_client_actor_run(client_actor_run)
 
     return _run_actor