Refactor initialization of storages & memory resource clients (#143)

### Description

- Refactor initialization of storages & memory resource clients.
- `BaseStorage` and `BaseResourceClients` are only ABC with abstract
methods; all the creation-related code is moved to a separate module.
- I created `_creation_management.py` helper modules in both `storages/`
and `memory_storage_client/`.
- I had to move `crawlee/storages/models.py` to `crawlee/models.py`
because of the import loops, I merged it with `request.py`. I am open to
other ideas in this regard.

### Related issues

- #85

### Testing

- Tests pass

Author: vdusek

src/crawlee/_utils/env_vars.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import os
-from enum import Enum
 from typing import Any, Literal, get_args
 
 from crawlee._utils.data_processing import (
@@ -12,15 +11,6 @@
     maybe_parse_int,
 )
 
-
-class CrawleeEnvVars(str, Enum):
-    """Enum for the environment variables used by Crawlee."""
-
-    LOCAL_STORAGE_DIR = 'CRAWLEE_LOCAL_STORAGE_DIR'
-    PERSIST_STORAGE = 'CRAWLEE_PERSIST_STORAGE'
-    PURGE_ON_START = 'CRAWLEE_PURGE_ON_START'
-
-
 INTEGER_ENV_VARS_TYPE = Literal[None]
 
 INTEGER_ENV_VARS: list[INTEGER_ENV_VARS_TYPE] = list(get_args(INTEGER_ENV_VARS_TYPE))

src/crawlee/_utils/file.py

@@ -5,15 +5,13 @@
 import io
 import json
 import mimetypes
-import os
 import re
 from enum import Enum
 from typing import Any
 
-import aiofiles
 import aioshutil
 from aiofiles import ospath
-from aiofiles.os import makedirs, remove, rename
+from aiofiles.os import remove, rename
 
 
 class ContentType(Enum):
@@ -108,29 +106,3 @@ async def json_dumps(obj: Any) -> str:
         A string containing the JSON representation of the input object.
     """
     return await asyncio.to_thread(json.dumps, obj, ensure_ascii=False, indent=2, default=str)
-
-
-async def persist_metadata_if_enabled(*, data: dict, entity_directory: str, write_metadata: bool) -> None:
-    """Updates or writes metadata to a specified directory.
-
-    The function writes a given metadata dictionary to a JSON file within a specified directory.
-    The writing process is skipped if `write_metadata` is False. Before writing, it ensures that
-    the target directory exists, creating it if necessary.
-
-    Args:
-        data: A dictionary containing metadata to be written.
-        entity_directory: The directory path where the metadata file should be stored.
-        write_metadata: A boolean flag indicating whether the metadata should be written to file.
-    """
-    # Skip metadata write; ensure directory exists first
-    if not write_metadata:
-        return
-
-    # Ensure the directory for the entity exists
-    await makedirs(entity_directory, exist_ok=True)
-
-    # Write the metadata to the file
-    file_path = os.path.join(entity_directory, '__metadata__.json')
-    async with aiofiles.open(file_path, mode='wb') as f:
-        s = await json_dumps(data)
-        await f.write(s.encode('utf-8'))

src/crawlee/base_storage_client/base_dataset_client.py

@@ -4,7 +4,7 @@
 from typing import TYPE_CHECKING, AsyncIterator
 
 if TYPE_CHECKING:
-    from crawlee.storages.models import DatasetItemsListPage, DatasetMetadata
+    from crawlee.models import DatasetItemsListPage, DatasetMetadata
     from crawlee.types import JSONSerializable
 
 
@@ -128,16 +128,16 @@ async def get_items_as_bytes(
         item_format: str = 'json',
         offset: int | None = None,
         limit: int | None = None,
-        desc: bool | None = None,
-        clean: bool | None = None,
-        bom: bool | None = None,
+        desc: bool = False,
+        clean: bool = False,
+        bom: bool = False,
         delimiter: str | None = None,
         fields: list[str] | None = None,
         omit: list[str] | None = None,
         unwind: str | None = None,
-        skip_empty: bool | None = None,
-        skip_header_row: bool | None = None,
-        skip_hidden: bool | None = None,
+        skip_empty: bool = False,
+        skip_header_row: bool = False,
+        skip_hidden: bool = False,
         xml_root: str | None = None,
         xml_row: str | None = None,
         flatten: list[str] | None = None,
@@ -173,16 +173,16 @@ async def stream_items(
         item_format: str = 'json',
         offset: int | None = None,
         limit: int | None = None,
-        desc: bool | None = None,
-        clean: bool | None = None,
-        bom: bool | None = None,
+        desc: bool = False,
+        clean: bool = False,
+        bom: bool = False,
         delimiter: str | None = None,
         fields: list[str] | None = None,
         omit: list[str] | None = None,
         unwind: str | None = None,
-        skip_empty: bool | None = None,
-        skip_header_row: bool | None = None,
-        skip_hidden: bool | None = None,
+        skip_empty: bool = False,
+        skip_header_row: bool = False,
+        skip_hidden: bool = False,
         xml_root: str | None = None,
         xml_row: str | None = None,
     ) -> AsyncIterator[dict]:

src/crawlee/base_storage_client/base_dataset_collection_client.py

@@ -4,7 +4,7 @@
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from crawlee.storages.models import DatasetListPage, DatasetMetadata
+    from crawlee.models import DatasetListPage, DatasetMetadata
 
 
 class BaseDatasetCollectionClient(ABC):
@@ -40,10 +40,10 @@ async def get_or_create(
     async def list(
         self,
         *,
-        unnamed: bool | None = None,
+        unnamed: bool = False,
         limit: int | None = None,
         offset: int | None = None,
-        desc: bool | None = None,
+        desc: bool = False,
     ) -> DatasetListPage:
         """List the available datasets.
 

src/crawlee/base_storage_client/base_key_value_store_client.py

@@ -4,7 +4,7 @@
 from typing import TYPE_CHECKING, Any, AsyncIterator
 
 if TYPE_CHECKING:
-    from crawlee.storages.models import KeyValueStoreListKeysPage, KeyValueStoreMetadata, KeyValueStoreRecord
+    from crawlee.models import KeyValueStoreListKeysPage, KeyValueStoreMetadata, KeyValueStoreRecord
 
 
 class BaseKeyValueStoreClient(ABC):

src/crawlee/base_storage_client/base_key_value_store_collection_client.py

@@ -4,7 +4,7 @@
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from crawlee.storages.models import KeyValueStoreListPage, KeyValueStoreMetadata
+    from crawlee.models import KeyValueStoreListPage, KeyValueStoreMetadata
 
 
 class BaseKeyValueStoreCollectionClient(ABC):
@@ -40,10 +40,10 @@ async def get_or_create(
     async def list(
         self,
         *,
-        unnamed: bool | None = None,
+        unnamed: bool = False,
         limit: int | None = None,
         offset: int | None = None,
-        desc: bool | None = None,
+        desc: bool = False,
     ) -> KeyValueStoreListPage:
         """List the available key-value stores.
 

src/crawlee/base_storage_client/base_request_queue_client.py

@@ -4,8 +4,7 @@
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from crawlee.request import Request
-    from crawlee.storages.models import RequestQueueHead, RequestQueueMetadata, RequestQueueOperationInfo
+    from crawlee.models import Request, RequestQueueHead, RequestQueueMetadata, RequestQueueOperationInfo
 
 
 class BaseRequestQueueClient(ABC):

src/crawlee/base_storage_client/base_request_queue_collection_client.py

@@ -4,7 +4,7 @@
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from crawlee.storages.models import RequestQueueListPage, RequestQueueMetadata
+    from crawlee.models import RequestQueueListPage, RequestQueueMetadata
 
 
 class BaseRequestQueueCollectionClient(ABC):
@@ -40,10 +40,10 @@ async def get_or_create(
     async def list(
         self,
         *,
-        unnamed: bool | None = None,
+        unnamed: bool = False,
         limit: int | None = None,
         offset: int | None = None,
-        desc: bool | None = None,
+        desc: bool = False,
     ) -> RequestQueueListPage:
         """List the available request queues.
 

src/crawlee/base_storage_client/base_storage_client.py

@@ -44,3 +44,12 @@ def request_queue(self, id: str) -> BaseRequestQueueClient:
     @abstractmethod
     def request_queues(self) -> BaseRequestQueueCollectionClient:
         """Gets a subclient for request queue collection operations."""
+
+    @abstractmethod
+    async def purge_on_start(self) -> None:
+        """Performs a purge of the default storages.
+
+        This method ensures that the purge is executed only once during the lifetime of the instance.
+        It is primarily used to clean up residual data from previous runs to maintain a clean state.
+        If the storage client does not support purging, leave it empty.
+        """

src/crawlee/basic_crawler/basic_crawler.py

@@ -38,7 +38,7 @@
 from crawlee.enqueue_strategy import EnqueueStrategy
 from crawlee.events.local_event_manager import LocalEventManager
 from crawlee.http_clients.httpx_client import HttpxClient
-from crawlee.request import BaseRequestData, Request, RequestState
+from crawlee.models import BaseRequestData, Request, RequestState
 from crawlee.sessions import SessionPool
 from crawlee.storages.request_queue import RequestQueue