Remove helper methods as suggested

Author: Pijukatel

docs/upgrading/upgrading_to_v1.md

@@ -214,50 +214,13 @@ service_locator.set_storage_client(MemoryStorageClient())  # Raises an error
 
 ## BasicCrawler changes
 
-### Renamed methods for opening storages
-- `BasicCrawler.get_dataset` renamed to `BasicCrawler.open_dataset`
-- `BasicCrawler.get_key_value_store` renamed to `BasicCrawler.open_key_value_store`
+### Removed helper methods for opening storages
+- `BasicCrawler.get_dataset` was removed.
+- `BasicCrawler.get_key_value_store` was removed.
 
-### Added method for opening RequestQueue that uses configuration and storage client of the crawler
-- `BasicCrawler.open_request_queue`
 
 ### BasicCrawler has its own instance of ServiceLocator to track its own services
-Explicitly passed services to the crawler can be different the global ones accessible in `crawlee.service_locator`. `BasicCrawler` no longer causes the global services in `service_locator` to be set to the crawler's explicitly passed services.
-
-**Before (v0.6):**
-
-```python
-from crawlee import service_locator
-from crawlee.crawlers import BasicCrawler
-from crawlee.storage_clients import MemoryStorageClient
-from crawlee.storages import Dataset
-
-
-async def main() -> None:
-    custom_storage_client = MemoryStorageClient()
-    crawler = BasicCrawler(storage_client=custom_storage_client)
-
-    assert service_locator.get_storage_client() is custom_storage_client
-    assert await crawler.get_dataset() is await Dataset.open()
-```
-**Now (v1.0):**
-
-```python
-from crawlee import service_locator
-from crawlee.crawlers import BasicCrawler
-from crawlee.storage_clients import MemoryStorageClient
-from crawlee.storages import Dataset
-
-
-async def main() -> None:
-    custom_storage_client = MemoryStorageClient()
-    crawler = BasicCrawler(storage_client=custom_storage_client)
-
-    assert service_locator.get_storage_client() is not custom_storage_client
-    assert await crawler.open_dataset() is not await Dataset.open()
-```
-
-This allows two crawlers with different services at the same time.
+Explicitly passed services to the crawler can be different the global ones accessible in `crawlee.service_locator`. `BasicCrawler` no longer causes the global services in `service_locator` to be set to the crawler's explicitly passed services. This allows two crawlers with different services at the same time.
 
 **Now (v1.0):**
 

src/crawlee/crawlers/_adaptive_playwright/_adaptive_playwright_crawler.py

@@ -291,7 +291,7 @@ async def get_input_state(
             use_state_function = context.use_state
 
         # New result is created and injected to newly created context. This is done to ensure isolation of sub crawlers.
-        result = RequestHandlerRunResult(key_value_store_getter=self.open_key_value_store)
+        result = RequestHandlerRunResult(key_value_store_getter=self._open_key_value_store)
         context_linked_to_result = BasicCrawlingContext(
             request=deepcopy(context.request),
             session=deepcopy(context.session),

src/crawlee/crawlers/_basic/_basic_crawler.py

@@ -563,42 +563,13 @@ async def _get_proxy_info(self, request: Request, session: Session | None) -> Pr
     async def get_request_manager(self) -> RequestManager:
         """Return the configured request manager. If none is configured, open and return the default request queue."""
         if not self._request_manager:
-            self._request_manager = await self.open_request_queue()
+            self._request_manager = await RequestQueue.open(
+                storage_client=self._service_locator.get_storage_client(),
+                configuration=self._service_locator.get_configuration(),
+            )
         return self._request_manager
 
-    async def open_request_queue(
-        self,
-        *,
-        id: str | None = None,
-        name: str | None = None,
-        alias: str | None = None,
-    ) -> RequestQueue:
-        """Return `RequestQueue` with the given ID or name or alias. If none is provided, return the default one."""
-        return await RequestQueue.open(
-            id=id,
-            name=name,
-            alias=alias,
-            storage_client=self._service_locator.get_storage_client(),
-            configuration=self._service_locator.get_configuration(),
-        )
-
-    async def open_dataset(
-        self,
-        *,
-        id: str | None = None,
-        name: str | None = None,
-        alias: str | None = None,
-    ) -> Dataset:
-        """Return `Dataset` with the given ID or name or alias. If none is provided, return the default one."""
-        return await Dataset.open(
-            id=id,
-            name=name,
-            alias=alias,
-            storage_client=self._service_locator.get_storage_client(),
-            configuration=self._service_locator.get_configuration(),
-        )
-
-    async def open_key_value_store(
+    async def _open_key_value_store(
         self,
         *,
         id: str | None = None,
@@ -671,7 +642,10 @@ async def run(
             request_manager = await self.get_request_manager()
             if purge_request_queue and isinstance(request_manager, RequestQueue):
                 await request_manager.drop()
-                self._request_manager = await self.open_request_queue()
+                self._request_manager = await RequestQueue.open(
+                    storage_client=self._service_locator.get_storage_client(),
+                    configuration=self._service_locator.get_configuration(),
+                )
 
         if requests is not None:
             await self.add_requests(requests)
@@ -805,11 +779,11 @@ async def _use_state(
         self,
         default_value: dict[str, JsonSerializable] | None = None,
     ) -> dict[str, JsonSerializable]:
-        kvs = await self.open_key_value_store()
+        kvs = await self._open_key_value_store()
         return await kvs.get_auto_saved_value(self._CRAWLEE_STATE_KEY, default_value)
 
     async def _save_crawler_state(self) -> None:
-        store = await self.open_key_value_store()
+        store = await self._open_key_value_store()
         await store.persist_autosaved_values()
 
     async def get_data(
@@ -899,7 +873,13 @@ async def _push_data(
             dataset_alias: The alias of the `Dataset` (run scope, unnamed storage).
             kwargs: Keyword arguments to be passed to the `Dataset.push_data()` method.
         """
-        dataset = await self.open_dataset(id=dataset_id, name=dataset_name, alias=dataset_alias)
+        dataset = await Dataset.open(
+            id=dataset_id,
+            name=dataset_name,
+            alias=dataset_alias,
+            storage_client=self._service_locator.get_storage_client(),
+            configuration=self._service_locator.get_configuration(),
+        )
         await dataset.push_data(data, **kwargs)
 
     def _should_retry_request(self, context: BasicCrawlingContext, error: Exception) -> bool:
@@ -1281,7 +1261,7 @@ async def _commit_request_handler_result(self, context: BasicCrawlingContext) ->
         for push_data_call in result.push_data_calls:
             await self._push_data(**push_data_call)
 
-        await self._commit_key_value_store_changes(result, get_kvs=self.open_key_value_store)
+        await self._commit_key_value_store_changes(result, get_kvs=self._open_key_value_store)
 
     @staticmethod
     async def _commit_key_value_store_changes(
@@ -1348,7 +1328,7 @@ async def __run_task_function(self) -> None:
         else:
             session = await self._get_session()
         proxy_info = await self._get_proxy_info(request, session)
-        result = RequestHandlerRunResult(key_value_store_getter=self.open_key_value_store)
+        result = RequestHandlerRunResult(key_value_store_getter=self._open_key_value_store)
 
         context = BasicCrawlingContext(
             request=request,

tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py

@@ -30,7 +30,7 @@
     AdaptiveContextError,
 )
 from crawlee.statistics import Statistics
-from crawlee.storages import KeyValueStore
+from crawlee.storages import Dataset, KeyValueStore
 
 if TYPE_CHECKING:
     from collections.abc import AsyncGenerator, Iterator
@@ -461,7 +461,7 @@ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
 
     await crawler.run(test_urls[:1])
 
-    dataset = await crawler.open_dataset()
+    dataset = await Dataset.open()
     stored_results = [item async for item in dataset.iterate_items()]
 
     if error_in_pw_crawler:

tests/unit/crawlers/_basic/test_basic_crawler.py

@@ -611,12 +611,6 @@ async def test_crawler_get_storages() -> None:
     rp = await crawler.get_request_manager()
     assert isinstance(rp, RequestQueue)
 
-    dataset = await crawler.open_dataset()
-    assert isinstance(dataset, Dataset)
-
-    kvs = await crawler.open_key_value_store()
-    assert isinstance(kvs, KeyValueStore)
-
 
 async def test_crawler_run_requests() -> None:
     crawler = BasicCrawler()
@@ -725,7 +719,7 @@ async def handler(context: BasicCrawlingContext) -> None:
 
     await crawler.run(['https://hello.world'])
 
-    store = await crawler.open_key_value_store()
+    store = await crawler._open_key_value_store()
     assert (await store.get_value('foo')) == 'bar'
 
 
@@ -738,7 +732,7 @@ async def handler(context: BasicCrawlingContext) -> None:
 
     await crawler.run(['https://hello.world'])
 
-    kvs = await crawler.open_key_value_store()
+    kvs = await crawler._open_key_value_store()
     value = await kvs.get_value(BasicCrawler._CRAWLEE_STATE_KEY)
 
     assert value == {'hello': 'world'}
@@ -781,7 +775,7 @@ async def handler_three(context: BasicCrawlingContext) -> None:
     # The state in handler_three must match the final state updated in previous run
     assert state_in_handler_three == {'hello': 'last_world'}
 
-    store = await crawler.open_key_value_store()
+    store = await crawler._open_key_value_store()
 
     # The state in the KVS must match with the last set state
     assert (await store.get_value(BasicCrawler._CRAWLEE_STATE_KEY)) == {'hello': 'last_world'}
@@ -1103,14 +1097,10 @@ async def test_crawler_uses_default_storages(tmp_path: Path) -> None:
     )
     service_locator.set_configuration(configuration)
 
-    dataset = await Dataset.open()
-    kvs = await KeyValueStore.open()
     rq = await RequestQueue.open()
 
     crawler = BasicCrawler()
 
-    assert dataset is await crawler.open_dataset()
-    assert kvs is await crawler.open_key_value_store()
     assert rq is await crawler.get_request_manager()
 
 
@@ -1121,14 +1111,10 @@ async def test_crawler_can_use_other_storages(tmp_path: Path) -> None:
     )
     service_locator.set_configuration(configuration)
 
-    dataset = await Dataset.open()
-    kvs = await KeyValueStore.open()
     rq = await RequestQueue.open()
 
     crawler = BasicCrawler(storage_client=MemoryStorageClient())
 
-    assert dataset is not await crawler.open_dataset()
-    assert kvs is not await crawler.open_key_value_store()
     assert rq is not await crawler.get_request_manager()
 
 
@@ -1156,16 +1142,12 @@ async def test_crawler_can_use_other_storages_of_same_type(tmp_path: Path) -> No
     service_locator.set_configuration(configuration_a)
     service_locator.set_storage_client(FileSystemStorageClient())
     # Create storages based on the global services
-    dataset = await Dataset.open()
-    kvs = await KeyValueStore.open()
     rq = await RequestQueue.open()
 
     # Set the crawler to use different storage client
     crawler = BasicCrawler(storage_client=FileSystemStorageClient(), configuration=configuration_b)
 
     # Assert that the storages are different
-    assert dataset is not await crawler.open_dataset()
-    assert kvs is not await crawler.open_key_value_store()
     assert rq is not await crawler.get_request_manager()
 
     # Assert that all storages exists on the filesystem
@@ -1193,7 +1175,7 @@ async def handler(context: BasicCrawlingContext) -> None:
         await crawler.run(['https://does-not-matter.com'])
         assert spy.call_count >= 1
 
-    dataset = await crawler.open_dataset()
+    dataset = await Dataset.open()
     data = await dataset.get_data()
     assert data.items == [{'foo': 'bar'}]
 
@@ -1208,7 +1190,7 @@ async def test_context_use_state_race_condition_in_handlers(key_value_store: Key
     from asyncio import Barrier  # type:ignore[attr-defined] # noqa: PLC0415
 
     crawler = BasicCrawler()
-    store = await crawler.open_key_value_store()
+    store = await crawler._open_key_value_store()
     await store.set_value(BasicCrawler._CRAWLEE_STATE_KEY, {'counter': 0})
     handler_barrier = Barrier(2)
 
@@ -1221,7 +1203,7 @@ async def handler(context: BasicCrawlingContext) -> None:
 
     await crawler.run(['https://crawlee.dev/', 'https://crawlee.dev/docs/quick-start'])
 
-    store = await crawler.open_key_value_store()
+    store = await crawler._open_key_value_store()
     # Ensure that local state is pushed back to kvs.
     await store.persist_autosaved_values()
     assert (await store.get_value(BasicCrawler._CRAWLEE_STATE_KEY))['counter'] == 2

tests/unit/crawlers/_http/test_http_crawler.py

@@ -562,7 +562,7 @@ async def request_handler(context: HttpCrawlingContext) -> None:
 
     await crawler.run([str(server_url)])
 
-    kvs = await crawler.open_key_value_store()
+    kvs = await crawler._open_key_value_store()
     kvs_content = {}
     async for key_info in kvs.iterate_keys():
         kvs_content[key_info.key] = await kvs.get_value(key_info.key)

tests/unit/crawlers/_playwright/test_playwright_crawler.py

@@ -627,7 +627,7 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None:
         [str(server_url), str(server_url / 'page_1'), str(server_url / 'page_2'), str(server_url / 'headers')]
     )
 
-    kvs = await crawler.open_key_value_store()
+    kvs = await crawler._open_key_value_store()
     kvs_content = {}
 
     async for key_info in kvs.iterate_keys():