Test with crawlee branch storage-clients-and-configurations

Author: Pijukatel

pyproject.toml

@@ -36,7 +36,7 @@ keywords = [
 dependencies = [
     "apify-client>=2.0.0,<3.0.0",
     "apify-shared>=2.0.0,<3.0.0",
-    "crawlee==1.0.0rc1",
+    "crawlee @ git+https://github.com/apify/crawlee-python.git@storage-clients-and-configurations",
     "cachetools>=5.5.0",
     "cryptography>=42.0.0",
     # TODO: ensure compatibility with the latest version of lazy-object-proxy

src/apify/_actor.py

@@ -429,7 +429,6 @@ async def open_dataset(
         return await Dataset.open(
             id=id,
             name=name,
-            configuration=self._configuration,
             storage_client=storage_client,
         )
 
@@ -463,7 +462,6 @@ async def open_key_value_store(
         return await KeyValueStore.open(
             id=id,
             name=name,
-            configuration=self._configuration,
             storage_client=storage_client,
         )
 
@@ -500,7 +498,6 @@ async def open_request_queue(
         return await RequestQueue.open(
             id=id,
             name=name,
-            configuration=self._configuration,
             storage_client=storage_client,
         )
 

src/apify/scrapy/extensions/_httpcache.py

@@ -56,7 +56,6 @@ async def open_kvs() -> KeyValueStore:
                 storage_client = ApifyStorageClient()
                 return await KeyValueStore.open(
                     name=kvs_name,
-                    configuration=configuration,
                     storage_client=storage_client,
                 )
             return await KeyValueStore.open(name=kvs_name)

src/apify/scrapy/scheduler.py

@@ -53,7 +53,6 @@ async def open_rq() -> RequestQueue:
             if configuration.is_at_home:
                 storage_client = ApifyStorageClient()
                 return await RequestQueue.open(
-                    configuration=configuration,
                     storage_client=storage_client,
                 )
             return await RequestQueue.open()

tests/integration/actor_source_base/requirements.txt

@@ -1,4 +1,4 @@
 # The test fixture will put the Apify SDK wheel path on the next line
 APIFY_SDK_WHEEL_PLACEHOLDER
 uvicorn[standard]
-crawlee[parsel]==1.0.0rc1
+crawlee[parsel] @ git+https://github.com/apify/crawlee-python.git@storage-clients-and-configurations

tests/integration/conftest.py

@@ -60,16 +60,8 @@ def _prepare_test_env() -> None:
         service_locator._storage_client = None
         service_locator._storage_instance_manager = None
 
-        # Reset the retrieval flags.
-        service_locator._configuration_was_retrieved = False
-        service_locator._event_manager_was_retrieved = False
-        service_locator._storage_client_was_retrieved = False
-
         # Verify that the test environment was set up correctly.
         assert os.environ.get(ApifyEnvVars.LOCAL_STORAGE_DIR) == str(tmp_path)
-        assert service_locator._configuration_was_retrieved is False
-        assert service_locator._storage_client_was_retrieved is False
-        assert service_locator._event_manager_was_retrieved is False
 
     return _prepare_test_env
 

tests/unit/conftest.py

@@ -51,16 +51,8 @@ def _prepare_test_env() -> None:
         service_locator._storage_client = None
         service_locator._storage_instance_manager = None
 
-        # Reset the retrieval flags.
-        service_locator._configuration_was_retrieved = False
-        service_locator._event_manager_was_retrieved = False
-        service_locator._storage_client_was_retrieved = False
-
         # Verify that the test environment was set up correctly.
         assert os.environ.get(ApifyEnvVars.LOCAL_STORAGE_DIR) == str(tmp_path)
-        assert service_locator._configuration_was_retrieved is False
-        assert service_locator._storage_client_was_retrieved is False
-        assert service_locator._event_manager_was_retrieved is False
 
     return _prepare_test_env