Merge pull request #276 from scrapy-plugins/scrapy-2.15

Add support for Scrapy 2.15

Author: wRAR

.github/workflows/scrapy-unreleased.yml

@@ -0,0 +1,33 @@
+# This runs the tests for the unreleased Scrapy master branch.
+# It doesn't run by default, to run it label your PR with the "test-scrapy-unreleased" label.
+name: scrapy-unreleased
+
+permissions:
+  contents: read
+
+on:
+  pull_request:
+    types: [ opened, synchronize, reopened, labeled ]
+
+jobs:
+  test:
+    if: contains(github.event.pull_request.labels.*.name, 'test-scrapy-unreleased')
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set up Python 3.13
+      uses: actions/setup-python@v6
+      with:
+        python-version: "3.13"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install tox
+    - name: tox
+      run: |
+        tox -e scrapy-unreleased -- -n auto
+    - name: coverage
+      uses: codecov/codecov-action@v5
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/test.yml

@@ -3,6 +3,9 @@
 
 name: tox
 
+permissions:
+  contents: read
+
 on:
   push:
     branches: [ main ]

tests/__init__.py

@@ -86,7 +86,12 @@ class DummySpider(Spider):
 
 
 async def get_crawler(
-    settings=None, spider_cls=DummySpider, setup_engine=True, use_addon=False, poet=True
+    settings=None,
+    spider_cls=DummySpider,
+    setup_engine=True,
+    start_handler=False,
+    use_addon=False,
+    poet=True,
 ):
     settings = settings or {}
     base_settings: SETTINGS_T = deepcopy(SETTINGS if not use_addon else SETTINGS_ADDON)
@@ -95,7 +100,7 @@ async def get_crawler(
         final_settings.setdefault("ADDONS", {})["scrapy_poet.Addon"] = 300
     crawler = _get_crawler(settings_dict=final_settings, spidercls=spider_cls)
     if setup_engine:
-        await setup_crawler_engine(crawler)
+        await setup_crawler_engine(crawler, start_handler=start_handler)
     return crawler
 
 
@@ -117,7 +122,9 @@ async def make_handler(
 ):
     if api_url is not None:
         settings["ZYTE_API_URL"] = api_url
-    crawler = await get_crawler(settings, use_addon=use_addon)
+    crawler = await get_crawler(
+        settings, setup_engine=True, start_handler=True, use_addon=use_addon
+    )
     handler = get_download_handler(crawler, "https")
     if not isinstance(handler, _ScrapyZyteAPIBaseDownloadHandler):
         # i.e. ZYTE_API_ENABLED=False
@@ -164,7 +171,7 @@ def set_env(**env_vars):
         environ.update(old_environ)
 
 
-async def setup_crawler_engine(crawler: Crawler):
+async def setup_crawler_engine(crawler: Crawler, start_handler: bool = False) -> None:
     """Run the crawl steps until engine setup, so that crawler.engine is not
     None.
 
@@ -175,9 +182,10 @@ async def setup_crawler_engine(crawler: Crawler):
     crawler.spider = crawler._create_spider()
     crawler.engine = crawler._create_engine()
 
-    handler = get_download_handler(crawler, "https")
-    if hasattr(handler, "engine_started"):
-        await handler.engine_started()
+    if start_handler:
+        handler = get_download_handler(crawler, "https")
+        if hasattr(handler, "engine_started"):
+            await handler.engine_started()
 
 
 async def download_request(handler, request) -> Response:

tests/test_api_requests.py

@@ -2945,7 +2945,7 @@ async def test_automap_all_cookies(meta):
         "ZYTE_API_EXPERIMENTAL_COOKIES_ENABLED": True,
         "ZYTE_API_TRANSPARENT_MODE": True,
     }
-    crawler = await get_crawler(settings)
+    crawler = await get_crawler(settings, start_handler=True)
     cookie_middleware = get_downloader_middleware(crawler, CookiesMiddleware)
     handler = get_download_handler(crawler, "https")
     param_parser = handler._param_parser
@@ -3029,6 +3029,7 @@ async def test_automap_all_cookies(meta):
             # {"name": "c", "value": "d", "domain": "b.example"},
         ]
     )
+    await handler._close()
 
 
 @pytest.mark.parametrize(
@@ -3053,7 +3054,7 @@ async def test_automap_cookie_jar(meta):
         "ZYTE_API_EXPERIMENTAL_COOKIES_ENABLED": True,
         "ZYTE_API_TRANSPARENT_MODE": True,
     }
-    crawler = await get_crawler(settings)
+    crawler = await get_crawler(settings, start_handler=True)
     cookie_middleware = get_downloader_middleware(crawler, CookiesMiddleware)
     handler = get_download_handler(crawler, "https")
     param_parser = handler._param_parser
@@ -3090,6 +3091,7 @@ async def test_automap_cookie_jar(meta):
             {"name": "z", "value": "y", "domain": "example.com"},
         ]
     )
+    await handler._close()
 
 
 @pytest.mark.parametrize(
@@ -3106,7 +3108,7 @@ async def test_automap_cookie_limit(meta, caplog):
         "ZYTE_API_MAX_COOKIES": 1,
         "ZYTE_API_TRANSPARENT_MODE": True,
     }
-    crawler = await get_crawler(settings)
+    crawler = await get_crawler(settings, start_handler=True)
     cookie_middleware = get_downloader_middleware(crawler, CookiesMiddleware)
     handler = get_download_handler(crawler, "https")
     param_parser = handler._param_parser
@@ -3196,6 +3198,7 @@ async def test_automap_cookie_limit(meta, caplog):
     assert "would get 2 cookies" in caplog.text
     assert "limited to 1 cookies" in caplog.text
     caplog.clear()
+    await handler._close()
 
 
 class CustomCookieJar(CookieJar):
@@ -3241,7 +3244,7 @@ async def test_automap_custom_cookie_middleware():
         "ZYTE_API_EXPERIMENTAL_COOKIES_ENABLED": True,
         "ZYTE_API_TRANSPARENT_MODE": True,
     }
-    crawler = await get_crawler(settings)
+    crawler = await get_crawler(settings, start_handler=True)
     cookie_middleware = get_downloader_middleware(crawler, mw_cls)
     handler = get_download_handler(crawler, "https")
     param_parser = handler._param_parser
@@ -3252,6 +3255,7 @@ async def test_automap_custom_cookie_middleware():
     assert api_params["experimental"]["requestCookies"] == [
         {"name": "z", "value": "y", "domain": "example.com"}
     ]
+    await handler._close()
 
 
 @pytest.mark.parametrize(

tests/test_handler.py

@@ -65,10 +65,11 @@ async def test_concurrency_configuration(concurrency):
         **SETTINGS,
         "CONCURRENT_REQUESTS": concurrency,
     }
-    crawler = await get_crawler_zyte_api(settings=settings)
+    crawler = await get_crawler_zyte_api(settings=settings, start_handler=True)
     handler = get_download_handler(crawler, "https")
     assert handler._client.n_conn == concurrency
     assert handler._session._session.connector.limit == concurrency
+    await handler._close()
 
 
 ETH_KEY = "c85ef7d79691fe79573b1a7064c5232332f53bb1b44a08f1a737f57a68a4706e"
@@ -274,6 +275,7 @@ async def test_retry_policy(
     async with make_handler(settings) as handler:
         req = Request("https://example.com", meta=meta)
         unmocked_session = handler._session
+        await unmocked_session.close()
         handler._session = mock.AsyncMock(unmocked_session)
         handler._session.get.return_value = {
             "browserHtml": "",
@@ -528,6 +530,7 @@ async def test_log_request_truncate(
         meta = {"zyte_api": input_params}
         request = Request("https://example.com", meta=meta)
         unmocked_session = handler._session
+        await unmocked_session.close()
         handler._session = mock.AsyncMock(unmocked_session)
         handler._session.get.return_value = {
             "browserHtml": "",
@@ -575,9 +578,10 @@ async def test_trust_env(enabled):
         settings["ZYTE_API_USE_ENV_PROXY"] = enabled
     else:
         enabled = False
-    crawler = await get_crawler_zyte_api(settings=settings)
+    crawler = await get_crawler_zyte_api(settings=settings, start_handler=True)
     handler = get_download_handler(crawler, "https")
     assert handler._session._session._trust_env == enabled
+    await handler._close()
 
 
 @pytest.mark.parametrize(
@@ -765,6 +769,7 @@ async def test_download_request_limits(
 ):
     settings: SETTINGS_T = {"DOWNLOAD_WARNSIZE": warnsize, "DOWNLOAD_MAXSIZE": maxsize}
     async with make_handler(settings, mockserver.urljoin("/")) as handler:
+        await handler._session.close()
         handler._session = mock.AsyncMock()
         handler._session.get.return_value = mock.Mock(body=b"x" * body_size)
 

tox.ini

@@ -173,6 +173,12 @@ extras = provider
 [testenv:x402]
 extras = x402
 
+[testenv:scrapy-unreleased]
+deps =
+    {[testenv]deps}
+    scrapy @ git+https://github.com/scrapy/scrapy@master
+extras = provider
+
 [testenv:twine]
 deps =
     twine==6.1.0