test: add e2e tests for Scrapy spiders running as Apify Actors (#788)

## Summary

Add 5 new e2e tests covering key Scrapy spider types and integration
patterns, all using a local HTTP server for reliability. Tests cover
basic Spider, CrawlSpider with rules, ItemLoader, cb_kwargs
serialization, and custom pipeline merging with apply_apify_settings.

## Motivation

- Better test coverage for Scrapy integration, up until now, we have
only 1 E2E test, and that was all.

## Issue

- Relates: #785

## Test plan

- [x] CI passes

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: vdusek

tests/e2e/test_scrapy/__init__.py

@@ -0,0 +1 @@
+

tests/e2e/test_scrapy/actor_source/__init__.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from scrapy.utils.reactor import install_reactor
+
+install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
+
+import os  # noqa: E402, I001
+
+from apify.scrapy import initialize_logging, run_scrapy_actor  # noqa: E402
+
+from .main import main  # noqa: E402
+
+os.environ['SCRAPY_SETTINGS_MODULE'] = 'src.settings'
+
+if __name__ == '__main__':
+    initialize_logging()
+    run_scrapy_actor(main())

tests/e2e/test_scrapy/actor_source/__main__.py

@@ -0,0 +1,10 @@
+from __future__ import annotations
+
+from scrapy import Field, Item
+
+
+class ProductItem(Item):
+    name = Field()
+    url = Field()
+    price = Field()
+    description = Field()

tests/e2e/test_scrapy/actor_source/items.py

@@ -0,0 +1,16 @@
+from __future__ import annotations  # noqa: I001
+
+from scrapy.crawler import CrawlerRunner
+from scrapy.utils.defer import deferred_to_future
+
+from apify import Actor
+from apify.scrapy import apply_apify_settings
+
+from .spiders import Spider  # ty: ignore[unresolved-import]
+
+
+async def main() -> None:
+    async with Actor:
+        settings = apply_apify_settings()
+        runner = CrawlerRunner(settings)
+        await deferred_to_future(runner.crawl(Spider, start_urls=['http://localhost:8080/']))

tests/e2e/test_scrapy/actor_source/main.py

@@ -0,0 +1,19 @@
+from __future__ import annotations  # noqa: I001
+
+import os
+
+from scrapy.crawler import CrawlerRunner
+from scrapy.utils.defer import deferred_to_future
+
+from apify import Actor
+from apify.scrapy import apply_apify_settings
+
+from .spiders import Spider  # ty: ignore[unresolved-import]
+
+
+async def main() -> None:
+    async with Actor:
+        os.environ['SCRAPY_SETTINGS_MODULE'] = 'src.settings_custom_pipeline'
+        settings = apply_apify_settings()
+        runner = CrawlerRunner(settings)
+        await deferred_to_future(runner.crawl(Spider, start_urls=['http://localhost:8080/']))

tests/e2e/test_scrapy/actor_source/main_custom_pipeline.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from scrapy import Item, Spider
+
+
+class PriceCleanerPipeline:
+    def process_item(
+        self,
+        item: Item,
+        _: Spider,
+    ) -> Item:
+        if 'price' in item and isinstance(item['price'], str):
+            item['price'] = item['price'].lstrip('$')
+        return item

tests/e2e/test_scrapy/actor_source/pipelines.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import Awaitable, Callable, Coroutine
+from typing import Any
+
+from uvicorn import Config
+from uvicorn.server import Server
+
+Receive = Callable[[], Awaitable[dict[str, Any]]]
+Send = Callable[[dict[str, Any]], Coroutine[None, None, None]]
+
+_PRODUCTS = {
+    '1': {'name': 'Widget A', 'price': '$19.99', 'description': 'A basic widget for everyday use'},
+    '2': {'name': 'Widget B', 'price': '$29.99', 'description': 'An advanced widget with extra features'},
+    '3': {'name': 'Widget C', 'price': '$39.99', 'description': 'A premium widget for professionals'},
+}
+
+
+async def _send_html(send: Send, html: str, status: int = 200) -> None:
+    await send(
+        {
+            'type': 'http.response.start',
+            'status': status,
+            'headers': [[b'content-type', b'text/html; charset=utf-8']],
+        }
+    )
+    await send({'type': 'http.response.body', 'body': html.encode()})
+
+
+async def app(scope: dict[str, Any], _receive: Receive, send: Send) -> None:
+    assert scope['type'] == 'http'
+    path = scope['path']
+
+    if path == '/':
+        await _send_html(
+            send,
+            '<html><head><title>E-commerce Test Store</title></head><body>'
+            '<h1>Welcome to Test Store</h1>'
+            '<a href="/products/1">Widget A</a>'
+            '<a href="/products/2">Widget B</a>'
+            '<a href="/products/3">Widget C</a>'
+            '<a href="/about">About Us</a>'
+            '</body></html>',
+        )
+    elif path.startswith('/products/'):
+        product = _PRODUCTS.get(path.split('/')[-1])
+        if product:
+            await _send_html(
+                send,
+                f'<html><head><title>{product["name"]}</title></head><body>'
+                f'<h1>{product["name"]}</h1>'
+                f'<span class="price">{product["price"]}</span>'
+                f'<p class="description">{product["description"]}</p>'
+                f'<a href="/">Back to Home</a>'
+                f'</body></html>',
+            )
+        else:
+            await _send_html(send, '<html><body>Not Found</body></html>', 404)
+    elif path == '/about':
+        await _send_html(
+            send,
+            '<html><head><title>About Us</title></head><body>'
+            '<h1>About Test Store</h1>'
+            '<p class="description">We sell the best widgets in the world.</p>'
+            '<a href="/">Back to Home</a>'
+            '</body></html>',
+        )
+    else:
+        await _send_html(send, '<html><body>Not Found</body></html>', 404)
+
+
+if __name__ == '__main__':
+    asyncio.run(
+        Server(
+            config=Config(
+                app=app,
+                lifespan='off',
+                loop='asyncio',
+                port=8080,
+                log_config=None,
+                log_level=logging.CRITICAL,
+            )
+        ).serve()
+    )

tests/e2e/test_scrapy/actor_source/server.py

@@ -0,0 +1,8 @@
+BOT_NAME = 'testbot'
+LOG_LEVEL = 'INFO'
+NEWSPIDER_MODULE = 'src.spiders'
+ROBOTSTXT_OBEY = False
+SPIDER_MODULES = ['src.spiders']
+TELNETCONSOLE_ENABLED = False
+TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor'
+HTTPCACHE_ENABLED = False

tests/e2e/test_scrapy/actor_source/settings.py

@@ -0,0 +1,5 @@
+from src.settings import *  # noqa: F403  # ty: ignore[unresolved-import]
+
+ITEM_PIPELINES = {
+    'src.pipelines.PriceCleanerPipeline': 100,
+}