Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 2 additions & 26 deletions scrapy_playwright/handler.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
import asyncio
import inspect
import logging
import platform
import warnings
from contextlib import suppress
from dataclasses import dataclass, field as dataclass_field
from functools import partial
Expand All @@ -26,7 +24,7 @@
from scrapy import Spider, signals, version_info as scrapy_version_info
from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
from scrapy.crawler import Crawler
from scrapy.exceptions import NotSupported, ScrapyDeprecationWarning
from scrapy.exceptions import NotSupported
from scrapy.http import Request, Response
from scrapy.http.headers import Headers
from scrapy.responsetypes import responsetypes
Expand Down Expand Up @@ -782,11 +780,7 @@ async def _request_handler(route: Route, playwright_request: PlaywrightRequest)

if self.process_request_headers is None:
final_headers = await playwright_request.all_headers()
elif (sig := inspect.signature(self.process_request_headers)) and (
"browser_type_name" in sig.parameters
and "playwright_request" in sig.parameters
and "scrapy_request_data" in sig.parameters
):
else:
overrides["headers"] = final_headers = await _maybe_await(
self.process_request_headers(
browser_type_name=self.config.browser_type_name,
Expand All @@ -800,24 +794,6 @@ async def _request_handler(route: Route, playwright_request: PlaywrightRequest)
},
)
)
else:
warnings.warn(
"Accepting positional arguments in the function passed to the"
" PLAYWRIGHT_PROCESS_REQUEST_HEADERS setting is deprecated. The function"
" should accept three (3) keyword arguments instead:"
" browser_type_name: str,"
" playwright_request: playwright.async_api.Request,"
" scrapy_request_data: dict",
category=ScrapyDeprecationWarning,
stacklevel=1,
)
overrides["headers"] = final_headers = await _maybe_await(
self.process_request_headers(
self.config.browser_type_name,
playwright_request,
headers,
)
)

# if the current request corresponds to the original scrapy one
if (
Expand Down
37 changes: 0 additions & 37 deletions tests/tests_asyncio/test_headers.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,43 +104,6 @@ async def important_headers(
assert headers.get("user-agent") not in (self.browser_type, "foobar")
assert "asdf" not in headers

@allow_windows
async def test_use_custom_headers_deprecated_arg_handling(self):
"""Custom header processing function that receives deprecated args"""

async def deprecated_args(
browser_name, pw_req, headers # pylint: disable=unused-argument
) -> dict:
return {"foo": "bar"}

settings_dict = {
"PLAYWRIGHT_BROWSER_TYPE": self.browser_type,
"PLAYWRIGHT_CONTEXTS": {"default": {"user_agent": self.browser_type}},
"PLAYWRIGHT_PROCESS_REQUEST_HEADERS": deprecated_args,
}
async with make_handler(settings_dict) as handler:
with MockServer() as server:
req = Request(
url=server.urljoin("/headers"),
meta={"playwright": True},
headers={"User-Agent": "foobar", "Asdf": "qwerty"},
)
with warnings.catch_warnings(record=True) as warning_list:
resp = await handler._download_request(req, Spider("foo"))
headers = json.loads(resp.css("pre::text").get())
headers = {key.lower(): value for key, value in headers.items()}
assert headers["foo"] == "bar"
assert headers.get("user-agent") not in (self.browser_type, "foobar")
assert "asdf" not in headers
assert str(warning_list[0].message) == (
"Accepting positional arguments in the function passed to the"
" PLAYWRIGHT_PROCESS_REQUEST_HEADERS setting is deprecated. The function"
" should accept three (3) keyword arguments instead:"
" browser_type_name: str,"
" playwright_request: playwright.async_api.Request,"
" scrapy_request_data: dict"
)


class TestProcessHeadersChromium(IsolatedAsyncioTestCase, MixinProcessHeadersTestCase):
browser_type = "chromium"
Expand Down