From 391cda2e18d64518dace4b38c11b3fc5da5c30eb Mon Sep 17 00:00:00 2001 From: Pawel Miech Date: Fri, 3 Apr 2015 13:46:55 +0200 Subject: [PATCH 01/10] [middleware] allow enabling splash per spider --- scrapy_splash/middleware.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scrapy_splash/middleware.py b/scrapy_splash/middleware.py index 24ab23a..69efcc3 100644 --- a/scrapy_splash/middleware.py +++ b/scrapy_splash/middleware.py @@ -257,8 +257,16 @@ def _argument_values(self): def _remote_keys(self): return self.crawler.spider.state[self.remote_keys_key] + def _get_splash_options(self, request, spider): + if request.meta.get("dont_proxy"): + return + spider_options = getattr(spider, "splash", {}) + request_options = request.meta.get("splash") + return request_options or spider_options + def process_request(self, request, spider): - if 'splash' not in request.meta: + splash_options = self._get_splash_options(request, spider) + if not splash_options: return if request.method not in {'GET', 'POST'}: @@ -274,7 +282,6 @@ def process_request(self, request, spider): # don't process the same request more than once return - splash_options = request.meta['splash'] request.meta['_splash_processed'] = True slot_policy = splash_options.get('slot_policy', self.slot_policy) @@ -368,7 +375,7 @@ def process_response(self, request, response, spider): if not request.meta.get("_splash_processed"): return response - splash_options = request.meta['splash'] + splash_options = self._get_splash_options(request, spider) if not splash_options: return response From 385d567bc4663949a3bcdd1f0880cfc7e8c50dbc Mon Sep 17 00:00:00 2001 From: Pawel Miech Date: Fri, 3 Apr 2015 13:48:10 +0200 Subject: [PATCH 02/10] [scrashtest] add another test spider --- example/scrashtest/spiders/dmoz_two.py | 31 ++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 example/scrashtest/spiders/dmoz_two.py diff --git a/example/scrashtest/spiders/dmoz_two.py b/example/scrashtest/spiders/dmoz_two.py new file mode 100644 index 0000000..e441fa6 --- /dev/null +++ b/example/scrashtest/spiders/dmoz_two.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +from urlparse import urljoin +import json + +import scrapy +from scrapy.contrib.linkextractors import LinkExtractor + + +class DmozSpider(scrapy.Spider): + name = "js_spider" + start_urls = ['http://www.isjavascriptenabled.com/'] + splash = {'args': {'har': 1, 'html': 1}} + + def parse(self, response): + is_js = response.xpath("//h1/text()").extract() + if "".join(is_js).lower() == "yes": + self.log("JS enabled!") + else: + self.log("Error! JS disabled!", scrapy.log.ERROR) + le = LinkExtractor() + + for link in le.extract_links(response): + url = urljoin(response.url, link.url) + yield scrapy.Request(url, self.parse_link) + break + + def parse_link(self, response): + title = response.xpath("//title").extract() + yes = response.xpath("//h1").extract() + self.log("response is: {}".format(repr(response))) + self.log(u"Html in response contains {} {}".format("".join(title), "".join(yes))) From a736785e0c80ebcc77f16e3fcadbd23650f94425 Mon Sep 17 00:00:00 2001 From: Pawel Miech Date: Tue, 26 May 2015 09:10:01 +0200 Subject: [PATCH 03/10] [spider attr] removed html_response * removed process_html (render response in html feature) from this PR * removed remote request tests with extra spider --- example/scrashtest/spiders/dmoz_two.py | 31 -------------------------- 1 file changed, 31 deletions(-) delete mode 100644 example/scrashtest/spiders/dmoz_two.py diff --git a/example/scrashtest/spiders/dmoz_two.py b/example/scrashtest/spiders/dmoz_two.py deleted file mode 100644 index e441fa6..0000000 --- a/example/scrashtest/spiders/dmoz_two.py +++ /dev/null @@ -1,31 +0,0 @@ -# -*- coding: utf-8 -*- -from urlparse import urljoin -import json - -import scrapy -from scrapy.contrib.linkextractors import LinkExtractor - - -class DmozSpider(scrapy.Spider): - name = "js_spider" - start_urls = ['http://www.isjavascriptenabled.com/'] - splash = {'args': {'har': 1, 'html': 1}} - - def parse(self, response): - is_js = response.xpath("//h1/text()").extract() - if "".join(is_js).lower() == "yes": - self.log("JS enabled!") - else: - self.log("Error! JS disabled!", scrapy.log.ERROR) - le = LinkExtractor() - - for link in le.extract_links(response): - url = urljoin(response.url, link.url) - yield scrapy.Request(url, self.parse_link) - break - - def parse_link(self, response): - title = response.xpath("//title").extract() - yes = response.xpath("//h1").extract() - self.log("response is: {}".format(repr(response))) - self.log(u"Html in response contains {} {}".format("".join(title), "".join(yes))) From 42fa592accab8a89e2ab8b2217e1c0785426fcfa Mon Sep 17 00:00:00 2001 From: Pawel Miech Date: Tue, 26 May 2015 09:13:37 +0200 Subject: [PATCH 04/10] [tests] refactors tests, adds tests for spider attr * refactors tests from functions to objects inheriting from unittest.TestCase * adds tests for enabling middleware with spider attribute --- tests/test_middleware.py | 177 +++++++++++++++++++++++++++++++-------- 1 file changed, 142 insertions(+), 35 deletions(-) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 66b79ce..089fe05 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -5,7 +5,6 @@ import base64 import scrapy -from scrapy.core.engine import ExecutionEngine from scrapy.utils.test import get_crawler from scrapy.http import Response, TextResponse from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware @@ -699,6 +698,8 @@ def test_float_wait_arg(): req = mw.process_request(req1, None) assert json.loads(to_native_str(req.body)) == {'url': req1.url, 'wait': 0.5} + def __init__(self, delay=0.0): + self.delay = delay def test_slot_policy_single_slot(): mw = _get_mw() @@ -706,15 +707,17 @@ def test_slot_policy_single_slot(): 'slot_policy': scrapy_splash.SlotPolicy.SINGLE_SLOT }} - req1 = scrapy.Request("http://example.com/path?key=value", meta=meta) - req1 = mw.process_request(req1, None) +class MockedDownloader(object): - req2 = scrapy.Request("http://fooexample.com/path?key=value", meta=meta) - req2 = mw.process_request(req2, None) + def __init__(self): + self.slots = {} - assert req1.meta.get('download_slot') - assert req1.meta['download_slot'] == req2.meta['download_slot'] + def _get_slot_key(self, request, spider): + if 'download_slot' in request.meta: + return request.meta['download_slot'] + key = urlparse_cached(request).hostname or '' + return key def test_slot_policy_per_domain(): mw = _get_mw() @@ -722,21 +725,29 @@ def test_slot_policy_per_domain(): 'slot_policy': scrapy_splash.SlotPolicy.PER_DOMAIN }} - req1 = scrapy.Request("http://example.com/path?key=value", meta=meta) - req1 = mw.process_request(req1, None) +class MockedEngine(object): + downloader = MockedDownloader() - req2 = scrapy.Request("http://example.com/path2", meta=meta) - req2 = mw.process_request(req2, None) - req3 = scrapy.Request("http://fooexample.com/path?key=value", meta=meta) - req3 = mw.process_request(req3, None) +class MiddlewareTest(unittest.TestCase): - assert req1.meta.get('download_slot') - assert req3.meta.get('download_slot') + def setUp(self): + self.crawler = get_crawler(settings_dict={ + 'DOWNLOAD_HANDLERS': {'s3': None}, # for faster test running + }) + if not hasattr(self.crawler, 'logformatter'): + self.crawler.logformatter = None + self.crawler.engine = MockedEngine() + self.mw = SplashMiddleware.from_crawler(self.crawler) - assert req1.meta['download_slot'] == req2.meta['download_slot'] - assert req1.meta['download_slot'] != req3.meta['download_slot'] + def test_nosplash(self): + req = scrapy.Request("http://example.com") + old_meta = copy.deepcopy(req.meta) + assert self.mw.process_request(req, None) is None + assert old_meta == req.meta + def test_splash_request(self): + req = SplashRequest("http://example.com?foo=bar&url=1&wait=100") def test_slot_policy_scrapy_default(): mw = _get_mw() @@ -746,22 +757,118 @@ def test_slot_policy_scrapy_default(): req = mw.process_request(req, None) assert 'download_slot' not in req.meta + expected_body = {'url': req.url} + expected_body.update(SplashRequest.default_splash_meta['args']) + assert json.loads(req2.body) == expected_body + + def test_splash_request_no_url(self): + lua_source = "function main(splash) return {result='ok'} end" + req1 = SplashRequest(meta={'splash': { + 'args': {'lua_source': lua_source}, + 'endpoint': 'execute', + }}) + req = self.mw.process_request(req1, None) + assert req.url == 'http://127.0.0.1:8050/execute' + assert json.loads(req.body) == { + 'url': 'about:blank', + 'lua_source': lua_source + } -def test_adjust_timeout(): - mw = _get_mw() - req1 = scrapy.Request("http://example.com", meta = { - 'splash': {'args': {'timeout': 60, 'html': 1}}, - - # download_timeout is always present, - # it is set by DownloadTimeoutMiddleware - 'download_timeout': 30, - }) - req1 = mw.process_request(req1, None) - assert req1.meta['download_timeout'] > 60 - - req2 = scrapy.Request("http://example.com", meta = { - 'splash': {'args': {'html': 1}}, - 'download_timeout': 30, - }) - req2 = mw.process_request(req2, None) - assert req2.meta['download_timeout'] == 30 + def test_override_splash_url(self): + req1 = scrapy.Request("http://example.com", meta={ + 'splash': { + 'endpoint': 'render.png', + 'splash_url': 'http://splash.example.com' + } + }) + req = self.mw.process_request(req1, None) + assert req.url == 'http://splash.example.com/render.png' + assert json.loads(req.body) == {'url': req1.url} + + def test_float_wait_arg(self): + req1 = scrapy.Request("http://example.com", meta={ + 'splash': { + 'endpoint': 'render.html', + 'args': {'wait': 0.5} + } + }) + req = self.mw.process_request(req1, None) + assert json.loads(req.body) == {'url': req1.url, 'wait': 0.5} + + def test_slot_policy_single_slot(self): + meta = {'splash': { + 'slot_policy': scrapyjs.SlotPolicy.SINGLE_SLOT + }} + + req1 = scrapy.Request("http://example.com/path?key=value", meta=meta) + req1 = self.mw.process_request(req1, None) + + req2 = scrapy.Request("http://fooexample.com/path?key=value", meta=meta) + req2 = self.mw.process_request(req2, None) + + assert req1.meta.get('download_slot') + assert req1.meta['download_slot'] == req2.meta['download_slot'] + + def test_slot_policy_per_domain(self): + meta = {'splash': { + 'slot_policy': scrapyjs.SlotPolicy.PER_DOMAIN + }} + + req1 = scrapy.Request("http://example.com/path?key=value", meta=meta) + req1 = self.mw.process_request(req1, None) + + req2 = scrapy.Request("http://example.com/path2", meta=meta) + req2 = self.mw.process_request(req2, None) + + req3 = scrapy.Request("http://fooexample.com/path?key=value", meta=meta) + req3 = self.mw.process_request(req3, None) + + assert req1.meta.get('download_slot') + assert req3.meta.get('download_slot') + + assert req1.meta['download_slot'] == req2.meta['download_slot'] + assert req1.meta['download_slot'] != req3.meta['download_slot'] + + def test_slot_policy_scrapy_default(self): + req = scrapy.Request("http://example.com", meta = {'splash': { + 'slot_policy': scrapyjs.SlotPolicy.SCRAPY_DEFAULT + }}) + req = self.mw.process_request(req, None) + assert 'download_slot' not in req.meta + + def test_adjust_timeout(self): + req1 = scrapy.Request("http://example.com", meta = { + 'splash': {'args': {'timeout': 60, 'html': 1}}, + + # download_timeout is always present, + # it is set by DownloadTimeoutMiddleware + 'download_timeout': 30, + }) + req1 = self.mw.process_request(req1, None) + assert req1.meta['download_timeout'] > 60 + + req2 = scrapy.Request("http://example.com", meta = { + 'splash': {'args': {'html': 1}}, + 'download_timeout': 30, + }) + req2 = self.mw.process_request(req2, None) + assert req2.meta['download_timeout'] == 30 + + def test_spider_attribute(self): + req_url = "http://scrapy.org" + req1 = scrapy.Request(req_url) + + spider = self.crawler._create_spider("foo") + spider.splash = {"args": {"images": 0}} + + req1 = self.mw.process_request(req1, spider) + self.assertIn("_splash_processed", req1.meta) + self.assertIn("render.json", req1.url) + self.assertIn("url", json.loads(req1.body)) + self.assertEqual(json.loads(req1.body).get("url"), req_url) + self.assertIn("images", json.loads(req1.body)) + + # spider attribute blank middleware disabled + spider.splash = {} + req2 = self.mw.process_request(req1, spider) + self.assertIsNone(req2) From 9e362e569b2c0e68660cce92f0334ecfdfa261ba Mon Sep 17 00:00:00 2001 From: Pawel Miech Date: Fri, 19 Jun 2015 14:27:48 +0200 Subject: [PATCH 05/10] Revert "[tests] refactors tests, adds tests for spider attr" This reverts commit bed8998a8a3a2db84d9ac115246d2de18b24b0ca. T# --- tests/test_middleware.py | 195 +++++++++++---------------------------- 1 file changed, 53 insertions(+), 142 deletions(-) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 089fe05..d1bbd02 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -5,6 +5,7 @@ import base64 import scrapy +from scrapy.core.engine import ExecutionEngine from scrapy.utils.test import get_crawler from scrapy.http import Response, TextResponse from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware @@ -698,8 +699,6 @@ def test_float_wait_arg(): req = mw.process_request(req1, None) assert json.loads(to_native_str(req.body)) == {'url': req1.url, 'wait': 0.5} - def __init__(self, delay=0.0): - self.delay = delay def test_slot_policy_single_slot(): mw = _get_mw() @@ -707,17 +706,20 @@ def test_slot_policy_single_slot(): 'slot_policy': scrapy_splash.SlotPolicy.SINGLE_SLOT }} -class MockedDownloader(object): +def test_slot_policy_single_slot(): + mw = _get_mw() + meta = {'splash': { + 'slot_policy': scrapyjs.SlotPolicy.SINGLE_SLOT + }} - def __init__(self): - self.slots = {} + req1 = scrapy.Request("http://example.com/path?key=value", meta=meta) + req1 = mw.process_request(req1, None) - def _get_slot_key(self, request, spider): - if 'download_slot' in request.meta: - return request.meta['download_slot'] + req2 = scrapy.Request("http://fooexample.com/path?key=value", meta=meta) + req2 = mw.process_request(req2, None) - key = urlparse_cached(request).hostname or '' - return key + assert req1.meta.get('download_slot') + assert req1.meta['download_slot'] == req2.meta['download_slot'] def test_slot_policy_per_domain(): mw = _get_mw() @@ -725,29 +727,26 @@ def test_slot_policy_per_domain(): 'slot_policy': scrapy_splash.SlotPolicy.PER_DOMAIN }} -class MockedEngine(object): - downloader = MockedDownloader() +def test_slot_policy_per_domain(): + mw = _get_mw() + meta = {'splash': { + 'slot_policy': scrapyjs.SlotPolicy.PER_DOMAIN + }} + req1 = scrapy.Request("http://example.com/path?key=value", meta=meta) + req1 = mw.process_request(req1, None) -class MiddlewareTest(unittest.TestCase): + req2 = scrapy.Request("http://example.com/path2", meta=meta) + req2 = mw.process_request(req2, None) - def setUp(self): - self.crawler = get_crawler(settings_dict={ - 'DOWNLOAD_HANDLERS': {'s3': None}, # for faster test running - }) - if not hasattr(self.crawler, 'logformatter'): - self.crawler.logformatter = None - self.crawler.engine = MockedEngine() - self.mw = SplashMiddleware.from_crawler(self.crawler) + req3 = scrapy.Request("http://fooexample.com/path?key=value", meta=meta) + req3 = mw.process_request(req3, None) - def test_nosplash(self): - req = scrapy.Request("http://example.com") - old_meta = copy.deepcopy(req.meta) - assert self.mw.process_request(req, None) is None - assert old_meta == req.meta + assert req1.meta.get('download_slot') + assert req3.meta.get('download_slot') - def test_splash_request(self): - req = SplashRequest("http://example.com?foo=bar&url=1&wait=100") + assert req1.meta['download_slot'] == req2.meta['download_slot'] + assert req1.meta['download_slot'] != req3.meta['download_slot'] def test_slot_policy_scrapy_default(): mw = _get_mw() @@ -757,118 +756,30 @@ def test_slot_policy_scrapy_default(): req = mw.process_request(req, None) assert 'download_slot' not in req.meta - expected_body = {'url': req.url} - expected_body.update(SplashRequest.default_splash_meta['args']) - assert json.loads(req2.body) == expected_body - - def test_splash_request_no_url(self): - lua_source = "function main(splash) return {result='ok'} end" - req1 = SplashRequest(meta={'splash': { - 'args': {'lua_source': lua_source}, - 'endpoint': 'execute', - }}) - req = self.mw.process_request(req1, None) - assert req.url == 'http://127.0.0.1:8050/execute' - assert json.loads(req.body) == { - 'url': 'about:blank', - 'lua_source': lua_source - } +def test_slot_policy_scrapy_default(): + mw = _get_mw() + req = scrapy.Request("http://example.com", meta = {'splash': { + 'slot_policy': scrapyjs.SlotPolicy.SCRAPY_DEFAULT + }}) + req = mw.process_request(req, None) + assert 'download_slot' not in req.meta - def test_override_splash_url(self): - req1 = scrapy.Request("http://example.com", meta={ - 'splash': { - 'endpoint': 'render.png', - 'splash_url': 'http://splash.example.com' - } - }) - req = self.mw.process_request(req1, None) - assert req.url == 'http://splash.example.com/render.png' - assert json.loads(req.body) == {'url': req1.url} - - def test_float_wait_arg(self): - req1 = scrapy.Request("http://example.com", meta={ - 'splash': { - 'endpoint': 'render.html', - 'args': {'wait': 0.5} - } - }) - req = self.mw.process_request(req1, None) - assert json.loads(req.body) == {'url': req1.url, 'wait': 0.5} - - def test_slot_policy_single_slot(self): - meta = {'splash': { - 'slot_policy': scrapyjs.SlotPolicy.SINGLE_SLOT - }} - - req1 = scrapy.Request("http://example.com/path?key=value", meta=meta) - req1 = self.mw.process_request(req1, None) - - req2 = scrapy.Request("http://fooexample.com/path?key=value", meta=meta) - req2 = self.mw.process_request(req2, None) - - assert req1.meta.get('download_slot') - assert req1.meta['download_slot'] == req2.meta['download_slot'] - - def test_slot_policy_per_domain(self): - meta = {'splash': { - 'slot_policy': scrapyjs.SlotPolicy.PER_DOMAIN - }} - - req1 = scrapy.Request("http://example.com/path?key=value", meta=meta) - req1 = self.mw.process_request(req1, None) - - req2 = scrapy.Request("http://example.com/path2", meta=meta) - req2 = self.mw.process_request(req2, None) - - req3 = scrapy.Request("http://fooexample.com/path?key=value", meta=meta) - req3 = self.mw.process_request(req3, None) - - assert req1.meta.get('download_slot') - assert req3.meta.get('download_slot') - - assert req1.meta['download_slot'] == req2.meta['download_slot'] - assert req1.meta['download_slot'] != req3.meta['download_slot'] - - def test_slot_policy_scrapy_default(self): - req = scrapy.Request("http://example.com", meta = {'splash': { - 'slot_policy': scrapyjs.SlotPolicy.SCRAPY_DEFAULT - }}) - req = self.mw.process_request(req, None) - assert 'download_slot' not in req.meta - - def test_adjust_timeout(self): - req1 = scrapy.Request("http://example.com", meta = { - 'splash': {'args': {'timeout': 60, 'html': 1}}, - - # download_timeout is always present, - # it is set by DownloadTimeoutMiddleware - 'download_timeout': 30, - }) - req1 = self.mw.process_request(req1, None) - assert req1.meta['download_timeout'] > 60 - - req2 = scrapy.Request("http://example.com", meta = { - 'splash': {'args': {'html': 1}}, - 'download_timeout': 30, - }) - req2 = self.mw.process_request(req2, None) - assert req2.meta['download_timeout'] == 30 - - def test_spider_attribute(self): - req_url = "http://scrapy.org" - req1 = scrapy.Request(req_url) - - spider = self.crawler._create_spider("foo") - spider.splash = {"args": {"images": 0}} - - req1 = self.mw.process_request(req1, spider) - self.assertIn("_splash_processed", req1.meta) - self.assertIn("render.json", req1.url) - self.assertIn("url", json.loads(req1.body)) - self.assertEqual(json.loads(req1.body).get("url"), req_url) - self.assertIn("images", json.loads(req1.body)) - - # spider attribute blank middleware disabled - spider.splash = {} - req2 = self.mw.process_request(req1, spider) - self.assertIsNone(req2) + +def test_adjust_timeout(): + mw = _get_mw() + req1 = scrapy.Request("http://example.com", meta = { + 'splash': {'args': {'timeout': 60, 'html': 1}}, + + # download_timeout is always present, + # it is set by DownloadTimeoutMiddleware + 'download_timeout': 30, + }) + req1 = mw.process_request(req1, None) + assert req1.meta['download_timeout'] > 60 + + req2 = scrapy.Request("http://example.com", meta = { + 'splash': {'args': {'html': 1}}, + 'download_timeout': 30, + }) + req2 = mw.process_request(req2, None) + assert req2.meta['download_timeout'] == 30 From 6a82190274021ad4f0b1c8acfc4cf06defdf2192 Mon Sep 17 00:00:00 2001 From: Pawel Miech Date: Fri, 19 Jun 2015 14:42:52 +0200 Subject: [PATCH 06/10] [#15/spider_attribute] adds proper tests --- tests/test_middleware.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index d1bbd02..0561af0 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -783,3 +783,25 @@ def test_adjust_timeout(): }) req2 = mw.process_request(req2, None) assert req2.meta['download_timeout'] == 30 + + +def test_spider_attribute(): + req_url = "http://scrapy.org" + req1 = scrapy.Request(req_url) + + spider = scrapy.Spider("example") + spider.splash = {"args": {"images": 0}} + + mw = _get_mw() + req1 = mw.process_request(req1, spider) + assert "_splash_processed" in req1.meta + assert "render.json" in req1.url + assert "url" in json.loads(req1.body) + assert json.loads(req1.body).get("url") == req_url + assert "images" in json.loads(req1.body) + assert req1.method == 'POST' + + # spider attribute blank middleware disabled + spider.splash = {} + req2 = mw.process_request(req1, spider) + assert req2 is None From 367add43452ed032e8d6311703a82db6f7195963 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 12 Aug 2019 17:06:19 +0200 Subject: [PATCH 07/10] Clean up after a complex rebase --- tests/test_middleware.py | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 0561af0..2bc3f8e 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -706,12 +706,6 @@ def test_slot_policy_single_slot(): 'slot_policy': scrapy_splash.SlotPolicy.SINGLE_SLOT }} -def test_slot_policy_single_slot(): - mw = _get_mw() - meta = {'splash': { - 'slot_policy': scrapyjs.SlotPolicy.SINGLE_SLOT - }} - req1 = scrapy.Request("http://example.com/path?key=value", meta=meta) req1 = mw.process_request(req1, None) @@ -721,16 +715,11 @@ def test_slot_policy_single_slot(): assert req1.meta.get('download_slot') assert req1.meta['download_slot'] == req2.meta['download_slot'] -def test_slot_policy_per_domain(): - mw = _get_mw() - meta = {'splash': { - 'slot_policy': scrapy_splash.SlotPolicy.PER_DOMAIN - }} def test_slot_policy_per_domain(): mw = _get_mw() meta = {'splash': { - 'slot_policy': scrapyjs.SlotPolicy.PER_DOMAIN + 'slot_policy': scrapy_splash.SlotPolicy.PER_DOMAIN }} req1 = scrapy.Request("http://example.com/path?key=value", meta=meta) @@ -748,18 +737,11 @@ def test_slot_policy_per_domain(): assert req1.meta['download_slot'] == req2.meta['download_slot'] assert req1.meta['download_slot'] != req3.meta['download_slot'] -def test_slot_policy_scrapy_default(): - mw = _get_mw() - req = scrapy.Request("http://example.com", meta = {'splash': { - 'slot_policy': scrapy_splash.SlotPolicy.SCRAPY_DEFAULT - }}) - req = mw.process_request(req, None) - assert 'download_slot' not in req.meta def test_slot_policy_scrapy_default(): mw = _get_mw() req = scrapy.Request("http://example.com", meta = {'splash': { - 'slot_policy': scrapyjs.SlotPolicy.SCRAPY_DEFAULT + 'slot_policy': scrapy_splash.SlotPolicy.SCRAPY_DEFAULT }}) req = mw.process_request(req, None) assert 'download_slot' not in req.meta From ed2ca004a2f41488bc2a49d094775f9e84fbcaf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 12 Aug 2019 17:16:30 +0200 Subject: [PATCH 08/10] Document how to use the splash spider attribute --- README.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.rst b/README.rst index 91b8855..e20f654 100644 --- a/README.rst +++ b/README.rst @@ -162,6 +162,18 @@ Alternatively, you can use regular scrapy.Request and } }) +It is also possible to configure Splash for all requests in a Spider by default +using a ``splash`` spider attribute:: + + class MySpider(Spider): + name = 'myspider' + splash = { + # … + } + +If you use a ``splash`` spider attribute, you can still override those Splash +settings for specific requests using the ``splash`` request meta key. + Use ``request.meta['splash']`` API in middlewares or when scrapy.Request subclasses are used (there is also ``SplashFormRequest`` described below). For example, ``meta['splash']`` allows to create a middleware which enables From 03aaab91d06b3e25b9ff252b976b985c950ee646 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 12 Aug 2019 17:20:29 +0200 Subject: [PATCH 09/10] Rename dont_proxy to dont_splash and document it --- README.rst | 4 +++- scrapy_splash/middleware.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index e20f654..f5c57f1 100644 --- a/README.rst +++ b/README.rst @@ -172,7 +172,9 @@ using a ``splash`` spider attribute:: } If you use a ``splash`` spider attribute, you can still override those Splash -settings for specific requests using the ``splash`` request meta key. +settings for specific requests using the ``splash`` request meta key, or +disable Splash completely setting the ``dont_splash`` request meta key to +``True``. Use ``request.meta['splash']`` API in middlewares or when scrapy.Request subclasses are used (there is also ``SplashFormRequest`` described below). diff --git a/scrapy_splash/middleware.py b/scrapy_splash/middleware.py index 69efcc3..70a03b2 100644 --- a/scrapy_splash/middleware.py +++ b/scrapy_splash/middleware.py @@ -258,7 +258,7 @@ def _remote_keys(self): return self.crawler.spider.state[self.remote_keys_key] def _get_splash_options(self, request, spider): - if request.meta.get("dont_proxy"): + if request.meta.get("dont_splash") is True: return spider_options = getattr(spider, "splash", {}) request_options = request.meta.get("splash") From 40961cb1746519309f95c0f73c7ef4fc1f9a6aba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 12 Aug 2019 17:34:42 +0200 Subject: [PATCH 10/10] Improve test coverage for the splash spider attribute --- tests/test_middleware.py | 52 +++++++++++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 2bc3f8e..12e53e8 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- + from __future__ import absolute_import import copy import json @@ -769,21 +770,54 @@ def test_adjust_timeout(): def test_spider_attribute(): req_url = "http://scrapy.org" + spider = scrapy.Spider("example") + mw = _get_mw() + req1 = scrapy.Request(req_url) + spider.splash = {"args": {"images": 0}} + + mw = _get_mw() + req2 = mw.process_request(req1, spider) + assert "_splash_processed" in req2.meta + assert "render.json" in req2.url + request_data = json.loads(req2.body.decode('utf8')) + assert "url" in request_data + assert request_data.get("url") == req_url + assert "images" in request_data + assert req2.method == 'POST' + + response = Response(req_url, request=req2) + response2 = mw.process_response(req2, response, spider) + assert response2 is not response + +def test_spider_attribute_dont_splash(): + req_url = "http://scrapy.org" spider = scrapy.Spider("example") + mw = _get_mw() + + req1 = scrapy.Request(req_url, meta={'dont_splash': True}) spider.splash = {"args": {"images": 0}} + req2 = mw.process_request(req1, spider) + assert req2 is None + + response = Response(req_url, request=req1) + response2 = mw.process_response(req1, response, spider) + assert response2 is response + + +def test_spider_attribute_blank(): + req_url = "http://scrapy.org" + spider = scrapy.Spider("example") mw = _get_mw() - req1 = mw.process_request(req1, spider) - assert "_splash_processed" in req1.meta - assert "render.json" in req1.url - assert "url" in json.loads(req1.body) - assert json.loads(req1.body).get("url") == req_url - assert "images" in json.loads(req1.body) - assert req1.method == 'POST' - - # spider attribute blank middleware disabled + + req1 = scrapy.Request(req_url) spider.splash = {} + req2 = mw.process_request(req1, spider) assert req2 is None + + response = Response(req_url, request=req1) + response2 = mw.process_response(req1, response, spider) + assert response2 is response