Skip to content

Commit dc9b99f

Browse files
committed
Update tesmplates and e2e tests to use specialized Playwright docker images
1 parent 7663404 commit dc9b99f

8 files changed

Lines changed: 57 additions & 22 deletions

File tree

src/crawlee/project_template/cookiecutter.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"project_name": "crawlee-python-project",
33
"__package_name": "{{ cookiecutter.project_name|lower|replace('-', '_') }}",
4-
"crawler_type": ["beautifulsoup", "parsel", "playwright", "playwright-camoufox"],
4+
"crawler_type": ["beautifulsoup", "parsel", "playwright", "playwright-camoufox", "playwright-chrome", "playwright-firefox", "playwright-webkit"],
55
"__crawler_type": "{{ cookiecutter.crawler_type|lower|replace('-', '_') }}",
66
"http_client": ["impit", "httpx", "curl-impersonate"],
77
"package_manager": ["poetry", "pip", "uv"],
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# % extends 'main.py'
2+
3+
# % block import
4+
from crawlee.crawlers import PlaywrightCrawler
5+
# % endblock
6+
7+
# % block instantiation
8+
crawler = PlaywrightCrawler(
9+
request_handler=router,
10+
headless=True,
11+
max_requests_per_crawl=10,
12+
browser_type="chrome",
13+
{{ self.http_client_instantiation() }})
14+
# % endblock
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# % extends 'main.py'
2+
3+
# % block import
4+
from crawlee.crawlers import PlaywrightCrawler
5+
# % endblock
6+
7+
# % block instantiation
8+
crawler = PlaywrightCrawler(
9+
request_handler=router,
10+
headless=True,
11+
max_requests_per_crawl=10,
12+
browser_type="firefox",
13+
{{ self.http_client_instantiation() }})
14+
# % endblock
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# % extends 'main.py'
2+
3+
# % block import
4+
from crawlee.crawlers import PlaywrightCrawler
5+
# % endblock
6+
7+
# % block instantiation
8+
crawler = PlaywrightCrawler(
9+
request_handler=router,
10+
headless=True,
11+
max_requests_per_crawl=10,
12+
browser_type="webkit",
13+
{{ self.http_client_instantiation() }})
14+
# % endblock

src/crawlee/project_template/templates/routes_playwright_camoufox.py

Lines changed: 0 additions & 19 deletions
This file was deleted.

src/crawlee/project_template/{{cookiecutter.project_name}}/Dockerfile

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,13 @@
44
# % if cookiecutter.crawler_type == 'playwright'
55
FROM apify/actor-python-playwright:3.13
66
# % elif cookiecutter.crawler_type == 'playwright-camoufox'
7-
# Currently camoufox has issues installing on Python 3.13
8-
FROM apify/actor-python-playwright:3.12
7+
FROM apify/actor-python-playwright-camoufox:3.13
8+
# % elif cookiecutter.crawler_type == 'playwright-chrome'
9+
FROM apify/actor-python-playwright-chrome:3.13
10+
# % elif cookiecutter.crawler_type == 'playwright-firefox'
11+
FROM apify/actor-python-playwright-firefox:3.13
12+
# % elif cookiecutter.crawler_type == 'playwright-webkit'
13+
FROM apify/actor-python-playwright-webkit:3.13
914
# % else
1015
FROM apify/actor-python:3.13
1116
# % endif
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1 +1,5 @@
1+
# % if cookiecutter.crawler_type.startswith('playwright')
2+
# % include 'routes_playwright.py'
3+
# % else
14
# % include 'routes_%s.py' % cookiecutter.__crawler_type
5+
# % endif

tests/e2e/project_template/test_static_crawlers_templates.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@
2020
'crawler_type',
2121
[
2222
pytest.param('playwright-camoufox', marks=pytest.mark.playwright_camoufox),
23+
pytest.param('playwright-chrome', marks=pytest.mark.playwright_camoufox),
24+
pytest.param('playwright-firefox', marks=pytest.mark.playwright_camoufox),
25+
pytest.param('playwright-webkit', marks=pytest.mark.playwright_camoufox),
2326
pytest.param('playwright', marks=pytest.mark.playwright),
2427
pytest.param('parsel', marks=pytest.mark.parsel),
2528
pytest.param('beautifulsoup', marks=pytest.mark.beautifulsoup),

0 commit comments

Comments
 (0)