crawlee-python/src/crawlee/crawlers/__init__.py at 39a52ef3e0911be3677d0377989f7a0ffa5badff · Mantisus/crawlee-python · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
from crawlee._utils.try_import import install_import_hook as _install_import_hook
from crawlee._utils.try_import import try_import as _try_import

from ._abstract_http import AbstractHttpCrawler, AbstractHttpParser, HttpCrawlerOptions, ParsedHttpCrawlingContext
from ._basic import BasicCrawler, BasicCrawlerOptions, BasicCrawlingContext, ContextPipeline
from ._http import HttpCrawler, HttpCrawlingContext, HttpCrawlingResult

_install_import_hook(__name__)

# The following imports use try_import to handle optional dependencies, as they may not always be available.

with _try_import(__name__, 'BeautifulSoupCrawler', 'BeautifulSoupCrawlingContext', 'BeautifulSoupParserType'):
    from ._beautifulsoup import BeautifulSoupCrawler, BeautifulSoupCrawlingContext, BeautifulSoupParserType

with _try_import(__name__, 'ParselCrawler', 'ParselCrawlingContext'):
    from ._parsel import ParselCrawler, ParselCrawlingContext

with _try_import(
    __name__,
    'PlaywrightCrawler',
    'PlaywrightCrawlingContext',
    'PlaywrightPostNavCrawlingContext',
    'PlaywrightPreNavCrawlingContext',
):
    from ._playwright import (
        PlaywrightCrawler,
        PlaywrightCrawlingContext,
        PlaywrightPostNavCrawlingContext,
        PlaywrightPreNavCrawlingContext,
    )

with _try_import(
    __name__,
    'AdaptivePlaywrightCrawler',
    'AdaptivePlaywrightCrawlingContext',
    'AdaptivePlaywrightPostNavCrawlingContext',
    'AdaptivePlaywrightPreNavCrawlingContext',
    'AdaptivePlaywrightCrawlerStatisticState',
    'RenderingType',
    'RenderingTypePrediction',
    'RenderingTypePredictor',
):
    from ._adaptive_playwright import (
        AdaptivePlaywrightCrawler,
        AdaptivePlaywrightCrawlerStatisticState,
        AdaptivePlaywrightCrawlingContext,
        AdaptivePlaywrightPostNavCrawlingContext,
        AdaptivePlaywrightPreNavCrawlingContext,
        RenderingType,
        RenderingTypePrediction,
        RenderingTypePredictor,
    )


__all__ = [
    'AbstractHttpCrawler',
    'AbstractHttpParser',
    'AdaptivePlaywrightCrawler',
    'AdaptivePlaywrightCrawlerStatisticState',
    'AdaptivePlaywrightCrawlingContext',
    'AdaptivePlaywrightPostNavCrawlingContext',
    'AdaptivePlaywrightPreNavCrawlingContext',
    'BasicCrawler',
    'BasicCrawlerOptions',
    'BasicCrawlingContext',
    'BeautifulSoupCrawler',
    'BeautifulSoupCrawlingContext',
    'BeautifulSoupParserType',
    'ContextPipeline',
    'HttpCrawler',
    'HttpCrawlerOptions',
    'HttpCrawlingContext',
    'HttpCrawlingResult',
    'ParsedHttpCrawlingContext',
    'ParselCrawler',
    'ParselCrawlingContext',
    'PlaywrightCrawler',
    'PlaywrightCrawlingContext',
    'PlaywrightPostNavCrawlingContext',
    'PlaywrightPreNavCrawlingContext',
    'RenderingType',
    'RenderingTypePrediction',
    'RenderingTypePredictor',
]