crawler-buddy/init_browser_setup.json at main · rumca-js/crawler-buddy · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
[
  {
    "crawler_name": "CurlCffiCrawler",
    "crawler_class_name": "CurlCffiCrawler",
    "settings": {
      "timeout_s": 40
    }
  },
  {
    "crawler_name": "RequestsCrawler",
    "crawler_class_name": "RequestsCrawler",
    "settings": {
      "timeout_s": 20
    }
  },
  {
    "crawler_name": "StealthRequestsCrawler",
    "crawler_class_name": "StealthRequestsCrawler",
    "settings": {
      "timeout_s": 40
    }
  },
  {
    "crawler_name": "HttpMorphCrawler",
    "crawler_class_name": "HttpMorphCrawler",
    "settings": {
      "timeout_s": 40
    }
  },
  {
    "crawler_name": "HttpCloakCrawler",
    "crawler_class_name": "HttpCloakCrawler",
    "settings": {
      "timeout_s": 40
    }
  },
  {
    "crawler_name": "HttpxCrawler",
    "crawler_class_name": "HttpxCrawler",
    "settings": {
      "timeout_s": 40
    }
  },
  {
    "crawler_name": "YtdlpCrawler",
    "crawler_class_name": "YtdlpCrawler",
    "settings": {
      "timeout_s": 40
    }
  },
  {
    "crawler_name": "SeleniumUndetected",
    "crawler_class_name": "ScriptCrawler",
    "settings": {
      "driver_executable": "/usr/bin/chromedriver",
      "timeout_s": 60
    }
  },
  {
    "crawler_name": "SeleniumChromeHeadless",
    "crawler_class_name": "ScriptCrawler",
    "settings": {
      "driver_executable": "/usr/bin/chromedriver",
      "timeout_s": 60
    }
  },
  {
    "crawler_name": "SeleniumChromeFull",
    "crawler_class_name": "SeleniumChromeFull",
    "settings": {
      "driver_executable": "/usr/bin/chromedriver",
      "timeout_s": 60,
      "delay_s": 10
    }
  },
  {
    "crawler_name": "CrawleeBeautifulScript",
    "crawler_class_name": "ScriptCrawler",
    "settings": {
      "script": "poetry run python crawleebeautifulsoup.py",
      "timeout_s": 50
    }
  },
  {
    "crawler_name": "CrawleePlaywrightScript",
    "crawler_class_name": "ScriptCrawler",
    "settings": {
      "script": "poetry run python crawleeplaywright.py",
      "timeout_s": 50
    }
  },
  {
    "crawler_name": "ScrapyCrawler",
    "crawler_class_name": "ScrapyCrawler",
    "settings": {
      "script": "poetry run python crawlerscrapy.py",
      "timeout_s": 50
    }
  },
  {
    "crawler_name": "SeleniumWireFull",
    "crawler_class_name": "ScriptCrawler",
    "settings": {
      "driver_executable": "/usr/bin/chromedriver",
      "timeout_s": 60,
      "delay_s": 10
    }
  },
  {
    "crawler_name": "SeleniumBase",
    "crawler_class_name": "ScriptCrawler",
    "settings": {
      "timeout_s": 60,
      "delay_s": 10
    }
  }
]