Skip to content

Commit 5657247

Browse files
committed
Added node-fetch as a scraper.
Added tests.
1 parent 683f0d0 commit 5657247

2 files changed

Lines changed: 33 additions & 0 deletions

File tree

src/mappings/crawlers.php

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -513,6 +513,7 @@ public static function get() : array {
513513
'http-client/' => new props('any', $fn['scraper']),
514514
'HttpClient/' => new props('any', $fn['scraper']),
515515
'PowerShell/' => new props('start', $fn['scraper']),
516+
'node-fetch' => new props('exact', $fn['scraper']),
516517
'OAI-SearchBot/' => new props('start', $fn['search']),
517518
'iaskspider/' => new props('start', $fn['search']),
518519
'MeltwaterNews' => new props('start', fn (string $value) : array => [

tests/crawlersTest.php

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1401,6 +1401,38 @@ public function testScrapers() : void {
14011401
'appname' => 'grub-client',
14021402
'appversion' => '1.5.3',
14031403
'url' => 'http://grub.org'
1404+
],
1405+
'node-fetch' => [
1406+
'string' => 'node-fetch',
1407+
'type' => 'robot',
1408+
'category' => 'scraper',
1409+
'app' => 'Node Fetch',
1410+
'appname' => 'node-fetch'
1411+
],
1412+
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) jsdom/13.0.0' => [
1413+
'string' => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) jsdom/13.0.0',
1414+
'type' => 'robot',
1415+
'category' => 'scraper',
1416+
'architecture' => 'x86',
1417+
'bits' => 64,
1418+
'kernel' => 'Windows NT',
1419+
'platform' => 'Windows',
1420+
'platformversion' => '10',
1421+
'app' => 'Jsdom',
1422+
'appname' => 'jsdom',
1423+
'appversion' => '13.0.0'
1424+
],
1425+
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) jsdom/13.0.0' => [
1426+
'string' => 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) jsdom/13.0.0',
1427+
'type' => 'robot',
1428+
'category' => 'scraper',
1429+
'architecture' => 'x86',
1430+
'bits' => 64,
1431+
'kernel' => 'Linux',
1432+
'platform' => 'Linux',
1433+
'app' => 'Jsdom',
1434+
'appname' => 'jsdom',
1435+
'appversion' => '13.0.0'
14041436
]
14051437
];
14061438
foreach ($strings AS $ua => $item) {

0 commit comments

Comments
 (0)