Skip to content

Commit 607587c

Browse files
hexydechexydec
committed
Added Blazer browser (PalmOS).
Added `PalmOS` and updated Windows captures to ignore PalmOS. Added a new `ai` category, to mark crawlers that are specifically for AI. Added specific mispellings that indicate a web scraper instead of the human it is trying to spoof. Added and updated tests.
1 parent 55ce30b commit 607587c

7 files changed

Lines changed: 224 additions & 16 deletions

File tree

‎src/mappings/browsers.php‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,12 @@ public static function get() : array {
198198
'browser' => $value,
199199
'browserversion' => $tokens[$i + 1]
200200
]),
201+
'Blazer' => new props('start', fn (string $value) : array => [
202+
'type' => 'human',
203+
'browser' => 'Blazer',
204+
'browserversion' => \mb_substr($value, 7),
205+
'engine' => 'Proprietary'
206+
]),
201207
'Edg/' => new props('start', $fn['browserslash']),
202208
'EdgA/' => new props('start', $fn['browserslash']),
203209
'Edge/' => new props('start', $fn['browserslash']),

‎src/mappings/categories.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,11 @@ public static function get() : array {
1919
'type' => 'human',
2020
'category' => 'mobile'
2121
]),
22-
'Moblie' => new props('exact', [ // some samsung devices mispelt it
22+
'Phone' => new props('exact', [
2323
'type' => 'human',
2424
'category' => 'mobile'
2525
]),
26-
'Phone' => new props('exact', [
26+
'PDA' => new props('exact', [
2727
'type' => 'human',
2828
'category' => 'mobile'
2929
]),

‎src/mappings/crawlers.php‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,7 @@ public static function get() : array {
132132
'search' => fn (string $value) : array => self::getApp($value, ['category' => 'search']),
133133
'ads' => fn (string $value) : array => self::getApp($value, ['category' => 'ads']),
134134
'validator' => fn (string $value) : array => self::getApp($value, ['category' => 'validator']),
135+
'ai' => fn (string $value) : array => self::getApp($value, ['category' => 'ai']),
135136
'feed' => fn (string $value) : array => self::getApp($value, \array_merge(
136137
\str_contains($value, 'WhatsApp/') ? [
137138
'app' => 'WhatsApp'
@@ -159,8 +160,17 @@ public static function get() : array {
159160
'map' => fn (string $value) : ?array => self::getApp($value)
160161
];
161162
return [
163+
'Mozlila/' => new props('start', [
164+
'type' => 'robot',
165+
'categpry' => 'scraper'
166+
]),
167+
'Moblie' => new props('exact', [ // some samsung devices mispelt it
168+
'type' => 'robot',
169+
'category' => 'scraper'
170+
]),
162171
'Yahoo! Slurp' => new props('exact', $fn['search']),
163172
'facebookexternalhit/' => new props('start', $fn['feed']),
173+
'facebookcatalog/' => new props('start', $fn['feed']),
164174
'Google-Site-Verification/' => new props('start', $fn['validator']),
165175
'Google-InspectionTool/' => new props('start', $fn['validator']),
166176
'Google-Read-Aloud' => new props('exact', $fn['feed']),
@@ -184,6 +194,7 @@ public static function get() : array {
184194
}),
185195
'okhttp' => new props('start', $fn['scraper']),
186196
'python' => new props('start', $fn['scraper']),
197+
'grpc-python/' => new props('start', $fn['scraper']),
187198
'jsdom/' => new props('start', $fn['scraper']),
188199
'Nessus' => new props('start', $fn['monitor']),
189200
'monitoring360bot' => new props('start', $fn['monitor']),
@@ -230,6 +241,7 @@ public static function get() : array {
230241
'PRTGCloudBot/' => new props('start', $fn['monitor']),
231242
'Site24x7' => new props('exact', $fn['monitor']),
232243
'StatusCake' => new props('exact', $fn['monitor']),
244+
'AWS Network Health' => new props('start', $fn['monitor']),
233245
'adbeat.com' => new props('start', fn (string $value) : array => [
234246
'type' => 'robot',
235247
'category' => 'ads',
@@ -271,6 +283,8 @@ public static function get() : array {
271283
'appversion' => \mb_substr($value, 37)
272284
]),
273285
'Pro-Sitemaps/' => new props('start', $fn['crawler']),
286+
'omgili/' => new props('start', $fn['crawler']),
287+
'CCBot/' => new props('start', $fn['crawler']),
274288
'Chrome Privacy Preserving Prefetch Proxy' => new props('exact', $fn['feed']),
275289
'ViberUrlDownloader' => new props('exact', $fn['feed']),
276290
'Google-Lens' => new props('exact', $fn['feed']),
@@ -281,7 +295,7 @@ public static function get() : array {
281295
'W3C-checklink/' => new props('start', $fn['validator']),
282296
'CSSCheck/' => new props('start', $fn['validator']),
283297
'Let\'s Encrypt validation server' => new props('exact', $fn['validator']),
284-
'SEO-Macroscope/' => new props('exact', $fn['validator']),
298+
'SEO-Macroscope/' => new props('start', $fn['validator']),
285299
'Electronic Frontier Foundation\'s Do Not Track Verifier' => new props('exact', $fn['validator']),
286300
'Expanse' => new props('start', $fn['crawler']),
287301
'eCairn-Grabber/' => new props('start', $fn['scraper']),
@@ -313,6 +327,14 @@ public static function get() : array {
313327
'http-client/' => new props('any', $fn['scraper']),
314328
'HttpClient/' => new props('any', $fn['scraper']),
315329
'PowerShell/' => new props('start', $fn['scraper']),
330+
'GPTBot/' => new props('start', $fn['ai']),
331+
'Diffbot/' => new props('start', $fn['ai']),
332+
'Amazonbot/' => new props('start', $fn['ai']),
333+
'Applebot/' => new props('start', $fn['ai']),
334+
'PerplexityBot/' => new props('start', $fn['ai']),
335+
'YouBot/' => new props('start', $fn['ai']),
336+
'Google-Extended' => new props('start', $fn['ai']),
337+
'ChatGPT-User/' => new props('start', $fn['feed']),
316338
'Validator' => new props('any', $fn['validator']),
317339
'feed' => new props('any', $fn['feed']),
318340
'spider' => new props('any', $fn['crawler']),

‎src/mappings/platforms.php‎

Lines changed: 42 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,24 @@ public static function get() : array {
7777
return [
7878

7979
// platforms
80+
'PalmSource' => new props('start', fn (string $value) : array => [
81+
'type' => 'human',
82+
'category' => 'mobile',
83+
'platform' => 'PalmOS',
84+
'platformversion' => \mb_substr($value, 11) ?: null,
85+
'kernel' => 'AMX 68000',
86+
'architecture' => 'arm',
87+
'bits' => 32,
88+
]),
89+
'PalmOS' => new props('start', fn (string $value) : array => [
90+
'type' => 'human',
91+
'category' => 'mobile',
92+
'platform' => 'PalmOS',
93+
'platformversion' => \mb_substr($value, 7) ?: null,
94+
'kernel' => 'AMX 68000',
95+
'architecture' => 'arm',
96+
'bits' => 32,
97+
]),
8098
'Windows NT ' => new props('any', $fn['platformwindows']),
8199
'Windows Phone' => new props('start', function (string $value) : array {
82100
$version = \mb_substr($value, 14);
@@ -88,15 +106,22 @@ public static function get() : array {
88106
'kernel' => \intval($version) >= 8 ? 'Windows NT' : 'Windows CE'
89107
];
90108
}),
91-
'Win98' => new props('start', [
92-
'type' => 'human',
93-
'category' => 'desktop',
94-
'architecture' => 'x86',
95-
'bits' => 32,
96-
'kernel' => 'MS-DOS',
97-
'platform' => 'Windows',
98-
'platformversion' => '98'
99-
]),
109+
'Win98' => new props('start', function (string $value, int $i, array $tokens) : array {
110+
foreach ($tokens AS $item) {
111+
if (\str_starts_with($item, 'PalmSource')) {
112+
return [];
113+
}
114+
}
115+
return [
116+
'type' => 'human',
117+
'category' => 'desktop',
118+
'architecture' => 'x86',
119+
'bits' => 32,
120+
'kernel' => 'MS-DOS',
121+
'platform' => 'Windows',
122+
'platformversion' => '98'
123+
];
124+
}),
100125
'Win32' => new props('exact', [
101126
'type' => 'human',
102127
'category' => 'desktop',
@@ -119,7 +144,14 @@ public static function get() : array {
119144
'platform' => 'Windows',
120145
'platformversion' => \mb_substr($value, 5)
121146
]),
122-
'Windows' => new props('any', $fn['platformwindows']),
147+
'Windows' => new props('any', function (string $value, int $i, array $tokens) use ($fn) : array {
148+
foreach ($tokens AS $item) {
149+
if (\str_starts_with($item, 'PalmSource')) {
150+
return [];
151+
}
152+
}
153+
return $fn['platformwindows']($value, $i, $tokens);
154+
}),
123155
'Mac OS X' => new props('any', function (string $value) : array {
124156
$version = \str_replace('_', '.', \mb_substr($value, \mb_stripos($value, 'Mac OS X') + 9));
125157
$register = $version && \intval(\explode('.', $version)[1] ?? 0) >= 6 ? 64 : null;

‎tests/browsersTest.php‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -342,7 +342,7 @@ public function testSafari() : void {
342342
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)' => [
343343
'string' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)',
344344
'type' => 'robot',
345-
'category' => 'search',
345+
'category' => 'ai',
346346
'app' => 'AppleBot',
347347
'appname' => 'Applebot',
348348
'appversion' => '0.1',
@@ -1863,6 +1863,21 @@ public function testOther() : void {
18631863
'engineversion' => '122.0.0.0',
18641864
'browser' => 'Huawei Browser',
18651865
'browserversion' => '14.0.0.322'
1866+
],
1867+
'Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; PalmSource/hspr-H102; Blazer/4.0) 16;320x320' => [
1868+
'string' => 'Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; PalmSource/hspr-H102; Blazer/4.0) 16;320x320',
1869+
'type' => 'human',
1870+
'category' => 'mobile',
1871+
'architecture' => 'arm',
1872+
'bits' => 32,
1873+
'kernel' => 'AMX 68000',
1874+
'platform' => 'PalmOS',
1875+
'platformversion' => 'hspr-H102',
1876+
'engine' => 'Proprietary',
1877+
'browser' => 'Blazer',
1878+
'browserversion' => '4.0',
1879+
'width' => '320',
1880+
'height' => '320'
18661881
]
18671882
];
18681883
foreach ($strings AS $ua => $item) {

‎tests/crawlersTest.php‎

Lines changed: 107 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ public function testSearch() : void {
9999
'appname' => 'Applebot',
100100
'appversion' => '0.1',
101101
'type' => 'robot',
102-
'category' => 'search',
102+
'category' => 'ai',
103103
'architecture' => 'x86',
104104
'bits' => 64,
105105
'processor' => 'Intel',
@@ -116,7 +116,7 @@ public function testSearch() : void {
116116
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)' => [
117117
'string' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15 (Applebot/0.1; +http://www.apple.com/go/applebot)',
118118
'type' => 'robot',
119-
'category' => 'search',
119+
'category' => 'ai',
120120
'app' => 'AppleBot',
121121
'appname' => 'Applebot',
122122
'appversion' => '0.1',
@@ -998,4 +998,109 @@ public function testMonitors() : void {
998998
$this->assertEquals($item, \array_filter((array) agentzero::parse($ua), fn(mixed $item) : mixed => $item !== null), $ua);
999999
}
10001000
}
1001+
1002+
public function testAi() : void {
1003+
$strings = [
1004+
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML\, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)' => [
1005+
'string' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML\, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)',
1006+
'type' => 'robot',
1007+
'category' => 'ai',
1008+
'vendor' => 'Apple',
1009+
'device' => 'Macintosh',
1010+
'processor' => 'Intel',
1011+
'architecture' => 'x86',
1012+
'bits' => 64,
1013+
'kernel' => 'Linux',
1014+
'platform' => 'Mac OS X',
1015+
'platformversion' => '10.10.1',
1016+
'engine' => 'WebKit',
1017+
'engineversion' => '600.2.5',
1018+
'browser' => 'Safari',
1019+
'browserversion' => '8.0.2',
1020+
'app' => 'Amazonbot',
1021+
'appname' => 'Amazonbot',
1022+
'appversion' => '0.1',
1023+
'url' => 'https://developer.amazon.com/support/amazonbot'
1024+
],
1025+
'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2 (.NET CLR 3.5.30729; Diffbot/0.1; +http://www.diffbot.com)' => [
1026+
'string' => 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2 (.NET CLR 3.5.30729; Diffbot/0.1; +http://www.diffbot.com)',
1027+
'type' => 'robot',
1028+
'category' => 'ai',
1029+
'kernel' => 'Windows NT',
1030+
'platform' => 'Windows',
1031+
'platformversion' => 'XP',
1032+
'engine' => 'Gecko',
1033+
'engineversion' => '20090729',
1034+
'browser' => 'Firefox',
1035+
'browserversion' => '3.5.2',
1036+
'language' => 'en-US',
1037+
'app' => 'Diffbot',
1038+
'appname' => 'Diffbot',
1039+
'appversion' => '0.1',
1040+
'url' => 'http://www.diffbot.com'
1041+
],
1042+
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Safari/605.1.15 (Applebot/0.1)' => [
1043+
'string' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1.1 Safari/605.1.15 (Applebot/0.1)',
1044+
'type' => 'robot',
1045+
'category' => 'ai',
1046+
'vendor' => 'Apple',
1047+
'device' => 'Macintosh',
1048+
'processor' => 'Intel',
1049+
'architecture' => 'x86',
1050+
'bits' => '64',
1051+
'kernel' => 'Linux',
1052+
'platform' => 'Mac OS X',
1053+
'platformversion' => '10.14.5',
1054+
'engine' => 'WebKit',
1055+
'engineversion' => '605.1.15',
1056+
'browser' => 'Safari',
1057+
'browserversion' => '12.1.1',
1058+
'app' => 'AppleBot',
1059+
'appname' => 'Applebot',
1060+
'appversion' => '0.1'
1061+
],
1062+
'Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4 (Applebot/0.1; +http://www.apple.com/go/applebot)' => [
1063+
'string' => 'Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4 (Applebot/0.1; +http://www.apple.com/go/applebot)',
1064+
'type' => 'robot',
1065+
'category' => 'ai',
1066+
'vendor' => 'Apple',
1067+
'device' => 'iPhone',
1068+
'model' => '12B410',
1069+
'architecture' => 'arm',
1070+
'bits' => 64,
1071+
'kernel' => 'Linux',
1072+
'platform' => 'iOS',
1073+
'platformversion' => '8.1',
1074+
'engine' => 'WebKit',
1075+
'engineversion' => '600.1.4',
1076+
'browser' => 'Safari',
1077+
'browserversion' => '8.0',
1078+
'app' => 'AppleBot',
1079+
'appname' => 'Applebot',
1080+
'appversion' => '0.1',
1081+
'url' => 'http://www.apple.com/go/applebot'
1082+
],
1083+
'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)' => [
1084+
'string' => 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)',
1085+
'type' => 'robot',
1086+
'category' => 'ai',
1087+
'app' => 'PerplexityBot',
1088+
'appname' => 'PerplexityBot',
1089+
'appversion' => '1.0',
1090+
'url' => 'https://perplexity.ai/perplexitybot'
1091+
],
1092+
'Mozilla/5.0 (compatible; YouBot/1.0; +https://about.you.com/youbot/)' => [
1093+
'string' => 'Mozilla/5.0 (compatible; YouBot/1.0; +https://about.you.com/youbot/)',
1094+
'type' => 'robot',
1095+
'category' => 'ai',
1096+
'app' => 'YouBot',
1097+
'appname' => 'YouBot',
1098+
'appversion' => '1.0',
1099+
'url' => 'https://about.you.com/youbot/'
1100+
]
1101+
];
1102+
foreach ($strings AS $ua => $item) {
1103+
$this->assertEquals($item, \array_filter((array) agentzero::parse($ua), fn(mixed $item) : mixed => $item !== null), $ua);
1104+
}
1105+
}
10011106
}

‎tests/platformsTest.php‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1450,6 +1450,34 @@ public function testOther() : void {
14501450
'browserversion' => '3.0.0',
14511451
'engine' => 'Blink',
14521452
'engineversion' => '83.0.4103.122'
1453+
],
1454+
'Mozilla/4.0 (PDA; PalmOS/sony/model prmr/Revision:1.1.54 (en)) NetFront/3.0' => [
1455+
'string' => 'Mozilla/4.0 (PDA; PalmOS/sony/model prmr/Revision:1.1.54 (en)) NetFront/3.0',
1456+
'type' => 'human',
1457+
'category' => 'mobile',
1458+
'architecture' => 'arm',
1459+
'bits' => 32,
1460+
'kernel' => 'AMX 68000',
1461+
'platform' => 'PalmOS',
1462+
'platformversion' => 'sony/model',
1463+
'browser' => 'NetFront',
1464+
'browserversion' => '3.0',
1465+
'language' => 'en'
1466+
],
1467+
'Palm680/RC1 Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; PalmSource/Palm-D053; Blazer/4.5) 16;320x320' => [
1468+
'string' => 'Palm680/RC1 Mozilla/4.0 (compatible; MSIE 6.0; Windows 98; PalmSource/Palm-D053; Blazer/4.5) 16;320x320',
1469+
'type' => 'human',
1470+
'category' => 'mobile',
1471+
'architecture' => 'arm',
1472+
'bits' => 32,
1473+
'kernel' => 'AMX 68000',
1474+
'platform' => 'PalmOS',
1475+
'platformversion' => 'Palm-D053',
1476+
'engine' => 'Proprietary',
1477+
'browser' => 'Blazer',
1478+
'browserversion' => '4.5',
1479+
'width' => '320',
1480+
'height' => '320'
14531481
]
14541482
];
14551483
foreach ($strings AS $ua => $item) {

0 commit comments

Comments
 (0)