Skip to content

Commit 4299397

Browse files
github-actions[bot]github-actions[bot]
committed
Update CrawlerScope data
1 parent 84862f2 commit 4299397

3 files changed

Lines changed: 447 additions & 192 deletions

File tree

‎data/current/crawlers.json‎

Lines changed: 44 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@
193193
}
194194
]
195195
},
196-
"generatedAt": "2026-07-21T14:26:07Z",
196+
"generatedAt": "2026-07-21T19:54:27Z",
197197
"insights": [
198198
{
199199
"detail": "Total CIDR prefix change since the previous snapshot.",
@@ -227,7 +227,7 @@
227227
"documentationUrl": null,
228228
"id": "google-common",
229229
"ipListAuthoritative": true,
230-
"lastCheckedAt": "2026-07-21T14:26:07Z",
230+
"lastCheckedAt": "2026-07-21T19:54:27Z",
231231
"note": null,
232232
"operator": "Google",
233233
"operatorCountry": "US",
@@ -332,7 +332,7 @@
332332
"documentationUrl": null,
333333
"id": "google-special",
334334
"ipListAuthoritative": true,
335-
"lastCheckedAt": "2026-07-21T14:26:07Z",
335+
"lastCheckedAt": "2026-07-21T19:54:27Z",
336336
"note": null,
337337
"operator": "Google",
338338
"operatorCountry": "US",
@@ -410,7 +410,7 @@
410410
"documentationUrl": null,
411411
"id": "google-user-triggered",
412412
"ipListAuthoritative": true,
413-
"lastCheckedAt": "2026-07-21T14:26:07Z",
413+
"lastCheckedAt": "2026-07-21T19:54:27Z",
414414
"note": null,
415415
"operator": "Google",
416416
"operatorCountry": "US",
@@ -670,7 +670,7 @@
670670
"documentationUrl": null,
671671
"id": "bingbot",
672672
"ipListAuthoritative": true,
673-
"lastCheckedAt": "2026-07-21T14:26:07Z",
673+
"lastCheckedAt": "2026-07-21T19:54:27Z",
674674
"note": null,
675675
"operator": "Microsoft",
676676
"operatorCountry": "US",
@@ -734,7 +734,7 @@
734734
"documentationUrl": null,
735735
"id": "duckduckbot",
736736
"ipListAuthoritative": true,
737-
"lastCheckedAt": "2026-07-21T14:26:07Z",
737+
"lastCheckedAt": "2026-07-21T19:54:27Z",
738738
"note": null,
739739
"operator": "DuckDuckGo",
740740
"operatorCountry": "US",
@@ -1248,7 +1248,7 @@
12481248
"documentationUrl": null,
12491249
"id": "duckassistbot",
12501250
"ipListAuthoritative": true,
1251-
"lastCheckedAt": "2026-07-21T14:26:07Z",
1251+
"lastCheckedAt": "2026-07-21T19:54:27Z",
12521252
"note": null,
12531253
"operator": "DuckDuckGo",
12541254
"operatorCountry": "US",
@@ -1762,7 +1762,7 @@
17621762
"documentationUrl": "https://support.apple.com/en-us/119829",
17631763
"id": "applebot",
17641764
"ipListAuthoritative": true,
1765-
"lastCheckedAt": "2026-07-21T14:26:07Z",
1765+
"lastCheckedAt": "2026-07-21T19:54:27Z",
17661766
"note": null,
17671767
"operator": "Apple",
17681768
"operatorCountry": "US",
@@ -1809,7 +1809,7 @@
18091809
"documentationUrl": null,
18101810
"id": "mojeekbot",
18111811
"ipListAuthoritative": true,
1812-
"lastCheckedAt": "2026-07-21T14:26:07Z",
1812+
"lastCheckedAt": "2026-07-21T19:54:27Z",
18131813
"note": null,
18141814
"operator": "Mojeek",
18151815
"operatorCountry": "GB",
@@ -1845,7 +1845,7 @@
18451845
"documentationUrl": null,
18461846
"id": "naver-yeti",
18471847
"ipListAuthoritative": true,
1848-
"lastCheckedAt": "2026-07-21T14:26:07Z",
1848+
"lastCheckedAt": "2026-07-21T19:54:27Z",
18491849
"note": null,
18501850
"operator": "Naver",
18511851
"operatorCountry": "KR",
@@ -1916,7 +1916,7 @@
19161916
"documentationUrl": null,
19171917
"id": "yandexbot",
19181918
"ipListAuthoritative": false,
1919-
"lastCheckedAt": "2026-07-21T14:26:07Z",
1919+
"lastCheckedAt": "2026-07-21T19:54:27Z",
19201920
"note": "Static known ranges; verify live requests with reverse DNS and forward-confirmation.",
19211921
"operator": "Yandex",
19221922
"operatorCountry": "RU",
@@ -1965,7 +1965,7 @@
19651965
"documentationUrl": null,
19661966
"id": "baiduspider",
19671967
"ipListAuthoritative": false,
1968-
"lastCheckedAt": "2026-07-21T14:26:07Z",
1968+
"lastCheckedAt": "2026-07-21T19:54:27Z",
19691969
"note": "Small static seed list; not a complete authoritative Baidu crawler range list.",
19701970
"operator": "Baidu",
19711971
"operatorCountry": "CN",
@@ -2003,7 +2003,7 @@
20032003
"documentationUrl": null,
20042004
"id": "openai-gptbot",
20052005
"ipListAuthoritative": true,
2006-
"lastCheckedAt": "2026-07-21T14:26:07Z",
2006+
"lastCheckedAt": "2026-07-21T19:54:27Z",
20072007
"note": null,
20082008
"operator": "OpenAI",
20092009
"operatorCountry": "US",
@@ -2053,7 +2053,7 @@
20532053
"documentationUrl": null,
20542054
"id": "openai-searchbot",
20552055
"ipListAuthoritative": true,
2056-
"lastCheckedAt": "2026-07-21T14:26:07Z",
2056+
"lastCheckedAt": "2026-07-21T19:54:27Z",
20572057
"note": null,
20582058
"operator": "OpenAI",
20592059
"operatorCountry": "US",
@@ -2118,7 +2118,7 @@
21182118
"documentationUrl": null,
21192119
"id": "openai-chatgpt-user",
21202120
"ipListAuthoritative": true,
2121-
"lastCheckedAt": "2026-07-21T14:26:07Z",
2121+
"lastCheckedAt": "2026-07-21T19:54:27Z",
21222122
"note": null,
21232123
"operator": "OpenAI",
21242124
"operatorCountry": "US",
@@ -2401,7 +2401,7 @@
24012401
"documentationUrl": null,
24022402
"id": "openai-adsbot",
24032403
"ipListAuthoritative": false,
2404-
"lastCheckedAt": "2026-07-21T14:26:07Z",
2404+
"lastCheckedAt": "2026-07-21T19:54:27Z",
24052405
"note": "OpenAI documents this user-agent but does not publish a separate IP JSON list for it.",
24062406
"operator": "OpenAI",
24072407
"operatorCountry": "US",
@@ -2433,7 +2433,7 @@
24332433
"documentationUrl": null,
24342434
"id": "perplexitybot",
24352435
"ipListAuthoritative": true,
2436-
"lastCheckedAt": "2026-07-21T14:26:07Z",
2436+
"lastCheckedAt": "2026-07-21T19:54:27Z",
24372437
"note": null,
24382438
"operator": "Perplexity",
24392439
"operatorCountry": "US",
@@ -2474,7 +2474,7 @@
24742474
"documentationUrl": null,
24752475
"id": "perplexity-user",
24762476
"ipListAuthoritative": true,
2477-
"lastCheckedAt": "2026-07-21T14:26:07Z",
2477+
"lastCheckedAt": "2026-07-21T19:54:27Z",
24782478
"note": "Official ranges may not cover stealth or browser-like crawler behavior reported by third parties.",
24792479
"operator": "Perplexity",
24802480
"operatorCountry": "US",
@@ -2511,7 +2511,7 @@
25112511
"documentationUrl": null,
25122512
"id": "claudebot",
25132513
"ipListAuthoritative": false,
2514-
"lastCheckedAt": "2026-07-21T14:26:07Z",
2514+
"lastCheckedAt": "2026-07-21T19:54:27Z",
25152515
"note": "Anthropic documents crawler user-agents but says it does not publish stable IP ranges.",
25162516
"operator": "Anthropic",
25172517
"operatorCountry": "US",
@@ -2545,7 +2545,7 @@
25452545
"documentationUrl": "https://developer.amazon.com/support/amazonbot",
25462546
"id": "amazonbot",
25472547
"ipListAuthoritative": true,
2548-
"lastCheckedAt": "2026-07-21T14:26:07Z",
2548+
"lastCheckedAt": "2026-07-21T19:54:27Z",
25492549
"note": "Amazon publishes the list embedded in its developer page rather than as a standalone JSON endpoint.",
25502550
"operator": "Amazon",
25512551
"operatorCountry": "US",
@@ -3104,7 +3104,7 @@
31043104
"documentationUrl": "https://developer.amazon.com/support/amazonbot",
31053105
"id": "amazon-searchbot",
31063106
"ipListAuthoritative": true,
3107-
"lastCheckedAt": "2026-07-21T14:26:07Z",
3107+
"lastCheckedAt": "2026-07-21T19:54:27Z",
31083108
"note": "Amazon publishes the list embedded in its developer page rather than as a standalone JSON endpoint.",
31093109
"operator": "Amazon",
31103110
"operatorCountry": "US",
@@ -3651,7 +3651,7 @@
36513651
"documentationUrl": "https://developer.amazon.com/support/amazonbot",
36523652
"id": "amazon-amzn-user",
36533653
"ipListAuthoritative": true,
3654-
"lastCheckedAt": "2026-07-21T14:26:07Z",
3654+
"lastCheckedAt": "2026-07-21T19:54:27Z",
36553655
"note": "Amazon publishes the live crawl list embedded in its developer page rather than as a standalone JSON endpoint.",
36563656
"operator": "Amazon",
36573657
"operatorCountry": "US",
@@ -4709,7 +4709,7 @@
47094709
"documentationUrl": null,
47104710
"id": "meta-ai-crawlers",
47114711
"ipListAuthoritative": false,
4712-
"lastCheckedAt": "2026-07-21T14:26:07Z",
4712+
"lastCheckedAt": "2026-07-21T19:54:27Z",
47134713
"note": "Static Meta platform ranges, not crawler-specific authoritative ranges.",
47144714
"operator": "Meta",
47154715
"operatorCountry": "US",
@@ -4748,7 +4748,7 @@
47484748
"documentationUrl": null,
47494749
"id": "bytespider",
47504750
"ipListAuthoritative": false,
4751-
"lastCheckedAt": "2026-07-21T14:26:07Z",
4751+
"lastCheckedAt": "2026-07-21T19:54:27Z",
47524752
"note": null,
47534753
"operator": "ByteDance",
47544754
"operatorCountry": "CN",
@@ -4780,7 +4780,7 @@
47804780
"documentationUrl": null,
47814781
"id": "mistralai-user",
47824782
"ipListAuthoritative": true,
4783-
"lastCheckedAt": "2026-07-21T14:26:07Z",
4783+
"lastCheckedAt": "2026-07-21T19:54:27Z",
47844784
"note": null,
47854785
"operator": "Mistral AI",
47864786
"operatorCountry": "FR",
@@ -4817,7 +4817,7 @@
48174817
"documentationUrl": "https://help.ahrefs.com/articles/78658-what-is-the-list-of-your-ip-ranges",
48184818
"id": "ahrefsbot",
48194819
"ipListAuthoritative": true,
4820-
"lastCheckedAt": "2026-07-21T14:26:07Z",
4820+
"lastCheckedAt": "2026-07-21T19:54:27Z",
48214821
"note": null,
48224822
"operator": "Ahrefs",
48234823
"operatorCountry": "SG",
@@ -4904,7 +4904,7 @@
49044904
"documentationUrl": null,
49054905
"id": "lumar-crawler",
49064906
"ipListAuthoritative": true,
4907-
"lastCheckedAt": "2026-07-21T14:26:07Z",
4907+
"lastCheckedAt": "2026-07-21T19:54:27Z",
49084908
"note": null,
49094909
"operator": "Lumar",
49104910
"operatorCountry": "GB",
@@ -5003,7 +5003,7 @@
50035003
"documentationUrl": null,
50045004
"id": "semrushbot",
50055005
"ipListAuthoritative": false,
5006-
"lastCheckedAt": "2026-07-21T14:26:07Z",
5006+
"lastCheckedAt": "2026-07-21T19:54:27Z",
50075007
"note": null,
50085008
"operator": "Semrush",
50095009
"operatorCountry": "US",
@@ -5037,7 +5037,7 @@
50375037
"documentationUrl": null,
50385038
"id": "censys",
50395039
"ipListAuthoritative": false,
5040-
"lastCheckedAt": "2026-07-21T14:26:07Z",
5040+
"lastCheckedAt": "2026-07-21T19:54:27Z",
50415041
"note": null,
50425042
"operator": "Censys",
50435043
"operatorCountry": "US",
@@ -5074,7 +5074,7 @@
50745074
"documentationUrl": null,
50755075
"id": "shodan",
50765076
"ipListAuthoritative": false,
5077-
"lastCheckedAt": "2026-07-21T14:26:07Z",
5077+
"lastCheckedAt": "2026-07-21T19:54:27Z",
50785078
"note": "Community-known/static ranges; verify with reverse DNS and current Shodan documentation.",
50795079
"operator": "Shodan",
50805080
"operatorCountry": "US",
@@ -5119,7 +5119,7 @@
51195119
"documentationUrl": null,
51205120
"id": "datadog-synthetics",
51215121
"ipListAuthoritative": true,
5122-
"lastCheckedAt": "2026-07-21T14:26:07Z",
5122+
"lastCheckedAt": "2026-07-21T19:54:27Z",
51235123
"note": null,
51245124
"operator": "Datadog",
51255125
"operatorCountry": "US",
@@ -5265,7 +5265,7 @@
52655265
"documentationUrl": null,
52665266
"id": "ias-crawler",
52675267
"ipListAuthoritative": true,
5268-
"lastCheckedAt": "2026-07-21T14:26:07Z",
5268+
"lastCheckedAt": "2026-07-21T19:54:27Z",
52695269
"note": null,
52705270
"operator": "Integral Ad Science",
52715271
"operatorCountry": "US",
@@ -5312,7 +5312,7 @@
53125312
"documentationUrl": null,
53135313
"id": "ttd-content",
53145314
"ipListAuthoritative": true,
5315-
"lastCheckedAt": "2026-07-21T14:26:07Z",
5315+
"lastCheckedAt": "2026-07-21T19:54:27Z",
53165316
"note": null,
53175317
"operator": "The Trade Desk",
53185318
"operatorCountry": "US",
@@ -7960,7 +7960,7 @@
79607960
"documentationUrl": "https://uptimerobot.com/locations",
79617961
"id": "uptimerobot",
79627962
"ipListAuthoritative": true,
7963-
"lastCheckedAt": "2026-07-21T14:26:07Z",
7963+
"lastCheckedAt": "2026-07-21T19:54:27Z",
79647964
"note": null,
79657965
"operator": "UptimeRobot",
79667966
"operatorCountry": "US",
@@ -8199,7 +8199,7 @@
81998199
"documentationUrl": null,
82008200
"id": "pingdom",
82018201
"ipListAuthoritative": true,
8202-
"lastCheckedAt": "2026-07-21T14:26:07Z",
8202+
"lastCheckedAt": "2026-07-21T19:54:27Z",
82038203
"note": null,
82048204
"operator": "SolarWinds Pingdom",
82058205
"operatorCountry": "US",
@@ -8388,7 +8388,7 @@
83888388
"documentationUrl": "https://www.statuscake.com/kb/knowledge-base/what-are-your-ips/",
83898389
"id": "statuscake",
83908390
"ipListAuthoritative": true,
8391-
"lastCheckedAt": "2026-07-21T14:26:07Z",
8391+
"lastCheckedAt": "2026-07-21T19:54:27Z",
83928392
"note": null,
83938393
"operator": "StatusCake",
83948394
"operatorCountry": "GB",
@@ -8718,7 +8718,7 @@
87188718
"documentationUrl": "https://betterstack.com/docs/uptime/frequently-asked-questions/",
87198719
"id": "better-stack",
87208720
"ipListAuthoritative": true,
8721-
"lastCheckedAt": "2026-07-21T14:26:07Z",
8721+
"lastCheckedAt": "2026-07-21T19:54:27Z",
87228722
"note": "Better Stack publishes probe addresses through its Better Uptime IP text endpoint.",
87238723
"operator": "Better Stack",
87248724
"operatorCountry": "CZ",
@@ -8786,7 +8786,7 @@
87868786
"documentationUrl": null,
87878787
"id": "ccbot",
87888788
"ipListAuthoritative": true,
8789-
"lastCheckedAt": "2026-07-21T14:26:07Z",
8789+
"lastCheckedAt": "2026-07-21T19:54:27Z",
87908790
"note": null,
87918791
"operator": "Common Crawl",
87928792
"operatorCountry": "US",
@@ -8826,7 +8826,7 @@
88268826
"documentationUrl": null,
88278827
"id": "flipboard-crawler",
88288828
"ipListAuthoritative": true,
8829-
"lastCheckedAt": "2026-07-21T14:26:07Z",
8829+
"lastCheckedAt": "2026-07-21T19:54:27Z",
88308830
"note": null,
88318831
"operator": "Flipboard",
88328832
"operatorCountry": "US",
@@ -8995,7 +8995,7 @@
89958995
"documentationUrl": null,
89968996
"id": "parsely-crawler",
89978997
"ipListAuthoritative": true,
8998-
"lastCheckedAt": "2026-07-21T14:26:07Z",
8998+
"lastCheckedAt": "2026-07-21T19:54:27Z",
89998999
"note": null,
90009000
"operator": "Parse.ly",
90019001
"operatorCountry": "US",
@@ -9038,7 +9038,7 @@
90389038
"documentationUrl": null,
90399039
"id": "pinterestbot",
90409040
"ipListAuthoritative": false,
9041-
"lastCheckedAt": "2026-07-21T14:26:07Z",
9041+
"lastCheckedAt": "2026-07-21T19:54:27Z",
90429042
"note": null,
90439043
"operator": "Pinterest",
90449044
"operatorCountry": "US",
@@ -9072,7 +9072,7 @@
90729072
"documentationUrl": null,
90739073
"id": "linkedinbot",
90749074
"ipListAuthoritative": false,
9075-
"lastCheckedAt": "2026-07-21T14:26:07Z",
9075+
"lastCheckedAt": "2026-07-21T19:54:27Z",
90769076
"note": null,
90779077
"operator": "LinkedIn",
90789078
"operatorCountry": "US",
@@ -9104,7 +9104,7 @@
91049104
"documentationUrl": "https://core.telegram.org/bots/webapps#link-preview",
91059105
"id": "telegram-link-preview",
91069106
"ipListAuthoritative": true,
9107-
"lastCheckedAt": "2026-07-21T14:26:07Z",
9107+
"lastCheckedAt": "2026-07-21T19:54:27Z",
91089108
"note": null,
91099109
"operator": "Telegram",
91109110
"operatorCountry": "AE",
@@ -9149,7 +9149,7 @@
91499149
"documentationUrl": "https://rssapi.net/faq",
91509150
"id": "rssapi-parser",
91519151
"ipListAuthoritative": true,
9152-
"lastCheckedAt": "2026-07-21T14:26:07Z",
9152+
"lastCheckedAt": "2026-07-21T19:54:27Z",
91539153
"note": null,
91549154
"operator": "RSS API",
91559155
"operatorCountry": "DE",

0 commit comments

Comments
 (0)