Merge pull request #6627 from SoftCreatR/feature/new-spiders

Added new spiders

Author: dtdesign

spiderList/spiderList.xml

@@ -1719,8 +1719,63 @@
 		<name>GPTBot</name>
 		<url>https://openai.com/gptbot</url>
 	</spider>
+	<spider ident="OAI-SearchBot/">
+		<name>OAI-SearchBot</name>
+		<url>https://openai.com/searchbot</url>
+	</spider>
+	<spider ident="ChatGPT-User/">
+		<name>ChatGPT-User</name>
+		<url>https://openai.com/bot</url>
+	</spider>
 	<spider ident="PerplexityBot/">
 		<name>PerplexityBot</name>
 		<url>https://perplexity.ai/perplexitybot</url>
 	</spider>
+	<spider ident="Perplexity-User/">
+		<name>Perplexity-User</name>
+		<url>https://perplexity.ai/perplexity-user</url>
+	</spider>
+	<spider ident="meta-webindexer/">
+		<name>Meta-WebIndexer</name>
+		<url>https://developers.facebook.com/docs/sharing/webmasters/web-crawlers</url>
+	</spider>
+	<spider ident="meta-externalads/">
+		<name>Meta-ExternalAds</name>
+		<url>https://developers.facebook.com/docs/sharing/webmasters/web-crawlers</url>
+	</spider>
+	<spider ident="meta-externalagent/">
+		<name>Meta-ExternalAgent</name>
+		<url>https://developers.facebook.com/docs/sharing/webmasters/web-crawlers</url>
+	</spider>
+	<spider ident="meta-externalfetcher/">
+		<name>Meta-ExternalFetcher</name>
+		<url>https://developers.facebook.com/docs/sharing/webmasters/web-crawlers</url>
+	</spider>
+	<spider ident="Bytespider/">
+		<name>Bytespider</name>
+	</spider>
+	<spider ident="PlagAwareBot/">
+		<name>PlagAware</name>
+		<url>https://www.plagaware.com/bot</url>
+	</spider>
+	<spider ident="DuckAssistBot/">
+		<name>DuckAssistBot</name>
+		<url>https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot</url>
+	</spider>
+	<spider ident="CCBot/">
+		<name>Common Crawl Bot</name>
+		<url>https://commoncrawl.org/faq</url>
+	</spider>
+	<spider ident="DataForSeoBot/">
+		<name>DataForSEO Link Bot</name>
+		<url>https://dataforseo.com/dataforseo-bot</url>
+	</spider>
+	<spider ident="Gemini-Deep-Research/">
+		<name>Gemini Deep Research</name>
+		<url>https://gemini.google/overview/deep-research/</url>
+	</spider>
+	<spider ident="quillbot/">
+		<name>Quillbot</name>
+		<url>https://quillbot.com/</url>
+	</spider>
 </data>

wcfsetup/install/files/lib/event/spider/SpiderCollecting.class.php

@@ -641,6 +641,57 @@ public function __construct()
         );
         $this->register(new Spider('360Spider', '360Spider'));
         $this->register(new Spider('GPTBot', 'GPTBot', 'https://openai.com/gptbot'));
+        $this->register(new Spider('OAI-SearchBot', 'OAI-SearchBot', 'https://openai.com/searchbot'));
+        $this->register(new Spider('ChatGPT-User', 'ChatGPT-User', 'https://openai.com/bot'));
+        $this->register(new Spider('PerplexityBot', 'PerplexityBot', 'https://perplexity.ai/perplexitybot'));
+        $this->register(new Spider('Perplexity-User', 'Perplexity-User', 'https://perplexity.ai/perplexity-user'));
+        $this->register(
+            new Spider(
+                'meta-webindexer', 
+                'Meta-WebIndexer', 
+                'https://developers.facebook.com/docs/sharing/webmasters/web-crawlers'
+            )
+        );
+        $this->register(
+            new Spider(
+                'meta-externalads',
+                'Meta-ExternalAds',
+                'https://developers.facebook.com/docs/sharing/webmasters/web-crawlers'
+            )
+        );
+        $this->register(
+            new Spider(
+                'meta-externalagent',
+                'Meta-ExternalAgent',
+                'https://developers.facebook.com/docs/sharing/webmasters/web-crawlers'
+            )
+        );
+        $this->register(
+            new Spider(
+                'meta-externalfetcher',
+                'Meta-ExternalFetcher',
+                'https://developers.facebook.com/docs/sharing/webmasters/web-crawlers'
+            )
+        );
+        $this->register(new Spider('Bytespider', 'Bytespider'));
+        $this->register(new Spider('PlagAwareBot', 'PlagAware', 'https://www.plagaware.com/bot'));
+        $this->register(
+            new Spider(
+                'DuckAssistBot',
+                'DuckAssistBot',
+                'https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot'
+            )
+        );
+        $this->register(new Spider('CCBot', 'Common Crawl Bot', 'https://commoncrawl.org/faq'));
+        $this->register(new Spider('DataForSeoBot', 'DataForSEO Link Bot', 'https://dataforseo.com/dataforseo-bot'));
+        $this->register(
+            new Spider(
+                'Gemini-Deep-Research',
+                'Gemini Deep Research',
+                'https://gemini.google/overview/deep-research/'
+            )
+        );
+        $this->register(new Spider('quillbot', 'Quillbot', 'https://quillbot.com/'));
     }
 
     /**