Added new AI crawlers (#248)

Updated config file to include additional AI crawlers in the doc.

Author: flordelisa-source

index.js

@@ -42,11 +42,24 @@ var prerender = module.exports = function(req, res, next) {
 
 prerender.crawlerUserAgents = [
   'googlebot',
+  'adsbot-google',
+  'apis-google',
+  'mediapartners-google',
+  'google-safety',
+  'feedfetcher-google',
+  'googleproducer',
+  'google-site-verification'
   'Google-InspectionTool',
   'Yahoo! Slurp',
   'bingbot',
   'yandex',
+  'yandexbot',
   'baiduspider',
+  'naver',
+  'seznambot',
+  'sznprohlizec',
+  'qwantbot',
+  'ecosia',
   'facebookexternalhit',
   'twitterbot',
   'rogerbot',
@@ -85,6 +98,10 @@ prerender.crawlerUserAgents = [
   'ChatGPT-User',
   'GPTBot',
   'ChatGPT-User/1.0',
+  'oai-searchbot',
+  'claudebot',
+  'claude-web',
+  'amazonbot',
   'Anthropic/Claude',
   'anthropic-ai',
   'Anthropic/Claude',
@@ -93,9 +110,10 @@ prerender.crawlerUserAgents = [
   'Microsoft/Bing AI',
   'BingBot/AI',
   'BingPreview',
-  'Perplexity',
   'Perplexitybot',
   'PerplexityBot/1.0',
+  'perplexity-user',
+  'youbot',
   'Cohere',
   'Cohere-ai',
   'cohere-crawler',
@@ -106,11 +124,14 @@ prerender.crawlerUserAgents = [
   'Baiduspider-AI',
   'YourBot',
   'You-bot',
+  'mistralai-user',
   'NeevaBot',
   'Hugging-Face-AI',
   'HuggingFaceBot',
   'LinkedInBot/AI',
   'facebookexternalhit/AI',
+  'meta-externalagent',
+  'facebookcatalog',
   'FacebookBot',
   'Twitterbot',
   'redditbot',
@@ -119,7 +140,21 @@ prerender.crawlerUserAgents = [
   'research-crawler',
   'Diffbot',
   'ScaleAI',
-  'scale-crawler'
+  'scale-crawler',
+  'rogerbot',
+  'semrushbot',
+  'ahrefsbot',
+  'chrome-lighthouse',
+  'screaming-frog',
+  'oncrawlbot',
+  'botifybot',
+  'deepcrawl',
+  'lumar',
+  'dotbot',
+  'duckduckbot',
+  'duckassistbot',
+  'instagram',
+  'tiktokspider',
 ];