[pull request #29] remove google, bing and yahoo from defaults crawler. add new crawler sync "https://github.com/prerender/prerender-node/blob/master/index.js#L32"

Author: greengerong

src/main/java/com/github/greengerong/PrerenderConfig.java

@@ -81,8 +81,9 @@ public String getForwardedURLHeader() {
     }
 
     public List<String> getCrawlerUserAgents() {
-        List<String> crawlerUserAgents = Lists.newArrayList("googlebot", "yahoo", "bingbot", "baiduspider",
-                "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly");
+        List<String> crawlerUserAgents = Lists.newArrayList("baiduspider",
+                "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly", "quora link preview"
+                , "showyoubo", "outbrain", "pinterest", "developers.google.com/+/web/snippet", "slackbot");
         final String crawlerUserAgentsFromConfig = config.get("crawlerUserAgents");
         if (isNotBlank(crawlerUserAgentsFromConfig)) {
             crawlerUserAgents.addAll(Arrays.asList(crawlerUserAgentsFromConfig.trim().split(",")));