readme & update prerender client

Author: greengerong

README.md

@@ -38,7 +38,7 @@ Use this java filter that prerenders a javascript-rendered page using an externa
 	2. Check to make sure we aren't requesting a resource (js, css, etc...)
 	3. (optional) Check to make sure the url is in the whitelist
 	4. (optional) Check to make sure the url isn't in the blacklist
-2. Make a `GET` request to the [prerender service](https://github.com/collectiveip/prerender)(phantomjs server) for the page's prerendered HTML
+2. Make a `GET` request to the [prerender service](https://github.com/prerender/prerender)(phantomjs server) for the page's prerendered HTML
 3. Return that HTML to the crawler
 
 ## Customization
@@ -56,7 +56,7 @@ We usually set the original url in an http header which is added by the reverse
 
 ### Using your own prerender service
 
-If you've deployed the prerender service on your own, set the `PRERENDER_SERVICE_URL` environment variable so that this package points there instead. Otherwise, it will default to the service already deployed at `http://prerender.herokuapp.com`
+If you've deployed the prerender service on your own, set the `PRERENDER_SERVICE_URL` environment variable so that this package points there instead. Otherwise, it will default to the service already deployed at `http://service.prerender.io/`
 
 	$ export PRERENDER_SERVICE_URL=<new url>
 
@@ -92,16 +92,21 @@ If you want to cache the caching, analytics, log or others, you can config it. I
 
 ## Testing
 
-If you want to make sure your pages are rendering correctly:
+If your URLs use a hash-bang:
 
-1. Open the Developer Tools in Chrome (Cmd + Atl + J)
-2. Click the Settings gear in the bottom right corner.
-3. Click "Overrides" on the left side of the settings panel.
-4. Check the "User Agent" checkbox.
-6. Choose "Other..." from the User Agent dropdown.
-7. Type `googlebot` into the input box.
-8. Refresh the page (make sure to keep the developer tools open).
+    If you want to see `http://localhost:3000/#!/profiles/1234`
+    Then go to `http://localhost:3000/?_escaped_fragment_=/profiles/1234`
 
+If your URLs use push-state:
+
+    If you want to see `http://localhost:3000/profiles/1234`
+    Then go to `http://localhost:3000/profiles/1234?_escaped_fragment_=`
+    
+
+project demo test url:
+
+    http://localhost:8080/test/?_escaped_fragment_=
+    
 ## License
 
 The MIT License (MIT)

demo/pom.xml

@@ -13,11 +13,11 @@
             <artifactId>prerender-java</artifactId>
             <version>1.6.2</version>
         </dependency>
-        <dependency>
-            <groupId>com.github.greengerong</groupId>
-            <artifactId>prerender-java-redis</artifactId>
-            <version>0.0.1-SNAPSHOT</version>
-        </dependency>
+        <!--<dependency>-->
+            <!--<groupId>com.github.greengerong</groupId>-->
+            <!--<artifactId>prerender-java-redis</artifactId>-->
+            <!--<version>0.0.1-SNAPSHOT</version>-->
+        <!--</dependency>-->
         <dependency>
             <groupId>commons-io</groupId>
             <artifactId>commons-io</artifactId>

demo/src/main/webapp/WEB-INF/web.xml

@@ -15,10 +15,10 @@
             <param-name>crawlerUserAgents</param-name>
             <param-value>YahooSeeker</param-value>
         </init-param>
-        <init-param>
-            <param-name>preRenderEventHandler</param-name>
-            <param-value>com.github.greengerong.PrerenderRedisCache</param-value>
-        </init-param>
+        <!--<init-param>-->
+        <!--<param-name>preRenderEventHandler</param-name>-->
+        <!--<param-value>com.github.greengerong.PrerenderRedisCache</param-value>-->
+        <!--</init-param>-->
     </filter>
     <filter-mapping>
         <filter-name>prerender</filter-name>

prerender-java.iml

@@ -23,5 +23,4 @@
     <orderEntry type="library" name="Maven: commons-codec:commons-codec:1.6" level="project" />
     <orderEntry type="library" name="Maven: org.slf4j:slf4j-api:1.7.5" level="project" />
   </component>
-</module>
-
+</module>

src/main/java/com/github/greengerong/PrerenderConfig.java

@@ -2,7 +2,6 @@
 
 
 import com.google.common.collect.Lists;
-import org.apache.commons.lang.StringUtils;
 import org.apache.http.HttpHost;
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.impl.client.CloseableHttpClient;
@@ -84,7 +83,8 @@ public String getForwardedURLHeader() {
     public List<String> getCrawlerUserAgents() {
         List<String> crawlerUserAgents = Lists.newArrayList("baiduspider",
                 "facebookexternalhit", "twitterbot", "rogerbot", "linkedinbot", "embedly", "quora link preview"
-                , "showyoubo", "outbrain", "pinterest", "developers.google.com/+/web/snippet", "slackbot");
+                , "showyoubo", "outbrain", "pinterest", "developers.google.com/+/web/snippet", "slackbot", "vkShare",
+                "W3C_Validator", "redditbot", "Applebot");
         final String crawlerUserAgentsFromConfig = config.get("crawlerUserAgents");
         if (isNotBlank(crawlerUserAgentsFromConfig)) {
             crawlerUserAgents.addAll(Arrays.asList(crawlerUserAgentsFromConfig.trim().split(",")));
@@ -94,10 +94,10 @@ public List<String> getCrawlerUserAgents() {
     }
 
     public List<String> getExtensionsToIgnore() {
-        List<String> extensionsToIgnore = Lists.newArrayList(".xml", ".js", ".css", ".less", ".png", ".jpg", ".jpeg",
-                ".gif", ".pdf", ".doc", ".txt", ".zip", ".mp3", ".rar", ".exe", ".wmv", ".doc", ".avi", ".ppt", ".mpg",
-                ".mpeg", ".tif", ".wav", ".mov", ".psd", ".ai", ".xls", ".mp4", ".m4a", ".swf", ".dat", ".dmg",
-                ".iso", ".flv", ".m4v", ".torrent");
+        List<String> extensionsToIgnore = Lists.newArrayList(".js", ".json", ".css", ".xml", ".less", ".png", ".jpg",
+                ".jpeg", ".gif", ".pdf", ".doc", ".txt", ".ico", ".rss", ".zip", ".mp3", ".rar", ".exe", ".wmv",
+                ".doc", ".avi", ".ppt", ".mpg", ".mpeg", ".tif", ".wav", ".mov", ".psd", ".ai", ".xls", ".mp4",
+                ".m4a", ".swf", ".dat", ".dmg", ".iso", ".flv", ".m4v", ".torrent", ".woff", ".ttf");
         final String extensionsToIgnoreFromConfig = config.get("extensionsToIgnore");
         if (isNotBlank(extensionsToIgnoreFromConfig)) {
             extensionsToIgnore.addAll(Arrays.asList(extensionsToIgnoreFromConfig.trim().split(",")));
@@ -124,6 +124,11 @@ public List<String> getBlacklist() {
 
     public String getPrerenderServiceUrl() {
         final String prerenderServiceUrl = config.get("prerenderServiceUrl");
-        return isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : PRERENDER_IO_SERVICE_URL;
+        return isNotBlank(prerenderServiceUrl) ? prerenderServiceUrl : getDefaultPrerenderIoServiceUrl();
+    }
+
+    private String getDefaultPrerenderIoServiceUrl() {
+        final String prerenderServiceUrlInEnv = System.getProperty("PRERENDER_SERVICE_URL");
+        return isNotBlank(prerenderServiceUrlInEnv) ? prerenderServiceUrlInEnv : PRERENDER_IO_SERVICE_URL;
     }
 }