Release v1.3.3 (#15)

* Added handling for badly formed URLs

* Fixed handling for badly formed URLs

* Added debug line to help with triage of issue

* Tests failed on Jenkins -  trigger rerun

* Added announcements of thread start

* Reduced thread start output to be included only in the verbose output

* Attempting to fix the tests again

* Now sleeping for random millisecond time before starting each thread

* Reduced thread count

* Fixed test - character has moved to moogle

* Added output of exceptions from 'does not exist'

* Error-code character pages now print status code to exception

* Recursively retry when experiencing 429 errors

* Fixed spelling errror

* Increased retry timeout

* Update CHANGELOG.md

* Update CHANGELOG.md

Author: ReidWeb

CHANGELOG.md

@@ -1,3 +1,7 @@
+v1.3.3
+======
+* If the Gatherer experiences a HTTP error code `429` (Too many requests), it will pause for 1-20 milliseconds and retry via  recursive method call.
+
 v1.3.2
 ======
 * New reputation columns accidentally deleted from CREATE TABLE script.

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>com.ffxivcensus.gatherer</groupId>
     <artifactId>XIVStats-Gatherer-Java</artifactId>
-    <version>v1.3.2</version>
+    <version>v1.3.3</version>
     <name>XIVStats Lodestone Gatherer</name>
     <url>https://github.com/xivstats</url>
 
@@ -180,6 +180,11 @@
             <artifactId>unirest-java</artifactId>
             <version>1.4.9</version>
         </dependency>
+        <dependency>
+            <groupId>commons-lang</groupId>
+            <artifactId>commons-lang</artifactId>
+            <version>2.6</version>
+        </dependency>
     </dependencies>
 
 </project>

src/main/java/com/ffxivcensus/gatherer/Gatherer.java

@@ -15,7 +15,7 @@
  */
 public class Gatherer implements Runnable {
 
-    GathererController parent;
+    private GathererController parent;
 
     /**
      * Default constructor
@@ -31,6 +31,9 @@ public void run() {
         int nextID = parent.getNextID();
         while (nextID != -1) { //While we still have IDs to parse
             try {
+                if(parent.isVerbose()) {
+                    System.out.println("Starting evaluation of player ID: " +nextID);
+                }
                 //Parse players and write them to DB
                 String out = parent.writeToDB(Player.getPlayer(nextID));
                 if (!parent.isQuiet()) { //If not running in quiet mode

src/main/java/com/ffxivcensus/gatherer/GathererController.java

@@ -14,6 +14,8 @@
 import java.sql.DriverManager;
 import java.sql.SQLException;
 import java.sql.Statement;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
 
 /**
  * GathererController class of character gathering program. This class makes calls to fetch records from the lodestone, and then
@@ -429,7 +431,20 @@ private void gatherRange() {
 
         //Start up threads
         for (Thread thread : threads) {
+            if(this.isVerbose()) {
+                System.out.println("Starting thread with id " + thread.getId());
+            }
             thread.start();
+            try {
+                //Generate random number 1-10 and sleep for it
+                Random rand = new Random();
+
+                int randomNum = rand.nextInt((10 - 1) + 1) + 1;
+
+                TimeUnit.MILLISECONDS.sleep(randomNum);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
         }
 
         //Spin down threads

src/main/java/com/ffxivcensus/gatherer/Player.java

@@ -10,12 +10,15 @@
 import org.jsoup.select.Elements;
 
 import java.io.IOException;
+import java.net.MalformedURLException;
 import java.text.DateFormat;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Date;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
 import java.util.regex.Pattern;
 
 /**
@@ -1864,14 +1867,14 @@ public static Player getPlayer(int playerID) throws Exception {
 
         try {
             //Fetch the specified URL
-            doc = Jsoup.connect(url).get();
+            doc = Jsoup.connect(url).timeout(5000).get();
             player.setPlayerName(getNameFromPage(doc));
             player.setRealm(getRealmFromPage(doc));
             player.setRace(getRaceFromPage(doc));
             player.setGender(getGenderFromPage(doc));
             player.setGrandCompany(getGrandCompanyFromPage(doc));
             player.setFreeCompany(getFreeCompanyFromPage(doc));
-            player.setDateImgLastModified(getDateLastUpdatedFromPage(doc));
+            player.setDateImgLastModified(getDateLastUpdatedFromPage(doc,playerID));
             player.setLevels(getLevelsFromPage(doc));
             player.setMounts(getMountsFromPage(doc));
             player.setMinions(getMinionsFromPage(doc));
@@ -1917,7 +1920,19 @@ public static Player getPlayer(int playerID) throws Exception {
             player.setIsLegacyPlayer(player.doesPlayerHaveMount("Legacy Chocobo"));
             player.setActive(player.isPlayerActiveInDateRange());
         } catch (IOException ioEx) {
-            throw new Exception("Character " + playerID + " does not exist.");
+            String strEx = org.apache.commons.lang.exception.ExceptionUtils.getStackTrace(ioEx);
+            String statusCode = strEx.split("\\s+")[5].replace("Status=","").replace(",","");
+            if(statusCode.equals("429")) {
+                //Generate random number 1-20 and sleep for it
+                Random rand = new Random();
+
+                int randomNum = rand.nextInt((20 - 1) + 1) + 1;
+                System.out.println("Experiencing rate limiting (HTTP 429) while fetching id " + playerID + ", waiting " + randomNum + "ms then retrying...");
+                TimeUnit.MILLISECONDS.sleep(randomNum);
+                player = Player.getPlayer(playerID);
+            } else {
+                throw new Exception("Character " + playerID + " does not exist. Status: " + statusCode);
+            }
         }
         return player;
     }
@@ -2146,7 +2161,7 @@ private static ArrayList getMountsFromPage(Document doc) {
      * @param doc the lodestone profile page to parse
      * @return the date on which the full body image was last modified.
      */
-    private static Date getDateLastUpdatedFromPage(Document doc) throws Exception {
+    private static Date getDateLastUpdatedFromPage(Document doc, int id) throws Exception {
         Date dateLastModified = new Date();
         //Get character image URL.
         String imgUrl = doc.getElementsByClass("bg_chara_264").get(0).getElementsByTag("img").get(0).attr("src");
@@ -2156,8 +2171,10 @@ private static Date getDateLastUpdatedFromPage(Document doc) throws Exception {
             HttpResponse<JsonNode> jsonResponse = Unirest.head(imgUrl).asJson();
 
             strLastModifiedDate = jsonResponse.getHeaders().get("Last-Modified").toString();
-        } catch (UnirestException e) {
-            e.printStackTrace();
+        }
+        catch (Exception e) {
+            System.out.println("Setting last-active date to ARR launch date due to an an error loading character " + id + "'s profile image: " + e.getMessage());
+            strLastModifiedDate = "[Sat, 24 Aug 2013 00:00:01 GMT]";
         }
 
         strLastModifiedDate = strLastModifiedDate.replace("[", "");
@@ -2167,7 +2184,7 @@ private static Date getDateLastUpdatedFromPage(Document doc) throws Exception {
         try {
             dateLastModified = dateFormat.parse(strLastModifiedDate);
         } catch (ParseException e) {
-            throw new Exception("Could not correctly parse date 'Last-Modified' header from full body image");
+            throw new Exception("Could not correctly parse date 'Last-Modified' header from full body image for character id" + id);
         }
         return dateLastModified;
     }

src/test/java/com/ffxivcensus/gatherer/GathererControllerTest.java

@@ -114,6 +114,8 @@ public void testRunBasic() throws IOException, SAXException, ParserConfiguration
         int endId = 11887010;
         GathererController gathererController = new GathererController(startId, endId);
         gathererController.setTableName("tblplayers_test_3");
+        gathererController.setVerbose(true);
+        gathererController.setThreadLimit(40);
         try {
             gathererController.run();
         } catch (Exception e) {
@@ -207,7 +209,7 @@ public void testRunSplitTables() {
         int startId = 1557260;
         int endId = 1558260;
 
-        GathererController gathererController = new GathererController(startId, endId, false, true, false, false, false, dbHost, dbName, dbUser, dbPassword, 71, "_test", true);
+        GathererController gathererController = new GathererController(startId, endId, false, true, false, false, false, dbHost, dbName, dbUser, dbPassword, 50, "_test", true);
         try {
             gathererController.run();
         } catch (Exception e) {
@@ -231,12 +233,12 @@ public void testRunSplitTables() {
         assertTrue(addedIDsCerberus.contains(1557648));
         assertTrue(addedIDsCerberus.contains(1558244));
 
-        //Test for ids that will exist on shiva (realm of end char)
-        assertTrue(addedIDsShiva.contains(endId));
+
         assertTrue(addedIDsShiva.contains(1557297));
 
         //Test for ids that will exist on Moogle
         assertTrue(addedIDsMoogle.contains(1557265));
+        assertTrue(addedIDsMoogle.contains(endId));
         assertTrue(addedIDsMoogle.contains(1557301));
 
         //Test that gatherer has not written records that don't exist on cerberus

src/test/java/com/ffxivcensus/gatherer/PlayerTest.java

@@ -163,7 +163,7 @@ public void testGetPlayer() throws Exception {
      * Perform a test of the getPlayer method using character #501646 (Omega Venom, Cerberus) to test data that could
      * not be tested with other tests.
      *
-     * @throws Exception exception thrown when reading non-existant character.
+     * @throws Exception exception thrown when reading non-existent character.
      */
     @org.junit.Test
     public void testGetVeteranPlayer() throws Exception {