NUTCH-2455 Use secondary sorting for memory-efficient HostDb integration in Generator

Make HostDatum in SelectorEntry optional for faster serialization

Author: sebastian-nagel

src/java/org/apache/nutch/crawl/Generator.java

@@ -60,6 +60,7 @@
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.BooleanWritable;
 import org.apache.hadoop.io.FloatWritable;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
@@ -158,12 +159,14 @@ public static class SelectorEntry implements Writable {
     public Text url;
     public CrawlDatum datum;
     public IntWritable segnum;
+    public BooleanWritable hasHostDatum;
     public HostDatum hostdatum;
 
     public SelectorEntry() {
       url = new Text();
       datum = new CrawlDatum();
       segnum = new IntWritable(0);
+      hasHostDatum = new BooleanWritable(false);
       hostdatum = new HostDatum();
     }
 
@@ -172,15 +175,21 @@ public void readFields(DataInput in) throws IOException {
       url.readFields(in);
       datum.readFields(in);
       segnum.readFields(in);
-      hostdatum.readFields(in);
+      hasHostDatum.readFields(in);
+      if (hasHostDatum.get()) {
+        hostdatum.readFields(in);
+      }
     }
 
     @Override
     public void write(DataOutput out) throws IOException {
       url.write(out);
       datum.write(out);
       segnum.write(out);
-      hostdatum.write(out);
+      hasHostDatum.write(out);
+      if (hasHostDatum.get()) {
+        hostdatum.write(out);
+      }
     }
 
     @Override
@@ -324,6 +333,7 @@ public static class HostDbReaderMapper
     public void map(Text hostname, HostDatum value, Context context)
         throws IOException, InterruptedException {
       SelectorEntry hostDataSelector = new SelectorEntry();
+      hostDataSelector.hasHostDatum.set(true);
       try {
         hostDataSelector.hostdatum = (HostDatum) value.clone();
       } catch (CloneNotSupportedException e) {