IncrementalCrawlingHistory persist technology switched from Lucene to

MapDb. Lucene dependency is provided now - add it on your own if you
want to use the ToLuceneContentHandler

Author: reuschling

pom.xml

@@ -81,18 +81,21 @@
             <groupId>org.apache.lucene</groupId>
             <artifactId>lucene-core</artifactId>
             <version>6.4.2</version>
+            <scope>provided</scope>
         </dependency>
 
         <dependency>
             <groupId>org.apache.lucene</groupId>
             <artifactId>lucene-analyzers-common</artifactId>
             <version>6.4.2</version>
+            <scope>provided</scope>
         </dependency>
 
         <dependency>
             <groupId>org.apache.lucene</groupId>
             <artifactId>lucene-queryparser</artifactId>
             <version>6.4.2</version>
+            <scope>provided</scope>
         </dependency>
 
 
@@ -169,11 +172,11 @@
                     </binFileExtensions>
                     <programs>
                         <program>
-                            <mainClass>de.dfki.km.leech.util.LuceneIndexCreator</mainClass>
+                            <mainClass>de.dfki.km.leech.lucene.LuceneIndexCreator</mainClass>
                             <name>createLuceneIndex</name>
                         </program>
                         <program>
-                            <mainClass>de.dfki.km.leech.util.SolrIndexCreator</mainClass>
+                            <mainClass>de.dfki.km.leech.solr.SolrIndexCreator</mainClass>
                             <name>createSolrIndex</name>
                         </program>
                         <program>

src/main/java/de/dfki/km/leech/io/HttpURLStreamProvider.java

@@ -122,7 +122,7 @@ public Metadata addFirstMetadata(URLName url2getMetadata, Metadata metadata2fill
 
         // müssen wir hier evtl. die Lucene-Teile auch wieder closen? das ist immerhin eine utility-Methode^^ och - wir haben ja auch noch nen
         // shutdown hook, und nach dem crawl wirds eh geschlossen. Klingt safe
-        if(crawlingHistory != null) crawlingHistory.openLuceneStuff();
+        if(crawlingHistory != null) crawlingHistory.openDBStuff();
 
 
         // keep a backup of the originally passed url

…ki/km/leech/util/IndexPostprocessor.java …/km/leech/lucene/IndexPostprocessor.javasrc/main/java/de/dfki/km/leech/util/IndexPostprocessor.java renamed to src/main/java/de/dfki/km/leech/lucene/IndexPostprocessor.java src/main/java/de/dfki/km/leech/util/IndexPostprocessor.java renamed to src/main/java/de/dfki/km/leech/lucene/IndexPostprocessor.java

@@ -1,4 +1,4 @@
-package de.dfki.km.leech.util;
+package de.dfki.km.leech.lucene;
 
 
 
@@ -28,14 +28,8 @@
 import org.apache.tika.metadata.Metadata;
 
 import de.dfki.inquisitor.file.FileUtilz;
-// import de.dfki.inquisitor.lucene.Buzzwords;
-// import de.dfki.inquisitor.lucene.DocumentFrqClass;
-// import de.dfki.inquisitor.lucene.FieldConfig;
-// import de.dfki.inquisitor.lucene.LuceneUtilz;
-// import de.dfki.inquisitor.lucene.PageCountEstimator;
 import de.dfki.inquisitor.processes.StopWatch;
 import de.dfki.inquisitor.text.StringUtils;
-import de.dfki.km.leech.lucene.ToLuceneContentHandler;
 import de.dfki.km.leech.metadata.LeechMetadata;
 
 

…ki/km/leech/util/LuceneIndexCreator.java …/km/leech/lucene/LuceneIndexCreator.javasrc/main/java/de/dfki/km/leech/util/LuceneIndexCreator.java renamed to src/main/java/de/dfki/km/leech/lucene/LuceneIndexCreator.java src/main/java/de/dfki/km/leech/util/LuceneIndexCreator.java renamed to src/main/java/de/dfki/km/leech/lucene/LuceneIndexCreator.java

@@ -1,16 +1,14 @@
-package de.dfki.km.leech.util;
+package de.dfki.km.leech.lucene;
 
 
 
-import java.io.File;
 import java.io.IOException;
 import java.nio.file.Paths;
 import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import de.dfki.km.leech.lucene.basic.FieldConfig;
@@ -30,9 +28,6 @@
 import de.dfki.inquisitor.text.StringUtils;
 import de.dfki.km.leech.Leech;
 import de.dfki.km.leech.config.CrawlerContext;
-import de.dfki.km.leech.lucene.LeechDefaultFieldConfig;
-import de.dfki.km.leech.lucene.ToLuceneContentHandler;
-import de.dfki.km.leech.parser.wikipedia.WikipediaDumpParser;
 import de.dfki.km.leech.parser.wikipedia.WikipediaDumpParser.WikipediaDumpParserConfig;
 import de.dfki.km.leech.sax.CrawlReportContentHandler;
 import de.dfki.km.leech.sax.PrintlnContentHandler;

src/main/java/de/dfki/km/leech/lucene/ToLuceneContentHandler.java

@@ -20,7 +20,6 @@
 
 import de.dfki.inquisitor.collections.MultiValueHashMap;
 import de.dfki.inquisitor.file.FileUtilz;
-// import de.dfki.inquisitor.lucene.FieldConfig;
 import de.dfki.km.leech.Leech;
 import de.dfki.km.leech.lucene.basic.FieldConfig;
 import de.dfki.km.leech.metadata.LeechMetadata;

src/main/java/de/dfki/km/leech/lucene/basic/FieldConfig.java

@@ -4,7 +4,6 @@
 
 import com.cedarsoftware.util.io.JsonReader;
 import com.cedarsoftware.util.io.JsonWriter;
-// import de.dfki.inquisitor.lucene.LuceneAnalyzerFactory;
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper;
 import org.apache.lucene.document.Field;

src/main/java/de/dfki/km/leech/parser/incremental/DataEntityHistoryEntry.java

@@ -0,0 +1,29 @@
+package de.dfki.km.leech.parser.incremental;
+
+
+
+import java.io.Serializable;
+
+
+
+public class DataEntityHistoryEntry implements Serializable
+{
+
+    public String dataEntityContentFingerprint;
+
+    public String dataEntityId;
+
+    public String masterDataEntityId;
+
+    public Long lastCrawledTime;
+
+
+
+    public DataEntityHistoryEntry(String dataEntityId, String dataEntityContentFingerprint, String masterDataEntityId, Long lastCrawledTime)
+    {
+        this.dataEntityContentFingerprint = dataEntityContentFingerprint;
+        this.dataEntityId = dataEntityId;
+        this.masterDataEntityId = masterDataEntityId;
+        this.lastCrawledTime = lastCrawledTime;
+    }
+}