Start to hook together all indexing

Author: boyter

src/main/java/com/searchcode/app/jobs/enqueue/EnqueueSearchcodeRepositoryJob.java

@@ -1,57 +1,68 @@
-package com.searchcode.app.jobs.enqueue;
-
-import com.searchcode.app.dao.IRepo;
-import com.searchcode.app.service.Singleton;
-import com.searchcode.app.service.index.IIndexService;
-import com.searchcode.app.util.Helpers;
-import com.searchcode.app.util.LoggerWrapper;
-import org.quartz.DisallowConcurrentExecution;
-import org.quartz.Job;
-import org.quartz.JobExecutionContext;
-import org.quartz.PersistJobDataAfterExecution;
-
-/**
- * Responsible for adding all of the repositories inside the database into the queues. There will be a queue
- * for GIT and SVN or any other repository added.
- * TODO add logic to test that the right queue has things added to it
- */
-@PersistJobDataAfterExecution
-@DisallowConcurrentExecution
-public class EnqueueSearchcodeRepositoryJob implements Job {
-    private final IIndexService indexService;
-    private final LoggerWrapper logger;
-    private final IRepo repo;
-    private final Helpers helpers;
-
-    public EnqueueSearchcodeRepositoryJob() {
-        this.indexService = Singleton.getIndexService();
-        this.repo = Singleton.getRepo();
-        this.logger = Singleton.getLogger();
-        this.helpers = Singleton.getHelpers();
-    }
-
-    public void execute(JobExecutionContext context) {
-        this.logger.info("fe39b962::starting enqueue job");
-        Thread.currentThread().setPriority(Thread.MIN_PRIORITY);
-
-        // TODO this never returns false, should probably implement back-pressure
-        if (this.indexService.shouldPause(IIndexService.JobType.REPO_ADDER)) {
-            return;
-        }
-
-        try {
-            Thread.currentThread().setPriority(Thread.MIN_PRIORITY);
-
-            var repoGitQueue = Singleton.getUniqueGitRepoQueue();
-            var repoSvnQueue = Singleton.getUniqueSvnRepoQueue();
-
-            // Determine where we have started from a repo collection
-            // determine where that ends
-            // update the point
-            // Get the batch of repositories from the database
-            // add them to the queue
-        } catch (Exception ex) {
-            this.logger.severe(String.format("8675db36::error in class %s exception %s", ex.getClass(), ex.getMessage()));
-        }
-    }
-}
+package com.searchcode.app.jobs.enqueue;
+
+import com.searchcode.app.dao.Data;
+import com.searchcode.app.dao.IRepo;
+import com.searchcode.app.service.Singleton;
+import com.searchcode.app.service.index.IIndexService;
+import com.searchcode.app.util.Helpers;
+import com.searchcode.app.util.LoggerWrapper;
+import org.quartz.DisallowConcurrentExecution;
+import org.quartz.Job;
+import org.quartz.JobExecutionContext;
+import org.quartz.PersistJobDataAfterExecution;
+
+/**
+ * Responsible for adding all of the repositories inside the database into the queues. There will be a queue
+ * for GIT and SVN or any other repository added.
+ * TODO add logic to test that the right queue has things added to it
+ */
+@PersistJobDataAfterExecution
+@DisallowConcurrentExecution
+public class EnqueueSearchcodeRepositoryJob implements Job {
+    private final IIndexService indexService;
+    private final LoggerWrapper logger;
+    private final IRepo repo;
+    private final Helpers helpers;
+    private final Data data;
+
+    public EnqueueSearchcodeRepositoryJob() {
+        this.indexService = Singleton.getIndexService();
+        this.repo = Singleton.getRepo();
+        this.logger = Singleton.getLogger();
+        this.helpers = Singleton.getHelpers();
+        this.data = Singleton.getData();
+    }
+
+    public void execute(JobExecutionContext context) {
+        this.logger.info("fe39b962::starting enqueue job");
+        Thread.currentThread().setPriority(Thread.MIN_PRIORITY);
+
+        // TODO this never returns false, should probably implement back-pressure
+        if (this.indexService.shouldPause(IIndexService.JobType.REPO_ADDER)) {
+            return;
+        }
+
+        try {
+            Thread.currentThread().setPriority(Thread.MIN_PRIORITY);
+
+            var repoGitQueue = Singleton.getUniqueGitRepoQueue();
+            var repoSvnQueue = Singleton.getUniqueSvnRepoQueue();
+
+//            this.repo.getAllRepo();
+
+//            for (var r : this.repo.getAllRepo()) {
+//                repoGitQueue.add(r);
+//            }
+
+            repoGitQueue.add(this.repo.getRepoById(1).get());
+
+            // Determine where we have started from a repo collection
+            // determine where that ends
+            // update the point
+            // Get the batch of repositories from the database
+            // add them to the queue
+        } catch (Exception ex) {
+            this.logger.severe(String.format("8675db36::error in class %s exception %s", ex.getClass(), ex.getMessage()));
+        }
+    }
+}

src/main/java/com/searchcode/app/jobs/searchcode/ReindexerJob.java

@@ -53,7 +53,9 @@ public void execute(JobExecutionContext context) {
         try {
             while (true) {
                 // Sleep for a jitter time to avoid many indexers converging on the same schedule
-                Thread.sleep(this.INDEXTIME * this.helpers.getRandomJitterSleepTimeMilliseconds());
+                var sleepTime = this.helpers.getRandomJitterSleepTimeMilliseconds();
+                this.logger.info(String.format("6d380ec6::reindexer sleep time %d", sleepTime));
+                Thread.sleep(sleepTime);
 
                 var codeIndexQueueSize = this.indexQueue.size();
 

src/main/java/com/searchcode/app/service/JobService.java

@@ -85,7 +85,7 @@ public JobService() {
      * Starts all of the above jobs as per their unique requirements between searchcode.com
      * and local runner
      */
-    public void initialJobs() {
+    public synchronized void initialJobs() {
         // Having this run multiple times can be an issue so ensure it can not happen
         if (this.initialJobsRun) {
             return;
@@ -102,6 +102,7 @@ public void initialJobs() {
             // searchcode.com path
             this.startHighlighter();
             this.startReIndexer();
+            this.startRepositoryJobs();
             this.startSearchcodeEnqueueJob();
         }