Updated Konstanz inserter and implemented wait time ETAs

Author: Nils Fo

src/main/java/de/rub/bph/omnineuro/client/Client.java

@@ -4,10 +4,12 @@
 import de.rub.bph.omnineuro.client.imported.log.Log;
 import de.rub.bph.omnineuro.client.ui.DialogFrame;
 import de.rub.bph.omnineuro.client.ui.OmniFrame;
+import de.rub.bph.omnineuro.client.util.TimeUtils;
 
 import javax.swing.*;
 import java.awt.*;
 import java.sql.SQLException;
+import java.util.Date;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
@@ -29,6 +31,7 @@ public static void main(String[] args) {
 			DateInterpreter.initMonthMap();
 
 			new OmniFrame();
+			Log.i("Current time: " + new TimeUtils().formatAbsolute(new Date(), 0, 0));
 		});
 	}
 

src/main/java/de/rub/bph/omnineuro/client/core/AXESSheetReaderManager.java

@@ -57,7 +57,7 @@ public ArrayList<JSONObject> startReading() {
 			AXESSheetReader reader = new AXESSheetReader(f, outDir);
 
 			readers.add(reader);
-			service.submit(reader);
+			submitTask(reader);
 		}
 
 		waitForTasks();
@@ -122,6 +122,11 @@ public ArrayList<File> discoverFiles(File currentDir) {
 		return list;
 	}
 
+	@Override
+	protected void onWaitUpdate() {
+	
+	}
+	
 	public FileNameExtensionFilter getFilter() {
 		return filter;
 	}

src/main/java/de/rub/bph/omnineuro/client/core/db/QueryExecutor.java

@@ -111,6 +111,11 @@ public synchronized String getFeatureViaID(String tableName, String columnName,
 		throw new IllegalStateException("Table '" + tableName + "' does not contain id " + id + "!");
 	}
 
+	public synchronized String getFeatureViaFeature(String tableName, String sourceColumn, String targetColumn, String feature) throws SQLException, IllegalStateException {
+		long id = getIDViaFeature(tableName, sourceColumn, feature);
+		return getFeatureViaID(tableName, targetColumn, id);
+	}
+	
 	public synchronized String getNameViaID(String tableName, long id) throws SQLException, IllegalStateException {
 		return getFeatureViaID(tableName, "name", id);
 	}
@@ -291,5 +296,13 @@ public void setLogEnabled(boolean logEnabled) {
 		this.logEnabled = logEnabled;
 	}
 
+	public synchronized boolean isExecutableQuery(String query) {
+		try {
+			executeQuery(query);
+		} catch (SQLException e) {
+			return false;
+		}
+		return true;
+	}
 
 }

src/main/java/de/rub/bph/omnineuro/client/core/db/in/AXESInserter.java

@@ -332,6 +332,7 @@ public void run() {
 			addError(" == FATAL ERROR! == Failed to insert Experiment " + getName() + " into the database! Error Type: " + e.getClass().getSimpleName() + ". Reason: '" + e.getMessage() + "'");
 		}
 		Log.i("Finished inserting responses for " + getName() + ". Count: " + getInsertedResponsesCount());
+		setFinished();
 	}
 
 	@Override

src/main/java/de/rub/bph/omnineuro/client/core/db/in/DBInserter.java

@@ -4,11 +4,12 @@
 import de.rub.bph.omnineuro.client.core.db.OmniNeuroQueryExecutor;
 import de.rub.bph.omnineuro.client.core.db.out.holder.CompoundHolder;
 import de.rub.bph.omnineuro.client.imported.log.Log;
+import de.rub.bph.omnineuro.client.util.concurrent.TimedRunnable;
 
 import java.sql.Connection;
 import java.util.ArrayList;
 
-public abstract class DBInserter implements Runnable {
+public abstract class DBInserter extends TimedRunnable {
 
 	protected static OmniNeuroQueryExecutor executor;
 	protected boolean attemptUnblinding;

src/main/java/de/rub/bph/omnineuro/client/core/db/in/InsertManager.java

@@ -8,13 +8,17 @@
 import de.rub.bph.omnineuro.client.core.sheet.reader.versions.meta.ExternalMetadata;
 import de.rub.bph.omnineuro.client.imported.filemanager.FileManager;
 import de.rub.bph.omnineuro.client.imported.log.Log;
+import de.rub.bph.omnineuro.client.util.TimeUtils;
+import de.rub.bph.omnineuro.client.util.concurrent.TimedRunnable;
 import org.json.JSONObject;
 
 import java.awt.*;
 import java.io.File;
 import java.io.IOException;
 import java.sql.SQLException;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
@@ -198,7 +202,20 @@ public void insert() {
 			} catch (InterruptedException e) {
 				e.printStackTrace();
 			}
-			Log.i("Still waiting for threads to finish...");
+			
+			ArrayList<TimedRunnable> timedRunnableList = new ArrayList<>();
+			int finishedCount = 0;
+			for (DBInserter in : inserters) {
+				if (in.isFinished()) finishedCount++;
+				TimedRunnable tr = in;
+				timedRunnableList.add(tr);
+			}
+			
+			long etaTime = TimedRunnable.calculateETA(timedRunnableList, inserters.size() - finishedCount, getThreads() / -3);
+			Date etaDate = new Date(new Date().getTime() + etaTime);
+			String etaText = new TimeUtils().formatAbsolute(etaDate, 3, 0);
+			
+			Log.i("Still waiting for inserter threads to finish: " + finishedCount + " / " + inserters.size() + ". ETA: " + etaText);
 		}
 		Log.i("Done waiting.");
 
@@ -207,10 +224,12 @@ public void insert() {
 		ArrayList<String> containsNaNList = new ArrayList<>();
 		ArrayList<String> insertedResponsesList = new ArrayList<>();
 		ArrayList<String> insertedBlindedCompounds = new ArrayList<>();
+		ArrayList<String> konstanzCasList = new ArrayList<>();
 		triviaList = new ArrayList<>();
 
 		int errorNaNCount = 0;
 		int insertedResponsesCount = 0;
+		konstanzCasList.add("Source;Sample ID;Plate ID;Read CAS;Existed in the DB;Database Compound;Database Abbreviation;Same DB Name");
 
 		for (DBInserter inserter : inserters) {
 			String name = inserter.getName();
@@ -229,6 +248,14 @@ public void insert() {
 			}
 
 			insertedBlindedCompounds.addAll(inserter.getBlindingInfo());
+			
+			if (inserter instanceof KonstanzInserter) {
+				KonstanzInserter ki = (KonstanzInserter) inserter;
+				
+				ArrayList<String> temp = new ArrayList<>(ki.getCASRows());
+				Collections.sort(temp);
+				konstanzCasList.addAll(temp);
+			}
 		}
 
 		triviaList.add("Total errors discovered: " + errors.size());
@@ -265,6 +292,7 @@ public void insert() {
 			manager.saveListFile(containsNaNList, new File(outDir, "contains_nan.csv"), false);
 			manager.saveListFile(insertedResponsesList, new File(outDir, "inserted_responses.csv"), false);
 			manager.saveListFile(triviaList, new File(outDir, "trivia.txt"), false);
+			manager.saveListFile(konstanzCasList, new File(outDir, "konstanz_cas.csv"), false);
 
 			manager.saveListFile(insertedBlindedCompounds, new File(outDir, "blindedCompounds.txt"), true, false);
 		} catch (IOException e) {

src/main/java/de/rub/bph/omnineuro/client/core/db/in/KonstanzInserter.java

@@ -19,11 +19,13 @@ public class KonstanzInserter extends DBInserter {
 	private static final int START_ROW_INDEX = 4;
 	private File sourceFile;
 	private HashMap<String, ArrayList<String>> lookupMapDMSO;
+	private ArrayList<String> casOutRows;
 
 	public KonstanzInserter(File sourceFile, boolean attemptUnblinding) {
 		super(attemptUnblinding);
 		this.sourceFile = sourceFile;
 		lookupMapDMSO = new HashMap<>();
+		casOutRows = new ArrayList<>();
 	}
 
 	private long decodeKonstanzEndpoint(String endpointName) throws SQLException {
@@ -66,6 +68,10 @@ private long decodeKonstanzEndpoint(String endpointName) throws SQLException {
 		}
 	}
 
+	protected void addCASRow(String row) {
+		casOutRows.add(row);
+	}
+	
 	@Override
 	public void run() {
 		long timestampExperiment = new Date(0).getTime();
@@ -81,6 +87,7 @@ public void run() {
 			FileInputStream excelFile = new FileInputStream(sourceFile);
 			XSSFWorkbook workbook = new XSSFWorkbook(excelFile);
 			workbook.setActiveSheet(0);
+			String CASCol = "K";
 
 			int rowIndex = START_ROW_INDEX;
 			SheetReader reader = new SheetReader(sourceFile, workbook);
@@ -109,25 +116,47 @@ public void run() {
 			if (reader.hasValueAt("J3")) {
 				extendedEndpoints = true;
 				endpoint3 = reader.getValueAt("J3");
+				CASCol = "L";
 			}
 
-			for (int i = START_ROW_INDEX; i < rowCount; i++) {
+			int rowTarget = rowCount + START_ROW_INDEX;
+			for (int i = START_ROW_INDEX; i < rowTarget; i++) {
 				cachedRow = i;
 				cachedWell = "<Unknown>";
 
 				// EXTRACT INFORMATION FROM SHEET
 				String sampleID = reader.getValueAt("A" + i);
 				String plateID = reader.getValueAt("B" + i);
 
-				String wellRow = reader.getValueAt("C" + i);
-				int wellCol = (int) Double.parseDouble(reader.getValueAt("D" + i, true));
-				WellBuilder wellBuilder = new WellBuilder(wellRow, wellCol);
-				String wellName = wellBuilder.getWellExtended();
+				boolean hasWell = true;
+				String wellName;
+				try {
+					String wellRow = reader.getValueAt("C" + i);
+					int wellCol = (int) Double.parseDouble(reader.getValueAt("D" + i, true));
+					WellBuilder wellBuilder = new WellBuilder(wellRow, wellCol);
+					wellName = wellBuilder.getWellExtended();
+				} catch (Exception e) {
+					hasWell = false;
+					wellName = "Z" + cachedRow;
+					CASCol = "J";
+				}
 				cachedWell = wellName;
 
 				String wellType = reader.getValueAt("E" + i);
 				String wellQuality = reader.getValueAt("F" + i);
 
+				String CAS = null;
+				try {
+					String CASCell = CASCol + i;
+					CAS = reader.getValueAt(CASCell, false).trim();
+					Log.v("Cas at " + CASCell + " found: " + CAS);
+					if (CAS.equals("") || CAS.equals("NaN")) CAS = null;
+				} catch (Exception e) {
+					Log.e("Failed to read the CAS Number. But that's okay, switching to 'No CAS Mode.'", e);
+					CAS = null;
+				}
+				boolean CASMode = CAS != null;
+				
 				long outlierID = confirmedOutlierID;
 				try {
 					int wellQualityNumeric = (int) Double.parseDouble(wellQuality);
@@ -155,10 +184,12 @@ public void run() {
 				} catch (Exception e) {
 					addError("Failed to the response to " + endpoint1 + " at H" + i + ". Error: " + e.getMessage());
 				}
-				try {
-					response2 = Double.parseDouble(reader.getValueAt("I" + i, true));
-				} catch (Exception e) {
-					addError("Failed to the response to " + endpoint2 + " at I" + i + ". Error: " + e.getMessage());
+				if (hasWell) {
+					try {
+						response2 = Double.parseDouble(reader.getValueAt("I" + i, true));
+					} catch (Exception e) {
+						addError("Failed to the response to " + endpoint2 + " at I" + i + ". Error: " + e.getMessage());
+					}
 				}
 				if (extendedEndpoints) {
 					try {
@@ -208,21 +239,62 @@ public void run() {
 							long workgroupID = 3;
 
 							long compoundID;
-							try {
-								compoundID = executor.getIDViaName("compound", sampleID);
-							} catch (Exception exx) {
+							if (CASMode) {
+								boolean foundCas;
+								try {
+									//Let's see if that CAS exists in the DB as a registered compound
+									compoundID = executor.getIDViaFeature("compound", "cas_no", CAS);
+									foundCas = true;
+								} catch (Exception exx) {
+									try {
+										String unblindedCAS = executor.getFeatureViaFeature("unblinded_compound_mapping", "name", "unblinded_cas_number", CAS);
+										addBlindingRow("Blinded entry: " + CAS + " was unblinded to: " + unblindedCAS);
+										compoundID = executor.getIDViaFeature("compound", "cas_no", unblindedCAS);
+										attemptUnblinding = false;
+										foundCas = true;
+									} catch (Exception exxx) {
+										//What if the CAS does not exists in the DB, but the Compound name?
+										String actualCAS = null;
+										try {
+											long actualID = executor.getIDViaName("compound", sampleID);
+											actualCAS = executor.getFeatureViaID("compound", "cas_no", actualID);
+										} catch (Exception exxxx) {
+											// all good. nvm.
+										}
+										if (actualCAS != null) {
+											String outRowExtra = sfName + ";" + sampleID + ";" + plateID + ";" + CAS + ";DUPLICATE;;;;DUPLICATE COMPOUND NAME BUT DIFFERENT CAS:;" + actualCAS;
+											addCASRow(outRowExtra);
+										}
+										
+										//So our CAS was not in the DB and not in the unblinded mapping. Let's insert it as a new compound then.
+										executor.insertCompound(sampleID, CAS, sampleID, true);
+										foundCas = false;
+										compoundID = executor.getIDViaName("compound", sampleID);
+									}
+								}
+								
+								String casName = executor.getNameViaID("compound", compoundID);
+								String abbreviation = executor.getFeatureViaID("compound", "abbreviation", compoundID);
+								boolean equals = casName.equals(sampleID);
+								String outRow = sfName + ";" + sampleID + ";" + plateID + ";" + CAS + ";" + String.valueOf(foundCas).toUpperCase() + ";" + casName + ";" + abbreviation + ";" + String.valueOf(equals).toUpperCase();
+								addCASRow(outRow);
+							} else {
 								try {
-									compoundID = executor.getIDViaFeature("compound", "abbreviation", sampleID);
-								} catch (Exception ex) {
-									//ex.printStackTrace();
-									executor.insertCompound(sampleID, sampleID, sampleID, true);
 									compoundID = executor.getIDViaName("compound", sampleID);
+								} catch (Exception exx) {
+									try {
+										compoundID = executor.getIDViaFeature("compound", "abbreviation", sampleID);
+									} catch (Exception ex) {
+										//ex.printStackTrace();
+										executor.insertCompound(sampleID, sampleID, sampleID, true);
+										compoundID = executor.getIDViaName("compound", sampleID);
+									}
 								}
 							}
 
 							if (attemptUnblinding) {
 								if (isControl) {
-									addBlindingRow(sampleID + " is not a blinded compound.");
+									addBlindingRow(sampleID + " [Line " + i + "] is not a blinded compound. It's a control.");
 								} else {
 									CompoundHolder compoundHolder = attemptUnblinding(sampleID);
 									if (compoundHolder != null) {
@@ -240,7 +312,7 @@ public void run() {
 							experimentID = executor.getNextSequenceTableVal("experiment");
 							executor.insertExperiment(experimentID, timestampExperiment, experimentName, projectID, workgroupID, individualID, compoundID, cellTypeID, assayID, plateFormatID, solventID, solventConcentration, controlPlateID);
 						}
-						Log.i("Experiment ID extracted: " + experimentID);
+						Log.v("Experiment ID extracted: " + experimentID);
 					}
 
 					long wellID;
@@ -295,19 +367,26 @@ public void run() {
 				} catch (Exception e) {
 					Log.e(e);
 					addError("FATAL Error! " + sfName + ": Error on row " + i + " to be inserted into the experiment: " + e.getMessage());
+					addCASRow(sfName + ";" + sampleID + ";" + plateID + ";FATAL ERROR on row " + i);
 					continue;
 				}
-				Log.i("Successfully inserted row " + i + " from file: " + sfName);
+				Log.v("Successfully inserted row " + i + " from file: " + sfName);
 			}
 			workbook.close();
 		} catch (Throwable e) {
 			Log.e("Fatal Error in sheet " + sourceFile.getName(), e);
 			addError("Fatal Error in: " + sourceFile.getName() + ". Type: " + e.getClass().getSimpleName() + ": '" + e.getMessage() + "'! Last known row: " + cachedRow + ". Last known well: " + cachedWell);
 		}
+		
+		setFinished();
 	}
 
 	@Override
 	public String getName() {
 		return sourceFile.getName();
 	}
+	
+	public ArrayList<String> getCASRows() {
+		return new ArrayList<>(casOutRows);
+	}
 }