v5.7.5

Self-synteny query and display improvements

Author: cariann09

README.md

@@ -1,10 +1,19 @@
-**Description**: SyMAP (Synteny Mapping and Analysis Program) is a software package for detecting, displaying, and querying syntenic relationships between a fully sequenced genome and (i) another sequenced genome, (ii) draft sequence, (iii) self-synteny, or (iv) FPC physical maps.  It is designed for medium-to-high divergent eukaryotic genomes (not bacteria).
+### SyMAP (Synteny Mapping and Analysis Program) ###
 
-**Download SyMAP tarball**: https://github.com/csoderlund/SyMAP/releases. The SyMAP tarball contains the executable jar files, external packages and demo files.
+**Description**: SyMAP is a software package for detecting, displaying, and querying syntenic relationships between a fully sequenced genome and   
+(i) another sequenced genome, (ii) draft sequence, and (iii) self-synteny.  It is designed for divergent eukaryotic genomes (not bacteria).
 
-**Documentation**: https://csoderlund.github.io/SyMAP
+**Download SyMAP tarball**: https://github.com/csoderlund/SyMAP/releases. The SyMAP tarball contains the executable jar file, all necessary external software, and demo files.
 
-**Requirements**: Java v17 or later and MySQL. SyMAP uses [MUMmer](https://mummer4.github.io) for the sequence to sequence alignment, which requires Perl. SyMAP has been tested on Linux and MacOS.
+**Requirements**: Java v17 or later and MySQL (or MariaDB). SyMAP uses [MUMmer](https://mummer4.github.io) for the sequence to sequence alignment, which requires Perl.   
+SyMAP has been tested on Linux and MacOS.
+
+**To use**: 
+1. Download the latest SyMAP tarball.
+2. Put the tarball in the location you want the symap_5 directory and untar it (tar xf symap_5.tar.gz).
+3. Follow the instructions in https://csoderlund.github.io/SyMAP/SystemGuide.html#demo to try the demo.
+
+**Full Documentation**: https://csoderlund.github.io/SyMAP
 
 **SyMAP github code**: To build, see SyMAP/java/README.
 

java/src/backend/AlignMain.java

@@ -9,6 +9,7 @@
 
 import java.util.TreeMap;
 import java.util.Queue;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.LinkedList;
@@ -34,7 +35,7 @@ public class AlignMain {
 	private static final int maxFileSize = 60000000; // 'Concat' parameter works for demo using this size
 
 	private ProgressDialog plog;
-	private String proj1Dir, proj2Dir; // also dbName
+	private String proj1Dir, proj2Dir; // also dbName, directory name
 	private Mproject mProj1, mProj2;
 
 	private String alignLogDirName;
@@ -140,8 +141,8 @@ public void run() {
 					plog.msg("Alignments:  success " + getNumCompleted());
 
 					String tmpDir = Constants.getNameTmpDir(proj1Dir, proj2Dir);// only delete tmp files if successful
-					if (Constants.PRT_STATS) {
-						System.out.println("Do not delete " + tmpDir);
+					if (Globals.TRACE) {// CAS575 changed from Constants.Prt_stats
+						Globals.prt("Do not delete " + tmpDir);
 					}
 					else {
 						File dir = new File(tmpDir);
@@ -168,7 +169,7 @@ private boolean alignExists() {
 		try {
 			String alignDir = Constants.getNameResultsDir(proj1Dir, proj2Dir) + Constants.alignDir;
 			File f = new File(alignDir);
-			if (!f.exists()) return false;	/*-- alignments to be done CAS571 was written .used... --*/
+			if (!f.exists()) return false;	/*-- alignments to be done ... --*/
 
 			boolean bdone = Utils.checkDoneFile(alignDir); // all.done
 			nAlignDone = Utils.checkDoneMaybe(alignDir);   // # existing .mum files
@@ -217,8 +218,7 @@ private void mummerParamsWrite() {
 			alignParams += mp.getChgAlign(Mpair.FILE, true); // true, do not add command line
 			if (Ext.isMummer4Path()) alignParams += Ext.getMummerPath() + " ";
 
-			//String resultDir = "./" + Constants.getNameResultsDir(mProj1.strDBName, mProj2.strDBName);
-			File pfile = Utils.getParamsFile(resultDir,Constants.usedFile); // CAS569 use this so renamed...
+			File pfile = Utils.getParamsFile(resultDir,Constants.usedFile); 
 			if (pfile==null) pfile = new File(resultDir,Constants.usedFile); // create 
 
 			PrintWriter out = new PrintWriter(pfile);
@@ -231,7 +231,7 @@ private void mummerParamsWrite() {
 	private void paramsRead(String dateTime) {
 		try {
 			alignParams = "Use previous " + nAlignDone + " MUMmer files " + dateTime; 
-			File pfile = Utils.getParamsFile(resultDir,Constants.usedFile); // CAS569 use this so renamed...
+			File pfile = Utils.getParamsFile(resultDir,Constants.usedFile); 
 			if (pfile==null) return;  // okay, not written yet
 
 			alignParams = "Use previous " + nAlignDone + " "; // completed with what is in file
@@ -250,12 +250,12 @@ private void paramsRead(String dateTime) {
 	 */
 	private void buildAlignments()  {
 		try {
-			plog.msg("\nAligning " + proj1Dir + " and " + proj2Dir + " with " + nMaxCPUs + " CPUs"); // CAS568 remove mp.getAlign, but add CPU
+			plog.msg("\nAligning " + proj1Dir + " and " + proj2Dir + " with " + nMaxCPUs + " CPUs"); 
 
 		/* create directories */
 			// result directory (e.g. data/seq_results/demo1_to_demo2)
 			Utilities.checkCreateDir(resultDir, true /* bPrt */);
-			mummerParamsWrite(); // CAS571 write here after creating the directory
+			mummerParamsWrite(); // write here after creating the directory
 
 			// temporary directories to put data for alignment
 			String tmpDir =  Constants.getNameTmpDir(proj1Dir, proj2Dir);
@@ -288,8 +288,8 @@ private void buildAlignments()  {
 			if (isSelf) program = Ext.exNucmer;
 
 			// user over-rides
-			if      (program.equals(Ext.exPromer) && mp.isNucmer(Mpair.FILE)) program = Ext.exNucmer; // CAS568 was backwards
-			else if (program.equals(Ext.exNucmer) && mp.isPromer(Mpair.FILE)) program = Ext.exPromer; // CAS568 was backwards
+			if      (program.equals(Ext.exPromer) && mp.isNucmer(Mpair.FILE)) program = Ext.exNucmer; 
+			else if (program.equals(Ext.exNucmer) && mp.isPromer(Mpair.FILE)) program = Ext.exPromer; 
 
 			String args = (program.contentEquals(Ext.exPromer)) ? mp.getPromerArgs(Mpair.FILE) : mp.getNucmerArgs(Mpair.FILE);
 			String self = (isSelf) ? (" " + mp.getSelfArgs(Mpair.FILE)) : ""; 
@@ -300,18 +300,22 @@ private void buildAlignments()  {
 			// For self, p1=p2
 			String alignDir = resultDir + Constants.alignDir;
 			Utilities.checkCreateDir(alignDir, false);
+			File[] fs1 = fh_tDir1.listFiles();		// order, not necessary, but nice; CAS575
+			Arrays.sort(fs1);	
+			File[] fs2 = fh_tDir2.listFiles();
+			Arrays.sort(fs2);	
 
-			for (File f1 : fh_tDir1.listFiles()) {    
+			for (File f1 : fs1) {    
 				if (!f1.isFile() || f1.isHidden()) continue; 
 
-				for (File f2 : fh_tDir2.listFiles()) {
+				for (File f2 : fs2) {
 					if (!f2.isFile() || f2.isHidden()) continue; 
 
 					// XXX
 					String aArgs = args;
 					if (isSelf) { // chr files have been written as is
 						if (f1.getName().equals(f2.getName())) aArgs += self;
-						else if (f1.getName().compareTo(f2.getName()) < 0)  continue; 
+						else if (f1.getName().compareTo(f2.getName()) < 0)  continue; // AnchorMain will correct order when wrong
 					}
 
 					AlignRun ps = new AlignRun(program, aArgs, f1, f2, alignDir, alignLogDirName);
@@ -321,7 +325,7 @@ private void buildAlignments()  {
 				}
 			}
 			if (alignList.size()>0) {
-				Collections.sort(alignList); // CAS566 add sort and add to queue after sort
+				Collections.sort(alignList); 
 				for (AlignRun ps : alignList) {
 					toDoQueue.add(ps);
 					ps.setStatus(AlignRun.STATUS_QUEUED);
@@ -330,7 +334,7 @@ private void buildAlignments()  {
 			}
 			if (nAlignDone>0) {
 				if (alignList.size()==0) Utils.writeDoneFile(alignDir);
-				plog.msg("Alignments to be completed: " + alignList.size()); // CAS566 add
+				plog.msg("Alignments to be completed: " + alignList.size()); 
 			}
 			else if (alignList.size() == 0  && !Cancelled.isCancelled()) {
 				plog.msg("Warning: no alignments between projects");
@@ -362,7 +366,7 @@ private boolean writePreprocSeq(File dir, Mproject mProj,  boolean concat, boole
 			}
 
 			boolean geneMask = (mp.mProj1.getIdx() == mProj.getIdx()) ? 
-								mp.isMask1(Mpair.FILE) : mp.isMask2(Mpair.FILE);	// CAS568 mask in pairs instead of proj
+								mp.isMask1(Mpair.FILE) : mp.isMask2(Mpair.FILE);	
 			String gmprop = (geneMask) ? "Masking non-genic sequence; " : "";
 			if (geneMask) plog.msg(projName + ": " + gmprop);
 
@@ -377,7 +381,7 @@ private boolean writePreprocSeq(File dir, Mproject mProj,  boolean concat, boole
 				if (concat) msg += "Concatenating all sequences into one file for " + who; 
 				else        msg += "Writing sequences into one or more files for " + who; 
 			}
-			else msg += "Writing separate chromosome files for self-synteny"; // CAS568
+			else msg += "Writing separate chromosome files for self-synteny";
 
 			// pseudos (files): grp_idx, fileName (e.g. chr3.seq), length
 			// xgroups (chrs):  idx, proj_idx, chr#, fullname where grp_idx is xgroups.idx
@@ -469,7 +473,7 @@ private boolean writePreprocSeq(File dir, Mproject mProj,  boolean concat, boole
 					if (interrupted) break;
 
 					rs = tdbc2.executeQuery("select fullname from xgroups where idx=" + gIdx);
-					rs.next(); // CAS560 was first()
+					rs.next(); 
 					String grpFullName = rs.getString(1);
 					fw.write(">" + grpFullName + "\n");
 
@@ -548,11 +552,11 @@ protected int getNumRunning() {
 	}
 	protected int getNumRemaining() {
 		synchronized(threads) {
-			return toDoQueue.size(); // CAS566 don't add error (was +getNumErrors();) 
+			return toDoQueue.size(); 
 		}
 	}
 	protected int getNumCompleted() {
-		return alignList.size() - getNumRunning() - getNumRemaining() - getNumErrors(); // CAS566 remove errors here
+		return alignList.size() - getNumRunning() - getNumRemaining() - getNumErrors(); 
 	}
 	protected int getNumErrors() {
 		int count = 0;

java/src/backend/AnchorMain.java

@@ -27,7 +27,8 @@
  * Mproj1 (arab) is query and Mproj2 (cabb) is target; target coords (cabb) comes before query (arab)
  */
 public class AnchorMain {
-	public boolean doAlgo1=false; 
+	public boolean doAlgo1=false;
+	private boolean isSelf=false;  // CAS575 add
 
 	// variables may be accessed directly by AnchorMain1 or AnchorMain2
 	public DBconn2 dbc2;
@@ -50,9 +51,10 @@ protected boolean run(Mproject pj1, Mproject pj2) throws Exception {
 		try {
 		/** setup **/
 			long startTime = Utils.getTimeMem();
+			isSelf = pj1.getIdx()==pj2.getIdx();
 
 			String proj1Name = pj1.getDBName(), proj2Name = pj2.getDBName();
-			if (proj1Name.equals(proj2Name)) plog.msg("\nStart calculating cluster hits for " + proj1Name + " self synteny"); // CAS572 add
+			if (isSelf) plog.msg("\nStart calculating cluster hits for " + proj1Name + " self synteny"); 
 			else plog.msg("\nStart calculating cluster hits for " + proj1Name + " and " + proj2Name);
 
 			String resultDir = Constants.getNameResultsDir(proj1Name, proj2Name); // e.g. data/seq_results/p1_to_p2
@@ -79,7 +81,7 @@ protected boolean run(Mproject pj1, Mproject pj2) throws Exception {
 			if (Cancelled.isCancelled()) {dbc2.close(); return false; } 
 
 			saveHitNum();
-			saveAnnoHitCnt();
+			saveAnnoHitCnt(true);
 
 			/** Run before pseudo **/
 			if (pj1.hasGenes() && pj2.hasGenes()) { 
@@ -104,6 +106,7 @@ protected boolean run(Mproject pj1, Mproject pj2) throws Exception {
 
 	/***************************************************************
 	 * Assign hitnum, with them ordered by start1 ASC, len DESC
+	 * isSelf: just do upper part of diagonal: only number if refidx=0
 	 */
 	private boolean saveHitNum() { 
 	try {
@@ -113,18 +116,21 @@ private boolean saveHitNum() {
 
 		for (int g1 : grpMap1.keySet()) {
 			for (int g2 : grpMap2.keySet()) {
+				if (isSelf && g1<g2) continue; // DIR_SELF lower diagonal 
+				
 				int nhits = dbc2.executeCount("select count(*) from pseudo_hits "
 						+ " where grp1_idx="+ g1 + " and grp2_idx=" + g2); 
 				if (nhits==0) continue;										// mysql error if 0 in next query
 
 				dbc2.executeUpdate("update pseudo_hits set hitnum=0 where grp1_idx=" + g1 + " and grp2_idx=" + g2);
 
+				// do not use grp1_idx<grp2_idx because this assumes chr01 is loaded before chr02
 				HashMap <Integer, Integer> hitNumMap = new HashMap <Integer, Integer> ();
-			// assign
 				int hitcnt=1;
 				ResultSet rs = dbc2.executeQuery("select idx, (end1-start1) as len from pseudo_hits "
-						+ " where grp1_idx= " + g1 + " and grp2_idx=" + g2
+						+ " where grp1_idx= " + g1 + " and grp2_idx=" + g2  + " and refidx=0"  // CAS575 add refidx for isSelf
 						+ " order by start1 ASC, len DESC");
+				
 				while (rs.next()) {
 					int hidx = rs.getInt(1);
 					hitNumMap.put(hidx, hitcnt);
@@ -145,15 +151,23 @@ private boolean saveHitNum() {
 			}
 		}
 		Globals.rclear();
+		
+		if (isSelf) { // mirror hitnum for self
+			dbc2.executeUpdate("update pseudo_hits as ph1, pseudo_hits as ph2 "
+				+ " set ph2.hitnum=ph1.hitnum "
+				+ " where ph1.pair_idx=" + pairIdx + " and ph2.pair_idx=" + pairIdx
+				+ " and ph1.idx=ph2.refidx");
+		}
 		return true;
 	}
 	catch (Exception e) {ErrorReport.print(e, "compute gene hit cnts"); return false;}
 	}
 	/********************************************************************
-	 * Count number of hits per gene; these include all pairwise projects except self; 
+	 * Count number of hits per gene; these include all pairwise projects
+	 * CAS575 this includes self, but self hits get counted for both sides of the diagonal
 	 */
-	public void saveAnnoHitCnt() { // public for Mpair.removePairFromDB
-		Globals.rprt("Compute and save gene numHits"); 
+	public void saveAnnoHitCnt(boolean bPrt) { // public for Mpair.removePairFromDB; add bPrt CAS575 so does not show on remove
+		if (bPrt) Globals.rprt("Compute and save gene numHits"); 
 
 		TreeMap <Integer, String> idxList1 = mp.mProj1.getGrpIdxMap();
 		TreeMap <Integer, String> idxList2 = mp.mProj2.getGrpIdxMap();
@@ -177,7 +191,7 @@ public boolean saveAnnotHitCnt(int grpIdx, String grpName) { // public for Mproj
 					+ "from pseudo_annot           as pa "
 					+ "join pseudo_hits_annot      as pha   on pha.annot_idx  = pa.idx "
 					+ "join pseudo_hits			   as ph	on pha.hit_idx    = ph.idx "
-					+ "where pa.type='gene' and ph.proj1_idx != ph.proj2_idx and pa.grp_idx=" + grpIdx);
+					+ "where pa.type='gene' and pa.grp_idx=" + grpIdx);		// CAS575 remove ph.proj1_idx != ph.proj2_idx
 
 			while (rs.next()) {
 				int aidx = rs.getInt(1);
@@ -301,15 +315,15 @@ private boolean addPseudo() {
 
 			int cnt=1;
 			for (int g1 : grpMap1.keySet()) {
-				chrPair = mProj1.getDisplayName() + " " + mProj1.getGrpNameFromIdx(g1);
+				chrPair = mProj1.getDisplayName() + " " + mProj1.getGrpFullNameFromIdx(g1);
 
 				String where = " where grp1_idx= " + g1 + " and grp2_idx IN (" + grpList2 + ") and annot1_idx=0 "; 
 
 				if (!addPseudoGrpGrp(g1, where, "annot1_idx", 0, cnt++, grpMap1.size())) return false;
 			}
 			cnt=1;
 			for (int g2 : grpMap2.keySet()) {
-				chrPair = mProj2.getDisplayName() + " " + mProj2.getGrpNameFromIdx(g2);
+				chrPair = mProj2.getDisplayName() + " " + mProj2.getGrpFullNameFromIdx(g2);
 
 				String where = " where grp1_idx IN (" + grpList1  + ") and grp2_idx=" + g2 + " and annot2_idx=0 "; 
 
@@ -363,23 +377,16 @@ private boolean addPseudoGrpGrp(int grpIdx, String where, String annoStr, int i,
 			rs.close();
 
 			String dn =  (i==0) ? mProj1.getDisplayName() : mProj2.getDisplayName();
-			String chr = (i==0) ? mProj1.getGrpNameFromIdx(grpIdx) : mProj2.getGrpNameFromIdx(grpIdx);
+			String chr = (i==0) ? mProj1.getGrpFullNameFromIdx(grpIdx) : mProj2.getGrpFullNameFromIdx(grpIdx);
 			String msg = String.format("Pseudo %-20s   Genes %,6d   Start %,6d ", (dn+" "+chr), genes, geneStart); 
-			/* too much output CAS572
-			if (Constants.VERBOSE && cntGrp==1) Utils.prtIndentNumMsgFile(plog, 1, hitGeneMap.size(), msg);  
-			if (Constants.VERBOSE && cntGrp==2) {
-				if (maxGrp>2) msg += "...";
-				Utils.prtIndentNumMsgFile(plog, 1, hitGeneMap.size(), msg);  
-			}
-			else */
 			Globals.rprt(String.format("%,5d %s", hitGeneMap.size(),msg));
 
 			if (hitGeneMap.size()==0) return true;
 
 			cntPseudo[i] += hitGeneMap.size();
 
 		// pseudo_annot
-			String name= "ID=pseudo-" + mProj1.getdbAbbrev()+"-"+mProj2.getdbAbbrev() +
+			String name= "ID=pseudo-" + mProj1.getDBName() +"-"+mProj2.getDBName() + // was abbrev, which can change; CAS575
 					";" + annoKey[i] + "=Not annotated; ";
 
 			int countBatch=0, cnt=0;