2.6 drafts 3 and 4

* Repair spelling of Chaetothyriomycetidae
* Suppress false equation of the two Morganellas, similarly for 7 other SILVA/IF conflicts
* Fix Myospalax (again) #62
* Various repairs in fungi #42
* Fix Norops (again) #31
* Remove a few erroneous extinct flags (Conus, etc.)
* New output file hidden.tsv lists all hidden taxa
* New flag "hidden_inherited"
* Fix bug in notSame

Author: jar398

Makefile

@@ -5,7 +5,7 @@
 # Get it from http://files.opentreeoflife.org/ott/
 # and if there's a file "taxonomy" change that to "taxonomy.tsv".
 
-WHICH=2.6draft2
+WHICH=2.6draft4
 PREV_WHICH=2.5
 
 #  $^ = all prerequisites

make-ott.py

@@ -9,8 +9,15 @@
 
 # Hibbett 2007 updated upper fungal taxonomy
 h2007 = Taxonomy.getNewick('feed/h2007/tree.tre', 'h2007')
+
+# 2014-04-08 Misspelling
+h2007.taxon('Chaetothryriomycetidae').rename('Chaetothyriomycetidae')
+
 ott.absorb(h2007)
 
+# h2007/if synonym https://github.com/OpenTreeOfLife/reference-taxonomy/issues/40
+ott.taxon('Urocystales').synonym('Urocystidales')
+
 # SILVA microbial taxonomy
 silva = Taxonomy.getTaxonomy('tax/silva/', 'silva')
 
@@ -60,14 +67,28 @@
 # 2014-03-07 Prevent a false match
 ott.notSame(silva.taxon('Phaeosphaeria'), fung.taxon('Phaeosphaeria'))
 
+# 2014-04-08 This was causing Agaricaceae to become 'tattered'
+ott.notSame(silva.taxon('Morganella'), fung.taxon('Morganella'))
+
+# 2014-04-08 More IF/SILVA bad matches (probably sample contamination)
+for name in ["Trichoderma harzianum",
+			 "Acantharia",
+			 "Bogoriella",
+			 "Steinia",
+			 "Sclerotinia homoeocarpa",
+			 "Epiphloea",
+			 "Campanella",
+			 "Lacrymaria"]:
+	ott.notSame(silva.taxon(name), fung.taxon(name))
+
 # analyzeMajorRankConflicts sets the "major_rank_conflict" flag when
 # intermediate ranks are missing (e.g. a family that's a child of a
 # class)
 fung.analyzeMajorRankConflicts()
 
 ott.absorb(fung)
 
-# Problem: Chamydotomus is an incertae sedis child of Fungi.
+# Problem: Chlamydotomus is an incertae sedis child of Fungi.
 # http://www.mycobank.org/BioloMICS.aspx?Link=T&TableKey=14682616000000067&Rec=35058&Fields=All
 # https://github.com/OpenTreeOfLife/reference-taxonomy/issues/20
 Cb = ott.taxon('Chlamydotomus beigelii')
@@ -142,7 +163,8 @@
 ott.notSame(silva.taxon('GN013951'), gbif.taxon('Gorkadinium')) #Tetrasphaera
 
 # Joseph 2013-07-23 https://github.com/OpenTreeOfLife/opentree/issues/62
-gbif.taxon('Myospalax','Muridae').absorb(gbif.taxon('2439119'))
+# GBIF has two copies of Myospalax
+gbif.taxon('6006429').absorb(gbif.taxon('2439119'))
 
 # Rick Ree 2014-03-28 https://github.com/OpenTreeOfLife/reference-taxonomy/issues/37
 ott.same(ncbi.taxon('Calothrix', 'Rivulariaceae'), gbif.taxon('Calothrix', 'Rivulariaceae'))
@@ -377,14 +399,26 @@
 ott.taxon('Cyphellopsis','Cyphellaceae').unhide()
 ott.taxon('Cyphellopsis','Cyphellaceae').absorb(ott.taxon('Cyphellopsis','Niaceae'))
 ott.taxon('Diaporthaceae').take(ott.taxon('Phomopsis'))
-ott.taxon('Valsaceae').take(ott.taxon('Valsa'))
-ott.taxon('Agaricaceae').take(ott.taxon('Cystoderma'))
-ott.taxon('Hypocreaceae').take(ott.taxon('Hypocrea'))
+ott.taxon('Valsaceae').take(ott.taxon('Valsa', 'Fungi'))
+ott.taxon('Agaricaceae').take(ott.taxon('Cystoderma','Fungi'))
+ott.taxon('Hypocrea').absorb(ott.taxonThatContains('Trichoderma', 'Trichoderma raseum'))
+# Invert the synonym relationship
+ott.taxon('Trichoderma deliquescens').rename('Hypocrea lutea')
 
 # Fold Norops into Anolis
 # https://github.com/OpenTreeOfLife/reference-taxonomy/issues/31
 # TBD: Change species names from Norops X to Anolis X for all X
-ott.taxon('Anolis').take(ott.taxon('Norops'))
+ott.taxon('Anolis').absorb(ott.taxon('Norops', 'Iguanidae'))
+
+# JAR 2014-4-08 - these are in study OTUs - see IRMNG
+ott.taxon('Inseliellum').extant()
+ott.taxon('Conus', 'Gastropoda').extant()
+ott.taxon('Patelloida').extant()
+ott.taxon('Phyllanthus', 'Phyllanthaceae').extant()
+ott.taxon('Stelis','Orchidaceae').extant()
+ott.taxon('Chloris', 'Poaceae').extant()
+ott.taxon('Acropora', 'Acroporidae').extant()
+
 
 # -----------------------------------------------------------------------------
 # Finish up

org/opentreeoflife/smasher/Taxon.java

@@ -32,7 +32,7 @@ public class Taxon {
 	Taxon mapped = null;			// source node -> union node
 	Taxon comapped = null;		// union node -> source node
 	boolean novelp = false;		// added to union in last round?
-	private String division = null;
+	String division = null;
 
 	static boolean windyp = true;
 
@@ -251,11 +251,14 @@ void unifyWith(Taxon unode) {
 		// A few are combined using |
 		unode.properFlags |=
 			((before | this.properFlags) &
-			 (Taxonomy.FORCED_VISIBLE | Taxonomy.TATTERED |
-			  Taxonomy.EDITED | Taxonomy.EXTINCT));
-		// This one is anomalous
+			 (Taxonomy.FORCED_VISIBLE | 
+			  Taxonomy.EDITED |
+			  Taxonomy.EXTINCT));
+		// This one is anomalous.  Propagate these flags from original setting
+		// ignoring how they're set in the merged taxon.
 		unode.properFlags |=
-			(before & Taxonomy.MAJOR_RANK_CONFLICT);
+			(before & (Taxonomy.MAJOR_RANK_CONFLICT |
+					   Taxonomy.TATTERED));
 	}
 
 	void reallyUnifyWith(Taxon unode) {
@@ -405,6 +408,15 @@ Taxon augment(UnionTaxonomy union) {
 				for (Taxon augChild: newChildren)
 					newnode.addChild(augChild);
 
+				if (this.taxonomy.tag.equals("if")) {
+					boolean javasux = false;
+					for (Taxon p = this.parent; p != null; p = p.parent)
+						if (p.name.equals("Fungi")) { javasux = true; break; }
+					if (javasux)
+						for (Taxon o : oldChildren)
+							System.err.format("** %s losing %s to %s\n", this, o, o.parent);
+				}
+
 				newflags |= Taxonomy.TATTERED;
 				union.logAndMark(Answer.yes(this, null, "new/tattered", null));
 				// fall through
@@ -1121,6 +1133,7 @@ else if (newchild == this)
 			System.err.format("** A taxon cannot be its own parent: %s\n", newchild, this);
 		else {
 			newchild.properFlags |= Taxonomy.EDITED;
+			newchild.properFlags &= ~(Taxonomy.MAJOR_RANK_CONFLICT | Taxonomy.INCERTAE_SEDIS);
 			newchild.changeParent(this);
 		}
 	}

org/opentreeoflife/smasher/Taxonomy.java

@@ -281,6 +281,7 @@ public void dump(String outprefix, String sep) throws IOException {
 		this.dumpNodes(this.roots, outprefix, sep);
 		this.dumpSynonyms(outprefix + "synonyms.tsv", sep);
 		this.dumpMetadata(outprefix + "about.json");
+		this.dumpHidden(outprefix + "hidden.tsv");
 	}
 
 	public void dump(String outprefix) throws IOException {
@@ -721,6 +722,28 @@ void dumpSynonyms(String filename, String sep) throws IOException {
 		out.close();
 	}
 
+	void dumpHidden(String filename) throws IOException {
+		PrintStream out = Taxonomy.openw(filename);
+		for (Taxon node : this) {
+			if (((node.properFlags | node.inheritedFlags) &
+				 (Taxonomy.HIDDEN |
+				  Taxonomy.EXTINCT |
+				  Taxonomy.MAJOR_RANK_CONFLICT |
+				  Taxonomy.TATTERED |
+				  Taxonomy.NOT_OTU |
+				  Taxonomy.HYBRID |
+				  Taxonomy.VIRAL |
+				  Taxonomy.UNCLASSIFIED |
+				  Taxonomy.ENVIRONMENTAL |
+				  Taxonomy.INCERTAE_SEDIS)) != 0) {
+				out.format("%s\t%s\t%s\t", node.id, node.name, node.division);
+				this.printFlags(node, out);
+				out.println();
+			}
+		}
+		out.close();
+	}
+
 	/*
 	   flags are:
 
@@ -788,7 +811,7 @@ flags are inherited (conservative approach), except for "incertaesedis", which i
 	// NCBI only (not SILVA)
 	public void analyzeOTUs() {
 		for (Taxon root : this.roots)
-			analyzeOTUs(root, 0);	// mutates the tree
+			analyzeOTUs(root);	// mutates the tree
 	}
 
 	// GBIF and IF only
@@ -920,10 +943,7 @@ static int analyzeRankConflicts(Taxon node, boolean majorp) {
 	// taxa being hidden.
 	// We use this for NCBI but not for SILVA.
 
-	static void analyzeOTUs(Taxon node, int inheritedFlags) {
-		// Before
-		node.inheritedFlags |= inheritedFlags;
-
+	static void analyzeOTUs(Taxon node) {
 		// Prepare for recursive descent
 		if (notOtuRegex.matcher(node.name).find()) 
 			node.properFlags |= NOT_OTU;
@@ -932,12 +952,10 @@ static void analyzeOTUs(Taxon node, int inheritedFlags) {
 		if (viralRegex.matcher(node.name).find()) 
 			node.properFlags |= VIRAL;
 
-		int bequest = inheritedFlags | node.properFlags;		// What the children inherit
-
 		// Recursive descent
 		if (node.children != null)
 			for (Taxon child : node.children)
-				analyzeOTUs(child, bequest);
+				analyzeOTUs(child);
 	}
 
 	// Flags to set for all taxonomies.	 Also elide container pseudo-taxa
@@ -1046,6 +1064,10 @@ static void printFlags(Taxon node, PrintStream out) {
 			if (needComma) out.print(","); else needComma = true;
 			out.print("hidden");
 		}
+		else if ((node.inheritedFlags & HIDDEN) != 0) {
+			if (needComma) out.print(","); else needComma = true;
+			out.print("hidden_inherited");
+		}
 
 		if ((node.properFlags & MAJOR_RANK_CONFLICT) != 0) {
 			if (needComma) out.print(","); else needComma = true;
@@ -1844,24 +1866,25 @@ public void sameness(Taxon node1, Taxon node2, boolean polarity) {
 							  node1, node2);
 			return;
 		}
-		if (snode.mapped == unode) return;	  // Already equated
 		if (!(snode.taxonomy instanceof SourceTaxonomy)) {
 			System.err.format("** One of the two nodes must come from a source taxonomy: %s %s\n", unode, snode);
 			return;
 		}
-		if (polarity) {
+		if (polarity) {			// same
 			if (snode.mapped != null) {
 				if (snode.mapped != unode)
 					System.err.format("** The taxa have already been determined to be different: %s\n", snode);
 				return;
 			}
 			snode.unifyWith(unode);
-		} else {
+		} else {				// notSame
 			if (snode.mapped != null) {
 				if (snode.mapped == unode)
 					System.err.format("** The taxa have already been determined to be the same: %s\n", snode);
 				return;
 			}
+			// Give the source node a place to go in the union that is
+			// different from the union node it's different from
 			Taxon evader = new Taxon(unode.taxonomy);
 			snode.unifyWithNew(evader);
 			evader.addSource(snode);
@@ -2200,6 +2223,17 @@ public void assignIds(SourceTaxonomy idsource) {
 		// idsource.tag = "ids";
 		idsource.mapInto(this, Criterion.idCriteria);
 
+		this.transferIds(idsource);
+
+		// Phase 2: give new ids to union nodes that didn't get them above.
+		long sourcemax = idsource.maxid();
+		this.assignNewIds(sourcemax);
+		// remember, this = union, idsource = previous version of ott
+
+		Taxon.printStats();		// Taxon id clash
+	}
+
+	public void transferIds(SourceTaxonomy idsource) {
 		Taxon.resetStats();
 		System.out.println("--- Assigning ids to union starting with " + idsource.getTag() + " ---");
 
@@ -2217,13 +2251,6 @@ public void assignIds(SourceTaxonomy idsource) {
 				unode.setId(node.id);
 			}
 		}
-
-		// Phase 2: give new ids to union nodes that didn't get them above.
-		long sourcemax = idsource.maxid();
-		this.assignNewIds(sourcemax);
-		// remember, this = union, idsource = previous version of ott
-
-		Taxon.printStats();		// Taxon id clash
 	}
 
 	// Cf. assignIds()
@@ -2347,6 +2374,7 @@ public void dump(String outprefix, String sep) throws IOException {
 
 		this.dumpNodes(this.roots, outprefix, sep);
 		this.dumpSynonyms(outprefix + "synonyms.tsv", sep);
+		this.dumpHidden(outprefix + "hidden.tsv");
 	}
 
 	// Overrides method in Taxonomy class