Skip to content

Commit 81be6ec

Browse files
danv61danv61
authored
Merge pull request #1496 from virtualcell/dan-bmdb-unsupported
Update BMDB model list - supported / failed
2 parents afd645b + 7bf43ea commit 81be6ec

6 files changed

Lines changed: 2327 additions & 28 deletions

File tree

‎vcell-client/src/main/resources/bioModelsNetInfo.xml‎

Lines changed: 22 additions & 22 deletions
Large diffs are not rendered by default.

‎vcell-core/src/test/java/org/vcell/sbml/BMDB_SBMLImportTest.java‎

Lines changed: 252 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,24 +3,38 @@
33
import cbit.util.xml.VCLogger;
44
import cbit.util.xml.VCLoggerException;
55
import cbit.vcell.biomodel.BioModel;
6+
import cbit.vcell.client.server.DynamicClientProperties;
7+
import cbit.vcell.resource.PropertyLoader;
8+
import cbit.vcell.resource.ResourceUtil;
9+
import cbit.vcell.xml.ExternalDocInfo;
10+
import cbit.vcell.xml.XMLSource;
11+
import cbit.vcell.xml.XMLTags;
12+
import cbit.vcell.xml.XmlHelper;
613
import org.jdom2.Document;
714
import org.jdom2.Element;
15+
import org.jdom2.Namespace;
16+
import org.jdom2.input.SAXBuilder;
817
import org.jdom2.output.Format;
918
import org.jdom2.output.XMLOutputter;
10-
import org.junit.jupiter.api.AfterAll;
11-
import org.junit.jupiter.api.Assertions;
12-
import org.junit.jupiter.api.BeforeAll;
13-
import org.junit.jupiter.api.Tag;
19+
import org.junit.jupiter.api.*;
1420
import org.junit.jupiter.params.ParameterizedTest;
1521
import org.junit.jupiter.params.provider.MethodSource;
1622
import org.sbml.jsbml.SBMLDocument;
1723
import org.sbml.jsbml.SBMLReader;
1824
import org.vcell.sbml.vcell.SBMLImporter;
25+
import org.vcell.util.UnzipUtility;
1926

2027
import java.io.*;
28+
import java.net.URL;
29+
import java.nio.charset.StandardCharsets;
2130
import java.nio.file.Files;
31+
import java.nio.file.Path;
32+
import java.nio.file.Paths;
33+
import java.nio.file.StandardOpenOption;
2234
import java.util.*;
2335
import java.util.stream.Collectors;
36+
import java.util.HashMap;
37+
import java.util.Map;
2438

2539
import static org.junit.jupiter.api.Assertions.fail;
2640

@@ -112,6 +126,8 @@ public static Map<Integer, SBMLTestSuiteTest.FAULT> knownFaults() {
112126
faults.put(340, SBMLTestSuiteTest.FAULT.MATHML_PARSING); // cause: UnsupportedConstruct: error parsing expression ' <math><apply><gt/><piecewise><piece><apply><minus/><csymbol encoding="text" definitionURL="http://www.sbml.org/sbml/symbols/time">
113127
faults.put(342, SBMLTestSuiteTest.FAULT.EXPRESSION_BINDING_EXCEPTION); // cause: Error binding global parameter 'TGF_beta_dose_mol_per_cell' to model: 'UNRESOLVED.initConc' is either not found in your model or is not allowed to be used in the current context.
114128
faults.put(353, SBMLTestSuiteTest.FAULT.NONINTEGER_STOICH); // cause: UnsupportedConstruct: Non-integer stoichiometry ('12345.7' for reactant 'cpd_C00369Glc_CS' in reaction 'rn_R02112CS_G2') or stoichiometryMath not handled in VCell at this time.
129+
faults.put(354, SBMLTestSuiteTest.FAULT.COMPARTMENT_CONSTANT_FALSE); // cause: Error adding Feature to vcModel CompartmentError: compartment 'cytosol' has constant attribute set to False, not currently supported.
130+
faults.put(355, SBMLTestSuiteTest.FAULT.COMPARTMENT_CONSTANT_FALSE); // cause: Error adding Feature to vcModel CompartmentError: compartment 'cytosol' has constant attribute set to False, not currently supported.
115131
faults.put(383, SBMLTestSuiteTest.FAULT.NONINTEGER_STOICH); // cause: UnsupportedConstruct: Non-integer stoichiometry ('1.5' for product 'PGA' in reaction 'PGA_prod_Vo') or stoichiometryMath not handled in VCell at this time.
116132
faults.put(384, SBMLTestSuiteTest.FAULT.NONINTEGER_STOICH); // cause: UnsupportedConstruct: Non-integer stoichiometry ('1.5' for product 'PGA' in reaction 'PGA_prod_Vo') or stoichiometryMath not handled in VCell at this time.
117133
faults.put(385, SBMLTestSuiteTest.FAULT.NONINTEGER_STOICH); // cause: UnsupportedConstruct: Non-integer stoichiometry ('1.5' for product 'PGA' in reaction 'PGA_prod_Vo') or stoichiometryMath not handled in VCell at this time.
@@ -258,6 +274,8 @@ public void testSbmlTestSuiteImport(Integer biomodelsDbModelNumber) throws Excep
258274
fault = SBMLTestSuiteTest.FAULT.MATHML_PARSING;
259275
}else if (cause.contains("class org.sbml.jsbml.Constraint cannot be cast to class org.sbml.jsbml.SBMLDocument")) {
260276
fault = SBMLTestSuiteTest.FAULT.CONSTRAINT_CLASS_CAST_EXCEPTION;
277+
}else if (cause.contains("CompartmentError: compartment") && cause.contains("constant attribute set to False")) {
278+
fault = SBMLTestSuiteTest.FAULT.COMPARTMENT_CONSTANT_FALSE;
261279
}
262280

263281
try (BufferedWriter codeProblemFileWriter = new BufferedWriter(new FileWriter(codeKnownProblemFile, true));
@@ -279,7 +297,236 @@ public void testSbmlTestSuiteImport(Integer biomodelsDbModelNumber) throws Excep
279297
}
280298
}
281299

282-
public static void main(String[] args) {
300+
/*
301+
* goes through the list of BMDB models (bioModelsNetInfo.xml)
302+
* downloads from the website (www.ebi.ac.uk) all those marked as supported
303+
* tries to import them in vcell
304+
* creates a list with those that fail
305+
*
306+
* for simplicity, place the bioModelsNetInfo.xml file in a convenient location and edit bioModelsNetInfoFile accordingly
307+
* this is not really a test and takes quite a long time to run, keep it commented out unless needed
308+
*/
309+
// @Test
310+
// public void importAllBmdbTest() {
311+
// try {
312+
// File bioModelsNetInfoFile = new File("C:/dan/projects/git/vcell/vcell-client/src/main/resources/bioModelsNetInfo.xml");
313+
// Map<String, BioModelData> modelInfoMap = parseXmlFile(bioModelsNetInfoFile);
314+
//// modelInfoMap.forEach((id, data) -> System.out.println("ID: " + id + " -> " + data));
315+
//
316+
// Set<Integer> failuresSet = new LinkedHashSet<> ();
317+
// for (Map.Entry<String, BioModelData> entry : modelInfoMap.entrySet()) {
318+
//
319+
// String id = entry.getKey();
320+
// String name = entry.getValue().name;
321+
// boolean supported = entry.getValue().supported;
322+
// if(supported == false) {
323+
//// System.out.println("Skipping: " + id + ", unsupported");
324+
// continue;
325+
// }
326+
// if(slowModelSet().contains(idToIndex(id))) {
327+
//// System.out.println("Skipping: " + id + ", slow");
328+
// continue; // skip the slow ones
329+
// }
330+
//
331+
// System.out.println("Reading: " + id);
332+
// ExternalDocInfo externalDocInfo = download(name, id);
333+
//
334+
// XMLSource xmlSource = externalDocInfo.createXMLSource();
335+
// org.jdom2.Element rootElement = xmlSource.getXmlDoc().getRootElement();
336+
// String xmlType = rootElement.getName();
337+
//
338+
// if (!xmlType.equals(XMLTags.SbmlRootNodeTag)) {
339+
// throw new RuntimeException("Expected SBML document");
340+
// }
341+
// try {
342+
// VCLogger transLogger = null;
343+
// Namespace namespace = rootElement.getNamespace(XMLTags.SBML_SPATIAL_NS_PREFIX);
344+
// boolean isBMDB = externalDocInfo.isBioModelsNet();
345+
// boolean bIsSpatial = (namespace == null) ? false : true;
346+
// BioModel doc = XmlHelper.importSBML(transLogger, xmlSource, bIsSpatial);
347+
//
348+
// } catch(Exception e) {
349+
// failuresSet.add(idToIndex(id));
350+
// System.err.println(" ...found new failure");
351+
// }
352+
// System.out.println(" ...success");
353+
// }
354+
// String failures = convertSetToString(failuresSet);
355+
// System.out.println(failures);
356+
// } catch(Exception e) {
357+
// System.out.println(e.getMessage());
358+
// }
359+
// }
360+
/*
361+
* same as above, goes to all the models marked as fails and tries to import them, makes a list with those that actually
362+
* can be imported
363+
*
364+
* WARNING: our SBML parser is "eating" some errors, if exceptions are being fired the imported model is probably wrong
365+
* even though it gets imported
366+
*/
367+
// @Test
368+
// public void checkExistingFailuresTest() {
369+
// try {
370+
// File bioModelsNetInfoFile = new File("C:/dan/projects/git/vcell/vcell-client/src/main/resources/bioModelsNetInfo.xml");
371+
// Map<String, BioModelData> modelInfoMap = parseXmlFile(bioModelsNetInfoFile);
372+
//
373+
// Set<Integer> failuresNowWorkingSet = new LinkedHashSet<> (); // models marked as failures, but now work
374+
// for (Map.Entry<String, BioModelData> entry : modelInfoMap.entrySet()) {
375+
//
376+
// String id = entry.getKey();
377+
// String name = entry.getValue().name;
378+
// boolean supported = entry.getValue().supported;
379+
// if(supported == true) {
380+
// System.out.println("Skipping: " + id + ", marked as supported");
381+
// continue;
382+
// }
383+
// if(slowModelSet().contains(idToIndex(id))) {
384+
// System.out.println("Skipping: " + id + ", slow");
385+
// continue; // skip the slow ones
386+
// }
387+
//
388+
// System.out.println("Reading: " + id);
389+
// ExternalDocInfo externalDocInfo = download(name, id);
390+
//
391+
// XMLSource xmlSource = externalDocInfo.createXMLSource();
392+
// org.jdom2.Element rootElement = xmlSource.getXmlDoc().getRootElement();
393+
// String xmlType = rootElement.getName();
394+
//
395+
// if (!xmlType.equals(XMLTags.SbmlRootNodeTag)) {
396+
// throw new RuntimeException("Expected SBML document");
397+
// }
398+
// try {
399+
// VCLogger transLogger = null;
400+
// Namespace namespace = rootElement.getNamespace(XMLTags.SBML_SPATIAL_NS_PREFIX);
401+
// boolean isBMDB = externalDocInfo.isBioModelsNet();
402+
// boolean bIsSpatial = (namespace == null) ? false : true;
403+
// BioModel doc = XmlHelper.importSBML(transLogger, xmlSource, bIsSpatial);
404+
// System.out.println(" ...success");
405+
// failuresNowWorkingSet.add(idToIndex(id));
406+
// } catch(Exception e) {
407+
// System.err.println(" ...still fails");
408+
// }
409+
// }
410+
// String failuresNowWorking = convertSetToString(failuresNowWorkingSet);
411+
// System.out.println(failuresNowWorking);
412+
// } catch(Exception e) {
413+
// System.out.println(e.getMessage());
414+
// }
415+
// }
416+
417+
private static final String IdPrefix = "BIOMD";
418+
public static String indexToId(int index) { // ex: 47 -> BIOMD0000000047
419+
String paddedSuffix = String.format("%010d", index);
420+
return IdPrefix + paddedSuffix;
421+
}
422+
public static int idToIndex(String id) { // ex: BIOMD0000000123 -> 123
423+
String suffixStr = id.substring(IdPrefix.length());
424+
return Integer.parseInt(suffixStr);
425+
}
426+
public static String convertSetToString(Set<Integer> numbers) {
427+
String stringOfNumbers = numbers.stream()
428+
.map(String::valueOf) // convert each integer to a string
429+
.collect(Collectors.joining(",")); // join with commas
430+
return stringOfNumbers;
431+
}
432+
433+
private ExternalDocInfo download(String name, String id) throws Exception {
434+
String simDataDir = ResourceUtil.getLocalRootDir().getAbsolutePath(); // C:\Users\myname\.vcell\simdata
435+
String tempDir = simDataDir + File.separator + "temp";
436+
String destDirectory = tempDir + File.separator + id;
437+
String zipFilePath = destDirectory + ".zip";
438+
439+
Path tempDirPath = Paths.get(tempDir);
440+
Files.createDirectories(tempDirPath); // temp may not be there, we make it
441+
442+
byte[] responseContent = null;
443+
URL url = new URL("https://www.ebi.ac.uk/biomodels/search/download?models=" + id);
444+
ByteArrayOutputStream baos = new ByteArrayOutputStream();
445+
InputStream is = null;
446+
try {
447+
try {
448+
is = url.openStream ();
449+
}catch (Exception e) {
450+
e.printStackTrace();
451+
if (is != null) { is.close(); }
452+
//Try with http instead of https
453+
String newUrlString = url.toString().replaceFirst("^https", "http");
454+
url = new URL(newUrlString);
455+
is = url.openStream ();
456+
}
457+
byte[] byteChunk = new byte[4096]; // Or whatever size you want to read in at a time.
458+
int n;
459+
460+
while ( (n = is.read(byteChunk)) > 0 ) {
461+
baos.write(byteChunk, 0, n);
462+
}
463+
responseContent = baos.toByteArray();
464+
}
465+
catch (IOException e) {
466+
System.err.printf ("Failed while reading bytes from %s: %s", url.toExternalForm(), e.getMessage());
467+
// e.printStackTrace ();
468+
throw new RuntimeException("Failed while reading bytes from: " + url.toExternalForm());
469+
}
470+
finally {
471+
if (is != null) { is.close(); }
472+
}
473+
if(responseContent == null) {
474+
throw new RuntimeException("Failed while reading bytes from: " + url.toExternalForm());
475+
}
476+
477+
try {
478+
File file = new File(zipFilePath);
479+
Files.write(file.toPath(), responseContent, StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING);
480+
UnzipUtility uu = new UnzipUtility();
481+
uu.unzip(zipFilePath, destDirectory);
482+
} catch(IOException e) {
483+
e.printStackTrace();
484+
}
485+
String unzippedPath = destDirectory + File.separator + id + ".xml";
486+
String bioModelSBML = new String(Files.readAllBytes(Paths.get(unzippedPath)), StandardCharsets.UTF_8);
487+
try {
488+
Files.deleteIfExists(Paths.get(zipFilePath)); // the original zip file
489+
Files.deleteIfExists(Paths.get(unzippedPath)); // the unzipped SBML file
490+
Files.deleteIfExists(Paths.get(destDirectory)); // its directory
491+
} catch(IOException e) {
492+
e.printStackTrace();
493+
}
494+
ExternalDocInfo externalDocInfo = ExternalDocInfo.createBioModelsNetExternalDocInfo(bioModelSBML, name);
495+
return externalDocInfo;
496+
}
497+
public Map<String, BioModelData> parseXmlFile(File xmlFile) throws Exception {
498+
Map<String, BioModelData> bioModelMap = new LinkedHashMap<>();
499+
500+
SAXBuilder builder = new SAXBuilder();
501+
Document document = builder.build(xmlFile);
502+
Element rootElement = document.getRootElement();
503+
504+
List<Element> bioModelInfos = rootElement.getChildren("BioModelInfo");
505+
for (Element bioModel : bioModelInfos) {
506+
String id = bioModel.getAttributeValue("ID");
507+
boolean supported = Boolean.parseBoolean(bioModel.getAttributeValue("Supported"));
508+
String name = bioModel.getAttributeValue("Name");
509+
bioModelMap.put(id, new BioModelData(supported, name));
510+
System.out.println("ID: " + id + ", Supported: " + supported + ", Name: " + name);
511+
}
512+
return bioModelMap;
513+
}
514+
515+
public class BioModelData implements Serializable {
516+
public final boolean supported;
517+
public final String name;
518+
519+
public BioModelData(boolean supported, String name) {
520+
this.supported = supported;
521+
this.name = name;
522+
}
523+
@Override
524+
public String toString() {
525+
return "BioModelData{name='" + name + "', supported=" + supported + "}";
526+
}
527+
}
528+
529+
public static void main(String[] args) {
283530
try {
284531
if (args.length != 1){
285532
System.out.println("usage: BMDB_SBMLImportTest xmlManifestFile");

‎vcell-core/src/test/java/org/vcell/sbml/BMDB_SBML_Files.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ public class BMDB_SBML_Files {
2222
public static int[] getBiomodelDB_curatedModelNumbers() {
2323
// only those Biomodels Database models which are committed to the vcell repo
2424
// Uncomment "includeFilter = n -> true;" to run all tests (must have the other models installed in proper directory).
25-
IntPredicate includeFilter = n -> Arrays.asList(15,48,71,205,264,427,452,453,591,594,595,596,623,643,696,826,952,
25+
IntPredicate includeFilter = n -> Arrays.asList(15,48,71,205,264,354,355,427,452,453,591,594,595,596,623,643,696,826,952,
2626
956, 302, 175, 618, 569, 973, 520, 731, 1064
2727
).contains(n);
2828
//IntPredicate includeFilter = n -> true;

‎vcell-core/src/test/java/org/vcell/sbml/SBMLTestSuiteTest.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -595,9 +595,11 @@ public enum FAULT {
595595
VALUE_NAN_INF_OR_MISSING,
596596
OVERDETERMINED_SYSTEM,
597597
NONINTEGER_STOICH,
598+
NONNUMERIC_STOICH,
598599
COMPARTMENT_0D,
599600
COMPARTMENT_1D,
600601
COMPARTMENT_DIM_MISSING,
602+
COMPARTMENT_CONSTANT_FALSE, // compartment size not constant - unsupported by vcell
601603
ALGEBRAIC_RULES,
602604
INCONSISTENT_UNIT_SYSTEM,
603605
EXPRESSION_BINDING_EXCEPTION,

0 commit comments

Comments
 (0)