Skip to content

Commit 9b9b383

Browse files
danv61danv61
authored
Merge pull request #1628 from virtualcell/dan-ss-results4
Dan ss results4
2 parents 91f3932 + df60911 commit 9b9b383

18 files changed

Lines changed: 1980 additions & 1 deletion

‎vcell-core/src/main/java/cbit/vcell/simdata/DataSetControllerImpl.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2411,6 +2411,8 @@ public LangevinBatchResultSet getLangevinBatchResultSet(VCDataIdentifier vcdID)
24112411

24122412
File avg_file = simData.getLangevinFile(LangevinBatchResultSet.LangevinFileType.Avg);
24132413
if (avg_file != null && avg_file.exists()) {
2414+
// want to keep all results, so pass 0 for keepAtMost
2415+
// don't care about functions, so pass null for functionsFile
24142416
ODESimData odeSimData_avg = ODESimData.readIDADataFile(vcdID, avg_file, keepAtMost, functionsFile);
24152417
lbrs.setOdeSimDataAvg(odeSimData_avg);
24162418
}
@@ -2429,6 +2431,23 @@ public LangevinBatchResultSet getLangevinBatchResultSet(VCDataIdentifier vcdID)
24292431
ODESimData odeSimData_std = ODESimData.readIDADataFile(vcdID, std_file, keepAtMost, functionsFile);
24302432
lbrs.setOdeSimDataStd(odeSimData_std);
24312433
}
2434+
2435+
// TODO: call ODESimData readCSVDataFile(VCDataIdentifier vcdId, File csvFile) to read cluster analysis results
2436+
// File counts = simData.getLangevinFile(LangevinFileType.ClusterCounts);
2437+
// if (counts != null && counts.exists()) {
2438+
// lbrs.setClusterCounts(readCSVDataFile(vcdID, counts));
2439+
// }
2440+
//
2441+
// File mean = simData.getLangevinFile(LangevinFileType.ClusterMean);
2442+
// if (mean != null && mean.exists()) {
2443+
// lbrs.setClusterMeans(readCSVDataFile(vcdID, mean));
2444+
// }
2445+
//
2446+
// File overall = simData.getLangevinFile(LangevinFileType.ClusterOverall);
2447+
// if (overall != null && overall.exists()) {
2448+
// lbrs.setClusterOverall(readCSVDataFile(vcdID, overall));
2449+
// }
2450+
24322451
return lbrs;
24332452
} catch (IOException e) {
24342453
throw new DataAccessException(e.getMessage(), e);

‎vcell-core/src/main/java/cbit/vcell/solver/ode/ODESimData.java‎

Lines changed: 66 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,10 @@
2525
import java.io.IOException;
2626
import java.io.InputStreamReader;
2727
import java.io.Serializable;
28+
import java.util.ArrayList;
2829
import java.util.StringTokenizer;
2930
import java.util.Vector;
31+
import java.util.List;
3032

3133
import org.apache.logging.log4j.LogManager;
3234
import org.apache.logging.log4j.Logger;
@@ -406,6 +408,60 @@ public void writeOut(DataOutputStream output) throws IOException {
406408
}
407409
}
408410

411+
// TODO: work in progress, untested
412+
public static ODESimData readCSVDataFile(VCDataIdentifier vcdId, File csvFile)
413+
throws DataAccessException {
414+
415+
lg.trace("reading csv file : " + csvFile);
416+
417+
ODESimData odeSimData = new ODESimData();
418+
odeSimData.formatID = ODESimData.IDA_DATA_FORMAT_ID;
419+
odeSimData.mathName = vcdId.getID();
420+
421+
BufferedReader bufferedReader = null;
422+
try {
423+
bufferedReader = new BufferedReader(new InputStreamReader(new FileInputStream(csvFile)));
424+
425+
String line = bufferedReader.readLine(); // read header line
426+
if (line == null) {
427+
return null;
428+
}
429+
430+
String[] headerTokens = line.split(","); // header is comma-separated
431+
for (String token : headerTokens) {
432+
odeSimData.addDataColumn(new ODESolverResultSetColumnDescription(token.trim()));
433+
}
434+
435+
while ((line = bufferedReader.readLine()) != null) { // read data rows
436+
if (line.trim().isEmpty()) {
437+
continue;
438+
}
439+
String[] tokens = line.split(",");
440+
double[] values = new double[odeSimData.getDataColumnCount()];
441+
if (tokens.length != values.length) {
442+
break; // malformed row → stop reading
443+
}
444+
for (int i = 0; i < tokens.length; i++) {
445+
values[i] = Double.parseDouble(tokens[i]);
446+
}
447+
odeSimData.addRow(values);
448+
}
449+
450+
} catch (Exception e) {
451+
lg.error(e.getMessage(), e);
452+
throw new DataAccessException(e.getMessage(), e);
453+
} finally {
454+
try {
455+
if (bufferedReader != null) {
456+
bufferedReader.close();
457+
}
458+
} catch (Exception ex) {
459+
lg.error(ex.getMessage(), ex);
460+
}
461+
}
462+
return odeSimData;
463+
}
464+
409465

410466
public static ODESimData readIDADataFile(VCDataIdentifier vcdId, File dataFile, int keepMost, File functionsFile) throws DataAccessException {
411467
// read ida file
@@ -456,7 +512,16 @@ public static ODESimData readIDADataFile(VCDataIdentifier vcdId, File dataFile,
456512
}
457513

458514
// read functions file
459-
515+
if(functionsFile == null) {
516+
// functionFile is being initialized in SimulationData.getJobFunctionsFile() which already
517+
// throws is the file doesn't exist, so we call explicitly with null here to skip functions loading
518+
// for langevin batch simulation results (where functionFile is meaningless)
519+
if (keepMost > 0) {
520+
odeSimData.trimRows(keepMost);
521+
}
522+
return odeSimData;
523+
}
524+
460525
if (!odeSimData.getColumnDescriptions(0).getName().equals(SimDataConstants.HISTOGRAM_INDEX_NAME)) {
461526
Vector<AnnotatedFunction> funcList;
462527
try {

‎vcell-core/src/test/java/cbit/vcell/simdata/LangevinResultsReadTest.java‎

Lines changed: 255 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,255 @@
1+
package cbit.vcell.simdata;
2+
3+
import cbit.vcell.export.server.ExportServiceImpl;
4+
import cbit.vcell.math.ODESolverResultSetColumnDescription;
5+
import cbit.vcell.resource.ResourceUtil;
6+
import cbit.vcell.server.DataSetControllerProvider;
7+
import cbit.vcell.solver.VCSimulationDataIdentifier;
8+
import cbit.vcell.solver.VCSimulationIdentifier;
9+
import cbit.vcell.solver.ode.ODESimData;
10+
import cbit.vcell.solver.ode.ODESolverResultSet;
11+
12+
13+
import com.google.common.io.Resources;
14+
import org.junit.jupiter.api.*;
15+
import org.vcell.util.CacheException;
16+
import org.vcell.util.DataAccessException;
17+
import org.vcell.util.document.KeyValue;
18+
import org.vcell.util.document.LocalVCDataIdentifier;
19+
import org.vcell.util.document.User;
20+
import org.vcell.util.document.VCDataIdentifier;
21+
22+
import java.io.*;
23+
import java.nio.file.Files;
24+
import java.nio.file.Path;
25+
import java.util.LinkedHashMap;
26+
import java.util.Map;
27+
28+
import static org.junit.jupiter.api.Assertions.*;
29+
30+
@Tag("Fast")
31+
public class LangevinResultsReadTest {
32+
33+
private static File primaryDir; // temporary working dir for this test
34+
private static File localSimDir;
35+
36+
// the files are in vcell-core/src/test/resources /cbit/vcell/simdata/langevin/batch
37+
private static File ida_avg_file;
38+
private static File ida_min_file;
39+
private static File ida_max_file;
40+
private static File ida_std_file;
41+
private static File clusters_counts_file;
42+
private static File clusters_mean_file;
43+
private static File clusters_overall_file;
44+
private static File functions_file;
45+
private static File simtask_xml_file;
46+
private static File langevin_input_file;
47+
private static File messaging_config_file;
48+
49+
// results for first task in the batch, named to match single run file (ending in "_") ex: SimID_303404574_0_
50+
private static File ida_0_file;
51+
// log produced by the solver for first task in the batch, named using the normal convention ex: SimID_303404574_0_0.log
52+
private static File log_0_file;
53+
// general default ida data log file, contains:
54+
// IDAData logfile
55+
// IDAData text format version 1
56+
// SimID_303404574_0_.ida
57+
private static File log_data_file;
58+
59+
60+
@BeforeAll
61+
public static void setUp() throws IOException {
62+
63+
Path tempDir = Files.createTempDirectory("langevin_");
64+
primaryDir = tempDir.toFile();
65+
// If you want it to delete on JVM exit as a fallback:
66+
primaryDir.deleteOnExit();
67+
68+
// Create subdirectory "temp" inside primaryDir
69+
localSimDir = new File(primaryDir, "temp");
70+
if (!localSimDir.mkdirs()) {
71+
throw new IOException("Failed to create subdirectory: " + localSimDir.getAbsolutePath());
72+
}
73+
74+
functions_file = copyToPrimaryDir("SimID_303404574_0_.functions");
75+
ida_0_file = copyToPrimaryDir("SimID_303404574_0_.ida"); // results for batch run0 (or for single run)
76+
log_0_file = copyToPrimaryDir("SimID_303404574_0_0.log"); // langevin-made log for run 0
77+
log_data_file = copyToPrimaryDir("SimID_303404574_0_.log"); // general log made by preprocessor
78+
langevin_input_file = copyToPrimaryDir("SimID_303404574_0_.langevinInput");
79+
messaging_config_file = copyToPrimaryDir("SimID_303404574_0_.langevinMessagingConfig");
80+
simtask_xml_file = copyToPrimaryDir("SimID_303404574_0__0.simtask.xml");
81+
82+
ida_avg_file = copyToPrimaryDir("SimID_303404574_0__Avg.ida");
83+
clusters_counts_file = copyToPrimaryDir("SimID_303404574_0__clusters_counts.csv");
84+
clusters_mean_file = copyToPrimaryDir("SimID_303404574_0__clusters_mean.csv");
85+
clusters_overall_file = copyToPrimaryDir("SimID_303404574_0__clusters_overall.csv");
86+
ida_max_file = copyToPrimaryDir("SimID_303404574_0__Max.ida");
87+
ida_min_file = copyToPrimaryDir("SimID_303404574_0__Min.ida");
88+
ida_std_file = copyToPrimaryDir("SimID_303404574_0__Std.ida");
89+
90+
91+
92+
93+
// ida_0_File = File.createTempFile("SimID_284673710_0_", ".ida");
94+
// Resources.asByteSource(Resources.getResource("cbit/vcell/simdata/SimID_284673710_0_.ida"))
95+
// .copyTo(com.google.common.io.Files.asByteSink(ida_0_File));
96+
97+
}
98+
99+
@AfterAll
100+
public static void tearDown() {
101+
102+
deleteRecursively(primaryDir);
103+
// ida_0_File.delete();
104+
// ida_1_File.delete();
105+
// ida_2_File.delete();
106+
// if (inputStream != null) {
107+
// inputStream.close();
108+
}
109+
110+
111+
@Test
112+
public void testReadData() throws IOException, DataAccessException, CacheException {
113+
114+
String simID = "303404574";
115+
KeyValue key = new KeyValue(simID);
116+
User usr = new User("temp",key);
117+
VCSimulationIdentifier vcSimID = new VCSimulationIdentifier(key, usr);
118+
119+
VCDataIdentifier vCDataIdentifier = new VCSimulationDataIdentifier(vcSimID, 0);
120+
VCData vcData = new SimulationData(vCDataIdentifier, primaryDir, localSimDir, null);
121+
122+
Cachetable aCacheTable = new Cachetable(10 * Cachetable.minute, 100000);
123+
aCacheTable.put(vCDataIdentifier, vcData);
124+
125+
DataSetControllerImpl dataSetControllerImpl = new DataSetControllerImpl(aCacheTable, primaryDir, localSimDir);
126+
127+
LangevinBatchResultSet lbrs = dataSetControllerImpl.getLangevinBatchResultSet(vCDataIdentifier);
128+
129+
assertNotNull(lbrs, "Result set should not be null");
130+
131+
ODEDataInfo odeDataInfo = lbrs.getOdeDataInfo();
132+
assertNotNull(odeDataInfo, "ODEDataInfo should not be null");
133+
assertTrue(odeDataInfo.getSimID().contains(simID), "ODEDataInfo SimID should contain the simulation ID");
134+
135+
// --- Avg data loaded ---
136+
ODESimData odeSimDataAvg = lbrs.getOdeSimDataAvg();
137+
assertNotNull(odeSimDataAvg, "Avg ODE data should be loaded");
138+
assertTrue(odeSimDataAvg.getMathName().contains(simID), "Avg ODE data mathName should contain the simulation ID");
139+
140+
// --- Minimal structural checks ---
141+
assertTrue(odeSimDataAvg.getDataColumnCount() > 0, "Avg ODE data should have at least one column");
142+
assertTrue(odeSimDataAvg.getRowCount() > 0, "Avg ODE data should have at least one row");
143+
144+
}
145+
146+
// @Test
147+
// @Disabled("Deprecated: postprocessing moved to solver")
148+
// public void testRead() throws IOException {
149+
//
150+
// // read the input data (3 .IDA files)
151+
// ODESolverResultSet osrs_0 = getOdeSolverResultSet(ida_0_File);
152+
// ODESolverResultSet osrs_1 = getOdeSolverResultSet(ida_1_File);
153+
// ODESolverResultSet osrs_2 = getOdeSolverResultSet(ida_2_File);
154+
//
155+
// Map<Integer, ODESolverResultSet> odeSolverResultSetMap = new LinkedHashMap<>();
156+
// odeSolverResultSetMap.put(0, osrs_0);
157+
// odeSolverResultSetMap.put(1, osrs_1);
158+
// odeSolverResultSetMap.put(2, osrs_2);
159+
//
160+
// LangevinPostProcessorInput lppInput = new LangevinPostProcessorInput(null, null);
161+
// lppInput.setFailed(false);
162+
// lppInput.setOdeSolverResultSetMap(odeSolverResultSetMap);
163+
//
164+
// // compute primary statistics
165+
// // note: LangevinPostProcessor is deprecated, but this test is kept for it contains some reusable code
166+
// LangevinPostProcessor lpp = new LangevinPostProcessor();
167+
// LangevinPostProcessorOutput lppOutput = lpp.postProcessLangevinResults(lppInput);
168+
//
169+
// assertFalse(lppOutput.isFailed(), "expected to not fail");
170+
// assertTrue(lppOutput.isMultiTrial(), "expected to be multi-trial");
171+
//
172+
// // get some timepoint for some variable
173+
// String name = osrs_0.getColumnDescriptions()[7].getName();
174+
// double anAverage = lppOutput.getAveragesResultSet().getRow(10)[7]; // TOTAL_MT0__Site1__state0
175+
// double aStd = lppOutput.getStdResultSet().getRow(10)[7];
176+
// double aMin = lppOutput.getMinResultSet().getRow(10)[7];
177+
// double aMax = lppOutput.getMaxResultSet().getRow(10)[7];
178+
//
179+
// // compare to what's expected
180+
// assertTrue("TOTAL_MT0__Site1__state0".contentEquals(name), "expecting column name 'TOTAL_MT0__Site1__state0', found: '" + name + "'");
181+
// assertTrue(anAverage == 21.0 ? true : false, "expecting 21.0, found " + anAverage);
182+
// assertTrue(aStd == 0.816496580927726 ? true : false, "expecting 0.816496580927726, found " + aStd);
183+
// assertTrue(aMin == 20.0 ? true : false, "expecting 20.0, found " + aMin);
184+
// assertTrue(aMax == 22.0 ? true : false, "expecting 22.0, found " + aMax);
185+
// }
186+
187+
188+
private static void deleteRecursively(File file) {
189+
if (file == null || !file.exists()) {
190+
return;
191+
}
192+
File[] children = file.listFiles();
193+
if (children != null) {
194+
for (File child : children) {
195+
deleteRecursively(child);
196+
}
197+
}
198+
file.delete();
199+
}
200+
201+
202+
private static File copyToPrimaryDir(String resourceName) throws IOException {
203+
File out = new File(primaryDir, resourceName);
204+
Resources.asByteSource(Resources.getResource("cbit/vcell/simdata/langevin/batch/" + resourceName))
205+
.copyTo(com.google.common.io.Files.asByteSink(out));
206+
return out;
207+
}
208+
209+
210+
private static ODESolverResultSet getOdeSolverResultSet(File idaFile) throws IOException {
211+
ODESolverResultSet odeSolverResultSet = new ODESolverResultSet();
212+
FileInputStream inputStream = null;
213+
inputStream = new FileInputStream(idaFile);
214+
if(readIDA(odeSolverResultSet, inputStream) == null) {
215+
return null;
216+
}
217+
return (odeSolverResultSet);
218+
}
219+
private static ODESolverResultSet readIDA(ODESolverResultSet odeSolverResultSet, FileInputStream inputStream) throws IOException {
220+
InputStreamReader inputStreamReader = new InputStreamReader(inputStream);
221+
BufferedReader bufferedReader = new BufferedReader(inputStreamReader);
222+
// read header
223+
String line = bufferedReader.readLine();
224+
if (line == null) {
225+
return null;
226+
}
227+
while (line.indexOf(':') > 0) {
228+
String name = line.substring(0, line.indexOf(':'));
229+
odeSolverResultSet.addDataColumn(new ODESolverResultSetColumnDescription(name));
230+
line = line.substring(line.indexOf(':') + 1);
231+
}
232+
// read data
233+
while ((line = bufferedReader.readLine()) != null) {
234+
line = line + " ";
235+
double[] values = new double[odeSolverResultSet.getDataColumnCount()];
236+
boolean bCompleteRow = true;
237+
for (int i = 0; i < odeSolverResultSet.getDataColumnCount(); i++) {
238+
if (line.indexOf(' ')==-1) { // here and below we assume separator is ' ', in other cases might also be '\t'
239+
bCompleteRow = false;
240+
break;
241+
}else{
242+
String value = line.substring(0, line.indexOf(' ')).trim();
243+
values[i] = Double.valueOf(value).doubleValue();
244+
line = line.substring(line.indexOf(' ') + 1);
245+
}
246+
}
247+
if (bCompleteRow){
248+
odeSolverResultSet.addRow(values);
249+
}else{
250+
break;
251+
}
252+
}
253+
return odeSolverResultSet;
254+
}
255+
}

‎vcell-core/src/test/resources/cbit/vcell/simdata/langevin/batch/SimID_303404574_0_.functions‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
##---------------------------------------------
2+
## /simdata/danv/SimID_303404574_0_.functions
3+
##---------------------------------------------
4+
5+

0 commit comments

Comments
 (0)