Working on EncogModel & time series

Author: jeffheaton

src/main/java/org/encog/ml/data/versatile/CSVDataSource.java

@@ -1,6 +1,8 @@
 package org.encog.ml.data.versatile;
 
 import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
 
 import org.encog.EncogError;
 import org.encog.util.csv.CSVFormat;
@@ -12,6 +14,7 @@ public class CSVDataSource implements VersatileDataSource {
 	private final File file;
 	private final boolean headers;
 	private final CSVFormat format;
+	private final Map<String,Integer> headerIndex = new HashMap<String,Integer>();
 
 	/**
 	 * Construct a CSV reader from a filename. The format parameter specifies
@@ -72,6 +75,20 @@ public String[] readLine() {
 	@Override
 	public void rewind() {
 		this.reader = new ReadCSV(this.file,this.headers,this.format);
+		if( this.headerIndex.size()==0 ) {
+			for(int i=0;i<this.reader.getColumnNames().size();i++) {
+				this.headerIndex.put(this.reader.getColumnNames().get(i), i);
+			}
+		}
+	}
+
+	@Override
+	public int columnIndex(String name) {
+		String name2 = name.toLowerCase();
+		if(!this.headerIndex.containsKey(name2)) {
+			return -1;
+		}
+		return this.headerIndex.get(name2);
 	}
 
 }

src/main/java/org/encog/ml/data/versatile/ColumnDefinition.java

@@ -14,6 +14,7 @@ public class ColumnDefinition {
 	private double mean;
 	private double sd;
 	private int count;
+	private int index;
 	private final List<String> classes = new ArrayList<String>();
 	private NormalizationHelper owner;
 
@@ -212,4 +213,20 @@ public void defineClass(String[] str) {
 	public void setOwner(NormalizationHelper theOwner) {
 		this.owner = theOwner;
 	}
+
+	/**
+	 * @return the index
+	 */
+	public int getIndex() {
+		return index;
+	}
+
+	/**
+	 * @param index the index to set
+	 */
+	public void setIndex(int index) {
+		this.index = index;
+	}
+	
+	
 }

src/main/java/org/encog/ml/data/versatile/MatrixMLDataSet.java

@@ -60,17 +60,7 @@ public final MLDataPair next() {
 				return null;
 			}
 
-			BasicMLData input = new BasicMLData(
-					MatrixMLDataSet.this.calculatedInputSize);
-			BasicMLData ideal = new BasicMLData(
-					MatrixMLDataSet.this.calculatedIdealSize);
-			MLDataPair pair = new BasicMLDataPair(input, ideal);
-			
-			MatrixMLDataSet.this.getRecord(this.currentIndex, pair);
-
-			this.currentIndex++;
-
-			return pair;
+			return MatrixMLDataSet.this.get(this.currentIndex++);
 		}
 
 		/**
@@ -141,6 +131,10 @@ public boolean isSupervised() {
 
 	@Override
 	public long getRecordCount() {
+		if( this.data==null ) {
+			throw new EncogError("You must normalize the dataset before using it.");
+		}
+		
 		if (this.mask == null) {
 			return this.data.length
 					- (this.lagWindowSize + this.leadWindowSize);
@@ -159,7 +153,10 @@ private int calculateLeadCount() {
 
 	@Override
 	public void getRecord(long index, MLDataPair pair) {
-
+		if( this.data==null ) {
+			throw new EncogError("You must normalize the dataset before using it.");
+		}
+		
 		// Copy the input, account for time windows.
 		int inputSize = calculateLagCount();
 		for (int i = 0; i < inputSize; i++) {

src/main/java/org/encog/ml/data/versatile/NormalizationHelper.java

@@ -74,8 +74,9 @@ public void addSourceColumn(ColumnDefinition def) {
 		def.setOwner(this);
 	}
 
-	public ColumnDefinition defineSourceColumn(String name, ColumnType colType) {
+	public ColumnDefinition defineSourceColumn(String name, int index, ColumnType colType) {
 		ColumnDefinition result = new ColumnDefinition(name,colType);
+		result.setIndex(index);
 		addSourceColumn(result);
 		return result;
 	}

src/main/java/org/encog/ml/data/versatile/VersatileDataSource.java

@@ -3,4 +3,5 @@
 public interface VersatileDataSource {
 	String[] readLine();
 	void rewind();
+	int columnIndex(String name);
 }

src/main/java/org/encog/ml/data/versatile/VersatileMLDataSet.java

@@ -15,6 +15,21 @@ public class VersatileMLDataSet extends MatrixMLDataSet {
 	public VersatileMLDataSet(VersatileDataSource theSource) {
 		this.source = theSource;
 	}
+	
+	private int findIndex(ColumnDefinition colDef) {
+		if( colDef.getIndex()!=-1 ) {
+			return colDef.getIndex();
+		}
+		
+		int index = this.source.columnIndex(colDef.getName());
+		colDef.setIndex(index);
+		
+		if( index==-1 ) {
+			throw new EncogError("Can't find column");
+		}
+		
+		return index;
+	}
 
 	public void analyze() {
 		String[] line;
@@ -26,7 +41,8 @@ public void analyze() {
 			c++;
 			for (int i = 0; i < this.helper.getSourceColumns().size(); i++) {
 				ColumnDefinition colDef = this.helper.getSourceColumns().get(i);
-				String value = line[i];
+				int index = findIndex(colDef);
+				String value = line[index];
 				colDef.analyze(value);
 			}
 		}
@@ -91,14 +107,14 @@ public void normalize() {
 		while ((line = this.source.readLine()) != null) {
 			int column = 0;
 			for (ColumnDefinition colDef : this.helper.getInputColumns()) {
-				int index = this.helper.getSourceColumns().indexOf(colDef);
+				int index = findIndex(colDef);
 				String value = line[index];
 
 				column = this.helper.normalizeToVector(colDef, column, getData()[row], true, value);
 			}
 
 			for (ColumnDefinition colDef : this.helper.getOutputColumns()) {
-				int index = this.helper.getSourceColumns().indexOf(colDef);
+				int index = findIndex(colDef);
 				String value = line[index];
 
 				column = this.helper.normalizeToVector(colDef, column, getData()[row], false, value);
@@ -107,8 +123,8 @@ public void normalize() {
 		}
 	}
 
-	public ColumnDefinition defineSourceColumn(String name, ColumnType colType) {
-		return this.helper.defineSourceColumn(name, colType);
+	public ColumnDefinition defineSourceColumn(String name, int index, ColumnType colType) {
+		return this.helper.defineSourceColumn(name, index, colType);
 	}
 
 	/**
@@ -137,17 +153,30 @@ public void divide(List<DataDivision> dataDivisionList, boolean shuffle,
 				getCalculatedIdealSize());
 
 	}
+	
+	public void defineOutput(ColumnDefinition col) {
+		this.helper.getOutputColumns().add(col);
+	}
+	
+	public void defineInput(ColumnDefinition col) {
+		this.helper.getInputColumns().add(col);
+	}
 
 	public void defineSingleOutputOthersInput(ColumnDefinition outputColumn) {
 		this.helper.clearInputOutput();
 
 		for (ColumnDefinition colDef : this.helper.getSourceColumns()) {
 			if (colDef == outputColumn) {
-				this.helper.getOutputColumns().add(colDef);
+				defineOutput(colDef);
 			} else if(colDef.getDataType()!=ColumnType.ignore) {
-				this.helper.getInputColumns().add(colDef);
+				defineInput(colDef);
 			}
 		}
 	}
 
+	public ColumnDefinition defineSourceColumn(String name,
+			ColumnType colType) {
+		return this.helper.defineSourceColumn(name, -1, colType);
+	}
+
 }

src/main/java/org/encog/ml/train/strategy/end/SimpleEarlyStoppingStrategy.java

@@ -110,7 +110,7 @@ public void postIteration() {
 
 			double currentValidationError = this.calc.calculateError(this.validationSet);
 
-			if( currentValidationError>this.lastValidationError ) {
+			if( currentValidationError>=this.lastValidationError ) {
 				stop = true;
 			}