Skip to content

Commit 3d3f856

Browse files
author
Tom Brauer
committed
Apply data conversion logic to all imports
Remove data conversion logic from DssDataWriter and move to DataConverter class. Call DataConverter when preparing data for both NetCDF and DSS writes.
1 parent 43d405f commit 3d3f856

5 files changed

Lines changed: 123 additions & 101 deletions

File tree

‎vortex-api/src/main/java/mil/army/usace/hec/vortex/convert/DataConverter.java‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
package mil.army.usace.hec.vortex.convert;
2+
3+
import mil.army.usace.hec.vortex.VortexGrid;
4+
import mil.army.usace.hec.vortex.VortexVariable;
5+
import mil.army.usace.hec.vortex.geo.RasterUtils;
6+
import mil.army.usace.hec.vortex.util.UnitUtil;
7+
8+
import javax.measure.Unit;
9+
import java.time.Duration;
10+
import java.time.ZonedDateTime;
11+
import java.util.ArrayList;
12+
import java.util.List;
13+
14+
import static javax.measure.MetricPrefix.MILLI;
15+
import static systems.uom.common.USCustomary.INCH;
16+
import static tech.units.indriya.AbstractUnit.ONE;
17+
import static tech.units.indriya.unit.Units.*;
18+
19+
/**
20+
* Performs common data conversions for use in HEC applications.
21+
*/
22+
23+
public class DataConverter {
24+
25+
private static final int SECONDS_PER_HOUR = 3600;
26+
private static final int SECONDS_PER_DAY = 86400;
27+
private static final float KELVIN_TO_CELSIUS = -273.15f;
28+
29+
private DataConverter() {
30+
}
31+
32+
public static List<VortexGrid> convert(List<VortexGrid> vortexGrids) {
33+
List<VortexGrid> convertedGrids = new ArrayList<>();
34+
for (VortexGrid vortexGrid : vortexGrids) {
35+
VortexGrid convertedGrid = convert(vortexGrid);
36+
convertedGrids.add(convertedGrid);
37+
}
38+
return convertedGrids;
39+
}
40+
41+
public static VortexGrid convert(VortexGrid vortexGrid) {
42+
VortexVariable variable = VortexVariable.fromGrid(vortexGrid);
43+
Unit<?> units = UnitUtil.getUnits(vortexGrid.units());
44+
45+
if (units == null)
46+
return vortexGrid;
47+
48+
float[] data = vortexGrid.data();
49+
float noDataValue = (float) vortexGrid.noDataValue();
50+
51+
ZonedDateTime startTime = vortexGrid.startTime();
52+
ZonedDateTime endTime = vortexGrid.endTime();
53+
Duration interval = vortexGrid.interval();
54+
55+
String convertedUnits;
56+
float[] convertedData;
57+
58+
if (variable == VortexVariable.PRECIPITATION
59+
&& startTime != null && endTime != null && !interval.isZero()
60+
&& (units.equals(MILLI(METRE).divide(SECOND))
61+
|| units.equals(MILLI(METRE).divide(HOUR))
62+
|| units.equals(MILLI(METRE).divide(DAY)))) {
63+
float conversion = getPrecipRateConversionFactor(interval, units);
64+
convertedData = RasterUtils.convert(data, conversion, noDataValue);
65+
convertedUnits = "mm";
66+
} else if (variable == VortexVariable.PRECIPITATION && units.equals(METRE)) {
67+
convertedData = RasterUtils.convert(data, 1000, noDataValue);
68+
convertedUnits = "mm";
69+
} else if (variable == VortexVariable.HUMIDITY && units.equals(ONE)) {
70+
convertedData = RasterUtils.convert(data, 100, noDataValue);
71+
convertedUnits = "%";
72+
} else if (units.equals(KELVIN)) {
73+
convertedData = new float[data.length];
74+
for (int i = 0; i < data.length; i++) {
75+
float value = data[i];
76+
convertedData[i] = Float.compare(noDataValue, value) == 0 ? noDataValue : data[i] + KELVIN_TO_CELSIUS;
77+
}
78+
convertedUnits = "celsius";
79+
} else if (units.equals(ONE.divide(INCH.multiply(1000)))) {
80+
convertedData = RasterUtils.convert(data, 1E-3f, noDataValue);
81+
convertedUnits = "in";
82+
} else if (units.equals(PASCAL)) {
83+
convertedData = RasterUtils.convert(data, 1E-3f, noDataValue);
84+
convertedUnits = "kpa";
85+
} else {
86+
return vortexGrid;
87+
}
88+
89+
return VortexGrid.toBuilder(vortexGrid)
90+
.data(convertedData)
91+
.units(convertedUnits)
92+
.build();
93+
}
94+
95+
private static float getPrecipRateConversionFactor(Duration interval, Unit<?> units) {
96+
if (units.equals(MILLI(METRE).divide(SECOND)))
97+
return interval.getSeconds();
98+
99+
if (units.equals(MILLI(METRE).divide(HOUR)))
100+
return (float) interval.getSeconds() / SECONDS_PER_HOUR;
101+
102+
if (units.equals(MILLI(METRE).divide(DAY)))
103+
return (float) interval.getSeconds() / SECONDS_PER_DAY;
104+
105+
return 1;
106+
}
107+
}

‎vortex-api/src/main/java/mil/army/usace/hec/vortex/io/DssDataWriter.java‎

Lines changed: 9 additions & 100 deletions
Original file line numberDiff line numberDiff line change
@@ -14,14 +14,12 @@
1414
import mil.army.usace.hec.vortex.VortexGrid;
1515
import mil.army.usace.hec.vortex.VortexPoint;
1616
import mil.army.usace.hec.vortex.VortexVariable;
17+
import mil.army.usace.hec.vortex.convert.DataConverter;
1718
import mil.army.usace.hec.vortex.geo.RasterUtils;
1819
import mil.army.usace.hec.vortex.geo.ZonalStatistics;
1920
import mil.army.usace.hec.vortex.util.DssUtil;
20-
import mil.army.usace.hec.vortex.util.UnitUtil;
2121

22-
import javax.measure.Unit;
2322
import java.time.Duration;
24-
import java.time.LocalDateTime;
2523
import java.time.ZonedDateTime;
2624
import java.time.format.DateTimeFormatter;
2725
import java.util.*;
@@ -30,12 +28,7 @@
3028
import java.util.stream.Collectors;
3129

3230
import static hec.heclib.util.Heclib.UNDEFINED_FLOAT;
33-
import static javax.measure.MetricPrefix.MILLI;
3431
import static mil.army.usace.hec.vortex.VortexVariable.*;
35-
import static systems.uom.common.USCustomary.FAHRENHEIT;
36-
import static systems.uom.common.USCustomary.INCH;
37-
import static tech.units.indriya.AbstractUnit.ONE;
38-
import static tech.units.indriya.unit.Units.*;
3932

4033
class DssDataWriter extends DataWriter {
4134

@@ -46,14 +39,13 @@ class DssDataWriter extends DataWriter {
4639
}
4740

4841
private static final int SECONDS_PER_MINUTE = 60;
49-
private static final int SECONDS_PER_HOUR = 3600;
50-
private static final int SECONDS_PER_DAY = 86400;
5142

5243
@Override
5344
public void write() {
5445
List<VortexGrid> grids = data.stream()
5546
.filter(VortexGrid.class::isInstance)
5647
.map(VortexGrid.class::cast)
48+
.map(DataConverter::convert)
5749
.toList();
5850

5951
for (VortexGrid grid : grids) {
@@ -74,8 +66,6 @@ public void write() {
7466
}
7567
}
7668

77-
Unit<?> units = UnitUtil.getUnits(grid.units());
78-
7969
DSSPathname dssPathname = new DSSPathname();
8070

8171
String cPart = getCPart(grid);
@@ -95,97 +85,16 @@ public void write() {
9585
options.put("dataType", "INST-VAL");
9686
}
9787

98-
if (cPart.equals("PRECIPITATION") && !grid.interval().isZero()
99-
&& (units.equals(MILLI(METRE).divide(SECOND))
100-
|| units.equals(MILLI(METRE).divide(HOUR))
101-
|| units.equals(MILLI(METRE).divide(DAY)))) {
102-
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("d MMMM yyyy, HH:mm", Locale.ENGLISH);
103-
LocalDateTime startTime = LocalDateTime.parse(gridInfo.getStartTime(), formatter);
104-
LocalDateTime endTime = LocalDateTime.parse(gridInfo.getEndTime(), formatter);
105-
Duration interval = Duration.between(startTime, endTime);
106-
107-
float conversion;
108-
if (units.equals(MILLI(METRE).divide(SECOND))) {
109-
conversion = interval.getSeconds();
110-
} else if (units.equals(MILLI(METRE).divide(HOUR))) {
111-
conversion = (float) interval.getSeconds() / SECONDS_PER_HOUR;
112-
} else if (units.equals(MILLI(METRE).divide(DAY))) {
113-
conversion = (float) interval.getSeconds() / SECONDS_PER_DAY;
114-
} else {
115-
conversion = 1;
116-
}
117-
118-
float[] convertedData = RasterUtils.convert(data, conversion, UNDEFINED_FLOAT);
119-
120-
gridInfo.setDataUnits("MM");
121-
gridInfo.setDataType(DssDataType.PER_CUM.value());
122-
123-
if (options.containsKey("partF") && options.get("partF").equals("*")) {
124-
DSSPathname pathnameIn = new DSSPathname();
125-
int status = pathnameIn.setPathname(grid.fullName());
126-
if (status == 0) {
127-
dssPathname.setFPart(pathnameIn.getFPart());
128-
}
129-
}
130-
131-
write(convertedData, gridInfo, dssPathname);
132-
133-
} else if (cPart.equals("PRECIPITATION") && units.equals(METRE)) {
134-
float[] convertedData = RasterUtils.convert(data, 1000, UNDEFINED_FLOAT);
135-
gridInfo.setDataUnits("MM");
136-
137-
write(convertedData, gridInfo, dssPathname);
138-
} else if (units.equals(FAHRENHEIT) || units.equals(KELVIN) || units.equals(CELSIUS)) {
139-
float[] convertedData = new float[data.length];
140-
if (units.equals(FAHRENHEIT)) {
141-
System.arraycopy(data, 0, convertedData, 0, data.length);
142-
gridInfo.setDataUnits("DEG F");
143-
} else if (units.equals(KELVIN)) {
144-
for (int i = 0; i < data.length; i++) {
145-
float value = data[i];
146-
convertedData[i] = Float.compare(UNDEFINED_FLOAT, value) == 0 ? UNDEFINED_FLOAT : (float) (data[i] - 273.15);
147-
}
148-
gridInfo.setDataUnits("DEG C");
149-
} else if (units.equals(CELSIUS)) {
150-
System.arraycopy(data, 0, convertedData, 0, data.length);
151-
gridInfo.setDataUnits("DEG C");
152-
}
153-
154-
if (options.containsKey("partF") && options.get("partF").equals("*")) {
155-
DSSPathname pathnameIn = new DSSPathname();
156-
int status = pathnameIn.setPathname(grid.fullName());
157-
if (status == 0) {
158-
dssPathname.setFPart(pathnameIn.getFPart());
159-
}
88+
if (options.containsKey("partF") && options.get("partF").equals("*")) {
89+
DSSPathname pathnameIn = new DSSPathname();
90+
int status = pathnameIn.setPathname(grid.fullName());
91+
if (status == 0) {
92+
dssPathname.setFPart(pathnameIn.getFPart());
16093
}
94+
}
16195

162-
write(convertedData, gridInfo, dssPathname);
163-
} else if (cPart.equals("HUMIDITY") && units.equals(ONE)) {
164-
float[] convertedData = RasterUtils.convert(data, 100, UNDEFINED_FLOAT);
165-
gridInfo.setDataUnits("%");
166-
167-
write(convertedData, gridInfo, dssPathname);
168-
} else if (units.equals(ONE.divide(INCH.multiply(1000)))) {
169-
float[] convertedData = RasterUtils.convert(data, 1E-3f, UNDEFINED_FLOAT);
170-
gridInfo.setDataUnits("IN");
171-
172-
write(convertedData, gridInfo, dssPathname);
173-
} else if (units.equals(PASCAL)) {
174-
float[] convertedData = RasterUtils.convert(data, 1E-3f, UNDEFINED_FLOAT);
175-
gridInfo.setDataUnits("KPA");
176-
177-
write(convertedData, gridInfo, dssPathname);
178-
} else {
179-
if (options.containsKey("partF") && options.get("partF").equals("*")) {
180-
DSSPathname pathnameIn = new DSSPathname();
181-
int status = pathnameIn.setPathname(grid.fullName());
182-
if (status == 0) {
183-
dssPathname.setFPart(pathnameIn.getFPart());
184-
}
185-
}
96+
write(data, gridInfo, dssPathname);
18697

187-
write(data, gridInfo, dssPathname);
188-
}
18998
}
19099

191100
List<VortexPoint> points = data.stream()

‎vortex-api/src/main/java/mil/army/usace/hec/vortex/io/ImportableUnit.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import mil.army.usace.hec.vortex.VortexData;
55
import mil.army.usace.hec.vortex.VortexGrid;
66
import mil.army.usace.hec.vortex.VortexProperty;
7+
import mil.army.usace.hec.vortex.convert.DataConverter;
78
import mil.army.usace.hec.vortex.geo.GeographicProcessor;
89

910
import java.beans.PropertyChangeListener;
@@ -101,9 +102,10 @@ public void process() {
101102
for (int i = 0; i < count; i++) {
102103
VortexGrid grid = (VortexGrid) reader.getDto(i);
103104
VortexGrid processed = geoProcessor.process(grid);
105+
VortexGrid converted = DataConverter.convert(processed);
104106

105107
List<VortexData> data = new ArrayList<>();
106-
data.add(processed);
108+
data.add(converted);
107109

108110
DataWriter writer = DataWriter.builder()
109111
.data(data)

‎vortex-api/src/main/java/mil/army/usace/hec/vortex/io/NetcdfBatchImporter.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import mil.army.usace.hec.vortex.VortexData;
55
import mil.army.usace.hec.vortex.VortexGrid;
66
import mil.army.usace.hec.vortex.VortexProperty;
7+
import mil.army.usace.hec.vortex.convert.DataConverter;
78
import mil.army.usace.hec.vortex.geo.GeographicProcessor;
89
import mil.army.usace.hec.vortex.util.Stopwatch;
910

@@ -55,6 +56,7 @@ public void process() {
5556
.filter(VortexGrid.class::isInstance)
5657
.map(VortexGrid.class::cast)
5758
.map(geoProcessor::process)
59+
.map(DataConverter::convert)
5860
.forEach(grid -> bufferedDataWriter.addAndProcessWhenFull(grid, bufferProcessFunction, false));
5961
bufferedDataWriter.processBufferAndClear(bufferProcessFunction, true);
6062

@@ -99,6 +101,7 @@ private static VortexGridCollection representativeCollection(List<DataReader> re
99101
.filter(VortexGrid.class::isInstance)
100102
.map(VortexGrid.class::cast)
101103
.map(geoProcessor::process)
104+
.map(DataConverter::convert)
102105
.toList();
103106
VortexGridCollection vortexGridCollection = VortexGridCollection.of(processedFirstGrids);
104107
List<VortexGrid> uniqueGrids = List.copyOf(vortexGridCollection.getRepresentativeGridNameMap().values());

‎vortex-api/src/main/java/mil/army/usace/hec/vortex/util/UnitUtil.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ public static Unit<?> getUnits(String units) {
3232
case "m s-1" -> METRE.divide(SECOND);
3333
case "%" -> PERCENT;
3434
case "hpa" -> HECTO(PASCAL);
35+
case "kpa" -> KILO(PASCAL);
3536
case "pa" -> PASCAL;
3637
case "m", "meter", "metre" -> METRE;
3738
case "min" -> MINUTE;

0 commit comments

Comments
 (0)