Skip to content

Commit 00f0976

Browse files
CaideyipiCaideyipi
committed
Fix library UDF edge cases
1 parent 5288816 commit 00f0976

41 files changed

Lines changed: 1333 additions & 133 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFIQR.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,11 @@ public void validate(UDFParameterValidator validator) throws Exception {
5959
"Parameter \"compute\" is illegal. Please use \"batch\" (for default) or \"stream\".",
6060
validator.getParameters().getStringOrDefault("compute", BATCH_COMPUTE))
6161
.validate(
62-
params -> (double) params[0] < (double) params[1],
63-
"parameter $q1$ should be smaller than $q3$",
62+
params ->
63+
Double.isFinite((double) params[0])
64+
&& Double.isFinite((double) params[1])
65+
&& (double) params[0] < (double) params[1],
66+
"parameter $q1$ and $q3$ should be finite, and $q1$ should be smaller than $q3$",
6467
validator.getParameters().getDoubleOrDefault("q1", -1),
6568
validator.getParameters().getDoubleOrDefault("q3", 1));
6669
if (validator

‎library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFKSigma.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,8 +55,8 @@ public void validate(UDFParameterValidator validator) throws Exception {
5555
"Window size should be larger than 0.",
5656
validator.getParameters().getIntOrDefault("window", DEFAULT_WINDOW_SIZE))
5757
.validate(
58-
x -> (double) x > 0,
59-
"Parameter k should be larger than 0.",
58+
x -> Double.isFinite((double) x) && (double) x > 0,
59+
"Parameter k should be finite and larger than 0.",
6060
validator.getParameters().getDoubleOrDefault("k", 3));
6161
}
6262

‎library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFLOF.java‎

Lines changed: 35 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@
3131
import org.apache.iotdb.udf.api.exception.UDFException;
3232
import org.apache.iotdb.udf.api.type.Type;
3333

34+
import java.util.ArrayList;
35+
import java.util.List;
36+
3437
/** This function is used to detect density anomaly of time series. */
3538
public class UDTFLOF implements UDTF {
3639
private int multipleK;
@@ -141,23 +144,30 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati
141144
@Override
142145
public void transform(RowWindow rowWindow, PointCollector collector) throws Exception {
143146
if (this.method.equals(DEFAULT_METHOD)) {
144-
int size = rowWindow.windowSize();
145-
Double[][] knn = new Double[size][dim];
146-
long[] timestamp = new long[size];
147-
int i = 0;
147+
int size = 0;
148+
Double[][] knn = new Double[rowWindow.windowSize()][dim];
149+
long[] timestamp = new long[rowWindow.windowSize()];
148150
int row = 0;
149151
while (row < rowWindow.windowSize()) {
150-
timestamp[i] = rowWindow.getRow(row).getTime();
152+
Double[] values = new Double[dim];
153+
boolean valid = true;
151154
for (int j = 0; j < dim; j++) {
152-
if (!rowWindow.getRow(row).isNull(j)) {
153-
knn[i][j] = Util.getValueAsDouble(rowWindow.getRow(row), j);
154-
} else {
155-
i--;
156-
size--;
155+
if (rowWindow.getRow(row).isNull(j)) {
156+
valid = false;
157+
break;
158+
}
159+
double value = Util.getValueAsDouble(rowWindow.getRow(row), j);
160+
if (!Double.isFinite(value)) {
161+
valid = false;
157162
break;
158163
}
164+
values[j] = value;
165+
}
166+
if (valid) {
167+
timestamp[size] = rowWindow.getRow(row).getTime();
168+
knn[size] = values;
169+
size++;
159170
}
160-
i++;
161171
row++;
162172
}
163173
if (size > multipleK) {
@@ -174,36 +184,32 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce
174184
} else if (this.method.equals("series")) {
175185
int size = rowWindow.windowSize() - window + 1;
176186
if (size > 0) {
177-
Double[][] knn = new Double[size][window];
178-
long[] timestamp = new long[rowWindow.windowSize()];
179-
double temp;
180-
int i = 0;
187+
List<Long> timestamp = new ArrayList<>();
188+
List<Double> values = new ArrayList<>();
181189
int row = 0;
182190
while (row < rowWindow.windowSize()) {
183-
timestamp[i] = rowWindow.getRow(row).getTime();
184191
if (!rowWindow.getRow(row).isNull(0)) {
185-
temp = Util.getValueAsDouble(rowWindow.getRow(row), 0);
186-
for (int p = 0; p < window; p++) {
187-
if (i - p < 0) {
188-
break;
189-
}
190-
if (i - p < size) {
191-
knn[i - p][p] = temp;
192-
}
192+
double value = Util.getValueAsDouble(rowWindow.getRow(row), 0);
193+
if (Double.isFinite(value)) {
194+
timestamp.add(rowWindow.getRow(row).getTime());
195+
values.add(value);
193196
}
194-
} else {
195-
i--;
196-
size--;
197197
}
198-
i++;
199198
row++;
200199
}
200+
size = values.size() - window + 1;
201201
if (size > multipleK) {
202+
Double[][] knn = new Double[size][window];
203+
for (int i = 0; i < size; i++) {
204+
for (int p = 0; p < window; p++) {
205+
knn[i][p] = values.get(i + p);
206+
}
207+
}
202208
double[] lof = new double[size];
203209
for (int m = 0; m < size; m++) {
204210
try {
205211
lof[m] = getLOF(knn, knn[m], size);
206-
collector.putDouble(timestamp[m], lof[m]);
212+
collector.putDouble(timestamp.get(m), lof[m]);
207213
} catch (Exception e) {
208214
throw new UDFException(LibraryUdfMessages.FAIL_TO_GET_LOF + m, e);
209215
}

‎library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFOutlier.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,8 @@ public void validate(UDFParameterValidator validator) throws Exception {
5858
"Parameter k should be a positive integer.",
5959
validator.getParameters().getIntOrDefault("k", 3))
6060
.validate(
61-
x -> (double) x >= 0,
62-
"Parameter r should be non-negative.",
61+
x -> Double.isFinite((double) x) && (double) x >= 0,
62+
"Parameter r should be finite and non-negative.",
6363
validator.getParameters().getDoubleOrDefault("r", 5))
6464
.validate(
6565
x -> (int) x > 0,

‎library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFRange.java‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,11 @@ public void validate(UDFParameterValidator validator) throws Exception {
4444
.validateRequiredAttribute("lower_bound")
4545
.validateRequiredAttribute("upper_bound")
4646
.validate(
47-
params -> (double) params[0] < (double) params[1],
48-
"parameter \"lower_bound\" should be smaller than \"upper_bound\".",
47+
params ->
48+
Double.isFinite((double) params[0])
49+
&& Double.isFinite((double) params[1])
50+
&& (double) params[0] < (double) params[1],
51+
"parameter \"lower_bound\" and \"upper_bound\" should be finite, and \"lower_bound\" should be smaller than \"upper_bound\".",
4952
validator.getParameters().getDouble("lower_bound"),
5053
validator.getParameters().getDouble("upper_bound"));
5154
}
@@ -87,13 +90,14 @@ public void transform(Row row, PointCollector collector) throws Exception {
8790
break;
8891
case FLOAT:
8992
floatValue = row.getFloat(0);
90-
if (floatValue > upperBound || floatValue < lowerBound) {
93+
if (Float.isFinite(floatValue) && (floatValue > upperBound || floatValue < lowerBound)) {
9194
collector.putFloat(timestamp, floatValue);
9295
}
9396
break;
9497
case DOUBLE:
9598
doubleValue = row.getDouble(0);
96-
if (doubleValue > upperBound || doubleValue < lowerBound) {
99+
if (Double.isFinite(doubleValue)
100+
&& (doubleValue > upperBound || doubleValue < lowerBound)) {
97101
collector.putDouble(timestamp, doubleValue);
98102
}
99103
break;

‎library-udf/src/main/java/org/apache/iotdb/library/anomaly/UDTFTwoSidedFilter.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,15 @@ public class UDTFTwoSidedFilter implements UDTF {
4343
public void validate(UDFParameterValidator validator) throws Exception {
4444
validator
4545
.validateInputSeriesNumber(1)
46-
.validateInputSeriesDataType(0, Type.INT32, Type.INT64, Type.FLOAT, Type.DOUBLE);
46+
.validateInputSeriesDataType(0, Type.INT32, Type.INT64, Type.FLOAT, Type.DOUBLE)
47+
.validate(
48+
x -> Double.isFinite((double) x) && (double) x > 0,
49+
"Parameter len should be finite and larger than 0.",
50+
validator.getParameters().getDoubleOrDefault("len", 5))
51+
.validate(
52+
x -> Double.isFinite((double) x) && (double) x >= 0,
53+
"Parameter threshold should be finite and non-negative.",
54+
validator.getParameters().getDoubleOrDefault("threshold", 0.4));
4755
}
4856

4957
@Override

‎library-udf/src/main/java/org/apache/iotdb/library/dlearn/UDTFAR.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,11 @@ public void beforeStart(UDFParameters udfParameters, UDTFConfigurations udtfConf
6666
@Override
6767
public void transform(Row row, PointCollector collector) throws Exception {
6868
if (!row.isNull(0)) {
69-
timeWindow.add(row.getTime());
70-
valueWindow.add(Util.getValueAsDouble(row));
69+
double value = Util.getValueAsDouble(row);
70+
if (Double.isFinite(value)) {
71+
timeWindow.add(row.getTime());
72+
valueWindow.add(value);
73+
}
7174
}
7275
}
7376

‎library-udf/src/main/java/org/apache/iotdb/library/dlearn/UDTFCluster.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -133,8 +133,11 @@ public void beforeStart(UDFParameters parameters, UDTFConfigurations configurati
133133
@Override
134134
public void transform(Row row, PointCollector collector) throws Exception {
135135
if (!row.isNull(0)) {
136-
timestamps.add(row.getTime());
137-
values.add(Util.getValueAsDouble(row));
136+
double value = Util.getValueAsDouble(row);
137+
if (Double.isFinite(value)) {
138+
timestamps.add(row.getTime());
139+
values.add(value);
140+
}
138141
}
139142
}
140143

‎library-udf/src/main/java/org/apache/iotdb/library/dmatch/UDAFDtw.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,8 +66,12 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce
6666
if (row.isNull(0) || row.isNull(1)) {
6767
continue;
6868
}
69-
a.add(Util.getValueAsDouble(row, 0));
70-
b.add(Util.getValueAsDouble(row, 1));
69+
double left = Util.getValueAsDouble(row, 0);
70+
double right = Util.getValueAsDouble(row, 1);
71+
if (Double.isFinite(left) && Double.isFinite(right)) {
72+
a.add(left);
73+
b.add(right);
74+
}
7175
}
7276
m = a.size();
7377
if (m == 0) {

‎library-udf/src/main/java/org/apache/iotdb/library/dmatch/UDTFPtnSym.java‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,8 @@ public void validate(UDFParameterValidator validator) throws Exception {
4848
"window has to be a positive integer.",
4949
validator.getParameters().getIntOrDefault("window", 10))
5050
.validate(
51-
x -> (double) x >= 0.0d,
52-
"threshold has to be non-negative.",
51+
x -> Double.isFinite((double) x) && (double) x >= 0.0d,
52+
"threshold has to be finite and non-negative.",
5353
validator.getParameters().getDoubleOrDefault("threshold", Double.MAX_VALUE));
5454
}
5555

@@ -74,7 +74,14 @@ public void transform(RowWindow rowWindow, PointCollector collector) throws Exce
7474
long time = rowWindow.getRow(0).getTime();
7575
for (int i = 0; i < n; i++) {
7676
Row row = rowWindow.getRow(i);
77-
a.add(Util.getValueAsDouble(row, 0));
77+
if (row.isNull(0)) {
78+
return;
79+
}
80+
double value = Util.getValueAsDouble(row, 0);
81+
if (!Double.isFinite(value)) {
82+
return;
83+
}
84+
a.add(value);
7885
}
7986
int m = a.size();
8087
double[][] dp = new double[m + 1][m + 1];

0 commit comments

Comments
 (0)