Skip to content

Commit 9d73507

Browse files
linrrzqqqlinrrzqqq
committed
[Enhancement](udf) support deterministic property for udf
1 parent c353518 commit 9d73507

21 files changed

Lines changed: 1061 additions & 20 deletions

File tree

‎fe/fe-catalog/src/main/java/org/apache/doris/catalog/Function.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,8 @@ public enum BinaryType {
114114
protected String runtimeVersion;
115115
@SerializedName("fc")
116116
protected String functionCode;
117+
@SerializedName("det")
118+
protected boolean deterministic = false;
117119

118120
// Only used for serialization
119121
protected Function() {
@@ -174,6 +176,7 @@ public Function(Function other) {
174176
this.expirationTime = other.expirationTime;
175177
this.runtimeVersion = other.runtimeVersion;
176178
this.functionCode = other.functionCode;
179+
this.deterministic = other.deterministic;
177180
}
178181

179182
public Function clone() {
@@ -301,6 +304,14 @@ public void setFunctionCode(String functionCode) {
301304
this.functionCode = functionCode;
302305
}
303306

307+
public boolean isDeterministic() {
308+
return deterministic;
309+
}
310+
311+
public void setDeterministic(boolean deterministic) {
312+
this.deterministic = deterministic;
313+
}
314+
304315
// TODO(cmy): Currently we judge whether it is UDF by wheter the 'location' is set.
305316
// Maybe we should use a separate variable to identify,
306317
// but additional variables need to modify the persistence information.
@@ -401,7 +412,8 @@ public boolean equals(Object o) {
401412
}
402413
Function function = (Function) o;
403414
return id == function.id && hasVarArgs == function.hasVarArgs && userVisible == function.userVisible
404-
&& vectorized == function.vectorized && Objects.equals(name, function.name)
415+
&& vectorized == function.vectorized && deterministic == function.deterministic
416+
&& Objects.equals(name, function.name)
405417
&& Objects.equals(retType, function.retType) && Arrays.equals(argTypes,
406418
function.argTypes) && Objects.equals(location, function.location)
407419
&& binaryType == function.binaryType && nullableMode == function.nullableMode && Objects.equals(
@@ -411,7 +423,7 @@ public boolean equals(Object o) {
411423
@Override
412424
public int hashCode() {
413425
int result = Objects.hash(id, name, retType, hasVarArgs, userVisible, location, binaryType, nullableMode,
414-
vectorized, checksum);
426+
vectorized, checksum, deterministic);
415427
result = 31 * result + Arrays.hashCode(argTypes);
416428
return result;
417429
}

‎fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionToSqlConverter.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,13 @@ public static String toSql(ScalarFunction fn, boolean ifNotExists) {
7575
.append("\"" + (fn.getLocation() == null ? "" : fn.getLocation().toString()) + "\"");
7676
boolean isReturnNull = fn.getNullableMode() == NullableMode.ALWAYS_NULLABLE;
7777
sb.append(",\n \"ALWAYS_NULLABLE\"=").append("\"" + isReturnNull + "\"");
78+
sb.append(",\n \"DETERMINISTIC\"=").append("\"" + fn.isDeterministic() + "\"");
79+
} else if (fn.getBinaryType() == Function.BinaryType.PYTHON_UDF) {
80+
sb.append(",\n \"FILE\"=")
81+
.append("\"" + (fn.getLocation() == null ? "" : fn.getLocation().toString()) + "\"");
82+
boolean isReturnNull = fn.getNullableMode() == NullableMode.ALWAYS_NULLABLE;
83+
sb.append(",\n \"ALWAYS_NULLABLE\"=").append("\"" + isReturnNull + "\"");
84+
sb.append(",\n \"DETERMINISTIC\"=").append("\"" + fn.isDeterministic() + "\"");
7885
} else {
7986
sb.append(",\n \"OBJECT_FILE\"=")
8087
.append("\"" + (fn.getLocation() == null ? "" : fn.getLocation().toString()) + "\"");
@@ -125,6 +132,13 @@ public static String toSql(AggregateFunction fn, boolean ifNotExists) {
125132
.append("\"" + (fn.getLocation() == null ? "" : fn.getLocation().toString()) + "\",");
126133
boolean isReturnNull = fn.getNullableMode() == NullableMode.ALWAYS_NULLABLE;
127134
sb.append("\n \"ALWAYS_NULLABLE\"=").append("\"" + isReturnNull + "\",");
135+
sb.append("\n \"DETERMINISTIC\"=").append("\"" + fn.isDeterministic() + "\",");
136+
} else if (fn.getBinaryType() == Function.BinaryType.PYTHON_UDF) {
137+
sb.append("\n \"FILE\"=")
138+
.append("\"" + (fn.getLocation() == null ? "" : fn.getLocation().toString()) + "\",");
139+
boolean isReturnNull = fn.getNullableMode() == NullableMode.ALWAYS_NULLABLE;
140+
sb.append("\n \"ALWAYS_NULLABLE\"=").append("\"" + isReturnNull + "\",");
141+
sb.append("\n \"DETERMINISTIC\"=").append("\"" + fn.isDeterministic() + "\",");
128142
} else {
129143
sb.append("\n \"OBJECT_FILE\"=")
130144
.append("\"" + (fn.getLocation() == null ? "" : fn.getLocation().toString()) + "\",");

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdaf.java‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ public class JavaUdaf extends AggregateFunction implements ExplicitlyCastableSig
6262
private final String checkSum;
6363
private final boolean isStaticLoad;
6464
private final long expirationTime;
65+
private final boolean deterministic;
6566

6667
/**
6768
* Constructor of UDAF
@@ -72,7 +73,8 @@ public JavaUdaf(String name, long functionId, String dbName, Function.BinaryType
7273
String objectFile, String symbol,
7374
String initFn, String updateFn, String mergeFn,
7475
String serializeFn, String finalizeFn, String getValueFn, String removeFn,
75-
boolean isDistinct, String checkSum, boolean isStaticLoad, long expirationTime, Expression... args) {
76+
boolean isDistinct, String checkSum, boolean isStaticLoad, long expirationTime,
77+
boolean deterministic, Expression... args) {
7678
super(name, isDistinct, args);
7779
this.dbName = dbName;
7880
this.functionId = functionId;
@@ -92,6 +94,7 @@ public JavaUdaf(String name, long functionId, String dbName, Function.BinaryType
9294
this.checkSum = checkSum;
9395
this.isStaticLoad = isStaticLoad;
9496
this.expirationTime = expirationTime;
97+
this.deterministic = deterministic;
9598
}
9699

97100
@Override
@@ -114,6 +117,11 @@ public NullableMode getNullableMode() {
114117
return nullableMode;
115118
}
116119

120+
@Override
121+
public boolean isDeterministic() {
122+
return deterministic;
123+
}
124+
117125
/**
118126
* withChildren.
119127
*/
@@ -122,7 +130,8 @@ public JavaUdaf withDistinctAndChildren(boolean isDistinct, List<Expression> chi
122130
Preconditions.checkArgument(children.size() == this.children.size());
123131
return new JavaUdaf(getName(), functionId, dbName, binaryType, signature, intermediateType, nullableMode,
124132
objectFile, symbol, initFn, updateFn, mergeFn, serializeFn, finalizeFn, getValueFn, removeFn,
125-
isDistinct, checkSum, isStaticLoad, expirationTime, children.toArray(new Expression[0]));
133+
isDistinct, checkSum, isStaticLoad, expirationTime, deterministic,
134+
children.toArray(new Expression[0]));
126135
}
127136

128137
/**
@@ -165,6 +174,7 @@ public static void translateToNereidsFunction(String dbName, org.apache.doris.ca
165174
aggregate.getChecksum(),
166175
aggregate.isStaticLoad(),
167176
aggregate.getExpirationTime(),
177+
aggregate.isDeterministic(),
168178
arguments);
169179

170180
JavaUdafBuilder builder = new JavaUdafBuilder(udaf);
@@ -201,6 +211,7 @@ public Function getCatalogFunction() {
201211
expr.setId(functionId);
202212
expr.setStaticLoad(isStaticLoad);
203213
expr.setExpirationTime(expirationTime);
214+
expr.setDeterministic(deterministic);
204215
return expr;
205216
} catch (Exception e) {
206217
throw new AnalysisException(e.getMessage(), e.getCause());

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdf.java‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,14 +56,15 @@ public class JavaUdf extends ScalarFunction implements ExplicitlyCastableSignatu
5656
private final String checkSum;
5757
private final boolean isStaticLoad;
5858
private final long expirationTime;
59+
private final boolean deterministic;
5960

6061
/**
6162
* Constructor of UDF
6263
*/
6364
public JavaUdf(String name, long functionId, String dbName, Function.BinaryType binaryType,
6465
FunctionSignature signature,
6566
NullableMode nullableMode, String objectFile, String symbol, String prepareFn, String closeFn,
66-
String checkSum, boolean isStaticLoad, long expirationTime, Expression... args) {
67+
String checkSum, boolean isStaticLoad, long expirationTime, boolean deterministic, Expression... args) {
6768
super(name, args);
6869
this.dbName = dbName;
6970
this.functionId = functionId;
@@ -77,6 +78,7 @@ public JavaUdf(String name, long functionId, String dbName, Function.BinaryType
7778
this.checkSum = checkSum;
7879
this.isStaticLoad = isStaticLoad;
7980
this.expirationTime = expirationTime;
81+
this.deterministic = deterministic;
8082
}
8183

8284
@Override
@@ -99,14 +101,19 @@ public NullableMode getNullableMode() {
99101
return nullableMode;
100102
}
101103

104+
@Override
105+
public boolean isDeterministic() {
106+
return deterministic;
107+
}
108+
102109
/**
103110
* withChildren.
104111
*/
105112
@Override
106113
public JavaUdf withChildren(List<Expression> children) {
107114
Preconditions.checkArgument(children.size() == this.children.size());
108115
return new JavaUdf(getName(), functionId, dbName, binaryType, signature, nullableMode,
109-
objectFile, symbol, prepareFn, closeFn, checkSum, isStaticLoad, expirationTime,
116+
objectFile, symbol, prepareFn, closeFn, checkSum, isStaticLoad, expirationTime, deterministic,
110117
children.toArray(new Expression[0]));
111118
}
112119

@@ -135,7 +142,7 @@ public static void translateToNereidsFunction(String dbName, org.apache.doris.ca
135142
scalar.getSymbolName(),
136143
scalar.getPrepareFnSymbol(),
137144
scalar.getCloseFnSymbol(),
138-
scalar.getChecksum(), scalar.isStaticLoad(), scalar.getExpirationTime(),
145+
scalar.getChecksum(), scalar.isStaticLoad(), scalar.getExpirationTime(), scalar.isDeterministic(),
139146
arguments);
140147

141148
JavaUdfBuilder builder = new JavaUdfBuilder(udf);
@@ -166,6 +173,7 @@ public Function getCatalogFunction() {
166173
expr.setId(functionId);
167174
expr.setStaticLoad(isStaticLoad);
168175
expr.setExpirationTime(expirationTime);
176+
expr.setDeterministic(deterministic);
169177
return expr;
170178
} catch (Exception e) {
171179
throw new AnalysisException(e.getMessage(), e.getCause());

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdtf.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,14 +56,15 @@ public class JavaUdtf extends TableGeneratingFunction implements ExplicitlyCasta
5656
private final String checkSum;
5757
private final boolean isStaticLoad;
5858
private final long expirationTime;
59+
private final boolean deterministic;
5960

6061
/**
6162
* Constructor of UDTF
6263
*/
6364
public JavaUdtf(String name, long functionId, String dbName, Function.BinaryType binaryType,
6465
FunctionSignature signature,
6566
NullableMode nullableMode, String objectFile, String symbol, String prepareFn, String closeFn,
66-
String checkSum, boolean isStaticLoad, long expirationTime, Expression... args) {
67+
String checkSum, boolean isStaticLoad, long expirationTime, boolean deterministic, Expression... args) {
6768
super(name, args);
6869
this.dbName = dbName;
6970
this.functionId = functionId;
@@ -77,6 +78,7 @@ public JavaUdtf(String name, long functionId, String dbName, Function.BinaryType
7778
this.checkSum = checkSum;
7879
this.isStaticLoad = isStaticLoad;
7980
this.expirationTime = expirationTime;
81+
this.deterministic = deterministic;
8082
}
8183

8284
/**
@@ -86,7 +88,7 @@ public JavaUdtf(String name, long functionId, String dbName, Function.BinaryType
8688
public JavaUdtf withChildren(List<Expression> children) {
8789
Preconditions.checkArgument(children.size() == this.children.size());
8890
return new JavaUdtf(getName(), functionId, dbName, binaryType, signature, nullableMode,
89-
objectFile, symbol, prepareFn, closeFn, checkSum, isStaticLoad, expirationTime,
91+
objectFile, symbol, prepareFn, closeFn, checkSum, isStaticLoad, expirationTime, deterministic,
9092
children.toArray(new Expression[0]));
9193
}
9294

@@ -95,6 +97,11 @@ public List<FunctionSignature> getSignatures() {
9597
return ImmutableList.of(signature);
9698
}
9799

100+
@Override
101+
public boolean isDeterministic() {
102+
return deterministic;
103+
}
104+
98105
@Override
99106
public boolean hasVarArguments() {
100107
return signature.hasVarArgs;
@@ -125,6 +132,7 @@ public Function getCatalogFunction() {
125132
expr.setStaticLoad(isStaticLoad);
126133
expr.setExpirationTime(expirationTime);
127134
expr.setUDTFunction(true);
135+
expr.setDeterministic(deterministic);
128136
return expr;
129137
} catch (Exception e) {
130138
throw new AnalysisException(e.getMessage(), e.getCause());
@@ -159,6 +167,7 @@ public static void translateToNereidsFunction(String dbName, org.apache.doris.ca
159167
scalar.getChecksum(),
160168
scalar.isStaticLoad(),
161169
scalar.getExpirationTime(),
170+
scalar.isDeterministic(),
162171
arguments);
163172

164173
JavaUdtfBuilder builder = new JavaUdtfBuilder(udf);

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdaf.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ public class PythonUdaf extends AggregateFunction implements ExplicitlyCastableS
6464
private final long expirationTime;
6565
private final String runtimeVersion;
6666
private final String functionCode;
67+
private final boolean deterministic;
6768

6869
/**
6970
* Constructor of UDAF
@@ -75,7 +76,7 @@ public PythonUdaf(String name, long functionId, String dbName, Function.BinaryTy
7576
String initFn, String updateFn, String mergeFn,
7677
String serializeFn, String finalizeFn, String getValueFn, String removeFn,
7778
boolean isDistinct, String checkSum, boolean isStaticLoad, long expirationTime,
78-
String runtimeVersion, String functionCode, Expression... args) {
79+
String runtimeVersion, String functionCode, boolean deterministic, Expression... args) {
7980
super(name, isDistinct, args);
8081
this.dbName = dbName;
8182
this.functionId = functionId;
@@ -97,6 +98,7 @@ public PythonUdaf(String name, long functionId, String dbName, Function.BinaryTy
9798
this.expirationTime = expirationTime;
9899
this.runtimeVersion = runtimeVersion;
99100
this.functionCode = functionCode;
101+
this.deterministic = deterministic;
100102
}
101103

102104
@Override
@@ -119,6 +121,11 @@ public NullableMode getNullableMode() {
119121
return nullableMode;
120122
}
121123

124+
@Override
125+
public boolean isDeterministic() {
126+
return deterministic;
127+
}
128+
122129
/**
123130
* withChildren.
124131
*/
@@ -127,7 +134,7 @@ public PythonUdaf withDistinctAndChildren(boolean isDistinct, List<Expression> c
127134
Preconditions.checkArgument(children.size() == this.children.size());
128135
return new PythonUdaf(getName(), functionId, dbName, binaryType, signature, intermediateType, nullableMode,
129136
objectFile, symbol, initFn, updateFn, mergeFn, serializeFn, finalizeFn, getValueFn, removeFn,
130-
isDistinct, checkSum, isStaticLoad, expirationTime, runtimeVersion, functionCode,
137+
isDistinct, checkSum, isStaticLoad, expirationTime, runtimeVersion, functionCode, deterministic,
131138
children.toArray(new Expression[0]));
132139
}
133140

@@ -173,6 +180,7 @@ public static void translateToNereidsFunction(String dbName, org.apache.doris.ca
173180
aggregate.getExpirationTime(),
174181
aggregate.getRuntimeVersion(),
175182
aggregate.getFunctionCode(),
183+
aggregate.isDeterministic(),
176184
arguments);
177185

178186
PythonUdafBuilder builder = new PythonUdafBuilder(udaf);
@@ -211,6 +219,7 @@ public Function getCatalogFunction() {
211219
expr.setExpirationTime(expirationTime);
212220
expr.setRuntimeVersion(runtimeVersion);
213221
expr.setFunctionCode(functionCode);
222+
expr.setDeterministic(deterministic);
214223
return expr;
215224
} catch (Exception e) {
216225
throw new AnalysisException(e.getMessage(), e.getCause());

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/PythonUdf.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ public class PythonUdf extends ScalarFunction implements ExplicitlyCastableSigna
5858
private final long expirationTime;
5959
private final String runtimeVersion;
6060
private final String functionCode;
61+
private final boolean deterministic;
6162

6263
/**
6364
* Constructor of UDF
@@ -66,7 +67,7 @@ public PythonUdf(String name, long functionId, String dbName, Function.BinaryTyp
6667
FunctionSignature signature,
6768
NullableMode nullableMode, String objectFile, String symbol, String prepareFn, String closeFn,
6869
String checkSum, boolean isStaticLoad, long expirationTime,
69-
String runtimeVersion, String functionCode, Expression... args) {
70+
String runtimeVersion, String functionCode, boolean deterministic, Expression... args) {
7071
super(name, args);
7172
this.dbName = dbName;
7273
this.functionId = functionId;
@@ -82,6 +83,7 @@ public PythonUdf(String name, long functionId, String dbName, Function.BinaryTyp
8283
this.expirationTime = expirationTime;
8384
this.runtimeVersion = runtimeVersion;
8485
this.functionCode = functionCode;
86+
this.deterministic = deterministic;
8587
}
8688

8789
@Override
@@ -104,6 +106,11 @@ public NullableMode getNullableMode() {
104106
return nullableMode;
105107
}
106108

109+
@Override
110+
public boolean isDeterministic() {
111+
return deterministic;
112+
}
113+
107114
/**
108115
* withChildren.
109116
*/
@@ -112,7 +119,7 @@ public PythonUdf withChildren(List<Expression> children) {
112119
Preconditions.checkArgument(children.size() == this.children.size());
113120
return new PythonUdf(getName(), functionId, dbName, binaryType, signature, nullableMode,
114121
objectFile, symbol, prepareFn, closeFn, checkSum, isStaticLoad, expirationTime,
115-
runtimeVersion, functionCode, children.toArray(new Expression[0]));
122+
runtimeVersion, functionCode, deterministic, children.toArray(new Expression[0]));
116123
}
117124

118125
/**
@@ -143,6 +150,7 @@ public static void translateToNereidsFunction(String dbName, org.apache.doris.ca
143150
scalar.getChecksum(), scalar.isStaticLoad(), scalar.getExpirationTime(),
144151
scalar.getRuntimeVersion(),
145152
scalar.getFunctionCode(),
153+
scalar.isDeterministic(),
146154
arguments);
147155

148156
PythonUdfBuilder builder = new PythonUdfBuilder(udf);
@@ -175,6 +183,7 @@ public Function getCatalogFunction() {
175183
expr.setExpirationTime(expirationTime);
176184
expr.setRuntimeVersion(runtimeVersion);
177185
expr.setFunctionCode(functionCode);
186+
expr.setDeterministic(deterministic);
178187
return expr;
179188
} catch (Exception e) {
180189
throw new AnalysisException(e.getMessage(), e.getCause());

0 commit comments

Comments
 (0)