Skip to content

Commit 94fb171

Browse files
LantaoJinLantaoJin
authored
Support functions isempty, isblank and ispresent with Calcite (opensearch-project#3627)
* Support functions isempty, isblank and ispresent with Calcite Signed-off-by: Lantao Jin <ltjin@amazon.com> * add version in doc Signed-off-by: Lantao Jin <ltjin@amazon.com> * fix doc Signed-off-by: Lantao Jin <ltjin@amazon.com> --------- Signed-off-by: Lantao Jin <ltjin@amazon.com>
1 parent 1d2c0f7 commit 94fb171

7 files changed

Lines changed: 202 additions & 5 deletions

File tree

‎core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,10 @@ public enum BuiltinFunctionName {
227227
NULLIF(FunctionName.of("nullif")),
228228
ISNULL(FunctionName.of("isnull")),
229229

230+
IS_PRESENT(FunctionName.of("ispresent")),
231+
IS_EMPTY(FunctionName.of("isempty")),
232+
IS_BLANK(FunctionName.of("isblank")),
233+
230234
ROW_NUMBER(FunctionName.of("row_number")),
231235
RANK(FunctionName.of("rank")),
232236
DENSE_RANK(FunctionName.of("dense_rank")),

‎core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,7 @@ void populate() {
202202
registerOperator(IS_NULL, SqlStdOperatorTable.IS_NULL);
203203
registerOperator(IF, SqlStdOperatorTable.CASE);
204204
registerOperator(IFNULL, SqlStdOperatorTable.COALESCE);
205+
registerOperator(IS_PRESENT, SqlStdOperatorTable.IS_NOT_NULL);
205206

206207
// Register library operator
207208
registerOperator(REGEXP, SqlLibraryOperators.REGEXP);
@@ -371,6 +372,28 @@ void populate() {
371372
builder.makeCall(SqlStdOperatorTable.EQUALS, arg1, arg2),
372373
builder.makeNullLiteral(arg1.getType()),
373374
arg1));
375+
register(
376+
IS_EMPTY,
377+
((FunctionImp1)
378+
(builder, arg) ->
379+
builder.makeCall(
380+
SqlStdOperatorTable.OR,
381+
builder.makeCall(SqlStdOperatorTable.IS_NULL, arg),
382+
builder.makeCall(SqlStdOperatorTable.IS_EMPTY, arg))));
383+
register(
384+
IS_BLANK,
385+
((FunctionImp1)
386+
(builder, arg) ->
387+
builder.makeCall(
388+
SqlStdOperatorTable.OR,
389+
builder.makeCall(SqlStdOperatorTable.IS_NULL, arg),
390+
builder.makeCall(
391+
SqlStdOperatorTable.IS_EMPTY,
392+
builder.makeCall(
393+
SqlStdOperatorTable.TRIM,
394+
builder.makeFlag(Flag.BOTH),
395+
builder.makeLiteral(" "),
396+
arg)))));
374397
}
375398
}
376399

‎docs/user/ppl/functions/condition.rst‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ Argument type: all the supported data type.
4545

4646
Return type: BOOLEAN
4747

48+
Synonyms: `ISPRESENT`_
49+
4850
Example::
4951

5052
os> source=accounts | where not isnotnull(employer) | fields account_number, employer
@@ -238,3 +240,85 @@ Example::
238240
| Dale | Adams | 33 |
239241
+-----------+----------+-----+
240242

243+
ISPRESENT
244+
---------
245+
246+
Description
247+
>>>>>>>>>>>
248+
249+
Version: 3.1.0
250+
251+
Usage: ispresent(field) return true if the field exists.
252+
253+
Argument type: all the supported data type.
254+
255+
Return type: BOOLEAN
256+
257+
Synonyms: `ISNOTNULL`_
258+
259+
Example::
260+
261+
PPL> source=accounts | where ispresent(employer) | fields employer, firstname
262+
fetched rows / total rows = 3/3
263+
+----------+-----------+
264+
| employer | firstname |
265+
|----------+-----------|
266+
| Pyrami | Amber |
267+
| Netagy | Hattie |
268+
| Quility | Nanette |
269+
+----------+-----------+
270+
271+
ISBLANK
272+
-------
273+
274+
Description
275+
>>>>>>>>>>>
276+
277+
Version: 3.1.0
278+
279+
Usage: isblank(field) returns true if the field is null, an empty string, or contains only white space.
280+
281+
Argument type: all the supported data type.
282+
283+
Return type: BOOLEAN
284+
285+
Example::
286+
287+
PPL> source=accounts | eval temp = ifnull(employer, ' ') | eval `isblank(employer)` = isblank(employer), `isblank(temp)` = isblank(temp) | fields `isblank(temp)`, temp, `isblank(employer)`, employer
288+
fetched rows / total rows = 4/4
289+
+---------------+---------+-------------------+----------+
290+
| isblank(temp) | temp | isblank(employer) | employer |
291+
|---------------+---------+-------------------+----------|
292+
| False | Pyrami | False | Pyrami |
293+
| False | Netagy | False | Netagy |
294+
| False | Quility | False | Quility |
295+
| True | | True | null |
296+
+---------------+---------+-------------------+----------+
297+
298+
299+
ISEMPTY
300+
-------
301+
302+
Description
303+
>>>>>>>>>>>
304+
305+
Version: 3.1.0
306+
307+
Usage: isempty(field) returns true if the field is null or is an empty string.
308+
309+
Argument type: all the supported data type.
310+
311+
Return type: BOOLEAN
312+
313+
Example::
314+
315+
PPL> source=accounts | eval temp = ifnull(employer, ' ') | eval `isempty(employer)` = isempty(employer), `isempty(temp)` = isempty(temp) | fields `isempty(temp)`, temp, `isempty(employer)`, employer
316+
fetched rows / total rows = 4/4
317+
+---------------+---------+-------------------+----------+
318+
| isempty(temp) | temp | isempty(employer) | employer |
319+
|---------------+---------+-------------------+----------|
320+
| False | Pyrami | False | Pyrami |
321+
| False | Netagy | False | Netagy |
322+
| False | Quility | False | Quility |
323+
| False | | True | null |
324+
+---------------+---------+-------------------+----------+

‎docs/user/ppl/functions/cryptographic.rst‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ MD5
1414
Description
1515
>>>>>>>>>>>
1616

17+
Version: 3.1.0
1718

1819
Usage: ``md5(str)`` calculates the MD5 digest and returns the value as a 32 character hex string.
1920

@@ -37,6 +38,8 @@ SHA1
3738
Description
3839
>>>>>>>>>>>
3940

41+
Version: 3.1.0
42+
4043
Usage: ``sha1(str)`` returns the hex string result of SHA-1.
4144

4245
Argument type: STRING
@@ -59,6 +62,8 @@ SHA2
5962
Description
6063
>>>>>>>>>>>
6164

65+
Version: 3.1.0
66+
6267
Usage: ``sha2(str, numBits)`` returns the hex string result of SHA-2 family of hash functions (SHA-224, SHA-256, SHA-384, and SHA-512).
6368
The numBits indicates the desired bit length of the result, which must have a value of 224, 256, 384, or 512.
6469

‎integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLConditionBuiltinFunctionIT.java‎

Lines changed: 80 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,25 @@
1313
import java.io.IOException;
1414
import org.json.JSONObject;
1515
import org.junit.jupiter.api.Test;
16+
import org.opensearch.client.Request;
1617

1718
public class CalcitePPLConditionBuiltinFunctionIT extends CalcitePPLIntegTestCase {
1819
@Override
1920
public void init() throws IOException {
2021
super.init();
2122
loadIndex(Index.STATE_COUNTRY);
2223
loadIndex(Index.STATE_COUNTRY_WITH_NULL);
24+
Request request1 =
25+
new Request("PUT", "/" + TEST_INDEX_STATE_COUNTRY_WITH_NULL + "/_doc/7?refresh=true");
26+
request1.setJsonEntity(
27+
"{\"name\":\" "
28+
+ " \",\"age\":27,\"state\":\"B.C\",\"country\":\"Canada\",\"year\":2023,\"month\":4}");
29+
client().performRequest(request1);
30+
Request request2 =
31+
new Request("PUT", "/" + TEST_INDEX_STATE_COUNTRY_WITH_NULL + "/_doc/8?refresh=true");
32+
request2.setJsonEntity(
33+
"{\"name\":\"\",\"age\":57,\"state\":\"B.C\",\"country\":\"Canada\",\"year\":2023,\"month\":4}");
34+
client().performRequest(request2);
2335
}
2436

2537
@Test
@@ -44,7 +56,15 @@ public void testIsNotNull() {
4456

4557
verifySchema(actual, schema("name", "string"));
4658

47-
verifyDataRows(actual, rows("John"), rows("Jane"), rows("Jake"), rows("Hello"), rows("Kevin"));
59+
verifyDataRows(
60+
actual,
61+
rows("John"),
62+
rows("Jane"),
63+
rows("Jake"),
64+
rows("Hello"),
65+
rows("Kevin"),
66+
rows(" "),
67+
rows(""));
4868
}
4969

5070
@Test
@@ -64,7 +84,9 @@ public void testNullIf() {
6484
rows(null, 10),
6585
rows("Jake", 70),
6686
rows("Kevin", null),
67-
rows("Hello", 30));
87+
rows("Hello", 30),
88+
rows(" ", 27),
89+
rows("", 57));
6890
}
6991

7092
@Test
@@ -85,7 +107,9 @@ public void testNullIfWithExpression() {
85107
rows(null, null),
86108
rows("Jake", "HJake"),
87109
rows("Kevin", "HKevin"),
88-
rows("Hello", null));
110+
rows("Hello", null),
111+
rows(" ", "H "),
112+
rows("", "H"));
89113
}
90114

91115
@Test
@@ -105,7 +129,9 @@ public void testIfNull() {
105129
rows("Unknown", 10),
106130
rows("Jake", 70),
107131
rows("Kevin", null),
108-
rows("Hello", 30));
132+
rows("Hello", 30),
133+
rows(" ", 27),
134+
rows("", 57));
109135
}
110136

111137
@Test
@@ -125,7 +151,9 @@ public void testIf() {
125151
rows("young", 20),
126152
rows("young", 10),
127153
rows("old", 70),
128-
rows("young", 30));
154+
rows("young", 30),
155+
rows("young", 27),
156+
rows("old", 57));
129157
}
130158

131159
@Test
@@ -141,4 +169,51 @@ public void testIfWithLike() {
141169
verifyDataRows(
142170
actual, rows(0.0, "John"), rows(0.0, "Jane"), rows(0.0, "Jake"), rows(1.0, "Hello"));
143171
}
172+
173+
@Test
174+
public void testIsPresent() throws IOException {
175+
JSONObject actual =
176+
executeQuery(
177+
String.format(
178+
"source=%s | where ispresent(name) | fields name, age",
179+
TEST_INDEX_STATE_COUNTRY_WITH_NULL));
180+
181+
verifySchema(actual, schema("name", "string"), schema("age", "integer"));
182+
183+
verifyDataRows(
184+
actual,
185+
rows("Jake", 70),
186+
rows("Hello", 30),
187+
rows("John", 25),
188+
rows("Jane", 20),
189+
rows("Kevin", null),
190+
rows(" ", 27),
191+
rows("", 57));
192+
}
193+
194+
@Test
195+
public void testIsEmpty() throws IOException {
196+
JSONObject actual =
197+
executeQuery(
198+
String.format(
199+
"source=%s | where isempty(name) | fields name, age",
200+
TEST_INDEX_STATE_COUNTRY_WITH_NULL));
201+
202+
verifySchema(actual, schema("name", "string"), schema("age", "integer"));
203+
204+
verifyDataRows(actual, rows(null, 10), rows("", 57));
205+
}
206+
207+
@Test
208+
public void testIsBlank() throws IOException {
209+
JSONObject actual =
210+
executeQuery(
211+
String.format(
212+
"source=%s | where isblank(name) | fields name, age",
213+
TEST_INDEX_STATE_COUNTRY_WITH_NULL));
214+
215+
verifySchema(actual, schema("name", "string"), schema("age", "integer"));
216+
217+
verifyDataRows(actual, rows(null, 10), rows(" ", 27), rows("", 57));
218+
}
144219
}

‎ppl/src/main/antlr/OpenSearchPPLLexer.g4‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -366,6 +366,9 @@ ISNULL: 'ISNULL';
366366
ISNOTNULL: 'ISNOTNULL';
367367
CIDRMATCH: 'CIDRMATCH';
368368
BETWEEN: 'BETWEEN';
369+
ISPRESENT: 'ISPRESENT';
370+
ISEMPTY: 'ISEMPTY';
371+
ISBLANK: 'ISBLANK';
369372

370373
// JSON FUNCTIONS
371374
JSON_VALID: 'JSON_VALID';

‎ppl/src/main/antlr/OpenSearchPPLParser.g4‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -834,6 +834,9 @@ conditionFunctionName
834834
| ISNOTNULL
835835
| CIDRMATCH
836836
| JSON_VALID
837+
| ISPRESENT
838+
| ISEMPTY
839+
| ISBLANK
837840
;
838841

839842
// flow control function return non-boolean value

0 commit comments

Comments
 (0)