Skip to content

Commit 35f5ff3

Browse files
yuancuyuancu
committed
Merge origin/main into pushdown-limit
Signed-off-by: Yuanchun Shen <yuanchu@amazon.com>
1 parent f984d85 commit 35f5ff3

26 files changed

Lines changed: 413 additions & 49 deletions

File tree

‎common/src/main/java/org/opensearch/sql/common/setting/Settings.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ public enum Key {
3131
CALCITE_ENGINE_ENABLED("plugins.calcite.enabled"),
3232
CALCITE_FALLBACK_ALLOWED("plugins.calcite.fallback.allowed"),
3333
CALCITE_PUSHDOWN_ENABLED("plugins.calcite.pushdown.enabled"),
34+
CALCITE_PUSHDOWN_ROWCOUNT_ESTIMATION_FACTOR(
35+
"plugins.calcite.pushdown.rowcount.estimation.factor"),
3436

3537
/** Query Settings. */
3638
FIELD_TYPE_TOLERANCE("plugins.query.field_type_tolerance"),

‎core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,11 @@ public enum BuiltinFunctionName {
138138
/** IP Functions. */
139139
CIDRMATCH(FunctionName.of("cidrmatch")),
140140

141+
/** Cryptographic Functions. */
142+
MD5(FunctionName.of("md5")),
143+
SHA1(FunctionName.of("sha1")),
144+
SHA2(FunctionName.of("sha2")),
145+
141146
/** Arithmetic Operators. */
142147
ADD(FunctionName.of("+")),
143148
ADDFUNCTION(FunctionName.of("add")),

‎core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,13 @@
1616
import org.apache.calcite.sql.SqlOperator;
1717
import org.apache.calcite.sql.util.ReflectiveSqlOperatorTable;
1818
import org.apache.calcite.util.BuiltInMethod;
19+
import org.opensearch.sql.expression.function.udf.CryptographicFunction;
1920

2021
/** Defines functions and operators that are implemented only by PPL */
2122
public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable {
2223

2324
public static final SqlOperator SPAN = new SpanFunctionImpl().toUDF("SPAN");
25+
public static final SqlOperator SHA2 = CryptographicFunction.sha2().toUDF("SHA2");
2426

2527
/**
2628
* Invoking an implementor registered in {@link RexImpTable}, need to use reflection since they're

‎core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
import static org.opensearch.sql.expression.function.BuiltinFunctionName.LOWER;
4444
import static org.opensearch.sql.expression.function.BuiltinFunctionName.LTE;
4545
import static org.opensearch.sql.expression.function.BuiltinFunctionName.LTRIM;
46+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MD5;
4647
import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTIPLY;
4748
import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOT;
4849
import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOTEQUAL;
@@ -58,6 +59,8 @@
5859
import static org.opensearch.sql.expression.function.BuiltinFunctionName.RIGHT;
5960
import static org.opensearch.sql.expression.function.BuiltinFunctionName.ROUND;
6061
import static org.opensearch.sql.expression.function.BuiltinFunctionName.RTRIM;
62+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.SHA1;
63+
import static org.opensearch.sql.expression.function.BuiltinFunctionName.SHA2;
6164
import static org.opensearch.sql.expression.function.BuiltinFunctionName.SIGN;
6265
import static org.opensearch.sql.expression.function.BuiltinFunctionName.SIN;
6366
import static org.opensearch.sql.expression.function.BuiltinFunctionName.SPAN;
@@ -270,11 +273,14 @@ void populate() {
270273
registerOperator(RIGHT, SqlLibraryOperators.RIGHT);
271274
registerOperator(LEFT, SqlLibraryOperators.LEFT);
272275
registerOperator(LOG2, SqlLibraryOperators.LOG2);
276+
registerOperator(MD5, SqlLibraryOperators.MD5);
277+
registerOperator(SHA1, SqlLibraryOperators.SHA1);
273278
registerOperator(INTERNAL_REGEXP_EXTRACT, SqlLibraryOperators.REGEXP_EXTRACT);
274279
registerOperator(INTERNAL_REGEXP_REPLACE_2, SqlLibraryOperators.REGEXP_REPLACE_2);
275280

276281
// Register PPL UDF operator
277282
registerOperator(SPAN, PPLBuiltinOperators.SPAN);
283+
registerOperator(SHA2, PPLBuiltinOperators.SHA2);
278284

279285
// Register implementation.
280286
// Note, make the implementation an individual class if too complex.

‎core/src/main/java/org/opensearch/sql/expression/function/udf/CryptographicFunction.java‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.expression.function.udf;
7+
8+
import java.util.List;
9+
import org.apache.calcite.adapter.enumerable.NotNullImplementor;
10+
import org.apache.calcite.adapter.enumerable.NullPolicy;
11+
import org.apache.calcite.adapter.enumerable.RexToLixTranslator;
12+
import org.apache.calcite.linq4j.tree.Expression;
13+
import org.apache.calcite.linq4j.tree.Expressions;
14+
import org.apache.calcite.rex.RexCall;
15+
import org.apache.calcite.sql.type.ReturnTypes;
16+
import org.apache.calcite.sql.type.SqlReturnTypeInference;
17+
import org.apache.calcite.sql.type.SqlTypeTransforms;
18+
import org.apache.commons.codec.binary.Hex;
19+
import org.apache.commons.codec.digest.DigestUtils;
20+
import org.apache.commons.codec.digest.MessageDigestAlgorithms;
21+
import org.opensearch.sql.expression.function.ImplementorUDF;
22+
23+
public class CryptographicFunction extends ImplementorUDF {
24+
private CryptographicFunction(NotNullImplementor implementor, NullPolicy nullPolicy) {
25+
super(implementor, nullPolicy);
26+
}
27+
28+
public static CryptographicFunction sha2() {
29+
return new CryptographicFunction(new Sha2Implementor(), NullPolicy.ANY);
30+
}
31+
32+
@Override
33+
public SqlReturnTypeInference getReturnTypeInference() {
34+
return ReturnTypes.VARCHAR.andThen(SqlTypeTransforms.FORCE_NULLABLE);
35+
}
36+
37+
public static class Sha2Implementor implements NotNullImplementor {
38+
@Override
39+
public Expression implement(
40+
RexToLixTranslator translator, RexCall call, List<Expression> translatedOperands) {
41+
return Expressions.call(Sha2Implementor.class, "getDigest", translatedOperands);
42+
}
43+
44+
public static String getDigest(String input, int algorithm) {
45+
return switch (algorithm) {
46+
case 224 -> Hex.encodeHexString(
47+
DigestUtils.getDigest(MessageDigestAlgorithms.SHA_224).digest(input.getBytes()));
48+
case 256 -> DigestUtils.sha256Hex(input);
49+
case 384 -> DigestUtils.sha384Hex(input);
50+
case 512 -> DigestUtils.sha512Hex(input);
51+
default -> throw new IllegalArgumentException(
52+
String.format(
53+
"Unsupported SHA2 algorithm: %d. Only 224, 256, 384, and 512 are supported.",
54+
algorithm));
55+
};
56+
}
57+
}
58+
}

‎docs/user/admin/settings.rst‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -757,3 +757,39 @@ This setting is present from 3.0.0-beta. You can enable Calcite as new query opt
757757

758758
Check `introduce v3 engine <../../../dev/intro-v3-engine.md>`_ for more details.
759759
Check `join doc <../../ppl/cmd/join.rst>`_ for example.
760+
761+
plugins.calcite.fallback.allowed
762+
=======================
763+
764+
Description
765+
-----------
766+
767+
This setting is present from 3.0.0-beta. If Calcite is enabled, you can use this setting to decide whether to allow fallback to v2 engine for some queries which are not supported by v3 engine.
768+
769+
1. The default value is true in 3.0.0-beta.
770+
2. This setting is node scope.
771+
3. This setting can be updated dynamically.
772+
773+
plugins.calcite.pushdown.enabled
774+
=======================
775+
776+
Description
777+
-----------
778+
779+
This setting is present from 3.0.0-beta. If Calcite is enabled, you can use this setting to decide whether to enable the operator pushdown optimization for v3 engine.
780+
781+
1. The default value is true in 3.0.0-beta.
782+
2. This setting is node scope.
783+
3. This setting can be updated dynamically.
784+
785+
plugins.calcite.pushdown.rowcount.estimation.factor
786+
=======================
787+
788+
Description
789+
-----------
790+
791+
This setting is present from 3.1.0. If Calcite pushdown optimization is enabled, this setting is used to estimate the row count of the query plan. The value is a factor to multiply the row count of the table scan to get the estimated row count.
792+
793+
1. The default value is 0.9 in 3.1.0.
794+
2. This setting is node scope.
795+
3. This setting can be updated dynamically.

‎docs/user/ppl/functions/cryptographic.rst‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
===========================
2+
PPL Cryptographic Functions
3+
===========================
4+
5+
.. rubric:: Table of contents
6+
7+
.. contents::
8+
:local:
9+
:depth: 1
10+
11+
MD5
12+
---
13+
14+
Description
15+
>>>>>>>>>>>
16+
17+
18+
Usage: ``md5(str)`` calculates the MD5 digest and returns the value as a 32 character hex string.
19+
20+
Argument type: STRING
21+
22+
Return type: STRING
23+
24+
Example::
25+
26+
os> source=people | eval `MD5('hello')` = MD5('hello') | fields `MD5('hello')`
27+
fetched rows / total rows = 1/1
28+
+----------------------------------+
29+
| MD5('hello') |
30+
|----------------------------------|
31+
| 5d41402abc4b2a76b9719d911017c592 |
32+
+----------------------------------+
33+
34+
SHA1
35+
----
36+
37+
Description
38+
>>>>>>>>>>>
39+
40+
Usage: ``sha1(str)`` returns the hex string result of SHA-1.
41+
42+
Argument type: STRING
43+
44+
Return type: STRING
45+
46+
Example::
47+
48+
os> source=people | eval `SHA1('hello')` = SHA1('hello') | fields `SHA1('hello')`
49+
fetched rows / total rows = 1/1
50+
+------------------------------------------+
51+
| SHA1('hello') |
52+
|------------------------------------------|
53+
| aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d |
54+
+------------------------------------------+
55+
56+
SHA2
57+
----
58+
59+
Description
60+
>>>>>>>>>>>
61+
62+
Usage: ``sha2(str, numBits)`` returns the hex string result of SHA-2 family of hash functions (SHA-224, SHA-256, SHA-384, and SHA-512).
63+
The numBits indicates the desired bit length of the result, which must have a value of 224, 256, 384, or 512.
64+
65+
Argument type: STRING, INTEGER
66+
67+
Return type: STRING
68+
69+
Example::
70+
71+
os> source=people | eval `SHA2('hello',256)` = SHA2('hello',256) | fields `SHA2('hello',256)`
72+
fetched rows / total rows = 1/1
73+
+------------------------------------------------------------------+
74+
| SHA2('hello',256) |
75+
|------------------------------------------------------------------|
76+
| 2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 |
77+
+------------------------------------------------------------------+
78+
79+
os> source=people | eval `SHA2('hello',512)` = SHA2('hello',512) | fields `SHA2('hello',512)`
80+
fetched rows / total rows = 1/1
81+
+----------------------------------------------------------------------------------------------------------------------------------+
82+
| SHA2('hello',512) |
83+
|----------------------------------------------------------------------------------------------------------------------------------|
84+
| 9b71d224bd62f3785d96d46ad3ea3d73319bfbc2890caadae2dff72519673ca72323c3d99ba5c11d7c7acc6e14b8c5da0c4663475c2e5c3adef46f73bcdec043 |
85+
+----------------------------------------------------------------------------------------------------------------------------------+

‎integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLAggregationIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -621,12 +621,12 @@ public void testSumGroupByNullValue() throws IOException {
621621
verifySchema(response, schema("a", null, "long"), schema("age", null, "integer"));
622622
verifyDataRows(
623623
response,
624-
rows(null, null),
624+
rows(isPushdownEnabled() ? 0 : null, null),
625625
rows(32838, 28),
626626
rows(39225, 32),
627627
rows(4180, 33),
628628
rows(48086, 34),
629-
rows(null, 36));
629+
rows(isPushdownEnabled() ? 0 : null, 36));
630630
}
631631

632632
@Test

‎integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLCryptographicFunctionIT.java‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.sql.calcite.standalone;
7+
8+
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STATE_COUNTRY;
9+
import static org.opensearch.sql.util.MatcherUtils.rows;
10+
import static org.opensearch.sql.util.MatcherUtils.schema;
11+
import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
12+
import static org.opensearch.sql.util.MatcherUtils.verifyErrorMessageContains;
13+
import static org.opensearch.sql.util.MatcherUtils.verifySchema;
14+
15+
import java.io.IOException;
16+
import org.json.JSONObject;
17+
import org.junit.jupiter.api.Test;
18+
19+
public class CalcitePPLCryptographicFunctionIT extends CalcitePPLIntegTestCase {
20+
@Override
21+
public void init() throws IOException {
22+
super.init();
23+
loadIndex(Index.STATE_COUNTRY);
24+
}
25+
26+
@Test
27+
public void testMd5() {
28+
JSONObject actual =
29+
executeQuery(
30+
String.format(
31+
"source=%s | where name = 'Jake' | eval hello = MD5('hello'), california ="
32+
+ " md5(state) | fields hello, california",
33+
TEST_INDEX_STATE_COUNTRY));
34+
verifySchema(actual, schema("hello", "string"), schema("california", "string"));
35+
verifyDataRows(
36+
actual, rows("5d41402abc4b2a76b9719d911017c592", "356779a9a1696714480f57fa3fb66d4c"));
37+
}
38+
39+
@Test
40+
public void testSha1() {
41+
JSONObject actual =
42+
executeQuery(
43+
String.format(
44+
"source=%s | where name = 'John' | eval hello = SHA1('hello'), ontario ="
45+
+ " SHA1(state) | fields hello, ontario",
46+
TEST_INDEX_STATE_COUNTRY));
47+
verifySchema(actual, schema("hello", "string"), schema("ontario", "string"));
48+
verifyDataRows(
49+
actual,
50+
rows(
51+
"aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d",
52+
"f9f742e1f653a74c4cd78d7ea283b5556539b96b"));
53+
}
54+
55+
@Test
56+
public void testSha2() {
57+
JSONObject actual =
58+
executeQuery(
59+
String.format(
60+
"source=%s | where name = 'Jane' | eval sha256 = SHA2('hello',256), sha512 ="
61+
+ " SHA2('hello',512), sha224 = SHA2(country, 224), sha384 = SHA2(country, 384)"
62+
+ " | fields sha256, sha512, sha224, sha384",
63+
TEST_INDEX_STATE_COUNTRY));
64+
verifySchema(
65+
actual,
66+
schema("sha256", "string"),
67+
schema("sha512", "string"),
68+
schema("sha224", "string"),
69+
schema("sha384", "string"));
70+
verifyDataRows(
71+
actual,
72+
rows(
73+
"2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824",
74+
"9b71d224bd62f3785d96d46ad3ea3d73319bfbc2890caadae2dff72519673ca72323c3d99ba5c11d7c7acc6e14b8c5da0c4663475c2e5c3adef46f73bcdec043",
75+
"c16f747ca3d2e267c76e7355429fb1583268d966887f237b8e1605c7",
76+
"de2abcb28b87d681830f3af25cd8dde7fdc2a4da9dcfde60b371fd2378a70ac39cef3e104bbe09aecda022aee7b4bf59"));
77+
}
78+
79+
@Test
80+
public void testSha2WrongAlgorithmShouldThrow() {
81+
Throwable e =
82+
assertThrows(
83+
IllegalArgumentException.class,
84+
() ->
85+
executeQuery(
86+
String.format(
87+
"source=%s | head 1 | eval sha100 = SHA2('hello', 100) | fields sha100",
88+
TEST_INDEX_STATE_COUNTRY)));
89+
verifyErrorMessageContains(e, "Unsupported SHA2 algorithm");
90+
}
91+
}

‎integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLExplainIT.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,8 @@ public void testExplainCommandCost() {
6060
String result = explainQuery("explain cost source=test | where age = 20 | fields name, age");
6161
assertTrue(
6262
result.contains(
63-
"CalciteEnumerableIndexScan(table=[[OpenSearch, test]]): rowcount = 100.0, cumulative"
64-
+ " cost = {100.0 rows, 101.0 cpu, 0.0 io}"));
63+
"CalciteEnumerableIndexScan(table=[[OpenSearch, test]]): rowcount = 10000.0, cumulative"
64+
+ " cost = {10000.0 rows, 10001.0 cpu, 0.0 io}"));
6565
}
6666

6767
@Test

0 commit comments

Comments
 (0)