Merge remote-tracking branch 'origin/main' into issues/4576

Signed-off-by: Yuanchun Shen <yuanchu@amazon.com>

Author: yuancu

core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java

@@ -418,6 +418,15 @@ public RexNode visitFunction(Function node, CalcitePlanContext context) {
       }
     }
 
+    if ("LIKE".equalsIgnoreCase(node.getFuncName()) && arguments.size() == 2) {
+      RexNode defaultCaseSensitive =
+          CalcitePlanContext.isLegacyPreferred()
+              ? context.rexBuilder.makeLiteral(false)
+              : context.rexBuilder.makeLiteral(true);
+      arguments = new ArrayList<>(arguments);
+      arguments.add(defaultCaseSensitive);
+    }
+
     RexNode resolvedNode =
         PPLFuncImpTable.INSTANCE.resolve(
             context.rexBuilder, node.getFuncName(), arguments.toArray(new RexNode[0]));

core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java

@@ -75,6 +75,7 @@ public enum BuiltinFunctionName {
   MVAPPEND(FunctionName.of("mvappend")),
   MVJOIN(FunctionName.of("mvjoin")),
   MVINDEX(FunctionName.of("mvindex")),
+  MVDEDUP(FunctionName.of("mvdedup")),
   FORALL(FunctionName.of("forall")),
   EXISTS(FunctionName.of("exists")),
   FILTER(FunctionName.of("filter")),
@@ -192,6 +193,7 @@ public enum BuiltinFunctionName {
   GREATER(FunctionName.of(">")),
   GTE(FunctionName.of(">=")),
   LIKE(FunctionName.of("like")),
+  ILIKE(FunctionName.of("ilike")),
   NOT_LIKE(FunctionName.of("not like")),
 
   /** Aggregation Function. */

core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java

@@ -79,6 +79,7 @@
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.HOUR_OF_DAY;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.IF;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.IFNULL;
+import static org.opensearch.sql.expression.function.BuiltinFunctionName.ILIKE;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_GROK;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_ITEM;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_PARSE;
@@ -150,6 +151,7 @@
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTIPLYFUNCTION;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.MULTI_MATCH;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.MVAPPEND;
+import static org.opensearch.sql.expression.function.BuiltinFunctionName.MVDEDUP;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.MVINDEX;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.MVJOIN;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.NOT;
@@ -326,6 +328,18 @@ default RexNode resolve(RexBuilder builder, RexNode... args) {
     }
   }
 
+  public interface FunctionImp3 extends FunctionImp {
+    RexNode resolve(RexBuilder builder, RexNode arg1, RexNode arg2, RexNode arg3);
+
+    @Override
+    default RexNode resolve(RexBuilder builder, RexNode... args) {
+      if (args.length != 3) {
+        throw new IllegalArgumentException("This function requires exactly 3 arguments");
+      }
+      return resolve(builder, args[0], args[1], args[2]);
+    }
+  }
+
   /** The singleton instance. */
   public static final PPLFuncImpTable INSTANCE;
 
@@ -989,6 +1003,7 @@ void populate() {
 
       registerOperator(ARRAY, PPLBuiltinOperators.ARRAY);
       registerOperator(MVAPPEND, PPLBuiltinOperators.MVAPPEND);
+      registerOperator(MVDEDUP, SqlLibraryOperators.ARRAY_DISTINCT);
       registerOperator(MAP_APPEND, PPLBuiltinOperators.MAP_APPEND);
       registerOperator(MAP_CONCAT, SqlLibraryOperators.MAP_CONCAT);
       registerOperator(MAP_REMOVE, PPLBuiltinOperators.MAP_REMOVE);
@@ -1214,17 +1229,22 @@ void populate() {
                               arg))),
           PPLTypeChecker.family(SqlTypeFamily.ANY));
       register(
-          LIKE,
+          ILIKE,
           (FunctionImp2)
               (builder, arg1, arg2) ->
                   builder.makeCall(
-                      SqlLibraryOperators.ILIKE,
-                      arg1,
-                      arg2,
-                      // TODO: Figure out escaping solution. '\\' is used for JSON input but is not
-                      // necessary for SQL function input
-                      builder.makeLiteral("\\")),
+                      SqlLibraryOperators.ILIKE, arg1, arg2, builder.makeLiteral("\\")),
           PPLTypeChecker.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING));
+      register(
+          LIKE,
+          (FunctionImp3)
+              (builder, arg1, arg2, arg3) ->
+                  ((RexLiteral) arg3).getValueAs(Boolean.class)
+                      ? builder.makeCall(
+                          SqlStdOperatorTable.LIKE, arg1, arg2, builder.makeLiteral("\\"))
+                      : builder.makeCall(
+                          SqlLibraryOperators.ILIKE, arg1, arg2, builder.makeLiteral("\\")),
+          PPLTypeChecker.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING, SqlTypeFamily.BOOLEAN));
     }
   }
 

core/src/main/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperators.java

@@ -387,7 +387,8 @@ private static DefaultFunctionResolver gte() {
   private static DefaultFunctionResolver like() {
     return define(
         BuiltinFunctionName.LIKE.getName(),
-        impl(nullMissingHandling(OperatorUtils::matches), BOOLEAN, STRING, STRING));
+        impl(nullMissingHandling(OperatorUtils::matches2), BOOLEAN, STRING, STRING),
+        impl(nullMissingHandling(OperatorUtils::matches3), BOOLEAN, STRING, STRING, BOOLEAN));
   }
 
   private static DefaultFunctionResolver regexp() {
@@ -401,10 +402,17 @@ private static DefaultFunctionResolver notLike() {
         BuiltinFunctionName.NOT_LIKE.getName(),
         impl(
             nullMissingHandling(
-                (v1, v2) -> UnaryPredicateOperators.not(OperatorUtils.matches(v1, v2))),
+                (v1, v2) -> UnaryPredicateOperators.not(OperatorUtils.matches2(v1, v2))),
             BOOLEAN,
             STRING,
-            STRING));
+            STRING),
+        impl(
+            nullMissingHandling(
+                (v1, v2, v3) -> UnaryPredicateOperators.not(OperatorUtils.matches3(v1, v2, v3))),
+            BOOLEAN,
+            STRING,
+            STRING,
+            BOOLEAN));
   }
 
   private static ExprValue lookupTableFunction(

core/src/main/java/org/opensearch/sql/utils/OperatorUtils.java

@@ -21,13 +21,31 @@ public class OperatorUtils {
    * @param pattern string pattern to match.
    * @return if text matches pattern returns true; else return false.
    */
-  public static ExprBooleanValue matches(ExprValue text, ExprValue pattern) {
+  public static ExprBooleanValue matches2(ExprValue text, ExprValue pattern) {
     return ExprBooleanValue.of(
         Pattern.compile(patternToRegex(pattern.stringValue()), Pattern.CASE_INSENSITIVE)
             .matcher(text.stringValue())
             .matches());
   }
 
+  /**
+   * Wildcard pattern matcher util.<br>
+   * Percent (%) character for wildcard,<br>
+   * Underscore (_) character for a single character match.
+   *
+   * @param pattern string pattern to match.
+   * @param caseSensitive indicate the case sensitivity of the pattern.
+   * @return if text matches pattern returns true; else return false.
+   */
+  public static ExprBooleanValue matches3(
+      ExprValue text, ExprValue pattern, ExprValue caseSensitive) {
+    Pattern p =
+        caseSensitive.booleanValue()
+            ? Pattern.compile(patternToRegex(pattern.stringValue()))
+            : Pattern.compile(patternToRegex(pattern.stringValue()), Pattern.CASE_INSENSITIVE);
+    return ExprBooleanValue.of(p.matcher(text.stringValue()).matches());
+  }
+
   /**
    * Checks if text matches regular expression pattern.
    *

core/src/test/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java

@@ -22,7 +22,7 @@
 import static org.opensearch.sql.data.type.ExprCoreType.STRUCT;
 import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP;
 import static org.opensearch.sql.utils.ComparisonUtil.compare;
-import static org.opensearch.sql.utils.OperatorUtils.matches;
+import static org.opensearch.sql.utils.OperatorUtils.matches2;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -554,7 +554,7 @@ public void test_gte(ExprValue v1, ExprValue v2) {
   public void test_like(ExprValue v1, ExprValue v2) {
     FunctionExpression like = DSL.like(DSL.literal(v1), DSL.literal(v2));
     assertEquals(BOOLEAN, like.type());
-    assertEquals(matches(v1, v2), like.valueOf(valueEnv()));
+    assertEquals(matches2(v1, v2), like.valueOf(valueEnv()));
     assertEquals(String.format("like(%s, %s)", v1.toString(), v2.toString()), like.toString());
   }
 

docs/user/ppl/functions/collection.rst

@@ -302,6 +302,44 @@ Example::
     | [1,text,2.5] |
     +--------------+
 
+MVDEDUP
+-------
+
+Description
+>>>>>>>>>>>
+
+Usage: mvdedup(array) removes duplicate values from a multivalue array while preserving the order of first occurrence. NULL elements are filtered out. Returns an array with duplicates removed, or null if the input is null.
+
+Argument type: array: ARRAY
+
+Return type: ARRAY
+
+Example::
+
+    os> source=people | eval array = array(1, 2, 2, 3, 1, 4), result = mvdedup(array) | fields result | head 1
+    fetched rows / total rows = 1/1
+    +-----------+
+    | result    |
+    |-----------|
+    | [1,2,3,4] |
+    +-----------+
+
+    os> source=people | eval array = array('z', 'a', 'z', 'b', 'a', 'c'), result = mvdedup(array) | fields result | head 1
+    fetched rows / total rows = 1/1
+    +-----------+
+    | result    |
+    |-----------|
+    | [z,a,b,c] |
+    +-----------+
+
+    os> source=people | eval array = array(), result = mvdedup(array) | fields result | head 1
+    fetched rows / total rows = 1/1
+    +--------+
+    | result |
+    |--------|
+    | []     |
+    +--------+
+
 MVINDEX
 -------
 

docs/user/ppl/functions/string.rst

@@ -80,33 +80,69 @@ Example::
     | 10                   |
     +----------------------+
 
-
 LIKE
 ----
 
 Description
 >>>>>>>>>>>
 
-Usage: like(string, PATTERN) return true if the string match the PATTERN, PATTERN is case insensitive.
+Usage: like(string, PATTERN[, case_sensitive]) return true if the string match the PATTERN. ``case_sensitive`` is optional. When set to ``true``, PATTERN is **case-sensitive**. **Default:** Determined by ``plugins.ppl.syntax.legacy.preferred``.
+
+ * When ``plugins.ppl.syntax.legacy.preferred=true``, ``case_sensitive`` defaults to ``false``
+ * When ``plugins.ppl.syntax.legacy.preferred=false``, ``case_sensitive`` defaults to ``true``
 
 There are two wildcards often used in conjunction with the LIKE operator:
 
 * ``%`` - The percent sign represents zero, one, or multiple characters
 * ``_`` - The underscore represents a single character
 
+Argument type: STRING, STRING [, BOOLEAN]
+
+Return type: INTEGER
+
 Example::
 
-    os> source=people | eval `LIKE('hello world', '_ello%')` = LIKE('hello world', '_ELLO%') | fields `LIKE('hello world', '_ello%')`
+    os> source=people | eval `LIKE('hello world', '_ello%')` = LIKE('hello world', '_ello%'), `LIKE('hello world', '_ELLo%', true)` = LIKE('hello world', '_ELLo%', true), `LIKE('hello world', '_ELLo%', false)` = LIKE('hello world', '_ELLo%', false) | fields `LIKE('hello world', '_ello%')`, `LIKE('hello world', '_ELLo%', true)`, `LIKE('hello world', '_ELLo%', false)`
     fetched rows / total rows = 1/1
-    +-------------------------------+
-    | LIKE('hello world', '_ello%') |
-    |-------------------------------|
-    | True                          |
-    +-------------------------------+
+    +-------------------------------+-------------------------------------+--------------------------------------+
+    | LIKE('hello world', '_ello%') | LIKE('hello world', '_ELLo%', true) | LIKE('hello world', '_ELLo%', false) |
+    |-------------------------------+-------------------------------------+--------------------------------------|
+    | True                          | False                               | True                                 |
+    +-------------------------------+-------------------------------------+--------------------------------------+
 
 
 Limitation: The pushdown of the LIKE function to a DSL wildcard query is supported only for keyword fields.
 
+ILIKE
+----
+
+Description
+>>>>>>>>>>>
+
+Usage: ilike(string, PATTERN) return true if the string match the PATTERN, PATTERN is **case-insensitive**.
+
+There are two wildcards often used in conjunction with the ILIKE operator:
+
+* ``%`` - The percent sign represents zero, one, or multiple characters
+* ``_`` - The underscore represents a single character
+
+Argument type: STRING, STRING
+
+Return type: INTEGER
+
+Example::
+
+    os> source=people | eval `ILIKE('hello world', '_ELLo%')` = ILIKE('hello world', '_ELLo%') | fields `ILIKE('hello world', '_ELLo%')`
+    fetched rows / total rows = 1/1
+    +--------------------------------+
+    | ILIKE('hello world', '_ELLo%') |
+    |--------------------------------|
+    | True                           |
+    +--------------------------------+
+
+
+Limitation: The pushdown of the ILIKE function to a DSL wildcard query is supported only for keyword fields.
+
 LOCATE
 -------
 

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteArrayFunctionIT.java

@@ -489,4 +489,82 @@ public void testMvindexRangeSingleElement() throws IOException {
     verifySchema(actual, schema("result", "array"));
     verifyDataRows(actual, rows(List.of(3)));
   }
+
+  @Test
+  public void testMvdedupWithDuplicates() throws IOException {
+    JSONObject actual =
+        executeQuery(
+            String.format(
+                "source=%s | eval arr = array(1, 2, 2, 3, 1, 4), result = mvdedup(arr) | head 1 |"
+                    + " fields result",
+                TEST_INDEX_BANK));
+
+    verifySchema(actual, schema("result", "array"));
+    verifyDataRows(actual, rows(List.of(1, 2, 3, 4)));
+  }
+
+  @Test
+  public void testMvdedupWithNoDuplicates() throws IOException {
+    JSONObject actual =
+        executeQuery(
+            String.format(
+                "source=%s | eval arr = array(1, 2, 3, 4), result = mvdedup(arr) | head 1 |"
+                    + " fields result",
+                TEST_INDEX_BANK));
+
+    verifySchema(actual, schema("result", "array"));
+    verifyDataRows(actual, rows(List.of(1, 2, 3, 4)));
+  }
+
+  @Test
+  public void testMvdedupWithAllDuplicates() throws IOException {
+    JSONObject actual =
+        executeQuery(
+            String.format(
+                "source=%s | eval arr = array(5, 5, 5, 5), result = mvdedup(arr) | head 1 |"
+                    + " fields result",
+                TEST_INDEX_BANK));
+
+    verifySchema(actual, schema("result", "array"));
+    verifyDataRows(actual, rows(List.of(5)));
+  }
+
+  @Test
+  public void testMvdedupWithEmptyArray() throws IOException {
+    JSONObject actual =
+        executeQuery(
+            String.format(
+                "source=%s | eval arr = array(), result = mvdedup(arr) | head 1 | fields result",
+                TEST_INDEX_BANK));
+
+    verifySchema(actual, schema("result", "array"));
+    verifyDataRows(actual, rows(List.of()));
+  }
+
+  @Test
+  public void testMvdedupWithStrings() throws IOException {
+    JSONObject actual =
+        executeQuery(
+            String.format(
+                "source=%s | eval arr = array('apple', 'banana', 'apple', 'cherry', 'banana'),"
+                    + " result = mvdedup(arr) | head 1 | fields result",
+                TEST_INDEX_BANK));
+
+    verifySchema(actual, schema("result", "array"));
+    verifyDataRows(actual, rows(List.of("apple", "banana", "cherry")));
+  }
+
+  @Test
+  public void testMvdedupPreservesOrder() throws IOException {
+    JSONObject actual =
+        executeQuery(
+            String.format(
+                "source=%s | eval arr = array('z', 'a', 'z', 'b', 'a', 'c'), result ="
+                    + " mvdedup(arr) | head 1 | fields result",
+                TEST_INDEX_BANK));
+
+    verifySchema(actual, schema("result", "array"));
+    // Should preserve first occurrence order: z, a, b, c
+    verifyDataRows(actual, rows(List.of("z", "a", "b", "c")));
+  }
 }