Update syntax: like(string, PATTERN[, case_sensitive]) (opensearch-project#4837) (opensearch-project#4864)

* Change like function to case-sensitive and add ilike for case-insensitive



* change plan in clickbench



* fix ut



* fix UT



* Update syntax: like(string, PATTERN[, case_sensitive])



* fix IT



* Fix IT



* revert import merging



* support like(string, PATTERN, bool) in v2



* dedup ut



---------


(cherry picked from commit 164c3f0)

Signed-off-by: Lantao Jin <ltjin@amazon.com>

Author: LantaoJin

core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java

@@ -419,6 +419,15 @@ public RexNode visitFunction(Function node, CalcitePlanContext context) {
       }
     }
 
+    if ("LIKE".equalsIgnoreCase(node.getFuncName()) && arguments.size() == 2) {
+      RexNode defaultCaseSensitive =
+          CalcitePlanContext.isLegacyPreferred()
+              ? context.rexBuilder.makeLiteral(false)
+              : context.rexBuilder.makeLiteral(true);
+      arguments = new ArrayList<>(arguments);
+      arguments.add(defaultCaseSensitive);
+    }
+
     RexNode resolvedNode =
         PPLFuncImpTable.INSTANCE.resolve(
             context.rexBuilder, node.getFuncName(), arguments.toArray(new RexNode[0]));

core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java

@@ -193,6 +193,7 @@ public enum BuiltinFunctionName {
   GREATER(FunctionName.of(">")),
   GTE(FunctionName.of(">=")),
   LIKE(FunctionName.of("like")),
+  ILIKE(FunctionName.of("ilike")),
   NOT_LIKE(FunctionName.of("not like")),
 
   /** Aggregation Function. */

core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java

@@ -79,6 +79,7 @@
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.HOUR_OF_DAY;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.IF;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.IFNULL;
+import static org.opensearch.sql.expression.function.BuiltinFunctionName.ILIKE;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_GROK;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_ITEM;
 import static org.opensearch.sql.expression.function.BuiltinFunctionName.INTERNAL_PARSE;
@@ -332,6 +333,18 @@ default RexNode resolve(RexBuilder builder, RexNode... args) {
     }
   }
 
+  public interface FunctionImp3 extends FunctionImp {
+    RexNode resolve(RexBuilder builder, RexNode arg1, RexNode arg2, RexNode arg3);
+
+    @Override
+    default RexNode resolve(RexBuilder builder, RexNode... args) {
+      if (args.length != 3) {
+        throw new IllegalArgumentException("This function requires exactly 3 arguments");
+      }
+      return resolve(builder, args[0], args[1], args[2]);
+    }
+  }
+
   /** The singleton instance. */
   public static final PPLFuncImpTable INSTANCE;
 
@@ -1225,17 +1238,22 @@ void populate() {
                               arg))),
           PPLTypeChecker.family(SqlTypeFamily.ANY));
       register(
-          LIKE,
+          ILIKE,
           (FunctionImp2)
               (builder, arg1, arg2) ->
                   builder.makeCall(
-                      SqlLibraryOperators.ILIKE,
-                      arg1,
-                      arg2,
-                      // TODO: Figure out escaping solution. '\\' is used for JSON input but is not
-                      // necessary for SQL function input
-                      builder.makeLiteral("\\")),
+                      SqlLibraryOperators.ILIKE, arg1, arg2, builder.makeLiteral("\\")),
           PPLTypeChecker.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING));
+      register(
+          LIKE,
+          (FunctionImp3)
+              (builder, arg1, arg2, arg3) ->
+                  ((RexLiteral) arg3).getValueAs(Boolean.class)
+                      ? builder.makeCall(
+                          SqlStdOperatorTable.LIKE, arg1, arg2, builder.makeLiteral("\\"))
+                      : builder.makeCall(
+                          SqlLibraryOperators.ILIKE, arg1, arg2, builder.makeLiteral("\\")),
+          PPLTypeChecker.family(SqlTypeFamily.STRING, SqlTypeFamily.STRING, SqlTypeFamily.BOOLEAN));
     }
   }
 

core/src/main/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperators.java

@@ -387,7 +387,8 @@ private static DefaultFunctionResolver gte() {
   private static DefaultFunctionResolver like() {
     return define(
         BuiltinFunctionName.LIKE.getName(),
-        impl(nullMissingHandling(OperatorUtils::matches), BOOLEAN, STRING, STRING));
+        impl(nullMissingHandling(OperatorUtils::matches2), BOOLEAN, STRING, STRING),
+        impl(nullMissingHandling(OperatorUtils::matches3), BOOLEAN, STRING, STRING, BOOLEAN));
   }
 
   private static DefaultFunctionResolver regexp() {
@@ -401,10 +402,17 @@ private static DefaultFunctionResolver notLike() {
         BuiltinFunctionName.NOT_LIKE.getName(),
         impl(
             nullMissingHandling(
-                (v1, v2) -> UnaryPredicateOperators.not(OperatorUtils.matches(v1, v2))),
+                (v1, v2) -> UnaryPredicateOperators.not(OperatorUtils.matches2(v1, v2))),
             BOOLEAN,
             STRING,
-            STRING));
+            STRING),
+        impl(
+            nullMissingHandling(
+                (v1, v2, v3) -> UnaryPredicateOperators.not(OperatorUtils.matches3(v1, v2, v3))),
+            BOOLEAN,
+            STRING,
+            STRING,
+            BOOLEAN));
   }
 
   private static ExprValue lookupTableFunction(

core/src/main/java/org/opensearch/sql/utils/OperatorUtils.java

@@ -21,13 +21,31 @@ public class OperatorUtils {
    * @param pattern string pattern to match.
    * @return if text matches pattern returns true; else return false.
    */
-  public static ExprBooleanValue matches(ExprValue text, ExprValue pattern) {
+  public static ExprBooleanValue matches2(ExprValue text, ExprValue pattern) {
     return ExprBooleanValue.of(
         Pattern.compile(patternToRegex(pattern.stringValue()), Pattern.CASE_INSENSITIVE)
             .matcher(text.stringValue())
             .matches());
   }
 
+  /**
+   * Wildcard pattern matcher util.<br>
+   * Percent (%) character for wildcard,<br>
+   * Underscore (_) character for a single character match.
+   *
+   * @param pattern string pattern to match.
+   * @param caseSensitive indicate the case sensitivity of the pattern.
+   * @return if text matches pattern returns true; else return false.
+   */
+  public static ExprBooleanValue matches3(
+      ExprValue text, ExprValue pattern, ExprValue caseSensitive) {
+    Pattern p =
+        caseSensitive.booleanValue()
+            ? Pattern.compile(patternToRegex(pattern.stringValue()))
+            : Pattern.compile(patternToRegex(pattern.stringValue()), Pattern.CASE_INSENSITIVE);
+    return ExprBooleanValue.of(p.matcher(text.stringValue()).matches());
+  }
+
   /**
    * Checks if text matches regular expression pattern.
    *

core/src/test/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java

@@ -23,7 +23,7 @@
 import static org.opensearch.sql.data.type.ExprCoreType.STRUCT;
 import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP;
 import static org.opensearch.sql.utils.ComparisonUtil.compare;
-import static org.opensearch.sql.utils.OperatorUtils.matches;
+import static org.opensearch.sql.utils.OperatorUtils.matches2;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -556,7 +556,7 @@ public void test_gte(ExprValue v1, ExprValue v2) {
   public void test_like(ExprValue v1, ExprValue v2) {
     FunctionExpression like = DSL.like(DSL.literal(v1), DSL.literal(v2));
     assertEquals(BOOLEAN, like.type());
-    assertEquals(matches(v1, v2), like.valueOf(valueEnv()));
+    assertEquals(matches2(v1, v2), like.valueOf(valueEnv()));
     assertEquals(String.format("like(%s, %s)", v1.toString(), v2.toString()), like.toString());
   }
 

docs/user/ppl/functions/string.rst

@@ -80,33 +80,69 @@ Example::
     | 10                   |
     +----------------------+
 
-
 LIKE
 ----
 
 Description
 >>>>>>>>>>>
 
-Usage: like(string, PATTERN) return true if the string match the PATTERN, PATTERN is case insensitive.
+Usage: like(string, PATTERN[, case_sensitive]) return true if the string match the PATTERN. ``case_sensitive`` is optional. When set to ``true``, PATTERN is **case-sensitive**. **Default:** Determined by ``plugins.ppl.syntax.legacy.preferred``.
+
+ * When ``plugins.ppl.syntax.legacy.preferred=true``, ``case_sensitive`` defaults to ``false``
+ * When ``plugins.ppl.syntax.legacy.preferred=false``, ``case_sensitive`` defaults to ``true``
 
 There are two wildcards often used in conjunction with the LIKE operator:
 
 * ``%`` - The percent sign represents zero, one, or multiple characters
 * ``_`` - The underscore represents a single character
 
+Argument type: STRING, STRING [, BOOLEAN]
+
+Return type: INTEGER
+
 Example::
 
-    os> source=people | eval `LIKE('hello world', '_ello%')` = LIKE('hello world', '_ELLO%') | fields `LIKE('hello world', '_ello%')`
+    os> source=people | eval `LIKE('hello world', '_ello%')` = LIKE('hello world', '_ello%'), `LIKE('hello world', '_ELLo%', true)` = LIKE('hello world', '_ELLo%', true), `LIKE('hello world', '_ELLo%', false)` = LIKE('hello world', '_ELLo%', false) | fields `LIKE('hello world', '_ello%')`, `LIKE('hello world', '_ELLo%', true)`, `LIKE('hello world', '_ELLo%', false)`
     fetched rows / total rows = 1/1
-    +-------------------------------+
-    | LIKE('hello world', '_ello%') |
-    |-------------------------------|
-    | True                          |
-    +-------------------------------+
+    +-------------------------------+-------------------------------------+--------------------------------------+
+    | LIKE('hello world', '_ello%') | LIKE('hello world', '_ELLo%', true) | LIKE('hello world', '_ELLo%', false) |
+    |-------------------------------+-------------------------------------+--------------------------------------|
+    | True                          | False                               | True                                 |
+    +-------------------------------+-------------------------------------+--------------------------------------+
 
 
 Limitation: The pushdown of the LIKE function to a DSL wildcard query is supported only for keyword fields.
 
+ILIKE
+----
+
+Description
+>>>>>>>>>>>
+
+Usage: ilike(string, PATTERN) return true if the string match the PATTERN, PATTERN is **case-insensitive**.
+
+There are two wildcards often used in conjunction with the ILIKE operator:
+
+* ``%`` - The percent sign represents zero, one, or multiple characters
+* ``_`` - The underscore represents a single character
+
+Argument type: STRING, STRING
+
+Return type: INTEGER
+
+Example::
+
+    os> source=people | eval `ILIKE('hello world', '_ELLo%')` = ILIKE('hello world', '_ELLo%') | fields `ILIKE('hello world', '_ELLo%')`
+    fetched rows / total rows = 1/1
+    +--------------------------------+
+    | ILIKE('hello world', '_ELLo%') |
+    |--------------------------------|
+    | True                           |
+    +--------------------------------+
+
+
+Limitation: The pushdown of the ILIKE function to a DSL wildcard query is supported only for keyword fields.
+
 LOCATE
 -------
 

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java

@@ -668,6 +668,26 @@ public void testStreamstatsResetExplain() throws IOException {
     assertYamlEqualsIgnoreId(expected, result);
   }
 
+  @Test
+  public void testKeywordILikeFunctionExplain() throws IOException {
+    // ilike is only supported in v3
+    String expected = loadExpectedPlan("explain_keyword_ilike_function.yaml");
+    assertYamlEqualsIgnoreId(
+        expected,
+        explainQueryYaml(
+            "source=opensearch-sql_test_index_account | where ilike(firstname, '%mbe%')"));
+  }
+
+  @Test
+  public void testTextILikeFunctionExplain() throws IOException {
+    // ilike is only supported in v3
+    String expected = loadExpectedPlan("explain_text_ilike_function.yaml");
+    assertYamlEqualsIgnoreId(
+        expected,
+        explainQueryYaml(
+            "source=opensearch-sql_test_index_account | where ilike(address, '%Holmes%')"));
+  }
+
   // Only for Calcite, as v2 gets unstable serialized string for function
   @Test
   public void testExplainOnAggregationWithSumEnhancement() throws IOException {

integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteLikeQueryIT.java

@@ -5,8 +5,15 @@
 
 package org.opensearch.sql.calcite.remote;
 
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WILDCARD;
+import static org.opensearch.sql.util.MatcherUtils.rows;
+import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
+import static org.opensearch.sql.util.MatcherUtils.verifyNumOfRows;
+
 import java.io.IOException;
+import org.json.JSONObject;
 import org.junit.Test;
+import org.opensearch.sql.common.setting.Settings;
 import org.opensearch.sql.ppl.LikeQueryIT;
 
 public class CalciteLikeQueryIT extends LikeQueryIT {
@@ -22,4 +29,61 @@ public void test_convert_field_text_to_keyword() throws IOException {
     enabledOnlyWhenPushdownIsEnabled();
     super.test_convert_field_text_to_keyword();
   }
+
+  @Test
+  public void test_ilike_is_case_insensitive() throws IOException {
+    String query =
+        "source="
+            + TEST_INDEX_WILDCARD
+            + " | WHERE ILike(KeywordBody, 'test Wildcard%') | fields KeywordBody";
+    JSONObject result = executeQuery(query);
+    verifyDataRows(
+        result,
+        rows("test wildcard"),
+        rows("test wildcard in the end of the text%"),
+        rows("test wildcard in % the middle of the text"),
+        rows("test wildcard %% beside each other"),
+        rows("test wildcard in the end of the text_"),
+        rows("test wildcard in _ the middle of the text"),
+        rows("test wildcard __ beside each other"));
+  }
+
+  @Test
+  public void test_the_default_3rd_option() throws IOException {
+    // only work in v3
+    String query =
+        "source="
+            + TEST_INDEX_WILDCARD
+            + " | WHERE Like(KeywordBody, 'test Wildcard%') | fields KeywordBody";
+    withSettings(
+        Settings.Key.PPL_SYNTAX_LEGACY_PREFERRED,
+        "true",
+        () -> {
+          try {
+            JSONObject result = executeQuery(query);
+            verifyDataRows(
+                result,
+                rows("test wildcard"),
+                rows("test wildcard in the end of the text%"),
+                rows("test wildcard in % the middle of the text"),
+                rows("test wildcard %% beside each other"),
+                rows("test wildcard in the end of the text_"),
+                rows("test wildcard in _ the middle of the text"),
+                rows("test wildcard __ beside each other"));
+          } catch (IOException e) {
+            throw new RuntimeException(e);
+          }
+        });
+    withSettings(
+        Settings.Key.PPL_SYNTAX_LEGACY_PREFERRED,
+        "false",
+        () -> {
+          try {
+            JSONObject result = executeQuery(query);
+            verifyNumOfRows(result, 0);
+          } catch (IOException e) {
+            throw new RuntimeException(e);
+          }
+        });
+  }
 }