Fix IT

Signed-off-by: Lantao Jin <ltjin@amazon.com>

Author: LantaoJin

integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java

@@ -604,6 +604,11 @@ public enum Index {
         "dog",
         getDogs3IndexMapping(),
         "src/test/resources/dogs3.json"),
+    DOG4(
+        TestsConstants.TEST_INDEX_DOG4,
+        "dog",
+        getDogIndexMapping(),
+        "src/test/resources/dogs4.json"),
     DOGSSUBQUERY(
         TestsConstants.TEST_INDEX_DOGSUBQUERY,
         "dog",

integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java

@@ -19,6 +19,7 @@ public class TestsConstants {
   public static final String TEST_INDEX_DOG = TEST_INDEX + "_dog";
   public static final String TEST_INDEX_DOG2 = TEST_INDEX + "_dog2";
   public static final String TEST_INDEX_DOG3 = TEST_INDEX + "_dog3";
+  public static final String TEST_INDEX_DOG4 = TEST_INDEX + "_dog4";
   public static final String TEST_INDEX_DOGSUBQUERY = TEST_INDEX + "_subquery";
   public static final String TEST_INDEX_PEOPLE = TEST_INDEX + "_people";
   public static final String TEST_INDEX_PEOPLE2 = TEST_INDEX + "_people2";

integ-test/src/test/java/org/opensearch/sql/ppl/SortCommandIT.java

@@ -8,6 +8,7 @@
 import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK;
 import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES;
 import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DOG;
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DOG4;
 import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WEBLOGS;
 import static org.opensearch.sql.util.MatcherUtils.rows;
 import static org.opensearch.sql.util.MatcherUtils.verifyOrder;
@@ -31,6 +32,7 @@ public void init() throws Exception {
     loadIndex(Index.BANK_WITH_NULL_VALUES);
     loadIndex(Index.DOG);
     loadIndex(Index.WEBLOG);
+    loadIndex(Index.DOG4);
   }
 
   @Test
@@ -161,4 +163,23 @@ public void testSortThenHead() throws IOException {
         executeQuery(String.format("source=%s | sort age | head 2 | fields age", TEST_INDEX_BANK));
     verifyOrder(result, rows(28), rows(32));
   }
+
+  @Test
+  public void testSortFielddata() throws IOException {
+    JSONObject result =
+        executeQuery(
+            String.format("source=%s | sort dog_name | fields dog_name, age", TEST_INDEX_DOG4));
+    verifyOrder(result, rows("M 1", 2), rows("M 2", 4));
+    // Even sort with DESC on dog_name, the order is no diff with ASC for fielddata field
+    // since the field is tokenized. And the behaviour in v3 should be same with v2.
+    // But the behaviours between pushdown enable and disable are different.
+    result =
+        executeQuery(
+            String.format("source=%s | sort - dog_name | fields dog_name, age", TEST_INDEX_DOG4));
+    if (isPushdownEnabled()) {
+      verifyOrder(result, rows("M 1", 2), rows("M 2", 4));
+    } else {
+      verifyOrder(result, rows("M 2", 2), rows("M 1", 4));
+    }
+  }
 }

integ-test/src/test/resources/dogs4.json

@@ -0,0 +1,4 @@
+{"index":{"_id":"1"}}
+{"dog_name":"M 1","holdersName":"Daenerys","age":2}
+{"index":{"_id":"6"}}
+{"dog_name":"M 2","holdersName":"Hattie","age":4}

opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java

@@ -255,9 +255,8 @@ public AbstractCalciteIndexScan pushDownSort(List<RelFieldCollation> collations)
           String fieldNameKeyword;
           if (fieldType instanceof OpenSearchTextType textType) {
             fieldNameKeyword = OpenSearchTextType.toKeywordSubField(fieldName, fieldType);
-            if (fieldName.equals(fieldNameKeyword) && !textType.isFieldData()) {
-              // can not convert text to keyword, skip pushdown
-              return null;
+            if (fieldNameKeyword == null && textType.isFieldData()) {
+              fieldNameKeyword = fieldName;
             }
           } else {
             fieldNameKeyword = fieldName;