'gender' in test data should contain keyword subfield

Signed-off-by: Lantao Jin <ltjin@amazon.com>

Author: LantaoJin

benchmarks/src/jmh/java/org/opensearch/sql/expression/operator/predicate/MergeArrayAndObjectMapBenchmark.java

@@ -12,7 +12,7 @@
 import java.util.Map;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.opensearch.sql.opensearch.data.type.OpenSearchDataType;
-import org.opensearch.sql.opensearch.request.system.OpenSearchDescribeIndexRequest;
+import org.opensearch.sql.opensearch.util.MergeRules.MergeRuleHelper;
 
 public class MergeArrayAndObjectMapBenchmark {
   private static final List<Map<String, OpenSearchDataType>> candidateMaps = prepareListOfMaps(120);
@@ -21,7 +21,7 @@ public class MergeArrayAndObjectMapBenchmark {
   public void testMerge() {
     Map<String, OpenSearchDataType> finalResult = new HashMap<>();
     for (Map<String, OpenSearchDataType> map : candidateMaps) {
-      OpenSearchDescribeIndexRequest.mergeObjectAndArrayInsideMap(finalResult, map);
+      MergeRuleHelper.merge(finalResult, map);
     }
   }
 

integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java

@@ -437,7 +437,7 @@ public void aggregationFunctionInHaving() throws IOException {
 
     JSONArray dataRows = getDataRows(response);
     assertEquals(1, dataRows.length());
-    assertEquals("m", dataRows.getJSONArray(0).getString(0));
+    assertEquals("M", dataRows.getJSONArray(0).getString(0));
   }
 
   /**

integ-test/src/test/java/org/opensearch/sql/ppl/RareCommandIT.java

@@ -59,7 +59,7 @@ public void testRareWithGroup() throws IOException {
           rows("F", "OK", 7),
           rows("F", "KS", 7),
           rows("F", "CO", 7),
-          rows("F", "NV", 8),
+          rows("F", "AR", 8),
           rows("M", "NE", 5),
           rows("M", "RI", 5),
           rows("M", "NV", 5),

integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java

@@ -11,7 +11,6 @@
 import static org.opensearch.sql.util.MatcherUtils.rows;
 import static org.opensearch.sql.util.MatcherUtils.schema;
 import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
-import static org.opensearch.sql.util.MatcherUtils.verifyDataRowsInOrder;
 import static org.opensearch.sql.util.MatcherUtils.verifySchema;
 import static org.opensearch.sql.util.MatcherUtils.verifySchemaInOrder;
 
@@ -358,27 +357,15 @@ public void testStatsBySpanAndMultipleFields() throws IOException {
         schema("span(age,10)", null, "int"),
         schema("gender", null, "string"),
         schema("state", null, "string"));
-    if (isCalciteEnabled()) {
-      verifyDataRows(
-          response,
-          rows(1, 20, "F", "VA"),
-          rows(1, 30, "F", "IN"),
-          rows(1, 30, "F", "PA"),
-          rows(1, 30, "M", "IL"),
-          rows(1, 30, "M", "MD"),
-          rows(1, 30, "M", "TN"),
-          rows(1, 30, "M", "WA"));
-    } else {
-      verifyDataRowsInOrder(
-          response,
-          rows(1, 20, "f", "VA"),
-          rows(1, 30, "f", "IN"),
-          rows(1, 30, "f", "PA"),
-          rows(1, 30, "m", "IL"),
-          rows(1, 30, "m", "MD"),
-          rows(1, 30, "m", "TN"),
-          rows(1, 30, "m", "WA"));
-    }
+    verifyDataRows(
+        response,
+        rows(1, 20, "F", "VA"),
+        rows(1, 30, "F", "IN"),
+        rows(1, 30, "F", "PA"),
+        rows(1, 30, "M", "IL"),
+        rows(1, 30, "M", "MD"),
+        rows(1, 30, "M", "TN"),
+        rows(1, 30, "M", "WA"));
   }
 
   @Test
@@ -395,27 +382,15 @@ public void testStatsByMultipleFieldsAndSpan() throws IOException {
         schema("span(age,10)", null, "int"),
         schema("gender", null, "string"),
         schema("state", null, "string"));
-    if (isCalciteEnabled()) {
-      verifyDataRows(
-          response,
-          rows(1, 20, "F", "VA"),
-          rows(1, 30, "F", "IN"),
-          rows(1, 30, "F", "PA"),
-          rows(1, 30, "M", "IL"),
-          rows(1, 30, "M", "MD"),
-          rows(1, 30, "M", "TN"),
-          rows(1, 30, "M", "WA"));
-    } else {
-      verifyDataRowsInOrder(
-          response,
-          rows(1, 20, "f", "VA"),
-          rows(1, 30, "f", "IN"),
-          rows(1, 30, "f", "PA"),
-          rows(1, 30, "m", "IL"),
-          rows(1, 30, "m", "MD"),
-          rows(1, 30, "m", "TN"),
-          rows(1, 30, "m", "WA"));
-    }
+    verifyDataRows(
+        response,
+        rows(1, 20, "F", "VA"),
+        rows(1, 30, "F", "IN"),
+        rows(1, 30, "F", "PA"),
+        rows(1, 30, "M", "IL"),
+        rows(1, 30, "M", "MD"),
+        rows(1, 30, "M", "TN"),
+        rows(1, 30, "M", "WA"));
   }
 
   @Test

integ-test/src/test/resources/indexDefinitions/account_index_mapping.json

@@ -3,21 +3,17 @@
     "properties": {
       "gender": {
         "type": "text",
+        "fielddata": true,
         "fields": {
-          "subfield1": {
+          "keyword": {
             "type": "keyword",
             "ignore_above": 256
           }
         }
       },
       "address": {
         "type": "text",
-        "fields": {
-          "subfield2": {
-            "type": "keyword",
-            "ignore_above": 256
-          }
-        }
+        "fielddata": true
       },
       "firstname": {
         "type": "text",

integ-test/src/test/resources/indexDefinitions/bank_index_mapping.json

@@ -30,7 +30,13 @@
       },
       "gender": {
         "type": "text",
-        "fielddata": true
+        "fielddata": true,
+        "fields": {
+          "keyword": {
+            "type": "keyword",
+            "ignore_above": 256
+          }
+        }
       },
       "lastname": {
         "type": "keyword"