fix UT and revert change

Signed-off-by: xinyual <xinyual@amazon.com>

Author: xinyual

docs/user/ppl/cmd/parse.rst

@@ -40,7 +40,7 @@ PPL query::
     |-----------------------+------------|
     | amberduke@pyrami.com  | pyrami.com |
     | hattiebond@netagy.com | netagy.com |
-    | null                  | null       |
+    | null                  |            |
     | daleadams@boink.com   | boink.com  |
     +-----------------------+------------+
 

docs/user/ppl/cmd/patterns.rst

@@ -69,7 +69,7 @@ PPL query::
     |-----------------------+----------------|
     | amberduke@pyrami.com  | @.             |
     | hattiebond@netagy.com | @.             |
-    | null                  | null           |
+    | null                  |                |
     | daleadams@boink.com   | @.             |
     +-----------------------+----------------+
 

opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java

@@ -8,6 +8,7 @@
 import static org.opensearch.core.xcontent.DeprecationHandler.IGNORE_DEPRECATIONS;
 import static org.opensearch.search.sort.FieldSortBuilder.DOC_FIELD_NAME;
 import static org.opensearch.search.sort.SortOrder.ASC;
+import static org.opensearch.sql.opensearch.storage.OpenSearchIndex.METADATA_FIELD_ID;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -191,6 +192,9 @@ public OpenSearchResponse searchWithPIT(Function<SearchRequest, SearchResponse>
       // Set sort field for search_after
       if (this.sourceBuilder.sorts() == null) {
         this.sourceBuilder.sort(DOC_FIELD_NAME, ASC);
+        // Workaround to preserve sort location more exactly,
+        // see https://github.com/opensearch-project/sql/pull/3061
+        this.sourceBuilder.sort(METADATA_FIELD_ID, ASC);
       }
       SearchRequest searchRequest = new SearchRequest().source(this.sourceBuilder);
       this.searchResponse = searchAction.apply(searchRequest);

opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java

@@ -299,7 +299,7 @@ public void pushDownHighlight(String field, Map<String, Literal> arguments) {
 
   /** Push down project list to DSL requests. */
   public void pushDownProjects(Set<ReferenceExpression> projects) {
-    pushDownProjectStream(projects.stream().map(ReferenceExpression::getAttr));
+    pushDownProjectStream(projects.stream().map(ReferenceExpression::getRawPath));
   }
 
   public void pushDownProjectStream(Stream<String> projects) {

opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java

@@ -88,7 +88,7 @@ public class OpenSearchSettings extends Settings {
   public static final Setting<?> CALCITE_ENGINE_ENABLED_SETTING =
       Setting.boolSetting(
           Key.CALCITE_ENGINE_ENABLED.getKeyValue(),
-          true,
+          false,
           Setting.Property.NodeScope,
           Setting.Property.Dynamic);
 

opensearch/src/test/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilderTest.java

@@ -123,7 +123,6 @@ void build_query_request_push_down_size() {
 
   @Test
   void build_PIT_request_with_correct_size() {
-    when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(true);
     when(client.createPit(any(CreatePitRequest.class))).thenReturn("samplePITId");
     Integer limit = 1;
     Integer offset = 0;
@@ -143,7 +142,6 @@ void build_PIT_request_with_correct_size() {
 
   @Test
   void buildRequestWithPit_pageSizeNull_sizeGreaterThanMaxResultWindow() {
-    when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(true);
     when(client.createPit(any(CreatePitRequest.class))).thenReturn("samplePITId");
     Integer limit = 600;
     Integer offset = 0;
@@ -166,7 +164,6 @@ void buildRequestWithPit_pageSizeNull_sizeGreaterThanMaxResultWindow() {
 
   @Test
   void buildRequestWithPit_pageSizeNull_sizeLessThanMaxResultWindow() {
-    when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(true);
     Integer limit = 400;
     Integer offset = 0;
     requestBuilder = new OpenSearchRequestBuilder(exprValueFactory, settings);
@@ -215,52 +212,8 @@ void buildRequestWithPit_pageSizeNotNull_startFromNonZero() {
         });
   }
 
-  @Test
-  void build_scroll_request_with_correct_size() {
-    when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(false);
-    Integer limit = 800;
-    Integer offset = 10;
-    requestBuilder.pushDownLimit(limit, offset);
-    requestBuilder.getSourceBuilder().fetchSource("a", "b");
-
-    assertEquals(
-        new OpenSearchScrollRequest(
-            new OpenSearchRequest.IndexName("test"),
-            TimeValue.timeValueMinutes(1),
-            new SearchSourceBuilder()
-                .from(offset)
-                .size(MAX_RESULT_WINDOW - offset)
-                .timeout(DEFAULT_QUERY_TIMEOUT),
-            exprValueFactory,
-            List.of()),
-        requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client));
-  }
-
-  @Test
-  void buildRequestWithScroll_pageSizeNotNull_startFromZero() {
-    when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(false);
-    int pageSize = 200;
-    int offset = 0;
-    int limit = 400;
-    requestBuilder.pushDownPageSize(pageSize);
-    requestBuilder.pushDownLimit(limit, offset);
-
-    assertEquals(
-        new OpenSearchScrollRequest(
-            new OpenSearchRequest.IndexName("test"),
-            TimeValue.timeValueMinutes(1),
-            new SearchSourceBuilder()
-                .from(offset)
-                .size(MAX_RESULT_WINDOW - offset)
-                .timeout(DEFAULT_QUERY_TIMEOUT),
-            exprValueFactory,
-            List.of()),
-        requestBuilder.build(indexName, MAX_RESULT_WINDOW, DEFAULT_QUERY_TIMEOUT, client));
-  }
-
   @Test
   void buildRequestWithScroll_pageSizeNotNull_startFromNonZero() {
-    when(settings.getSettingValue(Settings.Key.SQL_PAGINATION_API_SEARCH_AFTER)).thenReturn(false);
     int pageSize = 200;
     int offset = 100;
     int limit = 400;

ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLCastFunctionTest.java

@@ -68,7 +68,7 @@ public void testCastUnknownType() {
       RelNode root = getRelNode(ppl);
       fail("expected error, got " + root);
     } catch (Exception e) {
-      assertThat(e.getMessage(), containsString("cast(MGR as UNKNOWN' <-- HERE"));
+      assertThat(e.getMessage(), containsString("cast(MGR as UNKNOWN' <--- HERE"));
     }
   }
 

ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java

@@ -397,7 +397,7 @@ public void testPatterns() {
   }
 
   private String anonymize(String query) {
-    AstBuilder astBuilder = new AstBuilder(query);
+    AstBuilder astBuilder = new AstBuilder(query, settings);
     return anonymize(astBuilder.visit(parser.parse(query)));
   }