Switch percentile implementation to MergingDigest to align with OpenSearch (opensearch-project#3698)

* Switch percentile implementation to MergingDigest to align with OpenSearch

Signed-off-by: Lantao Jin <ltjin@amazon.com>

* fix IT

Signed-off-by: Lantao Jin <ltjin@amazon.com>

---------

Signed-off-by: Lantao Jin <ltjin@amazon.com>

Author: LantaoJin

core/src/main/java/org/opensearch/sql/calcite/udf/udaf/PercentileApproxFunction.java

@@ -5,7 +5,7 @@
 
 package org.opensearch.sql.calcite.udf.udaf;
 
-import com.tdunning.math.stats.AVLTreeDigest;
+import com.tdunning.math.stats.MergingDigest;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
@@ -85,7 +85,7 @@ public void evaluate(double value) {
     public Object value(Object... argList) {
       double percent = (double) argList[1];
       double compression = (double) argList[0];
-      AVLTreeDigest tree = new AVLTreeDigest(compression);
+      MergingDigest tree = new MergingDigest(compression);
       for (Number num : candidate) {
         tree.add(num.doubleValue());
       }

core/src/main/java/org/opensearch/sql/expression/aggregation/PercentileApproximateAggregator.java

@@ -8,7 +8,7 @@
 import static org.opensearch.sql.data.model.ExprValueUtils.doubleValue;
 import static org.opensearch.sql.utils.ExpressionUtils.format;
 
-import com.tdunning.math.stats.AVLTreeDigest;
+import com.tdunning.math.stats.MergingDigest;
 import java.util.List;
 import org.opensearch.sql.common.utils.StringUtils;
 import org.opensearch.sql.data.model.ExprNullValue;
@@ -56,11 +56,11 @@ public String toString() {
   }
 
   /**
-   * PercentileApproximateState is used to store the AVLTreeDigest state for percentile estimation.
+   * PercentileApproximateState is used to store the MergingDigest state for percentile estimation.
    */
-  protected static class PercentileApproximateState extends AVLTreeDigest
+  protected static class PercentileApproximateState extends MergingDigest
       implements AggregationState {
-    // The compression level for the AVLTreeDigest, keep the same default value as OpenSearch core.
+    // The compression level for the MergingDigest, keep the same default value as OpenSearch core.
     public static final double DEFAULT_COMPRESSION = 100.0;
     private final double percent;
 

core/src/test/java/org/opensearch/sql/expression/aggregation/PercentileApproxAggregatorTest.java

@@ -68,21 +68,21 @@ public void test_percentile_field_expression_with_user_defined_compression() {
         aggregation(
             DSL.percentile(DSL.ref("integer_value", INTEGER), DSL.literal(50), DSL.literal(0.1)),
             tuples);
-    assertEquals(2.5, result.value());
+    assertEquals(3.0, result.value());
     result =
         aggregation(
             DSL.percentile(DSL.ref("long_value", LONG), DSL.literal(50), DSL.literal(0.1)), tuples);
-    assertEquals(2.5, result.value());
+    assertEquals(3.0, result.value());
     result =
         aggregation(
             DSL.percentile(DSL.ref("double_value", DOUBLE), DSL.literal(50), DSL.literal(0.1)),
             tuples);
-    assertEquals(2.5, result.value());
+    assertEquals(3.0, result.value());
     result =
         aggregation(
             DSL.percentile(DSL.ref("float_value", FLOAT), DSL.literal(50), DSL.literal(0.1)),
             tuples);
-    assertEquals(2.5, result.value());
+    assertEquals(3.0, result.value());
   }
 
   @Test

docs/user/dql/aggregations.rst

@@ -378,6 +378,8 @@ Description
 
 Usage: PERCENTILE(expr, percent) or PERCENTILE_APPROX(expr, percent). Returns the approximate percentile value of `expr` at the specified percentage. `percent` must be a constant between 0 and 100.
 
+Note: From 3.1.0, the percentile implementation is switched to MergingDigest from AVLTreeDigest. Ref `issue link <https://github.com/opensearch-project/OpenSearch/issues/18122>`_.
+
 Example::
 
     os> SELECT gender, percentile(age, 90) as p90 FROM accounts GROUP BY gender;

docs/user/ppl/cmd/stats.rst

@@ -269,6 +269,8 @@ Usage: PERCENTILE(expr, percent) or PERCENTILE_APPROX(expr, percent). Return the
 
 * percent: The number must be a constant between 0 and 100.
 
+Note: From 3.1.0, the percentile implementation is switched to MergingDigest from AVLTreeDigest. Ref `issue link <https://github.com/opensearch-project/OpenSearch/issues/18122>`_.
+
 Example::
 
     os> source=accounts | stats percentile(age, 90) by gender;

integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java

@@ -436,9 +436,17 @@ public void testStatsPercentileWithNull() throws IOException {
   public void testStatsPercentileWithCompression() throws IOException {
     JSONObject response =
         executeQuery(
-            String.format("source=%s | stats percentile(balance, 50, 1)", TEST_INDEX_BANK));
-    verifySchema(response, schema("percentile(balance, 50, 1)", null, "bigint"));
+            String.format("source=%s | stats percentile(balance, 50, 20)", TEST_INDEX_BANK));
+    verifySchema(response, schema("percentile(balance, 50, 20)", null, "bigint"));
     verifyDataRows(response, rows(32838));
+
+    // disable pushdown by adding a eval command
+    JSONObject responsePushdownDisabled =
+        executeQuery(
+            String.format(
+                "source=%s | eval a = 1 | stats percentile(balance, 50, 20)", TEST_INDEX_BANK));
+    verifySchema(responsePushdownDisabled, schema("percentile(balance, 50, 20)", null, "bigint"));
+    verifyDataRows(responsePushdownDisabled, rows(32838));
   }
 
   @Test