Increase default TDigest compression for star-tree from 100 to 500

t-digest 3.3 has higher merge-order sensitivity than 3.2. At the
previous default compression=100, multi-level merge in star-tree
building produces ~0.35% error vs ground truth (compared to 0.06%
with 3.2). Increasing to 500 brings error back to ~0.09%.

Changes:
- PercentileTDigestValueAggregator: DEFAULT_TDIGEST_COMPRESSION 100->500
- StarTreeBuilderUtils: always inject default compression into
  expression context so the aggregator receives it explicitly
- StarTreeV2BuilderConfig: persist compression=500 in
  functionParameters when using old functionColumnPairs syntax,
  so canUseStarTree() can distinguish old segments (no metadata,
  compression=100) from new segments (metadata=500)

Backward compatibility:
- Old segments have no compressionFactor in metadata -> canUseStarTree()
  treats them as compression=100 -> matched by queries with default
  compression (100)
- New segments have compressionFactor=500 in metadata -> matched by
  queries explicitly specifying compression=500

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: xiangfu0

pinot-segment-local/src/main/java/org/apache/pinot/segment/local/aggregator/PercentileTDigestValueAggregator.java

@@ -29,8 +29,11 @@
 public class PercentileTDigestValueAggregator implements ValueAggregator<Object, TDigest> {
   public static final DataType AGGREGATED_VALUE_TYPE = DataType.BYTES;
 
-  // TODO: This is copied from PercentileTDigestAggregationFunction.
-  public static final int DEFAULT_TDIGEST_COMPRESSION = 100;
+  // NOTE: This is intentionally higher than the query-time default (100) in PercentileTDigestAggregationFunction.
+  // t-digest 3.3 has higher merge-order sensitivity, and star-tree building involves multi-level merge with
+  // intermediate serialization/deserialization. At compression=100, this causes ~0.35% error (vs 0.06% in 3.2).
+  // Compression=500 brings error back to ~0.09%, comparable to 3.2's accuracy at compression=100.
+  public static final int DEFAULT_TDIGEST_COMPRESSION = 500;
   private final int _compressionFactor;
   private int _maxByteSize;
 

pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/StarTreeBuilderUtils.java

@@ -34,6 +34,7 @@
 import org.apache.commons.io.FileUtils;
 import org.apache.pinot.common.request.Literal;
 import org.apache.pinot.common.request.context.ExpressionContext;
+import org.apache.pinot.segment.local.aggregator.PercentileTDigestValueAggregator;
 import org.apache.pinot.segment.local.startree.v2.builder.StarTreeV2BuilderConfig;
 import org.apache.pinot.segment.spi.AggregationFunctionType;
 import org.apache.pinot.segment.spi.Constants;
@@ -395,6 +396,12 @@ public static List<ExpressionContext> expressionContextFromFunctionParameters(
           expressionContexts.add(ExpressionContext.forLiteral(
               Literal.intValue(Integer.parseInt(String.valueOf(
                   functionParameters.get(Constants.PERCENTILETDIGEST_COMPRESSION_FACTOR_KEY))))));
+        } else {
+          // Always inject the default compression so it is stored in star-tree metadata.
+          // This allows canUseStarTree() to distinguish old segments (no metadata, built with compression=100)
+          // from new segments (metadata present, built with the current default).
+          expressionContexts.add(ExpressionContext.forLiteral(
+              Literal.intValue(PercentileTDigestValueAggregator.DEFAULT_TDIGEST_COMPRESSION)));
         }
         break;
       }

pinot-segment-local/src/main/java/org/apache/pinot/segment/local/startree/v2/builder/StarTreeV2BuilderConfig.java

@@ -32,8 +32,10 @@
 import org.apache.commons.configuration2.Configuration;
 import org.apache.commons.lang3.builder.ToStringBuilder;
 import org.apache.commons.lang3.builder.ToStringStyle;
+import org.apache.pinot.segment.local.aggregator.PercentileTDigestValueAggregator;
 import org.apache.pinot.segment.spi.AggregationFunctionType;
 import org.apache.pinot.segment.spi.ColumnMetadata;
+import org.apache.pinot.segment.spi.Constants;
 import org.apache.pinot.segment.spi.SegmentMetadata;
 import org.apache.pinot.segment.spi.index.startree.AggregationFunctionColumnPair;
 import org.apache.pinot.segment.spi.index.startree.AggregationSpec;
@@ -80,7 +82,8 @@ public static StarTreeV2BuilderConfig fromIndexConfig(StarTreeIndexConfig indexC
             AggregationFunctionColumnPair.resolveToStoredType(aggregationFunctionColumnPair);
         // If there is already an equivalent functionColumnPair in the map, do not load another.
         // This prevents the duplication of the aggregation when the StarTree is constructed.
-        aggregationSpecs.putIfAbsent(storedType, AggregationSpec.DEFAULT);
+        aggregationSpecs.putIfAbsent(storedType,
+            getDefaultAggregationSpec(aggregationFunctionColumnPair.getFunctionType()));
       }
     }
     if (indexConfig.getAggregationConfigs() != null) {
@@ -323,4 +326,22 @@ public String toString() {
         .append("skipStarNodeCreation", _skipStarNodeCreationForDimensions)
         .append("aggregationSpecs", _aggregationSpecs).append("maxLeafRecords", _maxLeafRecords).toString();
   }
+
+  /**
+   * Returns the default {@link AggregationSpec} for the given function type. For PERCENTILETDIGEST and
+   * PERCENTILERAWTDIGEST, the default compression factor is explicitly included in the function parameters so that
+   * it is persisted in star-tree metadata. This allows {@code canUseStarTree()} to distinguish old segments (built
+   * with the previous default of 100, which have no compression metadata) from new segments.
+   */
+  private static AggregationSpec getDefaultAggregationSpec(AggregationFunctionType functionType) {
+    switch (functionType) {
+      case PERCENTILETDIGEST:
+      case PERCENTILERAWTDIGEST:
+        return new AggregationSpec(null, null, null, null, null,
+            Map.of(Constants.PERCENTILETDIGEST_COMPRESSION_FACTOR_KEY,
+                PercentileTDigestValueAggregator.DEFAULT_TDIGEST_COMPRESSION));
+      default:
+        return AggregationSpec.DEFAULT;
+    }
+  }
 }