Merge branch 'main' into fix_comet_sum_compatbility_level

Author: coderfender

.claude/skills/bug-triage/SKILL.md

@@ -65,8 +65,7 @@ For each issue, review the title and body and determine:
 2. **Area labels** (zero or more): from the area table in the guide
    (`area:writer`, `area:shuffle`, `area:aggregation`, `area:scan`,
    `area:expressions`, `area:ffi`, `area:ci`) plus the pre-existing area
-   indicators (`native_datafusion`, `native_iceberg_compat`, `spark 4`,
-   `spark sql tests`).
+   indicators (`spark 4`, `spark sql tests`).
 3. **Escalation note**: if the issue matches an escalation trigger from the
    guide (e.g., a `priority:high` crash that may also produce wrong results),
    note it in the summary.

.github/workflows/codeql.yml

@@ -49,11 +49,11 @@ jobs:
         persist-credentials: false
 
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4
+      uses: github/codeql-action/init@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4
       with:
         languages: actions
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4
+      uses: github/codeql-action/analyze@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4
       with:
         category: "/language:actions"

.github/workflows/stale.yml

@@ -27,7 +27,7 @@ jobs:
       issues: write
       pull-requests: write
     steps:
-      - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f  # v10.2.0
+      - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899  # v10.3.0
         with:
           stale-pr-message: "Thank you for your contribution. Unfortunately, this pull request is stale because it has been open 60 days with no activity. Please remove the stale label or comment or this will be closed in 7 days."
           days-before-pr-stale: 60

native/Cargo.lock

@@ -56,7 +56,7 @@ regex = "1.12.3"
 thiserror = "2"
 object_store = { version = "0.13.1", features = ["gcp", "azure", "aws", "http"] }
 url = "2.2"
-aws-config = "1.8.16"
+aws-config = "1.8.17"
 aws-credential-types = "1.2.13"
 iceberg = { git = "https://github.com/apache/iceberg-rust", rev = "1ad4bfd" }
 iceberg-storage-opendal = { git = "https://github.com/apache/iceberg-rust", rev = "1ad4bfd", features = ["opendal-all"] }

native/Cargo.toml

@@ -1052,10 +1052,10 @@ impl ColumnarToRowContext {
                 })
             }
             (DataType::Int32, DataType::Decimal128(precision, scale)) => {
-                // Parquet stores small-precision decimals as Int32 for efficiency.
-                // When COMET_USE_DECIMAL_128 is false, BatchReader produces these types.
-                // The Int32 value is already scaled (e.g., -1 means -0.01 for scale 2).
-                // We need to reinterpret (not cast) to Decimal128 preserving the value.
+                // Parquet stores small-precision decimals as Int32 for efficiency, and the
+                // reader may surface them as the physical Int32 type. The value is already
+                // scaled (e.g., -1 means -0.01 for scale 2). Reinterpret (not cast) to
+                // Decimal128 preserving the value.
                 let int_array = array.as_any().downcast_ref::<Int32Array>().ok_or_else(|| {
                     CometError::Internal("Failed to downcast to Int32Array".to_string())
                 })?;
@@ -2581,8 +2581,7 @@ mod tests {
     #[test]
     fn test_convert_int32_to_decimal128() {
         // Test that Int32 arrays are correctly cast to Decimal128 when schema expects Decimal128.
-        // This can happen when COMET_USE_DECIMAL_128 is false and the parquet reader produces
-        // Int32 for small-precision decimals.
+        // This can happen when the parquet reader surfaces small-precision decimals as Int32.
 
         // Create an Int32 array representing decimals: [-1, -2, -3] which at scale 2 means
         // [-0.01, -0.02, -0.03]
@@ -2619,8 +2618,7 @@ mod tests {
     #[test]
     fn test_convert_int64_to_decimal128() {
         // Test that Int64 arrays are correctly cast to Decimal128 when schema expects Decimal128.
-        // This can happen when COMET_USE_DECIMAL_128 is false and the parquet reader produces
-        // Int64 for medium-precision decimals.
+        // This can happen when the parquet reader surfaces medium-precision decimals as Int64.
 
         // Create an Int64 array representing decimals
         let int_array: ArrayRef = Arc::new(Int64Array::from(vec![-100i64, -200, -300]));

native/core/src/execution/columnar_to_row.rs

@@ -38,8 +38,7 @@ use datafusion_datasource::TableSchema;
 use std::collections::HashMap;
 use std::sync::Arc;
 
-/// Initializes a DataSourceExec plan with a ParquetSource. This may be used by either the
-/// `native_datafusion` scan or the `native_iceberg_compat` scan.
+/// Initializes a DataSourceExec plan with a ParquetSource for Comet's native Parquet scan.
 ///
 ///   `required_schema`: Schema to be projected by the scan.
 ///

native/core/src/parquet/parquet_exec.rs

@@ -74,8 +74,6 @@ pub struct SparkParquetOptions {
     pub allow_incompat: bool,
     /// Support casting unsigned ints to signed ints (used by Parquet SchemaAdapter)
     pub allow_cast_unsigned_ints: bool,
-    /// Whether to always represent decimals using 128 bits. If false, the native reader may represent decimals using 32 or 64 bits, depending on the precision.
-    pub use_decimal_128: bool,
     /// Whether to read dates/timestamps that were written in the legacy hybrid Julian + Gregorian calendar as it is. If false, throw exceptions instead. If the spark type is TimestampNTZ, this should be true.
     pub use_legacy_date_timestamp_or_ntz: bool,
     // Whether schema field names are case sensitive
@@ -105,7 +103,6 @@ impl SparkParquetOptions {
             timezone: timezone.to_string(),
             allow_incompat,
             allow_cast_unsigned_ints: false,
-            use_decimal_128: false,
             use_legacy_date_timestamp_or_ntz: false,
             case_sensitive: false,
             return_null_struct_if_all_fields_missing: true,
@@ -121,7 +118,6 @@ impl SparkParquetOptions {
             timezone: "".to_string(),
             allow_incompat,
             allow_cast_unsigned_ints: false,
-            use_decimal_128: false,
             use_legacy_date_timestamp_or_ntz: false,
             case_sensitive: false,
             return_null_struct_if_all_fields_missing: true,

native/core/src/parquet/parquet_support.rs

@@ -40,13 +40,12 @@ public abstract class CometDecodedVector extends CometVector {
   private byte validityByteCache;
   protected boolean isUuid;
 
-  protected CometDecodedVector(ValueVector vector, Field valueField, boolean useDecimal128) {
-    this(vector, valueField, useDecimal128, false);
+  protected CometDecodedVector(ValueVector vector, Field valueField) {
+    this(vector, valueField, false);
   }
 
-  protected CometDecodedVector(
-      ValueVector vector, Field valueField, boolean useDecimal128, boolean isUuid) {
-    super(Utils.fromArrowField(valueField), useDecimal128);
+  protected CometDecodedVector(ValueVector vector, Field valueField, boolean isUuid) {
+    super(Utils.fromArrowField(valueField));
     this.valueVector = vector;
     this.numNulls = valueVector.getNullCount();
     this.numValues = valueVector.getValueCount();

spark/src/main/java/org/apache/comet/vector/CometDecodedVector.java

@@ -33,15 +33,11 @@ public class CometDelegateVector extends CometVector {
   protected CometVector delegate;
 
   public CometDelegateVector(DataType dataType) {
-    this(dataType, null, false);
+    this(dataType, null);
   }
 
-  public CometDelegateVector(DataType dataType, boolean useDecimal128) {
-    this(dataType, null, useDecimal128);
-  }
-
-  public CometDelegateVector(DataType dataType, CometVector delegate, boolean useDecimal128) {
-    super(dataType, useDecimal128);
+  public CometDelegateVector(DataType dataType, CometVector delegate) {
+    super(dataType);
     if (delegate instanceof CometDelegateVector) {
       throw new IllegalArgumentException("cannot have nested delegation");
     }