Merge remote-tracking branch 'origin/main' into user-jvm-udf

Author: andygrove

.asf.yaml

@@ -41,6 +41,7 @@ github:
   features:
     issues: true
     discussions: true
+    projects: true
   protected_branches:
     main:
       required_pull_request_reviews:

.github/workflows/docker-publish.yml

@@ -74,6 +74,6 @@ jobs:
         with:
           platforms: linux/amd64,linux/arm64
           push: true
-          tags: ghcr.io/apache/datafusion-comet:spark-3.5-scala-2.12-${{ env.COMET_VERSION }}
+          tags: ghcr.io/apache/datafusion-comet:spark-4.1-scala-2.13-${{ env.COMET_VERSION }}
           file: kube/Dockerfile
           no-cache: true

.github/workflows/pr_build_linux.yml

@@ -355,6 +355,7 @@ jobs:
               org.apache.comet.exec.CometWindowExecSuite
               org.apache.comet.exec.CometJoinSuite
               org.apache.comet.CometNativeSuite
+              org.apache.comet.CometSetOpWithGroupBySuite
               org.apache.comet.CometSparkSessionExtensionsSuite
               org.apache.comet.CometUserUdfSuite
               org.apache.spark.CometPluginsSuite
@@ -451,6 +452,8 @@ jobs:
     runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion-comet', github.run_id) || 'ubuntu-latest' }}
     container:
       image: amd64/rust
+    env:
+      JAVA_TOOL_OPTIONS: --add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED
     steps:
       - uses: runs-on/action@742bf56072eb4845a0f94b3394673e4903c90ff0  # v2.1.0
 
@@ -460,7 +463,7 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: ${{ env.RUST_VERSION }}
-          jdk-version: 11
+          jdk-version: 17
 
       - name: Download native library
         uses: actions/download-artifact@v8
@@ -505,6 +508,8 @@ jobs:
     runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a+m7a+c8a,cpu=16,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion-comet', github.run_id) || 'ubuntu-latest' }}
     container:
       image: amd64/rust
+    env:
+      JAVA_TOOL_OPTIONS: --add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED
     strategy:
       matrix:
         join: [sort_merge, broadcast, hash]
@@ -518,7 +523,7 @@ jobs:
         uses: ./.github/actions/setup-builder
         with:
           rust-version: ${{ env.RUST_VERSION }}
-          jdk-version: 11
+          jdk-version: 17
 
       - name: Download native library
         uses: actions/download-artifact@v8

.github/workflows/pr_build_macos.yml

@@ -194,6 +194,7 @@ jobs:
               org.apache.comet.exec.CometWindowExecSuite
               org.apache.comet.exec.CometJoinSuite
               org.apache.comet.CometNativeSuite
+              org.apache.comet.CometSetOpWithGroupBySuite
               org.apache.comet.CometSparkSessionExtensionsSuite
               org.apache.comet.CometUserUdfSuite
               org.apache.spark.CometPluginsSuite

common/src/main/scala/org/apache/comet/parquet/CometParquetUtils.scala

@@ -29,6 +29,12 @@ object CometParquetUtils {
   private val PARQUET_FIELD_ID_READ_ENABLED = "spark.sql.parquet.fieldId.read.enabled"
   private val IGNORE_MISSING_PARQUET_FIELD_ID = "spark.sql.parquet.fieldId.read.ignoreMissing"
 
+  // Field-metadata key arrow-rs writes when it lifts Parquet field IDs into the Arrow schema
+  // (`parquet::arrow::PARQUET_FIELD_ID_META_KEY`). Spark's local key for the same concept is
+  // `parquet.field.id` (`ParquetUtils.FIELD_ID_METADATA_KEY`). The serde translates at the proto
+  // boundary so the native side can match the same key it gets from arrow-rs.
+  val PARQUET_FIELD_ID_META_KEY = "PARQUET:field_id"
+
   // Map of encryption configuration key-value pairs that, if present, are only supported with
   // these specific values. Generally, these are the default values that won't be present,
   // but if they are present we want to check them.

dev/diffs/4.1.1.diff

@@ -150,50 +150,6 @@ index 4410fe50912..43bcce2a038 100644
        case _ => Map[String, String]()
      }
      val childrenInfo = children.flatMap {
-diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/intersect-all.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/intersect-all.sql.out
-index 69b4001ff34..6fda691652d 100644
---- a/sql/core/src/test/resources/sql-tests/analyzer-results/intersect-all.sql.out
-+++ b/sql/core/src/test/resources/sql-tests/analyzer-results/intersect-all.sql.out
-@@ -1,7 +1,7 @@
- -- Automatically generated by SQLQueryTestSuite
- -- !query
- CREATE TEMPORARY VIEW tab1 AS SELECT * FROM VALUES
--    (1, 2), 
-+    (1, 2),
-     (1, 2),
-     (1, 3),
-     (1, 3),
-@@ -11,7 +11,7 @@ CREATE TEMPORARY VIEW tab1 AS SELECT * FROM VALUES
-     AS tab1(k, v)
- -- !query analysis
- CreateViewCommand `tab1`, SELECT * FROM VALUES
--    (1, 2), 
-+    (1, 2),
-     (1, 2),
-     (1, 3),
-     (1, 3),
-@@ -26,8 +26,8 @@ CreateViewCommand `tab1`, SELECT * FROM VALUES
- 
- -- !query
- CREATE TEMPORARY VIEW tab2 AS SELECT * FROM VALUES
--    (1, 2), 
--    (1, 2), 
-+    (1, 2),
-+    (1, 2),
-     (2, 3),
-     (3, 4),
-     (null, null),
-@@ -35,8 +35,8 @@ CREATE TEMPORARY VIEW tab2 AS SELECT * FROM VALUES
-     AS tab2(k, v)
- -- !query analysis
- CreateViewCommand `tab2`, SELECT * FROM VALUES
--    (1, 2), 
--    (1, 2), 
-+    (1, 2),
-+    (1, 2),
-     (2, 3),
-     (3, 4),
-     (null, null),
 diff --git a/sql/core/src/test/resources/sql-tests/inputs/decimalArithmeticOperations.sql b/sql/core/src/test/resources/sql-tests/inputs/decimalArithmeticOperations.sql
 index 13bbd9d81b7..541cdfb1e04 100644
 --- a/sql/core/src/test/resources/sql-tests/inputs/decimalArithmeticOperations.sql
@@ -211,18 +167,6 @@ index 13bbd9d81b7..541cdfb1e04 100644
  CREATE TEMPORARY VIEW t AS SELECT 1.0 as a, 0.0 as b;
 
  -- division, remainder and pmod by 0 return NULL
-diff --git a/sql/core/src/test/resources/sql-tests/inputs/except-all.sql b/sql/core/src/test/resources/sql-tests/inputs/except-all.sql
-index e28f0721a64..788b43c242a 100644
---- a/sql/core/src/test/resources/sql-tests/inputs/except-all.sql
-+++ b/sql/core/src/test/resources/sql-tests/inputs/except-all.sql
-@@ -1,3 +1,7 @@
-+-- TODO(https://github.com/apache/datafusion-comet/issues/4122)
-+-- EXCEPT ALL with GROUP BY returns incorrect results on Spark 4.1
-+--SET spark.comet.enabled = false
-+
- CREATE TEMPORARY VIEW tab1 AS SELECT * FROM VALUES
-     (0), (1), (2), (2), (2), (2), (3), (null), (null) AS tab1(c1);
- CREATE TEMPORARY VIEW tab2 AS SELECT * FROM VALUES
 diff --git a/sql/core/src/test/resources/sql-tests/inputs/explain-aqe.sql b/sql/core/src/test/resources/sql-tests/inputs/explain-aqe.sql
 index 7aef901da4f..f3d6e18926d 100644
 --- a/sql/core/src/test/resources/sql-tests/inputs/explain-aqe.sql
@@ -280,32 +224,6 @@ index 35128da97fd..25b873ae859 100644
  -- Positive test cases
  -- Create a table with some testing data.
  DROP TABLE IF EXISTS t1;
-diff --git a/sql/core/src/test/resources/sql-tests/inputs/intersect-all.sql b/sql/core/src/test/resources/sql-tests/inputs/intersect-all.sql
-index 077caa5dd44..697457d4251 100644
---- a/sql/core/src/test/resources/sql-tests/inputs/intersect-all.sql
-+++ b/sql/core/src/test/resources/sql-tests/inputs/intersect-all.sql
-@@ -1,5 +1,9 @@
-+-- TODO(https://github.com/apache/datafusion-comet/issues/4122)
-+-- INTERSECT ALL with GROUP BY returns incorrect results on Spark 4.1
-+--SET spark.comet.enabled = false
-+
- CREATE TEMPORARY VIEW tab1 AS SELECT * FROM VALUES
--    (1, 2), 
-+    (1, 2),
-     (1, 2),
-     (1, 3),
-     (1, 3),
-@@ -8,8 +12,8 @@ CREATE TEMPORARY VIEW tab1 AS SELECT * FROM VALUES
-     (null, null)
-     AS tab1(k, v);
- CREATE TEMPORARY VIEW tab2 AS SELECT * FROM VALUES
--    (1, 2), 
--    (1, 2), 
-+    (1, 2),
-+    (1, 2),
-     (2, 3),
-     (3, 4),
-     (null, null),
 diff --git a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part3.sql b/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part3.sql
 index 41fd4de2a09..162d5a817b6 100644
 --- a/sql/core/src/test/resources/sql-tests/inputs/postgreSQL/aggregates_part3.sql
@@ -428,30 +346,6 @@ index 21a3ce1e122..f4762ab98f0 100644
  SET spark.sql.ansi.enabled = false;
 
  -- In COMPENSATION views get invalidated if the type can't cast
-diff --git a/sql/core/src/test/resources/sql-tests/results/intersect-all.sql.out b/sql/core/src/test/resources/sql-tests/results/intersect-all.sql.out
-index 44f95f225ab..361866fc298 100644
---- a/sql/core/src/test/resources/sql-tests/results/intersect-all.sql.out
-+++ b/sql/core/src/test/resources/sql-tests/results/intersect-all.sql.out
-@@ -1,7 +1,7 @@
- -- Automatically generated by SQLQueryTestSuite
- -- !query
- CREATE TEMPORARY VIEW tab1 AS SELECT * FROM VALUES
--    (1, 2), 
-+    (1, 2),
-     (1, 2),
-     (1, 3),
-     (1, 3),
-@@ -17,8 +17,8 @@ struct<>
- 
- -- !query
- CREATE TEMPORARY VIEW tab2 AS SELECT * FROM VALUES
--    (1, 2), 
--    (1, 2), 
-+    (1, 2),
-+    (1, 2),
-     (2, 3),
-     (3, 4),
-     (null, null),
 diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
 index 0d807aeae4d..6d7744e771b 100644
 --- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala

docs/source/contributor-guide/benchmarking_aws_ec2.md

@@ -104,7 +104,7 @@ make release
 Set `COMET_JAR` environment variable.
 
 ```shell
-export COMET_JAR=/home/ec2-user/datafusion-comet/spark/target/comet-spark-spark3.5_2.12-$COMET_VERSION.jar
+export COMET_JAR=/home/ec2-user/datafusion-comet/spark/target/comet-spark-spark4.1_2.13-$COMET_VERSION.jar
 ```
 
 ## Run Benchmarks

docs/source/contributor-guide/benchmarking_macos.md

@@ -55,13 +55,13 @@ export DF_BENCH=`pwd`
 
 ## Install Spark
 
-Install Apache Spark. This example refers to 3.5.4 version.
+Install Apache Spark. This example refers to 4.1.1 version.
 
 ```shell
-wget https://archive.apache.org/dist/spark/spark-3.5.4/spark-3.5.4-bin-hadoop3.tgz
-tar xzf spark-3.5.4-bin-hadoop3.tgz
-sudo mv spark-3.5.4-bin-hadoop3 /opt
-export SPARK_HOME=/opt/spark-3.5.4-bin-hadoop3/
+wget https://archive.apache.org/dist/spark/spark-4.1.1/spark-4.1.1-bin-hadoop3.tgz
+tar xzf spark-4.1.1-bin-hadoop3.tgz
+sudo mv spark-4.1.1-bin-hadoop3 /opt
+export SPARK_HOME=/opt/spark-4.1.1-bin-hadoop3/
 ```
 
 Start Spark in standalone mode:
@@ -129,7 +129,7 @@ make release COMET_FEATURES=mimalloc
 Set `COMET_JAR` to point to the location of the Comet jar file. Example for Comet 0.8
 
 ```shell
-export COMET_JAR=`pwd`/spark/target/comet-spark-spark3.5_2.12-0.8.0-SNAPSHOT.jar
+export COMET_JAR=`pwd`/spark/target/comet-spark-spark4.1_2.13-0.8.0-SNAPSHOT.jar
 ```
 
 Run the following command (the `--data` parameter will need to be updated to point to your S3 bucket):

docs/source/contributor-guide/benchmarking_spark_sql_perf.md

@@ -34,8 +34,8 @@ partitioning and writing to Parquet format automatically.
 
 ## Prerequisites
 
-- Java 17 (for Spark 3.5+)
-- Apache Spark 3.5.x
+- Java 17
+- Apache Spark 4.1.x
 - SBT (Scala Build Tool)
 - C compiler toolchain (`gcc`, `make`, `flex`, `bison`, `byacc`)
 
@@ -225,7 +225,7 @@ Build Comet from source and launch `spark-shell` with both the Comet and spark-s
 
 ```shell
 make release
-export COMET_JAR=$(pwd)/spark/target/comet-spark-spark3.5_2.12-*.jar
+export COMET_JAR=$(pwd)/spark/target/comet-spark-spark4.1_2.13-*.jar
 
 $SPARK_HOME/bin/spark-shell \
     --master $SPARK_MASTER \

docs/source/contributor-guide/debugging.md

@@ -136,7 +136,7 @@ make release COMET_FEATURES=backtrace
 Set `RUST_BACKTRACE=1` for the Spark worker/executor process, or for `spark-submit` if running in local mode.
 
 ```console
-RUST_BACKTRACE=1 $SPARK_HOME/spark-shell --jars spark/target/comet-spark-spark3.5_2.12-$COMET_VERSION.jar --conf spark.plugins=org.apache.spark.CometPlugin --conf spark.comet.enabled=true --conf spark.comet.exec.enabled=true
+RUST_BACKTRACE=1 $SPARK_HOME/spark-shell --jars spark/target/comet-spark-spark4.1_2.13-$COMET_VERSION.jar --conf spark.plugins=org.apache.spark.CometPlugin --conf spark.comet.enabled=true --conf spark.comet.exec.enabled=true
 ```
 
 Get the expanded exception details