Merge branch 'main' into fix/3429-spark4-ctas-union-native-writer

Author: andygrove

.claude/skills/bug-triage/SKILL.md

@@ -65,8 +65,7 @@ For each issue, review the title and body and determine:
 2. **Area labels** (zero or more): from the area table in the guide
    (`area:writer`, `area:shuffle`, `area:aggregation`, `area:scan`,
    `area:expressions`, `area:ffi`, `area:ci`) plus the pre-existing area
-   indicators (`native_datafusion`, `native_iceberg_compat`, `spark 4`,
-   `spark sql tests`).
+   indicators (`spark 4`, `spark sql tests`).
 3. **Escalation note**: if the issue matches an escalation trigger from the
    guide (e.g., a `priority:high` crash that may also produce wrong results),
    note it in the summary.

.github/workflows/codeql.yml

@@ -49,11 +49,11 @@ jobs:
         persist-credentials: false
 
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4
+      uses: github/codeql-action/init@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4
       with:
         languages: actions
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4
+      uses: github/codeql-action/analyze@9e0d7b8d25671d64c341c19c0152d693099fb5ba # v4
       with:
         category: "/language:actions"

.github/workflows/iceberg_spark_test.yml

@@ -25,27 +25,41 @@ on:
   push:
     branches:
       - main
-    paths-ignore:
-      - "benchmarks/**"
-      - "doc/**"
-      - "docs/**"
-      - "**.md"
-      - "native/core/benches/**"
-      - "native/spark-expr/benches/**"
-      - "spark/src/test/**"
-      - "spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
-      - "spark-integration/**"
+    paths:
+      - "native/**/src/**"
+      - "native/**/Cargo.toml"
+      - "native/Cargo.lock"
+      - "!native/hdfs/**"
+      - "!native/fs-hdfs/**"
+      - "common/src/main/**"
+      - "common/pom.xml"
+      - "spark/src/main/**"
+      - "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
+      - "spark/pom.xml"
+      - "dev/diffs/iceberg/**"
+      - "pom.xml"
+      - "rust-toolchain.toml"
+      - ".github/workflows/iceberg_spark_test.yml"
+      - ".github/actions/setup-builder/**"
+      - ".github/actions/setup-iceberg-builder/**"
   pull_request:
-    paths-ignore:
-      - "benchmarks/**"
-      - "doc/**"
-      - "docs/**"
-      - "**.md"
-      - "native/core/benches/**"
-      - "native/spark-expr/benches/**"
-      - "spark/src/test/**"
-      - "spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
-      - "spark-integration/**"
+    paths:
+      - "native/**/src/**"
+      - "native/**/Cargo.toml"
+      - "native/Cargo.lock"
+      - "!native/hdfs/**"
+      - "!native/fs-hdfs/**"
+      - "common/src/main/**"
+      - "common/pom.xml"
+      - "spark/src/main/**"
+      - "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
+      - "spark/pom.xml"
+      - "dev/diffs/iceberg/**"
+      - "pom.xml"
+      - "rust-toolchain.toml"
+      - ".github/workflows/iceberg_spark_test.yml"
+      - ".github/actions/setup-builder/**"
+      - ".github/actions/setup-iceberg-builder/**"
   # manual trigger
   # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
   workflow_dispatch:

.github/workflows/miri.yml

@@ -22,26 +22,9 @@ concurrency:
   cancel-in-progress: true
 
 on:
-  push:
-    branches:
-      - main
-    paths-ignore:
-      - "doc/**"
-      - "docs/**"
-      - "**.md"
-      - "native/core/benches/**"
-      - "native/spark-expr/benches/**"
-      - "spark/src/test/scala/org/apache/spark/sql/benchmark/**"
-      - "spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
-  pull_request:
-    paths-ignore:
-      - "doc/**"
-      - "docs/**"
-      - "**.md"
-      - "native/core/benches/**"
-      - "native/spark-expr/benches/**"
-      - "spark/src/test/scala/org/apache/spark/sql/benchmark/**"
-      - "spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
+  # nightly safety check
+  schedule:
+    - cron: '0 4 * * *'
   # manual trigger
   # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
   workflow_dispatch:

.github/workflows/pr_build_linux.yml

@@ -22,28 +22,55 @@ concurrency:
   cancel-in-progress: true
 
 on:
+  # Allow-list of paths that affect this workflow. A change must match a positive
+  # pattern (and not a trailing "!" exclusion) for the build to run. Editing
+  # pr_build_macos.yml does not trigger this workflow, and vice versa.
   push:
     branches:
       - main
-    paths-ignore:
-      - "benchmarks/**"
-      - "doc/**"
-      - "docs/**"
-      - "**.md"
-      - "native/core/benches/**"
-      - "native/spark-expr/benches/**"
-      - "spark/src/test/scala/org/apache/spark/sql/benchmark/**"
-      - "spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
+    paths:
+      - "native/**"
+      - "common/**"
+      - "spark/**"
+      - "spark-integration/**"
+      - "pom.xml"
+      - "**/pom.xml"
+      - ".mvn/**"
+      - "mvnw"
+      - "Makefile"
+      - "rust-toolchain.toml"
+      - "dev/ci/**"
+      - ".github/workflows/pr_build_linux.yml"
+      - ".github/actions/setup-builder/**"
+      - ".github/actions/java-test/**"
+      - ".github/actions/rust-test/**"
+      - "!**.md"
+      - "!native/core/benches/**"
+      - "!native/spark-expr/benches/**"
+      - "!spark/src/test/scala/org/apache/spark/sql/benchmark/**"
+      - "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
   pull_request:
-    paths-ignore:
-      - "benchmarks/**"
-      - "doc/**"
-      - "docs/**"
-      - "**.md"
-      - "native/core/benches/**"
-      - "native/spark-expr/benches/**"
-      - "spark/src/test/scala/org/apache/spark/sql/benchmark/**"
-      - "spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
+    paths:
+      - "native/**"
+      - "common/**"
+      - "spark/**"
+      - "spark-integration/**"
+      - "pom.xml"
+      - "**/pom.xml"
+      - ".mvn/**"
+      - "mvnw"
+      - "Makefile"
+      - "rust-toolchain.toml"
+      - "dev/ci/**"
+      - ".github/workflows/pr_build_linux.yml"
+      - ".github/actions/setup-builder/**"
+      - ".github/actions/java-test/**"
+      - ".github/actions/rust-test/**"
+      - "!**.md"
+      - "!native/core/benches/**"
+      - "!native/spark-expr/benches/**"
+      - "!spark/src/test/scala/org/apache/spark/sql/benchmark/**"
+      - "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala"
   # manual trigger
   # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
   workflow_dispatch:
@@ -295,14 +322,28 @@ jobs:
           - name: "Spark 4.2, JDK 17"
             java_version: "17"
             maven_opts: "-Pspark-4.2"
+        # Suites are grouped by functional area into balanced buckets so that no test
+        # job runs much longer than ~23 min. See
+        # docs/superpowers/specs/2026-05-22-pr-build-consolidation-design.md for the
+        # per-suite timing analysis behind this grouping.
         suite:
-          - name: "fuzz"
+          - name: "scans"
             value: |
+              org.apache.comet.parquet.CometParquetWriterSuite
+              org.apache.comet.parquet.ParquetReadV1Suite
+              org.apache.comet.parquet.ParquetReadV2Suite
+              org.apache.comet.parquet.ParquetReadFromFakeHadoopFsSuite
+              org.apache.comet.parquet.ParquetTimestampLtzAsNtzSuite
+              org.apache.spark.sql.comet.ParquetDatetimeRebaseV1Suite
+              org.apache.spark.sql.comet.ParquetDatetimeRebaseV2Suite
+              org.apache.spark.sql.comet.ParquetEncryptionITCase
+              org.apache.comet.exec.CometNativeReaderSuite
+              org.apache.comet.CometIcebergNativeSuite
+              org.apache.comet.CometIcebergRewriteActionSuite
+              org.apache.comet.iceberg.IcebergReflectionSuite
+              org.apache.comet.csv.CometCsvNativeReadSuite
               org.apache.comet.CometFuzzTestSuite
-              org.apache.comet.CometFuzzAggregateSuite
               org.apache.comet.CometFuzzIcebergSuite
-              org.apache.comet.CometFuzzMathSuite
-              org.apache.comet.CometCodegenFuzzSuite
               org.apache.comet.DataGeneratorSuite
           - name: "shuffle"
             value: |
@@ -316,23 +357,6 @@ jobs:
               org.apache.comet.exec.DisableAQECometShuffleSuite
               org.apache.comet.exec.DisableAQECometAsyncShuffleSuite
               org.apache.spark.shuffle.sort.SpillSorterSuite
-          - name: "parquet"
-            value: |
-              org.apache.comet.parquet.CometParquetWriterSuite
-              org.apache.comet.parquet.ParquetReadV1Suite
-              org.apache.comet.parquet.ParquetReadV2Suite
-              org.apache.comet.parquet.ParquetReadFromFakeHadoopFsSuite
-              org.apache.comet.parquet.ParquetTimestampLtzAsNtzSuite
-              org.apache.spark.sql.comet.ParquetDatetimeRebaseV1Suite
-              org.apache.spark.sql.comet.ParquetDatetimeRebaseV2Suite
-              org.apache.spark.sql.comet.ParquetEncryptionITCase
-              org.apache.comet.exec.CometNativeReaderSuite
-              org.apache.comet.CometIcebergNativeSuite
-              org.apache.comet.CometIcebergRewriteActionSuite
-              org.apache.comet.iceberg.IcebergReflectionSuite
-          - name: "csv"
-            value: |
-              org.apache.comet.csv.CometCsvNativeReadSuite
           - name: "exec"
             value: |
               org.apache.comet.exec.CometAggregateSuite
@@ -360,6 +384,9 @@ jobs:
               org.apache.spark.sql.comet.CometShuffleFallbackStickinessSuite
               org.apache.spark.sql.comet.CometDecimalArithmeticViewSuite
               org.apache.comet.objectstore.NativeConfigSuite
+              org.apache.spark.sql.CometToPrettyStringSuite
+              org.apache.spark.sql.CometCollationSuite
+              org.apache.comet.CometFuzzAggregateSuite
           - name: "expressions"
             value: |
               org.apache.comet.CometExpressionSuite
@@ -376,18 +403,15 @@ jobs:
               org.apache.comet.CometMapExpressionSuite
               org.apache.comet.CometCsvExpressionSuite
               org.apache.comet.CometJsonExpressionSuite
-              org.apache.comet.CometDateTimeUtilsSuite
               org.apache.comet.SparkErrorConverterSuite
               org.apache.comet.expressions.conditional.CometIfSuite
               org.apache.comet.expressions.conditional.CometCoalesceSuite
               org.apache.comet.expressions.conditional.CometCaseWhenSuite
               org.apache.comet.CometCodegenSuite
               org.apache.comet.CometCodegenSourceSuite
               org.apache.comet.CometCodegenHOFSuite
-          - name: "sql"
-            value: |
-              org.apache.spark.sql.CometToPrettyStringSuite
-              org.apache.spark.sql.CometCollationSuite
+              org.apache.comet.CometFuzzMathSuite
+              org.apache.comet.CometCodegenFuzzSuite
       fail-fast: false
     name: ${{ matrix.profile.name }} [${{ matrix.suite.name }}]
     runs-on: ubuntu-24.04
@@ -427,7 +451,7 @@ jobs:
         uses: ./.github/actions/java-test
         with:
           artifact_name: ${{ matrix.profile.name }}-${{ matrix.suite.name }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}
-          suites: ${{ matrix.suite.name == 'sql' && matrix.profile.name == 'Spark 3.4, JDK 11, Scala 2.12' && '' || matrix.suite.value }}
+          suites: ${{ matrix.suite.value }}
           maven_opts: ${{ matrix.profile.maven_opts }}
           upload-test-reports: true
           skip-native-build: true
@@ -486,19 +510,16 @@ jobs:
         run: |
           SPARK_HOME=`pwd` SPARK_TPCH_DATA=`pwd`/tpch/sf1_parquet ./mvnw -B -Prelease -Dsuites=org.apache.spark.sql.CometTPCHQuerySuite test
 
-  # TPC-DS correctness tests - verifies benchmark queries produce correct results
+  # TPC-DS correctness tests - verifies benchmark queries produce correct results.
+  # The three join strategies run sequentially in one job so the project is built once.
   verify-benchmark-results-tpcds:
     needs: build-native
-    name: Verify TPC-DS Results (${{ matrix.join }})
+    name: Verify TPC-DS Results
     runs-on: ubuntu-24.04
     container:
       image: amd64/rust
     env:
       JAVA_TOOL_OPTIONS: --add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED
-    strategy:
-      matrix:
-        join: [sort_merge, broadcast, hash]
-      fail-fast: false
     steps:
       - uses: actions/checkout@v6
 
@@ -555,7 +576,6 @@ jobs:
           cd spark && MAVEN_OPTS='-Xmx20g' ../mvnw -B -Prelease exec:java -Dexec.mainClass="org.apache.spark.sql.GenTPCDSData" -Dexec.classpathScope="test" -Dexec.cleanupDaemonThreads="false" -Dexec.args="--dsdgenDir `pwd`/../tpcds-kit/tools --location `pwd`/../tpcds-sf-1 --scaleFactor 1 --numPartitions 1"
 
       - name: Run TPC-DS queries (Sort merge join)
-        if: matrix.join == 'sort_merge'
         run: |
           SPARK_HOME=`pwd` SPARK_TPCDS_DATA=`pwd`/tpcds-sf-1 ./mvnw -B -Prelease -Dsuites=org.apache.spark.sql.CometTPCDSQuerySuite test
         env:
@@ -564,15 +584,13 @@ jobs:
             spark.sql.join.preferSortMergeJoin=true
 
       - name: Run TPC-DS queries (Broadcast hash join)
-        if: matrix.join == 'broadcast'
         run: |
           SPARK_HOME=`pwd` SPARK_TPCDS_DATA=`pwd`/tpcds-sf-1 ./mvnw -B -Prelease -Dsuites=org.apache.spark.sql.CometTPCDSQuerySuite test
         env:
           SPARK_TPCDS_JOIN_CONF: |
             spark.sql.autoBroadcastJoinThreshold=10485760
 
       - name: Run TPC-DS queries (Shuffled hash join)
-        if: matrix.join == 'hash'
         run: |
           SPARK_HOME=`pwd` SPARK_TPCDS_DATA=`pwd`/tpcds-sf-1 ./mvnw -B -Prelease -Dsuites=org.apache.spark.sql.CometTPCDSQuerySuite test
         env: