Bump Spark 4.0.1 to 4.0.2 (#4114)

Author: andygrove

.github/workflows/spark_sql_test.yml

@@ -140,7 +140,7 @@ jobs:
         config:
           - {spark-short: '3.4', spark-full: '3.4.3', java: 11, scan-impl: 'auto'}
           - {spark-short: '3.5', spark-full: '3.5.8', java: 11, scan-impl: 'auto'}
-          - {spark-short: '4.0', spark-full: '4.0.1', java: 17, scan-impl: 'auto'}
+          - {spark-short: '4.0', spark-full: '4.0.2', java: 17, scan-impl: 'auto'}
       fail-fast: false
     name: spark-sql-${{ matrix.config.scan-impl }}-${{ matrix.module.name }}/spark-${{ matrix.config.spark-full }}
     runs-on: ${{ matrix.os }}

dev/diffs/4.0.1.diff dev/diffs/4.0.2.diffdev/diffs/4.0.1.diff renamed to dev/diffs/4.0.2.diff

@@ -3774,7 +3774,7 @@ index 4b27082e188..057b2430872 100644
 -class HiveUDFDynamicLoadSuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
 +// Comet: mix in IgnoreCometSuite so these tests are reported as ignored when Comet is enabled
 +// (ENABLE_COMET=true). The jar these tests depend on (`hive-test-udfs.jar`) is stripped from the
-+// Spark 4.0.1 release source tag per the ASF binary-artifact policy, so the tests cannot run in
++// Spark 4.0.2 release source tag per the ASF binary-artifact policy, so the tests cannot run in
 +// Comet's CI. Ignoring keeps the suite passing without masking real regressions; the upstream
 +// tests still run in non-Comet Spark builds that ship the jar on branch-4.0.
 +class HiveUDFDynamicLoadSuite extends QueryTest with SQLTestUtils with TestHiveSingleton
@@ -3788,7 +3788,7 @@ index 4b27082e188..057b2430872 100644
      val jarPath = "src/test/noclasspath/hive-test-udfs.jar"
 -    assume(new java.io.File(jarPath).exists)
 +    // Comet: the upstream `assume(...)` runs here in the suite constructor (inside this foreach,
-+    // before `test(...)` registers a case). When the jar is missing - as it is on the v4.0.1
++    // before `test(...)` registers a case). When the jar is missing - as it is on the v4.0.2
 +    // release tag - `assume` throws TestCanceledException out of `<init>`, which ScalaTest
 +    // reports as a suite abort (not a per-test cancel) and fails the whole job. The
 +    // IgnoreCometSuite mixin above already reroutes these tests to `ignore` under Comet, so

docs/source/user-guide/latest/installation.md

@@ -50,7 +50,7 @@ use only and should not be used in production yet.
 
 | Spark Version | Java Version | Scala Version | Comet Tests in CI | Spark SQL Tests in CI |
 | ------------- | ------------ | ------------- | ----------------- | --------------------- |
-| 4.0.1         | 17           | 2.13          | Yes               | Yes                   |
+| 4.0.2         | 17           | 2.13          | Yes               | Yes                   |
 
 Note that Comet may not fully work with proprietary forks of Apache Spark such as the Spark versions offered by
 Cloud Service Providers.

pom.xml

@@ -668,7 +668,7 @@ under the License.
         <!-- Use Scala 2.13 by default -->
         <scala.version>2.13.16</scala.version>
         <scala.binary.version>2.13</scala.binary.version>
-        <spark.version>4.0.1</spark.version>
+        <spark.version>4.0.2</spark.version>
         <spark.version.short>4.0</spark.version.short>
         <parquet.version>1.15.2</parquet.version>
         <semanticdb.version>4.13.6</semanticdb.version>