chore: [branch-0.10] Support Spark 4.0.1 instead of 4.0.0 (#2414) (#2497)

Author: andygrove

.github/workflows/spark_sql_test_ansi.yml

@@ -43,7 +43,7 @@ jobs:
       matrix:
         os: [ubuntu-24.04]
         java-version: [17]
-        spark-version: [{short: '4.0', full: '4.0.0'}]
+        spark-version: [{short: '4.0', full: '4.0.1'}]
         module:
           - {name: "catalyst", args1: "catalyst/test", args2: ""}
           - {name: "sql/core-1", args1: "", args2: sql/testOnly * -- -l org.apache.spark.tags.ExtendedSQLTest -l org.apache.spark.tags.SlowSQLTest}

dev/diffs/4.0.0.diff dev/diffs/4.0.1.diffdev/diffs/4.0.0.diff renamed to dev/diffs/4.0.1.diff

@@ -33,9 +33,9 @@ releases. Once this integration is complete, we plan on switching from the `nati
 [#2060]: https://github.com/apache/datafusion-comet/issues/2060
 [#2189]: https://github.com/apache/datafusion-comet/issues/2189
 
-### Spark 4.0.0 Support
+### Spark 4.0 Support
 
-Comet has experimental support for Spark 4.0.0, but there is more work to do ([#1637]), such as enabling 
+Comet has experimental support for Spark 4.0, but there is more work to do ([#1637]), such as enabling 
 more Spark SQL tests and fully implementing ANSI support ([#313]) for all supported expressions.
 
 [#313]: https://github.com/apache/datafusion-comet/issues/313

docs/source/contributor-guide/roadmap.md

@@ -47,9 +47,9 @@ Note that we do not test the full matrix of supported Java and Scala versions in
 Experimental support is provided for the following versions of Apache Spark and is intended for development/testing
 use only and should not be used in production yet.
 
-| Spark Version  | Java Version | Scala Version | Comet Tests in CI | Spark SQL Tests in CI |
-| -------------- | ------------ | ------------- | ----------------- |-----------------------|
-| 4.0.0 | 17           | 2.13          | Yes               | Yes                   |
+| Spark Version | Java Version | Scala Version | Comet Tests in CI | Spark SQL Tests in CI |
+|---------------| ------------ | ------------- | ----------------- |-----------------------|
+| 4.0.1         | 17           | 2.13          | Yes               | Yes                   |
 
 Note that Comet may not fully work with proprietary forks of Apache Spark such as the Spark versions offered by
 Cloud Service Providers.

docs/source/user-guide/latest/installation.md

@@ -657,7 +657,7 @@ impl BitReader {
         debug_assert!(self.bit_offset == 0 || i == num_bits_to_read);
 
         // Check if there's opportunity to directly copy bytes using `memcpy`.
-        if (offset + i) % 8 == 0 && i < num_bits_to_read {
+        if (offset + i).is_multiple_of(8) && i < num_bits_to_read {
             let num_bytes = (num_bits_to_read - i) / 8;
             let dst_byte_offset = (offset + i) / 8;
             if num_bytes > 0 {
@@ -671,7 +671,7 @@ impl BitReader {
             }
         }
 
-        debug_assert!((offset + i) % 8 != 0 || num_bits_to_read - i < 8);
+        debug_assert!(!(offset + i).is_multiple_of(8) || num_bits_to_read - i < 8);
 
         // Now copy the remaining bits if there's any.
         while i < num_bits_to_read {

native/core/src/common/bit.rs

@@ -616,7 +616,7 @@ under the License.
         <!-- Use Scala 2.13 by default -->
         <scala.version>2.13.16</scala.version>
         <scala.binary.version>2.13</scala.binary.version>
-        <spark.version>4.0.0</spark.version>
+        <spark.version>4.0.1</spark.version>
         <spark.version.short>4.0</spark.version.short>
         <parquet.version>1.15.2</parquet.version>
         <semanticdb.version>4.13.6</semanticdb.version>