apache · zhouyuan · Mar 17, 2026 · Apr 1, 2026 · Apr 6, 2026 · Apr 8, 2026
diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml
@@ -107,7 +107,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ "ubuntu:20.04", "ubuntu:22.04" ]
-        spark: [ "spark-3.3", "spark-3.4", "spark-3.5", "spark-4.0", "spark-4.1"  ]
+        spark: [ "spark-3.3", "spark-3.4", "spark-3.5", "spark-4.0", "spark-4.1", "spark-4.2" ]
         java: [ "java-8", "java-11", "java-17", "java-21" ]
         # Spark supports JDK17 since 3.3.
         exclude:
@@ -139,6 +139,10 @@ jobs:
             java: java-8
           - spark: spark-4.1
             java: java-11
+          - spark: spark-4.2
+            java: java-8
+          - spark: spark-4.2
+            java: java-11
 
     runs-on: ubuntu-22.04
     container: ${{ matrix.os }}
@@ -156,7 +160,7 @@ jobs:
           path: /root/.m2/repository/org/apache/arrow/
       - name: Setup tzdata
         run: |
-          #sed -i 's|http://archive|http://us.archive|g' /etc/apt/sources.list
+          sed -i 's|http://archive|http://eu.archive|g' /etc/apt/sources.list
           if [ "${{ matrix.os }}" = "ubuntu:22.04" ]; then
             apt-get update
             TZ="Etc/GMT" DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata
@@ -197,7 +201,7 @@ jobs:
           export JAVA_HOME=/usr/lib/jvm/${{ matrix.java }}-openjdk-amd64
           echo "JAVA_HOME: $JAVA_HOME"
           case "${{ matrix.spark }}" in
-            spark-4.0|spark-4.1)
+            spark-4.0|spark-4.1|spark-4.2)
               $MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }} -Pscala-2.13 -Pbackends-velox -DskipTests
               ;;
             *)
@@ -240,7 +244,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ "centos:8" ]
-        spark: [ "spark-3.3", "spark-3.4", "spark-3.5", "spark-4.0", "spark-4.1" ]
+        spark: [ "spark-3.3", "spark-3.4", "spark-3.5", "spark-4.0", "spark-4.1", "spark-4.2"]
         java: [ "java-8", "java-11", "java-17" ]
         # Spark supports JDK17 since 3.3.
         exclude:
@@ -260,6 +264,10 @@ jobs:
             java: java-8
           - spark: spark-4.1
             java: java-11
+          - spark: spark-4.2
+            java: java-8
+          - spark: spark-4.2
+            java: java-11
 
     runs-on: ubuntu-22.04
     container: ${{ matrix.os }}
@@ -304,7 +312,7 @@ jobs:
           echo "JAVA_HOME: $JAVA_HOME"
           cd $GITHUB_WORKSPACE/
           case "${{ matrix.spark }}" in
-            spark-4.0|spark-4.1)
+            spark-4.0|spark-4.1|spark-4.2)
               $MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }} -Pscala-2.13 -Pbackends-velox -DskipTests
               ;;
             *)

diff --git a/.github/workflows/velox_nightly.yml b/.github/workflows/velox_nightly.yml
@@ -68,6 +68,71 @@ jobs:
           path: .m2/repository/org/apache/arrow/
           name: velox-arrow-jar-centos-7-${{github.sha}}
 
+  nightly-tpc-test-centos9-sparkmain:
+    needs: build-native-lib-x86
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ "centos:9" ]
+        spark: [ "spark-main" ]
+        java: [ "java-17" ]
+
+    runs-on: ubuntu-22.04
+    container: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Download All Native Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: velox-native-lib-${{github.sha}}
+          path: ./cpp/build/
+      - name: Download All Arrow Jar Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: velox-arrow-jar-centos-7-${{github.sha}}
+          path: /root/.m2/repository/org/apache/arrow/
+      - name: Setup java
+        run: |
+          if [ "${{ matrix.java }}" = "java-17" ]; then
+            yum update -y && yum install -y java-17-openjdk-devel wget
+          elif [ "${{ matrix.java }}" = "java-11" ]; then
+            yum update -y && yum install -y java-11-openjdk-devel wget
+          else
+            yum update -y && yum install -y java-1.8.0-openjdk-devel wget
+          fi
+      - name: Set environment variables
+        run: |
+          if [ "${{ matrix.java }}" = "java-17" ]; then
+            echo "JAVA_HOME=/usr/lib/jvm/java-17-openjdk" >> $GITHUB_ENV
+          elif [ "${{ matrix.java }}" = "java-11" ]; then
+            echo "JAVA_HOME=/usr/lib/jvm/java-11-openjdk" >> $GITHUB_ENV
+          else
+            echo "JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk" >> $GITHUB_ENV
+          fi
+      - name: Build gluten-it
+        run: |
+          echo "JAVA_HOME: $JAVA_HOME"
+          cd $GITHUB_WORKSPACE/
+          case "${{ matrix.spark }}" in
+            spark-4.0|spark-4.1|spark-4.2|spark-main)
+              $MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }} -Pscala-2.13 -Pbackends-velox -DskipTests
+              ;;
+            *)
+              $MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }} -Pbackends-velox -DskipTests
+              ;;
+          esac
+          cd $GITHUB_WORKSPACE/tools/gluten-it
+          $GITHUB_WORKSPACE/$MVN_CMD clean install -P${{ matrix.spark }} -P${{ matrix.java }}
+      - name: Run TPC-H / TPC-DS
+        run: |
+          echo "JAVA_HOME: $JAVA_HOME"
+          cd $GITHUB_WORKSPACE/tools/gluten-it
+          GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare \
+            --local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1
+          GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh queries-compare \
+            --local --preset=velox --benchmark-type=ds --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1
+
+
   build-bundle-package-centos8-jdk8-x86:
     if: ${{ startsWith(github.repository, 'apache/') }}
     needs: build-native-lib-x86

diff --git a/backends-velox/src/test/java/org/apache/gluten/fs/OnHeapFileSystemTest.java b/backends-velox/src/test/java/org/apache/gluten/fs/OnHeapFileSystemTest.java
@@ -35,8 +35,9 @@ public void testRoundTrip() {
     JniFilesystem.WriteFile writeFile = fs.openFileForWrite(path);
     try {
       byte[] bytes = text.getBytes(StandardCharsets.UTF_8);
-      ByteBuffer buf = PlatformDependent.allocateDirectNoCleaner(bytes.length);
+      ByteBuffer buf = ByteBuffer.allocateDirect(bytes.length);
       buf.put(bytes);
+      buf.flip();
       writeFile.append(bytes.length, PlatformDependent.directBufferAddress(buf));
       writeFile.flush();
       fileSize = writeFile.size();
@@ -47,8 +48,10 @@ public void testRoundTrip() {
 
     JniFilesystem.ReadFile readFile = fs.openFileForRead(path);
     Assert.assertEquals(fileSize, readFile.size());
-    ByteBuffer buf = PlatformDependent.allocateDirectNoCleaner((int) fileSize);
+    ByteBuffer buf = ByteBuffer.allocateDirect((int) fileSize);
     readFile.pread(0, fileSize, PlatformDependent.directBufferAddress(buf));
+    buf.limit((int) fileSize);
+    buf.position(0);
     byte[] out = new byte[(int) fileSize];
     buf.get(out);
     String decoded = new String(out, StandardCharsets.UTF_8);

diff --git a/gluten-substrait/src/main/scala/org/apache/gluten/expression/ConverterUtils.scala b/gluten-substrait/src/main/scala/org/apache/gluten/expression/ConverterUtils.scala
@@ -428,7 +428,7 @@ object ConverterUtils extends Logging {
         sigName = sigName.concat(getTypeSigName(valueType))
         sigName = sigName.concat(">")
         sigName
-      case CharType(_) =>
+      case c: CharType =>
         "fchar"
       case NullType =>
         "nothing"

diff --git a/.../src/main/scala/org/apache/gluten/extension/columnar/offload/OffloadSingleNodeRules.scala b/.../src/main/scala/org/apache/gluten/extension/columnar/offload/OffloadSingleNodeRules.scala
@@ -311,7 +311,7 @@ object OffloadOthers {
             plan.lowerBound,
             plan.upperBound,
             plan.withReplacement,
-            plan.seed,
+            SparkShimLoader.getSparkShims.getSampleExecSeed(plan),
             child)
         case plan: RDDScanExec if RDDScanTransformer.isSupportRDDScanExec(plan) =>
           RDDScanTransformer.getRDDScanTransform(plan)

diff --git a/gluten-substrait/src/main/scala/org/apache/spark/shuffle/sort/ColumnarShuffleManager.scala b/gluten-substrait/src/main/scala/org/apache/spark/shuffle/sort/ColumnarShuffleManager.scala
@@ -39,7 +39,7 @@ class ColumnarShuffleManager(conf: SparkConf)
   import ColumnarShuffleManager._
 
   private lazy val shuffleExecutorComponents = loadShuffleExecutorComponents(conf)
-  override val shuffleBlockResolver = new IndexShuffleBlockResolver(conf)
+  override lazy val shuffleBlockResolver = SparkIndexShuffleBlockResolverUtil.create(conf)
 
   /** A mapping from shuffle ids to the number of mappers producing output for those shuffles. */
   private[this] val taskIdMapsForShuffle = new ConcurrentHashMap[Int, OpenHashSet[Long]]()

diff --git a/pom.xml b/pom.xml
@@ -1388,6 +1388,180 @@
         </plugins>
       </build>
     </profile>
+    <profile>
+      <id>spark-4.2</id>
+      <properties>
+        <sparkbundle.version>4.2</sparkbundle.version>
+        <sparkshim.artifactId>spark-sql-columnar-shims-spark42</sparkshim.artifactId>
+        <spark.version>4.2.0-preview3</spark.version>
+        <iceberg.version>1.10.0</iceberg.version>
+        <delta.package.name>delta-spark</delta.package.name>
+        <delta.version>4.0.0</delta.version>
+        <delta.binary.version>40</delta.binary.version>
+        <hudi.version>1.1.0</hudi.version>
+        <paimon.version>1.3.0</paimon.version>
+        <fasterxml.version>2.18.2</fasterxml.version>
+        <fasterxml.jackson.version>2.18.2</fasterxml.jackson.version>
+        <fasterxml.jackson.databind.version>2.18.2</fasterxml.jackson.databind.version>
+        <hadoop.version>3.4.1</hadoop.version>
+        <antlr4.version>4.13.1</antlr4.version>
+        <guava.version>33.4.0-jre</guava.version>
+        <slf4j.version>2.0.16</slf4j.version>
+        <log4j.version>2.24.3</log4j.version>
+        <commons-lang3.version>3.17.0</commons-lang3.version>
+        <arrow.version>18.1.0</arrow.version>
+        <arrow-gluten.version>18.1.0</arrow-gluten.version>
+        <scala.compiler.version>4.9.5</scala.compiler.version>
+      </properties>
+      <dependencies>
+        <dependency>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-api</artifactId>
+          <version>${slf4j.version}</version>
+          <scope>provided</scope>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.logging.log4j</groupId>
+          <artifactId>log4j-slf4j2-impl</artifactId>
+          <version>${log4j.version}</version>
+          <scope>provided</scope>
+        </dependency>
+      </dependencies>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-enforcer-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>enforce-java-17+</id>
+                <goals>
+                  <goal>enforce</goal>
+                </goals>
+                <configuration>
+                  <rules>
+                    <requireActiveProfile>
+                      <!-- Spark 4.2 requires Java 17+ -->
+                      <profiles>java-17,java-21</profiles>
+                      <all>false</all>
+                      <message>"-P spark-4.2" requires JDK 17+</message>
+                    </requireActiveProfile>
+                  </rules>
+                </configuration>
+              </execution>
+              <execution>
+                <id>enforce-scala-213</id>
+                <goals>
+                  <goal>enforce</goal>
+                </goals>
+                <configuration>
+                  <rules>
+                    <requireActiveProfile>
+                      <!-- Spark 4.2 requires Scala 2.13 -->
+                      <profiles>scala-2.13</profiles>
+                      <message>"-P spark-4.2" requires Scala 2.13</message>
+                    </requireActiveProfile>
+                  </rules>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+    <profile>
+      <id>spark-main</id>
+      <properties>
+        <sparkbundle.version>4.2</sparkbundle.version>
+        <sparkshim.artifactId>spark-sql-columnar-shims-sparkmain</sparkshim.artifactId>
+        <spark.version>4.2.0-SNAPSHOT</spark.version>
+        <iceberg.version>1.10.0</iceberg.version>
+        <delta.package.name>delta-spark</delta.package.name>
+        <delta.version>4.0.0</delta.version>
+        <delta.binary.version>40</delta.binary.version>
+        <hudi.version>1.1.0</hudi.version>
+        <paimon.version>1.3.0</paimon.version>
+        <fasterxml.version>2.18.2</fasterxml.version>
+        <fasterxml.jackson.version>2.18.2</fasterxml.jackson.version>
+        <fasterxml.jackson.databind.version>2.18.2</fasterxml.jackson.databind.version>
+        <hadoop.version>3.4.1</hadoop.version>
+        <antlr4.version>4.13.1</antlr4.version>
+        <guava.version>33.4.0-jre</guava.version>
+        <slf4j.version>2.0.16</slf4j.version>
+        <log4j.version>2.24.3</log4j.version>
+        <commons-lang3.version>3.17.0</commons-lang3.version>
+        <arrow.version>18.1.0</arrow.version>
+        <arrow-gluten.version>18.1.0</arrow-gluten.version>
+        <scala.compiler.version>4.9.9</scala.compiler.version>
+      </properties>
+      <dependencies>
+        <dependency>
+          <groupId>org.slf4j</groupId>
+          <artifactId>slf4j-api</artifactId>
+          <version>${slf4j.version}</version>
+          <scope>provided</scope>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.logging.log4j</groupId>
+          <artifactId>log4j-slf4j2-impl</artifactId>
+          <version>${log4j.version}</version>
+          <scope>provided</scope>
+        </dependency>
+      </dependencies>
+      <repositories>
+        <repository>
+          <releases>
+            <enabled>false</enabled>
+          </releases>
+          <snapshots>
+            <enabled>true</enabled>
+          </snapshots>
+          <id>apache-spark-snapshots</id>
+          <url>https://repository.apache.org/content/repositories/snapshots/org/apache/spark/</url>
+        </repository>
+      </repositories>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-enforcer-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>enforce-java-17+</id>
+                <goals>
+                  <goal>enforce</goal>
+                </goals>
+                <configuration>
+                  <rules>
+                    <requireActiveProfile>
+                      <!-- Spark main branch requires Java 17+ -->
+                      <profiles>java-17,java-21</profiles>
+                      <all>false</all>
+                      <message>"-P spark-main" requires JDK 17+</message>
+                    </requireActiveProfile>
+                  </rules>
+                </configuration>
+              </execution>
+              <execution>
+                <id>enforce-scala-213</id>
+                <goals>
+                  <goal>enforce</goal>
+                </goals>
+                <configuration>
+                  <rules>
+                    <requireActiveProfile>
+                      <!-- Spark main branch requires Scala 2.13 -->
+                      <profiles>scala-2.13</profiles>
+                      <message>"-P spark-main" requires Scala 2.13</message>
+                    </requireActiveProfile>
+                  </rules>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
     <profile>
       <id>hadoop-2.7.4</id>
       <properties>

diff --git a/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala b/shims/common/src/main/scala/org/apache/gluten/sql/shims/SparkShims.scala
@@ -292,4 +292,10 @@ trait SparkShims {
    * similar to LeftOuter. Default implementation returns false for Spark 3.x compatibility.
    */
   def isLeftSingleJoinType(joinType: JoinType): Boolean = false
+
+  /**
+   * Extract seed value from SampleExec. In Spark 4.0+, seed is Option[Long], while in earlier
+   * versions it's Long. This method provides a unified interface across versions.
+   */
+  def getSampleExecSeed(plan: SampleExec): Long
 }
diff --git a/shims/pom.xml b/shims/pom.xml
@@ -98,6 +98,18 @@
         <module>spark41</module>
       </modules>
     </profile>
+    <profile>
+      <id>spark-4.2</id>
+      <modules>
+        <module>spark42</module>
+      </modules>
+    </profile>
+    <profile>
+      <id>spark-main</id>
+      <modules>
+        <module>sparkmain</module>
+      </modules>
+    </profile>
     <profile>
       <id>default</id>
       <activation>