Merge branch 'main' into table_maintenance_pypaimon

Author: nuts-rice

.github/workflows/ci.yml

@@ -38,7 +38,7 @@ jobs:
   check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
       - name: Validate .asf.yaml
         run: python3 scripts/validate_asf_yaml.py
@@ -69,7 +69,7 @@ jobs:
           - macos-latest
           - windows-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - name: Build
         run: cargo build --features fulltext,vortex,mosaic
 
@@ -82,7 +82,7 @@ jobs:
           - macos-latest
           - windows-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
       - name: Test
         run: cargo test -p paimon --all-targets --features fulltext,vortex,mosaic
@@ -101,12 +101,40 @@ jobs:
           RUST_BACKTRACE: full
 
   integration:
+    name: integration (${{ matrix.suite }})
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - suite: rust
+            needs_docker: true
+            needs_lumina: false
+            needs_uv: false
+          - suite: datafusion
+            needs_docker: true
+            needs_lumina: true
+            needs_uv: false
+          - suite: lumina
+            needs_docker: false
+            needs_lumina: true
+            needs_uv: false
+          - suite: python
+            needs_docker: true
+            needs_lumina: false
+            needs_uv: true
+          - suite: go
+            needs_docker: true
+            needs_lumina: false
+            needs_uv: false
+    env:
+      RUST_LOG: DEBUG
+      RUST_BACKTRACE: full
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
       - name: Rust Cache
-        uses: actions/cache@v5
+        uses: actions/cache@v6
         with:
           path: |
             ~/.cargo/registry
@@ -117,65 +145,60 @@ jobs:
             integration-${{ runner.os }}-
 
       - name: Start Docker containers
+        if: matrix.needs_docker == true
         run: make docker-up
 
       - name: Rust Integration Test
+        if: matrix.suite == 'rust'
         run: cargo test -p paimon-integration-tests --all-targets
-        env:
-          RUST_LOG: DEBUG
-          RUST_BACKTRACE: full
 
       - name: Install lumina native library
+        if: matrix.needs_lumina == true
         run: |
           pip install lumina-data
           echo "LUMINA_LIB_PATH=$(python3 -c 'import lumina_data; print(lumina_data.__path__[0])')/lib/liblumina_py.so" >> $GITHUB_ENV
 
       - name: Core Lumina Native Build Test
+        if: matrix.suite == 'lumina'
         run: >
           cargo test -p paimon
           table::lumina_index_build_builder::tests::test_execute_writes_lumina_index_manifest
           --features fulltext,vortex
           -- --ignored --exact
-        env:
-          RUST_LOG: DEBUG
-          RUST_BACKTRACE: full
 
       - name: DataFusion Lumina Build Query E2E Test
+        if: matrix.suite == 'lumina'
         run: >
           cargo test -p paimon-datafusion
           --features vortex
           vector_search_tests::test_lumina_build_then_vector_search_query
           -- --ignored --exact
-        env:
-          RUST_LOG: DEBUG
-          RUST_BACKTRACE: full
 
       - name: DataFusion Integration Test
+        if: matrix.suite == 'datafusion'
         run: cargo test -p paimon-datafusion --all-targets
-        env:
-          RUST_LOG: DEBUG
-          RUST_BACKTRACE: full
 
       - name: DataFusion Vortex Integration Test
+        if: matrix.suite == 'datafusion'
         run: cargo test -p paimon-datafusion --features vortex --test vortex_tables
-        env:
-          RUST_LOG: DEBUG
-          RUST_BACKTRACE: full
 
       - name: Install uv
-        uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b
+        if: matrix.needs_uv == true
+        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39
         with:
           version: "0.9.3"
           enable-cache: true
 
       - name: Python Binding Integration Test
+        if: matrix.suite == 'python'
         working-directory: bindings/python
         run: make install && make test
 
       - name: Go Integration Test
+        if: matrix.suite == 'go'
         working-directory: bindings/go
         run: make test
 
       - name: Stop Docker containers
-        if: always()
+        if: ${{ always() && matrix.needs_docker }}
         run: make docker-down

.github/workflows/release-go-binding.yml

@@ -47,7 +47,7 @@ jobs:
       tag: ${{ steps.meta.outputs.tag }}
       source_sha: ${{ steps.meta.outputs.source_sha }}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           fetch-depth: 0
           ref: ${{ inputs.source_ref || github.sha }}
@@ -111,7 +111,7 @@ jobs:
             expected_file: libpaimon_c.darwin.arm64.dylib.zst
 
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           ref: ${{ needs.validate.outputs.source_sha }}
 
@@ -121,7 +121,7 @@ jobs:
           rustup default stable
 
       - name: Cache Rust build outputs
-        uses: actions/cache@v5
+        uses: actions/cache@v6
         with:
           path: |
             ~/.cargo/bin/
@@ -162,7 +162,7 @@ jobs:
       - build
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           fetch-depth: 0
           ref: ${{ needs.validate.outputs.source_sha }}

.github/workflows/release-rust.yml

@@ -45,7 +45,7 @@ jobs:
           - paimon
           - paimon-datafusion
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
       - name: Setup Rust toolchain
         run: |

.github/workflows/release_python_binding.yml

@@ -41,7 +41,7 @@ jobs:
   sdist:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
       - uses: PyO3/maturin-action@v1
         with:
@@ -68,7 +68,7 @@ jobs:
           - { os: ubuntu-latest, target: "x86_64" }
           - { os: ubuntu-latest, target: "aarch64", manylinux: "manylinux_2_28" }
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
       - uses: PyO3/maturin-action@v1
         with:

Cargo.toml

@@ -30,14 +30,16 @@ rust-version = "1.91.0"
 [workspace.dependencies]
 arrow = "58.0"
 arrow-array = { version = "58.0", features = ["ffi"] }
+arrow-arith = "58.0"
 arrow-buffer = "58.0"
 arrow-schema = "58.0"
 arrow-cast = "58.0"
 arrow-ord = "58.0"
 arrow-row = "58.0"
 arrow-select = "58.0"
-datafusion = "53.0.0"
-datafusion-ffi = "53.0.0"
+arrow-string = "58.0"
+datafusion = "54.0.0"
+datafusion-ffi = "54.0.0"
 paimon = { version = "0.3.0", path = "crates/paimon" }
 parquet = "58.0"
 constant_time_eq = ">=0.4.0, <0.5.0"