Merge branch 'main' into table_maintenance_pypaimon

Author: nuts-rice

.github/workflows/ci.yml

@@ -38,7 +38,7 @@ jobs:
   check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
       - name: Validate .asf.yaml
         run: python3 scripts/validate_asf_yaml.py
@@ -58,7 +58,7 @@ jobs:
         run: cargo fmt --all -- --check
 
       - name: Clippy
-        run: cargo clippy --all-targets --workspace --features fulltext,vortex,mosaic -- -D warnings
+        run: cargo clippy --all-targets --workspace --features fulltext,vortex -- -D warnings
 
   build:
     runs-on: ${{ matrix.os }}
@@ -69,9 +69,9 @@ jobs:
           - macos-latest
           - windows-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
       - name: Build
-        run: cargo build --features fulltext,vortex,mosaic
+        run: cargo build --features fulltext,vortex
 
   unit:
     runs-on: ${{ matrix.os }}
@@ -82,10 +82,10 @@ jobs:
           - macos-latest
           - windows-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
       - name: Test
-        run: cargo test -p paimon --all-targets --features fulltext,vortex,mosaic
+        run: cargo test -p paimon --all-targets --features fulltext,vortex
         env:
           RUST_LOG: DEBUG
           RUST_BACKTRACE: full
@@ -101,12 +101,40 @@ jobs:
           RUST_BACKTRACE: full
 
   integration:
+    name: integration (${{ matrix.suite }})
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - suite: rust
+            needs_docker: true
+            needs_lumina: false
+            needs_uv: false
+          - suite: datafusion
+            needs_docker: true
+            needs_lumina: true
+            needs_uv: false
+          - suite: lumina
+            needs_docker: false
+            needs_lumina: true
+            needs_uv: false
+          - suite: python
+            needs_docker: true
+            needs_lumina: false
+            needs_uv: true
+          - suite: go
+            needs_docker: true
+            needs_lumina: false
+            needs_uv: false
+    env:
+      RUST_LOG: DEBUG
+      RUST_BACKTRACE: full
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
       - name: Rust Cache
-        uses: actions/cache@v5
+        uses: actions/cache@v6
         with:
           path: |
             ~/.cargo/registry
@@ -117,65 +145,60 @@ jobs:
             integration-${{ runner.os }}-
 
       - name: Start Docker containers
+        if: matrix.needs_docker == true
         run: make docker-up
 
       - name: Rust Integration Test
+        if: matrix.suite == 'rust'
         run: cargo test -p paimon-integration-tests --all-targets
-        env:
-          RUST_LOG: DEBUG
-          RUST_BACKTRACE: full
 
       - name: Install lumina native library
+        if: matrix.needs_lumina == true
         run: |
           pip install lumina-data
           echo "LUMINA_LIB_PATH=$(python3 -c 'import lumina_data; print(lumina_data.__path__[0])')/lib/liblumina_py.so" >> $GITHUB_ENV
 
       - name: Core Lumina Native Build Test
+        if: matrix.suite == 'lumina'
         run: >
           cargo test -p paimon
           table::lumina_index_build_builder::tests::test_execute_writes_lumina_index_manifest
           --features fulltext,vortex
           -- --ignored --exact
-        env:
-          RUST_LOG: DEBUG
-          RUST_BACKTRACE: full
 
       - name: DataFusion Lumina Build Query E2E Test
+        if: matrix.suite == 'lumina'
         run: >
           cargo test -p paimon-datafusion
           --features vortex
           vector_search_tests::test_lumina_build_then_vector_search_query
           -- --ignored --exact
-        env:
-          RUST_LOG: DEBUG
-          RUST_BACKTRACE: full
 
       - name: DataFusion Integration Test
+        if: matrix.suite == 'datafusion'
         run: cargo test -p paimon-datafusion --all-targets
-        env:
-          RUST_LOG: DEBUG
-          RUST_BACKTRACE: full
 
       - name: DataFusion Vortex Integration Test
+        if: matrix.suite == 'datafusion'
         run: cargo test -p paimon-datafusion --features vortex --test vortex_tables
-        env:
-          RUST_LOG: DEBUG
-          RUST_BACKTRACE: full
 
       - name: Install uv
-        uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b
+        if: matrix.needs_uv == true
+        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39
         with:
           version: "0.9.3"
           enable-cache: true
 
       - name: Python Binding Integration Test
+        if: matrix.suite == 'python'
         working-directory: bindings/python
         run: make install && make test
 
       - name: Go Integration Test
+        if: matrix.suite == 'go'
         working-directory: bindings/go
         run: make test
 
       - name: Stop Docker containers
-        if: always()
+        if: ${{ always() && matrix.needs_docker }}
         run: make docker-down

.github/workflows/release-go-binding.yml

@@ -47,7 +47,7 @@ jobs:
       tag: ${{ steps.meta.outputs.tag }}
       source_sha: ${{ steps.meta.outputs.source_sha }}
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           fetch-depth: 0
           ref: ${{ inputs.source_ref || github.sha }}
@@ -111,7 +111,7 @@ jobs:
             expected_file: libpaimon_c.darwin.arm64.dylib.zst
 
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           ref: ${{ needs.validate.outputs.source_sha }}
 
@@ -121,7 +121,7 @@ jobs:
           rustup default stable
 
       - name: Cache Rust build outputs
-        uses: actions/cache@v5
+        uses: actions/cache@v6
         with:
           path: |
             ~/.cargo/bin/
@@ -162,7 +162,7 @@ jobs:
       - build
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
         with:
           fetch-depth: 0
           ref: ${{ needs.validate.outputs.source_sha }}

.github/workflows/release-rust.yml

@@ -45,7 +45,7 @@ jobs:
           - paimon
           - paimon-datafusion
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
       - name: Setup Rust toolchain
         run: |

.github/workflows/release_python_binding.yml

@@ -41,7 +41,7 @@ jobs:
   sdist:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
       - uses: PyO3/maturin-action@v1
         with:
@@ -68,7 +68,7 @@ jobs:
           - { os: ubuntu-latest, target: "x86_64" }
           - { os: ubuntu-latest, target: "aarch64", manylinux: "manylinux_2_28" }
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v7
 
       - uses: PyO3/maturin-action@v1
         with:

CONTRIBUTING.md

@@ -45,6 +45,30 @@ This is a pure Rust project, so only `cargo` is needed. Here are some common com
 2. Open a pull request on the project's GitHub page. Provide a clear description of your changes and why they are necessary.
 3. Wait for reviews and address any feedback. Once the pull request is approved and merged, your changes will be part of the project.
 
+## AI-Assisted Contributions
+
+Apache Paimon Rust has the following policy for AI-assisted pull requests:
+
+- The PR author should **understand the core ideas** behind the implementation **end-to-end** and be able to justify the design and code during review.
+- **Call out unknowns and assumptions.** It is acceptable not to fully understand some details of AI-generated code, but authors should identify these cases and point them out to reviewers. This allows reviewers to apply their knowledge of the codebase and evaluate potential concerns, such as correctness, concurrency, compatibility, or performance risks.
+
+### Why fully AI-generated PRs without understanding are not helpful
+
+AI tools cannot reliably make complex changes to Apache Paimon Rust on their own, which is why we rely on pull requests and code review.
+
+The purposes of code review are:
+
+1. Complete the intended task correctly.
+2. Share knowledge between authors and reviewers as a long-term investment in the project.
+
+A fully AI-generated contribution without sufficient author understanding does not meet these purposes. Maintainers could use AI tools directly, while contributors acting only as a proxy gain little knowledge of the project.
+
+Review capacity is limited, so large pull requests that appear to lack the required understanding may not be reviewed and may eventually be closed or redirected.
+
+### Better ways to contribute than an "AI dump"
+
+Consider writing a high-quality issue with a clear problem statement and a minimal reproducible example. This often makes it easier for the community to investigate the problem and develop an appropriate solution.
+
 ### Read the design docs
 
 For a deeper understanding of the project, read the design documentation available on our [Paimon Rust](https://paimon.apache.org/docs/rust/).

Cargo.toml

@@ -30,14 +30,16 @@ rust-version = "1.91.0"
 [workspace.dependencies]
 arrow = "58.0"
 arrow-array = { version = "58.0", features = ["ffi"] }
+arrow-arith = "58.0"
 arrow-buffer = "58.0"
 arrow-schema = "58.0"
 arrow-cast = "58.0"
 arrow-ord = "58.0"
 arrow-row = "58.0"
 arrow-select = "58.0"
-datafusion = "53.0.0"
-datafusion-ffi = "53.0.0"
+arrow-string = "58.0"
+datafusion = "54.0.0"
+datafusion-ffi = "54.0.0"
 paimon = { version = "0.3.0", path = "crates/paimon" }
 parquet = "58.0"
 constant_time_eq = ">=0.4.0, <0.5.0"