Merge remote-tracking branch 'origin/main' into pk-vec/2-batch-multi-query

# Conflicts:
#	crates/paimon/src/table/vector_search_builder.rs

Author: lijunrui

.gitattributes

@@ -1,2 +1,2 @@
-# Auto-generated dependency files — collapse in GitHub diffs
-*.tsv linguist-generated=true
+# Generated TSVs keep trailing empty columns.
+*.tsv text eol=lf linguist-generated=true whitespace=-blank-at-eol

.github/workflows/ci.yml

@@ -47,18 +47,46 @@ jobs:
         uses: apache/skywalking-eyes/header@v0.8.0
 
       - name: Install cargo-deny
-        uses: taiki-e/install-action@v2
+        uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0
         with:
-          tool: cargo-deny@0.19.0
+          tool: cargo-deny@0.19.6
 
-      - name: Check dependency licenses (Apache-compatible)
-        run: cargo deny check licenses
+      - name: Fetch locked dependencies
+        run: cargo fetch --locked
+
+      - name: Check dependency policy
+        run: cargo deny --locked --all-features check --warn unmaintained advisories licenses
+
+      - name: Verify dependency reports
+        run: python3 scripts/dependencies.py verify
 
       - name: Format
         run: cargo fmt --all -- --check
 
       - name: Clippy
-        run: cargo clippy --all-targets --workspace --features fulltext,vortex -- -D warnings
+        run: cargo clippy --locked --all-targets --workspace --features fulltext,vortex -- -D warnings
+
+  msrv:
+    runs-on: ubuntu-latest
+    env:
+      PYO3_PYTHON: python3.11
+    steps:
+      - uses: actions/checkout@v7
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Install Rust 1.91.0
+        run: rustup toolchain install 1.91.0 --profile minimal
+
+      - name: Validate workspace MSRV metadata
+        run: |
+          cargo +1.91.0 metadata --locked --format-version 1 --no-deps |
+            jq -e '.packages | all(.rust_version == "1.91.0")' >/dev/null
+
+      - name: Check workspace at MSRV
+        run: cargo +1.91.0 check --locked --workspace --all-targets --all-features
 
   build:
     runs-on: ${{ matrix.os }}
@@ -71,7 +99,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: Build
-        run: cargo build --features fulltext,vortex
+        run: cargo build --locked --features fulltext,vortex
 
   unit:
     runs-on: ${{ matrix.os }}
@@ -85,7 +113,7 @@ jobs:
       - uses: actions/checkout@v7
 
       - name: Test
-        run: cargo test -p paimon --all-targets --features fulltext,vortex
+        run: cargo test --locked -p paimon --all-targets --features fulltext,vortex
         env:
           RUST_LOG: DEBUG
           RUST_BACKTRACE: full
@@ -95,7 +123,7 @@ jobs:
       # catalog tests are likewise gated with `#[cfg(not(windows))]`).
       - name: Test paimon-rest-server
         if: runner.os != 'Windows'
-        run: cargo test -p paimon-rest-server --all-targets
+        run: cargo test --locked -p paimon-rest-server --all-targets
         env:
           RUST_LOG: DEBUG
           RUST_BACKTRACE: full
@@ -150,7 +178,7 @@ jobs:
 
       - name: Rust Integration Test
         if: matrix.suite == 'rust'
-        run: cargo test -p paimon-integration-tests --all-targets
+        run: cargo test --locked -p paimon-integration-tests --all-targets
 
       - name: Install lumina native library
         if: matrix.needs_lumina == true
@@ -161,26 +189,26 @@ jobs:
       - name: Core Lumina Native Build Test
         if: matrix.suite == 'lumina'
         run: >
-          cargo test -p paimon
+          cargo test --locked -p paimon
           table::lumina_index_build_builder::tests::test_execute_writes_lumina_index_manifest
           --features fulltext,vortex
           -- --ignored --exact
 
       - name: DataFusion Lumina Build Query E2E Test
         if: matrix.suite == 'lumina'
         run: >
-          cargo test -p paimon-datafusion
+          cargo test --locked -p paimon-datafusion
           --features vortex
           vector_search_tests::test_lumina_build_then_vector_search_query
           -- --ignored --exact
 
       - name: DataFusion Integration Test
         if: matrix.suite == 'datafusion'
-        run: cargo test -p paimon-datafusion --all-targets
+        run: cargo test --locked -p paimon-datafusion --all-targets
 
       - name: DataFusion Vortex Integration Test
         if: matrix.suite == 'datafusion'
-        run: cargo test -p paimon-datafusion --features vortex --test vortex_tables
+        run: cargo test --locked -p paimon-datafusion --features vortex --test vortex_tables
 
       - name: Install uv
         if: matrix.needs_uv == true

.github/workflows/release_python_binding.yml

@@ -16,7 +16,7 @@
 # under the License.
 
 # Publish the paimon Python binding to PyPI.
-# Trigger: push tag only (e.g. v0.1.0).
+# Trigger: release tag push or manual dispatch.
 # Pre-release tags (containing '-') publish to TestPyPI; release tags publish to PyPI.
 #
 # Token auth: add secrets PYPI_API_TOKEN / TEST_PYPI_API_TOKEN for publishing.
@@ -29,21 +29,37 @@ on:
       - "v[0-9]+.[0-9]+.[0-9]+"
       - "v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+"
   workflow_dispatch:
+    inputs:
+      tag:
+        description: Release tag, for example v0.3.0-rc2
+        required: true
+        type: string
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
+  group: ${{ github.workflow }}-${{ inputs.tag || github.ref_name }}
   cancel-in-progress: true
 
 permissions:
   contents: read
 
+env:
+  RELEASE_TAG: ${{ inputs.tag || github.ref_name }}
+
 jobs:
   sdist:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7
+        with:
+          ref: ${{ env.RELEASE_TAG }}
+
+      - name: Set Python release version
+        run: >
+          python3 scripts/python_release_version.py
+          --tag "${RELEASE_TAG}"
+          --pyproject bindings/python/pyproject.toml
 
-      - uses: PyO3/maturin-action@v1
+      - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0
         with:
           working-directory: bindings/python
           command: sdist
@@ -69,8 +85,17 @@ jobs:
           - { os: ubuntu-latest, target: "aarch64", manylinux: "manylinux_2_28" }
     steps:
       - uses: actions/checkout@v7
+        with:
+          ref: ${{ env.RELEASE_TAG }}
+
+      - name: Set Python release version
+        shell: bash
+        run: >
+          python3 scripts/python_release_version.py
+          --tag "${RELEASE_TAG}"
+          --pyproject bindings/python/pyproject.toml
 
-      - uses: PyO3/maturin-action@v1
+      - uses: PyO3/maturin-action@e83996d129638aa358a18fbd1dfb82f0b0fb5d3b # v1.51.0
         with:
           working-directory: bindings/python
           target: ${{ matrix.target }}
@@ -143,7 +168,7 @@ jobs:
     permissions:
       contents: read
     needs: [sdist, wheels]
-    if: startsWith(github.ref, 'refs/tags/')
+    if: startsWith(inputs.tag || github.ref_name, 'v')
     steps:
       - uses: actions/download-artifact@v8
         with:
@@ -152,17 +177,17 @@ jobs:
           path: bindings/python/dist
 
       - name: Publish to TestPyPI
-        if: contains(github.ref, '-')
-        uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b
+        if: contains(env.RELEASE_TAG, '-')
+        uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
         with:
           repository-url: https://test.pypi.org/legacy/
           skip-existing: true
           packages-dir: bindings/python/dist
           password: ${{ secrets.TEST_PYPI_API_TOKEN }}
 
       - name: Publish to PyPI
-        if: ${{ !contains(github.ref, '-') }}
-        uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b
+        if: ${{ !contains(env.RELEASE_TAG, '-') }}
+        uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
         with:
           skip-existing: true
           packages-dir: bindings/python/dist

.gitignore

@@ -16,7 +16,6 @@
 # under the License.
 
 /target
-/Cargo.lock
 .idea
 .vscode
 **/.DS_Store

.licenserc.yaml

@@ -23,6 +23,7 @@ header:
   paths-ignore:
     - "LICENSE"
     - "NOTICE"
+    - "Cargo.lock"
     - ".gitattributes"
     - ".github/PULL_REQUEST_TEMPLATE.md"
     - "crates/paimon/tests/**/*.json"