build: harden Rust crate publishing

.gitattributes

@@ -1,2 +1,2 @@
-# Auto-generated dependency files — collapse in GitHub diffs
-*.tsv linguist-generated=true
+# Generated TSVs keep trailing empty columns.
+*.tsv text eol=lf linguist-generated=true whitespace=-blank-at-eol

.github/workflows/ci.yml

@@ -47,18 +47,46 @@ jobs:
         uses: apache/skywalking-eyes/header@v0.8.0
 
       - name: Install cargo-deny
-        uses: taiki-e/install-action@v2
+        uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0
         with:
-          tool: cargo-deny@0.19.0
+          tool: cargo-deny@0.19.6
 
-      - name: Check dependency licenses (Apache-compatible)
-        run: cargo deny check licenses
+      - name: Fetch locked dependencies
+        run: cargo fetch --locked
+
+      - name: Check dependency policy
+        run: cargo deny --locked --all-features check --warn unmaintained advisories licenses
+
+      - name: Verify dependency reports
+        run: python3 scripts/dependencies.py verify
 
       - name: Format
         run: cargo fmt --all -- --check
 
       - name: Clippy
-        run: cargo clippy --all-targets --workspace --features fulltext,vortex -- -D warnings
+        run: cargo clippy --locked --all-targets --workspace --features fulltext,vortex -- -D warnings
+
+  msrv:
+    runs-on: ubuntu-latest
+    env:
+      PYO3_PYTHON: python3.11
+    steps:
+      - uses: actions/checkout@v7
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Install Rust 1.91.0
+        run: rustup toolchain install 1.91.0 --profile minimal
+
+      - name: Validate workspace MSRV metadata
+        run: |
+          cargo +1.91.0 metadata --locked --format-version 1 --no-deps |
+            jq -e '.packages | all(.rust_version == "1.91.0")' >/dev/null
+
+      - name: Check workspace at MSRV
+        run: cargo +1.91.0 check --locked --workspace --all-targets --all-features
 
   build:
     runs-on: ${{ matrix.os }}
@@ -71,7 +99,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: Build
-        run: cargo build --features fulltext,vortex
+        run: cargo build --locked --features fulltext,vortex
 
   unit:
     runs-on: ${{ matrix.os }}
@@ -85,7 +113,7 @@ jobs:
       - uses: actions/checkout@v7
 
       - name: Test
-        run: cargo test -p paimon --all-targets --features fulltext,vortex
+        run: cargo test --locked -p paimon --all-targets --features fulltext,vortex
         env:
           RUST_LOG: DEBUG
           RUST_BACKTRACE: full
@@ -95,7 +123,7 @@ jobs:
       # catalog tests are likewise gated with `#[cfg(not(windows))]`).
       - name: Test paimon-rest-server
         if: runner.os != 'Windows'
-        run: cargo test -p paimon-rest-server --all-targets
+        run: cargo test --locked -p paimon-rest-server --all-targets
         env:
           RUST_LOG: DEBUG
           RUST_BACKTRACE: full
@@ -150,7 +178,7 @@ jobs:
 
       - name: Rust Integration Test
         if: matrix.suite == 'rust'
-        run: cargo test -p paimon-integration-tests --all-targets
+        run: cargo test --locked -p paimon-integration-tests --all-targets
 
       - name: Install lumina native library
         if: matrix.needs_lumina == true
@@ -161,26 +189,26 @@ jobs:
       - name: Core Lumina Native Build Test
         if: matrix.suite == 'lumina'
         run: >
-          cargo test -p paimon
+          cargo test --locked -p paimon
           table::lumina_index_build_builder::tests::test_execute_writes_lumina_index_manifest
           --features fulltext,vortex
           -- --ignored --exact
 
       - name: DataFusion Lumina Build Query E2E Test
         if: matrix.suite == 'lumina'
         run: >
-          cargo test -p paimon-datafusion
+          cargo test --locked -p paimon-datafusion
           --features vortex
           vector_search_tests::test_lumina_build_then_vector_search_query
           -- --ignored --exact
 
       - name: DataFusion Integration Test
         if: matrix.suite == 'datafusion'
-        run: cargo test -p paimon-datafusion --all-targets
+        run: cargo test --locked -p paimon-datafusion --all-targets
 
       - name: DataFusion Vortex Integration Test
         if: matrix.suite == 'datafusion'
-        run: cargo test -p paimon-datafusion --features vortex --test vortex_tables
+        run: cargo test --locked -p paimon-datafusion --features vortex --test vortex_tables
 
       - name: Install uv
         if: matrix.needs_uv == true

.github/workflows/release-rust.yml

@@ -15,12 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# Publish paimon and paimon-datafusion crates to crates.io.
-#
-# Trigger: push a version tag (e.g. v0.1.0, v0.1.0-rc1).
-# Pre-release tags (containing '-') only run dry-run checks without publishing.
-#
-# Token auth: add secret CARGO_REGISTRY_TOKEN for crates.io publishing.
+# GA tags publish; RC tags validate only.
+# Secret: CARGO_REGISTRY_TOKEN.
 
 name: Release Rust
 
@@ -31,19 +27,15 @@ on:
       - "v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+"
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
 jobs:
-  publish:
+  validate:
     runs-on: ubuntu-latest
     permissions:
       contents: read
-    strategy:
-      # Publish crates sequentially to respect dependency order
-      max-parallel: 1
-      matrix:
-        # Order matters: paimon must be published before paimon-datafusion
-        package:
-          - paimon
-          - paimon-datafusion
     steps:
       - uses: actions/checkout@v7
 
@@ -52,24 +44,190 @@ jobs:
           rustup update stable
           rustup default stable
 
-      - name: Dry run
-        # Only dry-run paimon since paimon-datafusion depends on it
-        if: matrix.package == 'paimon'
-        run: cargo publish -p ${{ matrix.package }} --all-features --dry-run
+      - name: Validate release tag
+        if: github.ref_type == 'tag'
+        shell: bash
+        run: |
+          set -euo pipefail
+          if [[ ! "${GITHUB_REF_NAME}" =~ ^v([0-9]+\.[0-9]+\.[0-9]+)(-rc[0-9]+)?$ ]]; then
+            echo "::error::invalid release tag: ${GITHUB_REF_NAME}"
+            exit 1
+          fi
+          TAG_VERSION="${BASH_REMATCH[1]}"
+          WORKSPACE_VERSION=$(
+            cargo metadata --locked --format-version 1 --no-deps |
+              jq -er '
+                [.packages[] |
+                  select(.name == "paimon" or .name == "paimon-datafusion") |
+                  .version] as $versions |
+                if ($versions | length) == 2 and
+                   ($versions | unique | length) == 1
+                then $versions[0]
+                else error("release crate versions differ")
+                end
+              '
+          )
+          if [ "${TAG_VERSION}" != "${WORKSPACE_VERSION}" ]; then
+            echo "::error::tag ${TAG_VERSION} does not match workspace ${WORKSPACE_VERSION}"
+            exit 1
+          fi
+
+      - name: Check package inventory
+        shell: bash
+        run: |
+          set -euo pipefail
+          for crate_dir in crates/paimon crates/integrations/datafusion; do
+            for legal_file in LICENSE NOTICE; do
+              cmp --silent "${legal_file}" "${crate_dir}/${legal_file}" || {
+                echo "::error::${crate_dir}/${legal_file} differs from ${legal_file}"
+                exit 1
+              }
+            done
+          done
+
+          for package in paimon paimon-datafusion; do
+            files="/tmp/${package}-package-files.txt"
+            cargo package --locked -p "${package}" --list > "${files}"
+            for required in LICENSE NOTICE DEPENDENCIES.rust.tsv test_utils.rs; do
+              grep --fixed-strings --line-regexp --quiet "${required}" "${files}" || {
+                echo "::error::${package} package is missing ${required}"
+                exit 1
+              }
+            done
+            grep --quiet '^testdata/' "${files}" || {
+              echo "::error::${package} package is missing testdata"
+              exit 1
+            }
+          done
+
+          grep --fixed-strings --line-regexp --quiet \
+            blob_test_utils.rs /tmp/paimon-package-files.txt || {
+            echo "::error::paimon package is missing blob_test_utils.rs"
+            exit 1
+          }
+
+      - name: Package release crates
+        # Cargo verifies workspace dependencies through its temporary registry.
+        run: >
+          cargo package --locked
+          -p paimon
+          -p paimon-datafusion
+          --all-features
+
+      - name: Dry run paimon
+        run: cargo publish --locked -p paimon --all-features --dry-run
+
+  publish-paimon:
+    needs: validate
+    if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-')
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Setup Rust toolchain
+        run: |
+          rustup update stable
+          rustup default stable
 
-      - name: Publish ${{ matrix.package }} to crates.io
-        if: startsWith(github.ref, 'refs/tags/') && !contains(github.ref, '-')
-        run: cargo publish -p ${{ matrix.package }} --all-features
+      - name: Publish paimon if absent
+        shell: bash
         env:
           CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+        run: |
+          set -euo pipefail
+          VERSION=$(
+            cargo metadata --locked --format-version 1 --no-deps |
+              jq -er '.packages[] | select(.name == "paimon") | .version'
+          )
+          USER_AGENT="paimon-rust-release/1.0 (${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY})"
+          STATUS=$(
+            curl --silent --show-error --retry 3 \
+              --user-agent "${USER_AGENT}" \
+              --output /tmp/paimon-version.json \
+              --write-out '%{http_code}' \
+              "https://crates.io/api/v1/crates/paimon/${VERSION}"
+          )
+          if [ "${STATUS}" = 200 ]; then
+            jq -e --arg version "${VERSION}" '.version.num == $version' \
+              /tmp/paimon-version.json >/dev/null
+            echo "paimon ${VERSION} already exists; skipping"
+          elif [ "${STATUS}" = 404 ]; then
+            cargo publish --locked -p paimon --all-features
+          else
+            echo "::error::crates.io returned HTTP ${STATUS} for paimon ${VERSION}"
+            exit 1
+          fi
 
-      - name: Wait for crates.io to update
-        if: startsWith(github.ref, 'refs/tags/') && !contains(github.ref, '-') && matrix.package == 'paimon'
+      - name: Wait for paimon on crates.io
+        shell: bash
         run: |
-          VERSION=${GITHUB_REF_NAME#v}
-          for i in {1..10}; do
-            CURRENT=$(curl -s https://crates.io/api/v1/crates/paimon | jq -r '.crate.max_version')
-            [ "$CURRENT" = "$VERSION" ] && echo "✅ paimon $VERSION available" && exit 0
-            echo "Waiting for $VERSION (currently $CURRENT)... $i/10"
+          set -euo pipefail
+          VERSION=$(
+            cargo metadata --locked --format-version 1 --no-deps |
+              jq -er '.packages[] | select(.name == "paimon") | .version'
+          )
+          USER_AGENT="paimon-rust-release/1.0 (${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY})"
+          for attempt in {1..20}; do
+            CURRENT=$(
+              curl --fail-with-body --silent --show-error --retry 3 \
+                --user-agent "${USER_AGENT}" \
+                "https://crates.io/api/v1/crates/paimon/${VERSION}" |
+                jq -r '.version.num // empty' || true
+            )
+            if [ "${CURRENT}" = "${VERSION}" ]; then
+              echo "paimon ${VERSION} is available"
+              exit 0
+            fi
+            echo "Waiting for paimon ${VERSION}... ${attempt}/20"
             sleep 30
           done
+          echo "::error::paimon ${VERSION} did not appear before timeout"
+          exit 1
+
+  publish-paimon-datafusion:
+    needs: publish-paimon
+    if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-')
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Setup Rust toolchain
+        run: |
+          rustup update stable
+          rustup default stable
+
+      - name: Dry run paimon-datafusion
+        run: cargo publish --locked -p paimon-datafusion --all-features --dry-run
+
+      - name: Publish paimon-datafusion if absent
+        shell: bash
+        env:
+          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+        run: |
+          set -euo pipefail
+          VERSION=$(
+            cargo metadata --locked --format-version 1 --no-deps |
+              jq -er '.packages[] | select(.name == "paimon-datafusion") | .version'
+          )
+          USER_AGENT="paimon-rust-release/1.0 (${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY})"
+          STATUS=$(
+            curl --silent --show-error --retry 3 \
+              --user-agent "${USER_AGENT}" \
+              --output /tmp/paimon-datafusion-version.json \
+              --write-out '%{http_code}' \
+              "https://crates.io/api/v1/crates/paimon-datafusion/${VERSION}"
+          )
+          if [ "${STATUS}" = 200 ]; then
+            jq -e --arg version "${VERSION}" '.version.num == $version' \
+              /tmp/paimon-datafusion-version.json >/dev/null
+            echo "paimon-datafusion ${VERSION} already exists; skipping"
+          elif [ "${STATUS}" = 404 ]; then
+            cargo publish --locked -p paimon-datafusion --all-features
+          else
+            echo "::error::crates.io returned HTTP ${STATUS} for paimon-datafusion ${VERSION}"
+            exit 1
+          fi

.gitignore

@@ -16,7 +16,6 @@
 # under the License.
 
 /target
-/Cargo.lock
 .idea
 .vscode
 **/.DS_Store

.licenserc.yaml

@@ -23,6 +23,7 @@ header:
   paths-ignore:
     - "LICENSE"
     - "NOTICE"
+    - "Cargo.lock"
     - ".gitattributes"
     - ".github/PULL_REQUEST_TEMPLATE.md"
     - "crates/paimon/tests/**/*.json"