build(bindings): package complete legal metadata (#561)

Author: XiaoHongbo-Hope

.gitattributes

@@ -1,2 +1,13 @@
 # Generated TSVs keep trailing empty columns.
 *.tsv text eol=lf linguist-generated=true whitespace=-blank-at-eol
+
+# Legal files are verified byte-for-byte.
+LICENSE text eol=lf
+NOTICE text eol=lf
+**/LICENSE text eol=lf
+**/NOTICE text eol=lf
+**/THIRD-PARTY-LICENSES*.html text eol=lf linguist-generated=true
+third-party-licenses/* text eol=lf
+
+# Generated wheel legal files must never enter the ASF source archive.
+bindings/python/licenses/** export-ignore

.github/workflows/ci.yml

@@ -39,18 +39,72 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+
+      - name: Detect release license inputs
+        id: release_license_inputs
+        shell: bash
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          BEFORE_SHA: ${{ github.event.before }}
+          BASE_REF: ${{ github.base_ref }}
+        run: |
+          set -euo pipefail
+          if [[ "$EVENT_NAME" == "pull_request" ]]; then
+            base="origin/$BASE_REF"
+          elif [[ -n "$BEFORE_SHA" && ! "$BEFORE_SHA" =~ ^0+$ ]] && \
+            git cat-file -e "$BEFORE_SHA^{commit}"; then
+            base="$BEFORE_SHA"
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+
+          if git diff --quiet "$base" HEAD -- \
+            .gitattributes \
+            LICENSE \
+            NOTICE \
+            ':(glob)**/LICENSE' \
+            ':(glob)**/LICENSE.*' \
+            ':(glob)**/NOTICE' \
+            ':(glob)**/NOTICE.*' \
+            Cargo.lock \
+            about.hbs \
+            about.toml \
+            bindings/go/LICENSE \
+            bindings/go/NOTICE \
+            ':(glob)bindings/go/THIRD-PARTY-LICENSES.*.html' \
+            bindings/python/licenses \
+            bindings/python/THIRD-PARTY-LICENSES.html \
+            scripts/release_licenses.py \
+            third-party-licenses \
+            ':(glob)**/Cargo.toml'; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
 
       - name: Validate .asf.yaml
         run: python3 scripts/validate_asf_yaml.py
 
       - name: Check License Header
         uses: apache/skywalking-eyes/header@v0.8.0
 
+      - name: Test release artifact verifiers
+        run: python3 -m unittest discover -s scripts/tests -p 'test_*.py'
+
       - name: Install cargo-deny
         uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0
         with:
           tool: cargo-deny@0.19.6
 
+      - name: Install cargo-about
+        if: steps.release_license_inputs.outputs.changed == 'true'
+        uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0
+        with:
+          tool: cargo-about@0.9.1
+
       - name: Fetch locked dependencies
         run: cargo fetch --locked
 
@@ -60,6 +114,10 @@ jobs:
       - name: Verify dependency reports
         run: python3 scripts/dependencies.py verify
 
+      - name: Verify release license files
+        if: steps.release_license_inputs.outputs.changed == 'true'
+        run: python3 scripts/release_licenses.py --check
+
       - name: Format
         run: cargo fmt --all -- --check
 

.github/workflows/release-go-binding.yml

@@ -36,7 +36,7 @@ on:
         type: string
 
 permissions:
-  contents: write
+  contents: read
 
 jobs:
   validate:
@@ -103,12 +103,20 @@ jobs:
         include:
           - runner: ubuntu-latest
             expected_file: libpaimon_c.linux.amd64.so.zst
+            rust_target: x86_64-unknown-linux-gnu
+            license_file: THIRD-PARTY-LICENSES.linux.amd64.html
           - runner: ubuntu-24.04-arm
             expected_file: libpaimon_c.linux.arm64.so.zst
+            rust_target: aarch64-unknown-linux-gnu
+            license_file: THIRD-PARTY-LICENSES.linux.arm64.html
           - runner: macos-15-intel
             expected_file: libpaimon_c.darwin.amd64.dylib.zst
+            rust_target: x86_64-apple-darwin
+            license_file: THIRD-PARTY-LICENSES.darwin.amd64.html
           - runner: macos-14
             expected_file: libpaimon_c.darwin.arm64.dylib.zst
+            rust_target: aarch64-apple-darwin
+            license_file: THIRD-PARTY-LICENSES.darwin.arm64.html
 
     steps:
       - uses: actions/checkout@v7
@@ -120,6 +128,10 @@ jobs:
           rustup update stable
           rustup default stable
 
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
       - name: Cache Rust build outputs
         uses: actions/cache@v6
         with:
@@ -131,6 +143,11 @@ jobs:
             target/
           key: ${{ runner.os }}-${{ runner.arch }}-cargo-${{ hashFiles('**/Cargo.lock') }}
 
+      - name: Install cargo-about
+        uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0
+        with:
+          tool: cargo-about@0.9.1
+
       - name: Install zstd on Linux
         if: runner.os == 'Linux'
         run: sudo apt-get update && sudo apt-get install -y zstd
@@ -143,48 +160,69 @@ jobs:
         working-directory: bindings/go
         run: make build
 
-      - name: Verify packaged library
-        working-directory: bindings/go
+      - name: Stage and verify packaged library
         shell: bash
         run: |
-          test -s '${{ matrix.expected_file }}'
+          mkdir -p release/go
+          cargo fetch --locked
+          python3 scripts/release_licenses.py \
+            --generate-go-report \
+            '${{ matrix.rust_target }}' \
+            release/go
+          cp 'bindings/go/${{ matrix.expected_file }}' release/go/
+          python3 scripts/verify_go_release_artifacts.py \
+            --artifacts-dir release/go \
+            --target '${{ matrix.rust_target }}'
 
       - name: Upload packaged library
         uses: actions/upload-artifact@v7
         with:
           name: ${{ matrix.expected_file }}
-          path: bindings/go/${{ matrix.expected_file }}
+          path: |
+            release/go/${{ matrix.expected_file }}
+            release/go/${{ matrix.license_file }}
           if-no-files-found: error
 
   publish:
     needs:
       - validate
       - build
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
     steps:
       - uses: actions/checkout@v7
         with:
           fetch-depth: 0
           ref: ${{ needs.validate.outputs.source_sha }}
 
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Install cargo-about
+        uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0
+        with:
+          tool: cargo-about@0.9.1
+
+      - name: Fetch locked release inputs
+        run: cargo fetch --locked
+
+      - name: Install artifact verification tools
+        run: sudo apt-get update && sudo apt-get install -y file zstd
+
       - name: Download packaged libraries
         uses: actions/download-artifact@v8
         with:
-          path: bindings/go
+          path: release/go
           merge-multiple: true
 
-      - name: Verify embedded assets
-        working-directory: bindings/go
+      - name: Generate and verify release legal files
         shell: bash
         run: |
-          for file in \
-            libpaimon_c.linux.amd64.so.zst \
-            libpaimon_c.linux.arm64.so.zst \
-            libpaimon_c.darwin.amd64.dylib.zst \
-            libpaimon_c.darwin.arm64.dylib.zst
-          do
-            test -s "$file"
-          done
+          python3 scripts/release_licenses.py \
+            --generate-go-release release/go
+          python3 scripts/verify_go_release_artifacts.py --artifacts-dir release/go
 
       - name: Create release tag commit
         env:
@@ -197,7 +235,17 @@ jobs:
           git config user.name 'github-actions[bot]'
           git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
 
-          git add bindings/go/libpaimon_c.*.zst
+          cp release/go/libpaimon_c.*.zst bindings/go/
+          cp \
+            release/go/LICENSE \
+            release/go/NOTICE \
+            release/go/THIRD-PARTY-LICENSES.*.html \
+            bindings/go/
+          git add \
+            bindings/go/libpaimon_c.*.zst \
+            bindings/go/LICENSE \
+            bindings/go/NOTICE \
+            bindings/go/THIRD-PARTY-LICENSES.*.html
 
           if ! git diff --cached --quiet; then
             git commit -m "bindings/go: release ${VERSION} (from ${SOURCE_REF}@${SOURCE_SHA})"
@@ -211,12 +259,18 @@ jobs:
           TAG: ${{ needs.validate.outputs.tag }}
           VERSION: ${{ needs.validate.outputs.version }}
           GH_TOKEN: ${{ github.token }}
-        working-directory: bindings/go
+        working-directory: release/go
         run: |
           gh release create "$TAG" \
             --title "Release Go binding $VERSION" \
             --generate-notes \
+            LICENSE \
+            NOTICE \
             libpaimon_c.linux.amd64.so.zst \
             libpaimon_c.linux.arm64.so.zst \
             libpaimon_c.darwin.amd64.dylib.zst \
-            libpaimon_c.darwin.arm64.dylib.zst
+            libpaimon_c.darwin.arm64.dylib.zst \
+            THIRD-PARTY-LICENSES.linux.amd64.html \
+            THIRD-PARTY-LICENSES.linux.arm64.html \
+            THIRD-PARTY-LICENSES.darwin.amd64.html \
+            THIRD-PARTY-LICENSES.darwin.arm64.html