Merge pull request #58 from RelationalAI/vs-sync-upstream-df-reenable

Restore datafusion and sync upstream

Author: vustef

.github/workflows/audit.yml

@@ -36,6 +36,9 @@ on:
   schedule:
     - cron: '0 0 * * *'
 
+permissions:
+  contents: read
+
 jobs:
   security_audit:
     runs-on: ubuntu-latest

.github/workflows/bindings_python_ci.yml

@@ -40,20 +40,23 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
-  # check-rust:
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #     - uses: actions/checkout@v6
-  #     - name: Check format
-  #       working-directory: "bindings/python"
-  #       run: cargo fmt --all -- --check
-  #     - name: Check clippy
-  #       working-directory: "bindings/python"
-  #       run: cargo clippy --all-targets --all-features -- -D warnings
+  check-rust:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - name: Check format
+        working-directory: "bindings/python"
+        run: cargo fmt --all -- --check
+      - name: Check clippy
+        working-directory: "bindings/python"
+        run: cargo clippy --all-targets --all-features -- -D warnings
 
   check-python:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-slim
     steps:
       - uses: actions/checkout@v6
       - uses: astral-sh/setup-uv@v7
@@ -72,40 +75,41 @@ jobs:
         run: |
           uvx ruff check .
 
-  # test:
-  #   runs-on: ${{ matrix.os }}
-  #   strategy:
-  #     matrix:
-  #       os:
-  #         - ubuntu-latest
-  #         - macos-latest
-  #         - windows-latest
-  #   steps:
-  #     - uses: actions/checkout@v6
-  #     - uses: actions/setup-python@v6
-  #       with:
-  #         python-version: 3.12
-  #     - uses: PyO3/maturin-action@v1
-  #       with:
-  #         working-directory: "bindings/python"
-  #         command: build
-  #         args: --out dist --sdist
-  #     - uses: astral-sh/setup-uv@v7
-  #       with:
-  #         version: "0.9.3"
-  #         enable-cache: true
-  #     - name: Sync dependencies
-  #       working-directory: "bindings/python"
-  #       shell: bash
-  #       run: |
-  #         make install
-  #     - name: Install built wheel
-  #       working-directory: "bindings/python"
-  #       shell: bash
-  #       run: |
-  #         uv pip install --reinstall dist/pyiceberg_core-*.whl
-  #     - name: Run tests
-  #       working-directory: "bindings/python"
-  #       shell: bash
-  #       run: |
-  #         make test
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      max-parallel: 15
+      matrix:
+        os:
+          - ubuntu-latest
+          - macos-latest
+          - windows-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-python@v6
+        with:
+          python-version: 3.12
+      - uses: PyO3/maturin-action@v1
+        with:
+          working-directory: "bindings/python"
+          command: build
+          args: --out dist
+      - uses: astral-sh/setup-uv@v7
+        with:
+          version: "0.9.3"
+          enable-cache: true
+      - name: Sync dependencies
+        working-directory: "bindings/python"
+        shell: bash
+        run: |
+          make install
+      - name: Install built wheel
+        working-directory: "bindings/python"
+        shell: bash
+        run: |
+          uv pip install --reinstall dist/pyiceberg_core-*.whl
+      - name: Run tests
+        working-directory: "bindings/python"
+        shell: bash
+        run: |
+          make test

.github/workflows/ci.yml

@@ -40,10 +40,14 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   check:
     runs-on: ${{ matrix.os }}
     strategy:
+      max-parallel: 15
       matrix:
         os:
           - ubuntu-latest
@@ -80,6 +84,7 @@ jobs:
   build:
     runs-on: ${{ matrix.os }}
     strategy:
+      max-parallel: 15
       matrix:
         os:
           - ubuntu-latest
@@ -105,6 +110,7 @@ jobs:
   build_with_no_default_features:
     runs-on: ${{ matrix.os }}
     strategy:
+      max-parallel: 15
       matrix:
         os:
           - ubuntu-latest
@@ -125,6 +131,7 @@ jobs:
   tests:
     runs-on: ubuntu-latest
     strategy:
+      max-parallel: 15
       matrix:
         test-suite:
           - { name: "default", args: "--all-targets --all-features --workspace" }

.github/workflows/ci_typos.yml

@@ -32,14 +32,17 @@ concurrency:
 env:
   RUST_BACKTRACE: 1
 
+permissions:
+  contents: read
+
 jobs:
   typos-check:
     name: typos check
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-slim
     timeout-minutes: 10
     env:
       FORCE_COLOR: 1
     steps:
       - uses: actions/checkout@v6
       - name: Check typos
-        uses: crate-ci/typos@v1.43.3
+        uses: crate-ci/typos@v1.43.5

.github/workflows/codeql.yml

@@ -0,0 +1,54 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+  schedule:
+    - cron: '16 4 * * 1'
+
+permissions:
+  contents: read
+
+jobs:
+  analyze:
+    name: Analyze Actions
+    runs-on: ubuntu-slim
+    permissions:
+      contents: read
+      security-events: write
+      packages: read
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v6
+
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v4
+      with:
+        languages: actions
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v4
+      with:
+        category: "/language:actions"

.github/workflows/publish.yml

@@ -26,12 +26,15 @@ on:
       - "v[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+"
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   publish:
     runs-on: ubuntu-latest
     strategy:
       # Publish package one by one instead of flooding the registry
-      max-parallel: 1
+      max-parallel: 15
       matrix:
         # Order here is sensitive, as it will be used to determine the order of publishing
         package:

.github/workflows/release_python.yml

@@ -55,11 +55,14 @@ jobs:
         # Use input for workflow_dispatch, otherwise use `workflow_run.head_branch`
         # Note, `workflow_run.head_branch` does not contain `refs/tags/` prefix, just the tag name, i.e. `v0.4.0` or `v0.4.0-rc.1`
         # Valid formats: v<major>.<minor>.<patch> OR v<major>.<minor>.<patch>-rc.<release_candidate>
+        env:
+          DISPATCH_RELEASE_TAG: ${{ github.event.inputs.release_tag }}
+          RUN_HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}
         run: |
           if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
-            RELEASE_TAG="${{ github.event.inputs.release_tag }}"
+            RELEASE_TAG="$DISPATCH_RELEASE_TAG"
           else
-            RELEASE_TAG="${{ github.event.workflow_run.head_branch }}"
+            RELEASE_TAG="$RUN_HEAD_BRANCH"
           fi
           echo "Validating release tag: $RELEASE_TAG"
           if [[ ! "$RELEASE_TAG" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-rc\.[0-9]+)?$ ]]; then
@@ -142,6 +145,7 @@ jobs:
     runs-on: "${{ matrix.os }}"
     needs: [validate-release-tag]
     strategy:
+      max-parallel: 15
       matrix:
         include:
           - { os: windows-latest }

.github/workflows/release_python_nightly.yml

@@ -63,6 +63,7 @@ jobs:
     if: github.repository == 'apache/iceberg-rust' # Only run for apache repo
     runs-on: "${{ matrix.os }}"
     strategy:
+      max-parallel: 15
       matrix:
         include:
           - { os: windows-latest }

.github/workflows/stale.yml

@@ -17,28 +17,28 @@
 # under the License.
 #
 
-name: "Close Stale Issues"
+name: "Close Stale Issues and PRs"
 on:
   schedule:
     - cron: '0 0 * * *'
 
 permissions:
   # All other permissions are set to none
   issues: write
+  pull-requests: write
 
 jobs:
   stale:
     if: github.repository_owner == 'apache'
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
-      - uses: actions/stale@v10.1.1
+      - uses: actions/stale@v10.2.0
         with:
-          stale-issue-label: 'stale'
+          # stale issues
+          stale-issue-label: 'stale,security'
           exempt-issue-labels: 'not-stale'
           days-before-issue-stale: 180
           days-before-issue-close: 14
-          # Only close stale issues, leave PRs alone
-          days-before-pr-stale: -1
           stale-issue-message: >
             This issue has been automatically marked as stale because it has been open for 180 days
             with no activity. It will be closed in next 14 days if no further activity occurs. To
@@ -47,3 +47,12 @@ jobs:
           close-issue-message: >
             This issue has been closed because it has not received any activity in the last 14 days
             since being marked as 'stale'
+          # stale PRs
+          stale-pr-label: 'stale'
+          exempt-pr-labels: 'not-stale,security'
+          stale-pr-message: 'This pull request has been marked as stale due to 30 days of inactivity. It will be closed in 1 week if no further activity occurs. If you think that’s incorrect or this pull request requires a review, please simply write any comment. If closed, you can revive the PR at any time and @mention a reviewer or discuss it on the dev@iceberg.apache.org list. Thank you for your contributions.'
+          close-pr-message: 'This pull request has been closed due to lack of activity. This is not a judgement on the merit of the PR in any way. It is just a way of keeping the PR queue manageable. If you think that is incorrect, or the pull request requires review, you can revive the PR at any time.'
+          days-before-pr-stale: 30
+          days-before-pr-close: 7
+          ascending: true
+          operations-per-run: 200

.github/workflows/website.yml

@@ -30,6 +30,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   build:
     runs-on: ubuntu-latest