Merge branch 'main' into dont-materialize

Author: Fokko

.asf.yaml

@@ -42,7 +42,9 @@ github:
         required_approving_review_count: 1
 
       required_linear_history: true
-  del_branch_on_merge: true
+  pull_requests:
+    # auto-delete head branches after being merged
+    del_branch_on_merge: true
   features:
     wiki: true
     issues: true
@@ -57,4 +59,5 @@ notifications:
     commits:      commits@iceberg.apache.org
     issues:       issues@iceberg.apache.org
     pullrequests: issues@iceberg.apache.org
+    jobs:         ci-jobs@iceberg.apache.org
     jira_options: link label link label

.github/ISSUE_TEMPLATE/iceberg_bug_report.yml

@@ -28,7 +28,8 @@ body:
       description: What Apache Iceberg version are you using?
       multiple: false
       options:
-        - "0.10.0 (latest release)"
+        - "0.11.0 (latest release)"
+        - "0.10.0"
         - "0.9.1"
         - "0.9.0"
         - "0.8.1"

.github/dependabot.yml

@@ -19,7 +19,7 @@
 
 version: 2
 updates:
-  - package-ecosystem: "pip"
+  - package-ecosystem: "uv"
     directory: "/"
     schedule:
       interval: "weekly"

.github/workflows/check-md-link.yml

@@ -30,10 +30,14 @@ on:
     paths:
       - '.github/workflows/check-md-link.yml'
       - 'mkdocs/**'
+  workflow_dispatch:
+
+permissions:
+  contents: read
 
 jobs:
   markdown-link-check:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@master
-    - uses: tcort/github-action-markdown-link-check@v1
+    - uses: tcort/github-action-markdown-link-check@e7c7a18363c842693fadde5d41a3bd3573a7a225

.github/workflows/codeql.yml

@@ -0,0 +1,54 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+  schedule:
+    - cron: '16 4 * * 1'
+
+permissions:
+  contents: read
+
+jobs:
+  analyze:
+    name: Analyze Actions
+    runs-on: ubuntu-slim
+    permissions:
+      contents: read
+      security-events: write
+      packages: read
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v6
+
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v4
+      with:
+        languages: actions
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v4
+      with:
+        category: "/language:actions"

.github/workflows/license_check.yml

@@ -20,9 +20,12 @@
 name: "Run License Check"
 on: pull_request
 
+permissions:
+  contents: read
+
 jobs:
   rat:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v5
+    - uses: actions/checkout@v6
     - run: dev/check-license

.github/workflows/nightly-pypi-build.yml

@@ -24,28 +24,31 @@ on:
     - cron: "0 0 * * *"  # Runs at midnight UTC every day
   workflow_dispatch:  # Allows manual triggering
 
+permissions:
+  contents: read
+
 jobs:
   set-version:
     if: github.repository == 'apache/iceberg-python'  # Only run for apache repo
     runs-on: ubuntu-latest
     outputs:
       VERSION: ${{ steps.set-version.outputs.VERSION }}
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 1
 
       - uses: actions/setup-python@v6
         with:
           python-version: 3.12
 
-      - name: Install Poetry
-        run: make install-poetry
+      - name: Install UV
+        uses: astral-sh/setup-uv@v7
 
       - name: Set version
         id: set-version
         run: |
-          CURRENT_VERSION=$(poetry version --short)
+          CURRENT_VERSION=$(uv version --short)
           TIMESTAMP=$(date +%Y%m%d%H%M%S)
           echo "VERSION=${CURRENT_VERSION}.dev${TIMESTAMP}" >> "$GITHUB_OUTPUT"
 
@@ -71,15 +74,42 @@ jobs:
 
     steps:
     - name: Download all the artifacts
-      uses: actions/download-artifact@v6
+      uses: actions/download-artifact@v8
       with:
         merge-multiple: true
         path: dist/
     - name: List downloaded artifacts
       run: ls -R dist/
     - name: Publish to TestPyPI
+      id: publish-testpypi
+      continue-on-error: true
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
         repository-url: https://test.pypi.org/legacy/
         skip-existing: true
         verbose: true
+    - name: Display error message on publish failure
+      if: steps.publish-testpypi.outcome == 'failure'
+      run: |
+        echo "::error::Failed to publish to TestPyPI"
+        echo ""
+        echo "⚠️  TestPyPI Publish Failed"
+        echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+        echo ""
+        echo "This may be due to TestPyPI storage limits."
+        echo "See: https://docs.pypi.org/project-management/storage-limits"
+        echo ""
+        echo "To resolve this issue, use the pypi-cleanup utility to clean up old TestPyPI artifacts:"
+        echo "https://pypi.org/project/pypi-cleanup/"
+        echo ""
+        echo "  uvx pypi-cleanup --package pyiceberg --host https://test.pypi.org/ \\"
+        echo "    --verbose -d 10 --do-it --username <username>"
+        echo ""
+        echo "Requirements:"
+        echo "  • Must be a maintainer for pyiceberg on TestPyPI"
+        echo "    (https://test.pypi.org/project/pyiceberg)"
+        echo "  • Requires TestPyPI password and 2FA"
+        echo "  • ⚠️  ONLY do this for TestPyPI, NOT for production PyPI!"
+        echo ""
+        echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+        exit 1

.github/workflows/pypi-build-artifacts.yml

@@ -26,16 +26,20 @@ on:
         required: true
         type: string
 
+permissions:
+  contents: read
+
 jobs:
   pypi-build-artifacts:
     name: Build artifacts for PyPi on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
+      max-parallel: 15
       matrix:
-        os: [ ubuntu-latest, windows-latest, macos-latest ]
+        os: [ ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-15-intel, macos-latest ]
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 1
 
@@ -45,38 +49,41 @@ jobs:
             3.10
             3.11
             3.12
+            3.13
 
-      - name: Install poetry
-        run: make install-poetry
+      - name: Install UV
+        uses: astral-sh/setup-uv@v7
 
       - name: Set version with RC
         env:
           VERSION: ${{ inputs.VERSION }}
-        run: python -m poetry version "${{ env.VERSION }}"
+        run: uv version "${{ env.VERSION }}"
 
       # Publish the source distribution with the version that's in
       # the repository, otherwise the tests will fail
       - name: Compile source distribution
-        run: python3 -m poetry build --format=sdist
-        if: startsWith(matrix.os, 'ubuntu')
+        run: uv build --sdist
+        if: matrix.os == 'ubuntu-latest'
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v3.2.1
+        uses: pypa/cibuildwheel@v3.3.1
         with:
           output-dir: wheelhouse
           config-file: "pyproject.toml"
         env:
           # Ignore 32 bit architectures
           CIBW_ARCHS: "auto64"
-          CIBW_PROJECT_REQUIRES_PYTHON: ">=3.10"
-          CIBW_TEST_REQUIRES: "pytest==7.4.2 moto==5.0.1"
-          CIBW_TEST_COMMAND: "pytest {project}/tests/avro/test_decoder.py"
-          # Ignore tests for pypy since not all dependencies are compiled for it
-          # and would require a local rust build chain
-          CIBW_TEST_SKIP: "pp*"
+          CIBW_PROJECT_REQUIRES_PYTHON: ">=3.10,<3.14"
+          # Keep these in sync with Python CI job `cibw-dev-env-smoke-test`
+          # in .github/workflows/python-ci.yml to catch import-time regressions early.
+          CIBW_BEFORE_TEST: "uv sync --directory {project} --only-group dev --no-install-project"
+          CIBW_TEST_COMMAND: "uv run --directory {project} pytest tests/avro/test_decoder.py"
+          # Skip free-threaded (PEP 703) builds until we evaluate decoder_fast support
+          CIBW_SKIP: "cp3*t-*"
+
 
       - name: Add source distribution
-        if: startsWith(matrix.os, 'ubuntu')
+        if: matrix.os == 'ubuntu-latest'
         run: ls -lah dist/* && cp dist/* wheelhouse/
 
       - uses: actions/upload-artifact@v4

.github/workflows/python-ci-docs.yml

@@ -25,24 +25,25 @@ on:
       - 'main'
   pull_request:
 
+permissions:
+  contents: read
+
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   docs:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-slim
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
       - uses: actions/setup-python@v6
         with:
           python-version: 3.12
-      - name: Install poetry
-        run: make install-poetry
-      - name: Install
-        run: make docs-install
+      - name: Install UV
+        uses: astral-sh/setup-uv@v7
       - name: Build docs
         run: make docs-build
       - name: Run linters