Merge branch 'main' of github.com:apache/iceberg-python into fd-rust

Author: Fokko

.github/workflows/python-ci.yml

@@ -43,9 +43,10 @@ concurrency:
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
-  lint-and-test:
+  lint-and-unit-test:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: true
       matrix:
         python: ['3.10', '3.11', '3.12', '3.13']
 
@@ -56,6 +57,8 @@ jobs:
         python-version: ${{ matrix.python }}
     - name: Install UV
       uses: astral-sh/setup-uv@v7
+      with:
+        enable-cache: true
     - name: Install system dependencies
       run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
     - name: Install
@@ -69,45 +72,130 @@ jobs:
 
   integration-test:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python: ['3.10', '3.11', '3.12', '3.13']
-
     steps:
     - uses: actions/checkout@v6
     - uses: actions/setup-python@v6
       with:
-        python-version: ${{ matrix.python }}
+        python-version: '3.12'
     - name: Install UV
       uses: astral-sh/setup-uv@v7
+      with:
+        enable-cache: true
     - name: Install system dependencies
       run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
     - name: Install
       run: make install
-
     - name: Run integration tests with coverage
       run: COVERAGE=1 make test-integration
     - name: Show debug logs
       if: ${{ failure() }}
-      run: docker compose -f dev/docker-compose.yml logs
+      run: docker compose -f dev/docker-compose-integration.yml logs
+    - name: Upload coverage data
+      uses: actions/upload-artifact@v4
+      with:
+        name: coverage-integration
+        path: .coverage*
+        include-hidden-files: true
 
+  integration-test-s3:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+    - uses: actions/setup-python@v6
+      with:
+        python-version: '3.12'
+    - name: Install UV
+      uses: astral-sh/setup-uv@v7
+      with:
+        enable-cache: true
+    - name: Install system dependencies
+      run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
+    - name: Install
+      run: make install
     - name: Run s3 integration tests with coverage
       run: COVERAGE=1 make test-s3
     - name: Show debug logs
       if: ${{ failure() }}
       run: docker compose -f dev/docker-compose.yml logs
+    - name: Upload coverage data
+      uses: actions/upload-artifact@v4
+      with:
+        name: coverage-s3
+        path: .coverage*
+        include-hidden-files: true
 
+  integration-test-adls:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+    - uses: actions/setup-python@v6
+      with:
+        python-version: '3.12'
+    - name: Install UV
+      uses: astral-sh/setup-uv@v7
+      with:
+        enable-cache: true
+    - name: Install system dependencies
+      run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
+    - name: Install
+      run: make install
     - name: Run adls integration tests with coverage
       run: COVERAGE=1 make test-adls
     - name: Show debug logs
       if: ${{ failure() }}
       run: docker compose -f dev/docker-compose-azurite.yml logs
+    - name: Upload coverage data
+      uses: actions/upload-artifact@v4
+      with:
+        name: coverage-adls
+        path: .coverage*
+        include-hidden-files: true
 
+  integration-test-gcs:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+    - uses: actions/setup-python@v6
+      with:
+        python-version: '3.12'
+    - name: Install UV
+      uses: astral-sh/setup-uv@v7
+      with:
+        enable-cache: true
+    - name: Install system dependencies
+      run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
+    - name: Install
+      run: make install
     - name: Run gcs integration tests with coverage
       run: COVERAGE=1 make test-gcs
     - name: Show debug logs
       if: ${{ failure() }}
       run: docker compose -f dev/docker-compose-gcs-server.yml logs
+    - name: Upload coverage data
+      uses: actions/upload-artifact@v4
+      with:
+        name: coverage-gcs
+        path: .coverage*
+        include-hidden-files: true
 
+  integration-coverage-report:
+    runs-on: ubuntu-latest
+    needs: [integration-test, integration-test-s3, integration-test-adls, integration-test-gcs]
+    steps:
+    - uses: actions/checkout@v6
+    - uses: actions/setup-python@v6
+      with:
+        python-version: '3.12'
+    - name: Install UV
+      uses: astral-sh/setup-uv@v7
+      with:
+        enable-cache: true
+    - name: Install dependencies
+      run: uv sync --group dev
+    - name: Download all coverage artifacts
+      uses: actions/download-artifact@v4
+      with:
+        pattern: coverage-*
+        merge-multiple: true
     - name: Generate coverage report (75%) # Coverage threshold should only increase over time — never decrease it!
       run: COVERAGE_FAIL_UNDER=75 make coverage-report

.gitignore

@@ -41,6 +41,9 @@ bin/
 .mypy_cache/
 htmlcov
 
+# Jupyter notebook checkpoints
+.ipynb_checkpoints/
+
 pyiceberg/avro/decoder_fast.c
 pyiceberg/avro/*.html
 pyiceberg/avro/*.so

.pre-commit-config.yaml

@@ -32,6 +32,11 @@ repos:
       - id: ruff
         args: [ --fix, --exit-non-zero-on-fix ]
       - id: ruff-format
+  - repo: https://github.com/nbQA-dev/nbQA
+    rev: 1.9.1
+    hooks:
+      - id: nbqa-ruff
+        args: [ --fix, --exit-non-zero-on-fix ]
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v1.18.2
     hooks:
@@ -70,3 +75,11 @@ repos:
     rev: v2.4.1
     hooks:
       - id: codespell
+  - repo: local
+    hooks:
+      - id: uv-lock-check
+        name: uv lock file check
+        entry: make uv-lock-check
+        language: system
+        pass_filenames: false
+        files: ^(pyproject\.toml|uv\.lock)$

Makefile

@@ -72,7 +72,10 @@ setup-venv: ## Create virtual environment
 install-dependencies: setup-venv ## Install all dependencies including extras
 	uv sync $(PYTHON_ARG) --all-extras --reinstall
 
-install: install-uv install-dependencies ## Install uv and dependencies
+install-hooks: ## Install pre-commit hooks
+	uv run $(PYTHON_ARG) prek install
+
+install: install-uv install-dependencies install-hooks ## Install uv, dependencies, and pre-commit hooks
 
 # ===============
 # Code Validation
@@ -97,7 +100,7 @@ test: ## Run all unit tests (excluding integration)
 
 test-integration: test-integration-setup test-integration-exec test-integration-cleanup ## Run integration tests
 
-test-integration-setup: ## Start Docker services for integration tests
+test-integration-setup: install ## Start Docker services for integration tests
 	docker compose -f dev/docker-compose-integration.yml kill
 	docker compose -f dev/docker-compose-integration.yml rm -f
 	docker compose -f dev/docker-compose-integration.yml up -d --build --wait
@@ -153,6 +156,21 @@ docs-serve: ## Serve local docs preview (hot reload)
 docs-build: ## Build the static documentation site
 	uv run $(PYTHON_ARG) mkdocs build -f mkdocs/mkdocs.yml --strict
 
+# ========================
+# Experimentation
+# ========================
+
+##@ Experimentation
+
+notebook-install: ## Install notebook dependencies
+	uv sync $(PYTHON_ARG) --all-extras --group notebook
+
+notebook: notebook-install ## Launch notebook for experimentation
+	uv run jupyter lab --notebook-dir=notebooks
+
+notebook-infra: notebook-install test-integration-setup ## Launch notebook with integration test infra (Spark, Iceberg Rest Catalog, object storage, etc.)
+	uv run jupyter lab --notebook-dir=notebooks
+
 # ===================
 # Project Maintenance
 # ===================
@@ -167,4 +185,14 @@ clean: ## Remove build artifacts and caches
 	@find . -name "__pycache__" -exec echo Deleting {} \; -exec rm -rf {} +
 	@find . -name "*.pyd" -exec echo Deleting {} \; -delete
 	@find . -name "*.pyo" -exec echo Deleting {} \; -delete
+	@echo "Cleaning up Jupyter notebook checkpoints..."
+	@find . -name ".ipynb_checkpoints" -exec echo Deleting {} \; -exec rm -rf {} +
 	@echo "Cleanup complete."
+
+uv-lock: ## Regenerate uv.lock file from pyproject.toml
+	uv lock $(PYTHON_ARG)
+
+uv-lock-check: ## Verify uv.lock is up to date
+	@command -v uv >/dev/null || \
+	  (echo "uv is required. Run 'make install' or 'make install-uv' first." && exit 1)
+	uv lock --check $(PYTHON_ARG)

dev/.rat-excludes

@@ -5,3 +5,4 @@ build
 .gitignore
 uv.lock
 mkdocs/*
+notebooks/*

dev/docker-compose-integration.yml

@@ -17,21 +17,22 @@
 
 services:
   spark-iceberg:
-    container_name: pyiceberg-spark
+    image: pyiceberg-spark:latest
     build: spark/
+    container_name: pyiceberg-spark
     networks:
       iceberg_net:
     depends_on:
       - rest
       - hive
       - minio
+    ports:
+      - 15002:15002 # Spark Connect
+      - 4040:4040 # Spark UI
     environment:
       - AWS_ACCESS_KEY_ID=admin
       - AWS_SECRET_ACCESS_KEY=password
       - AWS_REGION=us-east-1
-    ports:
-      - 15002:15002 # Spark Connect
-      - 4040:4040 # Spark UI
     links:
       - rest:rest
       - hive:hive
@@ -60,25 +61,25 @@ services:
   minio:
     image: minio/minio
     container_name: pyiceberg-minio
-    environment:
-      - MINIO_ROOT_USER=admin
-      - MINIO_ROOT_PASSWORD=password
-      - MINIO_DOMAIN=minio
     networks:
       iceberg_net:
         aliases:
           - warehouse.minio
     ports:
       - 9001:9001
       - 9000:9000
+    environment:
+      - MINIO_ROOT_USER=admin
+      - MINIO_ROOT_PASSWORD=password
+      - MINIO_DOMAIN=minio
     command: ["server", "/data", "--console-address", ":9001"]
   mc:
-    depends_on:
-      - minio
     image: minio/mc
     container_name: pyiceberg-mc
     networks:
       iceberg_net:
+    depends_on:
+      - minio
     environment:
       - AWS_ACCESS_KEY_ID=admin
       - AWS_SECRET_ACCESS_KEY=password
@@ -91,6 +92,7 @@ services:
       tail -f /dev/null
       "
   hive:
+    image: pyiceberg-hive:latest
     build: hive/
     container_name: pyiceberg-hive
     hostname: hive

dev/hive/Dockerfile

@@ -15,17 +15,28 @@
 
 FROM apache/hive:4.0.0
 
-ENV HADOOP_VERSION=3.3.6
-ENV AWS_SDK_BUNDLE=1.12.753
+# Dependency versions - changing these invalidates the JAR download layer
+ARG HADOOP_VERSION=3.3.6
+ARG AWS_SDK_BUNDLE=1.12.753
+ARG MAVEN_MIRROR=https://repo1.maven.org/maven2
 
 USER root
 
-# Install curl, download JARs, and cleanup in a single layer
-RUN apt-get update -qq && apt-get -qq -y install curl && \
-    curl https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar -Lo /opt/hive/lib/hadoop-aws-${HADOOP_VERSION}.jar && \
-    curl https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS_SDK_BUNDLE}/aws-java-sdk-bundle-${AWS_SDK_BUNDLE}.jar -Lo /opt/hive/lib/aws-java-sdk-bundle-${AWS_SDK_BUNDLE}.jar && \
-    apt-get clean && rm -rf /var/lib/apt/lists/*
+# Install curl (separate layer - rarely changes)
+RUN apt-get update -qq && \
+    apt-get -qq -y install --no-install-recommends curl && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
+# Download JARs with retry logic (slow layer - only changes when versions change)
+RUN curl -fsSL --retry 3 --retry-delay 5 \
+        -o /opt/hive/lib/hadoop-aws-${HADOOP_VERSION}.jar \
+        "${MAVEN_MIRROR}/org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar" && \
+    curl -fsSL --retry 3 --retry-delay 5 \
+        -o /opt/hive/lib/aws-java-sdk-bundle-${AWS_SDK_BUNDLE}.jar \
+        "${MAVEN_MIRROR}/com/amazonaws/aws-java-sdk-bundle/${AWS_SDK_BUNDLE}/aws-java-sdk-bundle-${AWS_SDK_BUNDLE}.jar"
+
+# Copy configuration last (changes more frequently than JARs)
 COPY core-site.xml /opt/hadoop/etc/hadoop/core-site.xml
 
 USER hive