Skip to content

Commit ed0f72f

Browse files
FokkoFokko
authored
Merge branch 'main' into upsert-optimization
2 parents 1d6dad1 + d587aab commit ed0f72f

File tree

101 files changed

+6942
-1346
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

101 files changed

+6942
-1346
lines changed

‎.github/workflows/codeql.yml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
name: "CodeQL"
21+
22+
on:
23+
push:
24+
branches: [ "main" ]
25+
pull_request:
26+
branches: [ "main" ]
27+
schedule:
28+
- cron: '16 4 * * 1'
29+
30+
jobs:
31+
analyze:
32+
name: Analyze Actions
33+
runs-on: ubuntu-latest
34+
permissions:
35+
security-events: write
36+
packages: read
37+
38+
steps:
39+
- name: Checkout repository
40+
uses: actions/checkout@v4
41+
42+
- name: Initialize CodeQL
43+
uses: github/codeql-action/init@v4
44+
with:
45+
languages: actions
46+
47+
- name: Perform CodeQL Analysis
48+
uses: github/codeql-action/analyze@v4
49+
with:
50+
category: "/language:actions"

‎.github/workflows/pypi-build-artifacts.yml‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,9 @@ jobs:
3131
name: Build artifacts for PyPi on ${{ matrix.os }}
3232
runs-on: ${{ matrix.os }}
3333
strategy:
34+
max-parallel: 15
3435
matrix:
35-
os: [ ubuntu-latest, windows-latest, macos-latest ]
36+
os: [ ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-15-intel, macos-latest ]
3637

3738
steps:
3839
- uses: actions/checkout@v6
@@ -59,7 +60,7 @@ jobs:
5960
# the repository, otherwise the tests will fail
6061
- name: Compile source distribution
6162
run: uv build --sdist
62-
if: startsWith(matrix.os, 'ubuntu')
63+
if: matrix.os == 'ubuntu-latest'
6364

6465
- name: Build wheels
6566
uses: pypa/cibuildwheel@v3.3.1
@@ -72,15 +73,12 @@ jobs:
7273
CIBW_PROJECT_REQUIRES_PYTHON: ">=3.10,<3.14"
7374
CIBW_BEFORE_TEST: "uv sync --directory {project} --only-group dev --no-install-project"
7475
CIBW_TEST_COMMAND: "uv run --directory {project} pytest tests/avro/test_decoder.py"
75-
# Ignore tests for pypy since not all dependencies are compiled for it
76-
# and would require a local rust build chain
77-
CIBW_TEST_SKIP: "pp*"
7876
# Skip free-threaded (PEP 703) builds until we evaluate decoder_fast support
7977
CIBW_SKIP: "cp3*t-*"
8078

8179

8280
- name: Add source distribution
83-
if: startsWith(matrix.os, 'ubuntu')
81+
if: matrix.os == 'ubuntu-latest'
8482
run: ls -lah dist/* && cp dist/* wheelhouse/
8583

8684
- uses: actions/upload-artifact@v4

‎.github/workflows/python-ci.yml‎

Lines changed: 98 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,11 @@ concurrency:
4343
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
4444

4545
jobs:
46-
lint-and-test:
46+
lint-and-unit-test:
4747
runs-on: ubuntu-latest
4848
strategy:
49+
max-parallel: 15
50+
fail-fast: true
4951
matrix:
5052
python: ['3.10', '3.11', '3.12', '3.13']
5153

@@ -56,10 +58,12 @@ jobs:
5658
python-version: ${{ matrix.python }}
5759
- name: Install UV
5860
uses: astral-sh/setup-uv@v7
61+
with:
62+
enable-cache: true
5963
- name: Install system dependencies
6064
run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
6165
- name: Install
62-
run: make install-dependencies
66+
run: make install
6367
- name: Run linters
6468
run: make lint
6569
- name: Run unit tests with coverage
@@ -69,45 +73,130 @@ jobs:
6973

7074
integration-test:
7175
runs-on: ubuntu-latest
72-
strategy:
73-
matrix:
74-
python: ['3.10', '3.11', '3.12', '3.13']
75-
7676
steps:
7777
- uses: actions/checkout@v6
7878
- uses: actions/setup-python@v6
7979
with:
80-
python-version: ${{ matrix.python }}
80+
python-version: '3.12'
8181
- name: Install UV
8282
uses: astral-sh/setup-uv@v7
83+
with:
84+
enable-cache: true
8385
- name: Install system dependencies
8486
run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
8587
- name: Install
8688
run: make install
87-
8889
- name: Run integration tests with coverage
8990
run: COVERAGE=1 make test-integration
9091
- name: Show debug logs
9192
if: ${{ failure() }}
92-
run: docker compose -f dev/docker-compose.yml logs
93+
run: docker compose -f dev/docker-compose-integration.yml logs
94+
- name: Upload coverage data
95+
uses: actions/upload-artifact@v4
96+
with:
97+
name: coverage-integration
98+
path: .coverage*
99+
include-hidden-files: true
93100

101+
integration-test-s3:
102+
runs-on: ubuntu-latest
103+
steps:
104+
- uses: actions/checkout@v6
105+
- uses: actions/setup-python@v6
106+
with:
107+
python-version: '3.12'
108+
- name: Install UV
109+
uses: astral-sh/setup-uv@v7
110+
with:
111+
enable-cache: true
112+
- name: Install system dependencies
113+
run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
114+
- name: Install
115+
run: make install
94116
- name: Run s3 integration tests with coverage
95117
run: COVERAGE=1 make test-s3
96118
- name: Show debug logs
97119
if: ${{ failure() }}
98120
run: docker compose -f dev/docker-compose.yml logs
121+
- name: Upload coverage data
122+
uses: actions/upload-artifact@v4
123+
with:
124+
name: coverage-s3
125+
path: .coverage*
126+
include-hidden-files: true
99127

128+
integration-test-adls:
129+
runs-on: ubuntu-latest
130+
steps:
131+
- uses: actions/checkout@v6
132+
- uses: actions/setup-python@v6
133+
with:
134+
python-version: '3.12'
135+
- name: Install UV
136+
uses: astral-sh/setup-uv@v7
137+
with:
138+
enable-cache: true
139+
- name: Install system dependencies
140+
run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
141+
- name: Install
142+
run: make install
100143
- name: Run adls integration tests with coverage
101144
run: COVERAGE=1 make test-adls
102145
- name: Show debug logs
103146
if: ${{ failure() }}
104147
run: docker compose -f dev/docker-compose-azurite.yml logs
148+
- name: Upload coverage data
149+
uses: actions/upload-artifact@v4
150+
with:
151+
name: coverage-adls
152+
path: .coverage*
153+
include-hidden-files: true
105154

155+
integration-test-gcs:
156+
runs-on: ubuntu-latest
157+
steps:
158+
- uses: actions/checkout@v6
159+
- uses: actions/setup-python@v6
160+
with:
161+
python-version: '3.12'
162+
- name: Install UV
163+
uses: astral-sh/setup-uv@v7
164+
with:
165+
enable-cache: true
166+
- name: Install system dependencies
167+
run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
168+
- name: Install
169+
run: make install
106170
- name: Run gcs integration tests with coverage
107171
run: COVERAGE=1 make test-gcs
108172
- name: Show debug logs
109173
if: ${{ failure() }}
110174
run: docker compose -f dev/docker-compose-gcs-server.yml logs
175+
- name: Upload coverage data
176+
uses: actions/upload-artifact@v4
177+
with:
178+
name: coverage-gcs
179+
path: .coverage*
180+
include-hidden-files: true
111181

182+
integration-coverage-report:
183+
runs-on: ubuntu-latest
184+
needs: [integration-test, integration-test-s3, integration-test-adls, integration-test-gcs]
185+
steps:
186+
- uses: actions/checkout@v6
187+
- uses: actions/setup-python@v6
188+
with:
189+
python-version: '3.12'
190+
- name: Install UV
191+
uses: astral-sh/setup-uv@v7
192+
with:
193+
enable-cache: true
194+
- name: Install dependencies
195+
run: uv sync --group dev
196+
- name: Download all coverage artifacts
197+
uses: actions/download-artifact@v7
198+
with:
199+
pattern: coverage-*
200+
merge-multiple: true
112201
- name: Generate coverage report (75%) # Coverage threshold should only increase over time — never decrease it!
113202
run: COVERAGE_FAIL_UNDER=75 make coverage-report

‎.github/workflows/svn-build-artifacts.yml‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,9 @@ jobs:
3131
name: Build artifacts for SVN on ${{ matrix.os }}
3232
runs-on: ${{ matrix.os }}
3333
strategy:
34+
max-parallel: 15
3435
matrix:
35-
os: [ ubuntu-latest, windows-latest, macos-latest ]
36+
os: [ ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-15-intel, macos-latest ]
3637

3738
steps:
3839
- uses: actions/checkout@v6
@@ -54,7 +55,7 @@ jobs:
5455
# the repository, otherwise the tests will fail
5556
- name: Compile source distribution
5657
run: uv build --sdist
57-
if: startsWith(matrix.os, 'ubuntu')
58+
if: matrix.os == 'ubuntu-latest'
5859

5960
- name: Build wheels
6061
uses: pypa/cibuildwheel@v3.3.1
@@ -67,14 +68,11 @@ jobs:
6768
CIBW_PROJECT_REQUIRES_PYTHON: ">=3.10,<3.14"
6869
CIBW_BEFORE_TEST: "uv sync --directory {project} --only-group dev --no-install-project"
6970
CIBW_TEST_COMMAND: "uv run --directory {project} pytest tests/avro/test_decoder.py"
70-
# Ignore tests for pypy since not all dependencies are compiled for it
71-
# and would require a local rust build chain
72-
CIBW_TEST_SKIP: "pp*"
7371
# Skip free-threaded (PEP 703) builds until we evaluate decoder_fast support
7472
CIBW_SKIP: "cp3*t-*"
7573

7674
- name: Add source distribution
77-
if: startsWith(matrix.os, 'ubuntu')
75+
if: matrix.os == 'ubuntu-latest'
7876
run: ls -lah dist/* && cp dist/* wheelhouse/
7977

8078
- uses: actions/upload-artifact@v4

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,9 @@ bin/
4141
.mypy_cache/
4242
htmlcov
4343

44+
# Jupyter notebook checkpoints
45+
.ipynb_checkpoints/
46+
4447
pyiceberg/avro/decoder_fast.c
4548
pyiceberg/avro/*.html
4649
pyiceberg/avro/*.so

‎.pre-commit-config.yaml‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ repos:
2323
hooks:
2424
- id: trailing-whitespace
2525
- id: end-of-file-fixer
26-
- id: debug-statements
2726
- id: check-yaml
2827
- id: check-ast
2928
- repo: https://github.com/astral-sh/ruff-pre-commit
@@ -32,6 +31,11 @@ repos:
3231
- id: ruff
3332
args: [ --fix, --exit-non-zero-on-fix ]
3433
- id: ruff-format
34+
- repo: https://github.com/nbQA-dev/nbQA
35+
rev: 1.9.1
36+
hooks:
37+
- id: nbqa-ruff
38+
args: [ --fix, --exit-non-zero-on-fix ]
3539
- repo: https://github.com/pre-commit/mirrors-mypy
3640
rev: v1.18.2
3741
hooks:
@@ -58,15 +62,15 @@ repos:
5862
]
5963
additional_dependencies:
6064
- tomli==2.0.1
61-
- repo: https://github.com/ikamensh/flynt
62-
rev: 1.0.6
63-
hooks:
64-
- id: flynt
65-
args:
66-
# --line-length is set to a high value to deal with very long lines
67-
- --line-length
68-
- '99999'
6965
- repo: https://github.com/codespell-project/codespell
7066
rev: v2.4.1
7167
hooks:
7268
- id: codespell
69+
- repo: local
70+
hooks:
71+
- id: uv-lock-check
72+
name: uv lock file check
73+
entry: make uv-lock-check
74+
language: system
75+
pass_filenames: false
76+
files: ^(pyproject\.toml|uv\.lock)$

‎MANIFEST.in‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ recursive-include tests *
2626

2727
# Include development files
2828
include Makefile
29+
include uv.lock
2930
recursive-include dev *
3031

3132
# Exclude build artifacts

0 commit comments

Comments
 (0)