Skip to content

Commit 30ec047

Browse files
author
ShreyeshArangath
committed
Merge branch 'main' into feat/support-metrics
# Conflicts: # crates/core/src/metrics.rs # uv.lock
2 parents d2b6c9f + 5be412b commit 30ec047

File tree

175 files changed

+25787
-4478
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

175 files changed

+25787
-4478
lines changed

‎.cargo/config.toml‎

Lines changed: 0 additions & 5 deletions
This file was deleted.

‎.github/workflows/build.yml‎

Lines changed: 48 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ on:
3636
env:
3737
CARGO_TERM_COLOR: always
3838
RUST_BACKTRACE: 1
39+
UV_LOCKED: true
3940

4041
jobs:
4142
# ============================================
@@ -47,7 +48,7 @@ jobs:
4748
- uses: actions/checkout@v6
4849

4950
- name: Setup Rust
50-
uses: dtolnay/rust-toolchain@stable
51+
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8
5152
with:
5253
toolchain: "nightly"
5354
components: rustfmt
@@ -68,7 +69,7 @@ jobs:
6869
with:
6970
python-version: "3.12"
7071

71-
- uses: astral-sh/setup-uv@v6
72+
- uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098
7273
with:
7374
enable-cache: true
7475

@@ -98,12 +99,21 @@ jobs:
9899
- name: Check Cargo.toml formatting
99100
run: taplo format --check
100101

102+
check-crates-patch:
103+
if: inputs.build_mode == 'release' && startsWith(github.ref, 'refs/tags/')
104+
runs-on: ubuntu-latest
105+
steps:
106+
- uses: actions/checkout@v6
107+
108+
- name: Ensure [patch.crates-io] is empty
109+
run: python3 dev/check_crates_patch.py
110+
101111
generate-license:
102112
runs-on: ubuntu-latest
103113
steps:
104114
- uses: actions/checkout@v6
105115

106-
- uses: astral-sh/setup-uv@v6
116+
- uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098
107117
with:
108118
enable-cache: true
109119

@@ -115,7 +125,7 @@ jobs:
115125
- name: Generate license file
116126
run: uv run --no-project python ./dev/create_license.py
117127

118-
- uses: actions/upload-artifact@v6
128+
- uses: actions/upload-artifact@v7
119129
with:
120130
name: python-wheel-license
121131
path: LICENSE.txt
@@ -132,20 +142,20 @@ jobs:
132142

133143
- run: rm LICENSE.txt
134144
- name: Download LICENSE.txt
135-
uses: actions/download-artifact@v7
145+
uses: actions/download-artifact@v8
136146
with:
137147
name: python-wheel-license
138148
path: .
139149

140150
- name: Setup Rust
141-
uses: dtolnay/rust-toolchain@stable
151+
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8
142152

143153
- name: Cache Cargo
144154
uses: Swatinem/rust-cache@v2
145155
with:
146156
key: ${{ inputs.build_mode }}
147157

148-
- uses: astral-sh/setup-uv@v6
158+
- uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098
149159
with:
150160
enable-cache: true
151161

@@ -177,13 +187,13 @@ jobs:
177187
rustup-components: rust-std
178188

179189
- name: Archive wheels
180-
uses: actions/upload-artifact@v6
190+
uses: actions/upload-artifact@v7
181191
with:
182192
name: dist-manylinux-x86_64
183193
path: dist/*
184194

185195
- name: Archive FFI test wheel
186-
uses: actions/upload-artifact@v6
196+
uses: actions/upload-artifact@v7
187197
with:
188198
name: test-ffi-manylinux-x86_64
189199
path: examples/datafusion-ffi-example/dist/*
@@ -200,23 +210,36 @@ jobs:
200210

201211
- run: rm LICENSE.txt
202212
- name: Download LICENSE.txt
203-
uses: actions/download-artifact@v7
213+
uses: actions/download-artifact@v8
204214
with:
205215
name: python-wheel-license
206216
path: .
207217

208218
- name: Setup Rust
209-
uses: dtolnay/rust-toolchain@stable
219+
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8
210220

211221
- name: Cache Cargo
212222
uses: Swatinem/rust-cache@v2
213223
with:
214224
key: ${{ inputs.build_mode }}
215225

216-
- uses: astral-sh/setup-uv@v6
226+
- uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098
217227
with:
218228
enable-cache: true
219229

230+
- name: Add extra swap for release build
231+
if: inputs.build_mode == 'release'
232+
run: |
233+
set -euxo pipefail
234+
sudo swapoff -a || true
235+
sudo rm -f /swapfile
236+
sudo fallocate -l 16G /swapfile || sudo dd if=/dev/zero of=/swapfile bs=1M count=16384
237+
sudo chmod 600 /swapfile
238+
sudo mkswap /swapfile
239+
sudo swapon /swapfile
240+
free -h
241+
swapon --show
242+
220243
- name: Build (release mode)
221244
uses: PyO3/maturin-action@v1
222245
if: inputs.build_mode == 'release'
@@ -236,7 +259,7 @@ jobs:
236259
rustup-components: rust-std
237260

238261
- name: Archive wheels
239-
uses: actions/upload-artifact@v6
262+
uses: actions/upload-artifact@v7
240263
if: inputs.build_mode == 'release'
241264
with:
242265
name: dist-manylinux-aarch64
@@ -257,11 +280,11 @@ jobs:
257280
steps:
258281
- uses: actions/checkout@v6
259282

260-
- uses: dtolnay/rust-toolchain@stable
283+
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8
261284

262285
- run: rm LICENSE.txt
263286
- name: Download LICENSE.txt
264-
uses: actions/download-artifact@v7
287+
uses: actions/download-artifact@v8
265288
with:
266289
name: python-wheel-license
267290
path: .
@@ -271,7 +294,7 @@ jobs:
271294
with:
272295
key: ${{ inputs.build_mode }}
273296

274-
- uses: astral-sh/setup-uv@v7
297+
- uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098
275298
with:
276299
enable-cache: true
277300

@@ -311,7 +334,7 @@ jobs:
311334
run: find target/wheels/
312335

313336
- name: Archive wheels
314-
uses: actions/upload-artifact@v6
337+
uses: actions/upload-artifact@v7
315338
if: inputs.build_mode == 'release'
316339
with:
317340
name: dist-${{ matrix.os }}
@@ -331,11 +354,11 @@ jobs:
331354
steps:
332355
- uses: actions/checkout@v6
333356

334-
- uses: dtolnay/rust-toolchain@stable
357+
- uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8
335358

336359
- run: rm LICENSE.txt
337360
- name: Download LICENSE.txt
338-
uses: actions/download-artifact@v7
361+
uses: actions/download-artifact@v8
339362
with:
340363
name: python-wheel-license
341364
path: .
@@ -345,7 +368,7 @@ jobs:
345368
with:
346369
key: ${{ inputs.build_mode }}
347370

348-
- uses: astral-sh/setup-uv@v7
371+
- uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098
349372
with:
350373
enable-cache: true
351374

@@ -366,7 +389,7 @@ jobs:
366389
run: find target/wheels/
367390

368391
- name: Archive wheels
369-
uses: actions/upload-artifact@v6
392+
uses: actions/upload-artifact@v7
370393
with:
371394
name: dist-macos-aarch64
372395
path: target/wheels/*
@@ -384,7 +407,7 @@ jobs:
384407
- uses: actions/checkout@v6
385408
- run: rm LICENSE.txt
386409
- name: Download LICENSE.txt
387-
uses: actions/download-artifact@v7
410+
uses: actions/download-artifact@v8
388411
with:
389412
name: python-wheel-license
390413
path: .
@@ -422,7 +445,7 @@ jobs:
422445
- build-sdist
423446
steps:
424447
- name: Merge Build Artifacts
425-
uses: actions/upload-artifact/merge@v6
448+
uses: actions/upload-artifact/merge@v7
426449
with:
427450
name: dist
428451
pattern: dist-*
@@ -469,13 +492,13 @@ jobs:
469492
python-version: "3.10"
470493

471494
- name: Install dependencies
472-
uses: astral-sh/setup-uv@v7
495+
uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098
473496
with:
474497
enable-cache: true
475498

476499
# Download the Linux wheel built in the previous job
477500
- name: Download pre-built Linux wheel
478-
uses: actions/download-artifact@v7
501+
uses: actions/download-artifact@v8
479502
with:
480503
name: dist-manylinux-x86_64
481504
path: wheels/

‎.github/workflows/test.yml‎

Lines changed: 14 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,9 @@ name: Test
2323
on:
2424
workflow_call:
2525

26+
env:
27+
UV_LOCKED: true
28+
2629
jobs:
2730
test-matrix:
2831
runs-on: ubuntu-latest
@@ -41,18 +44,6 @@ jobs:
4144
steps:
4245
- uses: actions/checkout@v6
4346

44-
- name: Verify example datafusion version
45-
run: |
46-
MAIN_VERSION=$(grep -A 1 "name = \"datafusion-common\"" Cargo.lock | grep "version = " | head -1 | sed 's/.*version = "\(.*\)"/\1/')
47-
EXAMPLE_VERSION=$(grep -A 1 "name = \"datafusion-common\"" examples/datafusion-ffi-example/Cargo.lock | grep "version = " | head -1 | sed 's/.*version = "\(.*\)"/\1/')
48-
echo "Main crate datafusion version: $MAIN_VERSION"
49-
echo "FFI example datafusion version: $EXAMPLE_VERSION"
50-
51-
if [ "$MAIN_VERSION" != "$EXAMPLE_VERSION" ]; then
52-
echo "❌ Error: FFI example datafusion versions don't match!"
53-
exit 1
54-
fi
55-
5647
- name: Setup Python
5748
uses: actions/setup-python@v6
5849
with:
@@ -62,23 +53,23 @@ jobs:
6253
uses: actions/cache@v5
6354
with:
6455
path: ~/.cargo
65-
key: cargo-cache-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('Cargo.lock') }}
56+
key: cargo-cache-${{ matrix.toolchain }}-${{ hashFiles('Cargo.lock') }}
6657

6758
- name: Install dependencies
68-
uses: astral-sh/setup-uv@v7
59+
uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098
6960
with:
70-
enable-cache: true
61+
enable-cache: true
7162

7263
# Download the Linux wheel built in the build workflow
7364
- name: Download pre-built Linux wheel
74-
uses: actions/download-artifact@v7
65+
uses: actions/download-artifact@v8
7566
with:
7667
name: dist-manylinux-x86_64
7768
path: wheels/
7869

7970
# Download the FFI test wheel
8071
- name: Download pre-built FFI test wheel
81-
uses: actions/download-artifact@v7
72+
uses: actions/download-artifact@v8
8273
with:
8374
name: test-ffi-manylinux-x86_64
8475
path: wheels/
@@ -106,28 +97,21 @@ jobs:
10697
RUST_BACKTRACE: 1
10798
run: |
10899
git submodule update --init
109-
uv run --no-project pytest -v . --import-mode=importlib
100+
uv run --no-project pytest -v --import-mode=importlib
110101
111102
- name: FFI unit tests
112103
run: |
113104
cd examples/datafusion-ffi-example
114105
uv run --no-project pytest python/tests/_test*.py
115106
116-
- name: Cache the generated dataset
117-
id: cache-tpch-dataset
118-
uses: actions/cache@v5
119-
with:
120-
path: benchmarks/tpch/data
121-
key: tpch-data-2.18.0
122-
123-
- name: Run dbgen to create 1 Gb dataset
124-
if: ${{ steps.cache-tpch-dataset.outputs.cache-hit != 'true' }}
107+
- name: Run tpchgen-cli to create 1 Gb dataset
125108
run: |
126-
cd benchmarks/tpch
127-
RUN_IN_CI=TRUE ./tpch-gen.sh 1
109+
mkdir examples/tpch/data
110+
cd examples/tpch/data
111+
uv pip install tpchgen-cli
112+
uv run --no-project tpchgen-cli -s 1 --format=parquet
128113
129114
- name: Run TPC-H examples
130115
run: |
131116
cd examples/tpch
132-
uv run --no-project python convert_data_to_parquet.py
133117
uv run --no-project pytest _tests.py

‎.github/workflows/verify-release-candidate.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -58,10 +58,10 @@ jobs:
5858
arch: x64
5959
runner: macos-15-intel
6060

61-
# Windows (disabled for now)
62-
# - os: windows
63-
# arch: x64
64-
# runner: windows-latest
61+
# Windows
62+
- os: windows
63+
arch: x64
64+
runner: windows-latest
6565
runs-on: ${{ matrix.runner }}
6666
steps:
6767
- name: Checkout repository

‎.pre-commit-config.yaml‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ repos:
2222
- id: actionlint-docker
2323
- repo: https://github.com/astral-sh/ruff-pre-commit
2424
# Ruff version.
25-
rev: v0.9.10
25+
rev: v0.15.1
2626
hooks:
2727
# Run the linter.
2828
- id: ruff
@@ -53,5 +53,12 @@ repos:
5353
additional_dependencies:
5454
- tomli
5555

56+
- repo: https://github.com/astral-sh/uv-pre-commit
57+
# uv version.
58+
rev: 0.10.7
59+
hooks:
60+
# Update the uv lockfile
61+
- id: uv-lock
62+
5663
default_language_version:
5764
python: python3

0 commit comments

Comments
 (0)