Skip to content

Commit 2258dd3

Browse files
authored
New Parquet-Variant encoding (#7130)
## Summary This PR introduces a new encoding to support Arrow's canonical extension type, but does not integrate it with anything else. It does include basic pieces like slice/take/filter, and it can (or at least should) roundtrip with the equivalent arrow array. ## Testing The encoding includes a bunch of basic tests, both for making sure it roundtrips with arrow and for the various nullability cases. --------- Signed-off-by: Adam Gutglick <adam@spiraldb.com>
1 parent d0ed3fc commit 2258dd3

14 files changed

Lines changed: 2082 additions & 21 deletions

File tree

Cargo.lock

Lines changed: 65 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ members = [
3333
"vortex-python",
3434
"vortex-tui",
3535
"vortex-web/crate",
36+
"vortex-sqllogictest",
3637
"vortex-test/compat-gen",
3738
"vortex-test/e2e-cuda",
3839
"xtask",
@@ -49,13 +50,13 @@ members = [
4950
"encodings/zigzag",
5051
"encodings/zstd",
5152
"encodings/bytebool",
53+
"encodings/parquet-variant",
5254
# Benchmarks
5355
"benchmarks/lance-bench",
5456
"benchmarks/compress-bench",
5557
"benchmarks/datafusion-bench",
5658
"benchmarks/duckdb-bench",
5759
"benchmarks/random-access-bench",
58-
"vortex-sqllogictest",
5960
]
6061
exclude = ["java/testfiles", "wasm-test"]
6162
resolver = "2"
@@ -184,6 +185,8 @@ opentelemetry-otlp = "0.31.0"
184185
opentelemetry_sdk = "0.31.0"
185186
parking_lot = { version = "0.12.3", features = ["nightly"] }
186187
parquet = "58"
188+
parquet-variant = "58"
189+
parquet-variant-compute = "58"
187190
paste = "1.0.15"
188191
pco = "1.0.1"
189192
pin-project-lite = "0.2.15"
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
[package]
2+
name = "vortex-parquet-variant"
3+
authors = { workspace = true }
4+
categories = { workspace = true }
5+
description = "Vortex Parquet Variant array"
6+
edition = { workspace = true }
7+
homepage = { workspace = true }
8+
include = { workspace = true }
9+
keywords = { workspace = true }
10+
license = { workspace = true }
11+
readme = { workspace = true }
12+
repository = { workspace = true }
13+
rust-version = { workspace = true }
14+
version = { workspace = true }
15+
publish = false
16+
17+
[lints]
18+
workspace = true
19+
20+
[dependencies]
21+
arrow-array = { workspace = true }
22+
arrow-buffer = { workspace = true }
23+
arrow-schema = { workspace = true }
24+
chrono = { workspace = true }
25+
parquet-variant = { workspace = true }
26+
parquet-variant-compute = { workspace = true }
27+
prost = { workspace = true }
28+
vortex-array = { workspace = true }
29+
vortex-buffer = { workspace = true }
30+
vortex-error = { workspace = true }
31+
vortex-mask = { workspace = true }
32+
vortex-proto = { workspace = true }
33+
vortex-session = { workspace = true }
34+
35+
[dev-dependencies]
36+
rstest = { workspace = true }
37+
vortex-array = { workspace = true, features = ["_test-harness"] }
38+
39+
[package.metadata.cargo-machete]
40+
ignored = ["getrandom_v03"]

0 commit comments

Comments
 (0)