Skip to content

Commit e8a8601

Browse files
committed
feat(vortex-file): gate the file writer behind editions
The writer is now gated by Vortex editions, defaulting to the newest frozen core edition, core2026.07.0: - The first-party edition declarations move from the `vortex` facade into `vortex_edition::declarations` (pure static data), so that `vortex-file` can resolve edition encoding sets without a session. `vortex::editions` re-exports them and still seeds the default session. - `Scheme::produced_encodings` is a new trait method declaring the array encodings a compression scheme may emit. All first-party schemes declare their outputs; undeclared schemes are excluded from gated writers. - `BtrBlocksCompressorBuilder::retain_allowed_encodings` filters the scheme list down to schemes whose outputs are all allowed. - `vortex-file`'s `ALLOWED_ENCODINGS` — the set the flat leaf writer validates every serialized encoding against (failing the write on a violation) — is now derived from `DEFAULT_WRITE_EDITIONS` (core2026.07.0, plus the newest unstable draft when the `unstable_encodings` feature opts in) instead of being hand maintained. `WriteStrategyBuilder::build` applies the same set to the data and stats compressors, so compression only ever chooses encodings from the enabled editions, and `WriteStrategyBuilder::with_editions` lets callers pin different editions. Experimental patches remain a runtime opt-out: `Patched` is only writable when the experimental environment variable is set. Signed-off-by: Claude <noreply@anthropic.com> Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CK2nCuXnyd2g3mNHQC8Lz2
1 parent 3e8141e commit e8a8601

48 files changed

Lines changed: 628 additions & 161 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

Cargo.lock

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

vortex-btrblocks/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ insta = { workspace = true }
4444
rstest = { workspace = true }
4545
test-with = { workspace = true }
4646
vortex-array = { workspace = true, features = ["_test-harness"] }
47+
vortex-edition = { workspace = true }
4748
vortex-session = { workspace = true }
4849

4950
[features]

vortex-btrblocks/src/builder.rs

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
//! Builder for configuring `BtrBlocksCompressor` instances.
55
6+
use vortex_array::ArrayId;
67
use vortex_utils::aliases::hash_set::HashSet;
78

89
use crate::BtrBlocksCompressor;
@@ -208,6 +209,20 @@ impl BtrBlocksCompressorBuilder {
208209
self
209210
}
210211

212+
/// Retains only schemes whose declared produced encodings all belong to `allowed`.
213+
///
214+
/// Schemes that do not declare their produced encodings (see
215+
/// [`Scheme::produced_encodings`]) are removed as well: the compressor cannot prove
216+
/// their output stays within the allowed set. The file writer uses this to restrict
217+
/// compression to the encodings of its configured editions.
218+
pub fn retain_allowed_encodings(mut self, allowed: &HashSet<ArrayId>) -> Self {
219+
self.schemes.retain(|s| {
220+
s.produced_encodings()
221+
.is_some_and(|ids| ids.iter().all(|id| allowed.contains(id)))
222+
});
223+
self
224+
}
225+
211226
/// Builds the configured [`BtrBlocksCompressor`].
212227
pub fn build(self) -> BtrBlocksCompressor {
213228
BtrBlocksCompressor(CascadingCompressor::new(self.schemes))
@@ -216,6 +231,12 @@ impl BtrBlocksCompressorBuilder {
216231

217232
#[cfg(test)]
218233
mod tests {
234+
use vortex_array::VTable;
235+
use vortex_edition::declarations::CORE_2026_07_0;
236+
use vortex_edition::declarations::UNSTABLE_2026_06_0;
237+
use vortex_edition::declarations::encodings_in_editions;
238+
use vortex_fastlanes::FoR;
239+
219240
use super::*;
220241

221242
#[test]
@@ -230,6 +251,42 @@ mod tests {
230251
assert_eq!(builder.schemes.len(), ALL_SCHEMES.len());
231252
}
232253

254+
#[test]
255+
fn all_schemes_declare_produced_encodings() {
256+
for scheme in ALL_SCHEMES {
257+
assert!(
258+
scheme.produced_encodings().is_some(),
259+
"scheme {} must declare its produced encodings",
260+
scheme.id()
261+
);
262+
}
263+
}
264+
265+
#[test]
266+
fn retain_allowed_encodings_filters_schemes() {
267+
let allowed: HashSet<ArrayId> = [FoR.id()].into_iter().collect();
268+
let builder = BtrBlocksCompressorBuilder::default().retain_allowed_encodings(&allowed);
269+
assert_eq!(builder.schemes.len(), 1);
270+
assert_eq!(builder.schemes[0].id(), integer::FoRScheme.id());
271+
272+
let none = BtrBlocksCompressorBuilder::default().retain_allowed_encodings(&HashSet::new());
273+
assert!(none.schemes.is_empty());
274+
}
275+
276+
/// Every default scheme must produce only encodings covered by the writer's default
277+
/// editions, otherwise the default file writer would silently drop it.
278+
#[test]
279+
fn default_schemes_stay_within_default_editions() {
280+
let editions = if cfg!(feature = "unstable_encodings") {
281+
vec![CORE_2026_07_0, UNSTABLE_2026_06_0]
282+
} else {
283+
vec![CORE_2026_07_0]
284+
};
285+
let allowed: HashSet<ArrayId> = encodings_in_editions(&editions).into_iter().collect();
286+
let builder = BtrBlocksCompressorBuilder::default().retain_allowed_encodings(&allowed);
287+
assert_eq!(builder.schemes.len(), ALL_SCHEMES.len());
288+
}
289+
233290
#[test]
234291
fn cuda_compatible_excludes_alprd() {
235292
let builder = BtrBlocksCompressorBuilder::default().only_cuda_compatible();

vortex-btrblocks/src/schemes/binary/zstd.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,12 @@
33

44
//! Zstd compression for binary arrays.
55
6+
use vortex_array::ArrayId;
67
use vortex_array::ArrayRef;
78
use vortex_array::Canonical;
89
use vortex_array::ExecutionCtx;
910
use vortex_array::IntoArray;
11+
use vortex_array::VTable;
1012
use vortex_compressor::scheme::CompressionEstimate;
1113
use vortex_compressor::scheme::DeferredEstimate;
1214
use vortex_error::VortexResult;
@@ -29,6 +31,10 @@ impl Scheme for ZstdScheme {
2931
canonical.dtype().is_binary()
3032
}
3133

34+
fn produced_encodings(&self) -> Option<Vec<ArrayId>> {
35+
Some(vec![vortex_zstd::Zstd.id()])
36+
}
37+
3238
fn expected_compression_ratio(
3339
&self,
3440
_data: &ArrayAndStats,

vortex-btrblocks/src/schemes/binary/zstd_buffers.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,12 @@
33

44
//! Zstd buffer-level binary compression preserving array layout for GPU decompression.
55
6+
use vortex_array::ArrayId;
67
use vortex_array::ArrayRef;
78
use vortex_array::Canonical;
89
use vortex_array::ExecutionCtx;
910
use vortex_array::IntoArray;
11+
use vortex_array::VTable;
1012
use vortex_compressor::scheme::CompressionEstimate;
1113
use vortex_compressor::scheme::DeferredEstimate;
1214
use vortex_error::VortexResult;
@@ -29,6 +31,10 @@ impl Scheme for ZstdBuffersScheme {
2931
canonical.dtype().is_binary()
3032
}
3133

34+
fn produced_encodings(&self) -> Option<Vec<ArrayId>> {
35+
Some(vec![vortex_zstd::ZstdBuffers.id()])
36+
}
37+
3238
fn expected_compression_ratio(
3339
&self,
3440
_data: &ArrayAndStats,

vortex-btrblocks/src/schemes/decimal.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,12 @@
33

44
//! Decimal compression scheme using byte-part decomposition.
55
6+
use vortex_array::ArrayId;
67
use vortex_array::ArrayRef;
78
use vortex_array::Canonical;
89
use vortex_array::ExecutionCtx;
910
use vortex_array::IntoArray;
11+
use vortex_array::VTable;
1012
use vortex_array::arrays::DecimalArray;
1113
use vortex_array::arrays::PrimitiveArray;
1214
use vortex_array::arrays::decimal::narrowed_decimal;
@@ -38,6 +40,10 @@ impl Scheme for DecimalScheme {
3840
matches!(canonical, Canonical::Decimal(_))
3941
}
4042

43+
fn produced_encodings(&self) -> Option<Vec<ArrayId>> {
44+
Some(vec![DecimalByteParts.id()])
45+
}
46+
4147
/// Children: primitive=0.
4248
fn num_children(&self) -> usize {
4349
1

vortex-btrblocks/src/schemes/float/alp.rs

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,12 @@ use vortex_alp::ALP;
77
use vortex_alp::ALPArrayExt;
88
use vortex_alp::ALPArraySlotsExt;
99
use vortex_alp::alp_encode;
10+
use vortex_array::ArrayId;
1011
use vortex_array::ArrayRef;
1112
use vortex_array::Canonical;
1213
use vortex_array::ExecutionCtx;
1314
use vortex_array::IntoArray;
15+
use vortex_array::VTable;
1416
use vortex_array::arrays::Patched;
1517
use vortex_array::arrays::patched::use_experimental_patches;
1618
use vortex_array::arrays::primitive::PrimitiveArrayExt;
@@ -40,6 +42,14 @@ impl Scheme for ALPScheme {
4042
canonical.dtype().is_float()
4143
}
4244

45+
fn produced_encodings(&self) -> Option<Vec<ArrayId>> {
46+
let mut encodings = vec![ALP.id()];
47+
if use_experimental_patches() {
48+
encodings.push(Patched.id());
49+
}
50+
Some(encodings)
51+
}
52+
4353
/// Children: encoded_ints=0.
4454
fn num_children(&self) -> usize {
4555
1

vortex-btrblocks/src/schemes/float/alprd.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,12 @@
66
use vortex_alp::ALPRDArrayExt;
77
use vortex_alp::ALPRDArrayOwnedExt;
88
use vortex_alp::RDEncoder;
9+
use vortex_array::ArrayId;
910
use vortex_array::ArrayRef;
1011
use vortex_array::Canonical;
1112
use vortex_array::ExecutionCtx;
1213
use vortex_array::IntoArray;
14+
use vortex_array::VTable;
1315
use vortex_array::arrays::primitive::PrimitiveArrayExt;
1416
use vortex_array::dtype::PType;
1517
use vortex_compressor::scheme::CompressionEstimate;
@@ -37,6 +39,10 @@ impl Scheme for ALPRDScheme {
3739
canonical.dtype().is_float()
3840
}
3941

42+
fn produced_encodings(&self) -> Option<Vec<ArrayId>> {
43+
Some(vec![vortex_alp::ALPRD.id()])
44+
}
45+
4046
fn expected_compression_ratio(
4147
&self,
4248
data: &ArrayAndStats,

vortex-btrblocks/src/schemes/float/pco.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,12 @@
33

44
//! Pco (pcodec) float compression.
55
6+
use vortex_array::ArrayId;
67
use vortex_array::ArrayRef;
78
use vortex_array::Canonical;
89
use vortex_array::ExecutionCtx;
910
use vortex_array::IntoArray;
11+
use vortex_array::VTable;
1012
use vortex_compressor::scheme::CompressionEstimate;
1113
use vortex_compressor::scheme::DeferredEstimate;
1214
use vortex_error::VortexResult;
@@ -29,6 +31,10 @@ impl Scheme for PcoScheme {
2931
canonical.dtype().is_float()
3032
}
3133

34+
fn produced_encodings(&self) -> Option<Vec<ArrayId>> {
35+
Some(vec![vortex_pco::Pco.id()])
36+
}
37+
3238
fn expected_compression_ratio(
3339
&self,
3440
_data: &ArrayAndStats,

vortex-btrblocks/src/schemes/float/rle.rs

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,18 @@
33

44
//! Run-length float encoding.
55
6+
use vortex_array::ArrayId;
67
use vortex_array::ArrayRef;
78
use vortex_array::Canonical;
89
use vortex_array::ExecutionCtx;
10+
use vortex_array::VTable;
911
use vortex_compressor::scheme::AncestorExclusion;
1012
use vortex_compressor::scheme::CompressionEstimate;
1113
use vortex_compressor::scheme::DeferredEstimate;
1214
use vortex_compressor::scheme::DescendantExclusion;
1315
use vortex_compressor::scheme::EstimateVerdict;
1416
use vortex_error::VortexResult;
17+
use vortex_fastlanes::RLE;
1518

1619
use crate::ArrayAndStats;
1720
use crate::CascadingCompressor;
@@ -35,6 +38,10 @@ impl Scheme for FloatRLEScheme {
3538
canonical.dtype().is_float()
3639
}
3740

41+
fn produced_encodings(&self) -> Option<Vec<ArrayId>> {
42+
Some(vec![RLE.id()])
43+
}
44+
3845
/// Children: values=0, indices=1, offsets=2.
3946
fn num_children(&self) -> usize {
4047
3

0 commit comments

Comments
 (0)