Skip to content

Commit 2da25df

Browse files
claudeclaude
committed
feat(vortex-file): gate the file writer behind editions
The session is the source of truth for what a writer may emit: EditionSession now stores the set of enabled editions alongside the declarations — enable() (validated, at most one edition per family), enabled(), and enabled_encodings(), the resolved union of the enabled editions' encoding sets. register_default_editions declares the editions and enables the default write editions: core2026.07.0, plus the newest unstable draft under the `unstable_encodings` feature. The writer resolves its allow set from the session at write time: - The flat leaf writer's normalize-with-error validation fails the write for any encoding outside the session's enabled editions. - The writer's pre-populated ArrayContext is seeded from the same set. - Compression is gated at scheme-selection time: the new required Scheme::produced_encodings method declares the array encodings a scheme may emit (empty means only canonical arrays), and CascadingCompressor::with_enabled_editions_gating skips schemes whose outputs are not all within the enabled editions, resolved from the execution context per compression. Sessions with no enabled editions fall back to the static ALLOWED_ENCODINGS registry, and explicit with_allow_encodings sets still override everything. Experimental patches remain a runtime opt-out: Patched is only writable when the experimental environment variable is set, mirroring when compression produces it. Signed-off-by: Claude <noreply@anthropic.com> Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CK2nCuXnyd2g3mNHQC8Lz2
1 parent 49c4753 commit 2da25df

46 files changed

Lines changed: 795 additions & 37 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Cargo.lock‎

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vortex-btrblocks/src/builder.rs‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
//! Builder for configuring `BtrBlocksCompressor` instances.
55
6+
use vortex_array::ArrayId;
67
use vortex_utils::aliases::hash_set::HashSet;
78

89
use crate::BtrBlocksCompressor;
@@ -90,12 +91,14 @@ pub const ALL_SCHEMES: &[&dyn Scheme] = &[
9091
#[derive(Debug, Clone)]
9192
pub struct BtrBlocksCompressorBuilder {
9293
schemes: Vec<&'static dyn Scheme>,
94+
gate_by_enabled_editions: bool,
9395
}
9496

9597
impl Default for BtrBlocksCompressorBuilder {
9698
fn default() -> Self {
9799
Self {
98100
schemes: ALL_SCHEMES.to_vec(),
101+
gate_by_enabled_editions: false,
99102
}
100103
}
101104
}
@@ -107,6 +110,7 @@ impl BtrBlocksCompressorBuilder {
107110
pub fn empty() -> Self {
108111
Self {
109112
schemes: Vec::new(),
113+
gate_by_enabled_editions: false,
110114
}
111115
}
112116

@@ -208,14 +212,40 @@ impl BtrBlocksCompressorBuilder {
208212
self
209213
}
210214

215+
/// Retains only schemes whose declared produced encodings (see
216+
/// [`Scheme::produced_encodings`]) all belong to `allowed`. The file writer uses this
217+
/// to restrict compression to an explicitly configured allow set.
218+
pub fn retain_allowed_encodings(mut self, allowed: &HashSet<ArrayId>) -> Self {
219+
self.schemes
220+
.retain(|s| s.produced_encodings().iter().all(|id| allowed.contains(id)));
221+
self
222+
}
223+
224+
/// Gate scheme selection by the session's enabled editions at compression time.
225+
///
226+
/// See [`CascadingCompressor::with_enabled_editions_gating`]. The file writer enables
227+
/// this by default so compression only chooses encodings from the session's enabled
228+
/// editions.
229+
pub fn gate_by_enabled_editions(mut self) -> Self {
230+
self.gate_by_enabled_editions = true;
231+
self
232+
}
233+
211234
/// Builds the configured [`BtrBlocksCompressor`].
212235
pub fn build(self) -> BtrBlocksCompressor {
213-
BtrBlocksCompressor(CascadingCompressor::new(self.schemes))
236+
let mut compressor = CascadingCompressor::new(self.schemes);
237+
if self.gate_by_enabled_editions {
238+
compressor = compressor.with_enabled_editions_gating();
239+
}
240+
BtrBlocksCompressor(compressor)
214241
}
215242
}
216243

217244
#[cfg(test)]
218245
mod tests {
246+
use vortex_array::VTable;
247+
use vortex_fastlanes::FoR;
248+
219249
use super::*;
220250

221251
#[test]
@@ -230,6 +260,17 @@ mod tests {
230260
assert_eq!(builder.schemes.len(), ALL_SCHEMES.len());
231261
}
232262

263+
#[test]
264+
fn retain_allowed_encodings_filters_schemes() {
265+
let allowed: HashSet<ArrayId> = [FoR.id()].into_iter().collect();
266+
let builder = BtrBlocksCompressorBuilder::default().retain_allowed_encodings(&allowed);
267+
assert_eq!(builder.schemes.len(), 1);
268+
assert_eq!(builder.schemes[0].id(), integer::FoRScheme.id());
269+
270+
let none = BtrBlocksCompressorBuilder::default().retain_allowed_encodings(&HashSet::new());
271+
assert!(none.schemes.is_empty());
272+
}
273+
233274
#[test]
234275
fn cuda_compatible_excludes_alprd() {
235276
let builder = BtrBlocksCompressorBuilder::default().only_cuda_compatible();

‎vortex-btrblocks/src/schemes/binary/zstd.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,12 @@
33

44
//! Zstd compression for binary arrays.
55
6+
use vortex_array::ArrayId;
67
use vortex_array::ArrayRef;
78
use vortex_array::Canonical;
89
use vortex_array::ExecutionCtx;
910
use vortex_array::IntoArray;
11+
use vortex_array::VTable;
1012
use vortex_compressor::scheme::CompressionEstimate;
1113
use vortex_compressor::scheme::DeferredEstimate;
1214
use vortex_error::VortexResult;
@@ -29,6 +31,10 @@ impl Scheme for ZstdScheme {
2931
canonical.dtype().is_binary()
3032
}
3133

34+
fn produced_encodings(&self) -> Vec<ArrayId> {
35+
vec![vortex_zstd::Zstd.id()]
36+
}
37+
3238
fn expected_compression_ratio(
3339
&self,
3440
_data: &ArrayAndStats,

‎vortex-btrblocks/src/schemes/binary/zstd_buffers.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,12 @@
33

44
//! Zstd buffer-level binary compression preserving array layout for GPU decompression.
55
6+
use vortex_array::ArrayId;
67
use vortex_array::ArrayRef;
78
use vortex_array::Canonical;
89
use vortex_array::ExecutionCtx;
910
use vortex_array::IntoArray;
11+
use vortex_array::VTable;
1012
use vortex_compressor::scheme::CompressionEstimate;
1113
use vortex_compressor::scheme::DeferredEstimate;
1214
use vortex_error::VortexResult;
@@ -29,6 +31,10 @@ impl Scheme for ZstdBuffersScheme {
2931
canonical.dtype().is_binary()
3032
}
3133

34+
fn produced_encodings(&self) -> Vec<ArrayId> {
35+
vec![vortex_zstd::ZstdBuffers.id()]
36+
}
37+
3238
fn expected_compression_ratio(
3339
&self,
3440
_data: &ArrayAndStats,

‎vortex-btrblocks/src/schemes/decimal.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,12 @@
33

44
//! Decimal compression scheme using byte-part decomposition.
55
6+
use vortex_array::ArrayId;
67
use vortex_array::ArrayRef;
78
use vortex_array::Canonical;
89
use vortex_array::ExecutionCtx;
910
use vortex_array::IntoArray;
11+
use vortex_array::VTable;
1012
use vortex_array::arrays::DecimalArray;
1113
use vortex_array::arrays::PrimitiveArray;
1214
use vortex_array::arrays::decimal::narrowed_decimal;
@@ -38,6 +40,10 @@ impl Scheme for DecimalScheme {
3840
matches!(canonical, Canonical::Decimal(_))
3941
}
4042

43+
fn produced_encodings(&self) -> Vec<ArrayId> {
44+
vec![DecimalByteParts.id()]
45+
}
46+
4147
/// Children: primitive=0.
4248
fn num_children(&self) -> usize {
4349
1

‎vortex-btrblocks/src/schemes/float/alp.rs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,12 @@ use vortex_alp::ALP;
77
use vortex_alp::ALPArrayExt;
88
use vortex_alp::ALPArraySlotsExt;
99
use vortex_alp::alp_encode;
10+
use vortex_array::ArrayId;
1011
use vortex_array::ArrayRef;
1112
use vortex_array::Canonical;
1213
use vortex_array::ExecutionCtx;
1314
use vortex_array::IntoArray;
15+
use vortex_array::VTable;
1416
use vortex_array::arrays::Patched;
1517
use vortex_array::arrays::patched::use_experimental_patches;
1618
use vortex_array::arrays::primitive::PrimitiveArrayExt;
@@ -40,6 +42,14 @@ impl Scheme for ALPScheme {
4042
canonical.dtype().is_float()
4143
}
4244

45+
fn produced_encodings(&self) -> Vec<ArrayId> {
46+
let mut encodings = vec![ALP.id()];
47+
if use_experimental_patches() {
48+
encodings.push(Patched.id());
49+
}
50+
encodings
51+
}
52+
4353
/// Children: encoded_ints=0.
4454
fn num_children(&self) -> usize {
4555
1

‎vortex-btrblocks/src/schemes/float/alprd.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,12 @@
66
use vortex_alp::ALPRDArrayExt;
77
use vortex_alp::ALPRDArrayOwnedExt;
88
use vortex_alp::RDEncoder;
9+
use vortex_array::ArrayId;
910
use vortex_array::ArrayRef;
1011
use vortex_array::Canonical;
1112
use vortex_array::ExecutionCtx;
1213
use vortex_array::IntoArray;
14+
use vortex_array::VTable;
1315
use vortex_array::arrays::primitive::PrimitiveArrayExt;
1416
use vortex_array::dtype::PType;
1517
use vortex_compressor::scheme::CompressionEstimate;
@@ -37,6 +39,10 @@ impl Scheme for ALPRDScheme {
3739
canonical.dtype().is_float()
3840
}
3941

42+
fn produced_encodings(&self) -> Vec<ArrayId> {
43+
vec![vortex_alp::ALPRD.id()]
44+
}
45+
4046
fn expected_compression_ratio(
4147
&self,
4248
data: &ArrayAndStats,

‎vortex-btrblocks/src/schemes/float/pco.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,12 @@
33

44
//! Pco (pcodec) float compression.
55
6+
use vortex_array::ArrayId;
67
use vortex_array::ArrayRef;
78
use vortex_array::Canonical;
89
use vortex_array::ExecutionCtx;
910
use vortex_array::IntoArray;
11+
use vortex_array::VTable;
1012
use vortex_compressor::scheme::CompressionEstimate;
1113
use vortex_compressor::scheme::DeferredEstimate;
1214
use vortex_error::VortexResult;
@@ -29,6 +31,10 @@ impl Scheme for PcoScheme {
2931
canonical.dtype().is_float()
3032
}
3133

34+
fn produced_encodings(&self) -> Vec<ArrayId> {
35+
vec![vortex_pco::Pco.id()]
36+
}
37+
3238
fn expected_compression_ratio(
3339
&self,
3440
_data: &ArrayAndStats,

‎vortex-btrblocks/src/schemes/float/rle.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,18 @@
33

44
//! Run-length float encoding.
55
6+
use vortex_array::ArrayId;
67
use vortex_array::ArrayRef;
78
use vortex_array::Canonical;
89
use vortex_array::ExecutionCtx;
10+
use vortex_array::VTable;
911
use vortex_compressor::scheme::AncestorExclusion;
1012
use vortex_compressor::scheme::CompressionEstimate;
1113
use vortex_compressor::scheme::DeferredEstimate;
1214
use vortex_compressor::scheme::DescendantExclusion;
1315
use vortex_compressor::scheme::EstimateVerdict;
1416
use vortex_error::VortexResult;
17+
use vortex_fastlanes::RLE;
1518

1619
use crate::ArrayAndStats;
1720
use crate::CascadingCompressor;
@@ -35,6 +38,10 @@ impl Scheme for FloatRLEScheme {
3538
canonical.dtype().is_float()
3639
}
3740

41+
fn produced_encodings(&self) -> Vec<ArrayId> {
42+
vec![RLE.id()]
43+
}
44+
3845
/// Children: values=0, indices=1, offsets=2.
3946
fn num_children(&self) -> usize {
4047
3

‎vortex-btrblocks/src/schemes/float/sparse.rs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,12 @@
33

44
//! Sparse encoding for null-dominated float arrays.
55
6+
use vortex_array::ArrayId;
67
use vortex_array::ArrayRef;
78
use vortex_array::Canonical;
89
use vortex_array::ExecutionCtx;
910
use vortex_array::IntoArray;
11+
use vortex_array::VTable;
1012
use vortex_array::arrays::PrimitiveArray;
1113
use vortex_array::arrays::primitive::PrimitiveArrayExt;
1214
use vortex_compressor::scheme::ChildSelection;
@@ -39,6 +41,10 @@ impl Scheme for NullDominatedSparseScheme {
3941
canonical.dtype().is_float()
4042
}
4143

44+
fn produced_encodings(&self) -> Vec<ArrayId> {
45+
vec![Sparse.id()]
46+
}
47+
4248
/// Children: indices=0.
4349
fn num_children(&self) -> usize {
4450
1

0 commit comments

Comments
 (0)