Skip to content

Commit 93bffa7

Browse files
connortsui20connortsui20
committed
add more exclusion rules (beyond the old compressor)
Signed-off-by: Connor Tsui <connor.tsui20@gmail.com>
1 parent cf820d2 commit 93bffa7

8 files changed

Lines changed: 104 additions & 10 deletions

File tree

‎vortex-btrblocks/public-api.lock‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,8 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::integer::F
284284

285285
impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::integer::FoRScheme
286286

287+
pub fn vortex_btrblocks::schemes::integer::FoRScheme::ancestor_exclusions(&self) -> alloc::vec::Vec<vortex_compressor::scheme::AncestorExclusion>
288+
287289
pub fn vortex_btrblocks::schemes::integer::FoRScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult<vortex_array::array::ArrayRef>
288290

289291
pub fn vortex_btrblocks::schemes::integer::FoRScheme::expected_compression_ratio(&self, _compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult<f64>
@@ -468,6 +470,8 @@ impl core::marker::StructuralPartialEq for vortex_btrblocks::schemes::integer::Z
468470

469471
impl vortex_compressor::scheme::Scheme for vortex_btrblocks::schemes::integer::ZigZagScheme
470472

473+
pub fn vortex_btrblocks::schemes::integer::ZigZagScheme::ancestor_exclusions(&self) -> alloc::vec::Vec<vortex_compressor::scheme::AncestorExclusion>
474+
471475
pub fn vortex_btrblocks::schemes::integer::ZigZagScheme::compress(&self, compressor: &vortex_compressor::compressor::CascadingCompressor, data: &mut vortex_compressor::stats::cache::ArrayAndStats, ctx: vortex_compressor::ctx::CompressorContext) -> vortex_error::VortexResult<vortex_array::array::ArrayRef>
472476

473477
pub fn vortex_btrblocks::schemes::integer::ZigZagScheme::descendant_exclusions(&self) -> alloc::vec::Vec<vortex_compressor::scheme::DescendantExclusion>

‎vortex-btrblocks/src/schemes/decimal.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ impl Scheme for DecimalScheme {
3636
matches!(canonical, Canonical::Decimal(_))
3737
}
3838

39+
/// Children: primitive=0.
3940
fn num_children(&self) -> usize {
4041
1
4142
}

‎vortex-btrblocks/src/schemes/float.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@ impl Scheme for ALPScheme {
110110
is_float_primitive(canonical)
111111
}
112112

113+
/// Children: encoded_ints=0.
113114
fn num_children(&self) -> usize {
114115
1
115116
}
@@ -220,11 +221,11 @@ impl Scheme for NullDominatedSparseScheme {
220221
is_float_primitive(canonical)
221222
}
222223

224+
/// Children: indices=0.
223225
fn num_children(&self) -> usize {
224226
1
225227
}
226228

227-
// TODO(connor): There seems to be stuff missing here...
228229
/// The indices of a null-dominated sparse array should not be sparse-encoded again.
229230
fn descendant_exclusions(&self) -> Vec<DescendantExclusion> {
230231
vec![DescendantExclusion {

‎vortex-btrblocks/src/schemes/integer.rs‎

Lines changed: 55 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,24 @@ impl Scheme for FoRScheme {
134134
is_integer_primitive(canonical)
135135
}
136136

137+
/// Dict codes always start at 0, so FoR (which subtracts the min) is a no-op.
138+
fn ancestor_exclusions(&self) -> Vec<AncestorExclusion> {
139+
vec![
140+
AncestorExclusion {
141+
ancestor: IntDictScheme.id(),
142+
children: ChildSelection::One(1),
143+
},
144+
AncestorExclusion {
145+
ancestor: FloatDictScheme.id(),
146+
children: ChildSelection::One(1),
147+
},
148+
AncestorExclusion {
149+
ancestor: StringDictScheme.id(),
150+
children: ChildSelection::One(1),
151+
},
152+
]
153+
}
154+
137155
fn expected_compression_ratio(
138156
&self,
139157
_compressor: &CascadingCompressor,
@@ -225,6 +243,7 @@ impl Scheme for ZigZagScheme {
225243
is_integer_primitive(canonical)
226244
}
227245

246+
/// Children: encoded=0.
228247
fn num_children(&self) -> usize {
229248
1
230249
}
@@ -249,6 +268,24 @@ impl Scheme for ZigZagScheme {
249268
]
250269
}
251270

271+
/// Dict codes are unsigned integers (0..cardinality). ZigZag only helps negatives.
272+
fn ancestor_exclusions(&self) -> Vec<AncestorExclusion> {
273+
vec![
274+
AncestorExclusion {
275+
ancestor: IntDictScheme.id(),
276+
children: ChildSelection::One(1),
277+
},
278+
AncestorExclusion {
279+
ancestor: FloatDictScheme.id(),
280+
children: ChildSelection::One(1),
281+
},
282+
AncestorExclusion {
283+
ancestor: StringDictScheme.id(),
284+
children: ChildSelection::One(1),
285+
},
286+
]
287+
}
288+
252289
fn expected_compression_ratio(
253290
&self,
254291
compressor: &CascadingCompressor,
@@ -365,6 +402,7 @@ impl Scheme for SparseScheme {
365402
}
366403
}
367404

405+
/// Children: values=0, indices=1.
368406
fn num_children(&self) -> usize {
369407
2
370408
}
@@ -494,6 +532,7 @@ impl Scheme for RunEndScheme {
494532
is_integer_primitive(canonical)
495533
}
496534

535+
/// Children: values=0, ends=1.
497536
fn num_children(&self) -> usize {
498537
2
499538
}
@@ -507,12 +546,23 @@ impl Scheme for RunEndScheme {
507546
}]
508547
}
509548

510-
// TODO(connor): There seems to be stuff missing here...
549+
/// Dict values (child 0) are all unique by definition, so run-end encoding them is
550+
/// pointless. Codes (child 1) can have runs and may benefit from RunEnd.
511551
fn ancestor_exclusions(&self) -> Vec<AncestorExclusion> {
512-
vec![AncestorExclusion {
513-
ancestor: FloatDictScheme.id(),
514-
children: ChildSelection::One(0),
515-
}]
552+
vec![
553+
AncestorExclusion {
554+
ancestor: IntDictScheme.id(),
555+
children: ChildSelection::One(0),
556+
},
557+
AncestorExclusion {
558+
ancestor: FloatDictScheme.id(),
559+
children: ChildSelection::One(0),
560+
},
561+
AncestorExclusion {
562+
ancestor: StringDictScheme.id(),
563+
children: ChildSelection::One(0),
564+
},
565+
]
516566
}
517567

518568
fn expected_compression_ratio(

‎vortex-btrblocks/src/schemes/rle.rs‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@ use vortex_array::Canonical;
99
use vortex_array::IntoArray;
1010
use vortex_array::ToCanonical;
1111
use vortex_array::arrays::PrimitiveArray;
12+
use vortex_compressor::builtins::FloatDictScheme;
13+
use vortex_compressor::builtins::StringDictScheme;
14+
use vortex_compressor::scheme::AncestorExclusion;
1215
use vortex_compressor::scheme::ChildSelection;
1316
use vortex_compressor::scheme::DescendantExclusion;
1417
use vortex_error::VortexResult;
@@ -90,6 +93,7 @@ impl<C: RLEConfig> Scheme for RLEScheme<C> {
9093
C::matches(canonical)
9194
}
9295

96+
/// Children: values=0, indices=1, offsets=2.
9397
fn num_children(&self) -> usize {
9498
3
9599
}
@@ -104,6 +108,24 @@ impl<C: RLEConfig> Scheme for RLEScheme<C> {
104108
}]
105109
}
106110

111+
/// Dict values (child 0) are all unique by definition, so RLE is pointless on them.
112+
fn ancestor_exclusions(&self) -> Vec<AncestorExclusion> {
113+
vec![
114+
AncestorExclusion {
115+
ancestor: IntDictScheme.id(),
116+
children: ChildSelection::One(0),
117+
},
118+
AncestorExclusion {
119+
ancestor: FloatDictScheme.id(),
120+
children: ChildSelection::One(0),
121+
},
122+
AncestorExclusion {
123+
ancestor: StringDictScheme.id(),
124+
children: ChildSelection::One(0),
125+
},
126+
]
127+
}
128+
107129
fn expected_compression_ratio(
108130
&self,
109131
compressor: &CascadingCompressor,

‎vortex-btrblocks/src/schemes/string.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ impl Scheme for FSSTScheme {
6262
is_utf8_string(canonical)
6363
}
6464

65+
/// Children: lengths=0, code_offsets=1.
6566
fn num_children(&self) -> usize {
6667
2
6768
}
@@ -124,11 +125,11 @@ impl Scheme for NullDominatedSparseScheme {
124125
is_utf8_string(canonical)
125126
}
126127

128+
/// Children: indices=0.
127129
fn num_children(&self) -> usize {
128130
1
129131
}
130132

131-
// TODO(connor): There seems to be stuff missing here...
132133
/// The indices of a null-dominated sparse array should not be sparse-encoded again.
133134
fn descendant_exclusions(&self) -> Vec<DescendantExclusion> {
134135
vec![

‎vortex-btrblocks/src/schemes/temporal.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ impl Scheme for TemporalScheme {
5353
true
5454
}
5555

56+
/// Children: days=0, seconds=1, subseconds=2.
5657
fn num_children(&self) -> usize {
5758
3
5859
}

‎vortex-compressor/src/builtins/dict/mod.rs‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,9 @@ impl Scheme for IntDictScheme {
4747
}
4848
}
4949

50+
/// Children: values=0, codes=1.
5051
fn num_children(&self) -> usize {
51-
1
52+
2
5253
}
5354

5455
fn expected_compression_ratio(
@@ -142,11 +143,19 @@ impl Scheme for FloatDictScheme {
142143
}
143144
}
144145

146+
/// Children: values=0, codes=1.
145147
fn num_children(&self) -> usize {
146148
2
147149
}
148150

149-
// TODO(connor): There seems to be stuff missing here...
151+
/// Float dict codes (child 1) are compact unsigned integers that should not be
152+
/// dict-encoded again. Float dict values (child 0) flow through ALP into integer-land,
153+
/// where integer dict encoding is redundant since the values are already deduplicated at
154+
/// the float level.
155+
///
156+
/// Additional exclusions for codes (IntSequenceScheme, IntRunEndScheme, FoRScheme,
157+
/// ZigZagScheme, SparseScheme, RLE) are expressed as pull rules on those schemes in
158+
/// vortex-btrblocks.
150159
fn descendant_exclusions(&self) -> Vec<DescendantExclusion> {
151160
vec![
152161
DescendantExclusion {
@@ -235,11 +244,16 @@ impl Scheme for StringDictScheme {
235244
}
236245
}
237246

247+
/// Children: values=0, codes=1.
238248
fn num_children(&self) -> usize {
239249
2
240250
}
241251

242-
// TODO(connor): There seems to be stuff missing here...
252+
/// String dict codes (child 1) are compact unsigned integers that should not be dict-encoded
253+
/// again.
254+
///
255+
/// Additional exclusions for codes (IntSequenceScheme, FoRScheme, ZigZagScheme, SparseScheme,
256+
/// RunEndScheme, RLE, etc.) are expressed as pull rules on those schemes in `vortex-btrblocks`.
243257
fn descendant_exclusions(&self) -> Vec<DescendantExclusion> {
244258
vec![DescendantExclusion {
245259
excluded: IntDictScheme.id(),

0 commit comments

Comments
 (0)