fix

Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk>

Author: joseph-isaacs

vortex-test/compat-gen/src/adapter.rs

@@ -52,6 +52,24 @@ pub fn write_file(path: &Path, chunk: ArrayRef) -> VortexResult<()> {
     })
 }
 
+/// Write a sequence of array chunks to an in-memory `.vortex` byte buffer with no compression.
+pub fn write_file_to_bytes(chunk: ArrayRef) -> VortexResult<ByteBuffer> {
+    let stream = ArrayStreamAdapter::new(chunk.dtype().clone(), stream::iter([Ok(chunk)]));
+
+    let strategy: Arc<dyn LayoutStrategy> = Arc::new(FlatLayoutStrategy::default());
+
+    runtime()?.block_on(async {
+        let session = VortexSession::default().with_tokio();
+        let mut bytes = Vec::new();
+        let _summary = session
+            .write_options()
+            .with_strategy(strategy)
+            .write(&mut bytes, stream)
+            .await?;
+        Ok(ByteBuffer::from(bytes))
+    })
+}
+
 /// Write a `.vortex` file using a caller-provided layout strategy (compressor pipeline).
 pub fn write_compressed(
     path: &Path,
@@ -74,6 +92,25 @@ pub fn write_compressed(
     })
 }
 
+/// Write a `.vortex` file into an in-memory byte buffer using a caller-provided layout strategy.
+pub fn write_compressed_to_bytes(
+    chunk: ArrayRef,
+    strategy: Arc<dyn LayoutStrategy>,
+) -> VortexResult<ByteBuffer> {
+    let stream = ArrayStreamAdapter::new(chunk.dtype().clone(), stream::iter([Ok(chunk)]));
+
+    runtime()?.block_on(async {
+        let session = VortexSession::default().with_tokio();
+        let mut bytes = Vec::new();
+        let _summary = session
+            .write_options()
+            .with_strategy(strategy)
+            .write(&mut bytes, stream)
+            .await?;
+        Ok(ByteBuffer::from(bytes))
+    })
+}
+
 /// Read a `.vortex` file from bytes, returning the arrays.
 pub fn read_file(bytes: ByteBuffer) -> VortexResult<ArrayRef> {
     runtime()?.block_on(async {

vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/decimal.rs

@@ -47,7 +47,7 @@ impl FlatLayoutFixture for DecimalFixture {
 
         // Decimal(18,0) stored as i64: large integers
         let dec_18_0 = DecimalArray::new(
-            buffer![0i64, i64::MAX, i64::MIN],
+            buffer![0i64, 999_999_999_999_999_999, -999_999_999_999_999_999],
             DecimalDType::new(18, 0),
             Validity::NonNullable,
         );

vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alp.rs

@@ -18,6 +18,34 @@ use crate::fixtures::FlatLayoutFixture;
 
 pub struct AlpFixture;
 
+fn special_f64(i: usize) -> f64 {
+    match i % 9 {
+        0 => 0.0,
+        1 => -0.0,
+        2 => f64::from_bits(0x7ff8_0000_0000_0001),
+        3 => f64::from_bits(0x7ff8_0000_0000_1234),
+        4 => f64::from_bits(0xfff8_0000_0000_5678),
+        5 => f64::INFINITY,
+        6 => f64::NEG_INFINITY,
+        7 => f64::MIN_POSITIVE,
+        _ => -f64::from_bits(1),
+    }
+}
+
+fn special_f32(i: usize) -> f32 {
+    match i % 9 {
+        0 => 0.0,
+        1 => -0.0,
+        2 => f32::from_bits(0x7fc0_0001),
+        3 => f32::from_bits(0x7fc0_1234),
+        4 => f32::from_bits(0xffc0_5678),
+        5 => f32::INFINITY,
+        6 => f32::NEG_INFINITY,
+        7 => f32::MIN_POSITIVE,
+        _ => -f32::from_bits(1),
+    }
+}
+
 impl FlatLayoutFixture for AlpFixture {
     fn name(&self) -> &str {
         "alp.vortex"
@@ -34,6 +62,9 @@ impl FlatLayoutFixture for AlpFixture {
     fn build(&self) -> VortexResult<ArrayRef> {
         let f64_prices: Vec<f64> = (0..N).map(|i| 100.0 + (i as f64) * 0.25).collect();
         let f32_near_int: Vec<f32> = (0..N).map(|i| i as f32).collect();
+        let f64_negative_near_int: Vec<f64> = (0..N)
+            .map(|i| -(i as f64) - ((i % 7) as f64) * 0.000_1)
+            .collect();
         let f64_currency: Vec<f64> = (0..N).map(|i| ((i % 10000) as f64) / 100.0).collect();
         let f64_nullable = PrimitiveArray::from_option_iter(
             (0..N as i64).map(|i| (i % 10 != 0).then_some(50.0 + (i as f64) * 0.125)),
@@ -47,14 +78,56 @@ impl FlatLayoutFixture for AlpFixture {
                 }
             })
             .collect();
+        let f64_patch_heavy: Vec<f64> = (0..N)
+            .map(|i| {
+                if i % 7 == 0 || i % 11 == 0 {
+                    10_000.0 + (i as f64).powi(2)
+                } else {
+                    250.0 + ((i % 37) as f64) * 0.01
+                }
+            })
+            .collect();
+        let f64_special_values: Vec<f64> = (0..N).map(special_f64).collect();
+        let f32_special_values: Vec<f32> = (0..N).map(special_f32).collect();
+        let f64_extremes: Vec<f64> = (0..N)
+            .map(|i| match i % 10 {
+                0 => f64::MAX,
+                1 => f64::MIN,
+                2 => f64::EPSILON,
+                3 => -f64::EPSILON,
+                4 => f64::MIN_POSITIVE,
+                5 => -f64::MIN_POSITIVE,
+                6 => std::f64::consts::PI,
+                7 => -std::f64::consts::E,
+                8 => f64::from_bits(1),
+                _ => -f64::from_bits(2),
+            })
+            .collect();
+        let f64_boundary_specials: Vec<f64> = (0..N)
+            .map(|i| match i {
+                0 => f64::from_bits(0x7ff8_0000_0000_0001),
+                1 => -0.0,
+                511 => f64::INFINITY,
+                512 => f64::NEG_INFINITY,
+                513 => f64::from_bits(0xfff8_0000_0000_5678),
+                1023 => f64::from_bits(1),
+                _ => 12.5 + (i as f64) * 0.000_001,
+            })
+            .collect();
 
         let arr = StructArray::try_new(
             FieldNames::from([
                 "f64_prices",
                 "f32_near_int",
+                "f64_negative_near_int",
                 "f64_currency",
                 "f64_nullable",
                 "f64_patched",
+                "f64_patch_heavy",
+                "f64_special_values",
+                "f32_special_values",
+                "f64_extremes",
+                "f64_boundary_specials",
             ]),
             vec![
                 alp_encode(
@@ -67,6 +140,14 @@ impl FlatLayoutFixture for AlpFixture {
                     None,
                 )?
                 .into_array(),
+                alp_encode(
+                    &PrimitiveArray::new(
+                        Buffer::from(f64_negative_near_int),
+                        Validity::NonNullable,
+                    ),
+                    None,
+                )?
+                .into_array(),
                 alp_encode(
                     &PrimitiveArray::new(Buffer::from(f64_currency), Validity::NonNullable),
                     None,
@@ -78,6 +159,34 @@ impl FlatLayoutFixture for AlpFixture {
                     None,
                 )?
                 .into_array(),
+                alp_encode(
+                    &PrimitiveArray::new(Buffer::from(f64_patch_heavy), Validity::NonNullable),
+                    None,
+                )?
+                .into_array(),
+                alp_encode(
+                    &PrimitiveArray::new(Buffer::from(f64_special_values), Validity::NonNullable),
+                    None,
+                )?
+                .into_array(),
+                alp_encode(
+                    &PrimitiveArray::new(Buffer::from(f32_special_values), Validity::NonNullable),
+                    None,
+                )?
+                .into_array(),
+                alp_encode(
+                    &PrimitiveArray::new(Buffer::from(f64_extremes), Validity::NonNullable),
+                    None,
+                )?
+                .into_array(),
+                alp_encode(
+                    &PrimitiveArray::new(
+                        Buffer::from(f64_boundary_specials),
+                        Validity::NonNullable,
+                    ),
+                    None,
+                )?
+                .into_array(),
             ],
             N,
             Validity::NonNullable,

vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/alprd.rs

@@ -18,6 +18,20 @@ use crate::fixtures::FlatLayoutFixture;
 
 pub struct AlprdFixture;
 
+fn special_f64(i: usize) -> f64 {
+    match i % 9 {
+        0 => 0.0,
+        1 => -0.0,
+        2 => f64::from_bits(0x7ff8_0000_0000_0001),
+        3 => f64::from_bits(0x7ff8_0000_0000_1234),
+        4 => f64::from_bits(0xfff8_0000_0000_5678),
+        5 => f64::INFINITY,
+        6 => f64::NEG_INFINITY,
+        7 => f64::MIN_POSITIVE,
+        _ => -f64::from_bits(1),
+    }
+}
+
 impl FlatLayoutFixture for AlprdFixture {
     fn name(&self) -> &str {
         "alprd.vortex"
@@ -42,6 +56,23 @@ impl FlatLayoutFixture for AlprdFixture {
         let drift: Vec<f64> = (0..N)
             .map(|i| 1000.0 + (i as f64) * 0.001 + ((i * 3) % 7) as f64 * 0.0001)
             .collect();
+        let constant_series = vec![12.125; N];
+        let decreasing: Vec<f64> = (0..N)
+            .map(|i| 512.0 - (i as f64) * 0.000_5 - ((i * 5 % 13) as f64) * 0.000_01)
+            .collect();
+        let oscillating: Vec<f64> = (0..N)
+            .map(|i| {
+                let phase = ((i % 9) as i32 - 4) as f64;
+                -0.25 + phase * 0.000_1 + (i as f64) * 0.000_001
+            })
+            .collect();
+        let periodic_resets: Vec<f64> = (0..N)
+            .map(|i| {
+                let block = i / 64;
+                let offset = i % 64;
+                block as f64 * 10.0 + (offset as f64) * 0.000_2
+            })
+            .collect();
 
         let sensor_nullable_vals: Vec<f64> = (0..N)
             .map(|i| {
@@ -57,20 +88,93 @@ impl FlatLayoutFixture for AlprdFixture {
                 Some(37.0 + noise)
             }
         }));
+        let special_values: Vec<f64> = (0..N)
+            .map(|i| {
+                if i % 16 == 0 {
+                    special_f64(i)
+                } else {
+                    42.125 + ((i * 5 % 17) as f64) * 0.000_01
+                }
+            })
+            .collect();
+        let boundary_specials: Vec<f64> = (0..N)
+            .map(|i| match i {
+                0 => f64::from_bits(0x7ff8_0000_0000_0001),
+                1 => -0.0,
+                511 => f64::INFINITY,
+                512 => f64::NEG_INFINITY,
+                513 => f64::from_bits(0xfff8_0000_0000_5678),
+                1023 => f64::from_bits(1),
+                _ => 9.875 + ((i * 3 % 11) as f64) * 0.000_1,
+            })
+            .collect();
+        let nullable_special_vals: Vec<f64> = (0..N)
+            .map(|i| {
+                if i % 32 == 7 {
+                    special_f64(i)
+                } else {
+                    11.5 + ((i * 13 % 19) as f64) * 0.000_01
+                }
+            })
+            .collect();
+        let nullable_specials = PrimitiveArray::from_option_iter((0..N).map(|i| {
+            if i % 29 == 0 || i == 0 || i == N - 1 {
+                None
+            } else {
+                Some(nullable_special_vals[i])
+            }
+        }));
 
         let sensor_prim = PrimitiveArray::new(Buffer::from(sensor), Validity::NonNullable);
         let drift_prim = PrimitiveArray::new(Buffer::from(drift), Validity::NonNullable);
+        let constant_prim =
+            PrimitiveArray::new(Buffer::from(constant_series), Validity::NonNullable);
+        let decreasing_prim = PrimitiveArray::new(Buffer::from(decreasing), Validity::NonNullable);
+        let oscillating_prim =
+            PrimitiveArray::new(Buffer::from(oscillating), Validity::NonNullable);
+        let periodic_resets_prim =
+            PrimitiveArray::new(Buffer::from(periodic_resets), Validity::NonNullable);
+        let special_prim = PrimitiveArray::new(Buffer::from(special_values), Validity::NonNullable);
+        let boundary_prim =
+            PrimitiveArray::new(Buffer::from(boundary_specials), Validity::NonNullable);
 
         let sensor_enc = RDEncoder::new::<f64>(sensor_prim.as_slice::<f64>());
         let drift_enc = RDEncoder::new::<f64>(drift_prim.as_slice::<f64>());
+        let constant_enc = RDEncoder::new::<f64>(constant_prim.as_slice::<f64>());
+        let decreasing_enc = RDEncoder::new::<f64>(decreasing_prim.as_slice::<f64>());
+        let oscillating_enc = RDEncoder::new::<f64>(oscillating_prim.as_slice::<f64>());
+        let periodic_resets_enc = RDEncoder::new::<f64>(periodic_resets_prim.as_slice::<f64>());
         let nullable_enc = RDEncoder::new::<f64>(&sensor_nullable_vals);
+        let special_enc = RDEncoder::new::<f64>(special_prim.as_slice::<f64>());
+        let boundary_enc = RDEncoder::new::<f64>(boundary_prim.as_slice::<f64>());
+        let nullable_special_enc = RDEncoder::new::<f64>(&nullable_special_vals);
 
         let arr = StructArray::try_new(
-            FieldNames::from(["sensor", "drift", "sensor_nullable"]),
+            FieldNames::from([
+                "sensor",
+                "drift",
+                "constant_series",
+                "decreasing",
+                "oscillating",
+                "periodic_resets",
+                "sensor_nullable",
+                "special_values",
+                "boundary_specials",
+                "nullable_specials",
+            ]),
             vec![
                 sensor_enc.encode(&sensor_prim).into_array(),
                 drift_enc.encode(&drift_prim).into_array(),
+                constant_enc.encode(&constant_prim).into_array(),
+                decreasing_enc.encode(&decreasing_prim).into_array(),
+                oscillating_enc.encode(&oscillating_prim).into_array(),
+                periodic_resets_enc
+                    .encode(&periodic_resets_prim)
+                    .into_array(),
                 nullable_enc.encode(&sensor_nullable).into_array(),
+                special_enc.encode(&special_prim).into_array(),
+                boundary_enc.encode(&boundary_prim).into_array(),
+                nullable_special_enc.encode(&nullable_specials).into_array(),
             ],
             N,
             Validity::NonNullable,