fix

Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk>

Author: joseph-isaacs

Cargo.lock

@@ -38,6 +38,7 @@ tpchgen = { workspace = true }
 tpchgen-arrow = { workspace = true }
 
 # ClickBench parquet reading
+bytes = { workspace = true }
 parquet = { workspace = true }
 
 # Async runtime

vortex-test/compat-gen/Cargo.toml

@@ -2,6 +2,7 @@
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
 use arrow_array::RecordBatch;
+use bytes::Bytes;
 use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
 use vortex_array::ArrayRef;
 use vortex_array::IntoArray;
@@ -12,31 +13,27 @@ use vortex_error::vortex_err;
 
 use crate::fixtures::DatasetFixture;
 
-/// First partition of ClickBench hits, limited to 1000 rows.
-const CLICKBENCH_URL: &str =
-    "https://pub-3ba949c0f0354ac18db1f0f14f0a2c52.r2.dev/clickbench/parquet_many/hits_0.parquet";
+/// 5×1000 rows sampled from deterministic random offsets in ClickBench hits partition 0.
+/// Offsets (seed=42): [26225, 116739, 288389, 670487, 777572].
+const CLICKBENCH_PARQUET: &[u8] = include_bytes!("../../../../data/clickbench_hits_5k.parquet");
 
-struct ClickBenchHits1kFixture;
+struct ClickBenchHits5kFixture;
 
-impl DatasetFixture for ClickBenchHits1kFixture {
+impl DatasetFixture for ClickBenchHits5kFixture {
     fn name(&self) -> &str {
-        "clickbench_hits_1k"
+        "clickbench_hits_5k"
     }
 
     fn description(&self) -> &str {
-        "First 1000 rows of ClickBench hits dataset with wide schema of primitives and strings"
+        "5000 rows (5x1000 from random offsets) of ClickBench hits dataset with wide schema of primitives and strings"
     }
 
     fn build(&self) -> VortexResult<ArrayRef> {
-        let bytes = reqwest::blocking::get(CLICKBENCH_URL)
-            .map_err(|e| vortex_err!("failed to download ClickBench parquet: {e}"))?
-            .bytes()
-            .map_err(|e| vortex_err!("failed to read ClickBench response body: {e}"))?;
+        let bytes = Bytes::from_static(CLICKBENCH_PARQUET);
 
         let reader = ParquetRecordBatchReaderBuilder::try_new(bytes)
             .map_err(|e| vortex_err!("failed to open parquet: {e}"))?
             .with_batch_size(1000)
-            .with_limit(1000)
             .build()
             .map_err(|e| vortex_err!("failed to build parquet reader: {e}"))?;
 
@@ -55,5 +52,5 @@ impl DatasetFixture for ClickBenchHits1kFixture {
 }
 
 pub fn fixtures() -> Vec<Box<dyn DatasetFixture>> {
-    vec![Box::new(ClickBenchHits1kFixture)]
+    vec![Box::new(ClickBenchHits5kFixture)]
 }

vortex-test/compat-gen/src/fixtures/arrays/datasets/clickbench.rs

@@ -14,3 +14,42 @@ pub fn fixtures() -> Vec<Box<dyn DatasetFixture>> {
     fixtures.extend(clickbench::fixtures());
     fixtures
 }
+
+#[cfg(test)]
+mod tests {
+    use vortex::file::WriteStrategyBuilder;
+
+    use super::fixtures;
+    use crate::adapter;
+
+    fn is_clickbench_fixture(name: &str) -> bool {
+        name.contains("clickbench")
+    }
+
+    #[test]
+    fn roundtrip_non_clickbench_fixtures_to_bytes() {
+        for dataset in fixtures()
+            .into_iter()
+            .filter(|fixture| !is_clickbench_fixture(fixture.name()))
+        {
+            eprintln!("--- writing {} regular to bytes ---", dataset.name());
+            let array = dataset.build().unwrap();
+            let regular_bytes = adapter::write_compressed_to_bytes(
+                array.clone(),
+                WriteStrategyBuilder::default().build(),
+            )
+            .unwrap();
+            let _regular = adapter::read_file(regular_bytes).unwrap();
+
+            eprintln!("--- writing {} compact to bytes ---", dataset.name());
+            let compact_bytes = adapter::write_compressed_to_bytes(
+                array,
+                WriteStrategyBuilder::default()
+                    .with_compact_encodings()
+                    .build(),
+            )
+            .unwrap();
+            let _compact = adapter::read_file(compact_bytes).unwrap();
+        }
+    }
+}

vortex-test/compat-gen/src/fixtures/arrays/datasets/mod.rs

@@ -44,7 +44,7 @@ impl FlatLayoutFixture for ByteBoolFixture {
         let sparse_true: Vec<bool> = (0..N).map(|i| i % 127 == 0).collect();
         let edge_null_vals: Vec<bool> = (0..N).map(|i| i % 4 == 0).collect();
         let edge_null_validity = Validity::from(
-            BoolArray::from_iter((0..N).map(|i| i >= 8 && i < N - 8)).to_bit_buffer(),
+            BoolArray::from_iter((0..N).map(|i| (8..N - 8).contains(&i))).to_bit_buffer(),
         );
 
         let arr = StructArray::try_new(

vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/bytebool.rs

@@ -90,7 +90,7 @@ impl FlatLayoutFixture for FsstFixture {
         let all_null_clustered = VarBinArray::from(
             (0..N)
                 .map(|i| {
-                    if i < 16 || i >= N - 16 {
+                    if !(16..N - 16).contains(&i) {
                         None
                     } else {
                         Some("clustered-null-middle")

vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/fsst.rs

@@ -52,7 +52,7 @@ impl FlatLayoutFixture for PcoFixture {
                 if i % 257 == 0 {
                     1_000_000.0 + i as f64
                 } else {
-                    3.14159
+                    std::f64::consts::PI
                 }
             })
             .collect();

vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/pco.rs

@@ -58,13 +58,9 @@ impl FlatLayoutFixture for SparseFixture {
             (0..N as i32).map(|i| if i % 75 == 0 { Some(99) } else { Some(10) }),
         );
         let all_default = ConstantArray::new(10i32, N).into_array();
-        let clustered_edges = PrimitiveArray::from_option_iter((0..N as i64).map(|i| {
-            if i < 8 || i >= N as i64 - 8 {
-                Some(i * 9)
-            } else {
-                None
-            }
-        }));
+        let clustered_edges = PrimitiveArray::from_option_iter(
+            (0..N as i64).map(|i| (i < 8 || i >= N as i64 - 8).then(|| i * 9)),
+        );
         let almost_dense = PrimitiveArray::from_option_iter(
             (0..N as i32).map(|i| if i % 32 == 0 { None } else { Some((i % 5) + 1) }),
         );

vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/sparse.rs

@@ -39,7 +39,7 @@ impl FlatLayoutFixture for ZstdFixture {
                 if i % 257 == 0 {
                     100_000.0 + i as f64
                 } else {
-                    3.14159
+                    std::f64::consts::PI
                 }
             })
             .collect();

vortex-test/compat-gen/src/fixtures/arrays/synthetic/encodings/zstd.rs

@@ -14,3 +14,81 @@ pub fn fixtures() -> Vec<Box<dyn FlatLayoutFixture>> {
     fixtures.extend(encodings::fixtures());
     fixtures
 }
+
+#[cfg(test)]
+mod tests {
+    use vortex_array::IntoArray;
+    use vortex_array::arrays::BoolArray;
+    use vortex_array::arrays::PrimitiveArray;
+    use vortex_array::arrays::StructArray;
+    use vortex_array::arrays::VarBinViewArray;
+    use vortex_array::assert_arrays_eq;
+    use vortex_array::dtype::FieldNames;
+    use vortex_array::validity::Validity;
+
+    use super::fixtures;
+    use crate::adapter;
+    use crate::fixtures::check_expected_encodings;
+
+    fn boundary_length_array(len: usize) -> vortex_error::VortexResult<vortex_array::ArrayRef> {
+        let ints = PrimitiveArray::from_iter((0..i32::try_from(len)?).map(|i| i - 17));
+        let nullable_ints = PrimitiveArray::from_option_iter(
+            (0..len as i64).map(|i| if i % 5 == 0 { None } else { Some(i * 3 - 7) }),
+        );
+        let bools = BoolArray::from_iter((0..len).map(|i| i % 3 == 0));
+        let strings = VarBinViewArray::from_iter_nullable_str((0..len).map(|i| match i % 5 {
+            0 => None,
+            1 => Some(""),
+            2 => Some("edge"),
+            3 => Some("boundary-length-string"),
+            _ => Some("zz"),
+        }));
+
+        Ok(StructArray::try_new(
+            FieldNames::from(["ints", "nullable_ints", "bools", "strings"]),
+            vec![
+                ints.into_array(),
+                nullable_ints.into_array(),
+                bools.into_array(),
+                strings.into_array(),
+            ],
+            len,
+            Validity::NonNullable,
+        )?
+        .into_array())
+    }
+
+    #[test]
+    fn roundtrip_fixtures_to_bytes() {
+        for fixture in fixtures() {
+            eprintln!("--- writing {} to bytes ---", fixture.name());
+            let array = fixture.build().unwrap();
+            check_expected_encodings(&array, fixture.as_ref()).unwrap();
+            let bytes = adapter::write_file_to_bytes(array.clone()).unwrap();
+            let roundtripped = adapter::read_file(bytes).unwrap();
+            assert_arrays_eq!(array, roundtripped);
+            eprintln!("  OK: {}", fixture.name());
+        }
+    }
+
+    #[test]
+    fn roundtrip_boundary_lengths_to_bytes() {
+        const BOUNDARY_LENGTHS: [usize; 15] = [
+            0, 1, 2, 31, 32, 63, 64, 127, 128, 255, 256, 511, 512, 1023, 1025,
+        ];
+
+        for len in BOUNDARY_LENGTHS {
+            eprintln!(
+                "--- writing shared boundary fixture length {} to bytes ---",
+                len
+            );
+            let boundary_array = boundary_length_array(len).unwrap();
+            if len == 0 {
+                assert!(adapter::write_file_to_bytes(boundary_array).is_err());
+                continue;
+            }
+            let bytes = adapter::write_file_to_bytes(boundary_array).unwrap();
+            let _array = adapter::read_file(bytes).unwrap();
+        }
+    }
+}