fix

Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk>

Author: joseph-isaacs

vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/datetime.rs

@@ -0,0 +1,104 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+use std::sync::Arc;
+
+use vortex_array::ArrayRef;
+use vortex_array::IntoArray;
+use vortex_array::arrays::Extension;
+use vortex_array::arrays::PrimitiveArray;
+use vortex_array::arrays::StructArray;
+use vortex_array::arrays::TemporalArray;
+use vortex_array::dtype::FieldNames;
+use vortex_array::extension::datetime::TimeUnit;
+use vortex_array::validity::Validity;
+use vortex_array::vtable::ArrayId;
+use vortex_buffer::buffer;
+use vortex_error::VortexResult;
+
+use crate::fixtures::FlatLayoutFixture;
+
+pub struct DateTimeFixture;
+
+impl FlatLayoutFixture for DateTimeFixture {
+    fn name(&self) -> &str {
+        "datetime.vortex"
+    }
+
+    fn description(&self) -> &str {
+        "Temporal arrays covering timestamps, dates, and times with various units"
+    }
+
+    fn expected_encodings(&self) -> Vec<ArrayId> {
+        vec![Extension::ID]
+    }
+
+    fn build(&self) -> VortexResult<ArrayRef> {
+        // Timestamps in seconds (i64): 2024-01-01T00:00:00Z, 2024-06-15T12:30:00Z, 2024-12-31T23:59:59Z
+        let ts_seconds = TemporalArray::new_timestamp(
+            PrimitiveArray::new(
+                buffer![1704067200i64, 1718451000, 1735689599],
+                Validity::NonNullable,
+            )
+            .into_array(),
+            TimeUnit::Seconds,
+            None,
+        );
+
+        // Timestamps in milliseconds with timezone (i64)
+        let ts_millis_tz = TemporalArray::new_timestamp(
+            PrimitiveArray::new(
+                buffer![1704067200000i64, 1718451000000, 1735689599000],
+                Validity::NonNullable,
+            )
+            .into_array(),
+            TimeUnit::Milliseconds,
+            Some(Arc::from("UTC")),
+        );
+
+        // Nullable timestamps in microseconds (i64)
+        let ts_nullable = TemporalArray::new_timestamp(
+            PrimitiveArray::from_option_iter([
+                Some(1704067200000000i64),
+                None,
+                Some(1735689599000000i64),
+            ])
+            .into_array(),
+            TimeUnit::Microseconds,
+            None,
+        );
+
+        // Date in days since epoch (i32): 2024-01-01, 2024-06-15, 2024-12-31
+        let date_days = TemporalArray::new_date(
+            PrimitiveArray::new(buffer![19723i32, 19889, 19723 + 365], Validity::NonNullable)
+                .into_array(),
+            TimeUnit::Days,
+        );
+
+        // Time in seconds since midnight (i32): 00:00:00, 12:30:00, 23:59:59
+        let time_secs = TemporalArray::new_time(
+            PrimitiveArray::new(buffer![0i32, 45000, 86399], Validity::NonNullable).into_array(),
+            TimeUnit::Seconds,
+        );
+
+        let arr = StructArray::try_new(
+            FieldNames::from([
+                "ts_seconds",
+                "ts_millis_tz",
+                "ts_nullable_micros",
+                "date_days",
+                "time_seconds",
+            ]),
+            vec![
+                ts_seconds.into_array(),
+                ts_millis_tz.into_array(),
+                ts_nullable.into_array(),
+                date_days.into_array(),
+                time_secs.into_array(),
+            ],
+            3,
+            Validity::NonNullable,
+        )?;
+        Ok(arr.into_array())
+    }
+}

vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/decimal.rs

@@ -0,0 +1,74 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+use vortex_array::ArrayRef;
+use vortex_array::IntoArray;
+use vortex_array::arrays::Decimal;
+use vortex_array::arrays::DecimalArray;
+use vortex_array::arrays::StructArray;
+use vortex_array::dtype::DecimalDType;
+use vortex_array::dtype::FieldNames;
+use vortex_array::validity::Validity;
+use vortex_array::vtable::ArrayId;
+use vortex_buffer::buffer;
+use vortex_error::VortexResult;
+
+use crate::fixtures::FlatLayoutFixture;
+
+pub struct DecimalFixture;
+
+impl FlatLayoutFixture for DecimalFixture {
+    fn name(&self) -> &str {
+        "decimal.vortex"
+    }
+
+    fn description(&self) -> &str {
+        "Decimal arrays with varying precisions and scales, including nullable variants"
+    }
+
+    fn expected_encodings(&self) -> Vec<ArrayId> {
+        vec![Decimal::ID]
+    }
+
+    fn build(&self) -> VortexResult<ArrayRef> {
+        // Decimal(5,2) stored as i32: represents 123.45, -678.90, 0.01
+        let dec_5_2 = DecimalArray::new(
+            buffer![12345i32, -67890, 1],
+            DecimalDType::new(5, 2),
+            Validity::NonNullable,
+        );
+
+        // Decimal(10,4) stored as i64: represents 123456.7890, -0.0001, 999999.9999
+        let dec_10_4 = DecimalArray::new(
+            buffer![1234567890i64, -1, 9999999999],
+            DecimalDType::new(10, 4),
+            Validity::NonNullable,
+        );
+
+        // Decimal(18,0) stored as i64: large integers
+        let dec_18_0 = DecimalArray::new(
+            buffer![0i64, i64::MAX, i64::MIN],
+            DecimalDType::new(18, 0),
+            Validity::NonNullable,
+        );
+
+        // Nullable Decimal(7,3) stored as i32
+        let dec_nullable = DecimalArray::from_option_iter(
+            [Some(1234567i32), None, Some(-9999999)],
+            DecimalDType::new(7, 3),
+        );
+
+        let arr = StructArray::try_new(
+            FieldNames::from(["dec_5_2", "dec_10_4", "dec_18_0", "dec_nullable_7_3"]),
+            vec![
+                dec_5_2.into_array(),
+                dec_10_4.into_array(),
+                dec_18_0.into_array(),
+                dec_nullable.into_array(),
+            ],
+            3,
+            Validity::NonNullable,
+        )?;
+        Ok(arr.into_array())
+    }
+}

vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/listview.rs

@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+use vortex_array::ArrayRef;
+use vortex_array::IntoArray;
+use vortex_array::arrays::ListView;
+use vortex_array::arrays::ListViewArray;
+use vortex_array::arrays::PrimitiveArray;
+use vortex_array::arrays::StructArray;
+use vortex_array::arrays::VarBinArray;
+use vortex_array::dtype::FieldNames;
+use vortex_array::validity::Validity;
+use vortex_array::vtable::ArrayId;
+use vortex_buffer::buffer;
+use vortex_error::VortexResult;
+
+use crate::fixtures::FlatLayoutFixture;
+
+pub struct ListViewFixture;
+
+impl FlatLayoutFixture for ListViewFixture {
+    fn name(&self) -> &str {
+        "listview.vortex"
+    }
+
+    fn description(&self) -> &str {
+        "ListView arrays with integer and string elements"
+    }
+
+    fn expected_encodings(&self) -> Vec<ArrayId> {
+        vec![ListView::ID]
+    }
+
+    fn build(&self) -> VortexResult<ArrayRef> {
+        // ListView of i32: [[1,2,3], [4,5], [6], [7,8,9,10]]
+        let int_elements = PrimitiveArray::new(
+            buffer![1i32, 2, 3, 4, 5, 6, 7, 8, 9, 10],
+            Validity::NonNullable,
+        );
+        let int_offsets = PrimitiveArray::new(buffer![0u32, 3, 5, 6], Validity::NonNullable);
+        let int_sizes = PrimitiveArray::new(buffer![3u32, 2, 1, 4], Validity::NonNullable);
+        let int_listview = ListViewArray::try_new(
+            int_elements.into_array(),
+            int_offsets.into_array(),
+            int_sizes.into_array(),
+            Validity::NonNullable,
+        )?;
+
+        // ListView of strings: [["a","b"], ["hello"], [], ["x","y","z"]]
+        let str_elements = VarBinArray::from(vec!["a", "b", "hello", "x", "y", "z"]);
+        let str_offsets = PrimitiveArray::new(buffer![0u32, 2, 3, 3], Validity::NonNullable);
+        let str_sizes = PrimitiveArray::new(buffer![2u32, 1, 0, 3], Validity::NonNullable);
+        let str_listview = ListViewArray::try_new(
+            str_elements.into_array(),
+            str_offsets.into_array(),
+            str_sizes.into_array(),
+            Validity::NonNullable,
+        )?;
+
+        let arr = StructArray::try_new(
+            FieldNames::from(["int_listview", "str_listview"]),
+            vec![int_listview.into_array(), str_listview.into_array()],
+            4,
+            Validity::NonNullable,
+        )?;
+        Ok(arr.into_array())
+    }
+}

vortex-test/compat-gen/src/fixtures/arrays/synthetic/arrays/mod.rs

@@ -7,8 +7,11 @@
 
 mod bool;
 mod chunked;
+mod datetime;
+mod decimal;
 mod fixed_size_list;
 mod list;
+mod listview;
 mod null;
 mod primitive;
 mod struct_nested;
@@ -29,5 +32,8 @@ pub fn fixtures() -> Vec<Box<dyn FlatLayoutFixture>> {
         Box::new(list::ListFixture),
         Box::new(fixed_size_list::FixedSizeListFixture),
         Box::new(null::NullFixture),
+        Box::new(datetime::DateTimeFixture),
+        Box::new(decimal::DecimalFixture),
+        Box::new(listview::ListViewFixture),
     ]
 }

vortex-test/compat-gen/src/lib.rs

@@ -5,3 +5,67 @@ pub mod adapter;
 pub mod fixtures;
 pub mod manifest;
 pub mod validate;
+
+#[cfg(test)]
+mod tests {
+    use vortex_buffer::ByteBuffer;
+
+    use crate::adapter;
+    use crate::fixtures::all_fixtures;
+
+    #[test]
+    fn roundtrip_all_fixtures() {
+        let tmp = tempfile::tempdir().unwrap();
+        let fixtures = all_fixtures();
+
+        for fixture in &fixtures {
+            eprintln!("--- writing {} ---", fixture.name());
+            let entries = fixture.write(tmp.path()).unwrap();
+            for entry in &entries {
+                let path = tmp.path().join(&entry.name);
+                let bytes = std::fs::read(&path).unwrap();
+                eprintln!("  reading back {}...", entry.name);
+                let _array = adapter::read_file(ByteBuffer::from(bytes)).unwrap();
+                eprintln!("  OK: {}", entry.name);
+            }
+        }
+    }
+
+    #[test]
+    fn roundtrip_dataset_regular() {
+        let tmp = tempfile::tempdir().unwrap();
+        for dataset in all_fixtures()
+            .into_iter()
+            .filter(|f| f.name().contains(".regular."))
+        {
+            eprintln!("--- {} ---", dataset.name());
+            let entries = dataset.write(tmp.path()).unwrap();
+            for entry in &entries {
+                let path = tmp.path().join(&entry.name);
+                let bytes = std::fs::read(&path).unwrap();
+                eprintln!("  reading back...");
+                let _array = adapter::read_file(ByteBuffer::from(bytes)).unwrap();
+                eprintln!("  OK");
+            }
+        }
+    }
+
+    #[test]
+    fn roundtrip_dataset_compact() {
+        let tmp = tempfile::tempdir().unwrap();
+        for dataset in all_fixtures()
+            .into_iter()
+            .filter(|f| f.name().contains(".compact."))
+        {
+            eprintln!("--- {} ---", dataset.name());
+            let entries = dataset.write(tmp.path()).unwrap();
+            for entry in &entries {
+                let path = tmp.path().join(&entry.name);
+                let bytes = std::fs::read(&path).unwrap();
+                eprintln!("  reading back...");
+                let _array = adapter::read_file(ByteBuffer::from(bytes)).unwrap();
+                eprintln!("  OK");
+            }
+        }
+    }
+}