Add TPC-H to sqllogictests (#6830)

This PR adds TPC-H to to our SLT suite, it requires generating data
(script provided), and currently doesn't work due to subtle formatting
differences between the crates.

One reason for those differences is that we actually run the queries on
different schemas as they each infer a different schema.

I've also added some duckdb functionality as I tried to get it working,
and I figure its worth merging it anyway.

---------

Signed-off-by: Adam Gutglick <adam@spiraldb.com>

Author: AdamGS

.github/workflows/ci.yml

@@ -821,8 +821,13 @@ jobs:
         uses: ./.github/actions/setup-rust
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install uv
+        uses: spiraldb/actions/.github/actions/setup-uv@0.18.5
+        with:
+          sync: false
       - name: Run sqllogictest tests
         run: |
+          ./vortex-sqllogictest/slt/tpch/generate_data.sh
           cargo test -p vortex-sqllogictest --test sqllogictests
 
 

Cargo.lock

@@ -100,6 +100,7 @@ async-fs = "2.2.0"
 async-lock = "3.4"
 async-stream = "0.3.6"
 async-trait = "0.1.89"
+bigdecimal = "0.4.8"
 bindgen = "0.72.0"
 bit-vec = "0.8.0"
 bitvec = "1.0.1"

Cargo.toml

@@ -8,7 +8,7 @@ extend-ignore-re = [
 ]
 
 [files]
-extend-exclude = ["/vortex-bench/**", "/docs/references.bib", "benchmarks/**"]
+extend-exclude = ["/vortex-bench/**", "/docs/references.bib", "benchmarks/**", "vortex-sqllogictest/slt/**"]
 
 [type.py]
 extend-ignore-identifiers-re = [

_typos.toml

@@ -253,6 +253,9 @@ impl<'a> TryFrom<&'a ValueRef> for Scalar {
             ExtractedValue::HugeInt(_) => {
                 vortex_bail!("DuckDB HugeInt is not yet supported in Vortex");
             }
+            ExtractedValue::UHugeInt(_) => {
+                vortex_bail!("DuckDB UHugeInt is not yet supported in Vortex");
+            }
             ExtractedValue::UTinyInt(v) => Ok(Scalar::primitive(v, Nullable)),
             ExtractedValue::USmallInt(v) => Ok(Scalar::primitive(v, Nullable)),
             ExtractedValue::UInteger(v) => Ok(Scalar::primitive(v, Nullable)),

vortex-duckdb/src/convert/scalar.rs

@@ -348,7 +348,7 @@ pub fn flat_vector_to_vortex(vector: &VectorRef, len: usize) -> VortexResult<Arr
             StructArray::try_new(names, children, len, vector.validity_ref(len).to_validity())
                 .map(|a| a.into_array())
         }
-        _ => todo!("missing impl for {type_id:?}"),
+        _ => unimplemented!("missing impl for {type_id:?}"),
     }
 }
 

vortex-duckdb/src/convert/vector.rs

@@ -166,6 +166,10 @@ impl LogicalType {
     pub fn timestamp_tz() -> Self {
         Self::new(DUCKDB_TYPE::DUCKDB_TYPE_TIMESTAMP_TZ)
     }
+
+    pub fn date() -> Self {
+        Self::new(DUCKDB_TYPE::DUCKDB_TYPE_DATE)
+    }
 }
 
 impl LogicalTypeRef {
@@ -182,6 +186,10 @@ impl LogicalTypeRef {
         }
     }
 
+    pub fn is_decimal(&self) -> bool {
+        matches!(self.as_type_id(), DUCKDB_TYPE::DUCKDB_TYPE_DECIMAL)
+    }
+
     pub fn array_child_type(&self) -> LogicalType {
         unsafe { LogicalType::own(duckdb_array_type_child_type(self.as_ptr())) }
     }

vortex-duckdb/src/duckdb/logical_type.rs

@@ -3,11 +3,13 @@
 
 use std::ffi::CStr;
 
+use vortex::error::VortexExpect;
 use vortex::error::VortexResult;
 use vortex::error::vortex_bail;
 use vortex::error::vortex_err;
 
 use crate::cpp;
+use crate::cpp::DUCKDB_TYPE;
 use crate::duckdb::DataChunk;
 use crate::lifetime_wrapper;
 
@@ -71,7 +73,15 @@ impl QueryResultRef {
     /// Get the type of a column by index.
     pub fn column_type(&self, col_idx: usize) -> LogicalType {
         let dtype = unsafe { cpp::duckdb_column_type(self.as_ptr(), col_idx as u64) };
-        LogicalType::new(dtype)
+        if dtype == DUCKDB_TYPE::DUCKDB_TYPE_DECIMAL {
+            let lt = unsafe { cpp::duckdb_column_logical_type(self.as_ptr(), col_idx as u64) };
+            let precision = unsafe { cpp::duckdb_decimal_width(lt) };
+            let scale = unsafe { cpp::duckdb_decimal_scale(lt) };
+
+            LogicalType::decimal_type(precision, scale).vortex_expect("valid decimal")
+        } else {
+            LogicalType::new(dtype)
+        }
     }
 }
 

vortex-duckdb/src/duckdb/query_result.rs

@@ -59,6 +59,10 @@ impl ValueRef {
             DUCKDB_TYPE::DUCKDB_TYPE_BIGINT => {
                 ExtractedValue::BigInt(unsafe { cpp::duckdb_get_int64(self.as_ptr()) })
             }
+            DUCKDB_TYPE::DUCKDB_TYPE_HUGEINT => {
+                let huge_int = unsafe { cpp::duckdb_get_hugeint(self.as_ptr()) };
+                ExtractedValue::HugeInt(i128_from_parts(huge_int.upper, huge_int.lower))
+            }
             DUCKDB_TYPE::DUCKDB_TYPE_UTINYINT => {
                 ExtractedValue::UTinyInt(unsafe { cpp::duckdb_get_uint8(self.as_ptr()) })
             }
@@ -71,6 +75,10 @@ impl ValueRef {
             DUCKDB_TYPE::DUCKDB_TYPE_UBIGINT => {
                 ExtractedValue::UBigInt(unsafe { cpp::duckdb_get_uint64(self.as_ptr()) })
             }
+            DUCKDB_TYPE::DUCKDB_TYPE_UHUGEINT => {
+                let huge_uint = unsafe { cpp::duckdb_get_uhugeint(self.as_ptr()) };
+                ExtractedValue::UHugeInt(u128_from_parts(huge_uint.upper, huge_uint.lower))
+            }
             DUCKDB_TYPE::DUCKDB_TYPE_FLOAT => {
                 ExtractedValue::Float(unsafe { cpp::duckdb_get_float(self.as_ptr()) })
             }
@@ -149,7 +157,7 @@ impl ValueRef {
                     .collect::<Vec<_>>(),
             ),
             // ...other types remain unimplemented..
-            _ => vortex_panic!("Unsupported DuckDB value type {:?}", self),
+            other => vortex_panic!("Unsupported DuckDB value type {other:?}"),
         }
     }
 }
@@ -264,6 +272,11 @@ pub fn i128_from_parts(high: i64, low: u64) -> i128 {
     ((high as i128) << 64) | (low as i128)
 }
 
+#[inline]
+pub fn u128_from_parts(high: u64, low: u64) -> u128 {
+    ((high as u128) << 64) | (low as u128)
+}
+
 impl<T> TryFrom<Option<T>> for Value
 where
     T: Into<Value> + NativeDType,
@@ -376,6 +389,7 @@ pub enum ExtractedValue {
     USmallInt(u16),
     UInteger(u32),
     UBigInt(u64),
+    UHugeInt(u128),
     Float(f32),
     Double(f64),
     Boolean(bool),
@@ -394,6 +408,7 @@ pub enum ExtractedValue {
 #[cfg(test)]
 mod tests {
     use crate::duckdb::i128_from_parts;
+    use crate::duckdb::u128_from_parts;
 
     #[test]
     fn test_huge_int_from_parts() {
@@ -408,4 +423,17 @@ mod tests {
             (1i128 << 64) + (u64::MAX as i128)
         );
     }
+
+    #[test]
+    fn test_uhuge_int_from_parts() {
+        assert_eq!(u128_from_parts(0, 0), 0u128);
+        assert_eq!(u128_from_parts(0, 34534912), 34534912u128);
+        assert_eq!(u128_from_parts(0, u64::MAX), u64::MAX as u128);
+        assert_eq!(u128_from_parts(u64::MAX, u64::MAX), u128::MAX);
+        assert_eq!(
+            u128_from_parts(1, u64::MAX),
+            (1u128 << 64) + (u64::MAX as u128)
+        );
+        assert_eq!(u128_from_parts(1, 0), 1u128 << 64);
+    }
 }

vortex-duckdb/src/duckdb/value.rs

@@ -39,6 +39,7 @@ impl DataSourceTableFunction for VortexMultiFileScan {
 
         // Parse the URL and separate the base URL (keep scheme, host, etc.) from the path.
         let glob_url_str = glob_url_parameter.as_string();
+
         let glob_url = match Url::parse(glob_url_str.as_str()) {
             Ok(url) => Ok(url),
             Err(_) => Url::from_file_path(Path::new(glob_url_str.as_str()))