Remove duckdbfs (#8468)

DuckdbFS implementation for Vortex was introduced in
#6198 as opt-out, but changed
to opt-in in #6564 due to
performance regressions.
There were multiple issues
(#6709,
#6565
#6685) associated with it
which differ from vortex's file system behaviour.

It also requires additional dependencies CI which are a blocker for 
#8486 since MacOS runner
doesn't bundle openssl for x86_64 on arm, and builds fail.

As a long term goal, calling duckdb's blocking IO inside our event loop
isn't the right abstraction. We want to allow duckdb to use its own IO
outside vortex.

Duckdb fs is  also not maintaned actively so we're removing it

Signed-off-by: Mikhail Kot <mikhail@spiraldb.com>

Author: myrrc

vortex-duckdb/README.md

@@ -77,6 +77,4 @@ loaded.
 ## Testing a custom DuckDB tag
 
 Change `DUCKDB_VERSION` environment variable value to a preferred hash or commit
-(local build), or change build.rs (for testing in CI). If you use a commit,
-DuckDB needs to link httpfs statically so you also need to install CURL
-development headers (e.g. `libcurl4-openssl-dev`).
+(local build), or change build.rs (for testing in CI).

vortex-duckdb/build.rs

@@ -358,13 +358,12 @@ fn build_duckdb(version: &DuckDBVersion, duckdb_repo_dir: &Path) {
             ("1", "0")
         };
 
-    // If we're building from a commit we need to build httpfs and benchmark
+    // If we're building from a commit  we need to build benchmark
     // extensions statically, otherwise DuckDB tries to load them from an http
     // endpoint with version 0.0.1 (all non-tagged builds) which doesn't exist.
-    // httpfs static build also requires CURL dev headers
     let static_extensions = match version {
         DuckDBVersion::Release(_) => "parquet;jemalloc",
-        DuckDBVersion::Commit(_) => "parquet;jemalloc;httpfs;tpch;tpcds",
+        DuckDBVersion::Commit(_) => "parquet;jemalloc;tpch;tpcds",
     };
 
     let envs = [

vortex-duckdb/cpp/copy_function.cpp

@@ -58,13 +58,12 @@ unique_ptr<FunctionData> copy_to_bind(ClientContext &,
 }
 
 unique_ptr<GlobalFunctionData>
-copy_to_initialize_global(ClientContext &context, FunctionData &bind_data, const string &file_path) {
+copy_to_initialize_global(ClientContext &, FunctionData &bind_data, const string &file_path) {
     void *const ffi_bind = bind_data.Cast<CopyBindData>().ffi_data->DataPtr();
-    const auto ffi_ctx = reinterpret_cast<duckdb_client_context>(&context);
 
     duckdb_vx_error error_out = nullptr;
     const duckdb_vx_data ffi_global =
-        duckdb_copy_function_copy_to_initialize_global(ffi_ctx, ffi_bind, file_path.c_str(), &error_out);
+        duckdb_copy_function_copy_to_initialize_global(ffi_bind, file_path.c_str(), &error_out);
     if (error_out) {
         throw ExecutorException(IntoErrString(error_out));
     }

vortex-duckdb/cpp/table_function.cpp

@@ -197,16 +197,14 @@ bool projection_expression_pushdown(ClientContext &, const TableFunctionProjecti
  * and after a query another file is added matching the glob, for second query
  * bind() will be called again.
  */
-unique_ptr<FunctionData> duckdb_vx_table_function_bind(ClientContext &context,
+unique_ptr<FunctionData> duckdb_vx_table_function_bind(ClientContext &,
                                                        TableFunctionBindInput &input,
                                                        vector<LogicalType> &return_types,
                                                        vector<string> &names) {
     CTableBindResult result = {return_types, names};
 
     duckdb_vx_error error_out = nullptr;
-    auto ctx = reinterpret_cast<duckdb_client_context>(&context);
-    auto ffi_bind_data = duckdb_table_function_bind(ctx,
-                                                    reinterpret_cast<duckdb_vx_tfunc_bind_input>(&input),
+    auto ffi_bind_data = duckdb_table_function_bind(reinterpret_cast<duckdb_vx_tfunc_bind_input>(&input),
                                                     reinterpret_cast<duckdb_vx_tfunc_bind_result>(&result),
                                                     &error_out);
     if (error_out) {

vortex-duckdb/include/vortex.h

@@ -59,8 +59,7 @@ duckdb_vx_data duckdb_table_function_init_global(const duckdb_vx_tfunc_init_inpu
 extern duckdb_vx_data duckdb_table_function_init_local(void *global_init_data);
 
 extern
-duckdb_vx_data duckdb_table_function_bind(duckdb_client_context ctx,
-                                          duckdb_vx_tfunc_bind_input bind_input,
+duckdb_vx_data duckdb_table_function_bind(duckdb_vx_tfunc_bind_input bind_input,
                                           duckdb_vx_tfunc_bind_result bind_result,
                                           duckdb_vx_error *error_out);
 
@@ -74,8 +73,7 @@ duckdb_vx_data duckdb_copy_function_copy_to_bind(const char *const *column_names
                                                  duckdb_vx_error *error_out);
 
 extern
-duckdb_vx_data duckdb_copy_function_copy_to_initialize_global(duckdb_client_context client_context,
-                                                              const void *bind_data,
+duckdb_vx_data duckdb_copy_function_copy_to_initialize_global(const void *bind_data,
                                                               const char *file_path,
                                                               duckdb_vx_error *error_out);
 

vortex-duckdb/src/copy.rs

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
+use async_fs::OpenOptions;
 use futures::SinkExt;
 use futures::TryStreamExt;
 use futures::channel::mpsc;
@@ -28,9 +29,7 @@ use crate::RUNTIME;
 use crate::SESSION;
 use crate::convert::FromLogicalType;
 use crate::convert::data_chunk_to_vortex;
-use crate::duckdb::ClientContextRef;
 use crate::duckdb::DataChunkRef;
-use crate::duckdb::DuckDbFsWriter;
 use crate::duckdb::LogicalTypeRef;
 
 #[derive(Clone)]
@@ -111,7 +110,6 @@ pub fn copy_to_finalize(init_global: &mut CopyFunctionGlobal) -> VortexResult<()
 }
 
 pub fn copy_to_initialize_global(
-    client_context: &ClientContextRef,
     bind_data: &CopyFunctionBind,
     file_path: String,
 ) -> VortexResult<CopyFunctionGlobal> {
@@ -120,16 +118,16 @@ pub fn copy_to_initialize_global(
     let array_stream = ArrayStreamAdapter::new(bind_data.dtype.clone(), rx.into_stream());
 
     let handle = SESSION.handle();
-    // SAFETY: The ClientContext is owned by the Connection and lives for the duration of
-    // query execution. DuckDB keeps the connection alive while this copy function runs.
-    let ctx = unsafe { client_context.erase_lifetime() };
 
-    // Use DuckDB FS exclusively to match the DuckDB client context configuration.
-    let writer = DuckDbFsWriter::new(ctx, &file_path)
-        .map_err(|e| vortex_err!("Failed to create DuckDB FS writer for {file_path}: {e}"))?;
-
-    let write_task =
-        handle.spawn(async move { SESSION.write_options().write(writer, array_stream).await });
+    let write_task = handle.spawn(async move {
+        let writer = OpenOptions::new()
+            .write(true)
+            .truncate(true)
+            .create(true)
+            .open(file_path)
+            .await?;
+        SESSION.write_options().write(writer, array_stream).await
+    });
 
     let worker_pool = RUNTIME.new_pool();
     worker_pool.set_workers_to_available_parallelism();

vortex-duckdb/src/duckdb/file_system.rs

@@ -10,7 +10,6 @@ mod data_chunk;
 mod database;
 mod ddb_string;
 mod expr;
-mod file_system;
 mod logical_type;
 mod macro_;
 mod query_result;
@@ -36,7 +35,6 @@ pub use data_chunk::*;
 pub use database::*;
 pub use ddb_string::*;
 pub use expr::*;
-pub use file_system::*;
 pub use logical_type::*;
 pub use query_result::*;
 pub use reusable_dict::*;