Shuffle project files for preparing to crates.io release (#405)

The way the project is laid out today make it unsuitable to be published
to crates.io, mainly because of the main crate having git dependencies.

This PR just shuffles files and modules around so that we can have a
clean main crate that can be sent to crates.io.

Author: gabotechs

Cargo.lock

@@ -45,31 +45,21 @@ crossbeam-queue = "0.3"
 sysinfo = { version = "0.30", optional = true }
 sketches-ddsketch = { version = "0.3", features = ["use_serde"] }
 bincode = "1"
+tonic-prost = "0.14.2"
 
 # integration_tests deps
 insta = { version = "1.46.0", features = ["filters"], optional = true }
-tpchgen = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "438e9c2dbc25b2fff82c0efc08b3f13b5707874f", optional = true }
-tpchgen-arrow = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "438e9c2dbc25b2fff82c0efc08b3f13b5707874f", optional = true }
 parquet = { version = "58", optional = true }
 arrow = { version = "58", optional = true, features = ["test_utils"] }
 hyper-util = { version = "0.1.16", optional = true }
-pretty_assertions = { version = "1.4", optional = true }
-reqwest = { version = "0.12", optional = true }
-zip = { version = "6.0", optional = true }
-tonic-prost = "0.14.2"
 
 [features]
 avro = ["datafusion/avro"]
 integration = [
   "insta",
-  "tpchgen",
-  "tpchgen-arrow",
   "parquet",
   "arrow",
   "hyper-util",
-  "pretty_assertions",
-  "reqwest",
-  "zip",
 ]
 
 system-metrics = ["sysinfo"]
@@ -80,15 +70,12 @@ clickbench = ["integration"]
 sysinfo = ["dep:sysinfo"]
 
 [dev-dependencies]
+datafusion-distributed-benchmarks = { path = "benchmarks" }
 structopt = "0.3"
 insta = { version = "1.46.0", features = ["filters"] }
-tpchgen = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "438e9c2dbc25b2fff82c0efc08b3f13b5707874f" }
-tpchgen-arrow = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "438e9c2dbc25b2fff82c0efc08b3f13b5707874f" }
 parquet = "58"
 arrow = { version = "58", features = ["test_utils"] }
 tokio-stream = { version = "0.1.17", features = ["sync"] }
 hyper-util = "0.1.16"
 pretty_assertions = "1.4"
-reqwest = "0.12"
-zip = "6.0"
 test-case = "3.3.1"

Cargo.toml

@@ -6,7 +6,6 @@ default-run = "dfbench"
 
 [dependencies]
 datafusion = { workspace = true }
-datafusion-proto = { workspace = true }
 datafusion-distributed = { path = "..", features = [
   "integration",
   "system-metrics",
@@ -19,19 +18,19 @@ serde = "1.0.219"
 serde_json = "1.0.141"
 env_logger = "0.11.8"
 async-trait = "0.1.89"
-chrono = "0.4.42"
 futures = "0.3.31"
-dashmap = "6.0.1"
-prost = "0.14.1"
 url = "2.5.7"
-arrow-flight = "58"
+arrow = { version = "58", features = ["test_utils"] }
 tonic = { version = "0.14.1", features = ["transport"] }
+tpchgen = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "438e9c2dbc25b2fff82c0efc08b3f13b5707874f" }
+tpchgen-arrow = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "438e9c2dbc25b2fff82c0efc08b3f13b5707874f" }
+reqwest = "0.12"
+zip = "6.0"
 axum = "0.7"
 object_store = { version = "0.13", features = ["aws"] }
 aws-config = "1"
 aws-sdk-ec2 = "1"
-openssl = { version = "0.10", features = ["vendored"] }
-clap = "4.5"
+openssl = { version = "0.10", features = ["vendored"] } # Keep this. Necessary for the remote benchmarks worker.
 mimalloc = "0.1"
 
 [dev-dependencies]

benchmarks/Cargo.toml

@@ -2,6 +2,7 @@
 !jest.config.js
 *.d.ts
 node_modules
+.cdk-outputs.json
 
 # CDK asset staging directory
 .cdk.staging

benchmarks/cdk/.gitignore

@@ -1,4 +1,4 @@
-use crate::test_utils::benchmarks_common;
+use super::common;
 use datafusion::common::DataFusionError;
 use std::fs;
 use std::io::Write;
@@ -10,11 +10,11 @@ const URL: &str =
     "https://datasets.clickhouse.com/hits_compatible/athena_partitioned/hits_{}.parquet";
 
 pub fn get_queries() -> Vec<String> {
-    benchmarks_common::get_queries("testdata/clickbench/queries")
+    common::get_queries("testdata/clickbench/queries")
 }
 
 pub fn get_query(id: &str) -> Result<String, DataFusionError> {
-    benchmarks_common::get_query("testdata/clickbench/queries", id)
+    common::get_query("testdata/clickbench/queries", id)
 }
 
 /// Downloads the datafusion-benchmarks repository as a zip file

src/test_utils/clickbench.rs benchmarks/src/datasets/clickbench.rssrc/test_utils/clickbench.rs renamed to benchmarks/src/datasets/clickbench.rs

@@ -3,8 +3,15 @@ use datafusion::prelude::{ParquetReadOptions, SessionContext};
 use std::fs;
 use std::path::Path;
 
-pub(crate) fn get_queries(path: &str) -> Vec<String> {
-    let queries_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join(path);
+/// Returns the workspace root directory (parent of the benchmarks crate).
+fn workspace_root() -> &'static Path {
+    Path::new(env!("CARGO_MANIFEST_DIR"))
+        .parent()
+        .expect("benchmarks crate should be inside a workspace")
+}
+
+pub fn get_queries(path: &str) -> Vec<String> {
+    let queries_dir = workspace_root().join(path);
     let mut result = vec![];
     for file in queries_dir.read_dir().unwrap() {
         let file = file.unwrap();
@@ -36,8 +43,8 @@ pub(crate) fn get_queries(path: &str) -> Vec<String> {
     result
 }
 
-pub(crate) fn get_query(path: &str, id: &str) -> Result<String, DataFusionError> {
-    let queries_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join(path);
+pub fn get_query(path: &str, id: &str) -> Result<String, DataFusionError> {
+    let queries_dir = workspace_root().join(path);
 
     if !queries_dir.exists() {
         return internal_err!(

src/test_utils/benchmarks_common.rs benchmarks/src/datasets/common.rssrc/test_utils/benchmarks_common.rs renamed to benchmarks/src/datasets/common.rs

@@ -0,0 +1,6 @@
+pub mod clickbench;
+mod common;
+pub mod tpcds;
+pub mod tpch;
+
+pub use common::register_tables;

benchmarks/src/datasets/mod.rs

@@ -1,4 +1,4 @@
-use crate::test_utils::benchmarks_common;
+use super::common;
 use arrow::datatypes::{DataType, Field};
 use datafusion::common::internal_err;
 use datafusion::error::DataFusionError;
@@ -18,11 +18,11 @@ use std::sync::Arc;
 const URL: &str = "https://github.com/apache/datafusion-benchmarks/archive/refs/heads/main.zip";
 
 pub fn get_queries() -> Vec<String> {
-    benchmarks_common::get_queries("testdata/tpcds/queries")
+    common::get_queries("testdata/tpcds/queries")
 }
 
 pub fn get_query(id: &str) -> Result<String, DataFusionError> {
-    benchmarks_common::get_query("testdata/tpcds/queries", id)
+    common::get_query("testdata/tpcds/queries", id)
 }
 
 /// Downloads the datafusion-benchmarks repository as a zip file

src/test_utils/tpcds.rs benchmarks/src/datasets/tpcds.rssrc/test_utils/tpcds.rs renamed to benchmarks/src/datasets/tpcds.rs

@@ -1,8 +1,9 @@
-use crate::test_utils::benchmarks_common;
+use super::common;
 use arrow::record_batch::RecordBatch;
 use datafusion::error::DataFusionError;
 use parquet::{arrow::arrow_writer::ArrowWriter, file::properties::WriterProperties};
 use std::fs;
+use std::path::Path;
 use tpchgen::generators::{
     CustomerGenerator, LineItemGenerator, NationGenerator, OrderGenerator, PartGenerator,
     PartSuppGenerator, RegionGenerator, SupplierGenerator,
@@ -13,19 +14,17 @@ use tpchgen_arrow::{
 };
 
 pub fn get_queries() -> Vec<String> {
-    benchmarks_common::get_queries("testdata/tpch/queries")
+    common::get_queries("testdata/tpch/queries")
 }
 
 pub fn get_query(id: &str) -> Result<String, DataFusionError> {
-    benchmarks_common::get_query("testdata/tpch/queries", id)
+    common::get_query("testdata/tpch/queries", id)
 }
 
-// generate_table creates a parquet file in the data directory from an arrow RecordBatch row
-// source.
 fn generate_table<A>(
     mut data_source: A,
     table_name: &str,
-    data_dir: &std::path::Path,
+    data_dir: &Path,
 ) -> Result<(), Box<dyn std::error::Error>>
 where
     A: Iterator<Item = RecordBatch>,
@@ -49,18 +48,16 @@ where
     Ok(())
 }
 
-// generate_tpch_data generates all TPC-H tables in the specified data directory.
-pub fn generate_tpch_data(data_dir: &std::path::Path, sf: f64, parts: i32) {
+/// Generates all TPC-H tables as parquet files in the specified data directory.
+pub fn generate_tpch_data(data_dir: &Path, sf: f64, parts: i32) {
     fs::create_dir_all(data_dir).expect("Failed to create data directory");
 
     macro_rules! must_generate_tpch_table {
         ($generator:ident, $arrow:ident, $name:literal) => {
             let data_dir = data_dir.join($name);
             fs::create_dir_all(data_dir.clone()).expect("Failed to create data directory");
-            // create three partitions for the table
             (1..=parts).for_each(|part| {
                 generate_table(
-                    // TODO: Consider adjusting the partitions and batch sizes.
                     $arrow::new($generator::new(sf, part, parts)).with_batch_size(1000),
                     &format!("{part}"),
                     &data_dir,

src/test_utils/tpch.rs benchmarks/src/datasets/tpch.rssrc/test_utils/tpch.rs renamed to benchmarks/src/datasets/tpch.rs

@@ -0,0 +1 @@
+pub mod datasets;