Feature-gate datafusion-substrait behind optional substrait feature

Makes datafusion-substrait an optional dependency in datafusion-sqllogictest,
gated behind a new `substrait` feature flag. This saves ~13s of compile time
when substrait is not needed.

Author: zhuqi-lucas

.github/workflows/rust.yml

@@ -524,7 +524,7 @@ jobs:
         #  command cannot be run for all the .slt files. Run it for just one that works (limit.slt)
         #  until most of the tickets in https://github.com/apache/datafusion/issues/16248 are addressed
         #  and this command can be run without filters.
-        run: cargo test --test sqllogictests -- --substrait-round-trip limit.slt
+        run: cargo test --test sqllogictests --features substrait -- --substrait-round-trip limit.slt
 
   #  Temporarily commenting out the Windows flow, the reason is enormously slow running build
   #  Waiting for new Windows 2025 github runner

datafusion/sqllogictest/Cargo.toml

@@ -48,7 +48,7 @@ chrono = { workspace = true, optional = true }
 clap = { version = "4.5.53", features = ["derive", "env"] }
 datafusion = { workspace = true, default-features = true, features = ["avro"] }
 datafusion-spark = { workspace = true, default-features = true }
-datafusion-substrait = { workspace = true, default-features = true }
+datafusion-substrait = { workspace = true, default-features = true, optional = true }
 futures = { workspace = true }
 half = { workspace = true, default-features = true }
 indicatif = "0.18"
@@ -79,6 +79,7 @@ postgres = [
 parquet_encryption = [
     "datafusion/parquet_encryption",
 ]
+substrait = ["datafusion-substrait"]
 
 [dev-dependencies]
 env_logger = { workspace = true }

datafusion/sqllogictest/bin/sqllogictests.rs

@@ -20,10 +20,12 @@ use datafusion::common::instant::Instant;
 use datafusion::common::utils::get_available_parallelism;
 use datafusion::common::{DataFusionError, Result, exec_datafusion_err, exec_err};
 use datafusion_sqllogictest::{
-    CurrentlyExecutingSqlTracker, DataFusion, DataFusionSubstraitRoundTrip, Filter,
-    TestContext, df_value_validator, read_dir_recursive, setup_scratch_dir,
-    should_skip_file, should_skip_record, value_normalizer,
+    CurrentlyExecutingSqlTracker, DataFusion, Filter, TestContext, df_value_validator,
+    read_dir_recursive, setup_scratch_dir, should_skip_file, should_skip_record,
+    value_normalizer,
 };
+#[cfg(feature = "substrait")]
+use datafusion_sqllogictest::DataFusionSubstraitRoundTrip;
 use futures::stream::StreamExt;
 use indicatif::{
     HumanDuration, MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle,
@@ -287,6 +289,7 @@ async fn run_tests() -> Result<()> {
     }
 }
 
+#[cfg(feature = "substrait")]
 async fn run_test_file_substrait_round_trip(
     test_file: TestFile,
     validator: Validator,
@@ -328,6 +331,19 @@ async fn run_test_file_substrait_round_trip(
     res
 }
 
+#[cfg(not(feature = "substrait"))]
+async fn run_test_file_substrait_round_trip(
+    _test_file: TestFile,
+    _validator: Validator,
+    _mp: MultiProgress,
+    _mp_style: ProgressStyle,
+    _filters: &[Filter],
+    _currently_executing_sql_tracker: CurrentlyExecutingSqlTracker,
+) -> Result<()> {
+    use datafusion::common::plan_err;
+    plan_err!("Can not run substrait round-trip as the substrait feature is not enabled")
+}
+
 async fn run_test_file(
     test_file: TestFile,
     validator: Validator,

datafusion/sqllogictest/src/engines/mod.rs

@@ -19,13 +19,15 @@
 mod conversion;
 mod currently_executed_sql;
 mod datafusion_engine;
+#[cfg(feature = "substrait")]
 mod datafusion_substrait_roundtrip_engine;
 mod output;
 
 pub use datafusion_engine::DFSqlLogicTestError;
 pub use datafusion_engine::DataFusion;
 pub use datafusion_engine::convert_batches;
 pub use datafusion_engine::convert_schema_to_types;
+#[cfg(feature = "substrait")]
 pub use datafusion_substrait_roundtrip_engine::DataFusionSubstraitRoundTrip;
 pub use output::DFColumnType;
 pub use output::DFOutput;

datafusion/sqllogictest/src/lib.rs

@@ -33,6 +33,7 @@ pub use engines::DFColumnType;
 pub use engines::DFOutput;
 pub use engines::DFSqlLogicTestError;
 pub use engines::DataFusion;
+#[cfg(feature = "substrait")]
 pub use engines::DataFusionSubstraitRoundTrip;
 pub use engines::convert_batches;
 pub use engines::convert_schema_to_types;