Merge branch 'main' of https://github.com/apache/datafusion into lambda4

Author: gstvg

.github/workflows/audit.yml

@@ -42,7 +42,7 @@ jobs:
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
       - name: Install cargo-audit
-        uses: taiki-e/install-action@94cb46f8d6e437890146ffbd78a778b78e623fb2  # v2.74.0
+        uses: taiki-e/install-action@85b24a67ef0c632dfefad70b9d5ce8fddb040754  # v2.75.10
         with:
           tool: cargo-audit
       - name: Run audit check

.github/workflows/rust.yml

@@ -430,7 +430,7 @@ jobs:
           sudo apt-get update -qq
           sudo apt-get install -y -qq clang
       - name: Setup wasm-pack
-        uses: taiki-e/install-action@94cb46f8d6e437890146ffbd78a778b78e623fb2  # v2.74.0
+        uses: taiki-e/install-action@85b24a67ef0c632dfefad70b9d5ce8fddb040754  # v2.75.10
         with:
           tool: wasm-pack
       - name: Run tests with headless mode
@@ -770,7 +770,7 @@ jobs:
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
       - name: Install cargo-msrv
-        uses: taiki-e/install-action@94cb46f8d6e437890146ffbd78a778b78e623fb2  # v2.74.0
+        uses: taiki-e/install-action@85b24a67ef0c632dfefad70b9d5ce8fddb040754  # v2.75.10
         with:
           tool: cargo-msrv
 

Cargo.lock

@@ -162,7 +162,7 @@ glob = "0.3.0"
 half = { version = "2.7.0", default-features = false }
 hashbrown = { version = "0.17.0" }
 hex = { version = "0.4.3" }
-indexmap = "2.13.1"
+indexmap = "2.14.0"
 insta = { version = "1.47.2", features = ["glob", "filters"] }
 itertools = "0.14"
 itoa = "1.0"

Cargo.toml

@@ -677,7 +677,7 @@ run_tpch() {
     echo "Running tpch benchmark..."
 
     FORMAT=$2
-    debug_run $CARGO_COMMAND --bin dfbench -- tpch --iterations 5 --path "${TPCH_DIR}" --prefer_hash_join "${PREFER_HASH_JOIN}" --format ${FORMAT} -o "${RESULTS_FILE}" ${QUERY_ARG} ${LATENCY_ARG}
+    debug_run $CARGO_COMMAND --bin dfbench -- tpch --iterations 5 --path "${TPCH_DIR}" --scale-factor "${SCALE_FACTOR}" --prefer_hash_join "${PREFER_HASH_JOIN}" --format ${FORMAT} -o "${RESULTS_FILE}" ${QUERY_ARG} ${LATENCY_ARG}
 }
 
 # Runs the tpch in memory (needs tpch parquet data)
@@ -693,7 +693,7 @@ run_tpch_mem() {
     echo "RESULTS_FILE: ${RESULTS_FILE}"
     echo "Running tpch_mem benchmark..."
     # -m means in memory
-    debug_run $CARGO_COMMAND --bin dfbench -- tpch --iterations 5 --path "${TPCH_DIR}" --prefer_hash_join "${PREFER_HASH_JOIN}" -m --format parquet -o "${RESULTS_FILE}" ${QUERY_ARG} ${LATENCY_ARG}
+    debug_run $CARGO_COMMAND --bin dfbench -- tpch --iterations 5 --path "${TPCH_DIR}" --scale-factor "${SCALE_FACTOR}" --prefer_hash_join "${PREFER_HASH_JOIN}" -m --format parquet -o "${RESULTS_FILE}" ${QUERY_ARG} ${LATENCY_ARG}
 }
 
 # Runs the tpcds benchmark
@@ -900,7 +900,7 @@ data_imdb() {
                 if [ "${DOWNLOADED_SIZE}" != "${expected_size}" ]; then
                     echo "Error: Download size mismatch"
                     echo "Expected: ${expected_size}"
-                    echo "Got: ${DOWNLADED_SIZE}"
+                    echo "Got: ${DOWNLOADED_SIZE}"
                     echo "Please re-initiate the download"
                     return 1
                 fi

benchmarks/bench.sh

@@ -164,12 +164,12 @@ def lineformat(
 ) -> None:
     baseline = BenchmarkRun.load_from_file(baseline)
     context = baseline.context
-    benchamrk_str = f"benchmark,name={context.name},version={context.benchmark_version},datafusion_version={context.datafusion_version},num_cpus={context.num_cpus}"
+    benchmark_str = f"benchmark,name={context.name},version={context.benchmark_version},datafusion_version={context.datafusion_version},num_cpus={context.num_cpus}"
     for query in baseline.queries:
         query_str = f"query=\"{query.query}\""
         timestamp = f"{query.start_time*10**9}"
         for iter_num, result in enumerate(query.iterations):
-            print(f"{benchamrk_str} {query_str},iteration={iter_num},row_count={result.row_count},elapsed_ms={result.elapsed*1000:.0f} {timestamp}\n")
+            print(f"{benchmark_str} {query_str},iteration={iter_num},row_count={result.row_count},elapsed_ms={result.elapsed*1000:.0f} {timestamp}\n")
 
 def main() -> None:
     parser = ArgumentParser()

benchmarks/lineprotocol.py

@@ -16,7 +16,7 @@ where
         c_custkey = o_custkey
   and l_orderkey = o_orderkey
   and o_orderdate >= date '1993-10-01'
-  and o_orderdate < date '1994-01-01'
+  and o_orderdate < date '1993-10-01' + interval '3' month
   and l_returnflag = 'R'
   and c_nationkey = n_nationkey
 group by

benchmarks/queries/q10.sql

@@ -13,7 +13,7 @@ group by
     ps_partkey having
     sum(ps_supplycost * ps_availqty) > (
     select
-    sum(ps_supplycost * ps_availqty) * 0.0001
+    sum(ps_supplycost * ps_availqty) * 0.0001 /* __TPCH_Q11_FRACTION__ */
     from
     partsupp,
     supplier,
@@ -24,4 +24,4 @@ group by
                   and n_name = 'GERMANY'
     )
 order by
-    value desc;
+    value desc;

benchmarks/queries/q11.sql

@@ -23,8 +23,8 @@ where
   and l_commitdate < l_receiptdate
   and l_shipdate < l_commitdate
   and l_receiptdate >= date '1994-01-01'
-  and l_receiptdate < date '1995-01-01'
+  and l_receiptdate < date '1994-01-01' + interval '1' year
 group by
     l_shipmode
 order by
-    l_shipmode;
+    l_shipmode;

benchmarks/queries/q12.sql

@@ -10,4 +10,4 @@ from
 where
         l_partkey = p_partkey
   and l_shipdate >= date '1995-09-01'
-  and l_shipdate < date '1995-10-01';
+  and l_shipdate < date '1995-09-01' + interval '1' month;