Updated datafusion-vortex submission with up-to-date versions and new dedicated tool

Author: AdamGS

datafusion-vortex-partitioned/README.md

@@ -0,0 +1,43 @@
+# DataFusion + Vortex
+
+Partitioned Vortex dataset, converted one-for-one from the 100 ClickBench Parquet files and queried with [`vortex-datafusion-cli`].
+
+[`vortex-datafusion-cli`]: https://github.com/vortex-data/vortex-datafusion-cli
+
+## Cookbook: Generate benchmark results
+
+Follow the same EC2 setup used by [datafusion-partitioned](../datafusion-partitioned/README.md), then run:
+
+```bash
+cd ClickBench/datafusion-vortex-partitioned
+bash benchmark.sh
+```
+
+The shared benchmark harness builds `vortex-datafusion-cli`, downloads the partitioned Parquet files, converts each `partitioned/hits_N.parquet` file into exactly one `vortex/hits_N.vortex` file, and runs the query set.
+
+The `install` script checks out `vortex-datafusion-cli` tag `0.70.0-53.1.0`. CLI tags use `<vortex-version>-<df-version>`, where the first component is the `vortex-datafusion` crate version and the second is the DataFusion/DataFusion CLI version.
+
+You can update/preview the results by running:
+
+```bash
+./make-json.sh <machine-name> # Example. ./make-json.sh c6a.xlarge
+```
+
+## Parquet To Vortex Conversion
+
+Each input file is converted independently through `vortex-datafusion-cli`:
+
+```sql
+CREATE EXTERNAL TABLE hits_parquet
+STORED AS PARQUET
+LOCATION 'partitioned/hits_0.parquet'
+OPTIONS ('binary_as_string' 'true');
+
+COPY (
+    SELECT * EXCEPT ("EventDate"),
+           CAST(CAST("EventDate" AS INTEGER) AS DATE) AS "EventDate"
+    FROM hits_parquet
+) TO 'vortex/hits_0.vortex' STORED AS VORTEX;
+```
+
+`binary_as_string=true` handles the incorrect Parquet logical annotation before Vortex is written. The produced Vortex files store those fields as strings, so benchmark reads use only the Vortex table registration.

datafusion-vortex-partitioned/benchmark.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 # Thin shim — actual flow is in lib/benchmark-common.sh.
-# query_bench (the vortex driver) handles its own dataset download/conversion.
-export BENCH_DOWNLOAD_SCRIPT=""
+export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-partitioned"
 export BENCH_DURABLE=yes
 export BENCH_RESTARTABLE=no
 exec ../lib/benchmark-common.sh

datafusion-vortex-partitioned/check

@@ -1,6 +1,5 @@
 #!/bin/bash
 set -e
 
-# Stateless system — confirm datafusion-cli (the playground's query
-# driver) is on PATH.
-command -v datafusion-cli >/dev/null
+DF=vortex-datafusion-cli/target/release/vortex-datafusion-cli
+"$DF" -q -c "SELECT 1" >/dev/null

datafusion-vortex-partitioned/create.sql

@@ -1,9 +1,3 @@
-CREATE EXTERNAL TABLE hits_raw
-STORED AS PARQUET
-LOCATION 'partitioned'
-OPTIONS ('binary_as_string' 'true');
-
-CREATE VIEW hits AS
-SELECT * EXCEPT ("EventDate"),
-       CAST(CAST("EventDate" AS INTEGER) AS DATE) AS "EventDate"
-FROM hits_raw;
+CREATE EXTERNAL TABLE hits
+STORED AS VORTEX
+LOCATION 'vortex';

datafusion-vortex-partitioned/data-size

@@ -1,5 +1,4 @@
 #!/bin/bash
 set -e
 
-# Sum the byte counts of all generated .vortex files.
-find . -name '*.vortex' -printf '%s\n' | awk '{s+=$1} END {print s+0}'
+find vortex -name '*.vortex' -printf '%s\n' | awk '{s+=$1} END {print s+0}'

datafusion-vortex-partitioned/install

@@ -1,41 +1,36 @@
 #!/bin/bash
 set -e
 
-VORTEX_VERSION=0.44.0
+if [ ! -x vortex-datafusion-cli/target/release/vortex-datafusion-cli ]; then
+    # <vortex-version>-<datafusion-version>
+    CLI_TAG=0.70.0-53.1.0
 
-if ! command -v cargo >/dev/null 2>&1; then
-    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > rust-init.sh
-    bash rust-init.sh -y
-fi
-export HOME=${HOME:=~}
-# shellcheck disable=SC1091
-source ~/.cargo/env
+    if ! command -v cargo >/dev/null 2>&1; then
+        curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs > rust-init.sh
+        bash rust-init.sh -y
+    fi
+    export HOME=${HOME:=~}
+    # shellcheck disable=SC1091
+    source "$HOME/.cargo/env"
 
-sudo apt-get update -y
-# vortex-duckdb's build.rs runs bindgen, which needs libclang plus the
-# clang freestanding headers (stdbool.h etc.); without libclang-dev the
-# build fails with `'stdbool.h' file not found`.
-sudo apt-get install -y gcc jq build-essential git clang libclang-dev
+    if [ "$(free -g | awk '/^Mem:/{print $2}')" -lt 12 ]; then
+        if [ "$(swapon --noheadings --show | wc -l)" -eq 0 ]; then
+            sudo fallocate -l 8G /swapfile
+            sudo chmod 600 /swapfile
+            sudo mkswap /swapfile
+            sudo swapon /swapfile
+        fi
+    fi
 
-if [ ! -d vortex ]; then
-    git clone https://github.com/spiraldb/vortex.git
-fi
-(
-    cd vortex
-    git fetch --tags
-    git checkout "$VORTEX_VERSION"
-    # See datafusion-vortex/install — submodule update isn't idempotent
-    # without sync + --force when a previous run left a partial clone.
-    git submodule sync --recursive
-    git submodule update --init --recursive --force
-    cargo build --release --bin query_bench --package bench-vortex
-)
+    sudo apt-get update -y
+    sudo apt-get install -y build-essential clang cmake git libclang-dev pkg-config
 
-# Build datafusion-cli so ./query can return actual rows instead of
-# the bench driver's JSON timing blob. See datafusion-vortex/install.
-if ! command -v datafusion-cli >/dev/null 2>&1; then
-    cargo install --locked --version 49.0.2 datafusion-cli
-    # Cargo installs into $HOME/.cargo/bin; the playground agent
-    # runs scripts with a stripped PATH, so symlink into /usr/local/bin.
-    sudo ln -sf "$HOME/.cargo/bin/datafusion-cli" /usr/local/bin/datafusion-cli
+    if [ ! -d vortex-datafusion-cli ]; then
+        git clone https://github.com/vortex-data/vortex-datafusion-cli.git
+    fi
+    cd vortex-datafusion-cli
+    git fetch --tags
+    git checkout "$CLI_TAG"
+    CARGO_PROFILE_RELEASE_LTO=true RUSTFLAGS="-C codegen-units=1" \
+        cargo build --release --bin vortex-datafusion-cli
 fi

datafusion-vortex-partitioned/load

@@ -1,9 +1,23 @@
 #!/bin/bash
 set -e
 
-# ./query uses datafusion-cli against the partitioned parquet files
-# under partitioned/. See datafusion-vortex/load for the rationale.
+DF=vortex-datafusion-cli/target/release/vortex-datafusion-cli
+
 mkdir -p partitioned
-../lib/download-hits-parquet-partitioned partitioned
+mv hits_*.parquet partitioned/ 2>/dev/null || true
+
+rm -rf vortex
+mkdir -p vortex
+
+seq 0 99 | xargs -P"$(nproc)" -I{} "$DF" -q \
+    -c "SET datafusion.execution.target_partitions = 1;" \
+    -c "CREATE EXTERNAL TABLE hits_parquet STORED AS PARQUET LOCATION 'partitioned/hits_{}.parquet' OPTIONS ('binary_as_string' 'true');" \
+    -c "COPY (SELECT * EXCEPT (\"EventDate\"), CAST(CAST(\"EventDate\" AS INTEGER) AS DATE) AS \"EventDate\" FROM hits_parquet) TO 'vortex/hits_{}.vortex' STORED AS VORTEX;"
+
+files=$(find vortex -maxdepth 1 -name 'hits_*.vortex' | wc -l)
+if [ "$files" -ne 100 ]; then
+    echo "Expected 100 Vortex files, found $files" >&2
+    exit 1
+fi
 
 sync

datafusion-vortex-partitioned/make-json.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# This script converts the raw `result.csv` data from `benchmark.sh` into the
+# final json format used by the benchmark dashboard.
+#
+# usage : ./make-json.sh <machine>
+#
+# example (save results/<YYYYMMDD>/c6a.4xlarge.json)
+#         ./make-json.sh c6a.4xlarge
+
+MACHINE=$1
+DATE=$(date -u +%Y-%m-%d)
+YYYYMMDD=${DATE//-/}
+mkdir -p "results/${YYYYMMDD}"
+OUTPUT_FILE="results/${YYYYMMDD}/${MACHINE}.json"
+SYSTEM_NAME="DataFusion (Vortex, partitioned)"
+LOAD_TIME=${LOAD_TIME:-null}
+DATA_SIZE=${DATA_SIZE:-$(./data-size 2>/dev/null || echo null)}
+DATA_SIZE=${DATA_SIZE:-null}
+
+# Read the CSV and build the result array using sed
+RESULT_ARRAY=$(awk -F, '{arr[$1]=arr[$1]","$3} END {for (i=1;i<=length(arr);i++) {gsub(/^,/, "", arr[i]); printf "        ["arr[i]"]"; if (i<length(arr)) printf ",\n"}}' result.csv)
+
+# form the final JSON structure from the template
+cat <<EOF > $OUTPUT_FILE
+{
+    "system": "$SYSTEM_NAME",
+    "date": "$DATE",
+    "machine": "$MACHINE",
+    "cluster_size": 1,
+    "proprietary": "no",
+    "tuned": "no",
+    "hardware": "cpu",
+    "tags": ["Rust","column-oriented","embedded","stateless"],
+    "load_time": $LOAD_TIME,
+    "data_size": $DATA_SIZE,
+    "result": [
+        $RESULT_ARRAY
+    ]
+}
+EOF

datafusion-vortex-partitioned/queries.sql

@@ -16,7 +16,7 @@ SELECT "SearchEngineID", "SearchPhrase", COUNT(*) AS c FROM hits WHERE "SearchPh
 SELECT "UserID", COUNT(*) FROM hits GROUP BY "UserID" ORDER BY COUNT(*) DESC LIMIT 10;
 SELECT "UserID", "SearchPhrase", COUNT(*) FROM hits GROUP BY "UserID", "SearchPhrase" ORDER BY COUNT(*) DESC LIMIT 10;
 SELECT "UserID", "SearchPhrase", COUNT(*) FROM hits GROUP BY "UserID", "SearchPhrase" LIMIT 10;
-SELECT "UserID", extract(minute FROM "EventTime") AS m, "SearchPhrase", COUNT(*) FROM hits GROUP BY "UserID", m, "SearchPhrase" ORDER BY COUNT(*) DESC LIMIT 10;
+SELECT "UserID", extract(minute FROM to_timestamp_seconds("EventTime")) AS m, "SearchPhrase", COUNT(*) FROM hits GROUP BY "UserID", m, "SearchPhrase" ORDER BY COUNT(*) DESC LIMIT 10;
 SELECT "UserID" FROM hits WHERE "UserID" = 435090932899640449;
 SELECT COUNT(*) FROM hits WHERE "URL" LIKE '%google%';
 SELECT "SearchPhrase", MIN("URL"), COUNT(*) AS c FROM hits WHERE "URL" LIKE '%google%' AND "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY c DESC LIMIT 10;
@@ -40,4 +40,4 @@ SELECT "URL", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventD
 SELECT "TraficSourceID", "SearchEngineID", "AdvEngineID", CASE WHEN ("SearchEngineID" = 0 AND "AdvEngineID" = 0) THEN "Referer" ELSE '' END AS Src, "URL" AS Dst, COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "IsRefresh" = 0 GROUP BY "TraficSourceID", "SearchEngineID", "AdvEngineID", Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;
 SELECT "URLHash", "EventDate", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "IsRefresh" = 0 AND "TraficSourceID" IN (-1, 6) AND "RefererHash" = 3594120000172545465 GROUP BY "URLHash", "EventDate" ORDER BY PageViews DESC LIMIT 10 OFFSET 100;
 SELECT "WindowClientWidth", "WindowClientHeight", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "IsRefresh" = 0 AND "DontCountHits" = 0 AND "URLHash" = 2868770270353813622 GROUP BY "WindowClientWidth", "WindowClientHeight" ORDER BY PageViews DESC LIMIT 10 OFFSET 10000;
-SELECT DATE_TRUNC('minute', "EventTime") AS M, COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-14' AND "EventDate" <= '2013-07-15' AND "IsRefresh" = 0 AND "DontCountHits" = 0 GROUP BY DATE_TRUNC('minute', "EventTime") ORDER BY DATE_TRUNC('minute', M) LIMIT 10 OFFSET 1000;
+SELECT DATE_TRUNC('minute', to_timestamp_seconds("EventTime")) AS M, COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-14' AND "EventDate" <= '2013-07-15' AND "IsRefresh" = 0 AND "DontCountHits" = 0 GROUP BY DATE_TRUNC('minute', to_timestamp_seconds("EventTime")) ORDER BY DATE_TRUNC('minute', M) LIMIT 10 OFFSET 1000;

datafusion-vortex-partitioned/query

@@ -1,23 +1,21 @@
 #!/bin/bash
-# Reads a SQL query from stdin, runs it via datafusion-cli against the
-# partitioned parquet files. See ../datafusion-vortex/query for the
-# rationale; the vortex bench binary is benchmark-only.
-# Stdout: query result.
-# Stderr: query runtime in fractional seconds on the last line.
+# Reads a SQL query from stdin and runs it via vortex-datafusion-cli.
+# Stdout: query result. Stderr: query runtime in fractional seconds.
 set -e
 
+DF=vortex-datafusion-cli/target/release/vortex-datafusion-cli
+
 query=$(cat)
-tmp=$(mktemp /tmp/datafusion.XXXXXX.sql)
+tmp=$(mktemp /tmp/datafusion-vortex.XXXXXX.sql)
 trap 'rm -f "$tmp"' EXIT
 printf '%s\n' "$query" > "$tmp"
 
-out=$(datafusion-cli -f create.sql "$tmp" 2>&1) && status=0 || status=$?
+out=$("$DF" -f create.sql -f "$tmp" 2>&1) && status=0 || status=$?
 
 if [ "$status" -ne 0 ]; then
     printf '%s\n' "$out" >&2
     exit "$status"
 fi
 
 printf '%s\n' "$out" | grep -v 'Elapsed' || true
-
 printf '%s\n' "$out" | awk '/Elapsed/ { e = $2 } END { print e }' >&2