update

Author: joseph-isaacs

duckdb-vortex/benchmark.sh

@@ -18,22 +18,38 @@ cd ../..
 # Load the data
 seq 0 99 | xargs -P100 -I{} bash -c 'wget --continue https://pub-3ba949c0f0354ac18db1f0f14f0a2c52.r2.dev/clickbench/parquet_many/hits_{}.parquet'
 
-# Convert parquet files to vortex
+# Convert parquet files to vortex partitioned
 seq 0 99 | xargs -P"$(nproc)" -I{} bash -c '
   if [ ! -f "hits_{}.vortex" ]; then
     duckdb -c "COPY 'hits_{}.parquet' TO hits_{}.vortex (FORMAT vortex)"
   fi
 '
 
-time duckdb hits.db -f create.sql
+# Convert parquet files to vortex single
+if [ ! -f "hits.vortex" ]; then
+  duckdb -c "COPY 'hits_*.parquet' TO hits.vortex (FORMAT vortex)"
+fi
+
+time duckdb hits-partitioned.db -f CREATE VIEW hits AS SELECT * FROM read_vortex('hits_*.vortex');
+
+time duckdb hits-single.db -f CREATE VIEW hits AS SELECT * FROM read_vortex('hits.vortex');
+
+
 
 # Run the queries
+echo 'partitioned'
 
-./run.sh 2>&1 | tee log.txt
+./run.sh 'hits-partitioned.db' 2>&1 | tee log-p.txt
+cat log-p.txt |
+  grep -P '^\d|Killed|Segmentation|^Run Time \(s\): real' |
+  sed -r -e 's/^.*(Killed|Segmentation).*$/null\nnull\nnull/; s/^Run Time \(s\): real\s*([0-9.]+).*$/\1/' |
+  awk '{ if (i % 3 == 0) { printf "[" }; printf $1; if (i % 3 != 2) { printf "," } else { print "]," }; ++i; }'
 
-wc -c hits.db
+echo ''
+echo 'single'
 
-cat log.txt |
+./run.sh 'hits-single.db' 2>&1 | tee log-s.txt
+cat log-s.txt |
   grep -P '^\d|Killed|Segmentation|^Run Time \(s\): real' |
   sed -r -e 's/^.*(Killed|Segmentation).*$/null\nnull\nnull/; s/^Run Time \(s\): real\s*([0-9.]+).*$/\1/' |
   awk '{ if (i % 3 == 0) { printf "[" }; printf $1; if (i % 3 != 2) { printf "," } else { print "]," }; ++i; }'

duckdb-vortex/create.sql

@@ -1,2 +1,2 @@
-CREATE VIEW hits AS SELECT * FROM read_vortex('hits_*.vortex');
+CREATE VIEW hits AS SELECT * FROM read_vortex('hits*.vortex');
 

duckdb-vortex/results/c6a.4xlarge.json

@@ -11,48 +11,48 @@
     "data_size": 16028685104,
 
     "result": [
-        [0.203,0.027,0.023],
-        [0.477,0.038,0.035],
-        [1.550,0.159,0.156],
-        [3.140,0.119,0.115],
-        [3.261,0.386,0.384],
-        [3.337,0.429,0.438],
-        [0.251,0.067,0.066],
-        [0.392,0.045,0.041],
-        [4.467,0.592,0.588],
-        [4.564,0.855,0.848],
-        [2.846,0.156,0.152],
-        [3.381,0.174,0.172],
-        [3.819,0.390,0.411],
-        [6.480,0.786,0.761],
-        [3.774,0.429,0.433],
-        [2.803,0.461,0.459],
-        [6.221,1.151,1.155],
-        [5.989,1.021,0.965],
-        [8.417,2.058,2.123],
-        [2.461,0.242,0.225],
-        [28.080,0.646,0.634],
-        [30.654,0.648,0.535],
-        [41.335,1.147,0.989],
-        [114.739,102.458,104.874],
-        [5.360,0.274,0.249],
-        [3.524,0.197,0.189],
-        [5.719,0.266,0.258],
-        [27.922,0.815,0.777],
-        [21.561,9.044,9.108],
-        [0.754,0.117,0.119],
-        [6.939,0.362,0.348],
-        [12.776,0.495,0.468],
-        [10.465,2.253,2.251],
-        [27.317,2.012,2.037],
-        [27.889,2.306,2.350],
-        [1.834,0.646,0.638],
-        [0.260,0.081,0.086],
-        [0.891,0.065,0.083],
-        [0.971,0.147,0.155],
-        [1.112,0.167,0.106],
-        [0.848,0.075,0.052],
-        [0.882,0.051,0.051],
-        [0.831,0.049,0.048]
+        [0.161,0.011,0.010],
+        [0.572,0.027,0.027],
+        [1.613,0.151,0.144],
+        [3.222,0.107,0.108],
+        [3.363,0.412,0.451],
+        [3.327,0.473,0.472],
+        [0.221,0.048,0.049],
+        [0.474,0.036,0.032],
+        [4.526,0.654,0.655],
+        [4.591,0.910,0.919],
+        [2.846,0.185,0.177],
+        [3.308,0.178,0.170],
+        [3.847,0.416,0.406],
+        [6.406,0.739,0.744],
+        [3.621,0.454,0.458],
+        [2.771,0.450,0.461],
+        [6.221,1.192,1.179],
+        [6.030,1.012,1.008],
+        [8.399,2.159,2.153],
+        [1.825,0.258,0.257],
+        [26.677,0.629,0.609],
+        [29.394,0.806,0.618],
+        [35.458,1.125,0.930],
+        [96.951,2.754,2.790],
+        [5.236,0.256,0.255],
+        [3.492,0.190,0.182],
+        [5.684,0.257,0.255],
+        [25.282,0.910,0.913],
+        [21.829,9.689,9.661],
+        [0.750,0.108,0.109],
+        [6.873,0.362,0.354],
+        [12.694,0.449,0.445],
+        [10.447,2.075,2.172],
+        [25.789,1.950,2.003],
+        [25.375,2.508,2.280],
+        [1.842,0.640,0.641],
+        [0.227,0.034,0.030],
+        [0.656,0.029,0.030],
+        [0.901,0.028,0.024],
+        [1.002,0.071,0.066],
+        [0.753,0.024,0.026],
+        [0.896,0.026,0.024],
+        [0.828,0.022,0.021]
     ]
 }

duckdb-vortex/run.sh

@@ -8,8 +8,6 @@ cat queries.sql | while read -r query; do
 
     echo "$query";
     cli_params=()
-#    cli_params+=("-c")
-#    cli_params+=("SET parquet_metadata_cache=true")
     cli_params+=("-c")
     cli_params+=(".timer on")
     for i in $(seq 1 $TRIES); do

index.html

@@ -439,7 +439,7 @@
 ,{"system":"DuckDB (memory)","date":"2024-11-26","machine":"c6a.metal, 500gb gp2","cluster_size":1,"tags":["C++","column-oriented","embedded"],"load_time":29.568019166999875,"data_size":102315057152,"result":[[28.37229389499771,0.01444745900153066,0.010460285000590375],[0.00818797199826804,0.018453592998412205,0.010097247002704535],[0.009875684001599438,0.014756544002011651,0.013079602998914197],[0.01480884600096033,0.018506574000639375,0.01780792100180406],[0.02132975700078532,0.8275158579999697,0.24229677399853244],[0.2527651769996737,0.19887691599797108,0.18186214999877848],[0.1886556860008568,0.01614768999934313,0.011127936999400845],[0.012829299001168692,0.017017687001498416,0.011138907000713516],[0.012957981001818553,0.24885185500170337,0.2766447909998533],[0.25002359700010857,0.2892966260005778,0.2779514459980419],[0.30130979000023217,0.08444572000007611,0.06657531800010474],[0.059446135001053335,0.07427780000216444,0.06103535399961402],[0.058512896997854114,0.2084252730019216,0.2086321480019251],[0.21721077699839952,0.46506098299869336,0.4530917009979021],[0.4697990730019228,0.22253320099844132,0.21058130000164965],[0.20870386100068572,0.3215249130007578,0.27229407300183084],[0.2499453230011568,0.3683013289992232,0.31991668999762624],[0.28497551399777876,0.3048137890000362,0.3115999949986872],[0.318867735000822,0.8751203950014315,0.7716744879980979],[0.7994624790007947,0.01659778099929099,0.01250473500113003],[0.012601426999026444,0.3383350500007509,0.09837473600055091],[0.09575407699958305,0.08805634300006204,0.10064015900206869],[0.0844710359997407,0.4643428749986924,0.19186990199887077],[0.19248208299904945,1.3182347119982296,0.5620290089973423],[0.5513104279998515,0.06911592700271285,0.05588764800268109],[0.062015663999773096,0.04412370800127974,0.049542190001375275],[0.04080415600037668,0.10599739000099362,0.1094067330013786],[0.10183726099785417,0.08797964200130082,0.08413135899900226],[0.08379066300039995,2.008289692999824,2.001949085999513],[1.925026742999762,0.0873897500023304,0.0680545590003021],[0.0829120550006337,0.18141677500170772,0.1736573039997893],[0.1532601569997496,0.18473046199869714,0.16197961100260727],[0.16216968500157236,0.6304102140020404,0.6036183989999699],[0.5863511949974054,0.5486020050011575,0.5516450720024295],[0.5755033479981648,0.5668365320016164,0.5793202010027017],[0.5245683149987599,0.24902728499728255,0.1521922360007011],[0.2654282980001881,0.07863214099779725,0.050029524998535635],[0.024858904998836806,0.06018337899877224,0.017208747998665785],[0.023967166998772882,0.04589027600013651,0.06055860699780169],[0.023993877999600954,0.08684039799845777,0.04880085199692985],[0.04033727000205545,0.013630619996547466,0.035837274001096375],[0.008728707001864677,0.011268315996858291,0.009220824998919852],[0.018453843000315828,0.014310172999103088,0.007255508000525879]],"source":"duckdb-memory/results/c6a.metal.json"}
 ,{"system":"DuckDB (Parquet, partitioned)","date":"2025-02-05","machine":"c6a.4xlarge, 500gb gp2","cluster_size":1,"comment":"","tags":["C++","column-oriented","embedded","stateless"],"load_time":0,"data_size":14737666736,"result":[[0.164,0.085,0.082],[0.119,0.060,0.061],[0.192,0.088,0.086],[0.359,0.081,0.079],[1.178,0.353,0.363],[0.828,0.460,0.457],[0.113,0.061,0.061],[0.127,0.063,0.060],[0.717,0.465,0.465],[1.059,0.596,0.578],[0.443,0.151,0.148],[1.016,0.185,0.182],[1.136,0.490,0.476],[2.430,0.919,0.880],[0.930,0.540,0.535],[0.564,0.420,0.420],[2.350,1.021,1.021],[2.122,0.791,0.789],[4.587,1.930,1.950],[0.251,0.074,0.073],[9.957,1.197,1.194],[10.999,0.684,0.668],[21.527,1.392,1.410],[33.496,5.369,5.336],[0.196,0.094,0.094],[0.975,0.248,0.253],[0.178,0.093,0.092],[9.984,0.896,0.889],[9.955,9.601,9.455],[0.160,0.073,0.074],[2.248,0.578,0.571],[5.922,0.682,0.676],[5.447,2.040,2.056],[9.888,2.372,2.386],[9.854,2.403,2.458],[0.746,0.587,0.590],[0.209,0.111,0.111],[0.155,0.088,0.087],[0.150,0.057,0.055],[0.394,0.219,0.217],[0.115,0.053,0.050],[0.116,0.056,0.055],[0.101,0.051,0.047]],"source":"duckdb-parquet/results/c6a.4xlarge.json"}
 ,{"system":"DuckDB (Parquet, partitioned)","date":"2025-02-05","machine":"c6a.metal, 500gb gp2","cluster_size":1,"comment":"","tags":["C++","column-oriented","embedded","stateless"],"load_time":0,"data_size":14737666736,"result":[[0.277,0.140,0.149],[0.129,0.072,0.068],[0.136,0.081,0.081],[0.329,0.080,0.065],[0.950,0.145,0.149],[0.829,0.200,0.199],[0.124,0.063,0.068],[0.149,0.092,0.078],[0.716,0.167,0.161],[1.077,0.188,0.177],[0.417,0.097,0.092],[0.807,0.103,0.100],[1.155,0.240,0.226],[2.313,0.407,0.340],[0.886,0.225,0.214],[0.492,0.172,0.186],[2.139,0.337,0.304],[2.103,0.282,0.281],[4.014,0.466,0.433],[0.177,0.070,0.060],[9.792,0.331,0.302],[10.963,0.216,0.191],[21.466,0.375,0.320],[48.898,1.545,1.231],[0.183,0.097,0.084],[0.890,0.110,0.094],[0.443,0.097,0.086],[9.775,0.274,0.228],[8.982,1.978,1.881],[0.157,0.092,0.078],[2.203,0.203,0.179],[5.725,0.218,0.214],[4.437,0.695,0.686],[9.814,0.960,0.713],[9.826,0.845,0.827],[0.313,0.170,0.167],[0.230,0.126,0.123],[0.192,0.107,0.101],[0.168,0.078,0.077],[0.349,0.214,0.213],[0.139,0.065,0.054],[0.130,0.078,0.091],[0.126,0.069,0.064]],"source":"duckdb-parquet/results/c6a.metal.json"}
-,{"system":"DuckDB (Vortex, partitioned)","date":"2025-05-20","machine":"c6a.4xlarge, 500gb gp2","cluster_size":1,"comment":"","tags":["rust","vortex","column-oriented","embedded","stateless"],"load_time":0,"data_size":16028685104,"result":[[0.203,0.027,0.023],[0.477,0.038,0.035],[1.550,0.159,0.156],[3.140,0.119,0.115],[3.261,0.386,0.384],[3.337,0.429,0.438],[0.251,0.067,0.066],[0.392,0.045,0.041],[4.467,0.592,0.588],[4.564,0.855,0.848],[2.846,0.156,0.152],[3.381,0.174,0.172],[3.819,0.390,0.411],[6.480,0.786,0.761],[3.774,0.429,0.433],[2.803,0.461,0.459],[6.221,1.151,1.155],[5.989,1.021,0.965],[8.417,2.058,2.123],[2.461,0.242,0.225],[28.080,0.646,0.634],[30.654,0.648,0.535],[41.335,1.147,0.989],[114.739,102.458,104.874],[5.360,0.274,0.249],[3.524,0.197,0.189],[5.719,0.266,0.258],[27.922,0.815,0.777],[21.561,9.044,9.108],[0.754,0.117,0.119],[6.939,0.362,0.348],[12.776,0.495,0.468],[10.465,2.253,2.251],[27.317,2.012,2.037],[27.889,2.306,2.350],[1.834,0.646,0.638],[0.260,0.081,0.086],[0.891,0.065,0.083],[0.971,0.147,0.155],[1.112,0.167,0.106],[0.848,0.075,0.052],[0.882,0.051,0.051],[0.831,0.049,0.048]],"source":"duckdb-vortex/results/c6a.4xlarge.json"}
+,{"system":"DuckDB (Vortex, partitioned)","date":"2025-05-20","machine":"c6a.4xlarge, 500gb gp2","cluster_size":1,"comment":"","tags":["rust","vortex","column-oriented","embedded","stateless"],"load_time":0,"data_size":16028685104,"result":[[0.161,0.011,0.010],[0.572,0.027,0.027],[1.613,0.151,0.144],[3.222,0.107,0.108],[3.363,0.412,0.451],[3.327,0.473,0.472],[0.221,0.048,0.049],[0.474,0.036,0.032],[4.526,0.654,0.655],[4.591,0.910,0.919],[2.846,0.185,0.177],[3.308,0.178,0.170],[3.847,0.416,0.406],[6.406,0.739,0.744],[3.621,0.454,0.458],[2.771,0.450,0.461],[6.221,1.192,1.179],[6.030,1.012,1.008],[8.399,2.159,2.153],[1.825,0.258,0.257],[26.677,0.629,0.609],[29.394,0.806,0.618],[35.458,1.125,0.930],[96.951,2.754,2.790],[5.236,0.256,0.255],[3.492,0.190,0.182],[5.684,0.257,0.255],[25.282,0.910,0.913],[21.829,9.689,9.661],[0.750,0.108,0.109],[6.873,0.362,0.354],[12.694,0.449,0.445],[10.447,2.075,2.172],[25.789,1.950,2.003],[25.375,2.508,2.280],[1.842,0.640,0.641],[0.227,0.034,0.030],[0.656,0.029,0.030],[0.901,0.028,0.024],[1.002,0.071,0.066],[0.753,0.024,0.026],[0.896,0.026,0.024],[0.828,0.022,0.021]],"source":"duckdb-vortex/results/c6a.4xlarge.json"}
 ,{"system":"DuckDB (Vortex, partitioned)","date":"2025-05-20","machine":"c6a.metal, 500gb gp2","cluster_size":1,"comment":"","tags":["rust","vortex","column-oriented","embedded","stateless"],"load_time":0,"data_size":16028685104,"result":[[0.173,0.025,0.025],[0.446,0.074,0.066],[1.410,0.192,0.193],[3.040,0.138,0.094],[3.125,0.233,0.243],[3.399,0.269,0.272],[0.242,0.130,0.128],[0.297,0.088,0.085],[4.286,0.391,0.367],[4.595,0.509,0.514],[2.896,0.154,0.139],[3.211,0.160,0.147],[3.633,0.230,0.238],[6.317,0.372,0.363],[3.718,0.251,0.228],[2.800,0.260,0.263],[6.121,0.477,0.484],[6.152,0.498,0.510],[8.184,0.749,0.726],[2.447,0.221,0.216],[29.004,0.334,0.301],[31.965,0.354,0.342],[43.794,0.523,0.446],[113.575,1.920,1.839],[5.334,0.124,0.085],[3.431,0.150,0.121],[5.527,0.178,0.147],[28.774,0.363,0.325],[23.202,1.569,1.582],[0.762,0.150,0.146],[6.704,0.252,0.208],[12.707,0.270,0.390],[9.594,0.884,0.907],[27.720,0.741,0.696],[27.784,0.749,0.769],[1.775,0.362,0.350],[0.398,0.145,0.160],[0.751,0.133,0.125],[1.108,0.147,0.142],[1.157,0.183,0.198],[0.825,0.098,0.099],[0.719,0.104,0.095],[0.633,0.081,0.082]],"source":"duckdb-vortex/results/c6a.metal.json"}
 ,{"system":"DuckDB","date":"2025-02-05","machine":"c6a.4xlarge, 500gb gp2","cluster_size":1,"tags":["C++","column-oriented","embedded"],"load_time":372.30,"data_size":23651168256,"result":[[0.035,0.001,0.000],[0.193,0.003,0.004],[1.080,0.018,0.017],[1.306,0.028,0.027],[1.303,0.301,0.302],[1.822,0.373,0.371],[0.140,0.009,0.009],[0.251,0.005,0.004],[1.995,0.362,0.364],[2.691,0.491,0.488],[1.622,0.081,0.080],[2.402,0.091,0.089],[2.148,0.348,0.347],[3.361,0.673,0.609],[2.063,0.356,0.352],[0.594,0.365,0.370],[3.252,0.824,0.819],[3.014,0.585,0.574],[6.040,1.556,1.574],[0.238,0.004,0.003],[18.681,0.671,0.666],[20.551,0.420,0.416],[34.795,0.637,0.628],[8.226,0.303,0.290],[0.187,0.025,0.030],[0.336,0.046,0.023],[0.724,0.022,0.022],[19.279,0.366,0.359],[13.702,9.139,9.062],[0.284,0.022,0.021],[5.209,0.311,0.300],[7.853,0.357,0.358],[6.336,1.732,1.749],[18.440,1.848,1.865],[18.459,1.931,1.953],[0.679,0.434,0.431],[0.176,0.029,0.031],[0.076,0.009,0.008],[0.102,0.015,0.016],[0.166,0.060,0.057],[0.121,0.004,0.004],[0.118,0.005,0.005],[0.116,0.006,0.006]],"source":"duckdb/results/c6a.4xlarge.json"}
 ,{"system":"DuckDB","date":"2025-02-05","machine":"c6a.metal, 500gb gp2","cluster_size":1,"tags":["C++","column-oriented","embedded"],"load_time":123.60,"data_size":23698354176,"result":[[0.040,0.000,0.001],[0.135,0.007,0.006],[2.057,0.029,0.019],[1.602,0.012,0.018],[1.761,0.169,0.158],[2.767,0.152,0.141],[0.096,0.024,0.016],[0.490,0.009,0.009],[2.799,0.190,0.183],[4.341,0.235,0.212],[2.758,0.065,0.069],[3.259,0.061,0.068],[3.131,0.164,0.144],[4.930,0.348,0.371],[3.221,0.168,0.148],[1.193,0.278,0.190],[4.721,0.368,0.336],[4.722,0.351,0.345],[8.144,0.425,0.401],[0.345,0.005,0.004],[23.726,0.348,0.121],[26.195,0.321,0.096],[41.895,0.464,0.158],[13.209,0.102,0.096],[1.076,0.015,0.013],[1.596,0.223,0.080],[1.331,0.058,0.020],[24.136,0.244,0.105],[18.221,1.461,1.325],[0.386,0.036,0.037],[7.530,0.147,0.135],[10.881,0.274,0.289],[7.814,0.778,0.618],[23.468,0.786,0.604],[23.248,0.571,0.554],[0.804,0.288,0.253],[0.137,0.027,0.027],[0.095,0.009,0.009],[0.121,0.018,0.016],[0.203,0.052,0.050],[0.090,0.006,0.005],[0.080,0.007,0.007],[0.071,0.007,0.006]],"source":"duckdb/results/c6a.metal.json"}