Skip to content

Commit 001f44b

Browse files
authored
Don't re-run benchmarks nightly (#7338)
We currently run some of the same benchmarks both nightly and per-commit, which pollutes the data file, which causes [failures](https://github.com/vortex-data/vortex/actions/runs/24129409553/job/70402005592) when comparing runs like in and is also just wasteful. ``` File "/home/runner/_work/vortex/vortex/scripts/compare-benchmark-jsons.py", line 629, in <module> main() File "/home/runner/_work/vortex/vortex/scripts/compare-benchmark-jsons.py", line 516, in main statistical_analysis = build_statistical_analysis(df3, threshold_pct) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/runner/_work/vortex/vortex/scripts/compare-benchmark-jsons.py", line 300, in build_statistical_analysis log_ratio_table = detail_df.pivot(index="query", columns="combo", values="log_ratio") ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/runner/.cache/uv/environments-v2/compare-benchmark-jsons-6c3240f590b6fe2d/lib/python3.11/site-packages/pandas/core/frame.py", line 10974, in pivot return pivot(self, index=index, columns=columns, values=values) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/runner/.cache/uv/environments-v2/compare-benchmark-jsons-6c3240f590b6fe2d/lib/python3.11/site-packages/pandas/core/reshape/pivot.py", line 910, in pivot result = cast("DataFrame", indexed.unstack(columns_listlike)) # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/runner/.cache/uv/environments-v2/compare-benchmark-jsons-6c3240f590b6fe2d/lib/python3.11/site-packages/pandas/core/series.py", line 4501, in unstack return unstack(self, level, fill_value, sort) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/runner/.cache/uv/environments-v2/compare-benchmark-jsons-6c3240f590b6fe2d/lib/python3.11/site-packages/pandas/core/reshape/reshape.py", line 596, in unstack unstacker = _Unstacker( ^^^^^^^^^^^ File "/home/runner/.cache/uv/environments-v2/compare-benchmark-jsons-6c3240f590b6fe2d/lib/python3.11/site-packages/pandas/core/reshape/reshape.py", line 176, in __init__ self._make_selectors() File "/home/runner/.cache/uv/environments-v2/compare-benchmark-jsons-6c3240f590b6fe2d/lib/python3.11/site-packages/pandas/core/reshape/reshape.py", line 232, in _make_selectors raise ValueError("Index contains duplicate entries, cannot reshape") ValueError: Index contains duplicate entries, cannot reshape ``` Signed-off-by: Adam Gutglick <adam@spiraldb.com>
1 parent 95befa7 commit 001f44b

File tree

1 file changed

+0
-28
lines changed

1 file changed

+0
-28
lines changed

.github/workflows/nightly-bench.yml

Lines changed: 0 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -23,33 +23,8 @@ jobs:
2323
with:
2424
mode: "develop"
2525
machine_type: ${{ matrix.machine_type.instance_name }}
26-
# datafusion:vortex uses a lot of memory
2726
benchmark_matrix: |
2827
[
29-
{
30-
"id": "clickbench-nvme",
31-
"subcommand": "clickbench",
32-
"name": "Clickbench on NVME",
33-
"targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb",
34-
"build_lance": true
35-
},
36-
{
37-
"id": "tpch-nvme",
38-
"subcommand": "tpch",
39-
"name": "TPC-H on NVME",
40-
"targets": "datafusion:parquet,datafusion:vortex,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:duckdb",
41-
"scale_factor": "10.0",
42-
"build_lance": true
43-
},
44-
{
45-
"id": "tpch-s3",
46-
"subcommand": "tpch",
47-
"name": "TPC-H on S3",
48-
"local_dir": "vortex-bench/data/tpch/10.0",
49-
"remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/",
50-
"targets": "datafusion:parquet,datafusion:vortex,duckdb:parquet,duckdb:vortex",
51-
"scale_factor": "10.0"
52-
},
5328
{
5429
"id": "tpch-nvme",
5530
"subcommand": "tpch",
@@ -74,6 +49,3 @@ jobs:
7449
machine_type:
7550
- id: x86
7651
instance_name: c6id.8xlarge
77-
# TODO(joe): support other arch
78-
# - id: arm64
79-
# instance_name: c6gd.8xlarge

0 commit comments

Comments
 (0)