From ed1ef3ed07bc3b90f2a00e2d87e09bc949d52ea9 Mon Sep 17 00:00:00 2001 From: Didi Hoffmann Date: Fri, 17 Jul 2026 14:50:39 +0200 Subject: [PATCH 1/3] Add Db2/MSSQL/Oracle engines and TPC-H, Wikipedia, YCSB, CH-benCHmark benchmarks Restructures the repo into two trees: benchmarks/ holds the GMT usage scenarios (one folder per benchmark), db/ holds the per-engine driver scripts (one folder per engine). Replaces the three root usage_scenario_*.yml files. Engines: adds IBM Db2, Microsoft SQL Server and Oracle Database Free alongside the existing Postgres/MariaDB/MySQL. Db2 needs a one-time image build (db/db2/build-image.sh) because HammerDB's db2tcl binding needs the full Db2 client. Benchmarks: adds TPC-H (HammerDB TPROC-H) plus Wikipedia, YCSB and CH-benCHmark driven by a new BenchBase container. Every engine gets an equal 4-CPU / 8-GB budget in compose.yml so the energy comparison is fair. Tooling: - run_on_cluster.py discovers every scenario and submits it to the GMT cluster, refusing to run on an unclean or unpushed tree. - check_repo.py enforces the fairness invariants: the per-benchmark compose.yml copies match the root, DB resource budgets are equal, and the driver scripts use the same knobs across engines. All scenarios here are stock/default engine configuration; tuning follows in a separate PR. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 100 ++++- benchmarks/benchbase/Dockerfile | 85 ++++ benchmarks/benchbase/build-image.sh | 45 +++ benchmarks/chbenchmark/compose.yml | 131 ++++++ benchmarks/chbenchmark/maria.yml | 50 +++ benchmarks/chbenchmark/mssql.yml | 50 +++ benchmarks/chbenchmark/mysql.yml | 50 +++ benchmarks/chbenchmark/oracle.yml | 50 +++ benchmarks/chbenchmark/pg.yml | 50 +++ benchmarks/tpcc/compose.yml | 131 ++++++ benchmarks/tpcc/db2.yml | 79 ++++ benchmarks/tpcc/maria.yml | 52 +++ benchmarks/tpcc/mssql.yml | 51 +++ benchmarks/tpcc/mysql.yml | 52 +++ benchmarks/tpcc/oracle.yml | 56 +++ benchmarks/tpcc/pg.yml | 52 +++ benchmarks/tpch/compose.yml | 131 ++++++ benchmarks/tpch/db2.yml | 84 ++++ benchmarks/tpch/maria.yml | 51 +++ benchmarks/tpch/mssql.yml | 51 +++ benchmarks/tpch/mysql.yml | 51 +++ benchmarks/tpch/oracle.yml | 55 +++ benchmarks/tpch/pg.yml | 51 +++ benchmarks/wikipedia/compose.yml | 131 ++++++ benchmarks/wikipedia/maria.yml | 50 +++ benchmarks/wikipedia/mssql.yml | 50 +++ benchmarks/wikipedia/mysql.yml | 50 +++ benchmarks/wikipedia/oracle.yml | 50 +++ benchmarks/wikipedia/pg.yml | 50 +++ benchmarks/ycsb/compose.yml | 131 ++++++ benchmarks/ycsb/maria.yml | 50 +++ benchmarks/ycsb/mssql.yml | 50 +++ benchmarks/ycsb/mysql.yml | 50 +++ benchmarks/ycsb/oracle.yml | 50 +++ benchmarks/ycsb/pg.yml | 50 +++ check_repo.py | 330 ++++++++++++++++ compose.yml | 111 +++++- db/db2/Dockerfile.hammerdb-db2 | 47 +++ db/db2/build-image.sh | 65 +++ db/db2/tpcc/db2_tprocc_buildschema.tcl | 29 ++ db/db2/tpcc/db2_tprocc_deleteschema.tcl | 18 + db/db2/tpcc/db2_tprocc_run.tcl | 34 ++ db/db2/tpch/db2_tproch_buildschema.tcl | 24 ++ db/db2/tpch/db2_tproch_deleteschema.tcl | 18 + db/db2/tpch/db2_tproch_run.tcl | 26 ++ .../chbenchmark/maria_chbenchmark_config.xml | 125 ++++++ .../maria/tpcc}/maria_tprocc_buildschema.tcl | 2 +- .../maria/tpcc}/maria_tprocc_deleteschema.tcl | 0 {maria => db/maria/tpcc}/maria_tprocc_run.tcl | 2 +- db/maria/tpch/maria_tproch_buildschema.tcl | 20 + db/maria/tpch/maria_tproch_deleteschema.tcl | 17 + db/maria/tpch/maria_tproch_run.tcl | 27 ++ db/maria/wikipedia/maria_wikipedia_config.xml | 50 +++ db/maria/ycsb/maria_ycsb_config.xml | 53 +++ .../chbenchmark/mssql_chbenchmark_config.xml | 125 ++++++ db/mssql/tpcc/mssql_tprocc_buildschema.tcl | 36 ++ db/mssql/tpcc/mssql_tprocc_deleteschema.tcl | 25 ++ db/mssql/tpcc/mssql_tprocc_run.tcl | 42 ++ db/mssql/tpch/mssql_tproch_buildschema.tcl | 31 ++ db/mssql/tpch/mssql_tproch_deleteschema.tcl | 25 ++ db/mssql/tpch/mssql_tproch_run.tcl | 35 ++ db/mssql/wikipedia/mssql_wikipedia_config.xml | 50 +++ db/mssql/ycsb/mssql_ycsb_config.xml | 53 +++ .../chbenchmark/mysql_chbenchmark_config.xml | 125 ++++++ .../mysql/tpcc}/mysql_tprocc_buildschema.tcl | 2 +- .../mysql/tpcc}/mysql_tprocc_deleteschema.tcl | 0 {mysql => db/mysql/tpcc}/mysql_tprocc_run.tcl | 2 +- db/mysql/tpch/mysql_tproch_buildschema.tcl | 20 + db/mysql/tpch/mysql_tproch_deleteschema.tcl | 17 + db/mysql/tpch/mysql_tproch_run.tcl | 27 ++ db/mysql/wikipedia/mysql_wikipedia_config.xml | 50 +++ db/mysql/ycsb/mysql_ycsb_config.xml | 53 +++ .../chbenchmark/oracle_chbenchmark_config.xml | 125 ++++++ db/oracle/tpcc/oracle_tprocc_buildschema.tcl | 29 ++ db/oracle/tpcc/oracle_tprocc_deleteschema.tcl | 18 + db/oracle/tpcc/oracle_tprocc_run.tcl | 34 ++ db/oracle/tpch/oracle_tproch_buildschema.tcl | 21 + db/oracle/tpch/oracle_tproch_deleteschema.tcl | 19 + db/oracle/tpch/oracle_tproch_run.tcl | 29 ++ .../wikipedia/oracle_wikipedia_config.xml | 50 +++ db/oracle/ycsb/oracle_ycsb_config.xml | 53 +++ db/pg/chbenchmark/pg_chbenchmark_config.xml | 125 ++++++ {pg => db/pg/tpcc}/pg_tprocc_buildschema.tcl | 2 +- {pg => db/pg/tpcc}/pg_tprocc_deleteschema.tcl | 0 {pg => db/pg/tpcc}/pg_tprocc_run.tcl | 4 +- db/pg/tpch/pg_tproch_buildschema.tcl | 24 ++ db/pg/tpch/pg_tproch_deleteschema.tcl | 21 + db/pg/tpch/pg_tproch_run.tcl | 28 ++ db/pg/wikipedia/pg_wikipedia_config.xml | 50 +++ db/pg/ycsb/pg_ycsb_config.xml | 53 +++ pg/benchmark_tests.tcl | 36 -- run_on_cluster.py | 372 ++++++++++++++++++ usage_scenario_maria.yml | 45 --- usage_scenario_mysql.yml | 45 --- usage_scenario_pg.yml | 45 --- 95 files changed, 5073 insertions(+), 191 deletions(-) create mode 100644 benchmarks/benchbase/Dockerfile create mode 100755 benchmarks/benchbase/build-image.sh create mode 100644 benchmarks/chbenchmark/compose.yml create mode 100644 benchmarks/chbenchmark/maria.yml create mode 100644 benchmarks/chbenchmark/mssql.yml create mode 100644 benchmarks/chbenchmark/mysql.yml create mode 100644 benchmarks/chbenchmark/oracle.yml create mode 100644 benchmarks/chbenchmark/pg.yml create mode 100644 benchmarks/tpcc/compose.yml create mode 100644 benchmarks/tpcc/db2.yml create mode 100644 benchmarks/tpcc/maria.yml create mode 100644 benchmarks/tpcc/mssql.yml create mode 100644 benchmarks/tpcc/mysql.yml create mode 100644 benchmarks/tpcc/oracle.yml create mode 100644 benchmarks/tpcc/pg.yml create mode 100644 benchmarks/tpch/compose.yml create mode 100644 benchmarks/tpch/db2.yml create mode 100644 benchmarks/tpch/maria.yml create mode 100644 benchmarks/tpch/mssql.yml create mode 100644 benchmarks/tpch/mysql.yml create mode 100644 benchmarks/tpch/oracle.yml create mode 100644 benchmarks/tpch/pg.yml create mode 100644 benchmarks/wikipedia/compose.yml create mode 100644 benchmarks/wikipedia/maria.yml create mode 100644 benchmarks/wikipedia/mssql.yml create mode 100644 benchmarks/wikipedia/mysql.yml create mode 100644 benchmarks/wikipedia/oracle.yml create mode 100644 benchmarks/wikipedia/pg.yml create mode 100644 benchmarks/ycsb/compose.yml create mode 100644 benchmarks/ycsb/maria.yml create mode 100644 benchmarks/ycsb/mssql.yml create mode 100644 benchmarks/ycsb/mysql.yml create mode 100644 benchmarks/ycsb/oracle.yml create mode 100644 benchmarks/ycsb/pg.yml create mode 100755 check_repo.py create mode 100644 db/db2/Dockerfile.hammerdb-db2 create mode 100755 db/db2/build-image.sh create mode 100644 db/db2/tpcc/db2_tprocc_buildschema.tcl create mode 100644 db/db2/tpcc/db2_tprocc_deleteschema.tcl create mode 100644 db/db2/tpcc/db2_tprocc_run.tcl create mode 100644 db/db2/tpch/db2_tproch_buildschema.tcl create mode 100644 db/db2/tpch/db2_tproch_deleteschema.tcl create mode 100644 db/db2/tpch/db2_tproch_run.tcl create mode 100644 db/maria/chbenchmark/maria_chbenchmark_config.xml rename {maria => db/maria/tpcc}/maria_tprocc_buildschema.tcl (96%) rename {maria => db/maria/tpcc}/maria_tprocc_deleteschema.tcl (100%) rename {maria => db/maria/tpcc}/maria_tprocc_run.tcl (97%) create mode 100644 db/maria/tpch/maria_tproch_buildschema.tcl create mode 100644 db/maria/tpch/maria_tproch_deleteschema.tcl create mode 100644 db/maria/tpch/maria_tproch_run.tcl create mode 100644 db/maria/wikipedia/maria_wikipedia_config.xml create mode 100644 db/maria/ycsb/maria_ycsb_config.xml create mode 100644 db/mssql/chbenchmark/mssql_chbenchmark_config.xml create mode 100644 db/mssql/tpcc/mssql_tprocc_buildschema.tcl create mode 100644 db/mssql/tpcc/mssql_tprocc_deleteschema.tcl create mode 100644 db/mssql/tpcc/mssql_tprocc_run.tcl create mode 100644 db/mssql/tpch/mssql_tproch_buildschema.tcl create mode 100644 db/mssql/tpch/mssql_tproch_deleteschema.tcl create mode 100644 db/mssql/tpch/mssql_tproch_run.tcl create mode 100644 db/mssql/wikipedia/mssql_wikipedia_config.xml create mode 100644 db/mssql/ycsb/mssql_ycsb_config.xml create mode 100644 db/mysql/chbenchmark/mysql_chbenchmark_config.xml rename {mysql => db/mysql/tpcc}/mysql_tprocc_buildschema.tcl (96%) rename {mysql => db/mysql/tpcc}/mysql_tprocc_deleteschema.tcl (100%) rename {mysql => db/mysql/tpcc}/mysql_tprocc_run.tcl (97%) create mode 100644 db/mysql/tpch/mysql_tproch_buildschema.tcl create mode 100644 db/mysql/tpch/mysql_tproch_deleteschema.tcl create mode 100644 db/mysql/tpch/mysql_tproch_run.tcl create mode 100644 db/mysql/wikipedia/mysql_wikipedia_config.xml create mode 100644 db/mysql/ycsb/mysql_ycsb_config.xml create mode 100644 db/oracle/chbenchmark/oracle_chbenchmark_config.xml create mode 100644 db/oracle/tpcc/oracle_tprocc_buildschema.tcl create mode 100644 db/oracle/tpcc/oracle_tprocc_deleteschema.tcl create mode 100644 db/oracle/tpcc/oracle_tprocc_run.tcl create mode 100644 db/oracle/tpch/oracle_tproch_buildschema.tcl create mode 100644 db/oracle/tpch/oracle_tproch_deleteschema.tcl create mode 100644 db/oracle/tpch/oracle_tproch_run.tcl create mode 100644 db/oracle/wikipedia/oracle_wikipedia_config.xml create mode 100644 db/oracle/ycsb/oracle_ycsb_config.xml create mode 100644 db/pg/chbenchmark/pg_chbenchmark_config.xml rename {pg => db/pg/tpcc}/pg_tprocc_buildschema.tcl (96%) rename {pg => db/pg/tpcc}/pg_tprocc_deleteschema.tcl (100%) rename {pg => db/pg/tpcc}/pg_tprocc_run.tcl (93%) create mode 100644 db/pg/tpch/pg_tproch_buildschema.tcl create mode 100644 db/pg/tpch/pg_tproch_deleteschema.tcl create mode 100644 db/pg/tpch/pg_tproch_run.tcl create mode 100644 db/pg/wikipedia/pg_wikipedia_config.xml create mode 100644 db/pg/ycsb/pg_ycsb_config.xml delete mode 100644 pg/benchmark_tests.tcl create mode 100755 run_on_cluster.py delete mode 100644 usage_scenario_maria.yml delete mode 100644 usage_scenario_mysql.yml delete mode 100644 usage_scenario_pg.yml diff --git a/README.md b/README.md index 052ce15..69ac731 100644 --- a/README.md +++ b/README.md @@ -2,19 +2,106 @@ A tool that benchmarks various DBMS systems agains each other and looks at energy usage The `compose.yml` defines all the containers we want to use to benchmark. We use the std containers provided by the -various DBMS and the [hammerdb](https://www.hammerdb.com/) container. For benchmarking we use the -hammerDB system. +various DBMS plus a load-driver container per benchmark family: + +- [HammerDB](https://www.hammerdb.com/) drives the TPC-C and TPC-H benchmarks. +- [BenchBase](https://github.com/cmu-db/benchbase) (CMU; the maintained OLTP-Bench successor) drives the Wikipedia, + YCSB and CH-benCHmark benchmarks. One image bakes in a distribution per engine, so the same driver runs every + database in a benchmark — see [BenchBase setup](#benchbase-setup). Currently we support: +- IBM Db2 (needs a one-time image build — see [Db2 setup](#db2-setup)) - MariaDB +- Microsoft SQL Server - MySql +- Oracle Database Free - PostgreSQL -For each DB vendor we have a benchmarking `usage_scenario_xxx.yml` that you can execute with the Green Metrics Tool to -get energy readings. +The repo is split into two trees: `benchmarks/` holds the GMT usage scenarios (one folder per benchmark), and `db/` +holds the per-engine driver scripts (one folder per database engine). + +For each DB vendor and benchmark we have a `benchmarks//.yml` (e.g. `benchmarks/tpcc/pg.yml`, +`benchmarks/tpch/oracle.yml`) that you can execute with the Green Metrics Tool to get energy readings. The driver's +per-DB scripts live under `db///`: HammerDB TCL scripts for TPC-C/TPC-H (e.g. `db/pg/tpcc/`), BenchBase +config XMLs for Wikipedia/YCSB/CH-benCHmark (e.g. `db/pg/wikipedia/pg_wikipedia_config.xml`). + +Each benchmark directory keeps its **own copy of `compose.yml`** (`benchmarks/tpcc/compose.yml`, +`benchmarks/tpch/compose.yml`, …): GMT's `!include` only resolves files inside the scenario's own directory, so +`!include ../compose.yml` is rejected. Treat the root `compose.yml` as the source of truth and re-sync the copies +whenever it changes: + +```sh +for d in tpcc tpch wikipedia ycsb chbenchmark; do cp compose.yml "benchmarks/$d/compose.yml"; done +``` + +We currently run five benchmarks: + +- [TPC-C](https://en.wikipedia.org/wiki/TPC-C) (HammerDB TPROC-C) — write-heavy OLTP; recorded metric is NOPM +- [TPC-H](https://en.wikipedia.org/wiki/TPC-H) (HammerDB TPROC-H) — analytical query workload; the SCI functional unit + is the number of queries completed (i.e. carbon per query) +- [Wikipedia](https://github.com/cmu-db/benchbase) (BenchBase) — read-mostly web-serving workload over real Wikipedia + article traces; the SCI functional unit is the number of requests completed +- [YCSB](https://github.com/cmu-db/benchbase) (BenchBase) — cloud key-value serving (point reads/writes/scans); the + SCI functional unit is the number of requests completed +- [CH-benCHmark](https://github.com/cmu-db/benchbase) (BenchBase, composite `tpcc,chbenchmark`) — HTAP: + TPC-C transactions and 22 TPC-H-style analytical queries run *concurrently* on one schema; the SCI functional unit is + the number of requests completed (OLTP txns + OLAP queries — see the per-transaction CSVs to split them) + +The three BenchBase benchmarks cover **5** engines — there is no Db2 BenchBase profile, so Db2 has TPC-C/TPC-H only. + +## Db2 setup + +Db2 needs a one-time prep the other engines don't. The `tpcorg/hammerdb` image has no Db2 client, and +HammerDB's `db2tcl` binding needs the *full* Db2 client (the free CLI driver lacks the `sqlefrce_api` +symbol). HammerDB also talks to Db2 through a *catalogued* database, so the client needs a real Db2 +instance. `db/db2/build-image.sh` handles this: it lifts the Db2 install + an initialised client instance out +of the `icr.io/db2_community/db2` image and bakes them onto the hammerdb (Ubuntu) base, then pushes the +result to Docker Hub as `ribalba/hammerdb-db2`, which the Db2 usage scenarios reference. + +Before running the Db2 scenarios (one-time): + +```sh +docker login # as the account that owns the image (ribalba) +./db/db2/build-image.sh # builds and pushes ribalba/hammerdb-db2:latest +``` + +Notes: + +- The Db2 server container requires `privileged: true`, and the host must allow it. +- Db2 is slow to start (minutes) and loads via INSERTs, so it builds slower than the bulk-loaded engines. + +## BenchBase setup + +The Wikipedia, YCSB and CH-benCHmark scenarios are driven by a `benchbase` container. BenchBase builds one +self-contained distribution per database (each bundles that engine's JDBC driver), so `benchmarks/benchbase/build-image.sh` builds +every engine's profile into a single image and pushes it to Docker Hub as `ribalba/benchbase:latest`, which the +scenarios pull. Each profile lands in `/benchbase/benchbase-/` and the scenario `cd`s into the right one. + +Before running the Wikipedia/YCSB/CH-benCHmark scenarios (one-time): + +```sh +docker login # as the account that owns the image (ribalba) +./benchmarks/benchbase/build-image.sh # builds + pushes ribalba/benchbase:latest +BENCHBASE_REF= ./benchmarks/benchbase/build-image.sh # pin a commit for a reproducible paper build +``` + +Notes: -We are currently only using the [TPC-C benchmark](https://en.wikipedia.org/wiki/TPC-C) +- Each scenario runs three flow steps: create the `benchbase` database/user on the DB container, then BenchBase + `--create --load` (schema + data), then `--execute` (the measured run). The functional-unit metric + (`wikipedia_requests` / `ycsb_requests` / `chbenchmark_requests`) is parsed from BenchBase's `measuredRequests` on + the execute step. +- CH-benCHmark is the composite `-b tpcc,chbenchmark` with a single config: the `tpcc` plugin supplies the OLTP schema + and transactions, the `chbenchmark` plugin adds the TPC-H tables and the 22 analytical queries. OLTP and OLAP run in + *separate terminal pools* (`` vs ``) — that split is the HTAP + knob. The execute step reports one combined `measuredRequests`; for the OLTP-vs-OLAP breakdown read BenchBase's + per-transaction `results/*.results..csv` files (TPC-C's five vs `Q1`…`Q22`). +- The workload (scalefactor, terminals, isolation, weights) is fixed in `db///__config.xml` + so only the *database* configuration is the variable when comparing default vs. tuned. Defaults: Wikipedia + `scalefactor=10`, YCSB `scalefactor=1000` (≈1 GB), CH-benCHmark `scalefactor=10` (TPC-C warehouses, ≈1 GB) — all + deliberately larger than default buffer pools so tuning shows. +- BenchBase requires Java 23 to build; the image handles this. It has no Db2 profile. ## Some background reading @@ -22,3 +109,6 @@ We are currently only using the [TPC-C benchmark](https://en.wikipedia.org/wiki/ - https://www.hammerdb.com/blog/uncategorized/how-to-deploy-hammerdb-cli-fast-with-docker/ - https://www.tpc.org/tpcc/default5.asp - https://www.hammerdb.com/blog/uncategorized/how-to-run-a-fixed-throughput-workloads/ +- https://github.com/cmu-db/benchbase (BenchBase) +- OLTP-Bench paper (BenchBase lineage): https://www.vldb.org/pvldb/vol7/p277-difallah.pdf +- CH-benCHmark paper (Cole et al., DBTest 2011): https://doi.org/10.1145/1988842.1988850 diff --git a/benchmarks/benchbase/Dockerfile b/benchmarks/benchbase/Dockerfile new file mode 100644 index 0000000..5e145ed --- /dev/null +++ b/benchmarks/benchbase/Dockerfile @@ -0,0 +1,85 @@ +# BenchBase load-driver image for the Wikipedia and YCSB scenarios. +# +# BenchBase builds one self-contained distribution per database profile (each +# bundles that engine's JDBC driver), so we build every profile we need and copy +# them all into one runtime image. A scenario then runs: +# cd /benchbase/benchbase- && java -jar benchbase.jar -b -c ... +# +# Build via ./benchmarks/benchbase/build-image.sh (sets the build args below). +# syntax=docker/dockerfile:1 + +# --- build stage: compile each requested profile from source --------------- +# BenchBase's pom.xml pins Java 23 (maven.compiler.release=23). +FROM eclipse-temurin:23-jdk AS build + +# Moving default; pin a commit SHA via build-image.sh for a reproducible build. +ARG BENCHBASE_REF=main +# Space-separated BenchBase Maven profiles to bake in. +ARG BENCHBASE_PROFILES="postgres mysql mariadb sqlserver oracle" + +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /src +RUN git clone https://github.com/cmu-db/benchbase.git . \ + && git checkout "${BENCHBASE_REF}" + +# Patch an upstream Wikipedia data-generator bug: UpdatePage truncates the +# generated revComment to 254 chars before writing revision.rev_comment, but +# writes the *untruncated* value into recentchanges.rc_comment (also +# varchar(255)). On any strict engine (PostgreSQL, and MySQL/MariaDB/Oracle/ +# SQL Server in their default strict modes) the occasional >255-char comment +# makes the rc_comment insert fail with "value too long" (SQLSTATE 22001), +# silently dropping that UpdatePage txn. We apply the same substring truncation +# already used for rev_comment. Anchored on the `// rc_comment` trailing comment +# so it survives line-number drift; the grep guard fails the build loudly if +# upstream changes shape and the patch silently no-ops. +# Upstream: https://github.com/cmu-db/benchbase (procedures/UpdatePage.java) +RUN set -eux; \ + f=src/main/java/com/oltpbenchmark/benchmarks/wikipedia/procedures/UpdatePage.java; \ + sed -i 's|revComment); // rc_comment|revComment.substring(0, Math.min(revComment.length(), 255 - 1))); // rc_comment|' "$f"; \ + grep -q 'revComment.substring(0, Math.min(revComment.length(), 255 - 1))); // rc_comment' "$f"; \ + ! grep -q 'param++, revComment); // rc_comment' "$f" + +# Patch the upstream Oracle Wikipedia DDL: five varchar2 columns (ipb_timestamp, +# page_touched, rev_timestamp, user_touched, user_token) carry a MySQL-style +# binary default of N literal NUL bytes written as '\0\0...'. Oracle does not +# process backslash escapes in string literals, so it reads each '\0' as the two +# characters backslash+zero -- e.g. varchar2(14) DEFAULT '\0\0\0\0\0\0\0\0\0\0\0\0\0\0' +# is a 28-char default into a 14-char column, which fails CREATE TABLE with +# ORA-01401 (inserted value too large for column). The sibling timestamp columns +# in the same file use DEFAULT '', so we normalise every varchar2 NUL-byte +# default to ''. The grep guard fails the build loudly if upstream changes shape +# and the patch silently no-ops. +# Upstream: https://github.com/cmu-db/benchbase (benchmarks/wikipedia/ddl-oracle.sql) +RUN set -eux; \ + f=src/main/resources/benchmarks/wikipedia/ddl-oracle.sql; \ + sed -i "s/\(varchar2([0-9]*)\) DEFAULT '[\\0]*'/\1 DEFAULT ''/g" "$f"; \ + ! grep -qF "\0" "$f" + +# Build each profile and unpack its tarball into /dist/benchbase-/. +# The Maven dependency cache (~/.m2) persists across profiles within this layer, +# so only the first profile pays the full download cost. Tests are skipped. +RUN set -eux; \ + mkdir -p /dist; \ + for p in ${BENCHBASE_PROFILES}; do \ + echo ">> building BenchBase profile: $p"; \ + ./mvnw -q -B clean package -P "$p" -DskipTests; \ + tar -xzf "target/benchbase-$p.tgz" -C /dist; \ + done; \ + ls -d /dist/benchbase-* + +# --- runtime stage: JRE + the unpacked distributions ----------------------- +FROM eclipse-temurin:23-jre +LABEL org.opencontainers.image.source="https://github.com/cmu-db/benchbase" \ + org.opencontainers.image.description="CMU BenchBase, multi-profile, load driver for DBMS-bench" + +WORKDIR /benchbase +COPY --from=build /dist/ /benchbase/ +# Fail the build early if no profile made it in. +RUN ls -d /benchbase/benchbase-* >/dev/null + +# Stay alive so GMT can exec each flow step (create+load, then execute) into the +# running container, the same way it drives the hammerdb container. +CMD ["sleep", "infinity"] diff --git a/benchmarks/benchbase/build-image.sh b/benchmarks/benchbase/build-image.sh new file mode 100755 index 0000000..d878b8b --- /dev/null +++ b/benchmarks/benchbase/build-image.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Builds the `benchbase` image used to drive the Wikipedia and YCSB scenarios. +# +# BenchBase (https://github.com/cmu-db/benchbase) builds one self-contained +# distribution per database profile (each bundles that engine's JDBC driver). +# This image bakes in a distribution for every engine we benchmark so a SINGLE +# image/container drives all databases of a benchmark — keeping the load +# generator identical across engines. Each profile lands in +# /benchbase/benchbase-/ inside the image; the usage scenarios cd into +# the right one and run `java -jar benchbase.jar ...`. +# +# Run this ONCE before running the Wikipedia/YCSB scenarios (after `docker login`): +# ./benchmarks/benchbase/build-image.sh +# It builds and pushes the image to Docker Hub as ribalba/benchbase:latest, which +# the usage scenarios pull. Set NO_PUSH=1 to build only. +# +# For a reproducible paper build, pin BENCHBASE_REF to a commit SHA: +# BENCHBASE_REF= ./benchmarks/benchbase/build-image.sh +set -euo pipefail + +OUT_IMAGE="${OUT_IMAGE:-ribalba/benchbase:latest}" +# BenchBase has no GitHub releases; `main` is the moving default. Pin a commit +# SHA via BENCHBASE_REF for a reproducible build. +BENCHBASE_REF="${BENCHBASE_REF:-main}" +# Engines we benchmark. BenchBase profile names (note: postgres->pg, mariadb->maria, +# sqlserver->mssql in this repo's directory naming). No Db2 profile exists. +BENCHBASE_PROFILES="${BENCHBASE_PROFILES:-postgres mysql mariadb sqlserver oracle}" +HERE="$(cd "$(dirname "$0")" && pwd)" + +echo ">> building $OUT_IMAGE" +echo " ref: $BENCHBASE_REF" +echo " profiles: $BENCHBASE_PROFILES" +docker build \ + --build-arg BENCHBASE_REF="$BENCHBASE_REF" \ + --build-arg BENCHBASE_PROFILES="$BENCHBASE_PROFILES" \ + -t "$OUT_IMAGE" \ + "$HERE" + +if [ "${NO_PUSH:-0}" = 1 ]; then + echo ">> built $OUT_IMAGE (push skipped via NO_PUSH=1)" +else + echo ">> pushing $OUT_IMAGE to Docker Hub (requires prior 'docker login')" + docker push "$OUT_IMAGE" + echo ">> done: pushed $OUT_IMAGE" +fi diff --git a/benchmarks/chbenchmark/compose.yml b/benchmarks/chbenchmark/compose.yml new file mode 100644 index 0000000..6aed8c5 --- /dev/null +++ b/benchmarks/chbenchmark/compose.yml @@ -0,0 +1,131 @@ +services: + postgres: + image: postgres:18.4-trixie + container_name: postgres_container + # Equal CPU/memory budget for every engine so the comparison is fair. GMT + # honours the native compose `cpus`/`mem_limit` keys. Tune these to the + # benchmark host; only one DB container is exercised per scenario (plus the + # hammerdb driver, which is left unconstrained so it never bottlenecks the + # load generation). + cpus: 4 + mem_limit: 8g + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + # ports: + # - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + mariadb: + image: mariadb:13.0.1-rc + container_name: mariadb_container + cpus: 4 + mem_limit: 8g + environment: + MYSQL_ROOT_PASSWORD: maria + MYSQL_DATABASE: maria + MYSQL_USER: maria + MYSQL_PASSWORD: maria + # ports: + # - "3306:3306" + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + mysql: + image: mysql:9.7.1-oraclelinux9 + container_name: mysql_container + cpus: 4 + mem_limit: 8g + environment: + MYSQL_ROOT_PASSWORD: mysql + MYSQL_DATABASE: mysql + MYSQL_USER: mysql + MYSQL_PASSWORD: mysql + # ports: + # - "3307:3306" + healthcheck: + test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -uroot -pmysql --silent"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + oracle: + image: container-registry.oracle.com/database/free:23.26.2.0 + container_name: oracle_container + cpus: 4 + mem_limit: 8g + environment: + ORACLE_PWD: oracle + # ports: + # - "1521:1521" + healthcheck: + test: ["CMD-SHELL", "echo 'SELECT 1 FROM dual;' | sqlplus -s -L system/oracle@//localhost:1521/FREEPDB1 | grep -qw 1"] + interval: 15s + timeout: 10s + retries: 40 + start_period: 120s + mssql: + image: mcr.microsoft.com/mssql/server:2022-CU23-ubuntu-22.04 + container_name: mssql_container + cpus: 4 + mem_limit: 8g + environment: + ACCEPT_EULA: "Y" + MSSQL_SA_PASSWORD: Hammerdb_2024 + MSSQL_PID: Developer + # ports: + # - "1433:1433" + healthcheck: + test: ["CMD-SHELL", "timeout 1 bash -c '/dev/null 2>&1"] + interval: 15s + timeout: 10s + retries: 40 + start_period: 300s + +# Benchmark drivers + + hammerdb: + image: tpcorg/hammerdb:latest + container_name: hammerdb_container + + benchbase: + image: ribalba/benchbase:latest + container_name: benchbase_container diff --git a/benchmarks/chbenchmark/maria.yml b/benchmarks/chbenchmark/maria.yml new file mode 100644 index 0000000..623fb26 --- /dev/null +++ b/benchmarks/chbenchmark/maria.yml @@ -0,0 +1,50 @@ +--- +name: BenchBase MariaDB CH-benCHmark benchmark +author: Didi Hoffmann /dev/null 2>&1"] + interval: 15s + timeout: 10s + retries: 40 + start_period: 300s + +# Benchmark drivers + + hammerdb: + image: tpcorg/hammerdb:latest + container_name: hammerdb_container + + benchbase: + image: ribalba/benchbase:latest + container_name: benchbase_container diff --git a/benchmarks/tpcc/db2.yml b/benchmarks/tpcc/db2.yml new file mode 100644 index 0000000..a54076a --- /dev/null +++ b/benchmarks/tpcc/db2.yml @@ -0,0 +1,79 @@ +--- +name: HammerDB Db2 TPC-C benchmark +author: Didi Hoffmann /dev/null 2>&1 && break; i=$((i+1)); sleep 3; done + db2 connect reset + db2 update db cfg for tpcc using LOGFILSIZ 8192 LOGPRIMARY 16 LOGSECOND 48 + db2 force application all + db2 deactivate db tpcc + db2 get db cfg for tpcc | grep -iE "Log file size|primary log|secondary log" + ' || true + shell: bash + log-stdout: true + - name: Catalog Db2 + container: hammerdb_container + commands: + - type: console + note: CATALOG REMOTE DB2 + command: chmod -R u+rwX /database/config/db2inst1/sqllib 2>/dev/null; rm -f /database/config/db2inst1/sqllib/db2nodes.cfg; echo "0 $(hostname) 0" > /database/config/db2inst1/sqllib/db2nodes.cfg && . /database/config/db2inst1/sqllib/db2profile && { db2 uncatalog database tpcc; db2 uncatalog node db2node; true; } && db2 catalog tcpip node db2node remote db2_container server 50000 && db2 catalog database tpcc at node db2node && db2 terminate && (i=0; while [ $i -lt 30 ]; do db2 connect to tpcc >/dev/null 2>&1 && db2 connect reset >/dev/null 2>&1 && break; i=$((i+1)); sleep 3; done) + shell: sh + log-stdout: true + - name: Build Warehouses + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB BUILD + command: . /database/config/db2inst1/sqllib/db2profile && cd /home/HammerDB-5.0 && ./hammerdbcli auto /tmp/repo/db/db2/tpcc/db2_tprocc_buildschema.tcl | tee /dev/stderr | awk '/System achieved [0-9]+ NOPM/ { match($0, /System achieved [0-9]+ NOPM/); "date +%s"|getline ts; printf("%s000000 sqlop=%s\n", ts, substr($0, RSTART+16, RLENGTH-21)) }' + shell: sh + log-stdout: true + read-sci-stdout: true + - name: Query Warehouses + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB SELECTS + command: . /database/config/db2inst1/sqllib/db2profile && cd /home/HammerDB-5.0 && ./hammerdbcli auto /tmp/repo/db/db2/tpcc/db2_tprocc_run.tcl | tee /dev/stderr | awk '/System achieved [0-9]+ NOPM/ { match($0, /System achieved [0-9]+ NOPM/); "date +%s"|getline ts; printf("%s000000 sqlop=%s\n", ts, substr($0, RSTART+16, RLENGTH-21)) }' + shell: sh + log-stdout: true + read-sci-stdout: true + - name: Drop Warehouses + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB DROP + command: . /database/config/db2inst1/sqllib/db2profile && cd /home/HammerDB-5.0 && ./hammerdbcli auto /tmp/repo/db/db2/tpcc/db2_tprocc_deleteschema.tcl | tee /dev/stderr | awk '/System achieved [0-9]+ NOPM/ { match($0, /System achieved [0-9]+ NOPM/); "date +%s"|getline ts; printf("%s000000 sqlop=%s\n", ts, substr($0, RSTART+16, RLENGTH-21)) }' + shell: sh + log-stdout: true + read-sci-stdout: true diff --git a/benchmarks/tpcc/maria.yml b/benchmarks/tpcc/maria.yml new file mode 100644 index 0000000..587e887 --- /dev/null +++ b/benchmarks/tpcc/maria.yml @@ -0,0 +1,52 @@ +--- +name: HammerDB MariaDB benchmark +author: Didi Hoffmann /dev/null 2>&1"] + interval: 15s + timeout: 10s + retries: 40 + start_period: 300s + +# Benchmark drivers + + hammerdb: + image: tpcorg/hammerdb:latest + container_name: hammerdb_container + + benchbase: + image: ribalba/benchbase:latest + container_name: benchbase_container diff --git a/benchmarks/tpch/db2.yml b/benchmarks/tpch/db2.yml new file mode 100644 index 0000000..91e72d9 --- /dev/null +++ b/benchmarks/tpch/db2.yml @@ -0,0 +1,84 @@ +--- +name: HammerDB Db2 TPC-H benchmark +author: Didi Hoffmann /dev/null 2>&1 && break; i=$((i+1)); sleep 3; done + db2 connect reset + db2 update db cfg for tpch using LOGFILSIZ 8192 LOGPRIMARY 16 LOGSECOND 48 + db2 force application all + db2 deactivate db tpch + db2 get db cfg for tpch | grep -iE "Log file size|primary log|secondary log" + ' || true + shell: bash + log-stdout: true + - name: Catalog Db2 + container: hammerdb_container + commands: + - type: console + note: CATALOG REMOTE DB2 + command: chmod -R u+rwX /database/config/db2inst1/sqllib 2>/dev/null; rm -f /database/config/db2inst1/sqllib/db2nodes.cfg; echo "0 $(hostname) 0" > /database/config/db2inst1/sqllib/db2nodes.cfg && . /database/config/db2inst1/sqllib/db2profile && { db2 uncatalog database tpch; db2 uncatalog node db2node; true; } && db2 catalog tcpip node db2node remote db2_container server 50000 && db2 catalog database tpch at node db2node && db2 terminate && (i=0; while [ $i -lt 30 ]; do db2 connect to tpch >/dev/null 2>&1 && db2 connect reset >/dev/null 2>&1 && break; i=$((i+1)); sleep 3; done) + shell: sh + log-stdout: true + - name: Build Schema + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB BUILD + command: . /database/config/db2inst1/sqllib/db2profile && cd /home/HammerDB-5.0 && ./hammerdbcli auto /tmp/repo/db/db2/tpch/db2_tproch_buildschema.tcl | tee /dev/stderr | awk '/query [0-9]+ completed in/{q++} END{if(q>0){"date +%s"|getline ts; printf("%s000000 tpch_queries=%d\n", ts, q)}}' + shell: sh + log-stdout: true + read-sci-stdout: true + - name: Run Queries + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB QUERIES + command: . /database/config/db2inst1/sqllib/db2profile && cd /home/HammerDB-5.0 && ./hammerdbcli auto /tmp/repo/db/db2/tpch/db2_tproch_run.tcl | tee /dev/stderr | awk 'END{"date +%s"|getline ts; printf("%s000000 tpch_queries=22\n", ts)}' + shell: sh + log-stdout: true + read-sci-stdout: true + - name: Drop Schema + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB DROP + command: . /database/config/db2inst1/sqllib/db2profile && cd /home/HammerDB-5.0 && ./hammerdbcli auto /tmp/repo/db/db2/tpch/db2_tproch_deleteschema.tcl | tee /dev/stderr | awk '/query [0-9]+ completed in/{q++} END{if(q>0){"date +%s"|getline ts; printf("%s000000 tpch_queries=%d\n", ts, q)}}' + shell: sh + log-stdout: true + read-sci-stdout: true diff --git a/benchmarks/tpch/maria.yml b/benchmarks/tpch/maria.yml new file mode 100644 index 0000000..c9de8b7 --- /dev/null +++ b/benchmarks/tpch/maria.yml @@ -0,0 +1,51 @@ +--- +name: HammerDB MariaDB TPC-H benchmark +author: Didi Hoffmann 0){"date +%s"|getline ts; printf("%s000000 tpch_queries=%d\n", ts, q)}}' + shell: sh + log-stdout: true + read-sci-stdout: true + - name: Run Queries + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB QUERIES + command: ./hammerdbcli auto /tmp/repo/db/maria/tpch/maria_tproch_run.tcl | tee /dev/stderr | awk 'END{"date +%s"|getline ts; printf("%s000000 tpch_queries=22\n", ts)}' + shell: sh + log-stdout: true + read-sci-stdout: true + - name: Drop Schema + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB DROP + command: ./hammerdbcli auto /tmp/repo/db/maria/tpch/maria_tproch_deleteschema.tcl | tee /dev/stderr | awk '/query [0-9]+ completed in/{q++} END{if(q>0){"date +%s"|getline ts; printf("%s000000 tpch_queries=%d\n", ts, q)}}' + shell: sh + log-stdout: true + read-sci-stdout: true diff --git a/benchmarks/tpch/mssql.yml b/benchmarks/tpch/mssql.yml new file mode 100644 index 0000000..eada32f --- /dev/null +++ b/benchmarks/tpch/mssql.yml @@ -0,0 +1,51 @@ +--- +name: HammerDB MSSQL TPC-H benchmark +author: Didi Hoffmann 0){"date +%s"|getline ts; printf("%s000000 tpch_queries=%d\n", ts, q)}}' + shell: sh + log-stdout: true + read-sci-stdout: true + - name: Run Queries + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB QUERIES + command: ./hammerdbcli auto /tmp/repo/db/mssql/tpch/mssql_tproch_run.tcl | tee /dev/stderr | awk 'END{"date +%s"|getline ts; printf("%s000000 tpch_queries=22\n", ts)}' + shell: sh + log-stdout: true + read-sci-stdout: true + - name: Drop Schema + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB DROP + command: ./hammerdbcli auto /tmp/repo/db/mssql/tpch/mssql_tproch_deleteschema.tcl | tee /dev/stderr | awk '/query [0-9]+ completed in/{q++} END{if(q>0){"date +%s"|getline ts; printf("%s000000 tpch_queries=%d\n", ts, q)}}' + shell: sh + log-stdout: true + read-sci-stdout: true diff --git a/benchmarks/tpch/mysql.yml b/benchmarks/tpch/mysql.yml new file mode 100644 index 0000000..fa20e49 --- /dev/null +++ b/benchmarks/tpch/mysql.yml @@ -0,0 +1,51 @@ +--- +name: HammerDB MySQL TPC-H benchmark +author: Didi Hoffmann 0){"date +%s"|getline ts; printf("%s000000 tpch_queries=%d\n", ts, q)}}' + shell: sh + log-stdout: true + read-sci-stdout: true + - name: Run Queries + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB QUERIES + command: ./hammerdbcli auto /tmp/repo/db/mysql/tpch/mysql_tproch_run.tcl | tee /dev/stderr | awk 'END{"date +%s"|getline ts; printf("%s000000 tpch_queries=22\n", ts)}' + shell: sh + log-stdout: true + read-sci-stdout: true + - name: Drop Schema + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB DROP + command: ./hammerdbcli auto /tmp/repo/db/mysql/tpch/mysql_tproch_deleteschema.tcl | tee /dev/stderr | awk '/query [0-9]+ completed in/{q++} END{if(q>0){"date +%s"|getline ts; printf("%s000000 tpch_queries=%d\n", ts, q)}}' + shell: sh + log-stdout: true + read-sci-stdout: true diff --git a/benchmarks/tpch/oracle.yml b/benchmarks/tpch/oracle.yml new file mode 100644 index 0000000..823448c --- /dev/null +++ b/benchmarks/tpch/oracle.yml @@ -0,0 +1,55 @@ +--- +name: HammerDB Oracle TPC-H benchmark +author: Didi Hoffmann 0){"date +%s"|getline ts; printf("%s000000 tpch_queries=%d\n", ts, q)}}' + shell: sh + log-stdout: true + read-sci-stdout: true + - name: Run Queries + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB QUERIES + command: ./hammerdbcli auto /tmp/repo/db/oracle/tpch/oracle_tproch_run.tcl | tee /dev/stderr | awk 'END{"date +%s"|getline ts; printf("%s000000 tpch_queries=22\n", ts)}' + shell: sh + log-stdout: true + read-sci-stdout: true + - name: Drop Schema + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB DROP + command: ./hammerdbcli auto /tmp/repo/db/oracle/tpch/oracle_tproch_deleteschema.tcl | tee /dev/stderr | awk '/query [0-9]+ completed in/{q++} END{if(q>0){"date +%s"|getline ts; printf("%s000000 tpch_queries=%d\n", ts, q)}}' + shell: sh + log-stdout: true + read-sci-stdout: true diff --git a/benchmarks/tpch/pg.yml b/benchmarks/tpch/pg.yml new file mode 100644 index 0000000..9ea43f1 --- /dev/null +++ b/benchmarks/tpch/pg.yml @@ -0,0 +1,51 @@ +--- +name: HammerDB Postgres TPC-H benchmark +author: Didi Hoffmann 0){"date +%s"|getline ts; printf("%s000000 tpch_queries=%d\n", ts, q)}}' + shell: sh + log-stdout: true + read-sci-stdout: true + - name: Run Queries + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB QUERIES + command: ./hammerdbcli auto /tmp/repo/db/pg/tpch/pg_tproch_run.tcl | tee /dev/stderr | awk 'END{"date +%s"|getline ts; printf("%s000000 tpch_queries=22\n", ts)}' + shell: sh + log-stdout: true + read-sci-stdout: true + - name: Drop Schema + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB DROP + command: ./hammerdbcli auto /tmp/repo/db/pg/tpch/pg_tproch_deleteschema.tcl | tee /dev/stderr | awk '/query [0-9]+ completed in/{q++} END{if(q>0){"date +%s"|getline ts; printf("%s000000 tpch_queries=%d\n", ts, q)}}' + shell: sh + log-stdout: true + read-sci-stdout: true diff --git a/benchmarks/wikipedia/compose.yml b/benchmarks/wikipedia/compose.yml new file mode 100644 index 0000000..6aed8c5 --- /dev/null +++ b/benchmarks/wikipedia/compose.yml @@ -0,0 +1,131 @@ +services: + postgres: + image: postgres:18.4-trixie + container_name: postgres_container + # Equal CPU/memory budget for every engine so the comparison is fair. GMT + # honours the native compose `cpus`/`mem_limit` keys. Tune these to the + # benchmark host; only one DB container is exercised per scenario (plus the + # hammerdb driver, which is left unconstrained so it never bottlenecks the + # load generation). + cpus: 4 + mem_limit: 8g + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + # ports: + # - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + mariadb: + image: mariadb:13.0.1-rc + container_name: mariadb_container + cpus: 4 + mem_limit: 8g + environment: + MYSQL_ROOT_PASSWORD: maria + MYSQL_DATABASE: maria + MYSQL_USER: maria + MYSQL_PASSWORD: maria + # ports: + # - "3306:3306" + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + mysql: + image: mysql:9.7.1-oraclelinux9 + container_name: mysql_container + cpus: 4 + mem_limit: 8g + environment: + MYSQL_ROOT_PASSWORD: mysql + MYSQL_DATABASE: mysql + MYSQL_USER: mysql + MYSQL_PASSWORD: mysql + # ports: + # - "3307:3306" + healthcheck: + test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -uroot -pmysql --silent"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + oracle: + image: container-registry.oracle.com/database/free:23.26.2.0 + container_name: oracle_container + cpus: 4 + mem_limit: 8g + environment: + ORACLE_PWD: oracle + # ports: + # - "1521:1521" + healthcheck: + test: ["CMD-SHELL", "echo 'SELECT 1 FROM dual;' | sqlplus -s -L system/oracle@//localhost:1521/FREEPDB1 | grep -qw 1"] + interval: 15s + timeout: 10s + retries: 40 + start_period: 120s + mssql: + image: mcr.microsoft.com/mssql/server:2022-CU23-ubuntu-22.04 + container_name: mssql_container + cpus: 4 + mem_limit: 8g + environment: + ACCEPT_EULA: "Y" + MSSQL_SA_PASSWORD: Hammerdb_2024 + MSSQL_PID: Developer + # ports: + # - "1433:1433" + healthcheck: + test: ["CMD-SHELL", "timeout 1 bash -c '/dev/null 2>&1"] + interval: 15s + timeout: 10s + retries: 40 + start_period: 300s + +# Benchmark drivers + + hammerdb: + image: tpcorg/hammerdb:latest + container_name: hammerdb_container + + benchbase: + image: ribalba/benchbase:latest + container_name: benchbase_container diff --git a/benchmarks/wikipedia/maria.yml b/benchmarks/wikipedia/maria.yml new file mode 100644 index 0000000..b5b7f83 --- /dev/null +++ b/benchmarks/wikipedia/maria.yml @@ -0,0 +1,50 @@ +--- +name: BenchBase MariaDB Wikipedia benchmark +author: Didi Hoffmann /dev/null 2>&1"] + interval: 15s + timeout: 10s + retries: 40 + start_period: 300s + +# Benchmark drivers + + hammerdb: + image: tpcorg/hammerdb:latest + container_name: hammerdb_container + + benchbase: + image: ribalba/benchbase:latest + container_name: benchbase_container diff --git a/benchmarks/ycsb/maria.yml b/benchmarks/ycsb/maria.yml new file mode 100644 index 0000000..981d6c5 --- /dev/null +++ b/benchmarks/ycsb/maria.yml @@ -0,0 +1,50 @@ +--- +name: BenchBase MariaDB YCSB benchmark +author: Didi Hoffmann //. HammerDB benchmarks are configured in +# .tcl scripts, BenchBase benchmarks in an .xml config. +HAMMERDB_BENCHMARKS = {"tpcc", "tpch"} +BENCHBASE_BENCHMARKS = {"wikipedia", "ycsb", "chbenchmark"} + +# Fairness knobs to extract from the HammerDB .tcl scripts, by benchmark. The +# values carry an engine-specific prefix in the scripts (pg_, maria_, mssqls_, +# ...; Oracle has none), so we match on the canonical suffix only. We compare a +# curated allow-list rather than every `diset` so engine-specific tuning that +# does not affect the comparison (connection strings, storage engines, ...) is +# ignored. +TCL_KNOBS = { + "tpcc": { + "vu": r"\bset\s+vu\s+(\d+)", + "vuset_vu": r"\bvuset\s+vu\s+(\d+)", + # A leading [\s_] absorbs the engine prefix (pg_rampup, maria_rampup, ...) + # as well as the bare Oracle form (`diset tpcc rampup 2`). + "rampup": r"[\s_]rampup\s+(\d+)", + "duration": r"[\s_]duration\s+(\d+)", + }, + "tpch": { + "scale_fact": r"scale_fact\s+(\d+)", + "num_tpch_threads": r"num_tpch_threads\s+(\d+)", + "total_querysets": r"total_querysets\s+(\d+)", + # Per-query intra-query parallelism. Each engine names this knob + # differently in HammerDB: pg_/oracle's `degree_of_parallel`, and + # MSSQL's `mssqls_maxdop` (MAXDOP) — the same fairness setting, so we + # match either. maria/mysql/db2 have no such HammerDB knob (no parallel + # query control there), so they legitimately never set it. + "degree_of_parallel": r"(?:degree_of_parallel|mssqls_maxdop)\s+(\d+)", + }, +} + +# Fairness knobs to extract from the BenchBase .xml configs.