diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c47a74f..9be099d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,3 +9,9 @@ updates: directory: "/" # Location of package manifests schedule: interval: "weekly" + ignore: + # MySQL must stay on the 8.4 LTS line: 9.x removed mysql_native_password and + # HammerDB 5.0 cannot authenticate to caching_sha2_password over plaintext TCP. + # See the comment on the mysql service in compose.yml. + - dependency-name: "mysql" + update-types: ["version-update:semver-major"] diff --git a/README.md b/README.md index 052ce15..6a4def4 100644 --- a/README.md +++ b/README.md @@ -2,19 +2,107 @@ A tool that benchmarks various DBMS systems agains each other and looks at energy usage The `compose.yml` defines all the containers we want to use to benchmark. We use the std containers provided by the -various DBMS and the [hammerdb](https://www.hammerdb.com/) container. For benchmarking we use the -hammerDB system. +various DBMS plus a load-driver container per benchmark family: + +- [HammerDB](https://www.hammerdb.com/) drives the TPC-C and TPC-H benchmarks. +- [BenchBase](https://github.com/cmu-db/benchbase) (CMU; the maintained OLTP-Bench successor) drives the Wikipedia, + YCSB and CH-benCHmark benchmarks. One image bakes in a distribution per engine, so the same driver runs every + database in a benchmark — see [BenchBase setup](#benchbase-setup). Currently we support: +- IBM Db2 (needs a one-time image build — see [Db2 setup](#db2-setup)) - MariaDB +- Microsoft SQL Server - MySql +- Oracle Database Free - PostgreSQL -For each DB vendor we have a benchmarking `usage_scenario_xxx.yml` that you can execute with the Green Metrics Tool to -get energy readings. +The repo is split into two trees: `benchmarks/` holds the GMT usage scenarios (one folder per benchmark), and `db/` +holds the per-engine driver scripts (one folder per database engine). + +For each DB vendor and benchmark we have a `benchmarks//.yml` (e.g. `benchmarks/tpcc/pg.yml`, +`benchmarks/tpch/oracle.yml`) that you can execute with the Green Metrics Tool to get energy readings. The driver's +per-DB scripts live under `db///`: HammerDB TCL scripts for TPC-C/TPC-H (e.g. `db/pg/tpcc/`), BenchBase +config XMLs for Wikipedia/YCSB/CH-benCHmark (e.g. `db/pg/wikipedia/pg_wikipedia_config.xml`). + +Each benchmark directory keeps its **own copy of `compose.yml`** (`benchmarks/tpcc/compose.yml`, +`benchmarks/tpch/compose.yml`, …): GMT's `!include` only resolves files inside the scenario's own directory, so +`!include ../compose.yml` is rejected. Treat the root `compose.yml` as the source of truth and re-sync the copies +whenever it changes: + +```sh +for d in tpcc tpch wikipedia ycsb chbenchmark; do cp compose.yml "benchmarks/$d/compose.yml"; done +``` + +We currently run five benchmarks: + +- [TPC-C](https://en.wikipedia.org/wiki/TPC-C) (HammerDB TPROC-C) — write-heavy OLTP; recorded metric is NOPM +- [TPC-H](https://en.wikipedia.org/wiki/TPC-H) (HammerDB TPROC-H) — analytical query workload; the SCI functional unit + is the number of queries completed (i.e. carbon per query) +- [Wikipedia](https://github.com/cmu-db/benchbase) (BenchBase) — read-mostly web-serving workload over real Wikipedia + article traces; the SCI functional unit is the number of requests completed +- [YCSB](https://github.com/cmu-db/benchbase) (BenchBase) — cloud key-value serving (point reads/writes/scans); the + SCI functional unit is the number of requests completed +- [CH-benCHmark](https://github.com/cmu-db/benchbase) (BenchBase, composite `tpcc,chbenchmark`) — HTAP: + TPC-C transactions and 22 TPC-H-style analytical queries run *concurrently* on one schema; the SCI functional unit is + the number of requests completed (OLTP txns + OLAP queries — see the per-transaction CSVs to split them) + +The three BenchBase benchmarks cover **5** engines — there is no Db2 BenchBase profile, so Db2 has TPC-C/TPC-H only. + +## Db2 setup + +Db2 needs a one-time prep the other engines don't. The `tpcorg/hammerdb` image has no Db2 client, and +HammerDB's `db2tcl` binding needs the *full* Db2 client (the free CLI driver lacks the `sqlefrce_api` +symbol). HammerDB also talks to Db2 through a *catalogued* database, so the client needs a real Db2 +instance. `db/db2/build-image.sh` handles this: it lifts the Db2 install + an initialised client instance out +of the `icr.io/db2_community/db2` image and bakes them onto the hammerdb (Ubuntu) base, then pushes the +result to Docker Hub as `ribalba/hammerdb-db2`, which the Db2 usage scenarios reference. + +Before running the Db2 scenarios (one-time): + +```sh +docker login # as the account that owns the image (ribalba) +./db/db2/build-image.sh # builds and pushes ribalba/hammerdb-db2:latest +``` + +Notes: + +- The Db2 server container requires the `CAP_IPC_OWNER` capability (granted via `--cap-add IPC_OWNER` in + `docker-run-args`), and the host must allow it. Full `privileged: true` is not needed. +- Db2 is slow to start (minutes) and loads via INSERTs, so it builds slower than the bulk-loaded engines. + +## BenchBase setup + +The Wikipedia, YCSB and CH-benCHmark scenarios are driven by a `benchbase` container. BenchBase builds one +self-contained distribution per database (each bundles that engine's JDBC driver), so `benchmarks/benchbase/build-image.sh` builds +every engine's profile into a single image and pushes it to Docker Hub as `ribalba/benchbase:latest`, which the +scenarios pull. Each profile lands in `/benchbase/benchbase-/` and the scenario `cd`s into the right one. + +Before running the Wikipedia/YCSB/CH-benCHmark scenarios (one-time): + +```sh +docker login # as the account that owns the image (ribalba) +./benchmarks/benchbase/build-image.sh # builds + pushes ribalba/benchbase:latest +BENCHBASE_REF= ./benchmarks/benchbase/build-image.sh # pin a commit for a reproducible paper build +``` + +Notes: -We are currently only using the [TPC-C benchmark](https://en.wikipedia.org/wiki/TPC-C) +- Each scenario runs three flow steps: create the `benchbase` database/user on the DB container, then BenchBase + `--create --load` (schema + data), then `--execute` (the measured run). The functional-unit metric + (`wikipedia_requests` / `ycsb_requests` / `chbenchmark_requests`) is parsed from BenchBase's `measuredRequests` on + the execute step. +- CH-benCHmark is the composite `-b tpcc,chbenchmark` with a single config: the `tpcc` plugin supplies the OLTP schema + and transactions, the `chbenchmark` plugin adds the TPC-H tables and the 22 analytical queries. OLTP and OLAP run in + *separate terminal pools* (`` vs ``) — that split is the HTAP + knob. The execute step reports one combined `measuredRequests`; for the OLTP-vs-OLAP breakdown read BenchBase's + per-transaction `results/*.results..csv` files (TPC-C's five vs `Q1`…`Q22`). +- The workload (scalefactor, terminals, isolation, weights) is fixed in `db///__config.xml` + so only the *database* configuration is the variable when comparing default vs. tuned. Defaults: Wikipedia + `scalefactor=10`, YCSB `scalefactor=1000` (≈1 GB), CH-benCHmark `scalefactor=10` (TPC-C warehouses, ≈1 GB) — all + deliberately larger than default buffer pools so tuning shows. +- BenchBase requires Java 23 to build; the image handles this. It has no Db2 profile. ## Some background reading @@ -22,3 +110,6 @@ We are currently only using the [TPC-C benchmark](https://en.wikipedia.org/wiki/ - https://www.hammerdb.com/blog/uncategorized/how-to-deploy-hammerdb-cli-fast-with-docker/ - https://www.tpc.org/tpcc/default5.asp - https://www.hammerdb.com/blog/uncategorized/how-to-run-a-fixed-throughput-workloads/ +- https://github.com/cmu-db/benchbase (BenchBase) +- OLTP-Bench paper (BenchBase lineage): https://www.vldb.org/pvldb/vol7/p277-difallah.pdf +- CH-benCHmark paper (Cole et al., DBTest 2011): https://doi.org/10.1145/1988842.1988850 diff --git a/benchmarks/benchbase/Dockerfile b/benchmarks/benchbase/Dockerfile new file mode 100644 index 0000000..5e145ed --- /dev/null +++ b/benchmarks/benchbase/Dockerfile @@ -0,0 +1,85 @@ +# BenchBase load-driver image for the Wikipedia and YCSB scenarios. +# +# BenchBase builds one self-contained distribution per database profile (each +# bundles that engine's JDBC driver), so we build every profile we need and copy +# them all into one runtime image. A scenario then runs: +# cd /benchbase/benchbase- && java -jar benchbase.jar -b -c ... +# +# Build via ./benchmarks/benchbase/build-image.sh (sets the build args below). +# syntax=docker/dockerfile:1 + +# --- build stage: compile each requested profile from source --------------- +# BenchBase's pom.xml pins Java 23 (maven.compiler.release=23). +FROM eclipse-temurin:23-jdk AS build + +# Moving default; pin a commit SHA via build-image.sh for a reproducible build. +ARG BENCHBASE_REF=main +# Space-separated BenchBase Maven profiles to bake in. +ARG BENCHBASE_PROFILES="postgres mysql mariadb sqlserver oracle" + +RUN apt-get update \ + && apt-get install -y --no-install-recommends git ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /src +RUN git clone https://github.com/cmu-db/benchbase.git . \ + && git checkout "${BENCHBASE_REF}" + +# Patch an upstream Wikipedia data-generator bug: UpdatePage truncates the +# generated revComment to 254 chars before writing revision.rev_comment, but +# writes the *untruncated* value into recentchanges.rc_comment (also +# varchar(255)). On any strict engine (PostgreSQL, and MySQL/MariaDB/Oracle/ +# SQL Server in their default strict modes) the occasional >255-char comment +# makes the rc_comment insert fail with "value too long" (SQLSTATE 22001), +# silently dropping that UpdatePage txn. We apply the same substring truncation +# already used for rev_comment. Anchored on the `// rc_comment` trailing comment +# so it survives line-number drift; the grep guard fails the build loudly if +# upstream changes shape and the patch silently no-ops. +# Upstream: https://github.com/cmu-db/benchbase (procedures/UpdatePage.java) +RUN set -eux; \ + f=src/main/java/com/oltpbenchmark/benchmarks/wikipedia/procedures/UpdatePage.java; \ + sed -i 's|revComment); // rc_comment|revComment.substring(0, Math.min(revComment.length(), 255 - 1))); // rc_comment|' "$f"; \ + grep -q 'revComment.substring(0, Math.min(revComment.length(), 255 - 1))); // rc_comment' "$f"; \ + ! grep -q 'param++, revComment); // rc_comment' "$f" + +# Patch the upstream Oracle Wikipedia DDL: five varchar2 columns (ipb_timestamp, +# page_touched, rev_timestamp, user_touched, user_token) carry a MySQL-style +# binary default of N literal NUL bytes written as '\0\0...'. Oracle does not +# process backslash escapes in string literals, so it reads each '\0' as the two +# characters backslash+zero -- e.g. varchar2(14) DEFAULT '\0\0\0\0\0\0\0\0\0\0\0\0\0\0' +# is a 28-char default into a 14-char column, which fails CREATE TABLE with +# ORA-01401 (inserted value too large for column). The sibling timestamp columns +# in the same file use DEFAULT '', so we normalise every varchar2 NUL-byte +# default to ''. The grep guard fails the build loudly if upstream changes shape +# and the patch silently no-ops. +# Upstream: https://github.com/cmu-db/benchbase (benchmarks/wikipedia/ddl-oracle.sql) +RUN set -eux; \ + f=src/main/resources/benchmarks/wikipedia/ddl-oracle.sql; \ + sed -i "s/\(varchar2([0-9]*)\) DEFAULT '[\\0]*'/\1 DEFAULT ''/g" "$f"; \ + ! grep -qF "\0" "$f" + +# Build each profile and unpack its tarball into /dist/benchbase-/. +# The Maven dependency cache (~/.m2) persists across profiles within this layer, +# so only the first profile pays the full download cost. Tests are skipped. +RUN set -eux; \ + mkdir -p /dist; \ + for p in ${BENCHBASE_PROFILES}; do \ + echo ">> building BenchBase profile: $p"; \ + ./mvnw -q -B clean package -P "$p" -DskipTests; \ + tar -xzf "target/benchbase-$p.tgz" -C /dist; \ + done; \ + ls -d /dist/benchbase-* + +# --- runtime stage: JRE + the unpacked distributions ----------------------- +FROM eclipse-temurin:23-jre +LABEL org.opencontainers.image.source="https://github.com/cmu-db/benchbase" \ + org.opencontainers.image.description="CMU BenchBase, multi-profile, load driver for DBMS-bench" + +WORKDIR /benchbase +COPY --from=build /dist/ /benchbase/ +# Fail the build early if no profile made it in. +RUN ls -d /benchbase/benchbase-* >/dev/null + +# Stay alive so GMT can exec each flow step (create+load, then execute) into the +# running container, the same way it drives the hammerdb container. +CMD ["sleep", "infinity"] diff --git a/benchmarks/benchbase/build-image.sh b/benchmarks/benchbase/build-image.sh new file mode 100755 index 0000000..d878b8b --- /dev/null +++ b/benchmarks/benchbase/build-image.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Builds the `benchbase` image used to drive the Wikipedia and YCSB scenarios. +# +# BenchBase (https://github.com/cmu-db/benchbase) builds one self-contained +# distribution per database profile (each bundles that engine's JDBC driver). +# This image bakes in a distribution for every engine we benchmark so a SINGLE +# image/container drives all databases of a benchmark — keeping the load +# generator identical across engines. Each profile lands in +# /benchbase/benchbase-/ inside the image; the usage scenarios cd into +# the right one and run `java -jar benchbase.jar ...`. +# +# Run this ONCE before running the Wikipedia/YCSB scenarios (after `docker login`): +# ./benchmarks/benchbase/build-image.sh +# It builds and pushes the image to Docker Hub as ribalba/benchbase:latest, which +# the usage scenarios pull. Set NO_PUSH=1 to build only. +# +# For a reproducible paper build, pin BENCHBASE_REF to a commit SHA: +# BENCHBASE_REF= ./benchmarks/benchbase/build-image.sh +set -euo pipefail + +OUT_IMAGE="${OUT_IMAGE:-ribalba/benchbase:latest}" +# BenchBase has no GitHub releases; `main` is the moving default. Pin a commit +# SHA via BENCHBASE_REF for a reproducible build. +BENCHBASE_REF="${BENCHBASE_REF:-main}" +# Engines we benchmark. BenchBase profile names (note: postgres->pg, mariadb->maria, +# sqlserver->mssql in this repo's directory naming). No Db2 profile exists. +BENCHBASE_PROFILES="${BENCHBASE_PROFILES:-postgres mysql mariadb sqlserver oracle}" +HERE="$(cd "$(dirname "$0")" && pwd)" + +echo ">> building $OUT_IMAGE" +echo " ref: $BENCHBASE_REF" +echo " profiles: $BENCHBASE_PROFILES" +docker build \ + --build-arg BENCHBASE_REF="$BENCHBASE_REF" \ + --build-arg BENCHBASE_PROFILES="$BENCHBASE_PROFILES" \ + -t "$OUT_IMAGE" \ + "$HERE" + +if [ "${NO_PUSH:-0}" = 1 ]; then + echo ">> built $OUT_IMAGE (push skipped via NO_PUSH=1)" +else + echo ">> pushing $OUT_IMAGE to Docker Hub (requires prior 'docker login')" + docker push "$OUT_IMAGE" + echo ">> done: pushed $OUT_IMAGE" +fi diff --git a/benchmarks/chbenchmark/compose.yml b/benchmarks/chbenchmark/compose.yml new file mode 100644 index 0000000..b08deb8 --- /dev/null +++ b/benchmarks/chbenchmark/compose.yml @@ -0,0 +1,138 @@ +services: + postgres: + image: postgres:18.4-trixie + container_name: postgres_container + # Equal CPU/memory budget for every engine so the comparison is fair. GMT + # honours the native compose `cpus`/`mem_limit` keys. Tune these to the + # benchmark host; only one DB container is exercised per scenario (plus the + # hammerdb driver, which is left unconstrained so it never bottlenecks the + # load generation). + cpus: 4 + mem_limit: 8g + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + # ports: + # - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + mariadb: + image: mariadb:12.3.2-noble + container_name: mariadb_container + cpus: 4 + mem_limit: 8g + environment: + MYSQL_ROOT_PASSWORD: maria + MYSQL_DATABASE: maria + MYSQL_USER: maria + MYSQL_PASSWORD: maria + # ports: + # - "3306:3306" + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + mysql: + # Pinned to the 8.4 LTS line on purpose — do not bump to 9.x. MySQL 9.0 removed + # mysql_native_password outright, and HammerDB 5.0's bundled mysqltcl has no + # TLS-free path for caching_sha2_password (no --get-server-public-key), so every + # virtual user fails to authenticate over plaintext TCP. The alternatives (TLS, or + # a Unix socket) would make MySQL the only engine not measured over plaintext TCP + # and so would bias the cross-engine energy comparison. + image: mysql:8.4.10-oraclelinux9 + command: ["--mysql-native-password=ON", "--authentication-policy=mysql_native_password"] + container_name: mysql_container + cpus: 4 + mem_limit: 8g + environment: + MYSQL_ROOT_PASSWORD: mysql + MYSQL_DATABASE: mysql + MYSQL_USER: mysql + MYSQL_PASSWORD: mysql + # ports: + # - "3307:3306" + healthcheck: + test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -uroot -pmysql --silent"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + oracle: + image: container-registry.oracle.com/database/free:23.26.2.0 + container_name: oracle_container + cpus: 4 + mem_limit: 8g + environment: + ORACLE_PWD: oracle + # ports: + # - "1521:1521" + healthcheck: + test: ["CMD-SHELL", "echo 'SELECT 1 FROM dual;' | sqlplus -s -L system/oracle@//localhost:1521/FREEPDB1 | grep -qw 1"] + interval: 15s + timeout: 10s + retries: 40 + start_period: 120s + mssql: + image: mcr.microsoft.com/mssql/server:2022-CU23-ubuntu-22.04 + container_name: mssql_container + cpus: 4 + mem_limit: 8g + environment: + ACCEPT_EULA: "Y" + MSSQL_SA_PASSWORD: Hammerdb_2024 + MSSQL_PID: Developer + # ports: + # - "1433:1433" + healthcheck: + test: ["CMD-SHELL", "timeout 1 bash -c '/dev/null 2>&1"] + interval: 15s + timeout: 10s + retries: 40 + start_period: 300s + +# Benchmark drivers + + hammerdb: + image: tpcorg/hammerdb:latest + container_name: hammerdb_container + + benchbase: + image: ribalba/benchbase:latest + container_name: benchbase_container diff --git a/benchmarks/chbenchmark/maria.yml b/benchmarks/chbenchmark/maria.yml new file mode 100644 index 0000000..4537f8d --- /dev/null +++ b/benchmarks/chbenchmark/maria.yml @@ -0,0 +1,50 @@ +--- +name: BenchBase MariaDB CH-benCHmark benchmark +author: Didi Hoffmann +description: Benchmarks MariaDB with BenchBase using CH-benCHmark (HTAP — concurrent TPC-C and TPC-H) + +compose-file: !include compose.yml + +custom_metrics: + chbenchmark_requests: # functional unit (work done): BenchBase requests completed (OLTP txns + OLAP queries) in the measured run + unit: CH-benCHmark request + sci: true + +services: + postgres: # unused — empty key removes the service + mysql: # unused — empty key removes the service + oracle: # unused — empty key removes the service + mssql: # unused — empty key removes the service + db2: # unused — empty key removes the service + hammerdb: # unused — empty key removes the service + benchbase: # load driver; wait for the database to be healthy first + depends_on: + mariadb: + condition: service_healthy + +flow: + - name: Create benchbase database + container: mariadb_container + commands: + - type: console + note: CREATE BENCHBASE DATABASE + command: mariadb -uroot -pmaria -e "CREATE DATABASE IF NOT EXISTS benchbase; GRANT ALL ON benchbase.* TO 'maria'@'%'; FLUSH PRIVILEGES;" + shell: sh + log-stdout: true + - name: Load CH-benCHmark schema and data + container: benchbase_container + commands: + - type: console + note: BENCHBASE CREATE + LOAD (TPC-C + CH tables) + command: cd /benchbase/benchbase-mariadb && java -jar benchbase.jar -b tpcc,chbenchmark -c /tmp/repo/db/maria/chbenchmark/maria_chbenchmark_config.xml --create=true --load=true --execute=false + shell: sh + log-stdout: true + - name: Run CH-benCHmark HTAP workload + container: benchbase_container + commands: + - type: console + note: BENCHBASE EXECUTE + command: rc=0; cd /benchbase/benchbase-mariadb && java -jar benchbase.jar -b tpcc,chbenchmark -c /tmp/repo/db/maria/chbenchmark/maria_chbenchmark_config.xml --create=false --load=false --execute=true > /tmp/benchbase-run.log 2>&1 || rc=$?; cat /tmp/benchbase-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; awk 'match($0,/measuredRequests=[0-9]+/) {v=substr($0,RSTART+17,RLENGTH-17)} END {if (v=="") exit 1; "date +%s"|getline ts; printf("%s000000 chbenchmark_requests=%s\n", ts, v)}' /tmp/benchbase-run.log + shell: sh + log-stdout: true + read-sci-stdout: true diff --git a/benchmarks/chbenchmark/mssql.yml b/benchmarks/chbenchmark/mssql.yml new file mode 100644 index 0000000..b1b3edf --- /dev/null +++ b/benchmarks/chbenchmark/mssql.yml @@ -0,0 +1,50 @@ +--- +name: BenchBase Microsoft SQL Server CH-benCHmark benchmark +author: Didi Hoffmann +description: Benchmarks Microsoft SQL Server with BenchBase using CH-benCHmark (HTAP — concurrent TPC-C and TPC-H) + +compose-file: !include compose.yml + +custom_metrics: + chbenchmark_requests: # functional unit (work done): BenchBase requests completed (OLTP txns + OLAP queries) in the measured run + unit: CH-benCHmark request + sci: true + +services: + postgres: # unused — empty key removes the service + mariadb: # unused — empty key removes the service + mysql: # unused — empty key removes the service + oracle: # unused — empty key removes the service + db2: # unused — empty key removes the service + hammerdb: # unused — empty key removes the service + benchbase: # load driver; wait for the database to be healthy first + depends_on: + mssql: + condition: service_healthy + +flow: + - name: Create benchbase database + container: mssql_container + commands: + - type: console + note: CREATE BENCHBASE DATABASE + command: /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P Hammerdb_2024 -C -Q "IF DB_ID('benchbase') IS NULL CREATE DATABASE benchbase" + shell: sh + log-stdout: true + - name: Load CH-benCHmark schema and data + container: benchbase_container + commands: + - type: console + note: BENCHBASE CREATE + LOAD (TPC-C + CH tables) + command: cd /benchbase/benchbase-sqlserver && java -jar benchbase.jar -b tpcc,chbenchmark -c /tmp/repo/db/mssql/chbenchmark/mssql_chbenchmark_config.xml --create=true --load=true --execute=false + shell: sh + log-stdout: true + - name: Run CH-benCHmark HTAP workload + container: benchbase_container + commands: + - type: console + note: BENCHBASE EXECUTE + command: rc=0; cd /benchbase/benchbase-sqlserver && java -jar benchbase.jar -b tpcc,chbenchmark -c /tmp/repo/db/mssql/chbenchmark/mssql_chbenchmark_config.xml --create=false --load=false --execute=true > /tmp/benchbase-run.log 2>&1 || rc=$?; cat /tmp/benchbase-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; awk 'match($0,/measuredRequests=[0-9]+/) {v=substr($0,RSTART+17,RLENGTH-17)} END {if (v=="") exit 1; "date +%s"|getline ts; printf("%s000000 chbenchmark_requests=%s\n", ts, v)}' /tmp/benchbase-run.log + shell: sh + log-stdout: true + read-sci-stdout: true diff --git a/benchmarks/chbenchmark/mysql.yml b/benchmarks/chbenchmark/mysql.yml new file mode 100644 index 0000000..7232662 --- /dev/null +++ b/benchmarks/chbenchmark/mysql.yml @@ -0,0 +1,50 @@ +--- +name: BenchBase MySQL CH-benCHmark benchmark +author: Didi Hoffmann +description: Benchmarks MySQL with BenchBase using CH-benCHmark (HTAP — concurrent TPC-C and TPC-H) + +compose-file: !include compose.yml + +custom_metrics: + chbenchmark_requests: # functional unit (work done): BenchBase requests completed (OLTP txns + OLAP queries) in the measured run + unit: CH-benCHmark request + sci: true + +services: + postgres: # unused — empty key removes the service + mariadb: # unused — empty key removes the service + oracle: # unused — empty key removes the service + mssql: # unused — empty key removes the service + db2: # unused — empty key removes the service + hammerdb: # unused — empty key removes the service + benchbase: # load driver; wait for the database to be healthy first + depends_on: + mysql: + condition: service_healthy + +flow: + - name: Create benchbase database + container: mysql_container + commands: + - type: console + note: CREATE BENCHBASE DATABASE + command: mysql -uroot -pmysql -e "CREATE DATABASE IF NOT EXISTS benchbase; GRANT ALL ON benchbase.* TO 'mysql'@'%'; FLUSH PRIVILEGES;" + shell: sh + log-stdout: true + - name: Load CH-benCHmark schema and data + container: benchbase_container + commands: + - type: console + note: BENCHBASE CREATE + LOAD (TPC-C + CH tables) + command: cd /benchbase/benchbase-mysql && java -jar benchbase.jar -b tpcc,chbenchmark -c /tmp/repo/db/mysql/chbenchmark/mysql_chbenchmark_config.xml --create=true --load=true --execute=false + shell: sh + log-stdout: true + - name: Run CH-benCHmark HTAP workload + container: benchbase_container + commands: + - type: console + note: BENCHBASE EXECUTE + command: rc=0; cd /benchbase/benchbase-mysql && java -jar benchbase.jar -b tpcc,chbenchmark -c /tmp/repo/db/mysql/chbenchmark/mysql_chbenchmark_config.xml --create=false --load=false --execute=true > /tmp/benchbase-run.log 2>&1 || rc=$?; cat /tmp/benchbase-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; awk 'match($0,/measuredRequests=[0-9]+/) {v=substr($0,RSTART+17,RLENGTH-17)} END {if (v=="") exit 1; "date +%s"|getline ts; printf("%s000000 chbenchmark_requests=%s\n", ts, v)}' /tmp/benchbase-run.log + shell: sh + log-stdout: true + read-sci-stdout: true diff --git a/benchmarks/chbenchmark/oracle.yml b/benchmarks/chbenchmark/oracle.yml new file mode 100644 index 0000000..8f4b2a1 --- /dev/null +++ b/benchmarks/chbenchmark/oracle.yml @@ -0,0 +1,50 @@ +--- +name: BenchBase Oracle Database Free CH-benCHmark benchmark +author: Didi Hoffmann +description: Benchmarks Oracle Database Free with BenchBase using CH-benCHmark (HTAP — concurrent TPC-C and TPC-H) + +compose-file: !include compose.yml + +custom_metrics: + chbenchmark_requests: # functional unit (work done): BenchBase requests completed (OLTP txns + OLAP queries) in the measured run + unit: CH-benCHmark request + sci: true + +services: + postgres: # unused — empty key removes the service + mariadb: # unused — empty key removes the service + mysql: # unused — empty key removes the service + mssql: # unused — empty key removes the service + db2: # unused — empty key removes the service + hammerdb: # unused — empty key removes the service + benchbase: # load driver; wait for the database to be healthy first + depends_on: + oracle: + condition: service_healthy + +flow: + - name: Create benchbase database + container: oracle_container + commands: + - type: console + note: CREATE BENCHBASE DATABASE + command: printf '%s\n' "WHENEVER SQLERROR EXIT FAILURE" "CREATE USER benchbase IDENTIFIED BY benchbase DEFAULT TABLESPACE users QUOTA UNLIMITED ON users;" "GRANT CONNECT, RESOURCE, CREATE VIEW TO benchbase;" "EXIT;" | sqlplus -s system/oracle@//localhost:1521/FREEPDB1 + shell: sh + log-stdout: true + - name: Load CH-benCHmark schema and data + container: benchbase_container + commands: + - type: console + note: BENCHBASE CREATE + LOAD (TPC-C + CH tables) + command: cd /benchbase/benchbase-oracle && java -jar benchbase.jar -b tpcc,chbenchmark -c /tmp/repo/db/oracle/chbenchmark/oracle_chbenchmark_config.xml --create=true --load=true --execute=false + shell: sh + log-stdout: true + - name: Run CH-benCHmark HTAP workload + container: benchbase_container + commands: + - type: console + note: BENCHBASE EXECUTE + command: rc=0; cd /benchbase/benchbase-oracle && java -jar benchbase.jar -b tpcc,chbenchmark -c /tmp/repo/db/oracle/chbenchmark/oracle_chbenchmark_config.xml --create=false --load=false --execute=true > /tmp/benchbase-run.log 2>&1 || rc=$?; cat /tmp/benchbase-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; awk 'match($0,/measuredRequests=[0-9]+/) {v=substr($0,RSTART+17,RLENGTH-17)} END {if (v=="") exit 1; "date +%s"|getline ts; printf("%s000000 chbenchmark_requests=%s\n", ts, v)}' /tmp/benchbase-run.log + shell: sh + log-stdout: true + read-sci-stdout: true diff --git a/benchmarks/chbenchmark/pg.yml b/benchmarks/chbenchmark/pg.yml new file mode 100644 index 0000000..0fb7fd4 --- /dev/null +++ b/benchmarks/chbenchmark/pg.yml @@ -0,0 +1,50 @@ +--- +name: BenchBase PostgreSQL CH-benCHmark benchmark +author: Didi Hoffmann +description: Benchmarks PostgreSQL with BenchBase using CH-benCHmark (HTAP — concurrent TPC-C and TPC-H) + +compose-file: !include compose.yml + +custom_metrics: + chbenchmark_requests: # functional unit (work done): BenchBase requests completed (OLTP txns + OLAP queries) in the measured run + unit: CH-benCHmark request + sci: true + +services: + mariadb: # unused — empty key removes the service + mysql: # unused — empty key removes the service + oracle: # unused — empty key removes the service + mssql: # unused — empty key removes the service + db2: # unused — empty key removes the service + hammerdb: # unused — empty key removes the service + benchbase: # load driver; wait for the database to be healthy first + depends_on: + postgres: + condition: service_healthy + +flow: + - name: Create benchbase database + container: postgres_container + commands: + - type: console + note: CREATE BENCHBASE DATABASE + command: psql "postgresql://postgres:postgres@localhost:5432/postgres" -tc "SELECT 1 FROM pg_database WHERE datname='benchbase'" | grep -q 1 || psql "postgresql://postgres:postgres@localhost:5432/postgres" -c "CREATE DATABASE benchbase" + shell: sh + log-stdout: true + - name: Load CH-benCHmark schema and data + container: benchbase_container + commands: + - type: console + note: BENCHBASE CREATE + LOAD (TPC-C + CH tables) + command: cd /benchbase/benchbase-postgres && java -jar benchbase.jar -b tpcc,chbenchmark -c /tmp/repo/db/pg/chbenchmark/pg_chbenchmark_config.xml --create=true --load=true --execute=false + shell: sh + log-stdout: true + - name: Run CH-benCHmark HTAP workload + container: benchbase_container + commands: + - type: console + note: BENCHBASE EXECUTE + command: rc=0; cd /benchbase/benchbase-postgres && java -jar benchbase.jar -b tpcc,chbenchmark -c /tmp/repo/db/pg/chbenchmark/pg_chbenchmark_config.xml --create=false --load=false --execute=true > /tmp/benchbase-run.log 2>&1 || rc=$?; cat /tmp/benchbase-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; awk 'match($0,/measuredRequests=[0-9]+/) {v=substr($0,RSTART+17,RLENGTH-17)} END {if (v=="") exit 1; "date +%s"|getline ts; printf("%s000000 chbenchmark_requests=%s\n", ts, v)}' /tmp/benchbase-run.log + shell: sh + log-stdout: true + read-sci-stdout: true diff --git a/benchmarks/tpcc/compose.yml b/benchmarks/tpcc/compose.yml new file mode 100644 index 0000000..b08deb8 --- /dev/null +++ b/benchmarks/tpcc/compose.yml @@ -0,0 +1,138 @@ +services: + postgres: + image: postgres:18.4-trixie + container_name: postgres_container + # Equal CPU/memory budget for every engine so the comparison is fair. GMT + # honours the native compose `cpus`/`mem_limit` keys. Tune these to the + # benchmark host; only one DB container is exercised per scenario (plus the + # hammerdb driver, which is left unconstrained so it never bottlenecks the + # load generation). + cpus: 4 + mem_limit: 8g + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + # ports: + # - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + mariadb: + image: mariadb:12.3.2-noble + container_name: mariadb_container + cpus: 4 + mem_limit: 8g + environment: + MYSQL_ROOT_PASSWORD: maria + MYSQL_DATABASE: maria + MYSQL_USER: maria + MYSQL_PASSWORD: maria + # ports: + # - "3306:3306" + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + mysql: + # Pinned to the 8.4 LTS line on purpose — do not bump to 9.x. MySQL 9.0 removed + # mysql_native_password outright, and HammerDB 5.0's bundled mysqltcl has no + # TLS-free path for caching_sha2_password (no --get-server-public-key), so every + # virtual user fails to authenticate over plaintext TCP. The alternatives (TLS, or + # a Unix socket) would make MySQL the only engine not measured over plaintext TCP + # and so would bias the cross-engine energy comparison. + image: mysql:8.4.10-oraclelinux9 + command: ["--mysql-native-password=ON", "--authentication-policy=mysql_native_password"] + container_name: mysql_container + cpus: 4 + mem_limit: 8g + environment: + MYSQL_ROOT_PASSWORD: mysql + MYSQL_DATABASE: mysql + MYSQL_USER: mysql + MYSQL_PASSWORD: mysql + # ports: + # - "3307:3306" + healthcheck: + test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -uroot -pmysql --silent"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + oracle: + image: container-registry.oracle.com/database/free:23.26.2.0 + container_name: oracle_container + cpus: 4 + mem_limit: 8g + environment: + ORACLE_PWD: oracle + # ports: + # - "1521:1521" + healthcheck: + test: ["CMD-SHELL", "echo 'SELECT 1 FROM dual;' | sqlplus -s -L system/oracle@//localhost:1521/FREEPDB1 | grep -qw 1"] + interval: 15s + timeout: 10s + retries: 40 + start_period: 120s + mssql: + image: mcr.microsoft.com/mssql/server:2022-CU23-ubuntu-22.04 + container_name: mssql_container + cpus: 4 + mem_limit: 8g + environment: + ACCEPT_EULA: "Y" + MSSQL_SA_PASSWORD: Hammerdb_2024 + MSSQL_PID: Developer + # ports: + # - "1433:1433" + healthcheck: + test: ["CMD-SHELL", "timeout 1 bash -c '/dev/null 2>&1"] + interval: 15s + timeout: 10s + retries: 40 + start_period: 300s + +# Benchmark drivers + + hammerdb: + image: tpcorg/hammerdb:latest + container_name: hammerdb_container + + benchbase: + image: ribalba/benchbase:latest + container_name: benchbase_container diff --git a/benchmarks/tpcc/db2.yml b/benchmarks/tpcc/db2.yml new file mode 100644 index 0000000..cb4910d --- /dev/null +++ b/benchmarks/tpcc/db2.yml @@ -0,0 +1,77 @@ +--- +name: HammerDB Db2 TPC-C benchmark +author: Didi Hoffmann +description: Benchmarks IBM Db2 with HammerDB using TPC-C + +compose-file: !include compose.yml + +custom_metrics: + sqlop: + unit: TPC-C SQL-op + sci: true + +services: + postgres: # empty key will remove the unused service + mariadb: # empty key will remove the unused service + mysql: # empty key will remove the unused service + oracle: # empty key will remove the unused service + mssql: # empty key will remove the unused service + hammerdb: # Db2-enabled image (build/push with ./db/db2/build-image.sh); wait for Db2 + image: ribalba/hammerdb-db2:latest + depends_on: + db2: + condition: service_healthy + +flow: + # Db2 is excluded from the paper (no BenchBase profile; Community-edition capped) and + # kept only for benchmarking, so enlarging its undersized default transaction log here + # is fine. Without this the 80-warehouse bulk load fails with SQL0964C (log full). + - name: Size Db2 transaction log + container: db2_container + commands: + - type: console + note: ENLARGE TRANSACTION LOG before the bulk load (avoids SQL0964C "log full") + command: | + su -l db2inst1 -c ' + i=0; while [ $i -lt 40 ]; do db2 connect to tpcc >/dev/null 2>&1 && break; i=$((i+1)); sleep 3; done + db2 connect reset + db2 update db cfg for tpcc using LOGFILSIZ 8192 LOGPRIMARY 16 LOGSECOND 48 + db2 force application all + db2 deactivate db tpcc + db2 get db cfg for tpcc | grep -iE "Log file size|primary log|secondary log" + ' || true + shell: bash + log-stdout: true + - name: Catalog Db2 + container: hammerdb_container + commands: + - type: console + note: CATALOG REMOTE DB2 + command: chmod -R u+rwX /database/config/db2inst1/sqllib 2>/dev/null; rm -f /database/config/db2inst1/sqllib/db2nodes.cfg; echo "0 $(hostname) 0" > /database/config/db2inst1/sqllib/db2nodes.cfg && . /database/config/db2inst1/sqllib/db2profile && { db2 uncatalog database tpcc; db2 uncatalog node db2node; true; } && db2 catalog tcpip node db2node remote db2_container server 50000 && db2 catalog database tpcc at node db2node && db2 terminate && (i=0; while [ $i -lt 30 ]; do db2 connect to tpcc >/dev/null 2>&1 && db2 connect reset >/dev/null 2>&1 && break; i=$((i+1)); sleep 3; done) + shell: sh + log-stdout: true + - name: Build Warehouses + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB BUILD + command: rc=0; . /database/config/db2inst1/sqllib/db2profile && cd /home/HammerDB-5.0 && ./hammerdbcli auto /tmp/repo/db/db2/tpcc/db2_tprocc_buildschema.tcl > /tmp/hammerdb-build.log 2>&1 || rc=$?; cat /tmp/hammerdb-build.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; if grep -qE 'FINISHED FAILED|Error in Virtual User' /tmp/hammerdb-build.log; then echo 'HammerDB reported virtual-user failures - see log above' >&2; exit 1; fi + shell: sh + log-stdout: true + - name: Query Warehouses + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB SELECTS + command: rc=0; . /database/config/db2inst1/sqllib/db2profile && cd /home/HammerDB-5.0 && ./hammerdbcli auto /tmp/repo/db/db2/tpcc/db2_tprocc_run.tcl > /tmp/hammerdb-run.log 2>&1 || rc=$?; cat /tmp/hammerdb-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; if grep -qE 'FINISHED FAILED|Error in Virtual User' /tmp/hammerdb-run.log; then echo 'HammerDB reported virtual-user failures - see log above' >&2; exit 1; fi; awk '/System achieved [0-9]+ NOPM/ { match($0, /System achieved [0-9]+ NOPM/); v=substr($0, RSTART+16, RLENGTH-21) } END { if (v=="") exit 1; "date +%s"|getline ts; printf("%s000000 sqlop=%s\n", ts, v) }' /tmp/hammerdb-run.log + shell: sh + log-stdout: true + read-sci-stdout: true + - name: Drop Warehouses + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB DROP + command: . /database/config/db2inst1/sqllib/db2profile && cd /home/HammerDB-5.0 && ./hammerdbcli auto /tmp/repo/db/db2/tpcc/db2_tprocc_deleteschema.tcl + shell: sh + log-stdout: true diff --git a/benchmarks/tpcc/maria.yml b/benchmarks/tpcc/maria.yml new file mode 100644 index 0000000..d15081b --- /dev/null +++ b/benchmarks/tpcc/maria.yml @@ -0,0 +1,49 @@ +--- +name: HammerDB MariaDB TPC-C benchmark +author: Didi Hoffmann +description: Benchmarks mariadb with HammerDB using TPC-C + +compose-file: !include compose.yml + +custom_metrics: + sqlop: + unit: TPC-C SQL-op + sci: true + +services: + postgres: # empty key will remove the unused service + mysql: # empty key will remove the service + oracle: # empty key will remove the unused service + mssql: # empty key will remove the unused service + db2: # empty key will remove the unused service + hammerdb: # wait for MariaDB to finish starting up before benchmarking + depends_on: + mariadb: + condition: service_healthy + +flow: + - name: Build Warehouses + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB BUILD + command: rc=0; ./hammerdbcli auto /tmp/repo/db/maria/tpcc/maria_tprocc_buildschema.tcl > /tmp/hammerdb-build.log 2>&1 || rc=$?; cat /tmp/hammerdb-build.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; if grep -qE 'FINISHED FAILED|Error in Virtual User' /tmp/hammerdb-build.log; then echo 'HammerDB reported virtual-user failures - see log above' >&2; exit 1; fi + shell: sh + log-stdout: true + - name: Query Warehouses + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB SELECTS + command: rc=0; ./hammerdbcli auto /tmp/repo/db/maria/tpcc/maria_tprocc_run.tcl > /tmp/hammerdb-run.log 2>&1 || rc=$?; cat /tmp/hammerdb-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; if grep -qE 'FINISHED FAILED|Error in Virtual User' /tmp/hammerdb-run.log; then echo 'HammerDB reported virtual-user failures - see log above' >&2; exit 1; fi; awk '/System achieved [0-9]+ NOPM/ { match($0, /System achieved [0-9]+ NOPM/); v=substr($0, RSTART+16, RLENGTH-21) } END { if (v=="") exit 1; "date +%s"|getline ts; printf("%s000000 sqlop=%s\n", ts, v) }' /tmp/hammerdb-run.log + shell: sh + log-stdout: true + read-sci-stdout: true + - name: Drop Warehouses + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB DROP + command: ./hammerdbcli auto /tmp/repo/db/maria/tpcc/maria_tprocc_deleteschema.tcl + shell: sh + log-stdout: true diff --git a/benchmarks/tpcc/mssql.yml b/benchmarks/tpcc/mssql.yml new file mode 100644 index 0000000..7d68634 --- /dev/null +++ b/benchmarks/tpcc/mssql.yml @@ -0,0 +1,49 @@ +--- +name: HammerDB MSSQL TPC-C benchmark +author: Didi Hoffmann +description: Benchmarks Microsoft SQL Server with HammerDB using TPC-C + +compose-file: !include compose.yml + +custom_metrics: + sqlop: + unit: TPC-C SQL-op + sci: true + +services: + postgres: # empty key will remove the unused service + mariadb: # empty key will remove the unused service + mysql: # empty key will remove the unused service + oracle: # empty key will remove the unused service + db2: # empty key will remove the unused service + hammerdb: # wait for SQL Server to finish starting up before benchmarking + depends_on: + mssql: + condition: service_healthy + +flow: + - name: Build Warehouses + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB BUILD + command: rc=0; TMP=/tmp PATH=/opt/mssql-tools18/bin:$PATH ./hammerdbcli auto /tmp/repo/db/mssql/tpcc/mssql_tprocc_buildschema.tcl > /tmp/hammerdb-build.log 2>&1 || rc=$?; cat /tmp/hammerdb-build.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; if grep -qE 'FINISHED FAILED|Error in Virtual User' /tmp/hammerdb-build.log; then echo 'HammerDB reported virtual-user failures - see log above' >&2; exit 1; fi + shell: sh + log-stdout: true + - name: Query Warehouses + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB SELECTS + command: rc=0; ./hammerdbcli auto /tmp/repo/db/mssql/tpcc/mssql_tprocc_run.tcl > /tmp/hammerdb-run.log 2>&1 || rc=$?; cat /tmp/hammerdb-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; if grep -qE 'FINISHED FAILED|Error in Virtual User' /tmp/hammerdb-run.log; then echo 'HammerDB reported virtual-user failures - see log above' >&2; exit 1; fi; awk '/System achieved [0-9]+ NOPM/ { match($0, /System achieved [0-9]+ NOPM/); v=substr($0, RSTART+16, RLENGTH-21) } END { if (v=="") exit 1; "date +%s"|getline ts; printf("%s000000 sqlop=%s\n", ts, v) }' /tmp/hammerdb-run.log + shell: sh + log-stdout: true + read-sci-stdout: true + - name: Drop Warehouses + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB DROP + command: ./hammerdbcli auto /tmp/repo/db/mssql/tpcc/mssql_tprocc_deleteschema.tcl + shell: sh + log-stdout: true diff --git a/benchmarks/tpcc/mysql.yml b/benchmarks/tpcc/mysql.yml new file mode 100644 index 0000000..36ecc76 --- /dev/null +++ b/benchmarks/tpcc/mysql.yml @@ -0,0 +1,49 @@ +--- +name: HammerDB MySQL TPC-C benchmark +author: Didi Hoffmann +description: Benchmarks mysql with HammerDB using TPC-C + +compose-file: !include compose.yml + +custom_metrics: + sqlop: + unit: TPC-C SQL-op + sci: true + +services: + postgres: # empty key will remove the unused service + mariadb: # empty key will remove the service + oracle: # empty key will remove the unused service + mssql: # empty key will remove the unused service + db2: # empty key will remove the unused service + hammerdb: # wait for MySQL to finish starting up before benchmarking + depends_on: + mysql: + condition: service_healthy + +flow: + - name: Build Warehouses + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB BUILD + command: rc=0; ./hammerdbcli auto /tmp/repo/db/mysql/tpcc/mysql_tprocc_buildschema.tcl > /tmp/hammerdb-build.log 2>&1 || rc=$?; cat /tmp/hammerdb-build.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; if grep -qE 'FINISHED FAILED|Error in Virtual User' /tmp/hammerdb-build.log; then echo 'HammerDB reported virtual-user failures - see log above' >&2; exit 1; fi + shell: sh + log-stdout: true + - name: Query Warehouses + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB SELECTS + command: rc=0; ./hammerdbcli auto /tmp/repo/db/mysql/tpcc/mysql_tprocc_run.tcl > /tmp/hammerdb-run.log 2>&1 || rc=$?; cat /tmp/hammerdb-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; if grep -qE 'FINISHED FAILED|Error in Virtual User' /tmp/hammerdb-run.log; then echo 'HammerDB reported virtual-user failures - see log above' >&2; exit 1; fi; awk '/System achieved [0-9]+ NOPM/ { match($0, /System achieved [0-9]+ NOPM/); v=substr($0, RSTART+16, RLENGTH-21) } END { if (v=="") exit 1; "date +%s"|getline ts; printf("%s000000 sqlop=%s\n", ts, v) }' /tmp/hammerdb-run.log + shell: sh + log-stdout: true + read-sci-stdout: true + - name: Drop Warehouses + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB DROP + command: ./hammerdbcli auto /tmp/repo/db/mysql/tpcc/mysql_tprocc_deleteschema.tcl + shell: sh + log-stdout: true diff --git a/benchmarks/tpcc/oracle.yml b/benchmarks/tpcc/oracle.yml new file mode 100644 index 0000000..cbff42a --- /dev/null +++ b/benchmarks/tpcc/oracle.yml @@ -0,0 +1,53 @@ +--- +name: HammerDB Oracle TPC-C benchmark +author: Didi Hoffmann +description: Benchmarks Oracle Database Free with HammerDB using TPC-C + +compose-file: !include compose.yml + +custom_metrics: + sqlop: + unit: TPC-C SQL-op + sci: true + +services: + postgres: # empty key will remove the unused service + mariadb: # empty key will remove the unused service + mysql: # empty key will remove the unused service + mssql: # empty key will remove the unused service + db2: # empty key will remove the unused service + hammerdb: # wait for Oracle to finish starting up before benchmarking + depends_on: + oracle: + condition: service_healthy + # the Oracle Instant Client ships in the hammerdb image but its dir is not on + # the loader path, so libclntsh.so can't find libnnz21.so / libclntshcore.so + environment: + LD_LIBRARY_PATH: /home/instantclient_21_18 + +flow: + - name: Build Warehouses + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB BUILD + command: rc=0; ./hammerdbcli auto /tmp/repo/db/oracle/tpcc/oracle_tprocc_buildschema.tcl > /tmp/hammerdb-build.log 2>&1 || rc=$?; cat /tmp/hammerdb-build.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; if grep -qE 'FINISHED FAILED|Error in Virtual User' /tmp/hammerdb-build.log; then echo 'HammerDB reported virtual-user failures - see log above' >&2; exit 1; fi + shell: sh + log-stdout: true + - name: Query Warehouses + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB SELECTS + command: rc=0; ./hammerdbcli auto /tmp/repo/db/oracle/tpcc/oracle_tprocc_run.tcl > /tmp/hammerdb-run.log 2>&1 || rc=$?; cat /tmp/hammerdb-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; if grep -qE 'FINISHED FAILED|Error in Virtual User' /tmp/hammerdb-run.log; then echo 'HammerDB reported virtual-user failures - see log above' >&2; exit 1; fi; awk '/System achieved [0-9]+ NOPM/ { match($0, /System achieved [0-9]+ NOPM/); v=substr($0, RSTART+16, RLENGTH-21) } END { if (v=="") exit 1; "date +%s"|getline ts; printf("%s000000 sqlop=%s\n", ts, v) }' /tmp/hammerdb-run.log + shell: sh + log-stdout: true + read-sci-stdout: true + - name: Drop Warehouses + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB DROP + command: ./hammerdbcli auto /tmp/repo/db/oracle/tpcc/oracle_tprocc_deleteschema.tcl + shell: sh + log-stdout: true diff --git a/benchmarks/tpcc/pg.yml b/benchmarks/tpcc/pg.yml new file mode 100644 index 0000000..2ae8e17 --- /dev/null +++ b/benchmarks/tpcc/pg.yml @@ -0,0 +1,49 @@ +--- +name: HammerDB Postgres TPC-C benchmark +author: Didi Hoffmann +description: Benchmarks postgresql with HammerDB using TPC-C + +compose-file: !include compose.yml + +custom_metrics: + sqlop: + unit: TPC-C SQL-op + sci: true + +services: + mariadb: # empty key will remove the unused service + mysql: # empty key will remove the service + oracle: # empty key will remove the unused service + mssql: # empty key will remove the unused service + db2: # empty key will remove the unused service + hammerdb: # wait for Postgres to finish starting up before benchmarking + depends_on: + postgres: + condition: service_healthy + +flow: + - name: Build Warehouses + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB BUILD + command: rc=0; ./hammerdbcli auto /tmp/repo/db/pg/tpcc/pg_tprocc_buildschema.tcl > /tmp/hammerdb-build.log 2>&1 || rc=$?; cat /tmp/hammerdb-build.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; if grep -qE 'FINISHED FAILED|Error in Virtual User' /tmp/hammerdb-build.log; then echo 'HammerDB reported virtual-user failures - see log above' >&2; exit 1; fi + shell: sh + log-stdout: true + - name: Query Warehouses + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB SELECTS + command: rc=0; ./hammerdbcli auto /tmp/repo/db/pg/tpcc/pg_tprocc_run.tcl > /tmp/hammerdb-run.log 2>&1 || rc=$?; cat /tmp/hammerdb-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; if grep -qE 'FINISHED FAILED|Error in Virtual User' /tmp/hammerdb-run.log; then echo 'HammerDB reported virtual-user failures - see log above' >&2; exit 1; fi; awk '/System achieved [0-9]+ NOPM/ { match($0, /System achieved [0-9]+ NOPM/); v=substr($0, RSTART+16, RLENGTH-21) } END { if (v=="") exit 1; "date +%s"|getline ts; printf("%s000000 sqlop=%s\n", ts, v) }' /tmp/hammerdb-run.log + shell: sh + log-stdout: true + read-sci-stdout: true + - name: Drop Warehouses + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB DROP + command: ./hammerdbcli auto /tmp/repo/db/pg/tpcc/pg_tprocc_deleteschema.tcl + shell: sh + log-stdout: true diff --git a/benchmarks/tpch/compose.yml b/benchmarks/tpch/compose.yml new file mode 100644 index 0000000..b08deb8 --- /dev/null +++ b/benchmarks/tpch/compose.yml @@ -0,0 +1,138 @@ +services: + postgres: + image: postgres:18.4-trixie + container_name: postgres_container + # Equal CPU/memory budget for every engine so the comparison is fair. GMT + # honours the native compose `cpus`/`mem_limit` keys. Tune these to the + # benchmark host; only one DB container is exercised per scenario (plus the + # hammerdb driver, which is left unconstrained so it never bottlenecks the + # load generation). + cpus: 4 + mem_limit: 8g + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + # ports: + # - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + mariadb: + image: mariadb:12.3.2-noble + container_name: mariadb_container + cpus: 4 + mem_limit: 8g + environment: + MYSQL_ROOT_PASSWORD: maria + MYSQL_DATABASE: maria + MYSQL_USER: maria + MYSQL_PASSWORD: maria + # ports: + # - "3306:3306" + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + mysql: + # Pinned to the 8.4 LTS line on purpose — do not bump to 9.x. MySQL 9.0 removed + # mysql_native_password outright, and HammerDB 5.0's bundled mysqltcl has no + # TLS-free path for caching_sha2_password (no --get-server-public-key), so every + # virtual user fails to authenticate over plaintext TCP. The alternatives (TLS, or + # a Unix socket) would make MySQL the only engine not measured over plaintext TCP + # and so would bias the cross-engine energy comparison. + image: mysql:8.4.10-oraclelinux9 + command: ["--mysql-native-password=ON", "--authentication-policy=mysql_native_password"] + container_name: mysql_container + cpus: 4 + mem_limit: 8g + environment: + MYSQL_ROOT_PASSWORD: mysql + MYSQL_DATABASE: mysql + MYSQL_USER: mysql + MYSQL_PASSWORD: mysql + # ports: + # - "3307:3306" + healthcheck: + test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -uroot -pmysql --silent"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + oracle: + image: container-registry.oracle.com/database/free:23.26.2.0 + container_name: oracle_container + cpus: 4 + mem_limit: 8g + environment: + ORACLE_PWD: oracle + # ports: + # - "1521:1521" + healthcheck: + test: ["CMD-SHELL", "echo 'SELECT 1 FROM dual;' | sqlplus -s -L system/oracle@//localhost:1521/FREEPDB1 | grep -qw 1"] + interval: 15s + timeout: 10s + retries: 40 + start_period: 120s + mssql: + image: mcr.microsoft.com/mssql/server:2022-CU23-ubuntu-22.04 + container_name: mssql_container + cpus: 4 + mem_limit: 8g + environment: + ACCEPT_EULA: "Y" + MSSQL_SA_PASSWORD: Hammerdb_2024 + MSSQL_PID: Developer + # ports: + # - "1433:1433" + healthcheck: + test: ["CMD-SHELL", "timeout 1 bash -c '/dev/null 2>&1"] + interval: 15s + timeout: 10s + retries: 40 + start_period: 300s + +# Benchmark drivers + + hammerdb: + image: tpcorg/hammerdb:latest + container_name: hammerdb_container + + benchbase: + image: ribalba/benchbase:latest + container_name: benchbase_container diff --git a/benchmarks/tpch/db2.yml b/benchmarks/tpch/db2.yml new file mode 100644 index 0000000..ea12920 --- /dev/null +++ b/benchmarks/tpch/db2.yml @@ -0,0 +1,82 @@ +--- +name: HammerDB Db2 TPC-H benchmark +author: Didi Hoffmann +description: Benchmarks IBM Db2 with HammerDB using TPC-H + +compose-file: !include compose.yml + +custom_metrics: + tpch_queries: # functional unit (work done): number of TPC-H queries completed + unit: TPC-H query + sci: true + +services: + postgres: # empty key will remove the unused service + mariadb: # empty key will remove the unused service + mysql: # empty key will remove the unused service + oracle: # empty key will remove the unused service + mssql: # empty key will remove the unused service + db2: # this benchmark needs the tpch database instead of the default tpcc + environment: + LICENSE: accept + DB2INST1_PASSWORD: ibmdb2 + DBNAME: tpch + hammerdb: # Db2-enabled image (build/push with ./db/db2/build-image.sh); wait for Db2 + image: ribalba/hammerdb-db2:latest + depends_on: + db2: + condition: service_healthy + +flow: + # Db2 is excluded from the paper (no BenchBase profile; Community-edition capped) and + # kept only for benchmarking, so enlarging its undersized default transaction log here + # is fine. Without this the bulk load fails with SQL0964C (log full). + - name: Size Db2 transaction log + container: db2_container + commands: + - type: console + note: ENLARGE TRANSACTION LOG before the bulk load (avoids SQL0964C "log full") + command: | + su -l db2inst1 -c ' + i=0; while [ $i -lt 40 ]; do db2 connect to tpch >/dev/null 2>&1 && break; i=$((i+1)); sleep 3; done + db2 connect reset + db2 update db cfg for tpch using LOGFILSIZ 8192 LOGPRIMARY 16 LOGSECOND 48 + db2 force application all + db2 deactivate db tpch + db2 get db cfg for tpch | grep -iE "Log file size|primary log|secondary log" + ' || true + shell: bash + log-stdout: true + - name: Catalog Db2 + container: hammerdb_container + commands: + - type: console + note: CATALOG REMOTE DB2 + command: chmod -R u+rwX /database/config/db2inst1/sqllib 2>/dev/null; rm -f /database/config/db2inst1/sqllib/db2nodes.cfg; echo "0 $(hostname) 0" > /database/config/db2inst1/sqllib/db2nodes.cfg && . /database/config/db2inst1/sqllib/db2profile && { db2 uncatalog database tpch; db2 uncatalog node db2node; true; } && db2 catalog tcpip node db2node remote db2_container server 50000 && db2 catalog database tpch at node db2node && db2 terminate && (i=0; while [ $i -lt 30 ]; do db2 connect to tpch >/dev/null 2>&1 && db2 connect reset >/dev/null 2>&1 && break; i=$((i+1)); sleep 3; done) + shell: sh + log-stdout: true + - name: Build Schema + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB BUILD + command: rc=0; . /database/config/db2inst1/sqllib/db2profile && cd /home/HammerDB-5.0 && ./hammerdbcli auto /tmp/repo/db/db2/tpch/db2_tproch_buildschema.tcl > /tmp/hammerdb-build.log 2>&1 || rc=$?; cat /tmp/hammerdb-build.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; if grep -qE 'FINISHED FAILED|Error in Virtual User' /tmp/hammerdb-build.log; then echo 'HammerDB reported virtual-user failures - see log above' >&2; exit 1; fi + shell: sh + log-stdout: true + - name: Run Queries + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB QUERIES + command: rc=0; . /database/config/db2inst1/sqllib/db2profile && cd /home/HammerDB-5.0 && ./hammerdbcli auto /tmp/repo/db/db2/tpch/db2_tproch_run.tcl > /tmp/hammerdb-run.log 2>&1 || rc=$?; cat /tmp/hammerdb-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; if grep -qE 'FINISHED FAILED|Error in Virtual User' /tmp/hammerdb-run.log; then echo 'HammerDB reported virtual-user failures - see log above' >&2; exit 1; fi; printf '%s000000 tpch_queries=22\n' "$(date +%s)" + shell: sh + log-stdout: true + read-sci-stdout: true + - name: Drop Schema + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB DROP + command: . /database/config/db2inst1/sqllib/db2profile && cd /home/HammerDB-5.0 && ./hammerdbcli auto /tmp/repo/db/db2/tpch/db2_tproch_deleteschema.tcl + shell: sh + log-stdout: true diff --git a/benchmarks/tpch/maria.yml b/benchmarks/tpch/maria.yml new file mode 100644 index 0000000..e569b93 --- /dev/null +++ b/benchmarks/tpch/maria.yml @@ -0,0 +1,49 @@ +--- +name: HammerDB MariaDB TPC-H benchmark +author: Didi Hoffmann +description: Benchmarks mariadb with HammerDB using TPC-H + +compose-file: !include compose.yml + +custom_metrics: + tpch_queries: # functional unit (work done): number of TPC-H queries completed + unit: TPC-H query + sci: true + +services: + postgres: # empty key will remove the unused service + mysql: # empty key will remove the unused service + oracle: # empty key will remove the unused service + mssql: # empty key will remove the unused service + db2: # empty key will remove the unused service + hammerdb: # wait for MariaDB to finish starting up before benchmarking + depends_on: + mariadb: + condition: service_healthy + +flow: + - name: Build Schema + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB BUILD + command: rc=0; ./hammerdbcli auto /tmp/repo/db/maria/tpch/maria_tproch_buildschema.tcl > /tmp/hammerdb-build.log 2>&1 || rc=$?; cat /tmp/hammerdb-build.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; if grep -qE 'FINISHED FAILED|Error in Virtual User' /tmp/hammerdb-build.log; then echo 'HammerDB reported virtual-user failures - see log above' >&2; exit 1; fi + shell: sh + log-stdout: true + - name: Run Queries + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB QUERIES + command: rc=0; ./hammerdbcli auto /tmp/repo/db/maria/tpch/maria_tproch_run.tcl > /tmp/hammerdb-run.log 2>&1 || rc=$?; cat /tmp/hammerdb-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; if grep -qE 'FINISHED FAILED|Error in Virtual User' /tmp/hammerdb-run.log; then echo 'HammerDB reported virtual-user failures - see log above' >&2; exit 1; fi; printf '%s000000 tpch_queries=22\n' "$(date +%s)" + shell: sh + log-stdout: true + read-sci-stdout: true + - name: Drop Schema + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB DROP + command: ./hammerdbcli auto /tmp/repo/db/maria/tpch/maria_tproch_deleteschema.tcl + shell: sh + log-stdout: true diff --git a/benchmarks/tpch/mssql.yml b/benchmarks/tpch/mssql.yml new file mode 100644 index 0000000..2d206d9 --- /dev/null +++ b/benchmarks/tpch/mssql.yml @@ -0,0 +1,49 @@ +--- +name: HammerDB MSSQL TPC-H benchmark +author: Didi Hoffmann +description: Benchmarks Microsoft SQL Server with HammerDB using TPC-H + +compose-file: !include compose.yml + +custom_metrics: + tpch_queries: # functional unit (work done): number of TPC-H queries completed + unit: TPC-H query + sci: true + +services: + postgres: # empty key will remove the unused service + mariadb: # empty key will remove the unused service + mysql: # empty key will remove the unused service + oracle: # empty key will remove the unused service + db2: # empty key will remove the unused service + hammerdb: # wait for SQL Server to finish starting up before benchmarking + depends_on: + mssql: + condition: service_healthy + +flow: + - name: Build Schema + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB BUILD + command: rc=0; ./hammerdbcli auto /tmp/repo/db/mssql/tpch/mssql_tproch_buildschema.tcl > /tmp/hammerdb-build.log 2>&1 || rc=$?; cat /tmp/hammerdb-build.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; if grep -qE 'FINISHED FAILED|Error in Virtual User' /tmp/hammerdb-build.log; then echo 'HammerDB reported virtual-user failures - see log above' >&2; exit 1; fi + shell: sh + log-stdout: true + - name: Run Queries + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB QUERIES + command: rc=0; ./hammerdbcli auto /tmp/repo/db/mssql/tpch/mssql_tproch_run.tcl > /tmp/hammerdb-run.log 2>&1 || rc=$?; cat /tmp/hammerdb-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; if grep -qE 'FINISHED FAILED|Error in Virtual User' /tmp/hammerdb-run.log; then echo 'HammerDB reported virtual-user failures - see log above' >&2; exit 1; fi; printf '%s000000 tpch_queries=22\n' "$(date +%s)" + shell: sh + log-stdout: true + read-sci-stdout: true + - name: Drop Schema + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB DROP + command: ./hammerdbcli auto /tmp/repo/db/mssql/tpch/mssql_tproch_deleteschema.tcl + shell: sh + log-stdout: true diff --git a/benchmarks/tpch/mysql.yml b/benchmarks/tpch/mysql.yml new file mode 100644 index 0000000..4819c13 --- /dev/null +++ b/benchmarks/tpch/mysql.yml @@ -0,0 +1,49 @@ +--- +name: HammerDB MySQL TPC-H benchmark +author: Didi Hoffmann +description: Benchmarks mysql with HammerDB using TPC-H + +compose-file: !include compose.yml + +custom_metrics: + tpch_queries: # functional unit (work done): number of TPC-H queries completed + unit: TPC-H query + sci: true + +services: + postgres: # empty key will remove the unused service + mariadb: # empty key will remove the unused service + oracle: # empty key will remove the unused service + mssql: # empty key will remove the unused service + db2: # empty key will remove the unused service + hammerdb: # wait for MySQL to finish starting up before benchmarking + depends_on: + mysql: + condition: service_healthy + +flow: + - name: Build Schema + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB BUILD + command: rc=0; ./hammerdbcli auto /tmp/repo/db/mysql/tpch/mysql_tproch_buildschema.tcl > /tmp/hammerdb-build.log 2>&1 || rc=$?; cat /tmp/hammerdb-build.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; if grep -qE 'FINISHED FAILED|Error in Virtual User' /tmp/hammerdb-build.log; then echo 'HammerDB reported virtual-user failures - see log above' >&2; exit 1; fi + shell: sh + log-stdout: true + - name: Run Queries + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB QUERIES + command: rc=0; ./hammerdbcli auto /tmp/repo/db/mysql/tpch/mysql_tproch_run.tcl > /tmp/hammerdb-run.log 2>&1 || rc=$?; cat /tmp/hammerdb-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; if grep -qE 'FINISHED FAILED|Error in Virtual User' /tmp/hammerdb-run.log; then echo 'HammerDB reported virtual-user failures - see log above' >&2; exit 1; fi; printf '%s000000 tpch_queries=22\n' "$(date +%s)" + shell: sh + log-stdout: true + read-sci-stdout: true + - name: Drop Schema + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB DROP + command: ./hammerdbcli auto /tmp/repo/db/mysql/tpch/mysql_tproch_deleteschema.tcl + shell: sh + log-stdout: true diff --git a/benchmarks/tpch/oracle.yml b/benchmarks/tpch/oracle.yml new file mode 100644 index 0000000..051e156 --- /dev/null +++ b/benchmarks/tpch/oracle.yml @@ -0,0 +1,53 @@ +--- +name: HammerDB Oracle TPC-H benchmark +author: Didi Hoffmann +description: Benchmarks Oracle Database Free with HammerDB using TPC-H + +compose-file: !include compose.yml + +custom_metrics: + tpch_queries: # functional unit (work done): number of TPC-H queries completed + unit: TPC-H query + sci: true + +services: + postgres: # empty key will remove the unused service + mariadb: # empty key will remove the unused service + mysql: # empty key will remove the unused service + mssql: # empty key will remove the unused service + db2: # empty key will remove the unused service + hammerdb: # wait for Oracle to finish starting up before benchmarking + depends_on: + oracle: + condition: service_healthy + # the Oracle Instant Client ships in the hammerdb image but its dir is not on + # the loader path, so libclntsh.so can't find libnnz21.so / libclntshcore.so + environment: + LD_LIBRARY_PATH: /home/instantclient_21_18 + +flow: + - name: Build Schema + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB BUILD + command: rc=0; ./hammerdbcli auto /tmp/repo/db/oracle/tpch/oracle_tproch_buildschema.tcl > /tmp/hammerdb-build.log 2>&1 || rc=$?; cat /tmp/hammerdb-build.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; if grep -qE 'FINISHED FAILED|Error in Virtual User' /tmp/hammerdb-build.log; then echo 'HammerDB reported virtual-user failures - see log above' >&2; exit 1; fi + shell: sh + log-stdout: true + - name: Run Queries + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB QUERIES + command: rc=0; ./hammerdbcli auto /tmp/repo/db/oracle/tpch/oracle_tproch_run.tcl > /tmp/hammerdb-run.log 2>&1 || rc=$?; cat /tmp/hammerdb-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; if grep -qE 'FINISHED FAILED|Error in Virtual User' /tmp/hammerdb-run.log; then echo 'HammerDB reported virtual-user failures - see log above' >&2; exit 1; fi; printf '%s000000 tpch_queries=22\n' "$(date +%s)" + shell: sh + log-stdout: true + read-sci-stdout: true + - name: Drop Schema + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB DROP + command: ./hammerdbcli auto /tmp/repo/db/oracle/tpch/oracle_tproch_deleteschema.tcl + shell: sh + log-stdout: true diff --git a/benchmarks/tpch/pg.yml b/benchmarks/tpch/pg.yml new file mode 100644 index 0000000..45622e1 --- /dev/null +++ b/benchmarks/tpch/pg.yml @@ -0,0 +1,49 @@ +--- +name: HammerDB Postgres TPC-H benchmark +author: Didi Hoffmann +description: Benchmarks postgresql with HammerDB using TPC-H + +compose-file: !include compose.yml + +custom_metrics: + tpch_queries: # functional unit (work done): number of TPC-H queries completed + unit: TPC-H query + sci: true + +services: + mariadb: # empty key will remove the unused service + mysql: # empty key will remove the unused service + oracle: # empty key will remove the unused service + mssql: # empty key will remove the unused service + db2: # empty key will remove the unused service + hammerdb: # wait for Postgres to finish starting up before benchmarking + depends_on: + postgres: + condition: service_healthy + +flow: + - name: Build Schema + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB BUILD + command: rc=0; ./hammerdbcli auto /tmp/repo/db/pg/tpch/pg_tproch_buildschema.tcl > /tmp/hammerdb-build.log 2>&1 || rc=$?; cat /tmp/hammerdb-build.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; if grep -qE 'FINISHED FAILED|Error in Virtual User' /tmp/hammerdb-build.log; then echo 'HammerDB reported virtual-user failures - see log above' >&2; exit 1; fi + shell: sh + log-stdout: true + - name: Run Queries + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB QUERIES + command: rc=0; ./hammerdbcli auto /tmp/repo/db/pg/tpch/pg_tproch_run.tcl > /tmp/hammerdb-run.log 2>&1 || rc=$?; cat /tmp/hammerdb-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; if grep -qE 'FINISHED FAILED|Error in Virtual User' /tmp/hammerdb-run.log; then echo 'HammerDB reported virtual-user failures - see log above' >&2; exit 1; fi; printf '%s000000 tpch_queries=22\n' "$(date +%s)" + shell: sh + log-stdout: true + read-sci-stdout: true + - name: Drop Schema + container: hammerdb_container + commands: + - type: console + note: RUN HAMMERDB DROP + command: ./hammerdbcli auto /tmp/repo/db/pg/tpch/pg_tproch_deleteschema.tcl + shell: sh + log-stdout: true diff --git a/benchmarks/wikipedia/compose.yml b/benchmarks/wikipedia/compose.yml new file mode 100644 index 0000000..b08deb8 --- /dev/null +++ b/benchmarks/wikipedia/compose.yml @@ -0,0 +1,138 @@ +services: + postgres: + image: postgres:18.4-trixie + container_name: postgres_container + # Equal CPU/memory budget for every engine so the comparison is fair. GMT + # honours the native compose `cpus`/`mem_limit` keys. Tune these to the + # benchmark host; only one DB container is exercised per scenario (plus the + # hammerdb driver, which is left unconstrained so it never bottlenecks the + # load generation). + cpus: 4 + mem_limit: 8g + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + # ports: + # - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + mariadb: + image: mariadb:12.3.2-noble + container_name: mariadb_container + cpus: 4 + mem_limit: 8g + environment: + MYSQL_ROOT_PASSWORD: maria + MYSQL_DATABASE: maria + MYSQL_USER: maria + MYSQL_PASSWORD: maria + # ports: + # - "3306:3306" + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + mysql: + # Pinned to the 8.4 LTS line on purpose — do not bump to 9.x. MySQL 9.0 removed + # mysql_native_password outright, and HammerDB 5.0's bundled mysqltcl has no + # TLS-free path for caching_sha2_password (no --get-server-public-key), so every + # virtual user fails to authenticate over plaintext TCP. The alternatives (TLS, or + # a Unix socket) would make MySQL the only engine not measured over plaintext TCP + # and so would bias the cross-engine energy comparison. + image: mysql:8.4.10-oraclelinux9 + command: ["--mysql-native-password=ON", "--authentication-policy=mysql_native_password"] + container_name: mysql_container + cpus: 4 + mem_limit: 8g + environment: + MYSQL_ROOT_PASSWORD: mysql + MYSQL_DATABASE: mysql + MYSQL_USER: mysql + MYSQL_PASSWORD: mysql + # ports: + # - "3307:3306" + healthcheck: + test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -uroot -pmysql --silent"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + oracle: + image: container-registry.oracle.com/database/free:23.26.2.0 + container_name: oracle_container + cpus: 4 + mem_limit: 8g + environment: + ORACLE_PWD: oracle + # ports: + # - "1521:1521" + healthcheck: + test: ["CMD-SHELL", "echo 'SELECT 1 FROM dual;' | sqlplus -s -L system/oracle@//localhost:1521/FREEPDB1 | grep -qw 1"] + interval: 15s + timeout: 10s + retries: 40 + start_period: 120s + mssql: + image: mcr.microsoft.com/mssql/server:2022-CU23-ubuntu-22.04 + container_name: mssql_container + cpus: 4 + mem_limit: 8g + environment: + ACCEPT_EULA: "Y" + MSSQL_SA_PASSWORD: Hammerdb_2024 + MSSQL_PID: Developer + # ports: + # - "1433:1433" + healthcheck: + test: ["CMD-SHELL", "timeout 1 bash -c '/dev/null 2>&1"] + interval: 15s + timeout: 10s + retries: 40 + start_period: 300s + +# Benchmark drivers + + hammerdb: + image: tpcorg/hammerdb:latest + container_name: hammerdb_container + + benchbase: + image: ribalba/benchbase:latest + container_name: benchbase_container diff --git a/benchmarks/wikipedia/maria.yml b/benchmarks/wikipedia/maria.yml new file mode 100644 index 0000000..810a63a --- /dev/null +++ b/benchmarks/wikipedia/maria.yml @@ -0,0 +1,50 @@ +--- +name: BenchBase MariaDB Wikipedia benchmark +author: Didi Hoffmann +description: Benchmarks MariaDB with BenchBase using the Wikipedia workload + +compose-file: !include compose.yml + +custom_metrics: + wikipedia_requests: # functional unit (work done): BenchBase requests completed in the measured run + unit: Wikipedia request + sci: true + +services: + postgres: # unused — empty key removes the service + mysql: # unused — empty key removes the service + oracle: # unused — empty key removes the service + mssql: # unused — empty key removes the service + db2: # unused — empty key removes the service + hammerdb: # unused — empty key removes the service + benchbase: # load driver; wait for the database to be healthy first + depends_on: + mariadb: + condition: service_healthy + +flow: + - name: Create benchbase database + container: mariadb_container + commands: + - type: console + note: CREATE BENCHBASE DATABASE + command: mariadb -uroot -pmaria -e "CREATE DATABASE IF NOT EXISTS benchbase; GRANT ALL ON benchbase.* TO 'maria'@'%'; FLUSH PRIVILEGES;" + shell: sh + log-stdout: true + - name: Load Wikipedia schema and data + container: benchbase_container + commands: + - type: console + note: BENCHBASE CREATE + LOAD + command: cd /benchbase/benchbase-mariadb && java -jar benchbase.jar -b wikipedia -c /tmp/repo/db/maria/wikipedia/maria_wikipedia_config.xml --create=true --load=true --execute=false + shell: sh + log-stdout: true + - name: Run Wikipedia workload + container: benchbase_container + commands: + - type: console + note: BENCHBASE EXECUTE + command: rc=0; cd /benchbase/benchbase-mariadb && java -jar benchbase.jar -b wikipedia -c /tmp/repo/db/maria/wikipedia/maria_wikipedia_config.xml --create=false --load=false --execute=true > /tmp/benchbase-run.log 2>&1 || rc=$?; cat /tmp/benchbase-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; awk 'match($0,/measuredRequests=[0-9]+/) {v=substr($0,RSTART+17,RLENGTH-17)} END {if (v=="") exit 1; "date +%s"|getline ts; printf("%s000000 wikipedia_requests=%s\n", ts, v)}' /tmp/benchbase-run.log + shell: sh + log-stdout: true + read-sci-stdout: true diff --git a/benchmarks/wikipedia/mssql.yml b/benchmarks/wikipedia/mssql.yml new file mode 100644 index 0000000..a9b461b --- /dev/null +++ b/benchmarks/wikipedia/mssql.yml @@ -0,0 +1,50 @@ +--- +name: BenchBase Microsoft SQL Server Wikipedia benchmark +author: Didi Hoffmann +description: Benchmarks Microsoft SQL Server with BenchBase using the Wikipedia workload + +compose-file: !include compose.yml + +custom_metrics: + wikipedia_requests: # functional unit (work done): BenchBase requests completed in the measured run + unit: Wikipedia request + sci: true + +services: + postgres: # unused — empty key removes the service + mariadb: # unused — empty key removes the service + mysql: # unused — empty key removes the service + oracle: # unused — empty key removes the service + db2: # unused — empty key removes the service + hammerdb: # unused — empty key removes the service + benchbase: # load driver; wait for the database to be healthy first + depends_on: + mssql: + condition: service_healthy + +flow: + - name: Create benchbase database + container: mssql_container + commands: + - type: console + note: CREATE BENCHBASE DATABASE + command: /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P Hammerdb_2024 -C -Q "IF DB_ID('benchbase') IS NULL CREATE DATABASE benchbase" + shell: sh + log-stdout: true + - name: Load Wikipedia schema and data + container: benchbase_container + commands: + - type: console + note: BENCHBASE CREATE + LOAD + command: cd /benchbase/benchbase-sqlserver && java -jar benchbase.jar -b wikipedia -c /tmp/repo/db/mssql/wikipedia/mssql_wikipedia_config.xml --create=true --load=true --execute=false + shell: sh + log-stdout: true + - name: Run Wikipedia workload + container: benchbase_container + commands: + - type: console + note: BENCHBASE EXECUTE + command: rc=0; cd /benchbase/benchbase-sqlserver && java -jar benchbase.jar -b wikipedia -c /tmp/repo/db/mssql/wikipedia/mssql_wikipedia_config.xml --create=false --load=false --execute=true > /tmp/benchbase-run.log 2>&1 || rc=$?; cat /tmp/benchbase-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; awk 'match($0,/measuredRequests=[0-9]+/) {v=substr($0,RSTART+17,RLENGTH-17)} END {if (v=="") exit 1; "date +%s"|getline ts; printf("%s000000 wikipedia_requests=%s\n", ts, v)}' /tmp/benchbase-run.log + shell: sh + log-stdout: true + read-sci-stdout: true diff --git a/benchmarks/wikipedia/mysql.yml b/benchmarks/wikipedia/mysql.yml new file mode 100644 index 0000000..91786c2 --- /dev/null +++ b/benchmarks/wikipedia/mysql.yml @@ -0,0 +1,50 @@ +--- +name: BenchBase MySQL Wikipedia benchmark +author: Didi Hoffmann +description: Benchmarks MySQL with BenchBase using the Wikipedia workload + +compose-file: !include compose.yml + +custom_metrics: + wikipedia_requests: # functional unit (work done): BenchBase requests completed in the measured run + unit: Wikipedia request + sci: true + +services: + postgres: # unused — empty key removes the service + mariadb: # unused — empty key removes the service + oracle: # unused — empty key removes the service + mssql: # unused — empty key removes the service + db2: # unused — empty key removes the service + hammerdb: # unused — empty key removes the service + benchbase: # load driver; wait for the database to be healthy first + depends_on: + mysql: + condition: service_healthy + +flow: + - name: Create benchbase database + container: mysql_container + commands: + - type: console + note: CREATE BENCHBASE DATABASE + command: mysql -uroot -pmysql -e "CREATE DATABASE IF NOT EXISTS benchbase; GRANT ALL ON benchbase.* TO 'mysql'@'%'; FLUSH PRIVILEGES;" + shell: sh + log-stdout: true + - name: Load Wikipedia schema and data + container: benchbase_container + commands: + - type: console + note: BENCHBASE CREATE + LOAD + command: cd /benchbase/benchbase-mysql && java -jar benchbase.jar -b wikipedia -c /tmp/repo/db/mysql/wikipedia/mysql_wikipedia_config.xml --create=true --load=true --execute=false + shell: sh + log-stdout: true + - name: Run Wikipedia workload + container: benchbase_container + commands: + - type: console + note: BENCHBASE EXECUTE + command: rc=0; cd /benchbase/benchbase-mysql && java -jar benchbase.jar -b wikipedia -c /tmp/repo/db/mysql/wikipedia/mysql_wikipedia_config.xml --create=false --load=false --execute=true > /tmp/benchbase-run.log 2>&1 || rc=$?; cat /tmp/benchbase-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; awk 'match($0,/measuredRequests=[0-9]+/) {v=substr($0,RSTART+17,RLENGTH-17)} END {if (v=="") exit 1; "date +%s"|getline ts; printf("%s000000 wikipedia_requests=%s\n", ts, v)}' /tmp/benchbase-run.log + shell: sh + log-stdout: true + read-sci-stdout: true diff --git a/benchmarks/wikipedia/oracle.yml b/benchmarks/wikipedia/oracle.yml new file mode 100644 index 0000000..9fa1b9f --- /dev/null +++ b/benchmarks/wikipedia/oracle.yml @@ -0,0 +1,50 @@ +--- +name: BenchBase Oracle Database Free Wikipedia benchmark +author: Didi Hoffmann +description: Benchmarks Oracle Database Free with BenchBase using the Wikipedia workload + +compose-file: !include compose.yml + +custom_metrics: + wikipedia_requests: # functional unit (work done): BenchBase requests completed in the measured run + unit: Wikipedia request + sci: true + +services: + postgres: # unused — empty key removes the service + mariadb: # unused — empty key removes the service + mysql: # unused — empty key removes the service + mssql: # unused — empty key removes the service + db2: # unused — empty key removes the service + hammerdb: # unused — empty key removes the service + benchbase: # load driver; wait for the database to be healthy first + depends_on: + oracle: + condition: service_healthy + +flow: + - name: Create benchbase database + container: oracle_container + commands: + - type: console + note: CREATE BENCHBASE DATABASE + command: printf '%s\n' "WHENEVER SQLERROR EXIT FAILURE" "CREATE USER benchbase IDENTIFIED BY benchbase DEFAULT TABLESPACE users QUOTA UNLIMITED ON users;" "GRANT CONNECT, RESOURCE, CREATE VIEW TO benchbase;" "EXIT;" | sqlplus -s system/oracle@//localhost:1521/FREEPDB1 + shell: sh + log-stdout: true + - name: Load Wikipedia schema and data + container: benchbase_container + commands: + - type: console + note: BENCHBASE CREATE + LOAD + command: cd /benchbase/benchbase-oracle && java -jar benchbase.jar -b wikipedia -c /tmp/repo/db/oracle/wikipedia/oracle_wikipedia_config.xml --create=true --load=true --execute=false + shell: sh + log-stdout: true + - name: Run Wikipedia workload + container: benchbase_container + commands: + - type: console + note: BENCHBASE EXECUTE + command: rc=0; cd /benchbase/benchbase-oracle && java -jar benchbase.jar -b wikipedia -c /tmp/repo/db/oracle/wikipedia/oracle_wikipedia_config.xml --create=false --load=false --execute=true > /tmp/benchbase-run.log 2>&1 || rc=$?; cat /tmp/benchbase-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; awk 'match($0,/measuredRequests=[0-9]+/) {v=substr($0,RSTART+17,RLENGTH-17)} END {if (v=="") exit 1; "date +%s"|getline ts; printf("%s000000 wikipedia_requests=%s\n", ts, v)}' /tmp/benchbase-run.log + shell: sh + log-stdout: true + read-sci-stdout: true diff --git a/benchmarks/wikipedia/pg.yml b/benchmarks/wikipedia/pg.yml new file mode 100644 index 0000000..a9c19a9 --- /dev/null +++ b/benchmarks/wikipedia/pg.yml @@ -0,0 +1,50 @@ +--- +name: BenchBase PostgreSQL Wikipedia benchmark +author: Didi Hoffmann +description: Benchmarks PostgreSQL with BenchBase using the Wikipedia workload + +compose-file: !include compose.yml + +custom_metrics: + wikipedia_requests: # functional unit (work done): BenchBase requests completed in the measured run + unit: Wikipedia request + sci: true + +services: + mariadb: # unused — empty key removes the service + mysql: # unused — empty key removes the service + oracle: # unused — empty key removes the service + mssql: # unused — empty key removes the service + db2: # unused — empty key removes the service + hammerdb: # unused — empty key removes the service + benchbase: # load driver; wait for the database to be healthy first + depends_on: + postgres: + condition: service_healthy + +flow: + - name: Create benchbase database + container: postgres_container + commands: + - type: console + note: CREATE BENCHBASE DATABASE + command: psql "postgresql://postgres:postgres@localhost:5432/postgres" -tc "SELECT 1 FROM pg_database WHERE datname='benchbase'" | grep -q 1 || psql "postgresql://postgres:postgres@localhost:5432/postgres" -c "CREATE DATABASE benchbase" + shell: sh + log-stdout: true + - name: Load Wikipedia schema and data + container: benchbase_container + commands: + - type: console + note: BENCHBASE CREATE + LOAD + command: cd /benchbase/benchbase-postgres && java -jar benchbase.jar -b wikipedia -c /tmp/repo/db/pg/wikipedia/pg_wikipedia_config.xml --create=true --load=true --execute=false + shell: sh + log-stdout: true + - name: Run Wikipedia workload + container: benchbase_container + commands: + - type: console + note: BENCHBASE EXECUTE + command: rc=0; cd /benchbase/benchbase-postgres && java -jar benchbase.jar -b wikipedia -c /tmp/repo/db/pg/wikipedia/pg_wikipedia_config.xml --create=false --load=false --execute=true > /tmp/benchbase-run.log 2>&1 || rc=$?; cat /tmp/benchbase-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; awk 'match($0,/measuredRequests=[0-9]+/) {v=substr($0,RSTART+17,RLENGTH-17)} END {if (v=="") exit 1; "date +%s"|getline ts; printf("%s000000 wikipedia_requests=%s\n", ts, v)}' /tmp/benchbase-run.log + shell: sh + log-stdout: true + read-sci-stdout: true diff --git a/benchmarks/ycsb/compose.yml b/benchmarks/ycsb/compose.yml new file mode 100644 index 0000000..b08deb8 --- /dev/null +++ b/benchmarks/ycsb/compose.yml @@ -0,0 +1,138 @@ +services: + postgres: + image: postgres:18.4-trixie + container_name: postgres_container + # Equal CPU/memory budget for every engine so the comparison is fair. GMT + # honours the native compose `cpus`/`mem_limit` keys. Tune these to the + # benchmark host; only one DB container is exercised per scenario (plus the + # hammerdb driver, which is left unconstrained so it never bottlenecks the + # load generation). + cpus: 4 + mem_limit: 8g + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + # ports: + # - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d postgres"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + mariadb: + image: mariadb:12.3.2-noble + container_name: mariadb_container + cpus: 4 + mem_limit: 8g + environment: + MYSQL_ROOT_PASSWORD: maria + MYSQL_DATABASE: maria + MYSQL_USER: maria + MYSQL_PASSWORD: maria + # ports: + # - "3306:3306" + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + mysql: + # Pinned to the 8.4 LTS line on purpose — do not bump to 9.x. MySQL 9.0 removed + # mysql_native_password outright, and HammerDB 5.0's bundled mysqltcl has no + # TLS-free path for caching_sha2_password (no --get-server-public-key), so every + # virtual user fails to authenticate over plaintext TCP. The alternatives (TLS, or + # a Unix socket) would make MySQL the only engine not measured over plaintext TCP + # and so would bias the cross-engine energy comparison. + image: mysql:8.4.10-oraclelinux9 + command: ["--mysql-native-password=ON", "--authentication-policy=mysql_native_password"] + container_name: mysql_container + cpus: 4 + mem_limit: 8g + environment: + MYSQL_ROOT_PASSWORD: mysql + MYSQL_DATABASE: mysql + MYSQL_USER: mysql + MYSQL_PASSWORD: mysql + # ports: + # - "3307:3306" + healthcheck: + test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -uroot -pmysql --silent"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + oracle: + image: container-registry.oracle.com/database/free:23.26.2.0 + container_name: oracle_container + cpus: 4 + mem_limit: 8g + environment: + ORACLE_PWD: oracle + # ports: + # - "1521:1521" + healthcheck: + test: ["CMD-SHELL", "echo 'SELECT 1 FROM dual;' | sqlplus -s -L system/oracle@//localhost:1521/FREEPDB1 | grep -qw 1"] + interval: 15s + timeout: 10s + retries: 40 + start_period: 120s + mssql: + image: mcr.microsoft.com/mssql/server:2022-CU23-ubuntu-22.04 + container_name: mssql_container + cpus: 4 + mem_limit: 8g + environment: + ACCEPT_EULA: "Y" + MSSQL_SA_PASSWORD: Hammerdb_2024 + MSSQL_PID: Developer + # ports: + # - "1433:1433" + healthcheck: + test: ["CMD-SHELL", "timeout 1 bash -c '/dev/null 2>&1"] + interval: 15s + timeout: 10s + retries: 40 + start_period: 300s + +# Benchmark drivers + + hammerdb: + image: tpcorg/hammerdb:latest + container_name: hammerdb_container + + benchbase: + image: ribalba/benchbase:latest + container_name: benchbase_container diff --git a/benchmarks/ycsb/maria.yml b/benchmarks/ycsb/maria.yml new file mode 100644 index 0000000..12b0e88 --- /dev/null +++ b/benchmarks/ycsb/maria.yml @@ -0,0 +1,50 @@ +--- +name: BenchBase MariaDB YCSB benchmark +author: Didi Hoffmann +description: Benchmarks MariaDB with BenchBase using the YCSB workload + +compose-file: !include compose.yml + +custom_metrics: + ycsb_requests: # functional unit (work done): BenchBase requests completed in the measured run + unit: YCSB request + sci: true + +services: + postgres: # unused — empty key removes the service + mysql: # unused — empty key removes the service + oracle: # unused — empty key removes the service + mssql: # unused — empty key removes the service + db2: # unused — empty key removes the service + hammerdb: # unused — empty key removes the service + benchbase: # load driver; wait for the database to be healthy first + depends_on: + mariadb: + condition: service_healthy + +flow: + - name: Create benchbase database + container: mariadb_container + commands: + - type: console + note: CREATE BENCHBASE DATABASE + command: mariadb -uroot -pmaria -e "CREATE DATABASE IF NOT EXISTS benchbase; GRANT ALL ON benchbase.* TO 'maria'@'%'; FLUSH PRIVILEGES;" + shell: sh + log-stdout: true + - name: Load YCSB schema and data + container: benchbase_container + commands: + - type: console + note: BENCHBASE CREATE + LOAD + command: cd /benchbase/benchbase-mariadb && java -jar benchbase.jar -b ycsb -c /tmp/repo/db/maria/ycsb/maria_ycsb_config.xml --create=true --load=true --execute=false + shell: sh + log-stdout: true + - name: Run YCSB workload + container: benchbase_container + commands: + - type: console + note: BENCHBASE EXECUTE + command: rc=0; cd /benchbase/benchbase-mariadb && java -jar benchbase.jar -b ycsb -c /tmp/repo/db/maria/ycsb/maria_ycsb_config.xml --create=false --load=false --execute=true > /tmp/benchbase-run.log 2>&1 || rc=$?; cat /tmp/benchbase-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; awk 'match($0,/measuredRequests=[0-9]+/) {v=substr($0,RSTART+17,RLENGTH-17)} END {if (v=="") exit 1; "date +%s"|getline ts; printf("%s000000 ycsb_requests=%s\n", ts, v)}' /tmp/benchbase-run.log + shell: sh + log-stdout: true + read-sci-stdout: true diff --git a/benchmarks/ycsb/mssql.yml b/benchmarks/ycsb/mssql.yml new file mode 100644 index 0000000..a702977 --- /dev/null +++ b/benchmarks/ycsb/mssql.yml @@ -0,0 +1,50 @@ +--- +name: BenchBase Microsoft SQL Server YCSB benchmark +author: Didi Hoffmann +description: Benchmarks Microsoft SQL Server with BenchBase using the YCSB workload + +compose-file: !include compose.yml + +custom_metrics: + ycsb_requests: # functional unit (work done): BenchBase requests completed in the measured run + unit: YCSB request + sci: true + +services: + postgres: # unused — empty key removes the service + mariadb: # unused — empty key removes the service + mysql: # unused — empty key removes the service + oracle: # unused — empty key removes the service + db2: # unused — empty key removes the service + hammerdb: # unused — empty key removes the service + benchbase: # load driver; wait for the database to be healthy first + depends_on: + mssql: + condition: service_healthy + +flow: + - name: Create benchbase database + container: mssql_container + commands: + - type: console + note: CREATE BENCHBASE DATABASE + command: /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P Hammerdb_2024 -C -Q "IF DB_ID('benchbase') IS NULL CREATE DATABASE benchbase" + shell: sh + log-stdout: true + - name: Load YCSB schema and data + container: benchbase_container + commands: + - type: console + note: BENCHBASE CREATE + LOAD + command: cd /benchbase/benchbase-sqlserver && java -jar benchbase.jar -b ycsb -c /tmp/repo/db/mssql/ycsb/mssql_ycsb_config.xml --create=true --load=true --execute=false + shell: sh + log-stdout: true + - name: Run YCSB workload + container: benchbase_container + commands: + - type: console + note: BENCHBASE EXECUTE + command: rc=0; cd /benchbase/benchbase-sqlserver && java -jar benchbase.jar -b ycsb -c /tmp/repo/db/mssql/ycsb/mssql_ycsb_config.xml --create=false --load=false --execute=true > /tmp/benchbase-run.log 2>&1 || rc=$?; cat /tmp/benchbase-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; awk 'match($0,/measuredRequests=[0-9]+/) {v=substr($0,RSTART+17,RLENGTH-17)} END {if (v=="") exit 1; "date +%s"|getline ts; printf("%s000000 ycsb_requests=%s\n", ts, v)}' /tmp/benchbase-run.log + shell: sh + log-stdout: true + read-sci-stdout: true diff --git a/benchmarks/ycsb/mysql.yml b/benchmarks/ycsb/mysql.yml new file mode 100644 index 0000000..ba0c2bc --- /dev/null +++ b/benchmarks/ycsb/mysql.yml @@ -0,0 +1,50 @@ +--- +name: BenchBase MySQL YCSB benchmark +author: Didi Hoffmann +description: Benchmarks MySQL with BenchBase using the YCSB workload + +compose-file: !include compose.yml + +custom_metrics: + ycsb_requests: # functional unit (work done): BenchBase requests completed in the measured run + unit: YCSB request + sci: true + +services: + postgres: # unused — empty key removes the service + mariadb: # unused — empty key removes the service + oracle: # unused — empty key removes the service + mssql: # unused — empty key removes the service + db2: # unused — empty key removes the service + hammerdb: # unused — empty key removes the service + benchbase: # load driver; wait for the database to be healthy first + depends_on: + mysql: + condition: service_healthy + +flow: + - name: Create benchbase database + container: mysql_container + commands: + - type: console + note: CREATE BENCHBASE DATABASE + command: mysql -uroot -pmysql -e "CREATE DATABASE IF NOT EXISTS benchbase; GRANT ALL ON benchbase.* TO 'mysql'@'%'; FLUSH PRIVILEGES;" + shell: sh + log-stdout: true + - name: Load YCSB schema and data + container: benchbase_container + commands: + - type: console + note: BENCHBASE CREATE + LOAD + command: cd /benchbase/benchbase-mysql && java -jar benchbase.jar -b ycsb -c /tmp/repo/db/mysql/ycsb/mysql_ycsb_config.xml --create=true --load=true --execute=false + shell: sh + log-stdout: true + - name: Run YCSB workload + container: benchbase_container + commands: + - type: console + note: BENCHBASE EXECUTE + command: rc=0; cd /benchbase/benchbase-mysql && java -jar benchbase.jar -b ycsb -c /tmp/repo/db/mysql/ycsb/mysql_ycsb_config.xml --create=false --load=false --execute=true > /tmp/benchbase-run.log 2>&1 || rc=$?; cat /tmp/benchbase-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; awk 'match($0,/measuredRequests=[0-9]+/) {v=substr($0,RSTART+17,RLENGTH-17)} END {if (v=="") exit 1; "date +%s"|getline ts; printf("%s000000 ycsb_requests=%s\n", ts, v)}' /tmp/benchbase-run.log + shell: sh + log-stdout: true + read-sci-stdout: true diff --git a/benchmarks/ycsb/oracle.yml b/benchmarks/ycsb/oracle.yml new file mode 100644 index 0000000..88a91c2 --- /dev/null +++ b/benchmarks/ycsb/oracle.yml @@ -0,0 +1,50 @@ +--- +name: BenchBase Oracle Database Free YCSB benchmark +author: Didi Hoffmann +description: Benchmarks Oracle Database Free with BenchBase using the YCSB workload + +compose-file: !include compose.yml + +custom_metrics: + ycsb_requests: # functional unit (work done): BenchBase requests completed in the measured run + unit: YCSB request + sci: true + +services: + postgres: # unused — empty key removes the service + mariadb: # unused — empty key removes the service + mysql: # unused — empty key removes the service + mssql: # unused — empty key removes the service + db2: # unused — empty key removes the service + hammerdb: # unused — empty key removes the service + benchbase: # load driver; wait for the database to be healthy first + depends_on: + oracle: + condition: service_healthy + +flow: + - name: Create benchbase database + container: oracle_container + commands: + - type: console + note: CREATE BENCHBASE DATABASE + command: printf '%s\n' "WHENEVER SQLERROR EXIT FAILURE" "CREATE USER benchbase IDENTIFIED BY benchbase DEFAULT TABLESPACE users QUOTA UNLIMITED ON users;" "GRANT CONNECT, RESOURCE, CREATE VIEW TO benchbase;" "EXIT;" | sqlplus -s system/oracle@//localhost:1521/FREEPDB1 + shell: sh + log-stdout: true + - name: Load YCSB schema and data + container: benchbase_container + commands: + - type: console + note: BENCHBASE CREATE + LOAD + command: cd /benchbase/benchbase-oracle && java -jar benchbase.jar -b ycsb -c /tmp/repo/db/oracle/ycsb/oracle_ycsb_config.xml --create=true --load=true --execute=false + shell: sh + log-stdout: true + - name: Run YCSB workload + container: benchbase_container + commands: + - type: console + note: BENCHBASE EXECUTE + command: rc=0; cd /benchbase/benchbase-oracle && java -jar benchbase.jar -b ycsb -c /tmp/repo/db/oracle/ycsb/oracle_ycsb_config.xml --create=false --load=false --execute=true > /tmp/benchbase-run.log 2>&1 || rc=$?; cat /tmp/benchbase-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; awk 'match($0,/measuredRequests=[0-9]+/) {v=substr($0,RSTART+17,RLENGTH-17)} END {if (v=="") exit 1; "date +%s"|getline ts; printf("%s000000 ycsb_requests=%s\n", ts, v)}' /tmp/benchbase-run.log + shell: sh + log-stdout: true + read-sci-stdout: true diff --git a/benchmarks/ycsb/pg.yml b/benchmarks/ycsb/pg.yml new file mode 100644 index 0000000..05b37e5 --- /dev/null +++ b/benchmarks/ycsb/pg.yml @@ -0,0 +1,50 @@ +--- +name: BenchBase PostgreSQL YCSB benchmark +author: Didi Hoffmann +description: Benchmarks PostgreSQL with BenchBase using the YCSB workload + +compose-file: !include compose.yml + +custom_metrics: + ycsb_requests: # functional unit (work done): BenchBase requests completed in the measured run + unit: YCSB request + sci: true + +services: + mariadb: # unused — empty key removes the service + mysql: # unused — empty key removes the service + oracle: # unused — empty key removes the service + mssql: # unused — empty key removes the service + db2: # unused — empty key removes the service + hammerdb: # unused — empty key removes the service + benchbase: # load driver; wait for the database to be healthy first + depends_on: + postgres: + condition: service_healthy + +flow: + - name: Create benchbase database + container: postgres_container + commands: + - type: console + note: CREATE BENCHBASE DATABASE + command: psql "postgresql://postgres:postgres@localhost:5432/postgres" -tc "SELECT 1 FROM pg_database WHERE datname='benchbase'" | grep -q 1 || psql "postgresql://postgres:postgres@localhost:5432/postgres" -c "CREATE DATABASE benchbase" + shell: sh + log-stdout: true + - name: Load YCSB schema and data + container: benchbase_container + commands: + - type: console + note: BENCHBASE CREATE + LOAD + command: cd /benchbase/benchbase-postgres && java -jar benchbase.jar -b ycsb -c /tmp/repo/db/pg/ycsb/pg_ycsb_config.xml --create=true --load=true --execute=false + shell: sh + log-stdout: true + - name: Run YCSB workload + container: benchbase_container + commands: + - type: console + note: BENCHBASE EXECUTE + command: rc=0; cd /benchbase/benchbase-postgres && java -jar benchbase.jar -b ycsb -c /tmp/repo/db/pg/ycsb/pg_ycsb_config.xml --create=false --load=false --execute=true > /tmp/benchbase-run.log 2>&1 || rc=$?; cat /tmp/benchbase-run.log >&2; [ "$rc" -eq 0 ] || exit "$rc"; awk 'match($0,/measuredRequests=[0-9]+/) {v=substr($0,RSTART+17,RLENGTH-17)} END {if (v=="") exit 1; "date +%s"|getline ts; printf("%s000000 ycsb_requests=%s\n", ts, v)}' /tmp/benchbase-run.log + shell: sh + log-stdout: true + read-sci-stdout: true diff --git a/check_repo.py b/check_repo.py new file mode 100755 index 0000000..3551b34 --- /dev/null +++ b/check_repo.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python3 +""" +check_repo.py — Consistency checks for the DBMS-bench repo. + +The whole point of this repo is a *fair* comparison between database engines: +every engine has to run with the same resource budget and the same benchmark +parameters, otherwise the energy/throughput numbers are not comparable. + +Because GMT refuses to `!include` a compose file from a parent directory, each +benchmark folder (``benchmarks/tpcc/``, ``benchmarks/tpch/``, ...) carries its +own copy of ``compose.yml`` next to the usage scenarios. Copies drift, so this +script enforces three invariants: + + 1. Every ``compose.yml`` in the repo is byte-for-byte identical (the per-folder + copies must match the root source of truth). + 2. Inside ``compose.yml`` every database service gets the same resource budget + (``cpus`` and ``mem_limit``) — the load drivers are intentionally + unconstrained and are skipped. + 3. For each benchmark, the per-engine driver scripts use the same fairness + knobs (virtual users, scale factor, terminals, duration, ...) across all + databases. + +Exit code is 0 when everything is consistent, 1 otherwise. Run it from anywhere; +paths are resolved relative to this file. +""" + +from __future__ import annotations + +import re +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +REPO = Path(__file__).resolve().parent + +# Database engines that may take part in a benchmark, and the benchmark folders +# that hold the GMT usage scenarios + the compose.yml copy. +DATABASES = ["pg", "maria", "mysql", "oracle", "mssql", "db2"] +BENCHMARKS = ["tpcc", "tpch", "wikipedia", "ycsb", "chbenchmark"] + +# Driver scripts live at db///. HammerDB benchmarks are configured in +# .tcl scripts, BenchBase benchmarks in an .xml config. +HAMMERDB_BENCHMARKS = {"tpcc", "tpch"} +BENCHBASE_BENCHMARKS = {"wikipedia", "ycsb", "chbenchmark"} + +# Fairness knobs to extract from the HammerDB .tcl scripts, by benchmark. The +# values carry an engine-specific prefix in the scripts (pg_, maria_, mssqls_, +# ...; Oracle has none), so we match on the canonical suffix only. We compare a +# curated allow-list rather than every `diset` so engine-specific tuning that +# does not affect the comparison (connection strings, storage engines, ...) is +# ignored. +TCL_KNOBS = { + "tpcc": { + "vu": r"\bset\s+vu\s+(\d+)", + "vuset_vu": r"\bvuset\s+vu\s+(\d+)", + # A leading [\s_] absorbs the engine prefix (pg_rampup, maria_rampup, ...) + # as well as the bare Oracle form (`diset tpcc rampup 2`). + "rampup": r"[\s_]rampup\s+(\d+)", + "duration": r"[\s_]duration\s+(\d+)", + }, + "tpch": { + "scale_fact": r"scale_fact\s+(\d+)", + "num_tpch_threads": r"num_tpch_threads\s+(\d+)", + "total_querysets": r"total_querysets\s+(\d+)", + # Per-query intra-query parallelism. Each engine names this knob + # differently in HammerDB: pg_/oracle's `degree_of_parallel`, and + # MSSQL's `mssqls_maxdop` (MAXDOP) — the same fairness setting, so we + # match either. maria/mysql/db2 have no such HammerDB knob (no parallel + # query control there), so they legitimately never set it. + "degree_of_parallel": r"(?:degree_of_parallel|mssqls_maxdop)\s+(\d+)", + }, +} + +# Fairness knobs to extract from the BenchBase .xml configs.