Add some Dacapo benchmarks to be run on PRs (#6911)

Author: manuel-alvarez-alvarez

.gitignore

@@ -73,3 +73,4 @@ upstream.env
 # Benchmarks #
 benchmark/reports
 benchmark/tracer
+benchmark/dacapo/scratch

.gitlab/benchmarks.yml

@@ -42,6 +42,14 @@ benchmarks-load:
     - ./steps/run-benchmarks.sh load
     - ./steps/analyze-results.sh load
 
+benchmarks-dacapo:
+  extends: .benchmarks
+  script:
+    - !reference [ .benchmarks, script ]
+    - ./steps/capture-hardware-software-info.sh
+    - ./steps/run-benchmarks.sh dacapo
+    - ./steps/analyze-results.sh dacapo
+
 benchmarks-post-results:
   extends: .benchmarks
   script:
@@ -53,6 +61,8 @@ benchmarks-post-results:
       artifacts: true
     - job: benchmarks-load
       artifacts: true
+    - job: benchmarks-dacapo
+      artifacts: true
 
 .dsm-kafka-benchmarks:
   stage: benchmarks

benchmark/Dockerfile

@@ -31,6 +31,22 @@ RUN cd insecure-bank \
   && ./gradlew bootWar \
   && cp build/libs/*.war /insecure-bank.war
 
+# Dacapo download
+FROM debian:bookworm-slim as dacapo
+RUN apt-get update \
+  && apt-get -y install wget unzip \
+  && apt-get -y clean \
+  && rm -rf /var/lib/apt/lists/*
+
+ARG DACAPO_VERSION=23.11-chopin
+# The data for the big benchmarks is removed too ensure the final docker image is not too big
+RUN wget -nv -O dacapo.zip https://download.dacapobench.org/chopin/dacapo-$DACAPO_VERSION.zip \
+	&& mkdir /dacapo \
+	&& unzip dacapo.zip -d /dacapo/ \
+	&& rm -rf /dacapo/dacapo-$DACAPO_VERSION/dat/luindex \
+	&& rm -rf /dacapo/dacapo-$DACAPO_VERSION/dat/lusearch \
+	&& rm -rf /dacapo/dacapo-$DACAPO_VERSION/dat/graphchi \
+	&& rm dacapo.zip
 
 FROM debian:bookworm-slim
 
@@ -75,3 +91,7 @@ ENV PETCLINIC=/app/spring-petclinic.jar
 
 COPY --from=insecure-bank /insecure-bank.war /app/insecure-bank.war
 ENV INSECURE_BANK=/app/insecure-bank.war
+
+COPY --from=dacapo /dacapo/ /app/
+ARG DACAPO_VERSION=23.11-chopin
+ENV DACAPO=/app/dacapo-$DACAPO_VERSION.jar

benchmark/benchmarks.sh

@@ -38,7 +38,7 @@ rm -rf "${REPORTS_DIR}"
 mkdir -p "${REPORTS_DIR}"
 
 if [[ "$#" == '0' ]]; then
-  for type in 'startup' 'load'; do
+  for type in 'startup' 'load' 'dacapo'; do
     run_benchmarks "$type"
   done
 else

benchmark/dacapo/benchmark.json

@@ -0,0 +1,45 @@
+{
+  "name": "dacapo_${BENCHMARK}",
+  "setup": "bash -c \"mkdir -p ${OUTPUT_DIR}/${VARIANT}\"",
+  "run": "bash -c \"java ${JAVA_OPTS} -jar ${DACAPO} --converge --scratch-directory=${OUTPUT_DIR}/${VARIANT}/scratch --latency-csv ${BENCHMARK} &> ${OUTPUT_DIR}/${VARIANT}/dacapo.log\"",
+  "timeout": 150,
+  "iterations": 1,
+  "variants": {
+    "${NO_AGENT_VARIANT}": {
+      "env": {
+        "VARIANT": "${NO_AGENT_VARIANT}",
+        "JAVA_OPTS": ""
+      }
+    },
+    "tracing": {
+      "env": {
+        "VARIANT": "tracing",
+        "JAVA_OPTS": "-javaagent:${TRACER}"
+      }
+    },
+    "profiling": {
+      "env": {
+        "VARIANT": "profiling",
+        "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.profiling.enabled=true"
+      }
+    },
+    "appsec": {
+      "env": {
+        "VARIANT": "appsec",
+        "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.appsec.enabled=true -Ddd.iast.enabled=false"
+      }
+    },
+    "iast": {
+      "env": {
+        "VARIANT": "iast",
+        "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.iast.enabled=true"
+      }
+    },
+    "iast_GLOBAL": {
+      "env": {
+        "VARIANT": "iast_GLOBAL",
+        "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.context.mode=GLOBAL"
+      }
+    }
+  }
+}

benchmark/dacapo/run.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+set -eu
+
+source "${UTILS_DIR}/update-java-version.sh" 11
+
+function message() {
+  echo "$(date +"%T"): $1"
+}
+
+run_benchmark() {
+  local type=$1
+
+  message "dacapo benchmark: ${type} started"
+
+  # export the benchmark
+  export BENCHMARK="${type}"
+
+  # create output folder for the test
+  export OUTPUT_DIR="${REPORTS_DIR}/dacapo/${type}"
+  mkdir -p "${OUTPUT_DIR}"
+
+  # substitute environment variables in the json file
+  benchmark=$(mktemp)
+  # shellcheck disable=SC2046
+  # shellcheck disable=SC2016
+  envsubst "$(printf '${%s} ' $(env | cut -d'=' -f1))" <benchmark.json >"${benchmark}"
+
+  # run the sirun test
+  sirun "${benchmark}" &>"${OUTPUT_DIR}/${type}.json"
+
+  message "dacapo benchmark: ${type} finished"
+}
+
+if [ "$#" == '2' ]; then
+  run_benchmark "$2"
+else
+  for benchmark in biojava tomcat ; do
+    run_benchmark "${benchmark}"
+  done
+fi
+

benchmark/startup/petclinic/benchmark.json

@@ -27,7 +27,7 @@
     "appsec_no_iast": {
       "env": {
         "VARIANT": "appsec",
-        "JAVA_OPTS": "-javaagent:${TRACER} -Ddd.benchmark.enabled=true -Ddd.benchmark.output.dir=${OUTPUT_DIR}/appsec_no_iast -Ddd.appsec.enabled=true -Ddd.iast.enabled=false"
+        "JAVA_OPTS": "-Ddd.appsec.enabled=true -Ddd.iast.enabled=false"
       }
     },
     "iast": {