Skip to content

Commit 901a12d

Browse files
ddyurchenkoddyurchenko
authored
Fix benchmark failures and improve results quality (#8962)
Fix benchmark failures and improve results quality
1 parent 72f45a8 commit 901a12d

11 files changed

Lines changed: 242 additions & 120 deletions

File tree

‎.gitlab/benchmarks.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
script:
1616
- export ARTIFACTS_DIR="$(pwd)/reports" && mkdir -p "${ARTIFACTS_DIR}"
1717
- git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
18-
- git clone --branch dd-trace-java/tracer-benchmarks https://github.com/DataDog/benchmarking-platform.git /platform && cd /platform
18+
- git clone --branch dd-trace-java/tracer-benchmarks-parallel https://github.com/DataDog/benchmarking-platform.git /platform && cd /platform
1919
artifacts:
2020
name: "reports"
2121
paths:

‎benchmark/benchmarks.sh‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,6 @@ if [[ ! -f "${TRACER}" ]]; then
3333
cd "${SCRIPT_DIR}"
3434
fi
3535

36-
# Cleanup previous reports
37-
rm -rf "${REPORTS_DIR}"
38-
mkdir -p "${REPORTS_DIR}"
39-
4036
if [[ "$#" == '0' ]]; then
4137
for type in 'startup' 'load' 'dacapo'; do
4238
run_benchmarks "$type"

‎benchmark/load/insecure-bank/benchmark.json‎

Lines changed: 0 additions & 40 deletions
This file was deleted.

‎benchmark/load/insecure-bank/k6.js‎

Lines changed: 53 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,62 @@
11
import http from 'k6/http';
22
import {checkResponse, isOk, isRedirect} from "../../utils/k6.js";
33

4-
const baseUrl = 'http://localhost:8080';
4+
const variants = {
5+
"no_agent": {
6+
"APP_URL": 'http://localhost:8080',
7+
},
8+
"tracing": {
9+
"APP_URL": 'http://localhost:8081',
10+
},
11+
"profiling": {
12+
"APP_URL": 'http://localhost:8082',
13+
},
14+
"iast": {
15+
"APP_URL": 'http://localhost:8083',
16+
},
17+
"iast_GLOBAL": {
18+
"APP_URL": 'http://localhost:8084',
19+
},
20+
"iast_FULL": {
21+
"APP_URL": 'http://localhost:8085',
22+
},
23+
}
24+
25+
export const options = function (variants) {
26+
let scenarios = {};
27+
for (const variant of Object.keys(variants)) {
28+
scenarios[`load--insecure-bank--${variant}--warmup`] = {
29+
executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors
30+
vus: 5,
31+
duration: '20s',
32+
gracefulStop: '2s',
33+
env: {
34+
"APP_URL": variants[variant]["APP_URL"]
35+
}
36+
};
37+
38+
scenarios[`load--insecure-bank--${variant}--high_load`] = {
39+
executor: 'constant-vus',
40+
vus: 5,
41+
startTime: '22s',
42+
duration: '15s',
43+
gracefulStop: '2s',
44+
env: {
45+
"APP_URL": variants[variant]["APP_URL"]
46+
}
47+
};
48+
}
549

6-
export const options = {
7-
discardResponseBodies: true,
8-
vus: 5,
9-
iterations: 40000
10-
};
50+
return {
51+
discardResponseBodies: true,
52+
scenarios,
53+
}
54+
}(variants);
1155

1256
export default function () {
1357

1458
// login form
15-
const loginResponse = http.post(`${baseUrl}/login`, {
59+
const loginResponse = http.post(`${__ENV.APP_URL}/login`, {
1660
username: 'john',
1761
password: 'test'
1862
}, {
@@ -21,11 +65,11 @@ export default function () {
2165
checkResponse(loginResponse, isRedirect);
2266

2367
// dashboard
24-
const dashboard = http.get(`${baseUrl}/dashboard`);
68+
const dashboard = http.get(`${__ENV.APP_URL}/dashboard`);
2569
checkResponse(dashboard, isOk);
2670

2771
// logout
28-
const logout = http.get(`${baseUrl}/j_spring_security_logout`, {
72+
const logout = http.get(`${__ENV.APP_URL}/j_spring_security_logout`, {
2973
redirects: 0
3074
});
3175
checkResponse(logout, isRedirect);

‎benchmark/load/insecure-bank/start-servers.sh‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
5+
start_server() {
6+
local VARIANT=$1
7+
local JAVA_OPTS=$2
8+
9+
if [ -n "$CI_JOB_TOKEN" ]; then
10+
# Inside BP, so we can assume 24 CPU cores available and set CPU affinity
11+
CPU_AFFINITY_APP=$3
12+
else
13+
CPU_AFFINITY_APP=""
14+
fi
15+
16+
mkdir -p "${LOGS_DIR}/${VARIANT}"
17+
${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${LOGS_DIR}/${VARIANT}/insecure-bank.log &PID=$!
18+
echo "${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms3G -Xmx3G -jar ${INSECURE_BANK} &> ${LOGS_DIR}/${VARIANT}/insecure-bank.log [PID=$PID]"
19+
}
20+
21+
start_server "no_agent" "-Dserver.port=8080" "taskset -c 47 " &
22+
start_server "tracing" "-javaagent:${TRACER} -Dserver.port=8081" "taskset -c 46 " &
23+
start_server "profiling" "-javaagent:${TRACER} -Ddd.profiling.enabled=true -Dserver.port=8082" "taskset -c 45 " &
24+
start_server "iast" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Dserver.port=8083" "taskset -c 44 " &
25+
start_server "iast_GLOBAL" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.context.mode=GLOBAL -Dserver.port=8084" "taskset -c 43 " &
26+
start_server "iast_FULL" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Ddd.iast.detection.mode=FULL -Dserver.port=8085" "taskset -c 42 " &
27+
28+
wait

‎benchmark/load/petclinic/benchmark.json‎

Lines changed: 0 additions & 46 deletions
This file was deleted.

‎benchmark/load/petclinic/k6.js‎

Lines changed: 51 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,61 @@
11
import http from 'k6/http';
22
import {checkResponse, isOk} from "../../utils/k6.js";
33

4-
const baseUrl = 'http://localhost:8080';
4+
const variants = {
5+
"no_agent": {
6+
"APP_URL": 'http://localhost:8080',
7+
},
8+
"tracing": {
9+
"APP_URL": 'http://localhost:8081',
10+
},
11+
"profiling": {
12+
"APP_URL": 'http://localhost:8082',
13+
},
14+
"appsec": {
15+
"APP_URL": 'http://localhost:8083',
16+
},
17+
"iast": {
18+
"APP_URL": 'http://localhost:8084',
19+
},
20+
"code_origins": {
21+
"APP_URL": 'http://localhost:8085',
22+
}
23+
}
24+
25+
export const options = function (variants) {
26+
const scenarios = {};
27+
for (const variant of Object.keys(variants)) {
28+
scenarios[`load--petclinic--${variant}--warmup`] = {
29+
executor: 'constant-vus', // https://grafana.com/docs/k6/latest/using-k6/scenarios/executors/#all-executors
30+
vus: 5,
31+
duration: '20s',
32+
gracefulStop: '2s',
33+
env: {
34+
"APP_URL": variants[variant]["APP_URL"]
35+
}
36+
};
37+
38+
scenarios[`load--petclinic--${variant}--high_load`] = {
39+
executor: 'constant-vus',
40+
vus: 5,
41+
startTime: '22s',
42+
duration: '15s',
43+
gracefulStop: '2s',
44+
env: {
45+
"APP_URL": variants[variant]["APP_URL"]
46+
}
47+
};
48+
}
549

6-
export const options = {
7-
discardResponseBodies: true,
8-
vus: 5,
9-
iterations: 80000
10-
};
50+
return {
51+
discardResponseBodies: true,
52+
scenarios,
53+
}
54+
}(variants);
1155

1256
export default function () {
1357

1458
// find owner
15-
const ownersList = http.get(`${baseUrl}/owners?lastName=`);
59+
const ownersList = http.get(`${__ENV.APP_URL}/owners?lastName=`);
1660
checkResponse(ownersList, isOk);
1761
}

‎benchmark/load/petclinic/start-servers.sh‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
5+
start_server() {
6+
local VARIANT=$1
7+
local JAVA_OPTS=$2
8+
9+
if [ -n "$CI_JOB_TOKEN" ]; then
10+
# Inside BP, so we can assume 24 CPU cores available and set CPU affinity
11+
CPU_AFFINITY_APP=$3
12+
else
13+
CPU_AFFINITY_APP=""
14+
fi
15+
16+
mkdir -p "${LOGS_DIR}/${VARIANT}"
17+
${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${LOGS_DIR}/${VARIANT}/petclinic.log &PID=$!
18+
echo "${CPU_AFFINITY_APP}java ${JAVA_OPTS} -Xms2G -Xmx2G -jar ${PETCLINIC} &> ${LOGS_DIR}/${VARIANT}/petclinic.log [PID=$!]"
19+
}
20+
21+
start_server "no_agent" "-Dserver.port=8080" "taskset -c 31-32 " &
22+
start_server "tracing" "-javaagent:${TRACER} -Dserver.port=8081" "taskset -c 33-34 " &
23+
start_server "profiling" "-javaagent:${TRACER} -Ddd.profiling.enabled=true -Dserver.port=8082" "taskset -c 35-36 " &
24+
start_server "appsec" "-javaagent:${TRACER} -Ddd.appsec.enabled=true -Dserver.port=8083" "taskset -c 37-38 " &
25+
start_server "iast" "-javaagent:${TRACER} -Ddd.iast.enabled=true -Dserver.port=8084" "taskset -c 39-40 " &
26+
start_server "code_origins" "-javaagent:${TRACER} -Ddd.code.origin.for.spans.enabled=true -Dserver.port=8085" "taskset -c 41-42 " &
27+
28+
wait

‎benchmark/load/run.sh‎

Lines changed: 74 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,77 @@
11
#!/usr/bin/env bash
2-
set -eu
2+
3+
set -e
4+
5+
function message() {
6+
echo "$(date +"%T"): $1"
7+
}
8+
9+
function healthcheck() {
10+
local url=$1
11+
12+
while true; do
13+
if [[ $(curl -fso /dev/null -w "%{http_code}" "${url}") = 200 ]]; then
14+
break
15+
fi
16+
done
17+
}
18+
19+
type=$1
20+
21+
if [ -n "$CI_JOB_TOKEN" ]; then
22+
# Inside BP, so we can assume 24 CPU cores on the second socket available and set CPU affinity
23+
export CPU_AFFINITY_K6="taskset -c 24-27 "
24+
else
25+
export CPU_AFFINITY_K6=""
26+
fi
327

428
source "${UTILS_DIR}/update-java-version.sh" 17
5-
"${UTILS_DIR}/run-sirun-benchmarks.sh" "$@"
29+
30+
for app in *; do
31+
if [[ ! -d "${app}" ]]; then
32+
continue
33+
fi
34+
35+
message "${type} benchmark: ${app} started"
36+
37+
export OUTPUT_DIR="${REPORTS_DIR}/${type}/${app}"
38+
mkdir -p ${OUTPUT_DIR}
39+
40+
export LOGS_DIR="${ARTIFACTS_DIR}/${type}/${app}"
41+
mkdir -p ${LOGS_DIR}
42+
43+
# Using profiler variants for healthcheck as they are the slowest
44+
if [ "${app}" == "petclinic" ]; then
45+
HEALTHCHECK_URL=http://localhost:8082
46+
REPETITIONS_COUNT=5
47+
elif [ "${app}" == "insecure-bank" ]; then
48+
HEALTHCHECK_URL=http://localhost:8082/login
49+
REPETITIONS_COUNT=2
50+
else
51+
echo "Unknown app ${app}"
52+
exit 1
53+
fi
54+
55+
for i in $(seq 1 $REPETITIONS_COUNT); do
56+
bash -c "${UTILS_DIR}/../${type}/${app}/start-servers.sh" &
57+
58+
echo "Waiting for serves to start..."
59+
if [ "${app}" == "petclinic" ]; then
60+
for port in $(seq 8080 8085); do
61+
healthcheck http://localhost:$port
62+
done
63+
elif [ "${app}" == "insecure-bank" ]; then
64+
for port in $(seq 8080 8085); do
65+
healthcheck http://localhost:$port/login
66+
done
67+
fi
68+
echo "Servers are up!"
69+
70+
(
71+
cd ${app} &&
72+
bash -c "${CPU_AFFINITY_K6}${UTILS_DIR}/run-k6-load-test.sh 'pkill java'"
73+
)
74+
done
75+
76+
message "${type} benchmark: ${app} finished"
77+
done

0 commit comments

Comments
 (0)