Skip to content

Commit 03c9e2a

Browse files
sarahchen6sarahchen6
committed
Add benchmark conversion stage
1 parent 437578c commit 03c9e2a

2 files changed

Lines changed: 269 additions & 0 deletions

File tree

‎.gitlab-ci.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ stages:
2525
- java-startup-microbenchmarks
2626
- java-load-microbenchmarks
2727
- java-dacapo-microbenchmarks
28+
- benchmark-conversion
2829
- generate-slos
2930
- macrobenchmarks
3031
- tests

‎.gitlab/benchmarks.yml‎

Lines changed: 268 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,66 @@
3030
UPSTREAM_BRANCH: $CI_COMMIT_REF_NAME # The branch or tag name for which project is built.
3131
UPSTREAM_COMMIT_SHA: $CI_COMMIT_SHA # The commit revision the project is built for.
3232

33+
# Conversion template for benchmark artifacts
34+
.convert-benchmarks:
35+
stage: benchmarks
36+
timeout: 1h
37+
tags: ["arch:amd64"]
38+
image: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:88265497
39+
rules:
40+
- if: '$POPULATE_CACHE'
41+
when: never
42+
- if: '$CI_COMMIT_TAG =~ /^v?[0-9]+\.[0-9]+\.[0-9]+$/'
43+
when: manual
44+
allow_failure: true
45+
- if: '$CI_COMMIT_BRANCH == "master"'
46+
when: on_success
47+
interruptible: false
48+
- when: on_success
49+
interruptible: true
50+
before_script:
51+
- export ARTIFACTS_DIR="$(pwd)/reports" && mkdir -p "${ARTIFACTS_DIR}"
52+
- export CONVERTED_DIR="${ARTIFACTS_DIR}/converted" && mkdir -p "${CONVERTED_DIR}"
53+
# Determine baseline_or_candidate based on branch
54+
- |
55+
if [ "$CI_COMMIT_BRANCH" == "master" ]; then
56+
export BASELINE_OR_CANDIDATE="baseline"
57+
else
58+
export BASELINE_OR_CANDIDATE="candidate"
59+
fi
60+
# Extract version from upstream.env if available
61+
- |
62+
if [ -f upstream.env ]; then
63+
source upstream.env
64+
export UPSTREAM_TRACER_VERSION="${UPSTREAM_TRACER_VERSION:-unknown}"
65+
else
66+
export UPSTREAM_TRACER_VERSION="unknown"
67+
fi
68+
# Get CPU model and kernel version
69+
- |
70+
if command -v lscpu >/dev/null 2>&1; then
71+
export CPU_MODEL=$(lscpu | grep -m1 "Model name:" | sed 's/Model name:[[:space:]]*//' || echo "unknown")
72+
elif [ -f /proc/cpuinfo ]; then
73+
export CPU_MODEL=$(grep -m1 "model name" /proc/cpuinfo 2>/dev/null | cut -d: -f2 | sed 's/^[[:space:]]*//' || echo "unknown")
74+
else
75+
export CPU_MODEL="unknown"
76+
fi
77+
- export KERNEL_VERSION=$(uname -a || echo "Unknown")
78+
- export CI_JOB_DATE=$(date +%s)
79+
- export CI_COMMIT_SHORT_SHA="${CI_COMMIT_SHORT_SHA:-${CI_COMMIT_SHA:0:7}}"
80+
- export CI_COMMIT_TIMESTAMP="${CI_COMMIT_TIMESTAMP:-$(date +%s)}"
81+
artifacts:
82+
name: "converted-benchmarks"
83+
paths:
84+
- reports/converted/
85+
expire_in: 3 months
86+
when: always
87+
variables:
88+
UPSTREAM_PROJECT_ID: $CI_PROJECT_ID
89+
UPSTREAM_PROJECT_NAME: $CI_PROJECT_NAME
90+
UPSTREAM_BRANCH: $CI_COMMIT_REF_NAME
91+
UPSTREAM_COMMIT_SHA: $CI_COMMIT_SHA
92+
3393
# benchmarks-startup:
3494
# extends: .benchmarks
3595
# script:
@@ -54,6 +114,214 @@
54114
# - ./steps/run-benchmarks.sh dacapo
55115
# - ./steps/analyze-results.sh dacapo
56116

117+
# Convert startup benchmark artifacts
118+
convert-startup-benchmarks:
119+
extends: .convert-benchmarks
120+
stage: benchmark-conversion
121+
needs: []
122+
rules:
123+
- if: '$POPULATE_CACHE'
124+
when: never
125+
- if: '$CI_COMMIT_TAG =~ /^v?[0-9]+\.[0-9]+\.[0-9]+$/'
126+
when: manual
127+
allow_failure: true
128+
- if: '$CI_COMMIT_BRANCH == "master"'
129+
when: on_success
130+
interruptible: false
131+
- when: on_success
132+
interruptible: true
133+
script:
134+
- !reference [.convert-benchmarks, before_script]
135+
- |
136+
echo "=== Converting startup benchmark artifacts ==="
137+
# Find all startup benchmark artifacts
138+
# Artifacts are in artifacts/startup-{app}/{variant}/startup_*.csv
139+
find artifacts -type d -name "startup-*" 2>/dev/null | while read app_dir; do
140+
app_name=$(basename "$app_dir" | sed 's/startup-//')
141+
echo "Processing startup artifacts for application: $app_name"
142+
143+
# Find all variant directories
144+
find "$app_dir" -mindepth 1 -maxdepth 1 -type d | while read variant_dir; do
145+
variant=$(basename "$variant_dir")
146+
echo " Processing variant: $variant"
147+
148+
# Check if there are CSV files
149+
if [ -n "$(find "$variant_dir" -name "startup_*.csv" 2>/dev/null | head -1)" ]; then
150+
# Build extra_params JSON
151+
extra_params="{\
152+
\"baseline_or_candidate\":\"${BASELINE_OR_CANDIDATE}\", \
153+
\"application\":\"${app_name}\", \
154+
\"release_version\":\"${UPSTREAM_TRACER_VERSION}\", \
155+
\"cpu_model\":\"${CPU_MODEL}\", \
156+
\"kernel_version\":\"${KERNEL_VERSION}\", \
157+
\"ci_job_date\":\"${CI_JOB_DATE}\", \
158+
\"ci_job_id\":\"${CI_JOB_ID}\", \
159+
\"ci_pipeline_id\":\"${CI_PIPELINE_ID}\", \
160+
\"git_commit_sha\":\"${CI_COMMIT_SHORT_SHA}\", \
161+
\"git_commit_date\":\"${CI_COMMIT_TIMESTAMP}\", \
162+
\"git_branch\":\"${CI_COMMIT_REF_NAME}\" \
163+
}"
164+
165+
# Convert using JavaStartup converter from relenv-benchmark-analyzer
166+
output_file="${CONVERTED_DIR}/startup/${app_name}/${variant}/benchmark-${BASELINE_OR_CANDIDATE}.json"
167+
mkdir -p "$(dirname "$output_file")"
168+
169+
benchmark_analyzer convert \
170+
--framework=javastartup \
171+
--extra-params="$extra_params" \
172+
--outpath="$output_file" \
173+
"$variant_dir" || {
174+
echo "ERROR: Failed to convert startup artifacts for $app_name/$variant"
175+
continue
176+
}
177+
178+
echo "Converted to: $output_file"
179+
else
180+
echo "WARNING: No CSV files found in $variant_dir"
181+
fi
182+
done
183+
done
184+
- |
185+
if [ ! -d "${CONVERTED_DIR}/startup" ] || [ -z "$(find "${CONVERTED_DIR}/startup" -name "*.json" 2>/dev/null)" ]; then
186+
echo "WARNING: No startup benchmark artifacts were converted"
187+
fi
188+
189+
# Convert dacapo benchmark artifacts
190+
convert-dacapo-benchmarks:
191+
extends: .convert-benchmarks
192+
stage: benchmark-conversion
193+
needs: []
194+
rules:
195+
- if: '$POPULATE_CACHE'
196+
when: never
197+
- if: '$CI_COMMIT_TAG =~ /^v?[0-9]+\.[0-9]+\.[0-9]+$/'
198+
when: manual
199+
allow_failure: true
200+
- if: '$CI_COMMIT_BRANCH == "master"'
201+
when: on_success
202+
interruptible: false
203+
- when: on_success
204+
interruptible: true
205+
script:
206+
- !reference [.convert-benchmarks, before_script]
207+
- |
208+
echo "=== Converting dacapo benchmark artifacts ==="
209+
# Find all dacapo benchmark artifacts
210+
# Artifacts are in artifacts/dacapo/{variant}/{benchmark}/
211+
find artifacts -type d -path "*/dacapo/*" 2>/dev/null | while read dacapo_dir; do
212+
# Extract variant and benchmark from path: artifacts/dacapo/{variant}/{benchmark}
213+
path_parts=$(echo "$dacapo_dir" | sed 's|artifacts/dacapo/||')
214+
variant=$(echo "$path_parts" | cut -d'/' -f1)
215+
benchmark=$(echo "$path_parts" | cut -d'/' -f2)
216+
217+
if [ -z "$benchmark" ] || [ "$benchmark" == "$variant" ]; then
218+
# Try alternative path structure
219+
benchmark=$(basename "$dacapo_dir")
220+
variant=$(basename "$(dirname "$dacapo_dir")")
221+
fi
222+
223+
echo "Processing dacapo artifacts: variant=$variant, benchmark=$benchmark"
224+
225+
# Check if there are CSV files in scratch/ or log files
226+
if [ -n "$(find "$dacapo_dir" -name "dacapo-latency-usec-simple-*.csv" -path "*/scratch/*" 2>/dev/null | head -1)" ] || \
227+
[ -n "$(find "$dacapo_dir" -name "dacapo-*.log" 2>/dev/null | head -1)" ]; then
228+
# Build extra_params JSON
229+
extra_params="{\
230+
\"baseline_or_candidate\":\"${BASELINE_OR_CANDIDATE}\", \
231+
\"application\":\"${benchmark}\", \
232+
\"release_version\":\"${UPSTREAM_TRACER_VERSION}\", \
233+
\"cpu_model\":\"${CPU_MODEL}\", \
234+
\"kernel_version\":\"${KERNEL_VERSION}\", \
235+
\"ci_job_date\":\"${CI_JOB_DATE}\", \
236+
\"ci_job_id\":\"${CI_JOB_ID}\", \
237+
\"ci_pipeline_id\":\"${CI_PIPELINE_ID}\", \
238+
\"git_commit_sha\":\"${CI_COMMIT_SHORT_SHA}\", \
239+
\"git_commit_date\":\"${CI_COMMIT_TIMESTAMP}\", \
240+
\"git_branch\":\"${CI_COMMIT_REF_NAME}\", \
241+
\"variant\":\"${variant}\" \
242+
}"
243+
244+
# Convert using JavaDacapo converter from relenv-benchmark-analyzer
245+
output_file="${CONVERTED_DIR}/dacapo/${variant}/${benchmark}/benchmark-${BASELINE_OR_CANDIDATE}.json"
246+
mkdir -p "$(dirname "$output_file")"
247+
248+
benchmark_analyzer convert \
249+
--framework=javadacapo \
250+
--extra-params="$extra_params" \
251+
--outpath="$output_file" \
252+
"$dacapo_dir" || {
253+
echo "ERROR: Failed to convert dacapo artifacts for $variant/$benchmark"
254+
continue
255+
}
256+
257+
echo "Converted to: $output_file"
258+
else
259+
echo "WARNING: No CSV or log files found in $dacapo_dir"
260+
fi
261+
done
262+
- |
263+
if [ ! -d "${CONVERTED_DIR}/dacapo" ] || [ -z "$(find "${CONVERTED_DIR}/dacapo" -name "*.json" 2>/dev/null)" ]; then
264+
echo "WARNING: No dacapo benchmark artifacts were converted"
265+
fi
266+
267+
# Convert load benchmark artifacts
268+
convert-load-benchmarks:
269+
extends: .convert-benchmarks
270+
stage: benchmark-conversion
271+
needs: []
272+
rules:
273+
- if: '$POPULATE_CACHE'
274+
when: never
275+
- if: '$CI_COMMIT_TAG =~ /^v?[0-9]+\.[0-9]+\.[0-9]+$/'
276+
when: manual
277+
allow_failure: true
278+
- if: '$CI_COMMIT_BRANCH == "master"'
279+
when: on_success
280+
interruptible: false
281+
- when: on_success
282+
interruptible: true
283+
script:
284+
- !reference [.convert-benchmarks, before_script]
285+
- |
286+
echo "=== Processing load benchmark artifacts ==="
287+
find artifacts -name "candidate-*.converted.json" -o -name "baseline-*.converted.json" 2>/dev/null | while read json_file; do
288+
echo "Processing load benchmark file: $json_file"
289+
290+
# Verify it's valid JSON
291+
if ! python3 -m json.tool "$json_file" > /dev/null 2>&1; then
292+
echo "WARNING: Invalid JSON file: $json_file"
293+
continue
294+
fi
295+
296+
# Extract information from filename
297+
filename=$(basename "$json_file")
298+
if [[ "$filename" =~ candidate-([^-]+)--([^-]+)--([0-9]+)\.converted\.json ]]; then
299+
stage="${BASH_REMATCH[1]}"
300+
product="${BASH_REMATCH[2]}"
301+
run_id="${BASH_REMATCH[3]}"
302+
type="candidate"
303+
elif [[ "$filename" =~ baseline-([^-]+)--([^-]+)--([0-9]+)\.converted\.json ]]; then
304+
stage="${BASH_REMATCH[1]}"
305+
product="${BASH_REMATCH[2]}"
306+
run_id="${BASH_REMATCH[3]}"
307+
type="baseline"
308+
else
309+
echo "WARNING: Could not parse filename: $filename"
310+
continue
311+
fi
312+
313+
# Copy to organized structure
314+
output_dir="${CONVERTED_DIR}/load/${product}/${stage}"
315+
mkdir -p "$output_dir"
316+
cp "$json_file" "$output_dir/benchmark-${type}-run${run_id}.json"
317+
318+
echo "Organized to: $output_dir/benchmark-${type}-run${run_id}.json"
319+
done
320+
- |
321+
if [ ! -d "${CONVERTED_DIR}/load" ] || [ -z "$(find "${CONVERTED_DIR}/load" -name "*.json" 2>/dev/null)" ]; then
322+
echo "WARNING: No load benchmark artifacts were found"
323+
fi
324+
57325
# benchmarks-post-results:
58326
# extends: .benchmarks
59327
# tags: ["arch:amd64"]

0 commit comments

Comments
 (0)