Skip to content

Commit ffc85ca

Browse files
vmelamedvmelamed
committed
slurp all the yaml scripts, rearranged benchmark.yml
1 parent 72d5337 commit ffc85ca

3 files changed

Lines changed: 85 additions & 75 deletions

File tree

‎.github/workflows/benchmarks.yml‎

Lines changed: 13 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ env:
4747
ARTIFACTS_DIR: BmResults
4848
SUMMARIES_DIR: BmResults/summaries
4949
BASELINE_DIR: BmResults/baseline
50+
FORCE_NEW_BASELINE: false
5051

5152
jobs:
5253
benchmarks:
@@ -70,12 +71,21 @@ jobs:
7071
**/packages.lock.json
7172
**/*.csproj
7273
73-
- name: Run benchmarks
74+
- name: Download baseline benchmark summaries artifact
75+
if: ${{ !inputs.force-new-baseline }}
76+
uses: actions/download-artifact@v4
77+
continue-on-error: true
78+
with:
79+
name: benchmark-baseline-${{ matrix.os }}
80+
path: ${{ env.BASELINE_DIR }}
81+
82+
- name: Run the benchmarks
7483
shell: bash
7584
env:
7685
FORCE_NEW_BASELINE: ${{ inputs.force-new-baseline }}
7786
CONFIGURATION: ${{ inputs.configuration }}
78-
run: ./scripts/bash/run-benchmarks.sh ${{ inputs.benchmark-project }}
87+
run: |
88+
./scripts/bash/run-benchmarks.sh ${{ inputs.benchmark-project }}
7989
8090
- name: Upload benchmark summaries artifact
8191
uses: actions/upload-artifact@v4
@@ -86,73 +96,8 @@ jobs:
8696
overwrite: true
8797
retention-days: 90
8898

89-
- name: Download baseline benchmark summaries artifact
90-
if: ${{ !inputs.force-new-baseline }}
91-
uses: actions/download-artifact@v4
92-
continue-on-error: true
93-
with:
94-
name: benchmark-baseline-${{ matrix.os }}
95-
path: ${{ env.BASELINE_DIR }}
96-
97-
- name: Compare benchmarks against baseline
98-
if: ${{ !inputs.force-new-baseline }}
99-
shell: bash
100-
env:
101-
MAX_REGRESSION_PCT: ${{ inputs.max-regression-pct }}
102-
run: |
103-
set -euo pipefail
104-
105-
fs=$(ls ${SUMMARIES_DIR}/*-summary.json 2>/dev/null || true)
106-
if [ -z "${fs}" ]; then
107-
echo "No current benchmark result JSON files found."
108-
exit 2
109-
fi
110-
SUM_CUR=0
111-
for f in ${fs}; do
112-
VAL=$(jq '( .Totals.Mean // 0)' "${f}")
113-
SUM_CUR=$(( SUM_CUR + VAL ))
114-
done
115-
if (( SUM_CUR == 0 )); then
116-
echo "Current sum is invalid (${SUM_CUR})."
117-
exit 2
118-
fi
119-
120-
fs=$(ls "${BASELINE_DIR}"/*-summary.json 2>/dev/null || true)
121-
if [[ -z "${fs}" ]]; then
122-
echo "Baseline reports were not found at ${BASELINE_DIR}."
123-
echo "FORCE_NEW_BASELINE=true" >> "$GITHUB_ENV"
124-
exit 0
125-
fi
126-
SUM_BASE=0
127-
for f in ${fs}; do
128-
VAL=$(jq '( .Totals.Mean // 0)' "${f}")
129-
SUM_BASE=$(( SUM_BASE + VAL ))
130-
done
131-
if (( SUM_BASE == 0 )); then
132-
echo "Baseline sum is invalid (${SUM_BASE})."
133-
exit 2
134-
fi
135-
136-
pct=$(( (SUM_CUR - SUM_BASE) * 100 / SUM_BASE ))
137-
echo "Percent change vs baseline: ${pct}% (allowed: ${MAX_REGRESSION}%)"
138-
139-
if (( pct > MAX_REGRESSION )); then
140-
echo "Performance regression exceeds threshold"
141-
exit 2
142-
elif (( pct > 0 )); then
143-
echo "Performance regression within acceptable threshold."
144-
elif (( pct < 0 )); then
145-
pct_abs=$(( -pct ))
146-
if (( pct_abs >= MAX_REGRESSION )); then
147-
echo "Significant improvement of ${pct_abs}% over baseline. Updating the baseline."
148-
echo "FORCE_NEW_BASELINE=true" >> "$GITHUB_ENV"
149-
else
150-
echo "Improvement of ${pct_abs}% over baseline."
151-
fi
152-
fi
153-
15499
- name: Upload benchmark results artifact as a new baseline
155-
if: ${{ inputs.force-new-baseline || false }}
100+
if: ${{ inputs.force-new-baseline || env.FORCE_NEW_BASELINE || false }}
156101
uses: actions/upload-artifact@v4
157102
with:
158103
name: benchmark-baseline-${{ matrix.os }}

‎scripts/bash/run-benchmarks-utils.sh‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,5 +162,7 @@ dump_all_variables()
162162
script_dir \
163163
SUMMARIES_DIR \
164164
--line \
165-
base_name
165+
results_dir \
166+
summaries_dir \
167+
baseline_dir
166168
}

‎scripts/bash/run-benchmarks.sh‎

Lines changed: 69 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,6 @@ bm_project=$(realpath -e "$bm_project") # ensure it's an absolute path and exis
1717
declare -x ARTIFACTS_DIR=${ARTIFACTS_DIR:="$solution_dir/BmResults"}
1818
ARTIFACTS_DIR=$(realpath -m "$ARTIFACTS_DIR") # ensure it's an absolute path
1919

20-
declare -x SUMMARIES_DIR=${SUMMARIES_DIR:="$ARTIFACTS_DIR/summaries"}
21-
SUMMARIES_DIR=$(realpath -m "$SUMMARIES_DIR") # ensure it's an absolute path
22-
2320
declare configuration=${CONFIGURATION:="Release"}
2421

2522
source "$script_dir/_common.sh"
@@ -29,10 +26,22 @@ get_arguments "$@"
2926
declare -r bm_project
3027
declare -r configuration
3128

29+
declare -x results_dir=${results_dir:="$ARTIFACTS_DIR/results"}
30+
results_dir=$(realpath -m "$results_dir") # ensure it's an absolute path
31+
declare -r results_dir
32+
33+
declare -x summaries_dir=${summaries_dir:="$ARTIFACTS_DIR/summaries"}
34+
summaries_dir=$(realpath -m "$summaries_dir")
35+
declare -r summaries_dir
36+
37+
declare -x baseline_dir=${baseline_dir:="$ARTIFACTS_DIR/baseline"}
38+
baseline_dir=$(realpath -m "$baseline_dir")
39+
declare -r baseline_dir
40+
3241
dump_all_variables
3342

3443
trace "Creating directory(s)..."
35-
execute mkdir -p "$SUMMARIES_DIR"
44+
execute mkdir -p "$summaries_dir"
3645

3746
trace "Running benchmark tests in project '$bm_project' with configuration '$configuration'..."
3847
execute mkdir -p "${ARTIFACTS_DIR}"
@@ -57,7 +66,7 @@ if [[ $dry_run != "true" ]]; then
5766
# it expands to an empty string instead of the default to leaving the pattern unchanged,
5867
# i.e. ${ARTIFACTS_DIR}/results/*-report.json - we don't want that
5968
shopt -s nullglob
60-
files=("${ARTIFACTS_DIR}/results/*-report.json")
69+
files=("$ARTIFACTS_DIR/results/*-report.json")
6170
shopt -u nullglob
6271

6372
if [ ${#files[@]} -eq 0 ]; then
@@ -67,6 +76,60 @@ if [[ $dry_run != "true" ]]; then
6776

6877
for f in "${files[@]}"; do
6978
sf=$(sed -nE 's/(.*)-report.json/\1-summary.json/p' <<< "$(basename "${f}")")
70-
jq -f "$solution_dir/.github/workflows/summary.jq" "${f}" > "${SUMMARIES_DIR}/${sf}"
79+
jq -f "$solution_dir/.github/workflows/summary.jq" "${f}" > "${summaries_dir}/${sf}"
7180
done
7281
fi
82+
83+
#-------------------------------------------------------------------------------
84+
85+
fs=$(ls "${summaries_dir}"/*-summary.json 2>/dev/null || true)
86+
if [ -z "${fs}" ]; then
87+
echo "No current benchmark result JSON files found." >&2
88+
exit 2
89+
fi
90+
sum_cur=0
91+
for f in ${fs}; do
92+
VAL=$(jq '( .Totals.Mean // 0)' "${f}")
93+
sum_cur=$(( sum_cur + VAL ))
94+
done
95+
if (( sum_cur == 0 )); then
96+
echo "Current sum is invalid (${sum_cur})." >&2
97+
exit 2
98+
fi
99+
100+
fs=$(ls "${baseline_dir}"/*-summary.json 2>/dev/null || true)
101+
if [[ -z "${fs}" ]]; then
102+
echo "Baseline reports were not found at ${baseline_dir}." >&2
103+
echo "FORCE_NEW_BASELINE=true" >> "$GITHUB_ENV"
104+
exit 0
105+
fi
106+
sum_base=0
107+
for f in ${fs}; do
108+
VAL=$(jq '( .Totals.Mean // 0)' "${f}")
109+
sum_base=$(( sum_base + VAL ))
110+
done
111+
if (( sum_base == 0 )); then
112+
echo "Baseline sum is invalid (${sum_base})." >&2
113+
exit 2
114+
fi
115+
116+
pct=$(( (sum_cur - sum_base) * 100 / sum_base ))
117+
echo "Percent change vs baseline: ${pct}% (allowed: ${max_regression_pct}%)"
118+
flush_stdout
119+
120+
if (( pct > max_regression_pct )); then
121+
echo "Performance regression exceeds threshold" >&2
122+
exit 2
123+
elif (( pct > 0 )); then
124+
echo "Performance regression within acceptable threshold."
125+
flush_stdout
126+
elif (( pct < 0 )); then
127+
pct_abs=$(( -pct ))
128+
if (( pct_abs >= max_regression_pct )); then
129+
echo "Significant improvement of ${pct_abs}% over baseline. Updating the baseline."
130+
echo "FORCE_NEW_BASELINE=true" >> "$GITHUB_ENV"
131+
else
132+
echo "Improvement of ${pct_abs}% over baseline."
133+
fi
134+
flush_stdout
135+
fi

0 commit comments

Comments
 (0)