Skip to content

Commit 8c57e1b

Browse files
OliverRietmannOliverRietmann
committed
Update ci scripts
1 parent 76e5be6 commit 8c57e1b

6 files changed

Lines changed: 63 additions & 61 deletions

File tree

‎.github/scripts/csv_to_md.py‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,16 @@ def get_2d_list(csv_filename):
1313
with open(csv_filename) as csv_file:
1414
csv_reader = csv.reader(csv_file)
1515
next(csv_reader)
16-
return [[str(name), float(mean), float(stdev)] for name, mean, stdev in csv_reader]
16+
return [[str(name), float(mean), float(stdev), int(count)] for name, mean, stdev, count in csv_reader]
1717

1818
table_baseline = get_2d_list(args.baseline)
1919
table_current = get_2d_list(args.current)
2020

21-
def student(meanX, stdevX, runsX, meanY, stdevY, runsY):
22-
s2 = ((runsX - 1) * stdevX**2 + (runsY - 1) * stdevY**2) / (runsX + runsY - 2)
23-
return (meanX - meanY) / math.sqrt(s2 / runsX + s2 / runsY) if s2 > 0.0 else 0.0
21+
def student(x, sx, m, y, sy, n):
22+
s = 0.0 if m < 2 or n < 2 else math.sqrt(((m - 1) * sx**2 + (n - 1) * sy**2) / (m + n - 2))
23+
d = x - y
24+
t = 0.0 if s == 0.0 else math.sqrt((n * m) / (n + m)) * d / s
25+
return d, s, t
2426

2527
def get_emoji(t):
2628
quantile = 2.0 # 95% confidence interval
@@ -33,13 +35,13 @@ def get_emoji(t):
3335

3436
table = []
3537
for baseline, current in zip(table_baseline, table_current):
36-
baseline_name, baseline_mean, baseline_stdev = baseline
37-
name, mean, stdev = current
38+
baseline_name, baseline_mean, baseline_stdev, count_baseline = baseline
39+
name, mean, stdev, count = current
3840
assert(baseline_name == name)
39-
diff = baseline_mean - mean
40-
t = student(baseline_mean, baseline_stdev, args.runs, mean, stdev, args.runs) if args.runs > 2 else 0.0
41+
total_time = mean * (count // args.runs)
42+
d, s, t = student(baseline_mean, baseline_stdev, args.runs, mean, stdev, args.runs)
4143
emoji = get_emoji(t)
42-
table.append([name, int(mean), f'{stdev:.2f}', int(diff), f'{t:.2f}', emoji])
44+
table.append([name, int(total_time), int(mean), f'{stdev:.2f}', f'{d:.2f}', f'{t:.2f}', emoji])
4345

44-
header = ['name', 'mean (\u03BCs)', 'stdev \u03C3', 'diff \u0394', 't', '']
46+
header = ['name', 'total time (\u03BCs)', 'mean (\u03BCs)', 'stdev \u03C3', 'diff \u0394', 't', '']
4547
print(tab.tabulate(table, header, tablefmt="github"))

‎.github/scripts/profiler_ncu.py‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import statistics
44

55
parser = argparse.ArgumentParser()
6-
parser.add_argument('-r', '--runs', type=int, required=True, help='Number of runs')
76
parser.add_argument('-i', '--input', required=True, help='Input CSV file')
87
parser.add_argument('-o', '--output', required=True, help='Output CSV file')
98
args = parser.parse_args()
@@ -22,12 +21,12 @@
2221
else:
2322
kernel_dict[name] = [time]
2423

25-
data = [["name", "time", "stdev"]]
24+
data = [["name", "time", "stdev", "count"]]
2625
for name, time_list in kernel_dict.items():
27-
count = len(time_list) // args.runs
28-
mean = statistics.mean(time_list) * count
29-
stdev = 0 if args.runs == 1 else statistics.stdev(time_list) * count
30-
data.append([name, mean, stdev])
26+
count = len(time_list)
27+
mean = statistics.mean(time_list)
28+
stdev = 0 if count == 1 else statistics.stdev(time_list)
29+
data.append([name, mean, stdev, count])
3130

3231
with open(args.output, 'w') as csv_file:
3332
csv_writer = csv.writer(csv_file)

‎.github/scripts/profiler_nsys.py‎

Lines changed: 8 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,11 @@
33
import statistics
44

55
parser = argparse.ArgumentParser()
6-
parser.add_argument('-r', '--runs', type=int, required=True, help='Number of runs')
76
parser.add_argument('-i', '--input', required=True, help='Input CSV file')
87
parser.add_argument('-o', '--output', required=True, help='Output CSV file')
98
args = parser.parse_args()
109

11-
ntsi_list = []
10+
kernel_list = []
1211
with open(args.input) as csv_file:
1312
csv_reader = csv.reader(csv_file)
1413
next(csv_reader)
@@ -17,21 +16,17 @@
1716
for row in csv_reader:
1817
if row:
1918
full_name = row[8]
20-
instances = int(row[2])
21-
time = float(row[3])
22-
sigma = float(row[7])
19+
count = int(row[2])
20+
mean = float(row[3])
21+
stdev = float(row[7])
2322
if len(full_name) > 5 and full_name[:5] == "krnl_":
2423
name = full_name[5:]
25-
ntsi_list.append([name, time, sigma, instances])
24+
kernel_list.append([name, mean, stdev, count])
2625

27-
ntsi_list.sort(key = lambda row: row[0])
26+
kernel_list.sort(key = lambda row: row[0])
2827

29-
data = [["name", "time", "stdev"]]
30-
for name, time, sigma, instances in ntsi_list:
31-
count = instances / args.runs
32-
mean = int(time * count)
33-
stdev = sigma * count
34-
data.append([name, mean, stdev])
28+
data = [["name", "mean", "stdev", "count"]]
29+
data += kernel_list
3530

3631
with open(args.output, 'w') as csv_file:
3732
csv_writer = csv.writer(csv_file)

‎.github/scripts/profiler_rocprofv2.py‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,11 @@
33
import statistics
44

55
parser = argparse.ArgumentParser()
6-
parser.add_argument('-r', '--runs', type=int, required=True, help='Number of runs')
76
parser.add_argument('-i', '--input', required=True, help='Input CSV file')
87
parser.add_argument('-o', '--output', required=True, help='Output CSV file')
98
args = parser.parse_args()
109

11-
time_dict = dict({})
10+
kernel_dict = dict({})
1211
with open(args.input) as csv_file:
1312
csv_reader = csv.reader(csv_file)
1413
next(csv_reader)
@@ -17,17 +16,19 @@
1716
time = (int(row[15]) - int(row[14])) / 1000.0
1817
if len(full_name) > 5 and full_name[:5] == "krnl_":
1918
name = full_name[5:-3]
20-
if name in time_dict.keys():
21-
time_dict[name].append(time)
19+
if name in kernel_dict.keys():
20+
kernel_dict[name].append(time)
2221
else:
23-
time_dict[name] = [time]
22+
kernel_dict[name] = [time]
2423

25-
data = [["name", "time", "stdev"]]
26-
for name, time_list in time_dict.items():
27-
count = len(time_list) / args.runs
28-
mean = int(statistics.mean(time_list) * count)
29-
stdev = 0 if args.runs == 1 else statistics.stdev(time_list) * count
30-
data.append([name, mean, stdev])
24+
kernel_list.sort(key = lambda row: row[0])
25+
26+
data = [["name", "mean", "stdev", "count"]]
27+
for name, time_list in kernel_dict.items():
28+
count = len(time_list)
29+
mean = statistics.mean(time_list)
30+
stdev = 0 if count == 1 else statistics.stdev(time_list)
31+
data.append([name, mean, stdev, count])
3132

3233
with open(args.output, 'w') as csv_file:
3334
csv_writer = csv.writer(csv_file)

.github/scripts/merge_runs.py renamed to .github/scripts/profiler_standalone.py

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import argparse
22
import csv
3+
import math
34
import statistics
45

56
parser = argparse.ArgumentParser()
@@ -9,23 +10,27 @@
910
args = parser.parse_args()
1011

1112
time_dict = dict({})
13+
count_dict = dict({})
1214
with open(args.input) as csv_file:
1315
csv_reader = csv.reader(csv_file)
1416
next(csv_reader)
1517
for row in csv_reader:
1618
name = row[2]
1719
time = float(row[3])
20+
count = row[1]
1821
if name in time_dict.keys():
1922
time_dict[name].append(time)
2023
else:
2124
time_dict[name] = [time]
25+
count_dict[name] = 1 if count == '' else int(count)
2226

23-
data = [["name", "time", "stdev"]]
27+
data = [["name", "mean", "stdev", "count"]]
2428
for name, time_list in time_dict.items():
25-
mean = int(statistics.mean(time_list[args.discard:]))
29+
count = count_dict[name]
30+
mean = statistics.mean(time_list[args.discard:]) / count
2631
runs = len(time_list[args.discard:])
27-
stdev = 0.0 if runs == 1 else statistics.stdev(time_list[args.discard:])
28-
data.append([name, mean, stdev])
32+
stdev = 0.0 if runs == 1 else statistics.stdev(time_list[args.discard:]) / math.sqrt(count)
33+
data.append([name, mean, stdev, runs * count])
2934

3035
with open(args.output, 'w') as csv_file:
3136
csv_writer = csv.writer(csv_file)

‎.github/workflows/standalone-benchmark.yml‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -104,8 +104,8 @@ jobs:
104104
source /etc/profile.d/modules.sh
105105
module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
106106
cd ${STANDALONE_DIR}
107-
${TIMING_CA} --debug 1 --runs ${{ matrix.standalone_runs }} --runsInit 2 --PROCdebugMarkdown 1 --PROCresetTimers 1 --PROCdebugCSV /root/${BENCHMARK_CSV}
108-
python3 ${GITHUB_WORKSPACE}/.github/scripts/merge_runs.py --discard 2 --input /root/${BENCHMARK_CSV} --output /root/summary_${BENCHMARK_CSV}
107+
${TIMING_CA} --debug 1 --runs ${{ matrix.standalone_runs }} --runsInit 0 --PROCdebugMarkdown 1 --PROCresetTimers 1 --PROCdebugCSV /root/${BENCHMARK_CSV}
108+
python3 ${GITHUB_WORKSPACE}/.github/scripts/profiler_standalone.py --discard 0 --input /root/${BENCHMARK_CSV} --output /root/summary_${BENCHMARK_CSV}
109109
110110
- name: Profiler - Nsight Compute
111111
if: ${{ matrix.name == 'nvidia-h100' }}
@@ -117,7 +117,7 @@ jobs:
117117
ncu --set none --metrics gpu__time_duration.avg --export ${{ matrix.name }} --clock-control none --force-overwrite ${TIMING_CA} --runs ${{ matrix.profiler_runs }} --debug 1 --PROCdebugMarkdown 1 # Generates ${{ matrix.name }}.ncu-rep
118118
ncu --import ${STANDALONE_DIR}/${{ matrix.name }}.ncu-rep --print-units base --csv > /root/${PROFILER_CSV}
119119
rm -rf ${STANDALONE_DIR}/events/50kHz ${STANDALONE_DIR}/build
120-
python3 ${GITHUB_WORKSPACE}/.github/scripts/profiler_ncu.py --runs ${{ matrix.profiler_runs }} --input /root/${PROFILER_CSV} --output /root/summary_${PROFILER_CSV}
120+
python3 ${GITHUB_WORKSPACE}/.github/scripts/profiler_ncu.py --input /root/${PROFILER_CSV} --output /root/summary_${PROFILER_CSV}
121121
122122
- name: Profiler - Nsight Systems
123123
if: ${{ matrix.name == 'nvidia-l40s' }}
@@ -130,7 +130,7 @@ jobs:
130130
nsys profile -o ${{ matrix.name }} ${TIMING_CA} --runs ${{ matrix.profiler_runs }} --debug 1 --PROCdebugMarkdown 1 # Generates ${{ matrix.name }}.nsys-rep
131131
nsys stats --report cuda_gpu_kern_sum --timeunit usec --force-export=true --format csv ${{ matrix.name }}.nsys-rep > /root/${PROFILER_CSV}
132132
rm -rf ${STANDALONE_DIR}/events/50kHz ${STANDALONE_DIR}/build
133-
python3 ${GITHUB_WORKSPACE}/.github/scripts/profiler_nsys.py --runs ${{ matrix.profiler_runs }} --input /root/${PROFILER_CSV} --output /root/summary_${PROFILER_CSV}
133+
python3 ${GITHUB_WORKSPACE}/.github/scripts/profiler_nsys.py --input /root/${PROFILER_CSV} --output /root/summary_${PROFILER_CSV}
134134
135135
- name: Profiler - rocprofv2
136136
if: ${{ matrix.name == 'amd-mi300x' || matrix.name == 'amd-w7900' }}
@@ -141,23 +141,23 @@ jobs:
141141
rocprofv2 --output-directory /root --output-file-name ${{ matrix.name }} ${TIMING_CA} --runs ${{ matrix.standalone_runs }} --debug 1 --PROCdebugMarkdown 1 # Generates results_${{ matrix.name }}.csv
142142
rm -rf ${STANDALONE_DIR}/events/50kHz ${STANDALONE_DIR}/build
143143
mv /root/results_${{ matrix.name }}.csv /root/${PROFILER_CSV}
144-
python3 ${GITHUB_WORKSPACE}/.github/scripts/profiler_rocprofv2.py --runs ${{ matrix.profiler_runs }} --input /root/${PROFILER_CSV} --output /root/summary_${PROFILER_CSV}
144+
python3 ${GITHUB_WORKSPACE}/.github/scripts/profiler_rocprofv2.py --input /root/${PROFILER_CSV} --output /root/summary_${PROFILER_CSV}
145145
146146
- name: Upload Artifact
147147
uses: actions/upload-artifact@v6
148148
with:
149149
name: ${{ matrix.name }}-artifact
150150
path: "/root/*.csv"
151151

152-
- name: Display table on GitHub web
153-
run: |
154-
source /etc/profile.d/modules.sh
155-
module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
156-
mkdir -p ${STANDALONE_DIR}/baseline
157-
curl -fL --retry 3 -o ${STANDALONE_DIR}/baseline/summary_${PROFILER_CSV} https://cernbox.cern.ch/remote.php/dav/public-files/SfYXgQOHFga2w75/baseline/summary_${PROFILER_CSV}
158-
curl -fL --retry 3 -o ${STANDALONE_DIR}/baseline/summary_${BENCHMARK_CSV} https://cernbox.cern.ch/remote.php/dav/public-files/SfYXgQOHFga2w75/baseline/summary_${BENCHMARK_CSV}
159-
python3 ${GITHUB_WORKSPACE}/.github/scripts/csv_to_md.py --runs ${{ matrix.profiler_runs }} --baseline ${STANDALONE_DIR}/baseline/summary_${PROFILER_CSV} --current /root/summary_${PROFILER_CSV} >> ${GITHUB_STEP_SUMMARY}
160-
echo -e "\n\n" >> ${GITHUB_STEP_SUMMARY}
161-
python3 ${GITHUB_WORKSPACE}/.github/scripts/csv_to_md.py --runs ${{ matrix.standalone_runs }} --baseline ${STANDALONE_DIR}/baseline/summary_${BENCHMARK_CSV} --current /root/summary_${BENCHMARK_CSV} >> ${GITHUB_STEP_SUMMARY}
162-
rm -rf ${STANDALONE_DIR}/baseline
163-
if: ${{ matrix.name != 'cpu' }}
152+
# - name: Display table on GitHub web
153+
# run: |
154+
# source /etc/profile.d/modules.sh
155+
# module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
156+
# mkdir -p ${STANDALONE_DIR}/baseline
157+
# curl -fL --retry 3 -o ${STANDALONE_DIR}/baseline/summary_${PROFILER_CSV} https://cernbox.cern.ch/remote.php/dav/public-files/SfYXgQOHFga2w75/baseline/summary_${PROFILER_CSV}
158+
# curl -fL --retry 3 -o ${STANDALONE_DIR}/baseline/summary_${BENCHMARK_CSV} https://cernbox.cern.ch/remote.php/dav/public-files/SfYXgQOHFga2w75/baseline/summary_${BENCHMARK_CSV}
159+
# python3 ${GITHUB_WORKSPACE}/.github/scripts/csv_to_md.py --runs ${{ matrix.profiler_runs }} --baseline ${STANDALONE_DIR}/baseline/summary_${PROFILER_CSV} --current /root/summary_${PROFILER_CSV} >> ${GITHUB_STEP_SUMMARY}
160+
# echo -e "\n\n" >> ${GITHUB_STEP_SUMMARY}
161+
# python3 ${GITHUB_WORKSPACE}/.github/scripts/csv_to_md.py --runs ${{ matrix.standalone_runs }} --baseline ${STANDALONE_DIR}/baseline/summary_${BENCHMARK_CSV} --current /root/summary_${BENCHMARK_CSV} >> ${GITHUB_STEP_SUMMARY}
162+
# rm -rf ${STANDALONE_DIR}/baseline
163+
# if: ${{ matrix.name != 'cpu' }}

0 commit comments

Comments
 (0)