Average over 12 runs and print table in github action

Author: OliverRietmann

.github/scripts/csv_to_md.py

@@ -1,33 +1,39 @@
-import sys
+import argparse
 import csv
 import tabulate as tab
 
-csv_benchmark = sys.argv[1]
-csv_baseline = sys.argv[2]
+parser = argparse.ArgumentParser()
+parser.add_argument('-b', '--baseline', required=True, help='Baseline CSV file')
+parser.add_argument('-c', '--current', required=True, help='Current CSV file')
+args = parser.parse_args()
 
-pretty = lambda x : "{:.1f}".format(x) if x <= 0 else "+{:.1f}".format(x)
+def get_2d_list(csv_filename):
+  with open(csv_filename) as csv_file:
+    csv_reader = csv.reader(csv_file)
+    next(csv_reader)
+    return [[str(name), float(mean), float(stdev)] for name, mean, stdev in csv_reader]
 
-with open(csv_benchmark) as csv_file:
-  csv_reader = csv.reader(csv_file)
-  next(csv_reader)
-  table_benchmark = [row for row in csv_reader]
- 
-with open(csv_baseline) as csv_file:
-  csv_reader = csv.reader(csv_file)
-  next(csv_reader)
-  table_baseline = [row for row in csv_reader]
+table_baseline = get_2d_list(args.baseline)
+table_current = get_2d_list(args.current)
+
+def get_emoji(d, stdev):
+  z = 1.96 # 95% confidence interval
+  if d < -z * stdev:
+    return ':green_circle:'
+  elif d > z * stdev:
+    return ':red_circle:'
+  else:
+    return ':white_circle:'
 
 table = []
-for benchmark, baseline in zip(table_benchmark, table_baseline):
-  assert(benchmark[0] == baseline[0])
-  name = benchmark[0]
-  time = benchmark[1]
-  stdev = u"\u00B1" + str(benchmark[2])
-  d = float(baseline[1]) - float(benchmark[1])
-  emoji = ':red_circle:' if 0 < d else ':green_circle:'
-  difference = pretty(d)
-  percent = pretty(100 * d / float(baseline[1]))
-  table.append([name, time, stdev, emoji, difference, percent])
+for baseline, current in zip(table_baseline, table_current):
+  baseline_name, baseline_mean, _ = baseline
+  name, mean, stdev = current
+  assert(baseline_name == name)
+  diff = baseline_mean - mean
+  impact = 0.0 if stdev == 0.0 else diff / stdev
+  emoji = get_emoji(diff, stdev)
+  table.append([name, int(mean), f'{stdev:.2f}', int(diff), f'{impact:.2f}', emoji])
 
-header = ["name", "time", "stdev", "", "difference", "percent"]
+header = ['name', 'mean (ms)', 'stdev \u03C3', 'diff \u0394', '\u0394 / \u03C3', '']
 print(tab.tabulate(table, header, tablefmt="github"))

.github/workflows/standalone-benchmark.yml

@@ -39,7 +39,7 @@ jobs:
     name: ${{ matrix.name }}
     steps:
       - name: Checkout Repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Download Files
         run: |
@@ -87,19 +87,19 @@ jobs:
           source /etc/profile.d/modules.sh
           module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
           cd ${STANDALONE_DIR}
-          ${STANDALONE_DIR}/ca -e 50kHz -g --memSize 15000000000 --sync --runs 42 --debug 1 --PROCtimingCSV ${BENCHMARK_CSV}
+          ${STANDALONE_DIR}/ca -e 50kHz -g --memSize 15000000000 --sync --runs 12 --debug 1 --PROCtimingCSV ${BENCHMARK_CSV}
           rm -rf ${STANDALONE_DIR}/events/50kHz ${STANDALONE_DIR}/build
 
       - name: Display table on GitHub web
         run: |
           source /etc/profile.d/modules.sh
           module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
           python3 ${GITHUB_WORKSPACE}/.github/scripts/merge_runs.py --discard 2 --input ${BENCHMARK_CSV} --output ${BENCHMARK_CSV}
-          #python3 ${GITHUB_WORKSPACE}/.github/scripts/csv_to_md.py ${BENCHMARK_CSV} ${STANDALONE_DIR}/baseline/${{ matrix.name }}.csv >> ${GITHUB_STEP_SUMMARY}
+          python3 ${GITHUB_WORKSPACE}/.github/scripts/csv_to_md.py --baseline ${STANDALONE_DIR}/baseline/${{ matrix.name }}.csv --current ${BENCHMARK_CSV} >> ${GITHUB_STEP_SUMMARY}
           rm -rf ${STANDALONE_DIR}/baseline
 
       - name: Upload Artifact
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v6
         with:
           name: ${{ matrix.name }}-artifact
           path: /root/${{ matrix.name }}.csv