Skip to content
This repository was archived by the owner on Jan 29, 2026. It is now read-only.

Commit 4f6d5f9

Browse files
author
Mingxin Wang
authored
Attach environment information to benchmarking report (#270)
1 parent bc7221f commit 4f6d5f9

11 files changed

Lines changed: 151 additions & 47 deletions

File tree

‎.github/workflows/bvt-appleclang.yml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,16 @@ jobs:
2323
cmake -B build -DCMAKE_CXX_STANDARD=23 -DCMAKE_BUILD_TYPE=Release
2424
cmake --build build -j
2525
ctest --test-dir build -j
26+
mkdir build/drop
27+
chmod +x tools/dump_build_env.sh
28+
./tools/dump_build_env.sh clang++ build/drop/env-info.json
2629
2730
- name: run benchmarks
2831
run: |
29-
cd build/benchmarks
30-
./msft_proxy_benchmarks --benchmark_min_warmup_time=0.1 --benchmark_min_time=0.1s --benchmark_repetitions=30 --benchmark_enable_random_interleaving=true --benchmark_report_aggregates_only=true --benchmark_format=json > benchmarking-results.json
32+
build/benchmarks/msft_proxy_benchmarks --benchmark_min_warmup_time=0.1 --benchmark_min_time=0.1s --benchmark_repetitions=30 --benchmark_enable_random_interleaving=true --benchmark_report_aggregates_only=true --benchmark_format=json > build/drop/benchmarking-results.json
3133
3234
- name: archive benchmarking results
3335
uses: actions/upload-artifact@v4
3436
with:
35-
name: benchmarking-results-appleclang
36-
path: build/benchmarks/benchmarking-results.json
37+
name: drop-appleclang
38+
path: build/drop/

‎.github/workflows/bvt-clang.yml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,16 @@ jobs:
2626
cmake -B build -DCMAKE_C_COMPILER=clang-19 -DCMAKE_CXX_COMPILER=clang++-19 -DCMAKE_CXX_FLAGS="-stdlib=libc++" -DCMAKE_CXX_STANDARD=23 -DCMAKE_BUILD_TYPE=Release
2727
cmake --build build -j
2828
ctest --test-dir build -j
29+
mkdir build/drop
30+
chmod +x tools/dump_build_env.sh
31+
./tools/dump_build_env.sh clang++-19 build/drop/env-info.json
2932
3033
- name: run benchmarks
3134
run: |
32-
cd build/benchmarks
33-
./msft_proxy_benchmarks --benchmark_min_warmup_time=0.1 --benchmark_min_time=0.1s --benchmark_repetitions=30 --benchmark_enable_random_interleaving=true --benchmark_report_aggregates_only=true --benchmark_format=json > benchmarking-results.json
35+
build/benchmarks/msft_proxy_benchmarks --benchmark_min_warmup_time=0.1 --benchmark_min_time=0.1s --benchmark_repetitions=30 --benchmark_enable_random_interleaving=true --benchmark_report_aggregates_only=true --benchmark_format=json > build/drop/benchmarking-results.json
3436
3537
- name: archive benchmarking results
3638
uses: actions/upload-artifact@v4
3739
with:
38-
name: benchmarking-results-clang
39-
path: build/benchmarks/benchmarking-results.json
40+
name: drop-clang
41+
path: build/drop/

‎.github/workflows/bvt-gcc.yml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,16 @@ jobs:
2626
cmake -B build -DCMAKE_C_COMPILER=gcc-14 -DCMAKE_CXX_COMPILER=g++-14 -DCMAKE_CXX_STANDARD=23 -DCMAKE_BUILD_TYPE=Release
2727
cmake --build build -j
2828
ctest --test-dir build -j
29+
mkdir build/drop
30+
chmod +x tools/dump_build_env.sh
31+
./tools/dump_build_env.sh g++-14 build/drop/env-info.json
2932
3033
- name: run benchmarks
3134
run: |
32-
cd build/benchmarks
33-
./msft_proxy_benchmarks --benchmark_min_warmup_time=0.1 --benchmark_min_time=0.1s --benchmark_repetitions=30 --benchmark_enable_random_interleaving=true --benchmark_report_aggregates_only=true --benchmark_format=json > benchmarking-results.json
35+
build/benchmarks/msft_proxy_benchmarks --benchmark_min_warmup_time=0.1 --benchmark_min_time=0.1s --benchmark_repetitions=30 --benchmark_enable_random_interleaving=true --benchmark_report_aggregates_only=true --benchmark_format=json > build/drop/benchmarking-results.json
3436
3537
- name: archive benchmarking results
3638
uses: actions/upload-artifact@v4
3739
with:
38-
name: benchmarking-results-gcc
39-
path: build/benchmarks/benchmarking-results.json
40+
name: drop-gcc
41+
path: build/drop/

‎.github/workflows/bvt-msvc.yml‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,19 +13,23 @@ jobs:
1313
with:
1414
ref: ${{ inputs.branch }}
1515

16+
- name: add cl.exe to PATH
17+
uses: ilammy/msvc-dev-cmd@v1
18+
1619
- name: build and run test with MSVC
1720
run: |
1821
cmake -B build -DCMAKE_CXX_STANDARD=23
1922
cmake --build build --config Release -j
2023
ctest --test-dir build -j
24+
mkdir build\drop > $null
25+
.\tools\dump_build_env_msvc.ps1 -OutputPath build\drop\env-info.json
2126
2227
- name: run benchmarks
2328
run: |
24-
cd build\benchmarks
25-
.\Release\msft_proxy_benchmarks.exe --benchmark_min_warmup_time=0.1 --benchmark_min_time=0.1s --benchmark_repetitions=30 --benchmark_enable_random_interleaving=true --benchmark_report_aggregates_only=true --benchmark_format=json > benchmarking-results.json
29+
build\benchmarks\Release\msft_proxy_benchmarks.exe --benchmark_min_warmup_time=0.1 --benchmark_min_time=0.1s --benchmark_repetitions=30 --benchmark_enable_random_interleaving=true --benchmark_report_aggregates_only=true --benchmark_format=json > build\drop\benchmarking-results.json
2630
2731
- name: archive benchmarking results
2832
uses: actions/upload-artifact@v4
2933
with:
30-
name: benchmarking-results-msvc
31-
path: build/benchmarks/benchmarking-results.json
34+
name: drop-msvc
35+
path: build/drop/

‎.github/workflows/bvt-nvhpc.yml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,16 @@ jobs:
2626
cmake -B build -DCMAKE_C_COMPILER=nvc -DCMAKE_CXX_COMPILER=nvc++ -DCMAKE_BUILD_TYPE=Release
2727
cmake --build build -j
2828
ctest --test-dir build -j
29+
mkdir build/drop
30+
chmod +x tools/dump_build_env.sh
31+
./tools/dump_build_env.sh nvc++ build/drop/env-info.json
2932
3033
- name: run benchmarks
3134
run: |
32-
cd build/benchmarks
33-
./msft_proxy_benchmarks --benchmark_min_warmup_time=0.1 --benchmark_min_time=0.1s --benchmark_repetitions=30 --benchmark_enable_random_interleaving=true --benchmark_report_aggregates_only=true --benchmark_format=json > benchmarking-results.json
35+
build/benchmarks/msft_proxy_benchmarks --benchmark_min_warmup_time=0.1 --benchmark_min_time=0.1s --benchmark_repetitions=30 --benchmark_enable_random_interleaving=true --benchmark_report_aggregates_only=true --benchmark_format=json > build/drop/benchmarking-results.json
3436
3537
- name: archive benchmarking results
3638
uses: actions/upload-artifact@v4
3739
with:
38-
name: benchmarking-results-nvhpc
39-
path: build/benchmarks/benchmarking-results.json
40+
name: drop-nvhpc
41+
path: build/drop/

‎.github/workflows/bvt-report.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ jobs:
2727
- name: generate report
2828
run: |
2929
cat <<EOF > benchmarking-report.md
30-
## Benchmarking Report
30+
# Benchmarking Report
3131
3232
- Generated for: [Microsoft "Proxy" library](https://github.com/microsoft/proxy)
3333
- Commit ID: [${{ github.sha }}](https://github.com/microsoft/proxy/commit/${{ github.sha }})

‎CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
cmake_minimum_required(VERSION 3.5)
1+
cmake_minimum_required(VERSION 3.10)
22

33
project(msft_proxy VERSION 3.2.1 LANGUAGES CXX)
44
add_library(msft_proxy INTERFACE)

‎tools/dump_build_env.sh‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/usr/bin/env bash
2+
3+
if [ $# -ne 2 ]; then
4+
echo "Usage: $0 <compiler> <output path>"
5+
exit 1
6+
fi
7+
8+
COMPILER_INFO="$("$1" --version 2>/dev/null | awk 'NF {print; exit}')"
9+
if [ -z "$COMPILER_INFO" ]; then
10+
echo "Unable to retrieve compiler info for '$1'."
11+
exit 1
12+
fi
13+
14+
OS_NAME="$(uname -s)"
15+
if [ "$OS_NAME" = "Linux" ]; then
16+
if [ -f /etc/os-release ]; then
17+
OS_NAME="$(grep ^PRETTY_NAME= /etc/os-release | cut -d= -f2 | tr -d '"')"
18+
fi
19+
elif [ "$OS_NAME" = "Darwin" ]; then
20+
OS_NAME="$(sw_vers -productName) $(sw_vers -productVersion)"
21+
fi
22+
23+
cat <<EOF > $2
24+
{
25+
"OS": "$OS_NAME",
26+
"KernelVersion": "$(uname -r)",
27+
"Architecture": "$(uname -m)",
28+
"Compiler": "$COMPILER_INFO"
29+
}
30+
EOF

‎tools/dump_build_env_msvc.ps1‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
[CmdletBinding()]
2+
param(
3+
[Parameter(Mandatory=$true)]
4+
[string]$OutputPath
5+
)
6+
7+
$osInfo = Get-CimInstance Win32_OperatingSystem
8+
$compiler = ((cl /? 2>&1 | Out-String) -split '\r')[0] -replace '^cl\s:\s',''
9+
$jsonObject = [ordered]@{
10+
OS = $osInfo.Caption
11+
KernelVersion = $osInfo.Version
12+
Architecture = $env:PROCESSOR_ARCHITECTURE
13+
Compiler = $compiler
14+
} | ConvertTo-Json
15+
Set-Content -Path $OutputPath -Value $jsonObject

‎tools/report_generator/main.cpp‎

Lines changed: 57 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -13,37 +13,63 @@
1313

1414
#include <nlohmann/json.hpp>
1515

16-
struct EnvironmentInfo {
16+
struct Environment {
1717
std::string Description;
18-
std::string Path;
18+
std::string InfoPath;
19+
std::string BenchmarkingResultsPath;
1920

20-
friend void to_json(nlohmann::json& j, const EnvironmentInfo& e) {
21+
friend void to_json(nlohmann::json& j, const Environment& e) {
2122
j = nlohmann::json{
2223
{"Description", e.Description},
23-
{"Path", e.Path}
24+
{"InfoPath", e.InfoPath},
25+
{"BenchmarkingResultsPath", e.BenchmarkingResultsPath},
2426
};
2527
}
2628

27-
friend void from_json(const nlohmann::json& j, EnvironmentInfo& e) {
29+
friend void from_json(const nlohmann::json& j, Environment& e) {
2830
j.at("Description").get_to(e.Description);
29-
j.at("Path").get_to(e.Path);
31+
j.at("InfoPath").get_to(e.InfoPath);
32+
j.at("BenchmarkingResultsPath").get_to(e.BenchmarkingResultsPath);
3033
}
3134
};
3235

33-
struct MetricInfo {
36+
struct EnvironmentInfo {
37+
std::string OS;
38+
std::string KernelVersion;
39+
std::string Architecture;
40+
std::string Compiler;
41+
42+
friend void to_json(nlohmann::json& j, const EnvironmentInfo& e) {
43+
j = nlohmann::json{
44+
{"OS", e.OS},
45+
{"KernelVersion", e.KernelVersion},
46+
{"Architecture", e.Architecture},
47+
{"Compiler", e.Compiler},
48+
};
49+
}
50+
51+
friend void from_json(const nlohmann::json& j, EnvironmentInfo& e) {
52+
j.at("OS").get_to(e.OS);
53+
j.at("KernelVersion").get_to(e.KernelVersion);
54+
j.at("Architecture").get_to(e.Architecture);
55+
j.at("Compiler").get_to(e.Compiler);
56+
}
57+
};
58+
59+
struct Metric {
3460
std::string Name;
3561
std::string TargetBenchmarkName;
3662
std::string BaselineBenchmarkName;
3763

38-
friend void to_json(nlohmann::json& j, const MetricInfo& m) {
64+
friend void to_json(nlohmann::json& j, const Metric& m) {
3965
j = nlohmann::json{
4066
{"Name", m.Name},
4167
{"TargetBenchmarkName", m.TargetBenchmarkName},
42-
{"BaselineBenchmarkName", m.BaselineBenchmarkName}
68+
{"BaselineBenchmarkName", m.BaselineBenchmarkName},
4369
};
4470
}
4571

46-
friend void from_json(const nlohmann::json& j, MetricInfo& m) {
72+
friend void from_json(const nlohmann::json& j, Metric& m) {
4773
j.at("Name").get_to(m.Name);
4874
j.at("TargetBenchmarkName").get_to(m.TargetBenchmarkName);
4975
j.at("BaselineBenchmarkName").get_to(m.BaselineBenchmarkName);
@@ -54,16 +80,16 @@ struct ReportConfig {
5480
std::string TargetName;
5581
double YellowIndicatorThreshold;
5682
std::string OutputPath;
57-
std::vector<EnvironmentInfo> Environments;
58-
std::vector<MetricInfo> Metrics;
83+
std::vector<Environment> Environments;
84+
std::vector<Metric> Metrics;
5985

6086
friend void to_json(nlohmann::json& j, const ReportConfig& rc) {
6187
j = nlohmann::json{
6288
{"TargetName", rc.TargetName},
6389
{"YellowIndicatorThreshold", rc.YellowIndicatorThreshold},
6490
{"OutputPath", rc.OutputPath},
6591
{"Environments", rc.Environments},
66-
{"Metrics", rc.Metrics}
92+
{"Metrics", rc.Metrics},
6793
};
6894
}
6995

@@ -78,7 +104,16 @@ struct ReportConfig {
78104

79105
const std::string_view MedianSuffix = "_median";
80106

81-
std::unordered_map<std::string, double> Parse(const std::filesystem::path& file) {
107+
EnvironmentInfo ParseEnvironmentInfo(const std::filesystem::path& file) {
108+
nlohmann::json obj;
109+
std::ifstream in;
110+
in.exceptions(std::ios_base::failbit | std::ios_base::badbit);
111+
in.open(file, std::ios_base::in | std::ios_base::binary);
112+
in >> obj;
113+
return obj.get<EnvironmentInfo>();
114+
}
115+
116+
std::unordered_map<std::string, double> ParseBenchmarkingResults(const std::filesystem::path& file) {
82117
nlohmann::json obj;
83118
{
84119
std::ifstream in;
@@ -111,7 +146,7 @@ void GenerateReport(const std::filesystem::path& config_path) {
111146
std::vector<std::unordered_map<std::string, double>> benchmarks;
112147
benchmarks.reserve(config.Environments.size());
113148
for (auto& environment : config.Environments) {
114-
benchmarks.push_back(Parse(environment.Path));
149+
benchmarks.push_back(ParseBenchmarkingResults(environment.BenchmarkingResultsPath));
115150
}
116151
std::ofstream out;
117152
out.exceptions(std::ios_base::failbit | std::ios_base::badbit);
@@ -155,6 +190,13 @@ void GenerateReport(const std::filesystem::path& config_path) {
155190
}
156191
out << "\n";
157192
}
193+
out << "\n## Environments\n\n";
194+
out << "| | Operating System | Kernel Version | Architecture | Compiler |\n";
195+
out << "| - | - | - | - | - |\n";
196+
for (auto& environment : config.Environments) {
197+
EnvironmentInfo env_info = ParseEnvironmentInfo(environment.InfoPath);
198+
out << "| **" << environment.Description << "** | " << env_info.OS << " | " << env_info.KernelVersion << " | " << env_info.Architecture << " | " << env_info.Compiler << " |\n";
199+
}
158200
}
159201

160202
int main(int argc, char** argv) {

0 commit comments

Comments
 (0)