Skip to content

Commit 46db760

Browse files
milsanoremilsanore
committed
feat: pushing benchmarks to InfluxDB, scheduled bench runs, housekeeping
1 parent 2ba2908 commit 46db760

9 files changed

Lines changed: 212 additions & 42 deletions

File tree

‎.github/workflows/nightly_bench.yml‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
name: Nightly Benchmark
2+
3+
# run a nightly performance benchmark report
4+
5+
on:
6+
schedule:
7+
- cron: '0 1 * * *' # Runs every day at 1:00 UTC
8+
workflow_dispatch: # Allows manual run from GitHub UI
9+
10+
jobs:
11+
build_and_test:
12+
runs-on: ubuntu-latest
13+
container:
14+
image: ghcr.io/milsanore/tradercppbuild:v1.7
15+
# set resource limits for some consistency
16+
options: --memory=7g --cpus=2
17+
steps:
18+
- name: Checkout repository
19+
uses: actions/checkout@v4
20+
with:
21+
fetch-depth: 0
22+
23+
- name: Restore Conan cache
24+
uses: actions/cache@v4
25+
with:
26+
path: ~/.conan2
27+
key: ${{ runner.os }}-conan-${{ hashFiles('conanfile.*') }}-Release
28+
29+
- name: Install conan dependencies
30+
run: |
31+
set -e
32+
rm -rf build
33+
conan profile detect --force
34+
conan install . --lockfile=conan.lock --build=missing -s build_type=Release
35+
36+
- name: Build
37+
run: |
38+
set -e
39+
cmake --preset=release
40+
cmake --build --preset=release
41+
42+
- name: Bench
43+
run: |
44+
set -e
45+
build/Release/benchmarks/benchmarks --benchmark_report_aggregates_only=false --benchmark_format=json > bench_results.json
46+
47+
- name: Upload benchmarks to InfluxDB
48+
env:
49+
INFLUX_URL: "https://us-east-1-1.aws.cloud2.influxdata.com"
50+
INFLUX_TOKEN: ${{ secrets.INFLUX_TOKEN }}
51+
INFLUX_ORG: "tradercpp"
52+
INFLUX_BUCKET: "benchmarks"
53+
GITHUB_SHA: ${{ github.sha }}
54+
GITHUB_REF_NAME: ${{ github.ref_name }}
55+
BENCH_SOURCE: github-actions
56+
run: python3 scripts/upload_to_influx.py

‎Makefile‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
#!make
22
SHELL:=/bin/bash
33

4+
#################################################
5+
# A set of basic execution recipes (all PHONY).
6+
# Call `make` on the command line for documentation.
7+
#################################################
8+
49
# pp - pretty print function
510
yellow := $(shell tput setaf 3)
611
normal := $(shell tput sgr0)
@@ -65,6 +70,7 @@ bench:
6570
$(call pp,assuming `make build-release` has been called)
6671
cmake --build --preset release
6772
build/Release/benchmarks/benchmarks --benchmark_report_aggregates_only=false
73+
# build/Release/benchmarks/benchmarks --benchmark_report_aggregates_only=false --benchmark_format=json > bench_results.json
6874

6975
## tidy: 🧹 tidy things up before committing code
7076
.PHONY: tidy

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,7 @@ NB: this app uses `make` as a recipe book, but it's not essential:
142142
- integration test with mocked Binance server
143143
- FTXUI snapshot testing
144144
- benchmarking
145-
- micro benchmarks
145+
- ✅ micro benchmarks
146146
- load test with mocked FIX server
147147
- profiling (valgrind/cachegrind)
148148
- profile-guided optimization (pgo)

‎benchmarks/core/book_side_benchmark.cpp‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#include <benchmark/benchmark.h>
2+
#include <fmt/ranges.h>
3+
4+
#include <random>
5+
6+
#include "core/bid_ask.h"
7+
#include "core/order_book.h"
8+
#include "spdlog/spdlog.h"
9+
10+
/// @brief the order book is composed of two asymetrically-sorted collections,
11+
/// this test is for one of those collections,
12+
/// (the bid side in this case, because of the additional complexity of a DESC sort)
13+
class BookSideFixture : public benchmark::Fixture {
14+
public:
15+
/// @brief deterministically build an initial, fully-populated, bid-side of the book
16+
void SetUp([[maybe_unused]] const benchmark::State& state) override {
17+
// bids
18+
for (uint64_t i = 0; i < DEPTH_LEVELS; ++i) {
19+
bids_[MID_PRICE - i] = 1;
20+
}
21+
}
22+
23+
void TearDown([[maybe_unused]] const benchmark::State& state) override {
24+
spdlog::info("Levels: [{}]", bids_);
25+
}
26+
27+
absl::btree_map<uint64_t, uint64_t, std::greater<>> bids_;
28+
static constexpr uint64_t DEPTH_LEVELS = 500;
29+
static constexpr uint64_t MID_PRICE = 100'000;
30+
};
31+
32+
/// @brief deterministically populate the order book
33+
BENCHMARK_DEFINE_F(BookSideFixture, BM_BookUpdate)(benchmark::State& state) {
34+
int i = 0;
35+
for (auto _ : state) {
36+
bids_[i] = MID_PRICE - i;
37+
if (i == MID_PRICE - 1) {
38+
i = 0;
39+
}
40+
i++;
41+
}
42+
43+
state.counters["Updates/sec"] =
44+
benchmark::Counter(state.iterations(), benchmark::Counter::kIsRate);
45+
}
46+
47+
BENCHMARK_REGISTER_F(BookSideFixture, BM_BookUpdate)->Iterations(500'000);

benchmarks/order_book_benchmark.cpp renamed to benchmarks/core/price_update_benchmark.cpp

Lines changed: 49 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,15 @@
77
#include "core/order_book.h"
88
#include "spdlog/spdlog.h"
99

10-
class OrderBookFixture : public benchmark::Fixture {
10+
class PriceUpdateFixture : public benchmark::Fixture {
1111
public:
12+
/// @brief deterministically build an initial, fully-populated, order book
1213
void SetUp([[maybe_unused]] const benchmark::State& state) override {
13-
// deterministically build an initial, fully-populated, order book
1414
book_ = std::make_unique<core::OrderBook>(bids_, asks_);
1515

1616
// bids
1717
uint64_t bid_px = MID_PRICE - 1;
18-
for (uint64_t i = 0; i <= DEPTH_LEVELS; ++i) {
18+
for (uint64_t i = 0; i < DEPTH_LEVELS; ++i) {
1919
auto msg = FIX44::MarketDataIncrementalRefresh();
2020
auto change = FIX44::MarketDataIncrementalRefresh::NoMDEntries();
2121
change.set(FIX::Symbol("BTCUSDT"));
@@ -25,14 +25,14 @@ class OrderBookFixture : public benchmark::Fixture {
2525
if (bid_px < MID_PRICE - DEPTH_LEVELS) {
2626
bid_px = MID_PRICE - 1;
2727
}
28-
change.set(FIX::MDEntrySize(i));
28+
change.set(FIX::MDEntrySize(1));
2929
msg.addGroup(change);
3030
book_->apply_increment(msg, false);
3131
}
3232

3333
// offers
3434
uint64_t ask_px = MID_PRICE + 1;
35-
for (uint64_t i = 0; i <= DEPTH_LEVELS; ++i) {
35+
for (uint64_t i = 0; i < DEPTH_LEVELS; ++i) {
3636
auto msg = FIX44::MarketDataIncrementalRefresh();
3737
auto change = FIX44::MarketDataIncrementalRefresh::NoMDEntries();
3838
change.set(FIX::Symbol("BTCUSDT"));
@@ -42,63 +42,73 @@ class OrderBookFixture : public benchmark::Fixture {
4242
if (ask_px > MID_PRICE + DEPTH_LEVELS) {
4343
ask_px = MID_PRICE + 1;
4444
}
45-
change.set(FIX::MDEntrySize(i));
45+
change.set(FIX::MDEntrySize(1));
4646
msg.addGroup(change);
4747
book_->apply_increment(msg, false);
4848

4949
// asks_[MID_PRICE + i] = 1;
5050
}
51+
52+
// test messages
53+
uint8_t tick_tock = 0;
54+
uint64_t volume = 1;
55+
bid_px = MID_PRICE - 1;
56+
ask_px = MID_PRICE + 1;
57+
for (int i = 0; i < MSG_COUNT; i++) {
58+
auto msg = FIX44::MarketDataIncrementalRefresh();
59+
auto change = FIX44::MarketDataIncrementalRefresh::NoMDEntries();
60+
change.set(FIX::Symbol("BTCUSDT"));
61+
change.set(FIX::MDUpdateAction(FIX::MDUpdateAction_NEW));
62+
if (tick_tock == 0) {
63+
change.set(FIX::MDEntryType(FIX::MDEntryType_BID));
64+
change.set(FIX::MDEntryPx(bid_px--));
65+
if (bid_px < MID_PRICE - DEPTH_LEVELS) {
66+
bid_px = MID_PRICE - 1;
67+
}
68+
change.set(FIX::MDEntrySize(volume));
69+
tick_tock = 1;
70+
} else {
71+
change.set(FIX::MDEntryType(FIX::MDEntryType_OFFER));
72+
change.set(FIX::MDEntryPx(ask_px++));
73+
if (ask_px > MID_PRICE + DEPTH_LEVELS) {
74+
ask_px = MID_PRICE + 1;
75+
}
76+
change.set(FIX::MDEntrySize(volume++));
77+
tick_tock = 0;
78+
}
79+
msg.addGroup(change);
80+
test_messages_[i] = msg;
81+
}
5182
}
5283

5384
void TearDown([[maybe_unused]] const benchmark::State& state) override {
54-
const auto vec = book_->to_vector();
55-
spdlog::info("Levels: [{}]", fmt::join(vec, ", "));
85+
// const auto vec = book_->to_vector();
86+
// spdlog::info("Levels: [{}]", fmt::join(vec, ", "));
5687
}
5788

5889
// order book
5990
std::unique_ptr<core::OrderBook> book_;
6091
absl::btree_map<uint64_t, uint64_t, std::greater<>> bids_;
6192
absl::btree_map<uint64_t, uint64_t> asks_;
62-
//
6393
static constexpr uint64_t DEPTH_LEVELS = 500;
6494
static constexpr uint64_t MID_PRICE = 100'000;
95+
// test messages
96+
static constexpr int MSG_COUNT = 1000;
97+
std::array<FIX44::MarketDataIncrementalRefresh, 1000> test_messages_;
6598
};
6699

67-
BENCHMARK_DEFINE_F(OrderBookFixture, BM_AddOrder)(benchmark::State& state) {
68-
FIX44::MarketDataIncrementalRefresh msg;
69-
FIX44::MarketDataIncrementalRefresh::NoMDEntries change;
70-
uint8_t tick_tock = 0;
71-
uint64_t volume = 1;
72-
uint64_t bid_px = MID_PRICE - 1;
73-
uint64_t ask_px = MID_PRICE + 1;
100+
/// @brief deterministically populate the order book
101+
BENCHMARK_DEFINE_F(PriceUpdateFixture, BM_PriceUpdate)(benchmark::State& state) {
102+
int i = 0;
74103
for (auto _ : state) {
75-
msg = FIX44::MarketDataIncrementalRefresh();
76-
change = FIX44::MarketDataIncrementalRefresh::NoMDEntries();
77-
change.set(FIX::Symbol("BTCUSDT"));
78-
change.set(FIX::MDUpdateAction(FIX::MDUpdateAction_NEW));
79-
if (tick_tock == 0) {
80-
change.set(FIX::MDEntryType(FIX::MDEntryType_BID));
81-
change.set(FIX::MDEntryPx(bid_px--));
82-
if (bid_px < MID_PRICE - DEPTH_LEVELS) {
83-
bid_px = MID_PRICE - 1;
84-
}
85-
change.set(FIX::MDEntrySize(volume));
86-
tick_tock = 1;
87-
} else {
88-
change.set(FIX::MDEntryType(FIX::MDEntryType_OFFER));
89-
change.set(FIX::MDEntryPx(ask_px++));
90-
if (ask_px > MID_PRICE + DEPTH_LEVELS) {
91-
ask_px = MID_PRICE + 1;
92-
}
93-
change.set(FIX::MDEntrySize(volume++));
94-
tick_tock = 0;
104+
book_->apply_increment(test_messages_[i], false);
105+
if (i == MSG_COUNT - 1) {
106+
i = 0;
95107
}
96-
msg.addGroup(change);
97-
book_->apply_increment(msg, false);
98108
}
99109

100110
state.counters["Updates/sec"] =
101111
benchmark::Counter(state.iterations(), benchmark::Counter::kIsRate);
102112
}
103113

104-
BENCHMARK_REGISTER_F(OrderBookFixture, BM_AddOrder)->Iterations(500'000);
114+
BENCHMARK_REGISTER_F(PriceUpdateFixture, BM_PriceUpdate)->Iterations(500'000);

‎binance/fixconfig‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ SocketConnectHost=127.0.0.1
1515
FileStorePath=qf_files
1616
FileLogPath=qf_logs
1717
FileLogHeartbeats=N
18-
FileLogMessages=Y
18+
FileLogMessages=N
1919
FileLogEvent=Y
2020

2121
[SESSION]

‎scripts/upload_to_influx.py‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
#!/usr/bin/env python3
2+
import json, os, requests, time
3+
4+
# GitHub Actions environment variables for commit info
5+
commit = os.getenv("GITHUB_SHA", "")[:8]
6+
branch = os.getenv("GITHUB_REF_NAME", "unknown")
7+
source = os.getenv("BENCH_SOURCE", "unknown")
8+
9+
# InfluxDB config from GitHub secrets
10+
INFLUX_URL = os.environ["INFLUX_URL"]
11+
INFLUX_ORG = os.environ["INFLUX_ORG"]
12+
INFLUX_BUCKET = os.environ["INFLUX_BUCKET"]
13+
INFLUX_TOKEN = os.environ["INFLUX_TOKEN"]
14+
15+
timestamp = int(time.time() * 1e9) # nanoseconds
16+
17+
with open("bench_results.json") as f:
18+
data = json.load(f)
19+
20+
lines = []
21+
for bm in data.get("benchmarks", []):
22+
name = bm["name"].replace(" ", "_").replace("/", "_")
23+
real_time = bm.get("real_time", 0.0)
24+
cpu_time = bm.get("cpu_time", 0.0)
25+
items_per_sec = bm.get("items_per_second", 0.0)
26+
iterations = bm.get("iterations", 0)
27+
28+
# Line Protocol with 'source' as a tag
29+
line = (
30+
f"benchmarks,name={name},commit={commit},branch={branch},source={source} "
31+
f"real_time={real_time},cpu_time={cpu_time},items_per_sec={items_per_sec},iterations={iterations} "
32+
f"{timestamp}"
33+
)
34+
lines.append(line)
35+
36+
body = "\n".join(lines)
37+
38+
resp = requests.post(
39+
f"{INFLUX_URL}/api/v2/write?org={INFLUX_ORG}&bucket={INFLUX_BUCKET}&precision=ns",
40+
headers={
41+
"Authorization": f"Token {INFLUX_TOKEN}",
42+
"Content-Type": "text/plain; charset=utf-8",
43+
},
44+
data=body,
45+
)
46+
47+
if resp.status_code != 204:
48+
print("❌ Failed to write to InfluxDB:", resp.text)
49+
resp.raise_for_status()
50+
else:
51+
print(f"✅ Uploaded {len(lines)} benchmarks to InfluxDB with source={source}")

‎tests/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ file(GLOB_RECURSE TEST_SOURCES
1010
)
1111

1212
add_executable(unit_tests
13-
test_main.cpp
13+
main.cpp
1414
${TEST_SOURCES}
1515
)
1616

tests/test_main.cpp renamed to tests/main.cpp

File renamed without changes.

0 commit comments

Comments
 (0)