feat: elevated process priority, realtime thread, cpu isolation, and pinned cpus as config

Author: milsanore

.env.example

@@ -1,6 +1,12 @@
+# LOGGING
 LOG_LEVEL="info"
 LOG_PATH="logs/log"
+# AUTHENTICATION
 API_KEY=""
 FIX_CONFIG_PATH="binance/fixconfig"
 PRIVATE_KEY_PATH="key.pem"
+#
 SYMBOLS=BTCUSDT,
+# CPU PINNING
+PX_SESSION_CPU=0
+TX_SESSION_CPU=1

Makefile

@@ -49,7 +49,7 @@ build-release:
 	cmake --preset=release
 	cmake --build --preset=release
 
-## test: 🧪 run google-test
+## test: 🧪 run unit tests
 .PHONY: test
 test:
 	cmake --preset debug
@@ -74,6 +74,7 @@ run-debug:
 ## run-release: 🏎️  run the app (prod)
 .PHONY: run-release
 run-release:
+	$(call pp,starting app. dont forget to run `scripts/cpu_shield_start.sh`)
 	build/Release/tradercpp
 
 # CONTAINERISATION RECIPES ----------------------------------------------------

README.md

@@ -12,6 +12,7 @@ A proof-of-concept, showcasing some modern c++ and some fintech concepts.
 - linux (tested with debian-trixie)
 - `stunnel` for TLS encryption (or a local proxy)
 - a Binance account, and an Ed25519 token with FIX read permissions enabled
+- `sudo` permissions (for elevated process and thread priorities)
 
 ## Run
 - start an SSL tunnel
@@ -24,7 +25,7 @@ A proof-of-concept, showcasing some modern c++ and some fintech concepts.
 - run:
   - download release
   - `chmod u+x tradercpp`
-  - `./tradercpp`
+  - `sudo ./tradercpp`
 
 ## Build Requirements
 - C++20
@@ -48,7 +49,7 @@ NB: this app uses `make` as a recipe book, but it's not essential:
 2. run an SSL tunnel (e.g. `stunnel binance/stunnel_prod.conf`)
 3. `make init`
 4. `make build-debug`
-5. `make withenv RECIPE=run-debug`
+5. `sudo make withenv RECIPE=run-debug`
 
 ## Test
 `make test`
@@ -149,9 +150,26 @@ NB: this app uses `make` as a recipe book, but it's not essential:
   - profiling (valgrind/cachegrind)
   - profile-guided optimization (pgo)
   - load test with mocked FIX server
-  - set process priority
-  - NIC affinity
-  - QoS
+  - CPU
+    - ✅ process priority
+    - ✅ FIX-thread "realtime"
+    - ✅ FIX-thread CPU affinity
+    - Disable hyperthreading
+  - NIC
+    - wired, kernel-bypass NICs
+    - NIC IRQ affinity to same CPU as FIX
+    - hardware queue affinity
+    - QoS (mark packets)
+    - AF_XDP (+ Zero-copy mode)
+    - ~dedicated NIC + DPDK~
+  - OS
+    - vacate OS services / move IRQs for all system devices to the last CPU
+    - RTOS
+  - BIOS
+    - disable hyperthreading, turbo boost
+    - disable C-states deeper than C1 (C1E, C6, etc)
+    - set cpu governor to "performance"
+  - Memory locking
   - sparse arrays & flat matrix
   - memory-mapped files
   - (analyse) find Binance's server location for a low-latency connection
@@ -162,6 +180,7 @@ NB: this app uses `make` as a recipe book, but it's not essential:
   - RT OS
 - ✅ logging
   - ✅ fast
+  - add console target for fatal messages
   - error handling
   - compiled out 'debug' logging for release builds
   - thread name in logs

scripts/cpu_shield_start.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# ------------------------------------------------------------
+# move all OS threads off of trader CPUs
+# ------------------------------------------------------------
+set -e
+
+sudo cset shield --cpu 0-1 --kthread=on
+
+echo "Isolated CPUs 0 and 1 for the trader app, all OS threads will be moved"

scripts/cpu_shield_stop.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# ------------------------------------------------------------
+# reset CPU isolation, allow OS threads to run on all CPUs
+# ------------------------------------------------------------
+set -e
+
+sudo cset shield --reset
+
+echo "Reset CPU isolation, OS threads can run on all CPUs"

src/binance/config.cpp

@@ -2,6 +2,7 @@
 
 #include <format>
 #include <ranges>
+#include <stdexcept>
 #include <string>
 #include <vector>
 
@@ -11,25 +12,43 @@
 namespace binance {
 // TODO(mils): for keys, use `std::vector<unsigned char>` instead of string
 
-/// @brief load Binance configuration from env
-/// (static member function)
-/// @return Config object
+// static function
 Config Config::from_env() {
-  const std::string apiKey = utils::Env::get_env_or_throw("API_KEY");
-  const std::string privateKey = utils::Env::get_env_or_throw("PRIVATE_KEY_PATH");
-  const std::string fixConfig = utils::Env::get_env_or_throw("FIX_CONFIG_PATH");
-  const std::string instStr = utils::Env::get_env_or_throw("SYMBOLS");
-
-  spdlog::info("fetched environment variable. key [API_KEY], value [{}]",
-               apiKey.substr(0, 3) + "..." + apiKey.substr(apiKey.length() - 3));
-  spdlog::info("fetched environment variable. key [PRIVATE_KEY_PATH], value [{}]",
-               privateKey);
-  spdlog::info("fetched environment variable. key [FIX_CONFIG_PATH], value [{}]",
-               fixConfig);
-  spdlog::info("fetched environment variable. key [SYMBOLS], value [{}]", instStr);
+  const std::string api_key = utils::Env::get_env_or_throw("API_KEY");
+  const std::string private_key = utils::Env::get_env_or_throw("PRIVATE_KEY_PATH");
+  const std::string fix_config = utils::Env::get_env_or_throw("FIX_CONFIG_PATH");
+  const std::string px_cpu_str = utils::Env::get_env_or_throw("PX_SESSION_CPU");
+  const std::string tx_cpu_str = utils::Env::get_env_or_throw("TX_SESSION_CPU");
+  const std::string inst_str = utils::Env::get_env_or_throw("SYMBOLS");
+
+  uint8_t px_cpu;
+  std::errc px_ec =
+      std::from_chars(px_cpu_str.data(), px_cpu_str.data() + px_cpu_str.size(), px_cpu)
+          .ec;
+  if (px_ec != std::errc()) {
+    throw std::runtime_error(
+        std::format("could not parse px cpu, value [{}]", px_cpu_str));
+  }
+
+  uint8_t tx_cpu;
+  std::errc tx_ec =
+      std::from_chars(tx_cpu_str.data(), tx_cpu_str.data() + tx_cpu_str.size(), tx_cpu)
+          .ec;
+  if (tx_ec != std::errc()) {
+    throw std::runtime_error(
+        std::format("could not parse tx cpu, value [{}]", tx_cpu_str));
+  }
+
+  spdlog::info("fetched envar. key [API_KEY], value [{}]",
+               api_key.substr(0, 3) + "..." + api_key.substr(api_key.length() - 3));
+  spdlog::info("fetched envar. key [PRIVATE_KEY_PATH], value [{}]", private_key);
+  spdlog::info("fetched envar. key [FIX_CONFIG_PATH], value [{}]", fix_config);
+  spdlog::info("fetched envar. key [SYMBOLS], value [{}]", inst_str);
+  spdlog::info("fetched envar. key [PX_SESSION_CPU], value [{}]", px_cpu_str);
+  spdlog::info("fetched envar. key [TX_SESSION_CPU], value [{}]", tx_cpu_str);
 
   std::vector<std::string> symbols;
-  for (auto inst : std::views::split(instStr, ',')) {
+  for (auto inst : std::views::split(inst_str, ',')) {
     if (!inst.empty()) {
       symbols.emplace_back(inst.begin(), inst.end());
     }
@@ -38,7 +57,7 @@ Config Config::from_env() {
   spdlog::info("MAX_DEPTH, value [{}]", MAX_DEPTH);
 
   // copy
-  return Config{apiKey, privateKey, fixConfig, symbols};
+  return Config{api_key, private_key, fix_config, symbols, px_cpu, tx_cpu};
 };
 
 }  // namespace binance

src/binance/config.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <charconv>
 #include <cmath>
 #include <cstdint>
 #include <format>
@@ -16,11 +17,27 @@ namespace binance {
 /// @brief Binance config parameters, fetched from env
 struct Config {
  public:
-  // TODO: MAKE UNIQUE POINTERS
   std::string api_key, private_key_path;
   const std::string fix_config_path;
   const std::vector<std::string> symbols;
-  //
+  const uint8_t px_cpu;
+  const uint8_t tx_cpu;
+
+  // Constructor that initializes all const members
+  Config(std::string api,
+         std::string private_key,
+         std::string fix_config,
+         std::vector<std::string> syms,
+         uint8_t px,
+         uint8_t tx)
+      : api_key(std::move(api)),
+        private_key_path(std::move(private_key)),
+        fix_config_path(std::move(fix_config)),
+        symbols(std::move(syms)),
+        px_cpu(px),
+        tx_cpu(tx) {}
+
+  /// @brief load Binance configuration parameters from environment variables
   static Config from_env();
 
   /// @brief convert a market price to a tick representation, for performance and
@@ -58,9 +75,6 @@ struct Config {
 
   /// @brief 1 == top level, otherwise 5000 is Binance's maximum depth
   static constexpr uint16_t MAX_DEPTH = 100;
-
-  static constexpr uint8_t PX_SESSION_CPU_AFFINITY = 0;
-  static constexpr uint8_t TX_SESSION_CPU_AFFINITY = 1;
 };
 
 }  // namespace binance

src/binance/fix_app.cpp

@@ -27,13 +27,13 @@ namespace binance {
 FixApp::FixApp(const std::vector<std::string>& symbols,
                std::unique_ptr<IAuth> auth,
                const uint16_t MAX_DEPTH,
-               const uint8_t PX_SESSION_CPU_AFFINITY,
-               const uint8_t TX_SESSION_CPU_AFFINITY)
+               const uint8_t px_cpu,
+               const uint8_t tx_cpu)
     : symbols_(symbols),
       auth_(std::move(auth)),
       MAX_DEPTH_(MAX_DEPTH),
-      PX_SESSION_CPU_AFFINITY_(PX_SESSION_CPU_AFFINITY),
-      TX_SESSION_CPU_AFFINITY_(TX_SESSION_CPU_AFFINITY) {}
+      px_cpu_(px_cpu),
+      tx_cpu_(tx_cpu) {}
 
 void FixApp::subscribe_to_prices(const FIX::SessionID& session_id) const {
   spdlog::info("subscribing to depth. qualifier [{}], id [{}]",
@@ -135,10 +135,11 @@ void FixApp::onLogon(const FIX::SessionID& sessionId) {
                utils::Threading::get_os_thread_id());
 
   if (sessionId.getSessionQualifier() == PX_SESSION_QUALIFIER_) {
-    utils::Threading::set_current_thread_affinity(PX_SESSION_CPU_AFFINITY_);
+    utils::Threading::set_thread_cpu(px_cpu_);
+    utils::Threading::set_thread_realtime();
     subscribe_to_prices(sessionId);
   } else if (sessionId.getSessionQualifier() == TX_SESSION_QUALIFIER_) {
-    utils::Threading::set_current_thread_affinity(TX_SESSION_CPU_AFFINITY_);
+    utils::Threading::set_thread_cpu(tx_cpu_);
     subscribe_to_trades(sessionId);
   } else if (sessionId.getSessionQualifier() == OX_SESSION_QUALIFIER_) {
     // do nothing for order session

src/binance/fix_app.h

@@ -21,8 +21,8 @@ class FixApp final : public FIX::Application, public FIX44::MessageCracker {
   FixApp(const std::vector<std::string>& symbols,
          std::unique_ptr<IAuth> auth,
          const uint16_t MAX_DEPTH,
-         const uint8_t PX_SESSION_CPU_AFFINITY,
-         const uint8_t TX_SESSION_CPU_AFFINITY);
+         const uint8_t px_cpu,
+         const uint8_t tx_cpu);
   ~FixApp() override = default;
 
   // Use the FIX44::MessageCracker to pull in the relevant overloads. This resolves the
@@ -51,8 +51,8 @@ class FixApp final : public FIX::Application, public FIX44::MessageCracker {
   const std::vector<std::string>& symbols_;
   const std::unique_ptr<IAuth> auth_;
   const uint16_t MAX_DEPTH_;
-  const uint8_t PX_SESSION_CPU_AFFINITY_;
-  const uint8_t TX_SESSION_CPU_AFFINITY_;
+  const uint8_t px_cpu_;
+  const uint8_t tx_cpu_;
 
   void onCreate(const FIX::SessionID&) override;
   void onLogon(const FIX::SessionID&) override;

src/binance/worker.cpp

@@ -32,8 +32,7 @@ Worker Worker::from_conf(Config& conf) {
   std::unique_ptr<IAuth> auth =
       std::make_unique<Auth>(conf.api_key, conf.private_key_path);
   auto app = std::make_unique<FixApp>(conf.symbols, std::move(auth), conf.MAX_DEPTH,
-                                      conf.PX_SESSION_CPU_AFFINITY,
-                                      conf.TX_SESSION_CPU_AFFINITY);
+                                      conf.px_cpu, conf.tx_cpu);
   auto settings = FIX::SessionSettings{conf.fix_config_path};
   auto store = std::make_unique<FIX::FileStoreFactory>(settings);
   auto log = std::make_unique<FIX::FileLogFactory>(settings);