feat: cpu isolation, and pinned cpus as config

milsanore · milsanore · commit b5196537647e · 2025-11-03T22:43:32.000+11:00
diff --git a/.env.example b/.env.example
@@ -1,6 +1,12 @@
+# LOGGING
 LOG_LEVEL="info"
 LOG_PATH="logs/log"
+# AUTHENTICATION
 API_KEY=""
 FIX_CONFIG_PATH="binance/fixconfig"
 PRIVATE_KEY_PATH="key.pem"
+#
 SYMBOLS=BTCUSDT,
+# CPU PINNING
+PX_SESSION_CPU=0
+TX_SESSION_CPU=1
diff --git a/Makefile b/Makefile
@@ -74,6 +74,7 @@ run-debug:
 ## run-release: 🏎️  run the app (prod)
 .PHONY: run-release
 run-release:
+	$(call pp,starting app. dont forget to run `scripts/cpu_shield_start.sh`)
 	build/Release/tradercpp
 
 # CONTAINERISATION RECIPES ----------------------------------------------------
diff --git a/README.md b/README.md
@@ -165,6 +165,10 @@ NB: this app uses `make` as a recipe book, but it's not essential:
   - OS
     - vacate OS services / move IRQs for all system devices to the last CPU
     - RTOS
+  - BIOS
+    - disable hyperthreading, turbo boost
+    - disable C-states deeper than C1 (C1E, C6, etc)
+    - set cpu governor to "performance"
   - Memory locking
   - sparse arrays & flat matrix
   - memory-mapped files
diff --git a/scripts/cpu_shield_start.sh b/scripts/cpu_shield_start.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# ------------------------------------------------------------
+# move all OS threads off of trader CPUs
+# ------------------------------------------------------------
+set -e
+
+sudo cset shield --cpu 0-1 --kthread=on
+
+echo "Isolated CPUs 0 and 1 for the trader app, all OS threads will be moved"
diff --git a/scripts/cpu_shield_stop.sh b/scripts/cpu_shield_stop.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# ------------------------------------------------------------
+# reset CPU isolation, allow OS threads to run on all CPUs
+# ------------------------------------------------------------
+set -e
+
+sudo cset shield --reset
+
+echo "Reset CPU isolation, OS threads can run on all CPUs"
diff --git a/src/binance/config.cpp b/src/binance/config.cpp
@@ -2,6 +2,7 @@
 
 #include <format>
 #include <ranges>
+#include <stdexcept>
 #include <string>
 #include <vector>
 
@@ -11,25 +12,43 @@
 namespace binance {
 // TODO(mils): for keys, use `std::vector<unsigned char>` instead of string
 
-/// @brief load Binance configuration from env
-/// (static member function)
-/// @return Config object
+// static function
 Config Config::from_env() {
-  const std::string apiKey = utils::Env::get_env_or_throw("API_KEY");
-  const std::string privateKey = utils::Env::get_env_or_throw("PRIVATE_KEY_PATH");
-  const std::string fixConfig = utils::Env::get_env_or_throw("FIX_CONFIG_PATH");
-  const std::string instStr = utils::Env::get_env_or_throw("SYMBOLS");
-
-  spdlog::info("fetched environment variable. key [API_KEY], value [{}]",
-               apiKey.substr(0, 3) + "..." + apiKey.substr(apiKey.length() - 3));
-  spdlog::info("fetched environment variable. key [PRIVATE_KEY_PATH], value [{}]",
-               privateKey);
-  spdlog::info("fetched environment variable. key [FIX_CONFIG_PATH], value [{}]",
-               fixConfig);
-  spdlog::info("fetched environment variable. key [SYMBOLS], value [{}]", instStr);
+  const std::string api_key = utils::Env::get_env_or_throw("API_KEY");
+  const std::string private_key = utils::Env::get_env_or_throw("PRIVATE_KEY_PATH");
+  const std::string fix_config = utils::Env::get_env_or_throw("FIX_CONFIG_PATH");
+  const std::string px_cpu_str = utils::Env::get_env_or_throw("PX_SESSION_CPU");
+  const std::string tx_cpu_str = utils::Env::get_env_or_throw("TX_SESSION_CPU");
+  const std::string inst_str = utils::Env::get_env_or_throw("SYMBOLS");
+
+  uint8_t px_cpu;
+  std::errc px_ec =
+      std::from_chars(px_cpu_str.data(), px_cpu_str.data() + px_cpu_str.size(), px_cpu)
+          .ec;
+  if (px_ec != std::errc()) {
+    throw std::runtime_error(
+        std::format("could not parse px cpu, value [{}]", px_cpu_str));
+  }
+
+  uint8_t tx_cpu;
+  std::errc tx_ec =
+      std::from_chars(tx_cpu_str.data(), tx_cpu_str.data() + tx_cpu_str.size(), tx_cpu)
+          .ec;
+  if (tx_ec != std::errc()) {
+    throw std::runtime_error(
+        std::format("could not parse tx cpu, value [{}]", tx_cpu_str));
+  }
+
+  spdlog::info("fetched envar. key [API_KEY], value [{}]",
+               api_key.substr(0, 3) + "..." + api_key.substr(api_key.length() - 3));
+  spdlog::info("fetched envar. key [PRIVATE_KEY_PATH], value [{}]", private_key);
+  spdlog::info("fetched envar. key [FIX_CONFIG_PATH], value [{}]", fix_config);
+  spdlog::info("fetched envar. key [SYMBOLS], value [{}]", inst_str);
+  spdlog::info("fetched envar. key [PX_SESSION_CPU], value [{}]", px_cpu_str);
+  spdlog::info("fetched envar. key [TX_SESSION_CPU], value [{}]", tx_cpu_str);
 
   std::vector<std::string> symbols;
-  for (auto inst : std::views::split(instStr, ',')) {
+  for (auto inst : std::views::split(inst_str, ',')) {
     if (!inst.empty()) {
       symbols.emplace_back(inst.begin(), inst.end());
     }
@@ -38,7 +57,7 @@ Config Config::from_env() {
   spdlog::info("MAX_DEPTH, value [{}]", MAX_DEPTH);
 
   // copy
-  return Config{apiKey, privateKey, fixConfig, symbols};
+  return Config{api_key, private_key, fix_config, symbols, px_cpu, tx_cpu};
 };
 
 }  // namespace binance
diff --git a/src/binance/config.h b/src/binance/config.h
@@ -1,5 +1,6 @@
 #pragma once
 
+#include <charconv>
 #include <cmath>
 #include <cstdint>
 #include <format>
@@ -16,11 +17,27 @@ namespace binance {
 /// @brief Binance config parameters, fetched from env
 struct Config {
  public:
-  // TODO: MAKE UNIQUE POINTERS
   std::string api_key, private_key_path;
   const std::string fix_config_path;
   const std::vector<std::string> symbols;
-  //
+  const uint8_t px_cpu;
+  const uint8_t tx_cpu;
+
+  // Constructor that initializes all const members
+  Config(std::string api,
+         std::string private_key,
+         std::string fix_config,
+         std::vector<std::string> syms,
+         uint8_t px,
+         uint8_t tx)
+      : api_key(std::move(api)),
+        private_key_path(std::move(private_key)),
+        fix_config_path(std::move(fix_config)),
+        symbols(std::move(syms)),
+        px_cpu(px),
+        tx_cpu(tx) {}
+
+  /// @brief load Binance configuration parameters from environment variables
   static Config from_env();
 
   /// @brief convert a market price to a tick representation, for performance and
@@ -58,9 +75,6 @@ struct Config {
 
   /// @brief 1 == top level, otherwise 5000 is Binance's maximum depth
   static constexpr uint16_t MAX_DEPTH = 100;
-
-  static constexpr uint8_t PX_SESSION_CPU_AFFINITY = 0;
-  static constexpr uint8_t TX_SESSION_CPU_AFFINITY = 1;
 };
 
 }  // namespace binance
diff --git a/src/binance/fix_app.cpp b/src/binance/fix_app.cpp
@@ -27,13 +27,13 @@ namespace binance {
 FixApp::FixApp(const std::vector<std::string>& symbols,
                std::unique_ptr<IAuth> auth,
                const uint16_t MAX_DEPTH,
-               const uint8_t PX_SESSION_CPU_AFFINITY,
-               const uint8_t TX_SESSION_CPU_AFFINITY)
+               const uint8_t px_cpu,
+               const uint8_t tx_cpu)
     : symbols_(symbols),
       auth_(std::move(auth)),
       MAX_DEPTH_(MAX_DEPTH),
-      PX_SESSION_CPU_AFFINITY_(PX_SESSION_CPU_AFFINITY),
-      TX_SESSION_CPU_AFFINITY_(TX_SESSION_CPU_AFFINITY) {}
+      px_cpu_(px_cpu),
+      tx_cpu_(tx_cpu) {}
 
 void FixApp::subscribe_to_prices(const FIX::SessionID& session_id) const {
   spdlog::info("subscribing to depth. qualifier [{}], id [{}]",
@@ -135,11 +135,11 @@ void FixApp::onLogon(const FIX::SessionID& sessionId) {
                utils::Threading::get_os_thread_id());
 
   if (sessionId.getSessionQualifier() == PX_SESSION_QUALIFIER_) {
-    utils::Threading::set_thread_affinity(PX_SESSION_CPU_AFFINITY_);
+    utils::Threading::set_thread_cpu(px_cpu_);
     utils::Threading::set_thread_realtime();
     subscribe_to_prices(sessionId);
   } else if (sessionId.getSessionQualifier() == TX_SESSION_QUALIFIER_) {
-    utils::Threading::set_thread_affinity(TX_SESSION_CPU_AFFINITY_);
+    utils::Threading::set_thread_cpu(tx_cpu_);
     subscribe_to_trades(sessionId);
   } else if (sessionId.getSessionQualifier() == OX_SESSION_QUALIFIER_) {
     // do nothing for order session
diff --git a/src/binance/fix_app.h b/src/binance/fix_app.h
@@ -21,8 +21,8 @@ class FixApp final : public FIX::Application, public FIX44::MessageCracker {
   FixApp(const std::vector<std::string>& symbols,
          std::unique_ptr<IAuth> auth,
          const uint16_t MAX_DEPTH,
-         const uint8_t PX_SESSION_CPU_AFFINITY,
-         const uint8_t TX_SESSION_CPU_AFFINITY);
+         const uint8_t px_cpu,
+         const uint8_t tx_cpu);
   ~FixApp() override = default;
 
   // Use the FIX44::MessageCracker to pull in the relevant overloads. This resolves the
@@ -51,8 +51,8 @@ class FixApp final : public FIX::Application, public FIX44::MessageCracker {
   const std::vector<std::string>& symbols_;
   const std::unique_ptr<IAuth> auth_;
   const uint16_t MAX_DEPTH_;
-  const uint8_t PX_SESSION_CPU_AFFINITY_;
-  const uint8_t TX_SESSION_CPU_AFFINITY_;
+  const uint8_t px_cpu_;
+  const uint8_t tx_cpu_;
 
   void onCreate(const FIX::SessionID&) override;
   void onLogon(const FIX::SessionID&) override;
diff --git a/src/binance/worker.cpp b/src/binance/worker.cpp
@@ -32,8 +32,7 @@ Worker Worker::from_conf(Config& conf) {
   std::unique_ptr<IAuth> auth =
       std::make_unique<Auth>(conf.api_key, conf.private_key_path);
   auto app = std::make_unique<FixApp>(conf.symbols, std::move(auth), conf.MAX_DEPTH,
-                                      conf.PX_SESSION_CPU_AFFINITY,
-                                      conf.TX_SESSION_CPU_AFFINITY);
+                                      conf.px_cpu, conf.tx_cpu);
   auto settings = FIX::SessionSettings{conf.fix_config_path};
   auto store = std::make_unique<FIX::FileStoreFactory>(settings);
   auto log = std::make_unique<FIX::FileLogFactory>(settings);
diff --git a/src/utils/threading.cpp b/src/utils/threading.cpp
@@ -153,7 +153,7 @@ void Threading::set_thread_affinity(std::thread& t, unsigned int cpu_id) {
 }
 
 // static function
-void Threading::set_thread_affinity(unsigned int cpu_id) {
+void Threading::set_thread_cpu(unsigned int cpu_id) {
 #if defined(__linux__) || defined(__APPLE__)
   return set_native_thread_affinity(pthread_self(), cpu_id);
 #elif defined(_WIN32)
diff --git a/src/utils/threading.h b/src/utils/threading.h
@@ -29,8 +29,6 @@ class Threading {
   static uint64_t get_os_thread_id();
 
   /// @brief Retrieves the number of logical CPU cores currently available to the process.
-  /// This function returns the count of online logical processors (hardware threads)
-  /// that the operating system makes available to the calling process.
   /// On Linux and macOS, it queries `sysconf(_SC_NPROCESSORS_ONLN)` for an accurate,
   /// real-time count that respects CPU hotplug events and container (cgroup) limits.
   /// On Windows, it uses `GetSystemInfo()`. If platform-specific calls fail, the function
@@ -40,26 +38,20 @@ class Threading {
   /// as multiple logical processors.
   /// @return The number of available logical CPUs (>= 1). If detection fails, returns 1.
   /// @note CPU indices are zero-based, ranging from 0 to (get_cpu_count() - 1).
-  /// @see set_thread_affinity(), set_native_thread_affinity()
   static unsigned int get_cpu_count();
 
   /// @brief Sets the CPU affinity for the calling (current) thread.
-  /// Attempts to pin the current thread to a specific logical CPU core, preventing
-  /// the operating system scheduler from migrating it to other cores.
-  /// CPU affinity can improve performance and reduce latency by keeping execution
-  /// on a fixed CPU, preserving cache locality and minimizing context switches.
+  /// Attempts to pin the current thread to a specific logical CPU core,
+  /// preserving cache locality and minimizing context switches.
   /// On Linux, this uses `pthread_setaffinity_np()` to bind the thread to the target CPU.
   /// On macOS, it applies a best-effort affinity hint via `thread_policy_set()`, since
-  /// strict CPU pinning is not supported by the kernel. On Windows, it uses
-  /// `SetThreadAffinityMask()` to achieve true CPU binding.
+  /// strict CPU pinning is not supported by the kernel.
+  /// On Windows, it uses `SetThreadAffinityMask()` to achieve true CPU binding.
   /// @note On macOS, affinity is advisory only and may not guarantee the thread remains
   ///       on the specified CPU.
-  /// @note This function only affects the calling thread. To set affinity for another
-  ///       thread, use @ref set_thread_affinity(std::thread&, unsigned int).
-  /// @see get_cpu_count(), set_thread_affinity(), set_native_thread_affinity()
   /// @param cpu_id The zero-based index of the target logical CPU.
   ///               Valid range is `[0, get_cpu_count() - 1]`.
-  static void set_thread_affinity(unsigned int cpu_id);
+  static void set_thread_cpu(unsigned int cpu_id);
 
   /// @brief Elevates the current thread to real-time priority.
   /// Attempts to set the calling thread to the highest real-time priority:
diff --git a/tests/binance/config_test.cpp b/tests/binance/config_test.cpp
@@ -12,23 +12,27 @@ TEST(ConfigTest, FromEnv) {
   setenv("PRIVATE_KEY_PATH", "/path/to/private.key", 1);
   setenv("FIX_CONFIG_PATH", "/path/to/fix.cfg", 1);
   setenv("SYMBOLS", "BTCUSDT,ETHUSDT,BNBUSDT", 1);
-
-  Config cfg = Config::from_env();
+  setenv("PX_SESSION_CPU", "0", 1);
+  setenv("TX_SESSION_CPU", "1", 1);
 
   // Validate
+  Config cfg = Config::from_env();
   EXPECT_EQ(cfg.api_key, "test-api-key");
   EXPECT_EQ(cfg.private_key_path, "/path/to/private.key");
   EXPECT_EQ(cfg.fix_config_path, "/path/to/fix.cfg");
-
   std::vector<std::string> expectedSymbols = {"BTCUSDT", "ETHUSDT", "BNBUSDT"};
   EXPECT_EQ(cfg.symbols, expectedSymbols);
+  EXPECT_EQ(cfg.px_cpu, 0);
+  EXPECT_EQ(cfg.tx_cpu, 1);
 }
 
 TEST(ConfigTest, ThrowsOnMissingEnv) {
   unsetenv("API_KEY");
   setenv("PRIVATE_KEY_PATH", "somekey", 1);
   setenv("FIX_CONFIG_PATH", "somecfg", 1);
   setenv("SYMBOLS", "BTCUSDT", 1);
+  setenv("PX_SESSION_CPU", "0", 1);
+  setenv("TX_SESSION_CPU", "1", 1);
 
   EXPECT_THROW(Config::from_env(), std::runtime_error);
 }
@@ -38,6 +42,8 @@ TEST(ConfigTest, HandlesEmptySymbols) {
   setenv("PRIVATE_KEY_PATH", "keypath", 1);
   setenv("FIX_CONFIG_PATH", "fix", 1);
   setenv("SYMBOLS", "", 1);
+  setenv("PX_SESSION_CPU", "0", 1);
+  setenv("TX_SESSION_CPU", "1", 1);
 
   EXPECT_THROW(Config::from_env(), std::runtime_error);
 }
@@ -47,6 +53,8 @@ TEST(ConfigTest, HandlesTrailingComma) {
   setenv("PRIVATE_KEY_PATH", "keypath", 1);
   setenv("FIX_CONFIG_PATH", "fix", 1);
   setenv("SYMBOLS", "BTCUSDT,", 1);
+  setenv("PX_SESSION_CPU", "0", 1);
+  setenv("TX_SESSION_CPU", "1", 1);
 
   Config cfg = Config::from_env();
   std::vector<std::string> expected = {"BTCUSDT"};
diff --git a/tests/ui/trade_box_test.cpp b/tests/ui/trade_box_test.cpp
@@ -27,7 +27,7 @@ class TradeBoxTest : public ::testing::Test {
   DummyScreen screen_;
   moodycamel::ConcurrentQueue<std::shared_ptr<const FIX44::Message>> queue_;
   std::unique_ptr<ui::TradeBox> trade_box_;
-  binance::Config config_;
+  binance::Config config_{"", "", "", std::vector<std::string>{}, 0, 0};
 
   void SetUp() override {
     trade_box_ = std::make_unique<ui::TradeBox>(screen_, config_, queue_);
diff --git a/tests/ui/ui_app_test.cpp b/tests/ui/ui_app_test.cpp
@@ -28,7 +28,7 @@ TEST(App, start) {
 
   constexpr int MAX_DEPTH = 50;
   auto book_box = std::make_unique<ui::OrderBookBox>(*screen, order_queue, MAX_DEPTH);
-  auto bconf = binance::Config();
+  binance::Config bconf{"", "", "", std::vector<std::string>{}, 0, 0};
   auto trade_box = std::make_unique<ui::TradeBox>(
       *screen, bconf, trade_queue,
       []([[maybe_unused]] const std::stop_token& stoken) { spdlog::info("mock task"); });
