Merge remote-tracking branch 'origin/master' into stats-box

Author: milsanore

.env.example

@@ -10,3 +10,6 @@ SYMBOLS=BTCUSDT,
 # CPU PINNING
 PX_SESSION_CPU=0
 TX_SESSION_CPU=1
+# CPU ISOLATION
+CPU_SET_NAME=tradercpp
+CPU_SET_RANGE="0-1"

.github/workflows/nightly_bench.yml

@@ -11,7 +11,7 @@ jobs:
   build_and_test:
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/milsanore/tradercppbuild:v1.7
+      image: ghcr.io/milsanore/tradercppbuild:v1.8
       # set resource limits for some consistency
       options: --memory=7g --cpus=2
     steps:
@@ -57,3 +57,7 @@ jobs:
           GITHUB_REF_NAME: ${{ github.ref_name }}
           BENCH_SOURCE: github-actions
         run: python3 scripts/upload_to_influx.py
+
+      - name: Prune Conan cache
+        run: |
+          conan cache clean --build

.github/workflows/release.yml

@@ -11,7 +11,7 @@ jobs:
   build_release:
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/milsanore/tradercppbuild:v1.7
+      image: ghcr.io/milsanore/tradercppbuild:v1.8
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -41,3 +41,7 @@ jobs:
         with:
           files: build/Release/tradercpp
           token: ${{ secrets.PAT_RELEASE_20261020 }}
+
+      - name: Prune Conan cache
+        run: |
+          conan cache clean --build

.github/workflows/reusable_build.yml

@@ -16,7 +16,7 @@ jobs:
   build_and_test:
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/milsanore/tradercppbuild:v1.7
+      image: ghcr.io/milsanore/tradercppbuild:v1.8
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -26,6 +26,9 @@ jobs:
       - name: clang-format
         run: hooks/check_clang_format.sh
 
+      - name: shellcheck
+        run: hooks/check_shell.sh
+
       - name: Restore Conan cache
         uses: actions/cache@v4
         with:
@@ -48,7 +51,7 @@ jobs:
       - name: Test
         run: |
           set -e
-          ctest -j$(nproc) --preset=debug
+          ctest -j$(nproc) --preset=debug -LE BENCHMARK
           # `lcov` FOR CODECOV
           lcov --gcov-tool gcov \
             --capture \
@@ -81,3 +84,7 @@ jobs:
             -Dsonar.sources=src
             -Dsonar.tests=tests
         continue-on-error: true
+
+      - name: Prune Conan cache
+        run: |
+          conan cache clean --build

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.16)
+cmake_minimum_required(VERSION 4.0)
 project(tradercpp)
 
 set(CMAKE_CXX_STANDARD 23)

Dockerfile_build

@@ -1,5 +1,5 @@
 #############################################
-# milss/tradercppbuild:v1.7
+# milss/tradercppbuild:v1.8
 # Build container for tradercpp
 # Published to dockerhub in order to accelerate github actions
 # Uses:
@@ -56,12 +56,6 @@ RUN apt update && apt install -y --no-install-recommends \
 
 
 # Install LLVM 18 (clang-format and clang-tidy)
-RUN apt update && apt install -y --no-install-recommends \
-    gnupg \
-    lsb-release \
-    software-properties-common \
-    wget \
-    && rm -rf /var/lib/apt/lists/*
 
 RUN wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
 
@@ -115,5 +109,6 @@ RUN apt update && apt install -y --no-install-recommends \
     make \
     ninja-build \
     perl \
+    shellcheck \
     unzip \
     && rm -rf /var/lib/apt/lists/*

Makefile

@@ -1,10 +1,11 @@
 #!make
 SHELL:=/bin/bash
 
-#################################################
-# A set of basic execution recipes (all PHONY).
-# Call `make` on the command line for documentation.
-#################################################
+# ###############################################
+# A set of execution recipes (all PHONY).
+# In essence, a programmatic entrypoint into the app.
+# Run `make` for help.
+# ###############################################
 
 # pp - pretty print function
 yellow := $(shell tput setaf 3)
@@ -18,33 +19,39 @@ help: Makefile
 	@echo " Choose a command to run:"
 	@sed -n 's/^##//p' $< | column -t -s ':' | sed -e 's/^/ /'
 
-## withenv: 😭 run `make` with envars from `.env`. like so `make withenv RECIPE=init`
+## withenv: 😭 execute `make` with environment variables defined in `.env`. like so > `make withenv RECIPE=init`
 .PHONY: withenv
 withenv:
 	test -e .env || cp .env.example .env
 	bash -c 'set -o allexport; source .env; set +o allexport; make "$$RECIPE"'
 
-## init: 🏌️ initialize the project, fetch dependencies
+## init: 🏌️ initialize the project
 .PHONY: init
 init:
+	$(call pp,initializing project)
+	git config core.hooksPath hooks
 	rm -rf build && mkdir build
 	python3 -m venv .venv
 	source .venv/bin/activate && \
 		pip install gcovr conan && \
 		conan install . --lockfile=conan.lock --build=missing -s build_type=Debug && \
 		conan install . --lockfile=conan.lock --build=missing -s build_type=Release
+	cmake --preset=release
 	cmake --preset=debug
+	$(MAKE) restore-cpus
 
-## lock-conan: 📦 run after adding (but before installing) conan dependencies
-.PHONY: lock-conan
-lock-conan:
-	conan lock create . --profile:host=default -s build_type=Debug   --lockfile-out=conan.lock
-	conan lock create . --profile:host=default -s build_type=Release --lockfile=conan.lock --lockfile-out=conan.lock
+## lock: 📦 update Conan dependency lock file. (NB run after adding conan dependencies to conanfile.txt, but before installing)
+.PHONY: lock
+lock:
+	$(call pp,updating conan lock file to match contents of conanfile.txt)
+	source .venv/bin/activate && \
+		conan lock create . --profile:host=default -s build_type=Debug   --lockfile-out=conan.lock && \
+		conan lock create . --profile:host=default -s build_type=Release --lockfile=conan.lock --lockfile-out=conan.lock
 
 ## build-debug: 🔨 compile (debug)
 .PHONY: build-debug
 build-debug:
-	$(call pp,assuming `make init` has been called)
+	$(call pp,NB assuming `make init` has been called)
 	cmake --preset=debug
 	cmake --build --preset=debug
 
@@ -68,7 +75,6 @@ test:
 ## bench: ⏱️ build and run benchmarks
 .PHONY: bench
 bench:
-	$(call pp,assuming `make build-release` has been called)
 	cmake --build --preset release
 	build/Release/benchmarks/benchmarks \
 	--benchmark_out=bench_results.json \
@@ -79,19 +85,29 @@ bench:
 .PHONY: tidy
 tidy:
 	find src/ tests/ benchmarks/ \( -name '*.cpp' -o -name '*.hpp' -o -name '*.c' -o -name '*.h' \) -exec clang-format -i {} +	
+	hooks/check_shell.sh
 	hooks/check_clang_tidy.sh
 
-## run-debug: 🏃‍♂️  run the app (debug) (don't forget `withenv`)
+## run-debug: 🏃‍♂️ run the app (debug) (don't forget `withenv`)
 .PHONY: run-debug
 run-debug:
 	ASAN_OPTIONS=detect_leaks=1:leak_check_at_exit=1:fast_unwind_on_malloc=0 \
 	build/Debug/tradercpp
 
-## run-release: 🏎️  run the app (prod)
+## run-release: 🏎️ run the app (prod)
 .PHONY: run-release
 run-release:
-	$(call pp,starting app. dont forget to run `scripts/cpu_shield_start.sh` and `scripts/irqs_move.sh`)
-	build/Release/tradercpp
+	$(call pp,moving IRQs)
+	set -o allexport; source .env; set +o allexport; sudo -E scripts/pin_irqs.sh
+	$(call pp,moving CPUs and starting app)
+	set -o allexport; source .env; set +o allexport; sudo -E scripts/pin_cpus.sh build/Release/tradercpp
+
+## restore-cpus: 🖥️ hand back pinned CPUs and IRQs to the operating system. (NB app must not be running) 
+.PHONY: restore-cpus
+restore-cpus:
+	$(call pp,handing CPU management back to the kernel (NB: app must not be running))
+	$(call pp,NB: for re-assigning IRQs reboot the machine)
+	set -o allexport; source .env; set +o allexport; sudo -E scripts/unpin_cpus.sh
 
 # CONTAINERISATION RECIPES ----------------------------------------------------
 
@@ -101,7 +117,7 @@ build-container:
 	IMAGE_VERSION=
 	@if [ -z "$(IMAGE_VERSION)" ]; then \
 		echo "Error: IMAGE_VERSION is not set"; \
-		echo "(you can set it on the command line like so: \`make build-container IMAGE_VERSION=1.7\`)"; \
+		echo "(you can set it on the command line like so: \`make build-container IMAGE_VERSION=1.8\`)"; \
 		exit 1; \
 	fi
 	docker build -f Dockerfile_build -t milss/tradercppbuild:latest -t milss/tradercppbuild:v$(IMAGE_VERSION) .

README.md

@@ -152,9 +152,9 @@ NB: this app uses `make` as a recipe book, but it's not essential:
   - ✅ cache line alignment
   - ✅ tcmalloc (Full) / gperftools
   - CPU
-    - ✅ process priority
-    - ✅ FIX-thread "realtime"
-    - ✅ FIX-thread CPU affinity
+    - ✅ isolated CPU cores
+    - ✅ thread-CPU affinity
+    - ✅ thread "realtime" priority
     - Disable hyperthreading
   - OS
     - ✅ vacate OS services
@@ -173,6 +173,7 @@ NB: this app uses `make` as a recipe book, but it's not essential:
     - ✅ debug quickfix to confirm if it's running in it's own thread
     - QuickFIX alternative (Fix8)
       - otherwise => QuickFIX + SSL
+  - hugepages
   - kernel space vs user space
   - intrinsics
   - compiler auto-vectorization

hooks/check_shell.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+################################################################
+# check_clang_tidy.sh
+# checks for any lint warnings and errors out.
+# does not modify files in-place.
+# uses run-clang-tidy for parallel execution.
+################################################################
+
+set -euo pipefail
+
+echo ">> Running shellcheck on all scripts"
+
+while IFS= read -r file; do
+    shellcheck "$file"
+# find files that start with a sh/bash shebang, exclude build/ and .git/hooks folders
+done < <(find . \( -type d -name build -o -path './.git/hooks' \) -prune -o -type f -exec grep -Iq '^#!.*sh' {} \; -print)

hooks/pre-commit

@@ -13,15 +13,17 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 # Run subscripts
 
-echo "Running strict pre-commit checks (clang-tidy, clang-format, build-debug)..."
+echo "Running strict pre-commit checks (clang-format, shellcheck, clang-tidy, build-debug)..."
+
+bash "$DIR/check_clang_format.sh"
+
+bash "$DIR/check_shell.sh"
 
 # TODO(mils): does this also need to be converted to:
 # `git diff -U0 origin/master...HEAD -- '*.cpp'`
 git diff -U0 master...HEAD -- '*.cpp' | \
   clang-tidy-diff-18.py -p1 -path "$DIR/../build/Debug"
 
-bash "$DIR/check_clang_format.sh"
-
 make build-debug
 
 echo "✅ Pre-commit checks passed on full codebase."