feat: working CPU and IRQ pinning scripts

Author: milsanore

.github/workflows/nightly_bench.yml

@@ -11,7 +11,7 @@ jobs:
   build_and_test:
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/milsanore/tradercppbuild:v1.7
+      image: ghcr.io/milsanore/tradercppbuild:v1.8
       # set resource limits for some consistency
       options: --memory=7g --cpus=2
     steps:

.github/workflows/release.yml

@@ -11,7 +11,7 @@ jobs:
   build_release:
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/milsanore/tradercppbuild:v1.7
+      image: ghcr.io/milsanore/tradercppbuild:v1.8
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4

.github/workflows/reusable_build.yml

@@ -16,7 +16,7 @@ jobs:
   build_and_test:
     runs-on: ubuntu-latest
     container:
-      image: ghcr.io/milsanore/tradercppbuild:v1.7
+      image: ghcr.io/milsanore/tradercppbuild:v1.8
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4

Dockerfile_build

@@ -1,5 +1,5 @@
 #############################################
-# milss/tradercppbuild:v1.7
+# milss/tradercppbuild:v1.8
 # Build container for tradercpp
 # Published to dockerhub in order to accelerate github actions
 # Uses:
@@ -56,12 +56,6 @@ RUN apt update && apt install -y --no-install-recommends \
 
 
 # Install LLVM 18 (clang-format and clang-tidy)
-RUN apt update && apt install -y --no-install-recommends \
-    gnupg \
-    lsb-release \
-    software-properties-common \
-    wget \
-    && rm -rf /var/lib/apt/lists/*
 
 RUN wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
 

Makefile

@@ -25,8 +25,6 @@ withenv:
 	test -e .env || cp .env.example .env
 	bash -c 'set -o allexport; source .env; set +o allexport; make "$$RECIPE"'
 
-# TODO: replace ☝️ with `set -a; . .env; set +a;`
-
 ## init: 🏌️ initialize the project
 .PHONY: init
 init:
@@ -99,15 +97,17 @@ run-debug:
 ## run-release: 🏎️ run the app (prod)
 .PHONY: run-release
 run-release:
-# 	set -a; . .env; set +a; sudo -E scripts/pin_irqs.sh
-	set -a; . .env; set +a; sudo -E scripts/pin_cpus.sh build/Release/tradercpp
+	$(call pp,moving IRQs)
+	set -o allexport; source .env; set +o allexport; sudo -E scripts/pin_irqs.sh
+	$(call pp,moving CPUs and starting app)
+	set -o allexport; source .env; set +o allexport; sudo -E scripts/pin_cpus.sh build/Release/tradercpp
 
 ## restore-cpus: 🖥️ hand back pinned CPUs and IRQs to the operating system. (NB app must not be running) 
 .PHONY: restore-cpus
 restore-cpus:
-	$(call pp,handing CPU and IRQ management back to the kernel. NB: app must not be running`)
-	set -a; . .env; set +a; sudo -E scripts/unpin_cpus.sh
-	set -a; . .env; set +a; sudo -E scripts/unpin_irqs.sh
+	$(call pp,handing CPU management back to the kernel (NB: app must not be running))
+	$(call pp,NB: for re-assigning IRQs reboot the machine)
+	set -o allexport; source .env; set +o allexport; sudo -E scripts/unpin_cpus.sh
 
 # CONTAINERISATION RECIPES ----------------------------------------------------
 
@@ -117,7 +117,7 @@ build-container:
 	IMAGE_VERSION=
 	@if [ -z "$(IMAGE_VERSION)" ]; then \
 		echo "Error: IMAGE_VERSION is not set"; \
-		echo "(you can set it on the command line like so: \`make build-container IMAGE_VERSION=1.7\`)"; \
+		echo "(you can set it on the command line like so: \`make build-container IMAGE_VERSION=1.8\`)"; \
 		exit 1; \
 	fi
 	docker build -f Dockerfile_build -t milss/tradercppbuild:latest -t milss/tradercppbuild:v$(IMAGE_VERSION) .

scripts/pin_cpus.sh

@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
-set -euo pipefail
 
 # ----------------------------------------------------------
 # CPU isolation launcher for Linux (cgroup v1 & v2)
@@ -9,64 +8,82 @@ set -euo pipefail
 # 2. Validation: Ensures CPU_SET_NAME and CPU_SET_RANGE exist.
 # 3. Cgroup detection: Automatically handles v1 and v2.
 # 4. CPU and memory configuration: Writes cpuset.cpus and cpuset.mems.
-# 5. Associating the application:
-#   - $$ is the PID of the script itself.
-#   - When we exec "$@", the application replaces the script and inherits the cgroup.
-#   - This is the standard way to bind the app to the cpuset.
+# 5. Associating the application
 # ----------------------------------------------------------
 
-# Validate environment variables
+set -euo pipefail
+
+# validate required environment variables
 : "${CPU_SET_NAME:?Environment variable CPU_SET_NAME is required}"
 : "${CPU_SET_RANGE:?Environment variable CPU_SET_RANGE is required}"
 echo "CPU isolation launcher starting..."
-echo "CPU_SET_NAME = $CPU_SET_NAME"
-echo "CPU_SET_RANGE = $CPU_SET_RANGE"
+echo "CPU_SET_NAME [$CPU_SET_NAME]"
+echo "CPU_SET_RANGE [$CPU_SET_RANGE]"
+
+# check for root
+if [ "$EUID" -ne 0 ]; then
+    echo "this script must be run as root."
+    echo "try: sudo $0"
+    exit 1
+fi
 
-# Check for root privileges
-if [[ $EUID -ne 0 ]]; then
-    echo "WARNING: root privileges are required for true CPU isolation."
-    echo "Proceeding without isolation..."
-    exec "$@"
+if ! [[ $CPU_SET_RANGE =~ ^[0-9,-]+$ ]]; then
+    echo "invalid CPU_SET_RANGE format. value [$CPU_SET_RANGE]"
+    exit 1
 fi
 
-# Detect cgroup version
+# --- minimal NUMA check ---
+# nodes_used=$(for cpu in $(echo "$CPU_SET_RANGE" | sed 's/,/ /g'); do
+#     for node_dir in /sys/devices/system/node/node*; do
+#         grep -qE "(^|,)$cpu($|,|-)" "$node_dir/cpulist" && echo $(basename $node_dir | sed 's/node//')
+#     done
+# done | sort -u)
+#
+# if [[ $(wc -w <<< "$nodes_used") -gt 1 ]]; then
+#     echo "error: requested CPUs span multiple NUMA nodes. nodes [$nodes_used]"
+#     exit 1
+# fi
+# echo "requested CPUs are within NUMA node. node [$nodes_used]"
+# --- end minimal NUMA check ---
+
+# build path based on cgroup version
 if [[ -f /sys/fs/cgroup/cgroup.controllers ]]; then
     CGROUP_VERSION=2
     BASE_CGROUP="/sys/fs/cgroup"
 else
     CGROUP_VERSION=1
     BASE_CGROUP="/sys/fs/cgroup/cpuset"
 fi
-echo "Detected cgroup version: $CGROUP_VERSION"
-
+echo "detected cgroup version. value [$CGROUP_VERSION]"
 GROUP_PATH="$BASE_CGROUP/$CPU_SET_NAME"
 
-# Create cpuset directory if it doesn't exist
+# create cpuset directory if it doesn't exist
 if [[ ! -d "$GROUP_PATH" ]]; then
     mkdir -p "$GROUP_PATH"
-    echo "Created cpuset directory: $GROUP_PATH"
+    echo "created cpuset directory. path [$GROUP_PATH]"
 fi
 
-# Write CPUs
+# write CPUs
 CPU_FILE="$GROUP_PATH/cpuset.cpus"
 echo "$CPU_SET_RANGE" > "$CPU_FILE"
-echo "Wrote CPUs to $CPU_FILE"
+echo "wrote CPUs to file. path [$CPU_FILE]"
 
-# Write memory nodes (required for cpuset)
+# write memory nodes (required for cpuset)
 MEM_FILE="$GROUP_PATH/cpuset.mems"
 if [[ ! -f "$MEM_FILE" ]]; then
-    # assume NUMA node 0
-    echo "0" > "$MEM_FILE"
+    echo "$nodes_used" > "$MEM_FILE"
 fi
-echo "Set memory nodes in $MEM_FILE"
-
-# TODO(MILS): WARN ABOUT MULTI-NUMA CPU RANGES AND REFUSE TO CONTINUE
+echo "set memory nodes in file. path [$MEM_FILE]"
 
-# Add current process to the cpuset
-# - $$ is the PID of the script itself.
-# - When we exec "$@", the application replaces the script and inherits the cgroup.
+# add current process to the cpuset
 PROCS_FILE="$GROUP_PATH/cgroup.procs"
+# NB: $$ is the PID of the script itself
 echo "$$" > "$PROCS_FILE"
-echo "Added PID $$ to $PROCS_FILE"
-echo "Launching application in isolated cpuset..."
-exec "$@"
+echo "added process to cpuset. PID [$$] path [$PROCS_FILE]"
+echo "launching application in isolated cpuset..."
+# NB: when we exec "$@", the application replaces the script and inherits the cgroup
+"$@"
+# print application return so that the script doesn't obfuscate it
+ret=$?
+echo "Application exited with code $ret"
+exit $ret

scripts/pin_irqs.sh

@@ -1,69 +1,83 @@
-#!/bin/bash
-# ------------------------------------------------------------
-# Move all IRQs (except NICs) to non-isolated CPUs dynamically
-# ------------------------------------------------------------
+#!/usr/bin/env bash
+
+# --------------------------------------------------------
+# IRQ isolation script
+# Moves all IRQs off of the specified isolated CPUs
+#
+# How it works:
+# 1. CPU range → bitmask
+#   - Converts something like 0-3,5 → 0b00101111 → 0x2F
+# 2. Iterates /proc/irq/*/smp_affinity
+#   - Reads current mask
+#   - Clears bits for isolated CPUs
+#   - Writes new mask back
+# 3. Safe
+#   - Only writes if a change is needed
+#   - Skips files that aren't writable
+#
+# How to undo: reboot
+# --------------------------------------------------------
 
 set -euo pipefail
 
-# Check for root
+: "${CPU_SET_RANGE:?Environment variable CPU_SET_RANGE is required}"
+echo "IRQ isolation starting..."
+echo "CPU_SET_RANGE [$CPU_SET_RANGE]"
+
+# check for root
 if [ "$EUID" -ne 0 ]; then
-    echo "This script must be run as root."
-    echo "Try: sudo $0"
+    echo "this script must be run as root."
+    echo "try: sudo $0"
     exit 1
 fi
 
-# Handle filenames with whitespace
-IFS=$'\n\t'
-
-# List of NIC interfaces to protect (keep IRQs on isolated cores)
-NIC_IFACES=("eth0" "eth1")  # modify as needed
-
-# Determine total CPU count
-CPU_COUNT=$(nproc)
-
-# Set last isolated CPU index (adjust if your isolated CPUs differ)
-# TODO: detect isolated CPUs dynamically (from /sys/devices/system/cpu/isolated)
-ISOLATED_END=1   # e.g., CPUs 0-1 are isolated
-
-# Build CPU mask for CPUs >= ISOLATED_END + 1
-# Example: if CPU_COUNT=8, ISOLATED_END=1 -> mask = 11111100 = FC
-# The mask is binary, then converted to hex
-CPU_MASK=$(printf '%X' "$(( (1 << CPU_COUNT) - 1 - ((1 << (ISOLATED_END + 1)) - 1) ))")
-
-echo "Detected CPU mask for IRQs: $CPU_MASK (CPUs >= $((ISOLATED_END + 1)))"
-
-# ------------------------------------------------------------
-# Helper function to check if an IRQ belongs to a NIC
-# ------------------------------------------------------------
-is_nic_irq() {
-    local irq_dir="$1"
-    local hint_file="$irq_dir/affinity_hint"
-    [ -f "$hint_file" ] || return 1
-
-    for nic in "${NIC_IFACES[@]}"; do
-        if grep -q "$nic" "$hint_file" 2>/dev/null; then
-            return 0  # It's a NIC IRQ
+# --- Convert CPU range to bitmask ---
+cpu_to_mask() {
+    # converts something like 0-3,5 to a hex bitmask string
+    local cpus=()
+    IFS=',' read -ra parts <<< "$1"
+    for part in "${parts[@]}"; do
+        if [[ $part =~ ^([0-9]+)-([0-9]+)$ ]]; then
+            for ((i=${BASH_REMATCH[1]}; i<=${BASH_REMATCH[2]}; i++)); do
+                cpus+=($i)
+            done
+        else
+            cpus+=($part)
         fi
     done
-    return 1  # Not a NIC IRQ
-}
 
-# ------------------------------------------------------------
-# Iterate through all IRQ directories
-# ------------------------------------------------------------
-for irq_dir in /proc/irq/*; do
-    [ -d "$irq_dir" ] || continue
-    irq_num=$(basename "$irq_dir")
-    smp_file="$irq_dir/smp_affinity"
-
-    [ -f "$smp_file" ] || continue
+    # build bitmask
+    local mask=0
+    for cpu in "${cpus[@]}"; do
+        mask=$((mask | (1 << cpu)))
+    done
+    printf "%x" "$mask"
+}
+ISOLATED_MASK=$(cpu_to_mask "$CPU_SET_RANGE")
+echo "isolated CPU mask: 0x[$ISOLATED_MASK]"
 
-    if is_nic_irq "$irq_dir"; then
-        echo "Skipping NIC IRQ $irq_num"
+# iterate over all IRQs
+for irq in /proc/irq/*/smp_affinity; do
+    # skip non-files
+    [[ -f $irq ]] || continue 
+    # Check if writable
+    if [[ ! -w "$irq" ]]; then
+        echo "skipping IRQ, smp_affinity not writable. IRQ [$(basename $(dirname $irq))]"
         continue
     fi
-
-    # Move IRQ to CPUs >= ISOLATED_END + 1
-    echo "$CPU_MASK" > "$smp_file"
-    echo "Moved IRQ $irq_num to CPUs >= $((ISOLATED_END + 1))"
+    # read current mask
+    current=$(cat "$irq")
+    # convert hex to decimal
+    current_dec=$((16#$current))
+    # remove isolated CPUs
+    new_dec=$((current_dec & ~0x$ISOLATED_MASK))
+    # calculate new mask
+    new_hex=$(printf "%x" "$new_dec")
+    # only write if different
+    if [[ "$current" != "$new_hex" ]]; then
+        echo "moving IRQ off isolated CPUs. IRQ [$(basename $(dirname $irq))] old [$current] new [$new_hex]"
+        echo "$new_hex" > "$irq"
+    fi
 done
+
+echo "IRQ isolation complete."