feat: moving IRQs

Author: milsanore

Makefile

@@ -74,7 +74,7 @@ run-debug:
 ## run-release: 🏎️  run the app (prod)
 .PHONY: run-release
 run-release:
-	$(call pp,starting app. dont forget to run `scripts/cpu_shield_start.sh`)
+	$(call pp,starting app. dont forget to run `scripts/cpu_shield_start.sh` and `scripts/irqs_move.sh`)
 	build/Release/tradercpp
 
 # CONTAINERISATION RECIPES ----------------------------------------------------

README.md

@@ -156,15 +156,16 @@ NB: this app uses `make` as a recipe book, but it's not essential:
     - ✅ FIX-thread CPU affinity
     - Disable hyperthreading
   - NIC
+    - ✅ NIC IRQ affinity to same CPU as FIX
     - wired, kernel-bypass NICs
-    - NIC IRQ affinity to same CPU as FIX
     - hardware queue affinity
     - QoS (mark packets)
     - AF_XDP (+ Zero-copy mode)
     - ~dedicated NIC + DPDK~
   - OS
-    - vacate OS services / move IRQs for all system devices to the last CPU
-    - RTOS
+    - ✅ vacate OS services
+    - move IRQs for all system devices to other CPUs
+    - RTOS / PREEMPT_RT kernel
   - BIOS
     - disable hyperthreading, turbo boost
     - disable C-states deeper than C1 (C1E, C6, etc)

scripts/irqs_move.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+# ------------------------------------------------------------
+# Move all IRQs (except NICs) to non-isolated CPUs dynamically
+# ------------------------------------------------------------
+
+# List of NIC interfaces to protect (keep IRQs on isolated cores)
+NIC_IFACES=("eth0" "eth1")  # modify as needed
+
+# Determine total CPU count
+CPU_COUNT=$(nproc)
+
+# Set first isolated CPU index (adjust if your isolated CPUs differ)
+ISOLATED_END=1   # e.g., CPUs 0-1 are isolated
+
+# Build CPU mask for CPUs >= ISOLATED_END + 1
+# Example: if CPU_COUNT=8, ISOLATED_END=1 -> mask = 11111100 = FC
+# The mask is binary, then converted to hex
+
+CPU_MASK=$(printf "%X" $(( (1 << (CPU_COUNT)) - 1 - ((1 << (ISOLATED_END + 1)) - 1) )))
+
+echo "Detected CPU mask for IRQs: $CPU_MASK (CPUs >= $((ISOLATED_END + 1)))"
+
+# Helper function to check if IRQ belongs to a NIC
+is_nic_irq() {
+    local irq_file="$1"
+    for nic in "${NIC_IFACES[@]}"; do
+        if grep -q "$nic" "$irq_file"; then
+            return 0  # It's a NIC IRQ
+        fi
+    done
+    return 1  # Not a NIC IRQ
+}
+
+# Iterate through all IRQ directories
+for irq_dir in /proc/irq/*; do
+    [ -d "$irq_dir" ] || continue
+    irq_num=$(basename "$irq_dir")
+    smp_file="$irq_dir/smp_affinity"
+
+    [ -f "$smp_file" ] || continue
+
+    if is_nic_irq "$irq_dir/affinity_hint"; then
+        echo "Skipping NIC IRQ $irq_num"
+        continue
+    fi
+
+    # Move IRQ to CPUs >= ISOLATED_END + 1
+    echo "$CPU_MASK" > "$smp_file"
+    echo "Moved IRQ $irq_num to CPUs >= $((ISOLATED_END + 1))"
+done

scripts/irqs_restore.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# ------------------------------------------------------------
+# Restore all IRQs to default CPU affinity (all CPUs)
+# ------------------------------------------------------------
+
+# Determine total number of CPUs
+CPU_COUNT=$(nproc)
+
+# Default mask: all CPUs (binary 111...1)
+DEFAULT_MASK=$(printf "%X" $(( (1 << CPU_COUNT) - 1 )))
+
+echo "Restoring all IRQs to default mask: $DEFAULT_MASK"
+
+# Iterate through all IRQ directories
+for irq_dir in /proc/irq/*; do
+    [ -d "$irq_dir" ] || continue
+    smp_file="$irq_dir/smp_affinity"
+
+    [ -f "$smp_file" ] || continue
+
+    # Restore default mask
+    echo "$DEFAULT_MASK" > "$smp_file"
+    echo "Restored IRQ $(basename $irq_dir) to all CPUs"
+done

src/utils/threading.cpp

@@ -147,11 +147,6 @@ unsigned int Threading::get_cpu_count() {
 #endif
 }
 
-// static function
-void Threading::set_thread_affinity(std::thread& t, unsigned int cpu_id) {
-  return set_native_thread_affinity(t.native_handle(), cpu_id);
-}
-
 // static function
 void Threading::set_thread_cpu(unsigned int cpu_id) {
 #if defined(__linux__) || defined(__APPLE__)

src/utils/threading.h

@@ -61,7 +61,6 @@ class Threading {
   static void set_thread_realtime();
 
  private:
-  static void set_thread_affinity(std::thread& t, unsigned int cpu_id);
   // platform-specific affinity for native thread handle
   static void set_native_thread_affinity(std::thread::native_handle_type handle,
                                          unsigned int cpu_id);