perf(embeddings): optimize thread budget for ORT sessions

- Introduce max thread budget based on CPU count
- Derive session count from thread budget and intra-op threads
- Add EMBEDDINGS_MAX_THREADS environment variable support
- Prevent thread oversubscription by capping session count

Author: donhardman

embeddings/src/model/local.rs

@@ -32,6 +32,9 @@ use tokenizers::Tokenizer;
 /// Default batch size per ONNX forward pass.
 const DEFAULT_BATCH_SIZE: usize = 8;
 
+/// Default intra-op threads per session.
+const DEFAULT_INTRA_THREADS: usize = 4;
+
 fn batch_size() -> usize {
     std::env::var("EMBEDDINGS_BATCH_SIZE")
         .ok()
@@ -45,22 +48,34 @@ fn available_cpus() -> usize {
         .unwrap_or(2)
 }
 
-/// Number of parallel ORT sessions. Default: num_cpus / 2 (min 1).
-/// Moderate oversubscription (total_threads ≈ 2×cores) fills idle gaps between ORT ops.
-fn num_sessions() -> usize {
-    std::env::var("EMBEDDINGS_NUM_SESSIONS")
+/// Max total ORT threads budget. Default: 2 × num_cpus (optimal oversubscription).
+/// Controls how many threads all sessions combined can use.
+fn max_threads() -> usize {
+    std::env::var("EMBEDDINGS_MAX_THREADS")
         .ok()
         .and_then(|v| v.parse().ok())
-        .unwrap_or_else(|| (available_cpus() / 2).max(1))
+        .unwrap_or_else(|| available_cpus() * 2)
 }
 
-/// Intra-op threads per session. Default: 4.
-/// With num_sessions = cpus/2, total threads = cpus/2 * 4 = 2×cpus — optimal oversubscription.
+/// Intra-op threads per session.
 fn intra_threads() -> usize {
     std::env::var("EMBEDDINGS_INTRA_THREADS")
         .ok()
         .and_then(|v| v.parse().ok())
-        .unwrap_or(4)
+        .unwrap_or(DEFAULT_INTRA_THREADS)
+}
+
+/// Number of parallel ORT sessions, derived from thread budget.
+/// total_threads = num_sessions × intra_threads ≤ max_threads
+fn num_sessions() -> usize {
+    let ns = std::env::var("EMBEDDINGS_NUM_SESSIONS")
+        .ok()
+        .and_then(|v| v.parse().ok())
+        .unwrap_or_else(|| (max_threads() / intra_threads()).max(1));
+    // Cap so total threads don't exceed budget
+    let it = intra_threads();
+    let mt = max_threads();
+    (mt / it).max(1).min(ns)
 }
 
 /// Model architecture type - determines pooling strategy