Support multiple concurrent TCP transcriptions sharing a single loaded model

TCP server now spawns a thread per connection, each getting an independent
pipeline (~7.3MB) while sharing the model weights in VRAM/RAM. Local audio
capture (--trigger) runs alongside TCP in a background thread. Output
simplified to raw text stream (no timestamp framing). CoreML encoder caches
moved from shared model struct to per-pipeline for thread safety. --input
flag deprecated in favour of --trigger for enabling local mode.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: danielbodart

CLAUDE.md

@@ -16,12 +16,15 @@ Zig and Bun are installed automatically via `bootstrap.sh` + mise.
 # Clean build artifacts
 ./run.ts clean
 
-# Run directly (loads model, grabs keyboard, CapsLock = push-to-talk)
+# Run directly (TCP server + local push-to-talk via CapsLock)
 # Linux:
 ./dist/linux/bin/capsper --trigger capslock --audio-channel FL --drop-terms drop-terms.txt
 # macOS:
 ./dist/macos/bin/capsper --trigger capslock --drop-terms drop-terms.txt
 
+# TCP-only mode (no local capture, for testing or remote transcription)
+./dist/linux/bin/capsper --drop-terms drop-terms.txt
+
 # First-time setup (builds, configures permissions, installs service)
 ./run.ts setup
 ```
@@ -55,7 +58,7 @@ On macOS, `./run.ts build` produces one binary. On Linux, it builds both `capspe
 
 ## Architecture
 
-Push-to-talk voice dictation for Linux and macOS. Self-contained binary per platform. On Linux: grabs keyboards via evdev, intercepts CapsLock, captures audio via PipeWire, transcribes with Nemotron RNNT (via onnxruntime), injects text via uinput. On macOS: CGEventTap input, CoreAudio capture, CoreML inference (93% ANE), CGEventPost injection.
+Push-to-talk voice dictation for Linux and macOS. Self-contained binary per platform. Supports multiple concurrent transcriptions — TCP server accepts multiple clients simultaneously, each getting an independent pipeline while sharing the single loaded model. Use `--trigger` to also enable local audio capture with push-to-talk alongside TCP. On Linux: grabs keyboards via evdev, intercepts CapsLock, captures audio via PipeWire, transcribes with Nemotron RNNT (via onnxruntime), injects text via uinput. On macOS: CGEventTap input, CoreAudio capture, CoreML inference (93% ANE), CGEventPost injection.
 
 > **History:** Capsper originally used whisper.cpp for ASR with Silero/TEN-VAD for voice activity detection. It now uses NVIDIA's Nemotron Speech 600M model (FastConformer RNNT) which is incremental and doesn't need a separate VAD — PTT (push-to-talk) is the sole gate. The name "Capsper" is a nod to Casper the friendly ghost — ghostwriting via CapsLock.
 
@@ -172,7 +175,7 @@ Do NOT manually download CI artifacts or stage releases by hand — the update s
 - Conversion scripts: [nemotron-speech-600m-coreml](https://github.com/danielbodart/nemotron-speech-600m-coreml) (CoreML), [nemotron-speech-600m-onnx](https://github.com/danielbodart/nemotron-speech-600m-onnx) (ONNX)
 - Audio format: 16kHz mono S16_LE PCM (32000 bytes/sec)
 - Default server port: 43007
-- CLI flags: `--audio-channel`, `--audio-target`, `--audio-gain`, `--audio-detect` (cross-platform names; `--pw-*` aliases kept for backwards compatibility)
+- CLI flags: `--audio-channel`, `--audio-target`, `--audio-gain`, `--audio-detect` (cross-platform names; `--pw-*` aliases kept for backwards compatibility). `--trigger` enables local audio capture + PTT alongside TCP. `--input` is deprecated.
 - Service management: `systemctl --user` on Linux, `launchctl bootstrap/bootout gui/$(id -u)` on macOS
 - Service files: `~/.config/systemd/user/capsper.service` (Linux), `~/Library/LaunchAgents/io.github.danielbodart.capsper.plist` (macOS)
 - Permissions: `input` group + udev rule on Linux; Accessibility + Microphone TCC on macOS

src/backend/coreml/helpers.m

@@ -18,10 +18,15 @@
 typedef struct {
     void *encoder;       // MLModel* (retained)
     void *decoder;       // MLModel* (retained)
+} CapsperCoreMLModels;
+
+// Per-pipeline encoder cache state. Each pipeline gets its own caches
+// so multiple transcriptions can run concurrently on the shared model.
+typedef struct {
     void *cache_channel; // MLMultiArray* (retained)
     void *cache_time;    // MLMultiArray* (retained)
     void *cache_len;     // MLMultiArray* (retained)
-} CapsperCoreMLModels;
+} CapsperCoreMLCaches;
 
 // ---------------------------------------------------------------------------
 // Helpers
@@ -230,39 +235,53 @@ static void copy_to_f32(MLMultiArray *src, float *dst, NSInteger count) {
         models->encoder = (void *)CFBridgingRetain(encoder);
         models->decoder = (void *)CFBridgingRetain(decoder);
 
-        // Initialize cache state (batch-first: [1, 24, 70, 1024], [1, 24, 1024, 8], [1])
-        models->cache_channel = (void *)CFBridgingRetain(
-            make_zeros(@[@1, @24, @70, @1024], MLMultiArrayDataTypeFloat32));
-        models->cache_time = (void *)CFBridgingRetain(
-            make_zeros(@[@1, @24, @1024, @8], MLMultiArrayDataTypeFloat32));
-        models->cache_len = (void *)CFBridgingRetain(
-            make_zeros(@[@1], MLMultiArrayDataTypeInt32));
-
         NSLog(@"capsper_coreml: models loaded from %@", dir);
         return models;
     }
 }
 
-/// Release all CoreML models and cache state.
+/// Release all CoreML models.
 void capsper_coreml_release(CapsperCoreMLModels *models) {
     if (!models) return;
     @autoreleasepool {
         if (models->encoder) CFBridgingRelease(models->encoder);
         if (models->decoder) CFBridgingRelease(models->decoder);
-        if (models->cache_channel) CFBridgingRelease(models->cache_channel);
-        if (models->cache_time) CFBridgingRelease(models->cache_time);
-        if (models->cache_len) CFBridgingRelease(models->cache_len);
         free(models);
     }
 }
 
+/// Create per-pipeline encoder cache state.
+CapsperCoreMLCaches *capsper_coreml_create_caches(void) {
+    @autoreleasepool {
+        CapsperCoreMLCaches *caches = (CapsperCoreMLCaches *)calloc(1, sizeof(CapsperCoreMLCaches));
+        caches->cache_channel = (void *)CFBridgingRetain(
+            make_zeros(@[@1, @24, @70, @1024], MLMultiArrayDataTypeFloat32));
+        caches->cache_time = (void *)CFBridgingRetain(
+            make_zeros(@[@1, @24, @1024, @8], MLMultiArrayDataTypeFloat32));
+        caches->cache_len = (void *)CFBridgingRetain(
+            make_zeros(@[@1], MLMultiArrayDataTypeInt32));
+        return caches;
+    }
+}
+
+/// Release per-pipeline encoder cache state.
+void capsper_coreml_release_caches(CapsperCoreMLCaches *caches) {
+    if (!caches) return;
+    @autoreleasepool {
+        if (caches->cache_channel) CFBridgingRelease(caches->cache_channel);
+        if (caches->cache_time) CFBridgingRelease(caches->cache_time);
+        if (caches->cache_len) CFBridgingRelease(caches->cache_len);
+        free(caches);
+    }
+}
+
 /// Reset encoder cache state to zeros (call between utterances).
-void capsper_coreml_reset_state(CapsperCoreMLModels *models) {
-    if (!models) return;
+void capsper_coreml_reset_state(CapsperCoreMLCaches *caches) {
+    if (!caches) return;
     @autoreleasepool {
-        MLMultiArray *ch = (__bridge MLMultiArray *)(models->cache_channel);
-        MLMultiArray *t = (__bridge MLMultiArray *)(models->cache_time);
-        MLMultiArray *l = (__bridge MLMultiArray *)(models->cache_len);
+        MLMultiArray *ch = (__bridge MLMultiArray *)(caches->cache_channel);
+        MLMultiArray *t = (__bridge MLMultiArray *)(caches->cache_time);
+        MLMultiArray *l = (__bridge MLMultiArray *)(caches->cache_len);
         memset(ch.dataPointer, 0, ch.count * sizeof(float));
         memset(t.dataPointer, 0, t.count * sizeof(float));
         memset(l.dataPointer, 0, l.count * sizeof(int32_t));
@@ -282,15 +301,16 @@ void capsper_coreml_reset_state(CapsperCoreMLModels *models) {
 /// Returns 0 on success, -1 on error.
 int capsper_coreml_run_encoder(
     CapsperCoreMLModels *models,
+    CapsperCoreMLCaches *caches,
     const float *mel_data,
     float *out_encoded,
     int32_t *out_encoded_len
 ) {
     @autoreleasepool {
         MLModel *encoder = (__bridge MLModel *)(models->encoder);
-        MLMultiArray *cache_ch = (__bridge MLMultiArray *)(models->cache_channel);
-        MLMultiArray *cache_time = (__bridge MLMultiArray *)(models->cache_time);
-        MLMultiArray *cache_len = (__bridge MLMultiArray *)(models->cache_len);
+        MLMultiArray *cache_ch = (__bridge MLMultiArray *)(caches->cache_channel);
+        MLMultiArray *cache_time = (__bridge MLMultiArray *)(caches->cache_time);
+        MLMultiArray *cache_len = (__bridge MLMultiArray *)(caches->cache_len);
 
         // Wrap mel input (zero-copy)
         MLMultiArray *mel = wrap_f32((float *)mel_data, @[@1, @128, @65], 1 * 128 * 65);

src/backend/coreml/pipeline.zig

@@ -31,8 +31,10 @@ const N_MELS = mel_state_mod.N_MELS;
 
 // CoreML C API (from coreml_helpers.m)
 pub const CapsperCoreMLModels = opaque {};
+pub const CapsperCoreMLCaches = opaque {};
 extern fn capsper_coreml_run_encoder(
     models: *CapsperCoreMLModels,
+    caches: *CapsperCoreMLCaches,
     mel_data: [*]const f32,
     out_encoded: [*]f32,
     out_encoded_len: *i32,
@@ -47,7 +49,9 @@ extern fn capsper_coreml_run_decoder(
     out_state_h: [*]f32,
     out_state_c: [*]f32,
 ) c_int;
-extern fn capsper_coreml_reset_state(models: *CapsperCoreMLModels) void;
+extern fn capsper_coreml_create_caches() ?*CapsperCoreMLCaches;
+extern fn capsper_coreml_release_caches(caches: *CapsperCoreMLCaches) void;
+extern fn capsper_coreml_reset_state(caches: *CapsperCoreMLCaches) void;
 
 /// Process-lifetime config. CoreML models are shared across connections.
 pub const CoreMLConfig = struct {
@@ -69,6 +73,9 @@ pub const CoreMLPipeline = struct {
     // Pre-encode cache: last PRE_ENCODE_CACHE mel frames from previous chunk
     pre_cache: [N_MELS * PRE_ENCODE_CACHE]f32 = [_]f32{0} ** (N_MELS * PRE_ENCODE_CACHE),
 
+    // Per-pipeline encoder cache state (CoreML MLMultiArrays)
+    caches: *CapsperCoreMLCaches,
+
     // RNNT decoder state
     dec_state1: []f32,
     dec_state2: []f32,
@@ -84,6 +91,8 @@ pub const CoreMLPipeline = struct {
     pub fn init(allocator: std.mem.Allocator, config: CoreMLConfig, verbose: bool) !CoreMLPipeline {
         const dec_state_size = PRED_LAYERS * 1 * PRED_HIDDEN;
 
+        const caches = capsper_coreml_create_caches() orelse return error.CoreMLCacheInitFailed;
+
         const dec_state1 = try allocator.alloc(f32, dec_state_size);
         errdefer allocator.free(dec_state1);
         @memset(dec_state1, 0);
@@ -97,6 +106,7 @@ pub const CoreMLPipeline = struct {
             .config = config,
             .verbose = verbose,
             .mel = NemoMelState.init(allocator, config.filterbank),
+            .caches = caches,
             .dec_state1 = dec_state1,
             .dec_state2 = dec_state2,
         };
@@ -107,6 +117,7 @@ pub const CoreMLPipeline = struct {
     }
 
     pub fn deinit(self: *CoreMLPipeline) void {
+        capsper_coreml_release_caches(self.caches);
         self.allocator.free(self.dec_state1);
         self.allocator.free(self.dec_state2);
         self.emitted_text.deinit(self.allocator);
@@ -154,7 +165,7 @@ pub const CoreMLPipeline = struct {
         self.mel.reset();
         self.mel_frame_cursor = 0;
         @memset(&self.pre_cache, 0);
-        capsper_coreml_reset_state(self.config.models);
+        capsper_coreml_reset_state(self.caches);
         @memset(self.dec_state1, 0);
         @memset(self.dec_state2, 0);
         self.last_token = tokenizer.BLANK_ID;
@@ -241,6 +252,7 @@ pub const CoreMLPipeline = struct {
 
         const status = capsper_coreml_run_encoder(
             self.config.models,
+            self.caches,
             &mel_input,
             &enc_output,
             &enc_len,

src/main.zig

@@ -22,7 +22,8 @@ const Recorder = @import("shared/recorder.zig").Recorder;
 pub fn main() !void {
     var gpa = std.heap.GeneralPurposeAllocator(.{ .enable_memory_limit = true }){};
     defer _ = gpa.deinit();
-    const allocator = gpa.allocator();
+    var ts_allocator = std.heap.ThreadSafeAllocator{ .child_allocator = gpa.allocator() };
+    const allocator = ts_allocator.allocator();
 
     const args = try std.process.argsAlloc(allocator);
     defer std.process.argsFree(allocator, args);
@@ -72,17 +73,9 @@ pub fn main() !void {
                 break :blk 43007;
             };
         } else if (std.mem.eql(u8, arg, "--input")) {
+            // Deprecated: local mode is now enabled by --trigger. Accept and skip for backwards compatibility.
             i += 1;
-            if (i < args.len) {
-                if (std.mem.eql(u8, args[i], "tcp")) {
-                    input_mode = .tcp;
-                } else if (std.mem.eql(u8, args[i], "local")) {
-                    input_mode = .local;
-                } else {
-                    std.debug.print("Invalid --input value '{s}', expected 'tcp' or 'local'\n", .{args[i]});
-                    return;
-                }
-            }
+            std.debug.print("Warning: --input is deprecated. Use --trigger to enable local mode alongside TCP.\n", .{});
         } else if (std.mem.eql(u8, arg, "--audio-target") or std.mem.eql(u8, arg, "--pw-target")) {
             i += 1;
             if (i < args.len) audio_target = args[i];
@@ -178,7 +171,7 @@ pub fn main() !void {
         return;
     }
 
-    // --trigger implies --input local (audio capture) and starts not-live (trigger key controls recording)
+    // --trigger enables local audio capture alongside TCP (trigger key controls PTT recording)
     if (trigger_key != null) {
         input_mode = .local;
     }
@@ -356,8 +349,8 @@ pub fn main() !void {
         };
 
         var server2 = Server.init(allocator, pipeline_factory, 0, .tcp, null, 0, verbose, false, null, drop_terms, null, 1.0, true);
-        server_mod.is_live.store(true, .monotonic);
-        server2.handleConnection(file.handle, 1, null) catch |err| {
+        var always_live = std.atomic.Value(bool).init(true);
+        server2.handleConnection(file.handle, 1, &always_live, null, null) catch |err| {
             std.debug.print("Stream error: {}\n", .{err});
         };
         file.close();
@@ -478,7 +471,7 @@ pub fn main() !void {
 fn printUsage() void {
     std.debug.print("Usage: capsper [--model PATH] [--port PORT]\n", .{});
     std.debug.print("       [--verbose|-v]\n", .{});
-    std.debug.print("       [--input tcp|local] [--audio-target NODE] [--audio-channel CHANNEL]\n", .{});
+    std.debug.print("       [--audio-target NODE] [--audio-channel CHANNEL]\n", .{});
     std.debug.print("       [--trigger KEY] [--trigger-passthrough] [--type-delay MICROSECONDS]\n", .{});
     std.debug.print("       [--drop-terms FILE]\n", .{});
     std.debug.print("       [--record-dir DIR [--record-keep N]]\n", .{});
@@ -487,4 +480,8 @@ fn printUsage() void {
     std.debug.print("       [--audio-detect [--detect-duration SECS]]\n", .{});
     std.debug.print("       [--warmup-file FILE] [--no-warmup]\n", .{});
     std.debug.print("       [--dry-run] [--version]\n", .{});
+    std.debug.print("\n", .{});
+    std.debug.print("TCP server is always active (default port 43007). Multiple clients can connect\n", .{});
+    std.debug.print("simultaneously, each getting an independent transcription pipeline.\n", .{});
+    std.debug.print("Use --trigger to also enable local audio capture with push-to-talk.\n", .{});
 }