Merge pull request #339 from altic-dev/B/model-benchmarking

Speed up fast dictation finalization

Author: altic-dev

Fluid.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved

@@ -0,0 +1,26 @@
+import Foundation
+
+enum ParakeetFinalizationMode: String, CaseIterable, Codable, Identifiable {
+    case stableFullFinal
+    case tokenTimedChunkMerge
+
+    var id: String { self.rawValue }
+
+    var displayName: String {
+        switch self {
+        case .stableFullFinal:
+            return "Standard"
+        case .tokenTimedChunkMerge:
+            return "Fast"
+        }
+    }
+
+    var detailText: String {
+        switch self {
+        case .stableFullFinal:
+            return "Most reliable."
+        case .tokenTimedChunkMerge:
+            return "Faster, but maybe inaccurate."
+        }
+    }
+}

Package.resolved

@@ -1304,6 +1304,16 @@ final class SettingsStore: ObservableObject {
         }
     }
 
+    var parakeetFinalizationMode: ParakeetFinalizationMode {
+        get {
+            self.defaults.string(forKey: Keys.parakeetFinalizationMode).flatMap(ParakeetFinalizationMode.init(rawValue:)) ?? .stableFullFinal
+        }
+        set {
+            objectWillChange.send()
+            self.defaults.set(newValue.rawValue, forKey: Keys.parakeetFinalizationMode)
+        }
+    }
+
     var copyTranscriptionToClipboard: Bool {
         get { self.defaults.bool(forKey: Keys.copyTranscriptionToClipboard) }
         set { self.defaults.set(newValue, forKey: Keys.copyTranscriptionToClipboard) }
@@ -3616,6 +3626,7 @@ private extension SettingsStore {
         static let hotkeyMode = "HotkeyMode"
         static let enableStreamingPreview = "EnableStreamingPreview"
         static let enableAIStreaming = "EnableAIStreaming"
+        static let parakeetFinalizationMode = "ParakeetFinalizationMode"
         static let copyTranscriptionToClipboard = "CopyTranscriptionToClipboard"
         static let textInsertionMode = "TextInsertionMode"
         static let autoUpdateCheckEnabled = "AutoUpdateCheckEnabled"

Sources/Fluid/Persistence/ParakeetFinalizationMode.swift

@@ -27,6 +27,12 @@ final class FluidAudioProvider: TranscriptionProvider {
 
     private var streamingAsrManager: AsrManager?
     private var finalAsrManager: AsrManager?
+    private var latestStreamingPreviewText: String = ""
+    private var latestStreamingPreviewSampleCount: Int = 0
+    private var latestStreamingPreviewFinishedAt: TimeInterval?
+    private let fastPreviewMinimumSamples = 32_000
+    private let fastPreviewTailSilenceRMS: Float = 0.002
+    private let fastPreviewTailAudioToleranceMs = 300
     private(set) var isReady: Bool = false
     private(set) var isWordBoostingActive: Bool = false
     private(set) var boostedVocabularyTermsCount: Int = 0
@@ -142,6 +148,9 @@ final class FluidAudioProvider: TranscriptionProvider {
 
         self.streamingAsrManager = streamingManager
         self.finalAsrManager = finalManager
+        self.latestStreamingPreviewText = ""
+        self.latestStreamingPreviewSampleCount = 0
+        self.latestStreamingPreviewFinishedAt = nil
         await progressSink.emit(0.98)
 
         self.isReady = true
@@ -156,16 +165,39 @@ final class FluidAudioProvider: TranscriptionProvider {
         try await self.transcribeFinal(samples)
     }
 
+    func resetStreamingPreviewCache() {
+        self.latestStreamingPreviewText = ""
+        self.latestStreamingPreviewSampleCount = 0
+        self.latestStreamingPreviewFinishedAt = nil
+    }
+
     func transcribeStreaming(_ samples: [Float]) async throws -> ASRTranscriptionResult {
-        guard let manager = self.streamingAsrManager else {
+        guard let fullPreviewManager = self.streamingAsrManager else {
             throw NSError(
                 domain: "FluidAudioProvider",
                 code: -1,
                 userInfo: [NSLocalizedDescriptionKey: "ASR manager not initialized"]
             )
         }
 
-        let result = try await manager.transcribe(samples, source: AudioSource.microphone)
+        let startedAt = Date().timeIntervalSince1970
+        let result = try await fullPreviewManager.transcribe(samples, source: AudioSource.microphone)
+        let text = result.text.trimmingCharacters(in: .whitespacesAndNewlines)
+        self.latestStreamingPreviewText = text
+        self.latestStreamingPreviewSampleCount = samples.count
+        self.latestStreamingPreviewFinishedAt = Date().timeIntervalSince1970
+        let elapsedMs = Int(((Date().timeIntervalSince1970 - startedAt) * 1000).rounded())
+        let audioMs = Int((Double(samples.count) / 16_000.0 * 1000).rounded())
+        let rtf = audioMs > 0 ? Double(elapsedMs) / Double(audioMs) : 0
+        let finalizationMode = SettingsStore.shared.parakeetFinalizationMode.rawValue
+        DebugLogger.shared.info(
+            """
+            ASR_BENCH provider_streaming_done samples=\(samples.count) audioMs=\(audioMs) \
+            elapsedMs=\(elapsedMs) textChars=\(text.trimmingCharacters(in: .whitespacesAndNewlines).count) \
+            rtf=\(String(format: "%.3f", rtf)) finalizationMode=\(finalizationMode)
+            """,
+            source: "ASRBenchmark"
+        )
         return ASRTranscriptionResult(text: result.text, confidence: result.confidence)
     }
 
@@ -178,10 +210,19 @@ final class FluidAudioProvider: TranscriptionProvider {
             )
         }
 
+        let startedAt = Date().timeIntervalSince1970
+        if SettingsStore.shared.parakeetFinalizationMode == .tokenTimedChunkMerge {
+            if let previewResult = await self.cachedStreamingPreviewResult(for: samples, startedAt: startedAt) {
+                return previewResult
+            }
+        }
+
         // If the boosted final manager fails, fall back to the unboosted streaming
         // manager so the user still gets a transcription (just without CTC rescoring).
         do {
+            let startedAt = Date().timeIntervalSince1970
             let result = try await manager.transcribe(samples, source: AudioSource.microphone)
+            self.logFinalBenchmark(samples: samples, text: result.text, startedAt: startedAt, usedFallback: false)
             return ASRTranscriptionResult(text: result.text, confidence: result.confidence)
         } catch {
             guard let fallback = self.streamingAsrManager, fallback !== manager else {
@@ -191,11 +232,128 @@ final class FluidAudioProvider: TranscriptionProvider {
                 "FluidAudioProvider: Boosted final transcription failed (\(error.localizedDescription)), retrying without vocab boost",
                 source: "FluidAudioProvider"
             )
+            let startedAt = Date().timeIntervalSince1970
             let result = try await fallback.transcribe(samples, source: AudioSource.microphone)
+            self.logFinalBenchmark(samples: samples, text: result.text, startedAt: startedAt, usedFallback: true)
             return ASRTranscriptionResult(text: result.text, confidence: result.confidence)
         }
     }
 
+    func transcribeCachedStreamingPreviewIfAvailable(_ samples: [Float]) async -> ASRTranscriptionResult? {
+        guard SettingsStore.shared.parakeetFinalizationMode == .tokenTimedChunkMerge else {
+            return nil
+        }
+        let startedAt = Date().timeIntervalSince1970
+        return await self.cachedStreamingPreviewResult(for: samples, startedAt: startedAt)
+    }
+
+    private func cachedStreamingPreviewResult(for samples: [Float], startedAt: TimeInterval) async -> ASRTranscriptionResult? {
+        let text = self.latestStreamingPreviewText.trimmingCharacters(in: .whitespacesAndNewlines)
+        let finalSampleCount = samples.count
+        let previewSampleCount = min(self.latestStreamingPreviewSampleCount, finalSampleCount)
+        let tailSamples = max(0, finalSampleCount - previewSampleCount)
+        let tailMs = Int((Double(tailSamples) / 16_000.0 * 1000).rounded())
+        let tailRMS = self.rms(samples: samples, startIndex: previewSampleCount)
+        let coverage = finalSampleCount > 0 ? Double(previewSampleCount) / Double(finalSampleCount) : 0
+        let ageMs: Int
+        if let latestStreamingPreviewFinishedAt {
+            ageMs = Int(((Date().timeIntervalSince1970 - latestStreamingPreviewFinishedAt) * 1000).rounded())
+        } else {
+            ageMs = Int.max
+        }
+
+        DebugLogger.shared.info(
+            """
+            ASR_BENCH provider_fast_preview_check finalSamples=\(finalSampleCount) previewSamples=\(previewSampleCount) \
+            tailMs=\(tailMs) coverage=\(String(format: "%.3f", coverage)) ageMs=\(ageMs) \
+            tailRMS=\(String(format: "%.5f", tailRMS)) textChars=\(text.count) wordBoosting=\(self.isWordBoostingActive)
+            """,
+            source: "ASRBenchmark"
+        )
+
+        guard !text.isEmpty else {
+            self.logFastPreviewMiss(reason: "empty", tailMs: tailMs, coverage: coverage, ageMs: ageMs, tailRMS: tailRMS)
+            return nil
+        }
+        guard self.latestStreamingPreviewFinishedAt != nil, self.latestStreamingPreviewSampleCount > 0 else {
+            self.logFastPreviewMiss(reason: "missing_preview", tailMs: tailMs, coverage: coverage, ageMs: ageMs, tailRMS: tailRMS)
+            return nil
+        }
+        guard ageMs <= 3000 else {
+            self.logFastPreviewMiss(reason: "stale", tailMs: tailMs, coverage: coverage, ageMs: ageMs, tailRMS: tailRMS)
+            return nil
+        }
+        guard finalSampleCount >= self.fastPreviewMinimumSamples else {
+            self.logFastPreviewMiss(reason: "short_recording", tailMs: tailMs, coverage: coverage, ageMs: ageMs, tailRMS: tailRMS)
+            return nil
+        }
+        guard coverage >= 0.88 else {
+            self.logFastPreviewMiss(reason: "low_coverage", tailMs: tailMs, coverage: coverage, ageMs: ageMs, tailRMS: tailRMS)
+            return nil
+        }
+        guard tailMs <= 1800 else {
+            self.logFastPreviewMiss(reason: "large_tail", tailMs: tailMs, coverage: coverage, ageMs: ageMs, tailRMS: tailRMS)
+            return nil
+        }
+        guard tailSamples == 0 || tailMs <= self.fastPreviewTailAudioToleranceMs || tailRMS <= self.fastPreviewTailSilenceRMS else {
+            self.logFastPreviewMiss(reason: "tail_has_audio", tailMs: tailMs, coverage: coverage, ageMs: ageMs, tailRMS: tailRMS)
+            return nil
+        }
+
+        DebugLogger.shared.info(
+            """
+            ASR_BENCH provider_fast_preview_hit tailMs=\(tailMs) coverage=\(String(format: "%.3f", coverage)) \
+            ageMs=\(ageMs) tailRMS=\(String(format: "%.5f", tailRMS)) textChars=\(text.count)
+            """,
+            source: "ASRBenchmark"
+        )
+        self.logFinalBenchmark(samples: samples, text: text, startedAt: startedAt, usedFallback: false, source: "livePreview")
+        return ASRTranscriptionResult(text: text, confidence: 0.95)
+    }
+
+    private func rms(samples: [Float], startIndex: Int) -> Float {
+        guard startIndex < samples.count else { return 0 }
+        var sum: Float = 0
+        var count: Float = 0
+        for sample in samples[startIndex...] {
+            sum += sample * sample
+            count += 1
+        }
+        guard count > 0 else { return 0 }
+        return sqrt(sum / count)
+    }
+
+    private func logFastPreviewMiss(reason: String, tailMs: Int, coverage: Double, ageMs: Int, tailRMS: Float) {
+        DebugLogger.shared.info(
+            """
+            ASR_BENCH provider_fast_preview_miss reason=\(reason) tailMs=\(tailMs) \
+            coverage=\(String(format: "%.3f", coverage)) ageMs=\(ageMs) tailRMS=\(String(format: "%.5f", tailRMS))
+            """,
+            source: "ASRBenchmark"
+        )
+    }
+
+    private func logFinalBenchmark(
+        samples: [Float],
+        text: String,
+        startedAt: TimeInterval,
+        usedFallback: Bool,
+        source: String = "full"
+    ) {
+        let elapsedMs = Int(((Date().timeIntervalSince1970 - startedAt) * 1000).rounded())
+        let audioMs = Int((Double(samples.count) / 16_000.0 * 1000).rounded())
+        let rtf = audioMs > 0 ? Double(elapsedMs) / Double(audioMs) : 0
+        let finalizationMode = SettingsStore.shared.parakeetFinalizationMode.rawValue
+        DebugLogger.shared.info(
+            """
+            ASR_BENCH provider_final_done samples=\(samples.count) audioMs=\(audioMs) \
+            elapsedMs=\(elapsedMs) textChars=\(text.trimmingCharacters(in: .whitespacesAndNewlines).count) \
+            rtf=\(String(format: "%.3f", rtf)) fallback=\(usedFallback) finalizationMode=\(finalizationMode) source=\(source)
+            """,
+            source: "ASRBenchmark"
+        )
+    }
+
     func modelsExistOnDisk() -> Bool {
         let baseCacheDir = AsrModels.defaultCacheDirectory().deletingLastPathComponent()
         let selectedModel = self.modelOverride ?? SettingsStore.shared.selectedSpeechModel
@@ -332,5 +490,11 @@ final class FluidAudioProvider: TranscriptionProvider {
     func detectBoostedTerms(in text: String, limit: Int = 2) -> [String] {
         []
     }
+
+    func resetStreamingPreviewCache() {}
+
+    func transcribeCachedStreamingPreviewIfAvailable(_ samples: [Float]) async -> ASRTranscriptionResult? {
+        nil
+    }
 }
 #endif

Sources/Fluid/Persistence/SettingsStore.swift

@@ -1385,6 +1385,47 @@ struct SettingsView: View {
                     }
                     .padding(16)
                 }
+
+                // Experimental Card
+                ThemedCard(style: .standard) {
+                    VStack(alignment: .leading, spacing: 14) {
+                        Label("Experimental Settings", systemImage: "exclamationmark.triangle")
+                            .font(.headline)
+                            .foregroundStyle(.primary)
+
+                        VStack(alignment: .leading, spacing: 8) {
+                            HStack(alignment: .center) {
+                                VStack(alignment: .leading, spacing: 2) {
+                                    Text("Dictation Processing Speed")
+                                        .font(.body)
+                                }
+
+                                Spacer()
+
+                                Picker("", selection: self.$settings.parakeetFinalizationMode) {
+                                    ForEach(ParakeetFinalizationMode.allCases) { mode in
+                                        Text(mode.displayName).tag(mode)
+                                    }
+                                }
+                                .pickerStyle(.menu)
+                                .frame(width: 170, alignment: .trailing)
+                                .disabled(self.asr.isRunning)
+                            }
+
+                            Text("Standard: most reliable. Fast: faster, but maybe inaccurate.")
+                                .font(.caption)
+                                .foregroundStyle(.secondary)
+
+                            if self.asr.isRunning {
+                                Text("Settings are disabled during active recording")
+                                    .font(.caption)
+                                    .foregroundStyle(.secondary)
+                                    .italic()
+                            }
+                        }
+                    }
+                    .padding(16)
+                }
             }
             .padding(16)
         }