Add RMVPE pitch extraction and NSF source to RVC pipeline

Integrated RMVPE pitch extraction model and neural source-filter (NSF) source module into the RVC inference pipeline. Updated Generator to use the NSF source and harmonics, and AudioProcessor to save audio as Int16 PCM for compatibility. Improved feature and F0 upsampling logic, added optional RMVPE loading, and clarified simulator limitations in documentation.

Author: Acelogic

Demos/iOS/RVCNative/RVCNative/ContentView.swift

@@ -9,6 +9,9 @@ struct ContentView: View {
     @State private var statusMessage: String = "Ready"
     @State private var isProcessing: Bool = false
     @State private var isImporting: Bool = false
+    @State private var logs: [String] = []
+    
+    @State private var isModelLoaded: Bool = false
 
     @StateObject private var inferenceEngine = RVCInference()
     @StateObject private var audioRecorder = AudioRecorder()
@@ -133,11 +136,11 @@ struct ContentView: View {
                 .font(.headline)
                 .frame(maxWidth: .infinity)
                 .padding()
-                .background(isProcessing ? Color.gray : Color.green)
+                .background(isProcessing || !isModelLoaded ? Color.gray : Color.green)
                 .foregroundColor(.white)
                 .cornerRadius(12)
             }
-            .disabled(isProcessing || inputURL == nil)
+            .disabled(isProcessing || inputURL == nil || !isModelLoaded)
 
             // Play Converted
             if let url = outputURL, !isProcessing {
@@ -182,7 +185,7 @@ struct ContentView: View {
         .padding()
         .onChange(of: inferenceEngine.status) { oldValue, newValue in
              statusMessage = newValue 
-             if newValue == "Done!" || newValue.starts(with: "Error") {
+             if newValue == "Done!" || newValue == "Models Loaded" || newValue.starts(with: "Error") {
                  isProcessing = false
              } else if newValue != "Idle" {
                  isProcessing = true
@@ -218,41 +221,97 @@ struct ContentView: View {
                 statusMessage = "Error: \(error.localizedDescription)"
             }
         }
+        .onAppear {
+            inferenceEngine.onLog = { msg in
+                self.log(msg)
+            }
+            loadModel(name: "Coder")
+        }
+    }
+    
+    func log(_ message: String) {
+        print("DEBUG: \(message)") // Keep in console
+        DispatchQueue.main.async {
+            self.logs.append(message)
+        }
     }
 
     func loadModel(name: String) {
+        log("loadModel called for \(name)")
         selectedModel = name
+        isModelLoaded = false // Reset state
+        
         // map name to file
-        let filename = name.lowercased() 
-        // Coder -> coder.npz
-        // Slim Shady -> placeholder?
+        let filename = name.lowercased().replacingOccurrences(of: " ", with: "_")
 
-        guard let url = RVCInference.bundle.url(forResource: filename, withExtension: "safetensors") else {
+        // Try finding in root or Assets subdir
+        let modelUrl = RVCInference.bundle.url(forResource: filename, withExtension: "safetensors") 
+            ?? RVCInference.bundle.url(forResource: filename, withExtension: "safetensors", subdirectory: "Assets")
+            
+        guard let url = modelUrl else {
+             log("Failed to find model file: \(filename).safetensors")
              statusMessage = "Model \(name) not found in bundle"
              return
         }
+        log("Found model at \(url.path)")
 
-        let hubertURL = RVCInference.bundle.url(forResource: "hubert_mlx", withExtension: "safetensors")
+        let hubertUrl = RVCInference.bundle.url(forResource: "hubert_base", withExtension: "safetensors")
+            ?? RVCInference.bundle.url(forResource: "hubert_base", withExtension: "safetensors", subdirectory: "Assets")
+            
+        guard let hubertURL = hubertUrl else {
+             log("Failed to find hubert_base.safetensors")
+             statusMessage = "Hubert model not found!"
+             return
+        }
+        log("Found hubert at \(hubertURL.path)")
+        
+        // Optional RMVPE
+        let rmvpeURL = RVCInference.bundle.url(forResource: "rmvpe", withExtension: "safetensors")
+            ?? RVCInference.bundle.url(forResource: "rmvpe", withExtension: "safetensors", subdirectory: "Assets")
+            ?? RVCInference.bundle.url(forResource: "rmvpe", withExtension: "npz")
+            ?? RVCInference.bundle.url(forResource: "rmvpe", withExtension: "npz", subdirectory: "Assets")
+            ?? RVCInference.bundle.url(forResource: "rmvpe_mlx", withExtension: "npz")
+            ?? RVCInference.bundle.url(forResource: "rmvpe_mlx", withExtension: "npz", subdirectory: "Assets")
+        
+        if let r = rmvpeURL {
+            log("Found rmvpe at \(r.path)")
+        } else {
+            log("RMVPE not found (optional)")
+        }
 
         Task {
             do {
-                try inferenceEngine.loadWeights(url: url, hubertURL: hubertURL)
+                log("Starting loadWeights task...")
+                try await inferenceEngine.loadWeights(hubertURL: hubertURL, modelURL: url, rmvpeURL: rmvpeURL)
+                log("loadWeights success")
                 statusMessage = "Loaded \(name)"
+                isModelLoaded = true
             } catch {
+                log("loadWeights failed: \(error)")
                 statusMessage = "Failed to load \(name): \(error.localizedDescription)"
+                isModelLoaded = false
             }
         }
     }
 
     func startInference() async {
         guard let input = inputURL else { return }
+        
+        if selectedModel == "Select Model" {
+            statusMessage = "Please select a model first."
+            log("Attempted inference without selecting model")
+            return
+        }
+        
+        log("Starting inference processing...")
         isProcessing = true
 
         // Temp output
         let output = FileManager.default.temporaryDirectory.appendingPathComponent("output.wav")
         self.outputURL = output
 
         await inferenceEngine.infer(audioURL: input, outputURL: output)
+        log("Inference complete.")
     }
 }
 

Demos/iOS/RVCNative/RVCNativePackage/Package.swift

@@ -31,6 +31,7 @@ let package = Package(
                 .product(name: "MLX", package: "mlx-swift"),
                 .product(name: "MLXNN", package: "mlx-swift"),
                 .product(name: "MLXRandom", package: "mlx-swift"),
+                .product(name: "MLXFFT", package: "mlx-swift"),
             ],
             resources: [
                 .copy("Assets")

Demos/iOS/RVCNative/RVCNativePackage/Sources/RVCNativeFeature/RVC/AudioProcessor.swift

@@ -43,18 +43,40 @@ final class AudioProcessor: @unchecked Sendable {
     // Save MLXArray to WAV
     func saveAudio(array: MLXArray, url: URL, sampleRate: Double = 40000) throws {
         let count = array.size
-        // Convert to Swift Array
-        // Assuming array is 1D float32
         let samples = array.asArray(Float.self)
 
-        let format = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: sampleRate, channels: 1, interleaved: false)!
-        let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(count))!
+        // 1. Define File Format: Int16 (Standard WAV)
+        // This ensures compatibility with all players
+        let fileSettings: [String: Any] = [
+            AVFormatIDKey: kAudioFormatLinearPCM,
+            AVSampleRateKey: sampleRate,
+            AVNumberOfChannelsKey: 1,
+            AVLinearPCMBitDepthKey: 16,
+            AVLinearPCMIsFloatKey: false,
+            AVLinearPCMIsBigEndianKey: false,
+            AVLinearPCMIsNonInterleaved: false
+        ]
+        
+        // 2. Define Buffer Format: Float32
+        // AVAudioFile processes in Float32 by default
+        let bufferFormat = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: sampleRate, channels: 1, interleaved: false)!
+        let buffer = AVAudioPCMBuffer(pcmFormat: bufferFormat, frameCapacity: AVAudioFrameCount(count))!
         buffer.frameLength = AVAudioFrameCount(count)
 
-        guard let channelData = buffer.floatChannelData?[0] else { return }
-        channelData.initialize(from: samples, count: count)
+        // 3. Fill Buffer (Clamp checks are handled by CoreAudio conversion usually, but safe to clamp)
+        if let channelData = buffer.floatChannelData?[0] {
+             for i in 0..<count {
+                 // conversion to Int16 implies clamping, but since we are keeping it as float
+                 // we just ensure it's in -1...1 range so conversion doesn't overflow weirdly
+                 channelData[i] = min(max(samples[i], -1.0), 1.0)
+             }
+        }
+        
+        // 4. Write
+        try? FileManager.default.removeItem(at: url)
 
-        let file = try AVAudioFile(forWriting: url, settings: format.settings)
+        // Initialize file with Int16 settings, but tell it we will provide Float32 data
+        let file = try AVAudioFile(forWriting: url, settings: fileSettings, commonFormat: .pcmFormatFloat32, interleaved: false)
         try file.write(from: buffer)
     }
 

Demos/iOS/RVCNative/RVCNativePackage/Sources/RVCNativeFeature/RVC/HubertModel.swift

@@ -309,7 +309,7 @@ public class HubertModel: Module {
         var x = feature_extractor(inputValues)
         x = feature_projection(x)
         x = encoder(x)
-        x = final_proj(x)
+        // x = final_proj(x) // Skip projection for RVC feature extraction
         return x
     }
 }