feat: add Kokoro TTS playback

Donach · Donach · commit 1bd88006e72d · 2026-05-25T21:19:42.000+02:00
diff --git a/README.md b/README.md
@@ -11,7 +11,8 @@ Works over Tailscale / ZeroTier — just point it at your server's VPN IP.
 3. Audio is streamed to WhisperLiveKit via native WebSocket (`/asr`) when PCM input is enabled
 4. Partial transcripts replace the in-progress text in the focused input field in real time
 5. If no editable field is focused, the final transcript is copied to the clipboard once the utterance is silent
-6. If live streaming is unavailable, the app falls back to the OpenAI-compatible REST API (`/v1/audio/transcriptions`)
+6. Optional Kokoro TTS can read clipboard text aloud through the overlay
+7. If live streaming is unavailable, the app falls back to the OpenAI-compatible REST API (`/v1/audio/transcriptions`)
 
 ## Setup
 
@@ -23,6 +24,8 @@ Run [WhisperLiveKit](https://github.com/QuentinFuxa/WhisperLiveKit) on your mach
 whisperlivekit-server --host 0.0.0.0 --port 8090 --pcm-input
 ```
 
+For TTS, run Kokoro-FastAPI on your machine or tailnet. The app discovers healthy TTS servers on port `8880` by default and uses the OpenAI-compatible `/v1/audio/speech` endpoint.
+
 ### App
 
 1. Install the APK (grab from [Actions artifacts](../../actions) or build yourself)
@@ -33,6 +36,7 @@ whisperlivekit-server --host 0.0.0.0 --port 8090 --pcm-input
    - **Notifications** — for the foreground service
 4. Enable the **Whisper Transcriber** accessibility service in Android Settings → Accessibility (needed to type into other apps' text fields)
 5. Tap **Start Overlay** — the floating bubble appears
+6. Optional: in **Settings → Text To Speech**, discover your Kokoro server, test the connection to load voices, pick a voice/speed, and play sample text
 
 ### Permissions
 
@@ -122,6 +126,7 @@ The app checks a rolling GitHub Release manifest at `app-latest`. When a newer `
 - **HTTP / ws://** works out of the box to any IP (cleartext traffic is allowed via network security config)
 - **HTTPS with self-signed certs** works — the client trusts all certificates (this is a private VPN tool, not a public app)
 - Works over **Tailscale**, **ZeroTier**, or any VPN — just use the VPN IP as the server URL
+- Long-press the overlay to open the panel, then tap **SPEAK** to read the current clipboard with the selected Kokoro voice
 
 ## Live endpoint probe
 
diff --git a/app/src/main/java/com/whispertranscriber/audio/TtsAudioPlayer.kt b/app/src/main/java/com/whispertranscriber/audio/TtsAudioPlayer.kt
@@ -0,0 +1,39 @@
+package com.whispertranscriber.audio
+
+import android.content.Context
+import android.media.MediaPlayer
+import java.io.File
+
+class TtsAudioPlayer(private val context: Context) {
+    private var mediaPlayer: MediaPlayer? = null
+
+    fun playWav(bytes: ByteArray) {
+        stop()
+        val file = File(context.cacheDir, "tts/kokoro-test.wav").apply {
+            parentFile?.mkdirs()
+            writeBytes(bytes)
+        }
+        mediaPlayer = MediaPlayer().apply {
+            setDataSource(file.absolutePath)
+            setOnCompletionListener {
+                it.release()
+                if (mediaPlayer == it) mediaPlayer = null
+            }
+            setOnErrorListener { player, _, _ ->
+                player.release()
+                if (mediaPlayer == player) mediaPlayer = null
+                true
+            }
+            prepare()
+            start()
+        }
+    }
+
+    fun stop() {
+        mediaPlayer?.let {
+            if (it.isPlaying) it.stop()
+            it.release()
+        }
+        mediaPlayer = null
+    }
+}
diff --git a/app/src/main/java/com/whispertranscriber/data/SettingsStore.kt b/app/src/main/java/com/whispertranscriber/data/SettingsStore.kt
@@ -4,6 +4,7 @@ import android.content.Context
 import androidx.datastore.core.DataStore
 import androidx.datastore.preferences.core.Preferences
 import androidx.datastore.preferences.core.edit
+import androidx.datastore.preferences.core.floatPreferencesKey
 import androidx.datastore.preferences.core.intPreferencesKey
 import androidx.datastore.preferences.core.stringPreferencesKey
 import androidx.datastore.preferences.preferencesDataStore
@@ -16,7 +17,11 @@ private val Context.dataStore: DataStore<Preferences> by preferencesDataStore(na
 data class AppSettings(
     val whisperServerUrl: String = "",
     val whisperServerPort: Int = WhisperServerDiscovery.DEFAULT_PORT,
-    val audioQuality: String = "medium"
+    val audioQuality: String = "medium",
+    val ttsServerUrl: String = "",
+    val ttsServerPort: Int = 8880,
+    val ttsVoice: String = "af_heart",
+    val ttsSpeed: Float = 1.0f
 )
 
 class SettingsStore(private val context: Context) {
@@ -25,13 +30,21 @@ class SettingsStore(private val context: Context) {
         private val KEY_SERVER_URL = stringPreferencesKey("whisper_server_url")
         private val KEY_SERVER_PORT = intPreferencesKey("whisper_server_port")
         private val KEY_AUDIO_QUALITY = stringPreferencesKey("audio_quality")
+        private val KEY_TTS_SERVER_URL = stringPreferencesKey("tts_server_url")
+        private val KEY_TTS_SERVER_PORT = intPreferencesKey("tts_server_port")
+        private val KEY_TTS_VOICE = stringPreferencesKey("tts_voice")
+        private val KEY_TTS_SPEED = floatPreferencesKey("tts_speed")
     }
 
     val settings: Flow<AppSettings> = context.dataStore.data.map { prefs ->
         AppSettings(
             whisperServerUrl = prefs[KEY_SERVER_URL] ?: AppSettings().whisperServerUrl,
             whisperServerPort = prefs[KEY_SERVER_PORT] ?: AppSettings().whisperServerPort,
-            audioQuality = prefs[KEY_AUDIO_QUALITY] ?: AppSettings().audioQuality
+            audioQuality = prefs[KEY_AUDIO_QUALITY] ?: AppSettings().audioQuality,
+            ttsServerUrl = prefs[KEY_TTS_SERVER_URL] ?: AppSettings().ttsServerUrl,
+            ttsServerPort = prefs[KEY_TTS_SERVER_PORT] ?: AppSettings().ttsServerPort,
+            ttsVoice = prefs[KEY_TTS_VOICE] ?: AppSettings().ttsVoice,
+            ttsSpeed = prefs[KEY_TTS_SPEED] ?: AppSettings().ttsSpeed
         )
     }
 
@@ -46,4 +59,20 @@ class SettingsStore(private val context: Context) {
     suspend fun updateAudioQuality(quality: String) {
         context.dataStore.edit { it[KEY_AUDIO_QUALITY] = quality }
     }
+
+    suspend fun updateTtsServerUrl(url: String) {
+        context.dataStore.edit { it[KEY_TTS_SERVER_URL] = url }
+    }
+
+    suspend fun updateTtsServerPort(port: Int) {
+        context.dataStore.edit { it[KEY_TTS_SERVER_PORT] = port }
+    }
+
+    suspend fun updateTtsVoice(voice: String) {
+        context.dataStore.edit { it[KEY_TTS_VOICE] = voice }
+    }
+
+    suspend fun updateTtsSpeed(speed: Float) {
+        context.dataStore.edit { it[KEY_TTS_SPEED] = speed }
+    }
 }
diff --git a/app/src/main/java/com/whispertranscriber/network/KokoroTtsClient.kt b/app/src/main/java/com/whispertranscriber/network/KokoroTtsClient.kt
@@ -0,0 +1,96 @@
+package com.whispertranscriber.network
+
+import com.google.gson.JsonObject
+import com.google.gson.JsonParser
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.suspendCancellableCoroutine
+import kotlinx.coroutines.withContext
+import okhttp3.Call
+import okhttp3.Callback
+import okhttp3.MediaType.Companion.toMediaType
+import okhttp3.OkHttpClient
+import okhttp3.Request
+import okhttp3.RequestBody.Companion.toRequestBody
+import okhttp3.Response
+import java.io.IOException
+import java.util.concurrent.TimeUnit
+import kotlin.coroutines.resume
+import kotlin.coroutines.resumeWithException
+
+class KokoroTtsClient {
+    private val client = OkHttpClient.Builder()
+        .connectTimeout(15, TimeUnit.SECONDS)
+        .readTimeout(120, TimeUnit.SECONDS)
+        .build()
+
+    suspend fun voices(serverUrl: String): List<String> = withContext(Dispatchers.IO) {
+        val request = Request.Builder()
+            .url(serverUrl.trimEnd('/') + "/v1/audio/voices")
+            .header("Cache-Control", "no-cache")
+            .build()
+        val response = client.newCall(request).await()
+        response.use {
+            if (!it.isSuccessful) throw IOException("Voices HTTP ${it.code}")
+            KokoroVoiceParser.parse(it.body?.string().orEmpty())
+        }
+    }
+
+    suspend fun synthesizeWav(
+        serverUrl: String,
+        text: String,
+        voice: String,
+        speed: Float
+    ): ByteArray = withContext(Dispatchers.IO) {
+        val request = Request.Builder()
+            .url(serverUrl.trimEnd('/') + "/v1/audio/speech")
+            .post(KokoroSpeechRequest.json(text, voice, speed).toRequestBody("application/json".toMediaType()))
+            .build()
+        val response = client.newCall(request).await()
+        response.use {
+            if (!it.isSuccessful) throw IOException("Speech HTTP ${it.code}: ${it.body?.string().orEmpty()}")
+            it.body?.bytes() ?: throw IOException("Empty speech response")
+        }
+    }
+
+    fun shutdown() {
+        client.dispatcher.executorService.shutdown()
+        client.connectionPool.evictAll()
+    }
+
+    private suspend fun Call.await(): Response = suspendCancellableCoroutine { continuation ->
+        continuation.invokeOnCancellation { cancel() }
+        enqueue(object : Callback {
+            override fun onResponse(call: Call, response: Response) {
+                continuation.resume(response)
+            }
+
+            override fun onFailure(call: Call, e: IOException) {
+                continuation.resumeWithException(e)
+            }
+        })
+    }
+}
+
+object KokoroVoiceParser {
+    fun parse(jsonText: String): List<String> {
+        val json = JsonParser.parseString(jsonText).asJsonObject
+        return json.getAsJsonArray("voices")
+            ?.mapNotNull { it.takeUnless { value -> value.isJsonNull }?.asString }
+            ?.filter { it.isNotBlank() }
+            .orEmpty()
+    }
+}
+
+object KokoroSpeechRequest {
+    fun json(text: String, voice: String, speed: Float): String {
+        val request = JsonObject().apply {
+            addProperty("model", "kokoro")
+            addProperty("input", text)
+            addProperty("voice", voice)
+            addProperty("response_format", "wav")
+            addProperty("speed", speed.coerceIn(0.25f, 4.0f))
+            addProperty("stream", false)
+        }
+        return request.toString()
+    }
+}
diff --git a/app/src/main/java/com/whispertranscriber/service/FloatingOverlayService.kt b/app/src/main/java/com/whispertranscriber/service/FloatingOverlayService.kt
@@ -27,8 +27,10 @@ import android.widget.Toast
 import com.whispertranscriber.MainActivity
 import com.whispertranscriber.R
 import com.whispertranscriber.audio.AudioRecorder
+import com.whispertranscriber.audio.TtsAudioPlayer
 import com.whispertranscriber.data.SettingsStore
 import com.whispertranscriber.data.TranscriptionLog
+import com.whispertranscriber.network.KokoroTtsClient
 import com.whispertranscriber.network.WhisperApiClient
 import com.whispertranscriber.network.WhisperLiveKitClient
 import com.whispertranscriber.network.WhisperLiveKitSession
@@ -58,7 +60,9 @@ class FloatingOverlayService : Service() {
     private val audioRecorder = AudioRecorder()
     private val whisperClient = WhisperApiClient()
     private val liveKitClient = WhisperLiveKitClient()
+    private val ttsClient = KokoroTtsClient()
     private val serviceScope = CoroutineScope(SupervisorJob() + Dispatchers.Main)
+    private lateinit var ttsAudioPlayer: TtsAudioPlayer
 
     private var bubbleView: View? = null
     private var expandedView: View? = null
@@ -80,6 +84,7 @@ class FloatingOverlayService : Service() {
         windowManager = getSystemService(WINDOW_SERVICE) as WindowManager
         settingsStore = SettingsStore(this)
         transcriptionLog = TranscriptionLog(this)
+        ttsAudioPlayer = TtsAudioPlayer(this)
         createNotificationChannel()
         startForeground(NOTIFICATION_ID, buildNotification())
         createBubbleView()
@@ -350,6 +355,14 @@ class FloatingOverlayService : Service() {
         return discovered.url
     }
 
+    private suspend fun resolveTtsServerUrl(configuredUrl: String, discoveryPort: Int): String {
+        if (configuredUrl.isNotBlank()) return configuredUrl
+        val discovered = WhisperServerDiscovery.discover(port = discoveryPort)
+            ?: throw IllegalStateException("No Kokoro TTS server found on local networks or Tailscale port $discoveryPort")
+        settingsStore.updateTtsServerUrl(discovered.url)
+        return discovered.url
+    }
+
     private fun toggleExpandedView() {
         if (isExpanded) {
             removeExpandedView()
@@ -394,6 +407,15 @@ class FloatingOverlayService : Service() {
         }
         titleBar.addView(copyButton)
 
+        val speakButton = TextView(this).apply {
+            text = "SPEAK"
+            textSize = 12f
+            setTextColor(0xFF6750A4.toInt())
+            setPadding((8 * density).toInt(), (4 * density).toInt(), (8 * density).toInt(), (4 * density).toInt())
+            setOnClickListener { speakClipboardText() }
+        }
+        titleBar.addView(speakButton)
+
         val closeButton = TextView(this).apply {
             text = "X"
             textSize = 14f
@@ -493,6 +515,40 @@ class FloatingOverlayService : Service() {
         Toast.makeText(this, "Copied to clipboard", Toast.LENGTH_SHORT).show()
     }
 
+    private fun speakClipboardText() {
+        val clipboard = getSystemService(Context.CLIPBOARD_SERVICE) as ClipboardManager
+        val text = clipboard.primaryClip
+            ?.takeIf { it.itemCount > 0 }
+            ?.getItemAt(0)
+            ?.coerceToText(this)
+            ?.toString()
+            ?.trim()
+            .orEmpty()
+
+        if (text.isBlank()) {
+            Toast.makeText(this, "Clipboard is empty", Toast.LENGTH_SHORT).show()
+            return
+        }
+
+        serviceScope.launch {
+            try {
+                val settings = settingsStore.settings.first()
+                val serverUrl = resolveTtsServerUrl(settings.ttsServerUrl, settings.ttsServerPort)
+                val audio = ttsClient.synthesizeWav(
+                    serverUrl = serverUrl,
+                    text = text,
+                    voice = settings.ttsVoice,
+                    speed = settings.ttsSpeed
+                )
+                ttsAudioPlayer.playWav(audio)
+                Toast.makeText(this@FloatingOverlayService, "Playing clipboard", Toast.LENGTH_SHORT).show()
+            } catch (e: Exception) {
+                Toast.makeText(this@FloatingOverlayService, "TTS failed: ${e.message}", Toast.LENGTH_LONG).show()
+                Log.e(TAG, "TTS playback failed", e)
+            }
+        }
+    }
+
     private fun createNotificationChannel() {
         val channel = NotificationChannel(
             CHANNEL_ID,
@@ -532,8 +588,10 @@ class FloatingOverlayService : Service() {
         transcriptionJob?.cancel()
         serviceScope.cancel()
         audioRecorder.release()
+        ttsAudioPlayer.stop()
         whisperClient.shutdown()
         liveKitClient.shutdown()
+        ttsClient.shutdown()
         removeExpandedView()
         bubbleView?.let {
             try {
diff --git a/app/src/main/java/com/whispertranscriber/ui/SettingsScreen.kt b/app/src/main/java/com/whispertranscriber/ui/SettingsScreen.kt
diff --git a/app/src/test/java/com/whispertranscriber/network/KokoroTtsClientTest.kt b/app/src/test/java/com/whispertranscriber/network/KokoroTtsClientTest.kt
