Skip to content

Commit 35d7e67

Browse files
claudeclaude
committed
feat: Auto-paste transcription into focused text field
- Add TranscriberAccessibilityService to type transcribed text directly into the currently focused input field - Auto-copy to clipboard on every successful transcription - Falls back to clipboard-only with toast if no field is focused or accessibility service is not enabled - Home screen shows card to enable accessibility when not active https://claude.ai/code/session_01ToKgc86ZWSEyGxQBcaridp
1 parent 248e06b commit 35d7e67

6 files changed

Lines changed: 141 additions & 5 deletions

File tree

‎app/src/main/AndroidManifest.xml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,17 @@
3838
android:value="floating_overlay_for_transcription" />
3939
</service>
4040

41+
<service
42+
android:name=".service.TranscriberAccessibilityService"
43+
android:exported="false"
44+
android:permission="android.permission.BIND_ACCESSIBILITY_SERVICE">
45+
<intent-filter>
46+
<action android:name="android.accessibilityservice.AccessibilityService" />
47+
</intent-filter>
48+
<meta-data
49+
android:name="android.accessibilityservice"
50+
android:resource="@xml/accessibility_service_config" />
51+
</service>
4152

4253
</application>
4354

‎app/src/main/java/com/whispertranscriber/MainActivity.kt‎

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import android.content.Intent
55
import android.os.Build
66
import android.os.Bundle
77
import android.provider.Settings
8+
import com.whispertranscriber.service.TranscriberAccessibilityService
89
import android.widget.Toast
910
import androidx.activity.ComponentActivity
1011
import androidx.activity.compose.setContent
@@ -20,11 +21,13 @@ import androidx.compose.foundation.layout.size
2021
import androidx.compose.foundation.layout.width
2122
import androidx.compose.material.icons.Icons
2223
import androidx.compose.material.icons.filled.History
24+
import androidx.compose.material.icons.filled.Accessibility
2325
import androidx.compose.material.icons.filled.Mic
2426
import androidx.compose.material.icons.filled.Settings
2527
import androidx.compose.material.icons.filled.Stop
2628
import androidx.compose.material3.Button
2729
import androidx.compose.material3.ButtonDefaults
30+
import androidx.compose.material3.OutlinedButton
2831
import androidx.compose.material3.Card
2932
import androidx.compose.material3.ExperimentalMaterial3Api
3033
import androidx.compose.material3.Icon
@@ -80,7 +83,11 @@ class MainActivity : ComponentActivity() {
8083
onSettingsClick = { navController.navigate("settings") },
8184
onLogClick = { navController.navigate("log") },
8285
onToggleOverlay = { toggleOverlayService() },
83-
overlayRunning = overlayRunning
86+
onEnableAccessibility = {
87+
startActivity(Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS))
88+
},
89+
overlayRunning = overlayRunning,
90+
accessibilityEnabled = TranscriberAccessibilityService.isAvailable()
8491
)
8592
}
8693
composable("settings") {
@@ -135,7 +142,9 @@ fun HomeScreen(
135142
onSettingsClick: () -> Unit,
136143
onLogClick: () -> Unit,
137144
onToggleOverlay: () -> Unit,
138-
overlayRunning: Boolean
145+
onEnableAccessibility: () -> Unit,
146+
overlayRunning: Boolean,
147+
accessibilityEnabled: Boolean
139148
) {
140149
Scaffold(
141150
topBar = {
@@ -200,6 +209,35 @@ fun HomeScreen(
200209
}
201210
}
202211

212+
if (!accessibilityEnabled) {
213+
Card(modifier = Modifier.fillMaxWidth()) {
214+
Column(
215+
modifier = Modifier.padding(16.dp),
216+
verticalArrangement = Arrangement.spacedBy(8.dp)
217+
) {
218+
Text("Type into Apps", style = MaterialTheme.typography.titleMedium)
219+
Text(
220+
"Enable the accessibility service to automatically paste transcriptions into the focused text field.",
221+
style = MaterialTheme.typography.bodyMedium,
222+
color = MaterialTheme.colorScheme.onSurfaceVariant
223+
)
224+
Spacer(Modifier.height(4.dp))
225+
OutlinedButton(
226+
onClick = onEnableAccessibility,
227+
modifier = Modifier.fillMaxWidth()
228+
) {
229+
Icon(
230+
Icons.Default.Accessibility,
231+
contentDescription = null,
232+
modifier = Modifier.size(18.dp)
233+
)
234+
Spacer(Modifier.width(8.dp))
235+
Text("Enable Accessibility")
236+
}
237+
}
238+
}
239+
}
240+
203241
Text(
204242
"Configure your Whisper server URL in Settings",
205243
style = MaterialTheme.typography.bodySmall,

‎app/src/main/java/com/whispertranscriber/service/FloatingOverlayService.kt‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -193,10 +193,13 @@ class FloatingOverlayService : Service() {
193193
audioData = wavData
194194
)
195195
val elapsed = System.currentTimeMillis() - startTime
196-
transcriptionText = if (result.success) {
197-
result.text.ifBlank { "(No speech detected)" }
196+
if (result.success && result.text.isNotBlank()) {
197+
transcriptionText = result.text
198+
outputText(result.text)
199+
} else if (result.success) {
200+
transcriptionText = "(No speech detected)"
198201
} else {
199-
"Error: ${result.error}"
202+
transcriptionText = "Error: ${result.error}"
200203
}
201204
transcriptionLog.addEntry(
202205
durationMs = elapsed,
@@ -322,6 +325,20 @@ class FloatingOverlayService : Service() {
322325
expandedView?.findViewWithTag<TextView>("transcription_content")?.text = transcriptionText
323326
}
324327

328+
private fun outputText(text: String) {
329+
// Always copy to clipboard
330+
val clipboard = getSystemService(Context.CLIPBOARD_SERVICE) as ClipboardManager
331+
clipboard.setPrimaryClip(ClipData.newPlainText("Transcription", text))
332+
333+
// Try to paste into the focused text field via accessibility
334+
if (TranscriberAccessibilityService.pasteText(text)) {
335+
Log.d(TAG, "Text pasted into focused field")
336+
} else {
337+
Toast.makeText(this, "Copied to clipboard", Toast.LENGTH_SHORT).show()
338+
Log.d(TAG, "No focused field, copied to clipboard")
339+
}
340+
}
341+
325342
private fun copyToClipboard() {
326343
if (transcriptionText.isBlank()) return
327344
val clipboard = getSystemService(Context.CLIPBOARD_SERVICE) as ClipboardManager

‎app/src/main/java/com/whispertranscriber/service/TranscriberAccessibilityService.kt‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
package com.whispertranscriber.service
2+
3+
import android.accessibilityservice.AccessibilityService
4+
import android.os.Bundle
5+
import android.util.Log
6+
import android.view.accessibility.AccessibilityEvent
7+
import android.view.accessibility.AccessibilityNodeInfo
8+
9+
class TranscriberAccessibilityService : AccessibilityService() {
10+
11+
companion object {
12+
private const val TAG = "TranscriberA11y"
13+
var instance: TranscriberAccessibilityService? = null
14+
private set
15+
16+
fun pasteText(text: String): Boolean {
17+
return instance?.commitText(text) ?: false
18+
}
19+
20+
fun isAvailable(): Boolean = instance != null
21+
}
22+
23+
override fun onServiceConnected() {
24+
super.onServiceConnected()
25+
instance = this
26+
Log.d(TAG, "Accessibility service connected")
27+
}
28+
29+
override fun onAccessibilityEvent(event: AccessibilityEvent?) {
30+
// Not needed — we only use this service to commit text
31+
}
32+
33+
override fun onInterrupt() {}
34+
35+
override fun onDestroy() {
36+
instance = null
37+
super.onDestroy()
38+
}
39+
40+
private fun commitText(text: String): Boolean {
41+
val rootNode = rootInActiveWindow ?: return false
42+
val focusedNode = rootNode.findFocus(AccessibilityNodeInfo.FOCUS_INPUT)
43+
if (focusedNode != null && focusedNode.isEditable) {
44+
val args = Bundle().apply {
45+
putCharSequence(
46+
AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE,
47+
getExistingText(focusedNode) + text
48+
)
49+
}
50+
val result = focusedNode.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, args)
51+
Log.d(TAG, "Text committed: $result")
52+
return result
53+
}
54+
Log.d(TAG, "No focused editable field found")
55+
return false
56+
}
57+
58+
private fun getExistingText(node: AccessibilityNodeInfo): String {
59+
return node.text?.toString() ?: ""
60+
}
61+
}

‎app/src/main/res/values/strings.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,5 @@
44
<string name="channel_overlay">Overlay Service</string>
55
<string name="notification_overlay_title">Whisper Transcriber Active</string>
66
<string name="notification_overlay_text">Floating bubble is running</string>
7+
<string name="accessibility_service_description">Allows Whisper Transcriber to paste transcribed text into the currently focused text field.</string>
78
</resources>

‎app/src/main/res/xml/accessibility_service_config.xml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
3+
android:accessibilityEventTypes="typeAllMask"
4+
android:accessibilityFeedbackType="feedbackGeneric"
5+
android:accessibilityFlags="flagDefault|flagRetrieveInteractiveWindows"
6+
android:canRetrieveWindowContent="true"
7+
android:description="@string/accessibility_service_description"
8+
android:notificationTimeout="100" />

0 commit comments

Comments
 (0)