Skip to content

Commit 096cd08

Browse files
enaboappsOwenMcGirr
enaboapps
and
OwenMcGirr
authored
feat: add Screen Highlights AI task (#2286) (#2287)
Second feature on the reusable AiBackend / AiTask / OnDeviceAi architecture. Captures the current screen, asks the on-device model to pull every actionable item out of it (URLs, phone numbers, emails, dates, addresses, plus a free-form "other" bucket), and lets the user tap one to copy it to the clipboard. Mirrors the Reply Drafter pattern end-to-end: foreground Compose activity, screenshot handed off via a process-internal holder, IAP gate on the menu entry. No changes to the shared AI infrastructure. Closes #2286 Co-authored-by: Owen McGirr <o.a.mcgirr@gmail.com>
1 parent 41292c0 commit 096cd08

13 files changed

Lines changed: 757 additions & 0 deletions

File tree

‎app/src/main/AndroidManifest.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,13 @@
5151
android:taskAffinity=""
5252
android:theme="@style/Theme.Switchify" />
5353

54+
<activity
55+
android:name=".screens.screenhighlights.ScreenHighlightsActivity"
56+
android:excludeFromRecents="true"
57+
android:exported="false"
58+
android:taskAffinity=""
59+
android:theme="@style/Theme.Switchify" />
60+
5461
<service
5562
android:name=".service.core.SwitchifyAccessibilityService"
5663
android:directBootAware="true"

‎app/src/main/java/com/enaboapps/switchify/screens/screenhighlights/ScreenHighlightsActivity.kt‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
package com.enaboapps.switchify.screens.screenhighlights
2+
3+
import android.os.Bundle
4+
import android.widget.Toast
5+
import androidx.activity.ComponentActivity
6+
import androidx.activity.compose.setContent
7+
import androidx.activity.viewModels
8+
import androidx.compose.runtime.getValue
9+
import androidx.compose.runtime.livedata.observeAsState
10+
import androidx.lifecycle.viewmodel.initializer
11+
import androidx.lifecycle.viewmodel.viewModelFactory
12+
import com.enaboapps.switchify.R
13+
import com.enaboapps.switchify.activities.ui.theme.SwitchifyTheme
14+
15+
/**
16+
* Visible, foreground screen that extracts Screen Highlights. Being the top
17+
* foreground activity is what lets AICore (Gemini Nano) run inference — it
18+
* refuses when the caller is in the background, so extraction is started from
19+
* [onResume].
20+
*/
21+
class ScreenHighlightsActivity : ComponentActivity() {
22+
private val viewModel: ScreenHighlightsViewModel by viewModels {
23+
viewModelFactory {
24+
initializer { ScreenHighlightsViewModel(applicationContext) }
25+
}
26+
}
27+
28+
override fun onCreate(savedInstanceState: Bundle?) {
29+
super.onCreate(savedInstanceState)
30+
setContent {
31+
val state by viewModel.uiState.observeAsState(ScreenHighlightsUiState.Loading)
32+
33+
SwitchifyTheme {
34+
ScreenHighlightsScreen(
35+
state = state,
36+
onSelect = { item ->
37+
viewModel.copyToClipboard(item.value)
38+
Toast.makeText(
39+
this@ScreenHighlightsActivity,
40+
R.string.screen_highlights_copied,
41+
Toast.LENGTH_SHORT
42+
).show()
43+
finish()
44+
},
45+
onRetry = { viewModel.extract() },
46+
onClose = { finish() }
47+
)
48+
}
49+
}
50+
}
51+
52+
// AICore only runs inference while this activity is the top foreground
53+
// app, so extraction starts here rather than from the view model's init.
54+
override fun onResume() {
55+
super.onResume()
56+
viewModel.start()
57+
}
58+
}

‎app/src/main/java/com/enaboapps/switchify/screens/screenhighlights/ScreenHighlightsScreen.kt‎

Lines changed: 208 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,208 @@
1+
package com.enaboapps.switchify.screens.screenhighlights
2+
3+
import androidx.compose.foundation.layout.Arrangement
4+
import androidx.compose.foundation.layout.Box
5+
import androidx.compose.foundation.layout.Column
6+
import androidx.compose.foundation.layout.Spacer
7+
import androidx.compose.foundation.layout.fillMaxSize
8+
import androidx.compose.foundation.layout.fillMaxWidth
9+
import androidx.compose.foundation.layout.height
10+
import androidx.compose.foundation.layout.padding
11+
import androidx.compose.foundation.layout.systemBarsPadding
12+
import androidx.compose.foundation.rememberScrollState
13+
import androidx.compose.foundation.shape.RoundedCornerShape
14+
import androidx.compose.foundation.verticalScroll
15+
import androidx.compose.material3.Card
16+
import androidx.compose.material3.CircularProgressIndicator
17+
import androidx.compose.material3.MaterialTheme
18+
import androidx.compose.material3.Surface
19+
import androidx.compose.material3.Text
20+
import androidx.compose.runtime.Composable
21+
import androidx.compose.ui.Alignment
22+
import androidx.compose.ui.Modifier
23+
import androidx.compose.ui.res.stringResource
24+
import androidx.compose.ui.text.font.FontWeight
25+
import androidx.compose.ui.text.style.TextAlign
26+
import androidx.compose.ui.unit.dp
27+
import com.enaboapps.switchify.R
28+
import com.enaboapps.switchify.components.ActionButton
29+
import com.enaboapps.switchify.components.ActionButtonType
30+
import com.enaboapps.switchify.service.llm.ExtractedItem
31+
import com.enaboapps.switchify.service.llm.HighlightType
32+
import com.enaboapps.switchify.theme.Dimens
33+
34+
@Composable
35+
fun ScreenHighlightsScreen(
36+
state: ScreenHighlightsUiState,
37+
onSelect: (ExtractedItem) -> Unit,
38+
onRetry: () -> Unit,
39+
onClose: () -> Unit
40+
) {
41+
Surface(
42+
modifier = Modifier.fillMaxSize(),
43+
color = MaterialTheme.colorScheme.background
44+
) {
45+
Column(
46+
modifier = Modifier
47+
.fillMaxSize()
48+
.systemBarsPadding()
49+
.padding(Dimens.spaceL)
50+
) {
51+
Text(
52+
text = stringResource(R.string.screen_highlights_title),
53+
style = MaterialTheme.typography.headlineSmall,
54+
fontWeight = FontWeight.SemiBold
55+
)
56+
Spacer(modifier = Modifier.height(Dimens.spaceL))
57+
58+
Box(
59+
modifier = Modifier
60+
.weight(1f)
61+
.fillMaxWidth()
62+
) {
63+
when (state) {
64+
is ScreenHighlightsUiState.Loading -> LoadingContent()
65+
66+
is ScreenHighlightsUiState.Items ->
67+
ItemsContent(state.items, onSelect)
68+
69+
is ScreenHighlightsUiState.Empty ->
70+
EmptyContent(onRetry)
71+
72+
is ScreenHighlightsUiState.Failed ->
73+
FailedContent(state, onRetry)
74+
}
75+
}
76+
77+
Spacer(modifier = Modifier.height(Dimens.spaceM))
78+
ActionButton(
79+
textResId = R.string.screen_highlights_close,
80+
onClick = onClose,
81+
modifier = Modifier.fillMaxWidth(),
82+
type = ActionButtonType.SECONDARY,
83+
applyPadding = false
84+
)
85+
}
86+
}
87+
}
88+
89+
@Composable
90+
private fun LoadingContent() {
91+
Column(
92+
modifier = Modifier.fillMaxSize(),
93+
horizontalAlignment = Alignment.CenterHorizontally,
94+
verticalArrangement = Arrangement.Center
95+
) {
96+
CircularProgressIndicator()
97+
Spacer(modifier = Modifier.height(Dimens.spaceM))
98+
Text(
99+
text = stringResource(R.string.screen_highlights_processing),
100+
style = MaterialTheme.typography.bodyLarge
101+
)
102+
}
103+
}
104+
105+
@Composable
106+
private fun ItemsContent(items: List<ExtractedItem>, onSelect: (ExtractedItem) -> Unit) {
107+
Column(
108+
modifier = Modifier
109+
.fillMaxSize()
110+
.verticalScroll(rememberScrollState()),
111+
verticalArrangement = Arrangement.spacedBy(Dimens.spaceS)
112+
) {
113+
Text(
114+
text = stringResource(R.string.screen_highlights_pick_hint),
115+
style = MaterialTheme.typography.bodyMedium,
116+
color = MaterialTheme.colorScheme.onSurfaceVariant
117+
)
118+
items.forEach { item ->
119+
HighlightCard(item = item, onClick = { onSelect(item) })
120+
}
121+
}
122+
}
123+
124+
@Composable
125+
private fun HighlightCard(item: ExtractedItem, onClick: () -> Unit) {
126+
Card(
127+
onClick = onClick,
128+
modifier = Modifier.fillMaxWidth()
129+
) {
130+
Column(
131+
modifier = Modifier.padding(Dimens.spaceM),
132+
verticalArrangement = Arrangement.spacedBy(Dimens.spaceXs)
133+
) {
134+
TypeChip(item.type)
135+
Text(
136+
text = item.value,
137+
style = MaterialTheme.typography.bodyLarge
138+
)
139+
}
140+
}
141+
}
142+
143+
@Composable
144+
private fun TypeChip(type: HighlightType) {
145+
Surface(
146+
color = MaterialTheme.colorScheme.secondaryContainer,
147+
contentColor = MaterialTheme.colorScheme.onSecondaryContainer,
148+
shape = RoundedCornerShape(8.dp)
149+
) {
150+
Text(
151+
text = stringResource(typeLabelRes(type)),
152+
style = MaterialTheme.typography.labelSmall,
153+
fontWeight = FontWeight.SemiBold,
154+
modifier = Modifier.padding(horizontal = Dimens.spaceXs, vertical = 4.dp)
155+
)
156+
}
157+
}
158+
159+
private fun typeLabelRes(type: HighlightType): Int = when (type) {
160+
HighlightType.URL -> R.string.screen_highlights_type_url
161+
HighlightType.PHONE -> R.string.screen_highlights_type_phone
162+
HighlightType.EMAIL -> R.string.screen_highlights_type_email
163+
HighlightType.DATE -> R.string.screen_highlights_type_date
164+
HighlightType.ADDRESS -> R.string.screen_highlights_type_address
165+
HighlightType.OTHER -> R.string.screen_highlights_type_other
166+
}
167+
168+
@Composable
169+
private fun EmptyContent(onRetry: () -> Unit) {
170+
Column(
171+
modifier = Modifier.fillMaxSize(),
172+
horizontalAlignment = Alignment.CenterHorizontally,
173+
verticalArrangement = Arrangement.Center
174+
) {
175+
Text(
176+
text = stringResource(R.string.screen_highlights_none_found),
177+
style = MaterialTheme.typography.bodyLarge,
178+
textAlign = TextAlign.Center
179+
)
180+
Spacer(modifier = Modifier.height(Dimens.spaceM))
181+
ActionButton(
182+
textResId = R.string.screen_highlights_retry,
183+
onClick = onRetry
184+
)
185+
}
186+
}
187+
188+
@Composable
189+
private fun FailedContent(state: ScreenHighlightsUiState.Failed, onRetry: () -> Unit) {
190+
Column(
191+
modifier = Modifier.fillMaxSize(),
192+
horizontalAlignment = Alignment.CenterHorizontally,
193+
verticalArrangement = Arrangement.Center
194+
) {
195+
Text(
196+
text = stringResource(state.messageRes),
197+
style = MaterialTheme.typography.bodyLarge,
198+
textAlign = TextAlign.Center
199+
)
200+
if (state.canRetry) {
201+
Spacer(modifier = Modifier.height(Dimens.spaceM))
202+
ActionButton(
203+
textResId = R.string.screen_highlights_retry,
204+
onClick = onRetry
205+
)
206+
}
207+
}
208+
}

‎app/src/main/java/com/enaboapps/switchify/screens/screenhighlights/ScreenHighlightsViewModel.kt‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
package com.enaboapps.switchify.screens.screenhighlights
2+
3+
import android.content.ClipData
4+
import android.content.ClipboardManager
5+
import android.content.Context
6+
import android.graphics.Bitmap
7+
import androidx.lifecycle.LiveData
8+
import androidx.lifecycle.MutableLiveData
9+
import androidx.lifecycle.ViewModel
10+
import androidx.lifecycle.viewModelScope
11+
import com.enaboapps.switchify.R
12+
import com.enaboapps.switchify.service.llm.AiFailure
13+
import com.enaboapps.switchify.service.llm.AiResult
14+
import com.enaboapps.switchify.service.llm.ExtractedItem
15+
import com.enaboapps.switchify.service.llm.OnDeviceAi
16+
import com.enaboapps.switchify.service.llm.ScreenHighlightsScreenshotHolder
17+
import com.enaboapps.switchify.service.llm.ScreenHighlightsTask
18+
import kotlinx.coroutines.launch
19+
20+
sealed interface ScreenHighlightsUiState {
21+
data object Loading : ScreenHighlightsUiState
22+
data class Items(val items: List<ExtractedItem>) : ScreenHighlightsUiState
23+
data object Empty : ScreenHighlightsUiState
24+
data class Failed(val messageRes: Int, val canRetry: Boolean) : ScreenHighlightsUiState
25+
}
26+
27+
class ScreenHighlightsViewModel(context: Context) : ViewModel() {
28+
private val appContext = context.applicationContext
29+
private val bitmap: Bitmap? = ScreenHighlightsScreenshotHolder.consume()
30+
31+
private val _uiState = MutableLiveData<ScreenHighlightsUiState>(ScreenHighlightsUiState.Loading)
32+
val uiState: LiveData<ScreenHighlightsUiState> = _uiState
33+
34+
private var hasStarted = false
35+
36+
/**
37+
* Start extracting once, when the activity first reaches the foreground.
38+
* AICore (Gemini Nano) only runs inference while the app is the top
39+
* foreground app, so this must not run from the view model's init.
40+
*/
41+
fun start() {
42+
if (hasStarted) return
43+
hasStarted = true
44+
extract()
45+
}
46+
47+
fun extract() {
48+
val image = bitmap
49+
if (image == null) {
50+
_uiState.value =
51+
ScreenHighlightsUiState.Failed(R.string.screen_highlights_llm_failed, canRetry = false)
52+
return
53+
}
54+
_uiState.value = ScreenHighlightsUiState.Loading
55+
viewModelScope.launch {
56+
_uiState.value = toUiState(OnDeviceAi.run(appContext, ScreenHighlightsTask, image))
57+
}
58+
}
59+
60+
private fun toUiState(result: AiResult<List<ExtractedItem>>): ScreenHighlightsUiState =
61+
when (result) {
62+
is AiResult.Success ->
63+
if (result.value.isEmpty()) {
64+
ScreenHighlightsUiState.Empty
65+
} else {
66+
ScreenHighlightsUiState.Items(result.value)
67+
}
68+
69+
is AiResult.Failure -> when (result.reason) {
70+
AiFailure.NOT_READY ->
71+
ScreenHighlightsUiState.Failed(
72+
R.string.screen_highlights_model_not_ready,
73+
canRetry = false
74+
)
75+
76+
AiFailure.INFERENCE_ERROR ->
77+
ScreenHighlightsUiState.Failed(
78+
R.string.screen_highlights_llm_failed,
79+
canRetry = true
80+
)
81+
}
82+
}
83+
84+
fun copyToClipboard(value: String) {
85+
val clipboard =
86+
appContext.getSystemService(Context.CLIPBOARD_SERVICE) as? ClipboardManager
87+
clipboard?.setPrimaryClip(ClipData.newPlainText("highlight", value))
88+
}
89+
}

0 commit comments

Comments
 (0)